From f0f189eed27b7816754dd9f77322aa7aa4d4921c Mon Sep 17 00:00:00 2001 From: George Tsoukalas Date: Sat, 26 Jul 2025 09:45:23 +0000 Subject: [PATCH] Minor modifications --- .gitignore | 5 +- frame/configs/funsearch-low-budget.yaml | 6 +- frame/configs/models/gpt4o-mini.yaml | 2 +- ...ro_generate_interestingness_funsearch.yaml | 42 +++++++------- .../learning/tests/gpt4o_test_1.py | 55 +++++++++++++++++++ frame/knowledge_base/knowledge_graph.py | 2 +- run_parallel_funsearch.sh | 2 +- ...run_experiment_existing_interestingness.sh | 6 +- scripts/run_funsearch_smaller_budget.sh | 2 +- 9 files changed, 90 insertions(+), 32 deletions(-) create mode 100644 frame/interestingness/learning/tests/gpt4o_test_1.py diff --git a/.gitignore b/.gitignore index ccbf299..d436a31 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,7 @@ multirun/ # Scratch notebooks -scratch_notebooks/ \ No newline at end of file +scratch_notebooks/ + +nohup.out +last_50_nohup.out \ No newline at end of file diff --git a/frame/configs/funsearch-low-budget.yaml b/frame/configs/funsearch-low-budget.yaml index 9503992..2c824f0 100644 --- a/frame/configs/funsearch-low-budget.yaml +++ b/frame/configs/funsearch-low-budget.yaml @@ -15,7 +15,7 @@ defaults: programs_database: functions_per_prompt: 2 - num_islands: 3 + num_islands: 1 reset_period: 360000 # 1 hour in seconds cluster_sampling_temperature_init: 0.01 cluster_sampling_temperature_period: 1000 @@ -55,12 +55,12 @@ output_directory: "${hydra:runtime.output_dir}" evaluations_dir: "${hydra:runtime.output_dir}/evaluations" # Iterations of the Funsearch sampling loop -iterations: 32 +iterations: 24 # --- FunSearch Abstraction Settings --- abstraction: enabled: true # Master switch: set to false to disable the entire abstraction mechanism - frequency: 3 # Run the abstraction step every N main FunSearch iterations (low for testing). + frequency: 2 # Run the abstraction step every N main FunSearch iterations (low for testing). programs_to_sample: 2 # How many top programs to analyze (low for testing). max_abstractions_per_step: 2 # Target new abstractions per step (low for testing). # Optional LLM override block for abstraction task diff --git a/frame/configs/models/gpt4o-mini.yaml b/frame/configs/models/gpt4o-mini.yaml index f1a779c..dfc3d32 100644 --- a/frame/configs/models/gpt4o-mini.yaml +++ b/frame/configs/models/gpt4o-mini.yaml @@ -1,7 +1,7 @@ name: gpt-4o provider: openai temperature: 0.9 -max_tokens: 8192 +max_tokens: 16384 top_p: 1.0 frequency_penalty: 0.0 presence_penalty: 0.0 \ No newline at end of file diff --git a/frame/configs/succ_zero_generate_interestingness_funsearch.yaml b/frame/configs/succ_zero_generate_interestingness_funsearch.yaml index f9a2485..20955f5 100644 --- a/frame/configs/succ_zero_generate_interestingness_funsearch.yaml +++ b/frame/configs/succ_zero_generate_interestingness_funsearch.yaml @@ -8,42 +8,42 @@ defaults: - override hydra/hydra_logging: default # # Note(George; 5/5): Due to some config loading issues, I've just pasted this directly here, change for diff FunSearch experiments. -initial_state: - _target_: frame.knowledge_base.knowledge_graph.KnowledgeGraph - name: "succ_zero_eq_initial" - description: "Initial knowledge graph with Zero, Successor, and Equality" - - # Import settings - import_from: "frame.knowledge_base.initial_concepts" - concepts: - - "create_zero_concept" - - "create_successor_concept" - - "create_equality_concept" - # initial_state: # _target_: frame.knowledge_base.knowledge_graph.KnowledgeGraph -# name: "arithmetic_basic_initial" -# description: "Initial knowledge graph with arithmetic basic concepts" +# name: "succ_zero_eq_initial" +# description: "Initial knowledge graph with Zero, Successor, and Equality" # # Import settings # import_from: "frame.knowledge_base.initial_concepts" # concepts: # - "create_zero_concept" -# - "create_one_concept" -# - "create_two_concept" -# - "create_addition_concept" -# - "create_multiplication_concept" -# - "create_divides_concept" -# - "create_leq_than_concept" +# - "create_successor_concept" # - "create_equality_concept" +initial_state: + _target_: frame.knowledge_base.knowledge_graph.KnowledgeGraph + name: "arithmetic_basic_initial" + description: "Initial knowledge graph with arithmetic basic concepts" + + # Import settings + import_from: "frame.knowledge_base.initial_concepts" + concepts: + - "create_zero_concept" + - "create_one_concept" + - "create_two_concept" + - "create_addition_concept" + - "create_multiplication_concept" + - "create_divides_concept" + - "create_leq_than_concept" + - "create_equality_concept" + # Experiment settings override experiment: name: "succ_zero_multi_generated_interestingness_eval" description: "Generate N interestingness functions and evaluate each M times, starting from successor and zero." max_steps: 1000 num_episodes: 16 - num_workers: 2 + num_workers: 4 seed: 0 #${oc.env:RANDOM_SEED,12345} episode_timeout_seconds: 30 # Timeout for the main process waiting for a worker episode result diff --git a/frame/interestingness/learning/tests/gpt4o_test_1.py b/frame/interestingness/learning/tests/gpt4o_test_1.py new file mode 100644 index 0000000..d256ec5 --- /dev/null +++ b/frame/interestingness/learning/tests/gpt4o_test_1.py @@ -0,0 +1,55 @@ +import math +import numpy as np +from typing import Dict, Any, List, Set, Optional, Union +from frame.knowledge_base.knowledge_graph import KnowledgeGraph, NodeType + +from frame.interestingness.learning.dsl_primitives import ( + get_ancestors, get_descendants, get_construction_depth, + get_in_degree, get_out_degree, get_construction_history_rule_names, + get_entity_step_age, get_num_concepts, get_num_conjectures, + get_entity_node_type, get_concept_category, get_input_arity, get_num_component_types, + get_examples, get_nonexamples, + get_num_construction_inputs, is_proven, create_weighted_interestingness_function +) + + +def calculate_interestingness(entity_id: str, graph: KnowledgeGraph) -> float: + """Calculate the interestingness score for a given entity. + + Args: + entity_id: The ID of the entity to score. + graph: The knowledge graph containing the entity. + + Returns: + A float value representing the interestingness score (higher is more interesting). + """ + try: + node_type = get_entity_node_type(entity_id, graph) + num_descendants = len(get_descendants(entity_id, graph)) + depth = get_construction_depth(entity_id, graph) + in_degree = get_in_degree(entity_id, graph) + out_degree = get_out_degree(entity_id, graph) + step_age = get_entity_step_age(entity_id, graph) + is_proven_value = is_proven(entity_id, graph) + + # Calculate base scores based on node type + base_score = 0.0 + if node_type == 'Theorem': + base_score = 2.0 + elif node_type == 'Conjecture': + base_score = 1.0 + elif node_type == 'Concept': + base_score = 0.5 + + # Calculate derived features + connectivity_score = (in_degree + out_degree) / (depth + 1) + temporal_score = math.exp(-step_age) * 5 + + print(f"temporal_score:, step_age: {step_age}, temporal_score: {temporal_score}") + + # Aggregate score + score = base_score + 0.5 * num_descendants + connectivity_score + temporal_score + 3 * is_proven_value + return score + + except Exception: + return 0.0 diff --git a/frame/knowledge_base/knowledge_graph.py b/frame/knowledge_base/knowledge_graph.py index f40f91c..3dba846 100644 --- a/frame/knowledge_base/knowledge_graph.py +++ b/frame/knowledge_base/knowledge_graph.py @@ -241,7 +241,7 @@ def get_entity_step_age(self, entity_id: str) -> int: if entity_id not in self: raise ValueError(f"Entity {entity_id} not found in graph") creation_step = self.nodes[entity_id].get("creation_step", 0) - return self._step_counter - creation_step + return max(self._step_counter - creation_step, 0) def get_node(self, node_id: str) -> Tuple[Entity, NodeType, Optional[ConstructionStep]]: """Get a node's data from the graph.""" diff --git a/run_parallel_funsearch.sh b/run_parallel_funsearch.sh index 8425338..9a6b89c 100755 --- a/run_parallel_funsearch.sh +++ b/run_parallel_funsearch.sh @@ -1,7 +1,7 @@ #!/bin/bash # Configuration -NUM_PARALLEL_RUNS=32 # N: Number of parallel FunSearch instances per iteration +NUM_PARALLEL_RUNS=16 # N: Number of parallel FunSearch instances per iteration FUNSEARCH_SCRIPT="scripts/run_funsearch_smaller_budget.sh" # Relative path from project root NUM_ITERATIONS=1 # M: Number of times to run the block of N parallel instances diff --git a/scripts/run_experiment_existing_interestingness.sh b/scripts/run_experiment_existing_interestingness.sh index 039b284..768f196 100755 --- a/scripts/run_experiment_existing_interestingness.sh +++ b/scripts/run_experiment_existing_interestingness.sh @@ -13,12 +13,12 @@ export INTERESTINGNESS_DEBUG=1 # TODO(George; 4/23): THIS IS FOR TESTING SPECIFIC PATHS! # Run the experiment with the existing interestingness config -nohup python -m frame.theory_builder \ +python -m frame.theory_builder \ --config-name succ_zero_existing_interestingness_experiment \ - experiment.num_episodes=128 \ + experiment.num_episodes=1 \ experiment.max_steps=1000 \ experiment.episode_timeout_seconds=30 \ - experiment.num_workers=128 \ + experiment.num_workers=1 \ z3_usage.use_z3_prover=true \ z3_usage.use_z3_example_search=true \ ++policy.params.interestingness_function_path="frame/interestingness/learning/tests/gpt4o_test_1.py" diff --git a/scripts/run_funsearch_smaller_budget.sh b/scripts/run_funsearch_smaller_budget.sh index ae26791..3b01d24 100755 --- a/scripts/run_funsearch_smaller_budget.sh +++ b/scripts/run_funsearch_smaller_budget.sh @@ -22,7 +22,7 @@ echo " Abstraction: ENABLED" # Only enabling abstraction explicitly. nohup python -m frame.funsearch.main \ --config-name funsearch-low-budget \ - abstraction.enabled=true + abstraction.enabled=false echo "FunSearch experiment with abstraction completed!" # Output directory location depends on Hydra configuration (e.g., ./outputs/YYYY-MM-DD/HH-MM-SS)