From 6b277b176ab505feb4e81aa19ac09b43ea2a8eaa Mon Sep 17 00:00:00 2001 From: TKS <32640296+bigsk1@users.noreply.github.com> Date: Mon, 29 Jul 2024 02:58:53 -0700 Subject: [PATCH] update project_state, add more logging, core changes --- backend.py | 34 +++++++++++++++---- project_state.json | 2 +- project_state.py | 44 +++++++++++++++++++++--- requirements.txt | 2 +- shared_utils.py | 84 +++++++++++++++++++++++++++++++++++----------- tools.py | 35 ++++++++++--------- 6 files changed, 153 insertions(+), 48 deletions(-) diff --git a/backend.py b/backend.py index 2616a45..1b46178 100644 --- a/backend.py +++ b/backend.py @@ -32,7 +32,10 @@ from dotenv import load_dotenv from automode_logic import AutomodeRequest, start_automode_logic, automode_messages, automode_progress from tools import tools, execute_tool -from project_state import sync_project_state_with_fs, clear_state_file, refresh_project_state, initialize_project_state +from project_state import ( + sync_project_state_with_fs, clear_state_file, refresh_project_state, + initialize_project_state, project_state, save_state_to_file +) from config import PROJECTS_DIR, UPLOADS_DIR, CLAUDE_MODEL, anthropic_client from shared_utils import ( system_prompt, perform_search, encode_image_to_base64, create_folder, create_file, @@ -187,8 +190,10 @@ async def create_folder_endpoint(path: str = Query(...)): @app.post("/create_file") async def create_file_endpoint(path: str = Query(...), content: str = ""): try: - result = await create_file(path, content) - logger.info(f"File created: {path}") + # Remove any leading slashes to ensure the path is relative + cleaned_path = path.lstrip('/') + result = await create_file(cleaned_path, content) + logger.info(f"File created: {cleaned_path}") return {"message": result} except Exception as e: logger.error(f"Error creating file: {str(e)}", exc_info=True) @@ -283,7 +288,7 @@ async def analyze_image(file: UploadFile = File(...)): logger.debug(f"Image encoded, length: {len(encoded_image)}") - analysis_result = await anthropic_client.messages.create( + analysis_result = anthropic_client.messages.create( model=CLAUDE_MODEL, max_tokens=1000, system=system_prompt, @@ -343,16 +348,26 @@ async def clear_project_state(): # Chat endpoint @app.post("/chat") async def chat(request: ChatRequest): - global conversation_history + global conversation_history, project_state try: message = request.message conversation_history.append({"role": "user", "content": message}) + # Sync project state before each interaction + project_state = await sync_project_state_with_fs() + + # Update system prompt with current project state + current_system_prompt = f"{system_prompt}\n\nCurrent project state:\nFolders: {', '.join(project_state['folders'])}\nFiles: {', '.join(project_state['files'])}" + logger.info(f"Sending message to AI: {message}") - response = anthropic_client.messages.create( + logger.debug(f"Current project state before AI response: {project_state}") + + # Use asyncio.to_thread to run the synchronous method in a separate thread + response = await asyncio.to_thread( + anthropic_client.messages.create, model=CLAUDE_MODEL, max_tokens=4096, - system=system_prompt, + system=current_system_prompt, messages=conversation_history, tools=tools ) @@ -382,6 +397,11 @@ async def chat(request: ChatRequest): response_content += "\nTask complete." conversation_history.append({"role": "assistant", "content": response_content}) + + # Sync project state after AI response + project_state = await sync_project_state_with_fs() + logger.debug(f"Current project state after AI response: {project_state}") + return {"response": response_content} except Exception as e: logger.error(f"Error in chat endpoint: {str(e)}", exc_info=True) diff --git a/project_state.json b/project_state.json index a82a8ca..533123e 100644 --- a/project_state.json +++ b/project_state.json @@ -1 +1 @@ -{"folders": ["uploads", "test_project"], "files": ["test_project/index.html", "test_project/styles.css"]} \ No newline at end of file +{"folders": ["uploads"], "files": []} \ No newline at end of file diff --git a/project_state.py b/project_state.py index 5f4945b..a49378a 100644 --- a/project_state.py +++ b/project_state.py @@ -1,6 +1,7 @@ import os import logging import json +from pathlib import Path from config import PROJECTS_DIR logger = logging.getLogger(__name__) @@ -26,25 +27,60 @@ async def clear_state_file(): async def sync_project_state_with_fs(): global project_state - new_state = {"folders": set(), "files": set()} + project_state = {"folders": set(), "files": set()} for root, dirs, files in os.walk(PROJECTS_DIR): for dir in dirs: rel_path = os.path.relpath(os.path.join(root, dir), PROJECTS_DIR).replace(os.sep, '/') - new_state["folders"].add(rel_path) + project_state["folders"].add(rel_path) for file in files: rel_path = os.path.relpath(os.path.join(root, file), PROJECTS_DIR).replace(os.sep, '/') - new_state["files"].add(rel_path) + project_state["files"].add(rel_path) - project_state = new_state await save_state_to_file(project_state) logger.debug(f"Synced project state with file system: {project_state}") return project_state +async def update_project_state(path: str, is_folder: bool, is_delete: bool = False): + global project_state + try: + # Normalize the path and make it relative to PROJECTS_DIR + normalized_path = os.path.normpath(path).lstrip(os.sep).replace('\\', '/') + projects_dir_path = Path(PROJECTS_DIR) + full_path = projects_dir_path / normalized_path + + # Ensure the path is within PROJECTS_DIR + try: + rel_path = str(full_path.relative_to(projects_dir_path)) + except ValueError: + logger.error(f"Path '{full_path}' is not within PROJECTS_DIR '{projects_dir_path}'") + return + + logger.debug(f"Updating project state for path: {rel_path}") + + if is_delete: + project_state["folders"].discard(rel_path) + project_state["files"].discard(rel_path) + logger.debug(f"Removed {'folder' if is_folder else 'file'} from project state: {rel_path}") + else: + if is_folder: + project_state["folders"].add(rel_path) + logger.debug(f"Added folder to project state: {rel_path}") + else: + project_state["files"].add(rel_path) + logger.debug(f"Added file to project state: {rel_path}") + + await save_state_to_file(project_state) + logger.debug(f"Project state after update: {project_state}") + except Exception as e: + logger.error(f"Error updating project state: {str(e)}", exc_info=True) + + async def save_state_to_file(state, filename=PROJECT_STATE_FILE): with open(filename, 'w') as f: json.dump({"folders": list(state["folders"]), "files": list(state["files"])}, f) logger.debug(f"Saved project state to file: {state}") + return state # Return the state to ensure it's not modified async def load_state_from_file(filename=PROJECT_STATE_FILE): try: diff --git a/requirements.txt b/requirements.txt index 7c9016f..621d4be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ pytest fastapi uvicorn pytest -httpx \ No newline at end of file +httpx diff --git a/shared_utils.py b/shared_utils.py index 9e5733d..08ba38b 100644 --- a/shared_utils.py +++ b/shared_utils.py @@ -20,18 +20,18 @@ import platform import base64 # from typing import Dict, Any -import base64 import requests from pathlib import Path from PIL import Image import io from fastapi import HTTPException from config import PROJECTS_DIR, SEARCH_RESULTS_LIMIT, SEARCH_PROVIDER, SEARXNG_URL, tavily_client -from project_state import project_state, save_state_to_file +from project_state import project_state, save_state_to_file, update_project_state from urllib.parse import urlparse from datetime import datetime + logger = logging.getLogger(__name__) @@ -140,7 +140,21 @@ async def sync_filesystem(): except Exception as e: logger.error(f"Error syncing file system: {str(e)}", exc_info=True) - +async def retry_file_operation(operation, *args, max_attempts=5, delay=0.5, **kwargs): + for attempt in range(max_attempts): + try: + logger.debug(f"Attempting operation {operation.__name__}, attempt {attempt + 1}/{max_attempts}") + result = await operation(*args, **kwargs) + logger.info(f"Operation {operation.__name__} successful on attempt {attempt + 1}") + return result + except Exception as e: + logger.warning(f"Attempt {attempt + 1} for {operation.__name__} failed: {str(e)}") + if attempt == max_attempts - 1: # Last attempt + logger.error(f"All {max_attempts} attempts for {operation.__name__} failed. Last error: {str(e)}") + raise + logger.info(f"Waiting {delay} seconds before next attempt") + await asyncio.sleep(delay) + async def encode_image_to_base64(image_data): try: logger.debug(f"Encoding image, data type: {type(image_data)}") @@ -276,6 +290,10 @@ async def create_folder(path: str) -> str: await sync_filesystem() if not full_path.exists(): raise FileNotFoundError(f"Failed to create folder: {full_path}") + + rel_path = str(full_path.relative_to(PROJECTS_DIR)).replace(os.sep, '/') + await update_project_state(rel_path, is_folder=True) + logger.info(f"Folder created and verified: {full_path}") return f"Folder created: {full_path}" except Exception as e: @@ -284,15 +302,38 @@ async def create_folder(path: str) -> str: async def create_file(path: str, content: str = "") -> str: try: - logger.debug(f"Creating file at path: {path} with content length: {len(content)}") - full_path = get_safe_path(path) + logger.debug(f"Attempting to create file at path: {path}") + + # Normalize the path and make it relative to PROJECTS_DIR + normalized_path = os.path.normpath(path).lstrip(os.sep).replace('\\', '/') + full_path = Path(PROJECTS_DIR) / normalized_path + + logger.debug(f"Normalized path: {normalized_path}") + logger.debug(f"Full path: {full_path}") + + # Ensure the directory exists full_path.parent.mkdir(parents=True, exist_ok=True) - full_path.write_text(content, encoding='utf-8') - await sync_filesystem() + + # Write content using asyncio.to_thread + await asyncio.to_thread(lambda: full_path.write_text(content, encoding='utf-8')) + + # Verify file exists and content is correct if not full_path.exists(): raise FileNotFoundError(f"Failed to create file: {full_path}") - logger.info(f"File created and verified: {full_path}") - return f"File created: {full_path}" + + # Read content using asyncio.to_thread to verify + written_content = await asyncio.to_thread(lambda: full_path.read_text(encoding='utf-8')) + + if written_content != content: + raise ValueError(f"File content verification failed for {full_path}") + + file_size = full_path.stat().st_size + logger.info(f"File created and verified: {full_path} (Size: {file_size} bytes)") + + await sync_filesystem() + await update_project_state(str(full_path.relative_to(PROJECTS_DIR)), is_folder=False) + + return f"File created: {full_path} (Size: {file_size} bytes)" except Exception as e: logger.error(f"Error creating file: {str(e)}", exc_info=True) raise HTTPException(status_code=500, detail=f"Error creating file: {str(e)}") @@ -301,26 +342,29 @@ async def write_to_file(path: str, content: str) -> str: try: logger.debug(f"Writing to file at path: {path} with content length: {len(content)}") full_path = get_safe_path(path) - logger.debug(f"Full path resolved to: {full_path}") # Ensure the directory exists os.makedirs(os.path.dirname(full_path), exist_ok=True) - with open(full_path, 'w', encoding='utf-8') as f: - f.write(content) - await sync_filesystem() + # Write content using asyncio.to_thread + await asyncio.to_thread(lambda: open(full_path, 'w', encoding='utf-8').write(content)) + # Verify file exists and content is correct if not os.path.exists(full_path): - raise FileNotFoundError(f"Failed to write to file: {full_path}") + raise FileNotFoundError(f"Failed to create file: {full_path}") + + # Read content using asyncio.to_thread + written_content = await asyncio.to_thread(lambda: open(full_path, 'r', encoding='utf-8').read()) - # Verify the content was written correctly - with open(full_path, 'r', encoding='utf-8') as f: - written_content = f.read() if written_content != content: raise ValueError(f"File content verification failed for {full_path}") file_size = os.path.getsize(full_path) logger.info(f"Content written to file and verified: {full_path} (Size: {file_size} bytes)") + + await sync_filesystem() + await update_project_state(path, is_folder=False) + return f"Content written to file: {full_path} (Size: {file_size} bytes)" except Exception as e: logger.error(f"Error writing to file: {str(e)}", exc_info=True) @@ -353,15 +397,14 @@ async def list_files(path: str = ".") -> list: } files.append(file_info) - # Update project_state + # Update project_state without overwriting if item.is_dir(): project_state["folders"].add(rel_path) else: project_state["files"].add(rel_path) logger.info(f"Listed files in {full_path}") - await save_state_to_file(project_state) - logger.debug(f"Updated project state: {project_state}") + logger.debug(f"Current project state: {project_state}") return files except Exception as e: logger.error(f"Error listing files: {str(e)}", exc_info=True) @@ -378,6 +421,7 @@ async def delete_file(path: str) -> str: raise FileNotFoundError(f"File or directory not found: {full_path}") logger.info(f"Deleted: {full_path}") await sync_filesystem() + await update_project_state(path, is_folder=full_path.is_dir(), is_delete=True) return f"Deleted: {full_path}" except Exception as e: logger.error(f"Error deleting file: {str(e)}", exc_info=True) diff --git a/tools.py b/tools.py index 171a5e2..26798a2 100644 --- a/tools.py +++ b/tools.py @@ -1,7 +1,10 @@ import os import logging -from shared_utils import create_file, read_file, write_to_file, create_folder, delete_file, perform_search, list_files, sync_filesystem -from project_state import save_state_to_file, project_state +from shared_utils import ( + create_file, read_file, write_to_file, create_folder, delete_file, perform_search, + list_files, sync_filesystem, retry_file_operation +) +from project_state import save_state_to_file, project_state, sync_project_state_with_fs from config import SEARCH_PROVIDER, PROJECTS_DIR @@ -111,8 +114,9 @@ async def execute_tool(tool_name, tool_input): full_path = os.path.normpath(tool_input["path"]).replace(os.sep, '/') if full_path in project_state["folders"]: return {"success": True, "result": f"Folder already exists: {full_path}"} - result = await create_folder(tool_input["path"]) + result = await retry_file_operation(create_folder, tool_input["path"]) project_state["folders"].add(full_path) + elif tool_name == "create_file": full_path = os.path.normpath(tool_input["path"]).replace(os.sep, '/') folder_path = os.path.dirname(full_path) @@ -122,8 +126,9 @@ async def execute_tool(tool_name, tool_input): return {"success": False, "error": f"Folder does not exist: {folder_path}"} if full_path in project_state["files"]: return {"success": True, "result": f"File already exists: {full_path}"} - result = await create_file(tool_input["path"], tool_input.get("content", "")) + result = await retry_file_operation(create_file, tool_input["path"], tool_input.get("content", "")) project_state["files"].add(full_path) + elif tool_name == "write_to_file": full_path = os.path.normpath(tool_input["path"]).replace(os.sep, '/') if full_path not in project_state["files"]: @@ -132,14 +137,9 @@ async def execute_tool(tool_name, tool_input): project_state["files"].add(full_path) else: return {"success": False, "error": f"File does not exist: {full_path}"} - try: - result = await write_to_file(tool_input["path"], tool_input["content"]) - project_state["files"].add(full_path) - # await save_state_to_file(project_state) - return {"success": True, "result": result} - except Exception as e: - logger.error(f"Error in write_to_file: {str(e)}", exc_info=True) - return {"success": False, "error": f"Error writing to file: {str(e)}"} + result = await retry_file_operation(write_to_file, tool_input["path"], tool_input["content"]) + project_state["files"].add(full_path) + elif tool_name == "read_file": full_path = os.path.normpath(tool_input["path"]).replace(os.sep, '/') if full_path not in project_state["files"]: @@ -148,18 +148,22 @@ async def execute_tool(tool_name, tool_input): project_state["files"].add(full_path) else: return {"success": False, "error": f"File does not exist: {full_path}"} - result = await read_file(tool_input["path"]) + result = await retry_file_operation(read_file, tool_input["path"]) + elif tool_name == "list_files": - result = await list_files(tool_input["path"]) + result = await retry_file_operation(list_files, tool_input["path"]) + elif tool_name == "delete_file": full_path = os.path.normpath(tool_input["path"]).replace(os.sep, '/') if full_path not in project_state["files"] and full_path not in project_state["folders"]: return {"success": False, "error": f"File or folder does not exist: {full_path}"} - result = await delete_file(tool_input["path"]) + result = await retry_file_operation(delete_file, tool_input["path"]) project_state["files"].discard(full_path) project_state["folders"].discard(full_path) + elif tool_name == "search": result = await perform_search(tool_input["query"]) + else: return {"success": False, "error": f"Unknown tool: {tool_name}"} @@ -170,4 +174,5 @@ async def execute_tool(tool_name, tool_input): return {"success": True, "result": result} except Exception as e: logger.error(f"Error executing tool {tool_name}: {str(e)}", exc_info=True) + await sync_project_state_with_fs() # Ensure state is synced even after an error return {"success": False, "error": f"Error executing tool {tool_name}: {str(e)}"} \ No newline at end of file