Added semantic code, made token counts n metrics uniform, updated readme

pyproject.toml

@@ -1,27 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "scaledown"
 version = "0.1.0"
-dependencies = [
-    "requests>=2.31.0",
-    "pydantic>=2.0.0",
-]
-authors = [
-  { name="Soham Chatterjee", email="soham@scaldown.ai" },
-]
 description = "A Context Optimization Package"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [
+  { name="Soham Chatterjee", email="soham@scaledown.ai" },
+]
 classifiers = [
     "Programming Language :: Python :: 3",
     "Operating System :: OS Independent",
 ]
-license = "MIT"
-license-files = ["LICEN[CS]E*"]
+dependencies = [
+    "requests>=2.31.0",
+    "pydantic>=2.0.0",
+]
+
+# --- Optional dependencies start here ---
+[project.optional-dependencies]
+semantic = [
+    "sentence-transformers>=3.0.0",
+    "faiss-cpu>=1.7.0",
+    "numpy>=1.20.0"
+]
+haste = [
+    "HasteContext>=0.2.4",
+]
 
 [project.urls]
 Homepage = "https://scaledown.ai"
 Issues = "https://github.com/scaledown-team/scaledown"
-
-[build-system]
-requires = ["setuptools >= 77.0.3"]
-build-backend = "setuptools.build_meta"

pytest.ini

@@ -0,0 +1,9 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+addopts = -v
+filterwarnings =
+    ignore::DeprecationWarning:importlib._bootstrap
+    ignore:builtin type SwigPyPacked:DeprecationWarning
+    ignore:builtin type SwigPyObject:DeprecationWarning
+    ignore:builtin type swigvarlink:DeprecationWarning

requirements.txt

@@ -1,2 +1,25 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml -o requirements.txt
+annotated-types==0.7.0
+    # via pydantic
+certifi==2025.11.12
+    # via requests
+charset-normalizer==3.4.4
+    # via requests
+idna==3.11
+    # via requests
+pydantic==2.12.5
+    # via scaledown (pyproject.toml)
+pydantic-core==2.41.5
+    # via pydantic
+requests==2.32.5
+    # via scaledown (pyproject.toml)
+typing-extensions==4.15.0
+    # via
+    #   pydantic
+    #   pydantic-core
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via pydantic
+urllib3==2.6.2
+    # via requests

scaledown/__init__.py

@@ -1,16 +1,42 @@
 import os
 from typing import Optional
-from .compressor import ScaleDownCompressor
 
-# Global configuration state
-_API_KEY: Optional[str] = os.environ.get("SCALEDOWN_API_KEY")
+# Configuration
+from scaledown.config import set_api_key, get_api_key
+
+# Core Components
+from scaledown.pipeline import Pipeline, make_pipeline
+# HasteOptimizer is optional, import from scaledown.optimizer if needed
+from scaledown.compressor.scaledown_compressor import ScaleDownCompressor
 
-def set_api_key(api_key: str) -> None:
-    """Sets the global API key for ScaleDown."""
-    global _API_KEY
-    _API_KEY = api_key
+# Types & Exceptions
+from scaledown.types import (
+    CompressedPrompt,
+    OptimizedContext,
+    PipelineResult,
+    StepMetadata
+)
 
-def get_api_key() -> Optional[str]:
-    """Retrieves the global API key."""
-    return _API_KEY
+from scaledown.exceptions import (
+    ScaleDownError,
+    AuthenticationError,
+    APIError
+)
+
+# Initialize global state if env var exists
+_API_KEY: Optional[str] = os.environ.get("SCALEDOWN_API_KEY")
 
+__all__ = [
+    "Pipeline",
+    "make_pipeline",
+    "ScaleDownCompressor",
+    "set_api_key",
+    "get_api_key",
+    "PipelineResult",
+    "StepMetadata",
+    "CompressedPrompt",
+    "OptimizedContext",
+    "ScaleDownError",
+    "AuthenticationError",
+    "APIError"
+]

scaledown/compressor/config.py

@@ -2,4 +2,4 @@
 default_scaledown_api="https://api.scaledown.xyz"
 
 def get_api_url():
-    return os.getenv("SCALEDOWN_API_URL", default_scaledown_api)
+    return os.getenv("SCALEDOWN_API_URL", default_scaledown_api)

scaledown/compressor/scaledown_compressor.py

@@ -93,7 +93,7 @@ def _compress_single(self, context, prompt, max_tokens=None, **kwargs) -> Compre
                 "original_prompt_tokens": data.get("total_original_tokens", results.get("original_prompt_tokens", 0)),
                 "compressed_prompt_tokens": data.get("total_compressed_tokens", results.get("compressed_prompt_tokens", 0)),
                 "latency_ms": data.get("latency_ms", 0),
-                "model_used": data.get("model_used"),
+                "target_model": data.get("target_model","model"),
                 "timestamp": data.get("request_metadata", {}).get("timestamp")
             }
 

scaledown/config.py

@@ -0,0 +1,14 @@
+import os
+from typing import Optional
+
+# Global configuration state
+_API_KEY: Optional[str] = os.environ.get("SCALEDOWN_API_KEY")
+
+def set_api_key(api_key: Optional[str]) -> None:
+    """Sets the global API key for ScaleDown."""
+    global _API_KEY
+    _API_KEY = api_key
+
+def get_api_key() -> Optional[str] :
+    """Retrieves the global API key."""
+    return _API_KEY

scaledown/exceptions.py

@@ -9,3 +9,11 @@ class AuthenticationError(ScaleDownError):
 class APIError(ScaleDownError):
     """Raised when the ScaleDown API returns an error."""
     pass
+
+class OptimizerError(ScaleDownError):
+    """Raised when an optimizer encounters an error."""
+    pass
+
+class PipelineError(ScaleDownError):
+    """Raised when pipeline execution fails."""
+    pass

scaledown/optimizer/__init__.py

@@ -0,0 +1,31 @@
+from typing import TYPE_CHECKING
+
+from .base import BaseOptimizer
+
+# Define what to expose
+__all__ = ["BaseOptimizer", "HasteOptimizer", "SemanticOptimizer"]
+
+def __getattr__(name):
+    if name == "HasteOptimizer":
+        try:
+            from .haste import HasteOptimizer
+            return HasteOptimizer
+        except ImportError as e:
+            raise ImportError(
+                "HasteOptimizer requires 'haste'. Install with `pip install scaledown[haste]`"
+            ) from e
+
+    if name == "SemanticOptimizer":
+        try:
+            from .semantic_code import SemanticOptimizer
+            return SemanticOptimizer
+        except ImportError as e:
+            raise ImportError(
+                "SemanticOptimizer requires 'semantic'. Install with `pip install scaledown[semantic]`"
+            ) from e
+
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+if TYPE_CHECKING:
+    from .haste import HasteOptimizer
+    from .semantic_code import SemanticOptimizer

scaledown/optimizer/base.py

@@ -0,0 +1,57 @@
+from abc import ABC, abstractmethod
+from typing import Union, List, Optional
+import scaledown
+
+class BaseOptimizer(ABC):
+    """
+    Base class for all context optimizers.
+    Optimizers process raw context before compression.
+    """
+
+    def __init__(self, api_key: Optional[str] = None, target_model:str="gpt-4o", **kwargs):
+        """
+        Initialize optimizer.
+
+        Parameters
+        ----------
+        api_key : str, optional
+            API key for optimizer services (if needed)
+        **kwargs : dict
+            Additional optimizer-specific parameters
+        """
+        self.api_key = api_key or scaledown.get_api_key()
+        self.target_model = target_model
+        self.config = kwargs
+
+    @abstractmethod
+    def optimize(
+        self,
+        context: Union[str, List[str]],
+        query: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+        **kwargs
+    ):
+        """
+        Optimize context for better relevance and structure.
+
+        Parameters
+        ----------
+        context : str or List[str]
+            Raw context to optimize (e.g., codebase, documents)
+        query : str, optional
+            Query to guide optimization
+        max_tokens : int, optional
+            Maximum token budget for optimized context
+        **kwargs : dict
+            Additional optimization parameters
+
+        Returns
+        -------
+        OptimizedContext or List[OptimizedContext]
+            Optimized context with metrics
+        """
+        pass
+
+    def update_config(self, **kwargs):
+        """Update optimizer configuration."""
+        self.config.update(kwargs)

scaledown/optimizer/config.py

@@ -0,0 +1,7 @@
+import os
+
+default_haste_api = "https://api.scaledown.xyz/haste"
+
+def get_haste_api_url():
+    """Get HASTE API URL from environment or default."""
+    return os.getenv("HASTE_API_URL", default_haste_api)