diff --git a/config.py b/config.py index 26b7b57..c27714e 100644 --- a/config.py +++ b/config.py @@ -1,9 +1,7 @@ from functools import lru_cache - from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict - class Settings(BaseSettings): model_config = SettingsConfigDict( env_file=".env", @@ -16,7 +14,7 @@ class Settings(BaseSettings): gemini_api_key: str = Field(default="test-key") - model_name: str = "gemini-1.5-flush" + model_name: str = "gemini-1.5-flash" temperature: float = Field(default=0.7, ge=0, le=2) top_p: float = Field(default=0.8, ge=0, le=1) @@ -49,6 +47,12 @@ class Settings(BaseSettings): port: int = Field(default=8000, ge=1) + turkish_support_enabled: bool = Field(default=True) + turkish_terminology_db_path: str = Field(default="data/turkish_islamic_terms.json") + turkish_ottoman_recognition_enabled: bool = Field(default=True) + turkish_arabic_mapping_enabled: bool = Field(default=True) + turkish_transliteration_enabled: bool = Field(default=True) + # Respectful Disagreement Enforcement settings enforce_respectful_disagreement: bool = True disrespectful_language_patterns: list[str] = Field( @@ -72,9 +76,8 @@ class Settings(BaseSettings): @field_validator("cors_origins", mode="before") @classmethod - def parse_cors_origins(cls, value): - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] + def parse_cors_origins(cls,value): + if isinstance(value,str): return [item.strip() for item in value.split(",") if item.strip()] return value @field_validator("disrespectful_language_patterns", mode="before") @@ -84,7 +87,5 @@ def parse_disrespectful_language_patterns(cls, value): return [item.strip() for item in value.split(",") if item.strip()] return value - @lru_cache -def get_settings() -> Settings: - return Settings() +def get_settings()->Settings: return Settings() \ No newline at end of file diff --git a/query_optimizer.py b/query_optimizer.py index 871bbbe..d2ebbae 100644 --- a/query_optimizer.py +++ b/query_optimizer.py @@ -40,6 +40,94 @@ router = APIRouter(prefix="/db-optimizer", tags=["db-optimizer"]) +# --------------------------------------------------------------------------- +# Turkish language support +# --------------------------------------------------------------------------- + +def turkish_lower(text: str) -> str: + """Turkish-aware lowercasing that respects the dotted/dotless I distinction.""" + return text.replace("I", "ı").replace("İ", "i").lower() + + +_TURKISH_SUFFIXES = ( + "lar", "ler", "dan", "den", "tan", "ten", + "yla", "yle", "sin", "sın", "sun", "sün", + "dir", "dır", "dur", "dür", "tir", "tır", "tur", "tür", + "ken", "ince", "erek", "arak", +) + +TURKISH_ISLAMIC_TERMS: dict[str, tuple[str, ...]] = { + "namaz": ("salat", "salât"), + "oruç": ("savm", "sawm"), + "zekat": ("zakat", "zekât"), + "hac": ("hajj", "hacc"), + "peygamber": ("nebi", "resul", "rasûl"), + "melek": ("malaik", "melekût"), + "kuran": ("qur'an", "kur'an", "quran"), + "hadis": ("hadith", "hadîs"), + "sünnet": ("sunna", "sünnet"), + "kelam": ("kalam", "kelâm"), + "tasavvuf": ("sufism", "tasawwuf"), + "tevhid": ("tawhid", "tevhit"), + "ibadet": ("ibada", "ibâdet"), + "sevap": ("thawab", "sevâb"), + "günah": ("dhanb", "gunah", "günâh"), + "helal": ("halal", "helâl"), + "haram": ("haram", "harâm"), + "dua": ("du'a", "duâ"), + "iman": ("iman", "îman"), + "islam": ("islam", "islâm"), +} + +_OTTOMAN_AFFIXES = ("ullah", "ül", "il", "el", "al", "ibn", "bin", "zade", "efendi") + + +def turkish_tokenize(text: str) -> list[str]: + """Split Turkish text into lowercase tokens, respecting Turkish characters.""" + return re.findall(r"[a-z0-9çğıöşüâîû']+", turkish_lower(text)) + + +def strip_turkish_suffixes(word: str) -> str: + """Remove common agglutinative suffixes for a conservative Turkish stem.""" + stem = word + changed = True + while changed and len(stem) > 3: + changed = False + for suffix in _TURKISH_SUFFIXES: + if stem.endswith(suffix) and len(stem) - len(suffix) >= 2: + stem = stem[: -len(suffix)] + changed = True + break + return stem + + +def normalize_turkish_islamic_term(term: str) -> tuple[str, ...] | str: + """Return known Arabic/Ottoman equivalents for a Turkish Islamic term.""" + normalized = turkish_lower(term).replace("'", "").replace(" ", "") + return TURKISH_ISLAMIC_TERMS.get(normalized, turkish_lower(term)) + + +def expand_turkish_islamic_query(text: str) -> str: + """Expand Turkish Islamic terms with Arabic/Ottoman equivalents for recall.""" + tokens = turkish_tokenize(text) + expanded: list[str] = [] + for token in tokens: + root = strip_turkish_suffixes(token) + equivalents = normalize_turkish_islamic_term(root) + if isinstance(equivalents, tuple): + expanded.append(token) + expanded.extend(equivalents) + else: + expanded.append(token) + return " ".join(expanded) + + +def detect_ottoman_turkish_influence(text: str) -> list[str]: + """Return Ottoman affixes found in a religious text fragment.""" + lowered = turkish_lower(text) + return [affix for affix in _OTTOMAN_AFFIXES if affix in lowered] + + # Cross-Surah Reference System class ReferenceType(str, Enum): @@ -105,6 +193,9 @@ def references_for(self, surah: int, ayah: int) -> list[CrossReferenceRecord]: def _tokenize(text: str) -> set[str]: return {token for token in re.findall(r"[a-z0-9']+", text.lower()) if + token not in _STOPWORDS} + + # Latency at or above this (milliseconds) counts a query as "slow". Mirrors the # issue's "zero queries exceeding 500ms" success criterion. DEFAULT_SLOW_QUERY_MS = 500.0 @@ -875,4 +966,4 @@ def _classify_relationship(a: AyahText, b: AyahText, jaccard: float, topic_overl return "example" if jaccard >= 0.15: return "elaboration" - return "complement" + return "complement" \ No newline at end of file diff --git a/swahili_processor.py b/swahili_processor.py index c77729e..8ba3054 100644 --- a/swahili_processor.py +++ b/swahili_processor.py @@ -51,6 +51,15 @@ def analyze_swahili(text: str) -> SwahiliAnalysisResult: ) +def analyze_turkish(text: str) -> dict: + """Analyze Turkish text. + + Placeholder for Turkish language support. The actual implementation + will be added in a dedicated Turkish processor module. + """ + raise NotImplementedError("Turkish language support is under development.") + + __all__ = [ "CodeSwitchResult", "CodeSwitchType", @@ -64,6 +73,7 @@ def analyze_swahili(text: str) -> SwahiliAnalysisResult: "SwahiliPromptEnhancement", "SwahiliToken", "analyze_swahili", + "analyze_turkish", "code_switch_processor", "cultural_context_engine", "dialect_classifier",