diff --git a/data/urdu_islamic_terms.json b/data/urdu_islamic_terms.json new file mode 100644 index 0000000..4256992 --- /dev/null +++ b/data/urdu_islamic_terms.json @@ -0,0 +1,227 @@ +{ + "version": "1.0.0", + "language": "ur", + "description": "Curated core Islamic terminology for Urdu query processing.", + "terms": [ + { + "id": "ur-ibadat-salah", + "urdu_term": "نماز", + "arabic_original": "صلاة", + "transliteration": "namaz / salah", + "english_equivalent": "ritual prayer", + "category": "ibadat", + "definition_ur": "روزانہ مقررہ اوقات میں ادا کی جانے والی فرض عبادت۔", + "variants": ["صلوٰۃ", "صلاۃ", "صلاح", "salah", "salat", "namaz"] + }, + { + "id": "ur-ibadat-wudu", + "urdu_term": "وضو", + "arabic_original": "وضوء", + "transliteration": "wudu", + "english_equivalent": "ritual ablution", + "category": "ibadat", + "definition_ur": "نماز وغیرہ کے لیے مقررہ طریقے سے حاصل کی جانے والی طہارت۔", + "variants": ["وضوء", "وضو کرنا", "wudu", "wudhu"] + }, + { + "id": "ur-ibadat-fasting", + "urdu_term": "روزہ", + "arabic_original": "صوم", + "transliteration": "roza / sawm", + "english_equivalent": "fasting", + "category": "ibadat", + "definition_ur": "نیت کے ساتھ فجر سے غروب آفتاب تک مفطرات سے رکنا۔", + "variants": ["صوم", "صیام", "roza", "sawm", "siyam"] + }, + { + "id": "ur-ibadat-zakat", + "urdu_term": "زکوٰۃ", + "arabic_original": "زكاة", + "transliteration": "zakat", + "english_equivalent": "obligatory almsgiving", + "category": "ibadat", + "definition_ur": "شرائط پوری ہونے پر مال کا مقررہ حصہ مستحقین کو دینا۔", + "variants": ["زکات", "زکاۃ", "زكاة", "zakat", "zakah"] + }, + { + "id": "ur-ibadat-hajj", + "urdu_term": "حج", + "arabic_original": "حج", + "transliteration": "hajj", + "english_equivalent": "pilgrimage to Makkah", + "category": "ibadat", + "definition_ur": "استطاعت رکھنے والے مسلمان کا مقررہ ایام میں بیت اللہ کا شرعی حج۔", + "variants": ["حج بیت اللہ", "hajj"] + }, + { + "id": "ur-quran-quran", + "urdu_term": "قرآن مجید", + "arabic_original": "القرآن الكريم", + "transliteration": "Quran Majid", + "english_equivalent": "the Noble Quran", + "category": "quran", + "definition_ur": "اللہ تعالیٰ کی آخری نازل کردہ کتاب۔", + "variants": ["قرآن", "قران", "قرآن کریم", "القرآن", "Quran", "Qur'an"] + }, + { + "id": "ur-quran-tafsir", + "urdu_term": "تفسیر", + "arabic_original": "تفسير", + "transliteration": "tafsir", + "english_equivalent": "Quranic exegesis", + "category": "quran", + "definition_ur": "قرآن مجید کے معانی اور مطالب کی علمی وضاحت۔", + "variants": ["تفسير", "tafsir"] + }, + { + "id": "ur-hadith-hadith", + "urdu_term": "حدیث", + "arabic_original": "حديث", + "transliteration": "hadith", + "english_equivalent": "Prophetic narration", + "category": "hadith", + "definition_ur": "رسول اللہ ﷺ کے اقوال، افعال، تقریرات یا اوصاف کی روایت۔", + "variants": ["حديث", "احادیث", "hadith", "hadees"] + }, + { + "id": "ur-hadith-sunnah", + "urdu_term": "سنت", + "arabic_original": "سنة", + "transliteration": "sunnah", + "english_equivalent": "Prophetic practice", + "category": "hadith", + "definition_ur": "رسول اللہ ﷺ کا منقول طریقہ اور تعلیم۔", + "variants": ["سنۃ", "سنة", "sunnah", "sunna"] + }, + { + "id": "ur-fiqh-fiqh", + "urdu_term": "فقہ", + "arabic_original": "فقه", + "transliteration": "fiqh", + "english_equivalent": "Islamic jurisprudence", + "category": "fiqh", + "definition_ur": "تفصیلی دلائل سے عملی شرعی احکام جاننے کا علم۔", + "variants": ["فقه", "fiqh"] + }, + { + "id": "ur-fiqh-fatwa", + "urdu_term": "فتویٰ", + "arabic_original": "فتوى", + "transliteration": "fatwa", + "english_equivalent": "formal legal opinion", + "category": "fiqh", + "definition_ur": "کسی شرعی سوال پر اہل علم کی مدلل رائے۔", + "variants": ["فتوی", "فتوى", "فتاویٰ", "fatwa"] + }, + { + "id": "ur-fiqh-shariah", + "urdu_term": "شریعت", + "arabic_original": "شريعة", + "transliteration": "shariah", + "english_equivalent": "Islamic revealed law and way", + "category": "fiqh", + "definition_ur": "اسلام کا الٰہی ہدایت پر مبنی مجموعی طریقۂ زندگی۔", + "variants": ["شریعہ", "شريعة", "sharia", "shariah"] + }, + { + "id": "ur-fiqh-hanafi", + "urdu_term": "فقہ حنفی", + "arabic_original": "المذهب الحنفي", + "transliteration": "Hanafi fiqh", + "english_equivalent": "Hanafi school of jurisprudence", + "category": "fiqh", + "definition_ur": "امام ابو حنیفہؒ سے منسوب اہل سنت کا فقہی مذہب۔", + "variants": ["حنفی", "مذہب حنفی", "Hanafi"] + }, + { + "id": "ur-belief-tawhid", + "urdu_term": "توحید", + "arabic_original": "توحيد", + "transliteration": "tawhid", + "english_equivalent": "oneness of Allah", + "category": "aqidah", + "definition_ur": "اللہ تعالیٰ کو ذات، صفات اور عبادت میں یکتا ماننا۔", + "variants": ["توحيد", "tawhid", "tauheed"] + }, + { + "id": "ur-belief-aqidah", + "urdu_term": "عقیدہ", + "arabic_original": "عقيدة", + "transliteration": "aqidah", + "english_equivalent": "creed or belief", + "category": "aqidah", + "definition_ur": "دین کے بنیادی ایمانی تصورات اور پختہ اعتقادات۔", + "variants": ["عقیدۂ", "عقائد", "عقيدة", "aqidah", "aqeedah"] + }, + { + "id": "ur-ethics-taqwa", + "urdu_term": "تقویٰ", + "arabic_original": "تقوى", + "transliteration": "taqwa", + "english_equivalent": "God-consciousness", + "category": "akhlaq", + "definition_ur": "اللہ کی جواب دہی کا شعور رکھتے ہوئے گناہ سے بچنا۔", + "variants": ["تقوی", "تقوى", "taqwa"] + }, + { + "id": "ur-ethics-sabr", + "urdu_term": "صبر", + "arabic_original": "صبر", + "transliteration": "sabr", + "english_equivalent": "patience and steadfastness", + "category": "akhlaq", + "definition_ur": "اطاعت، مصیبت اور گناہ سے اجتناب میں ثابت قدمی۔", + "variants": ["sabr"] + }, + { + "id": "ur-ethics-dua", + "urdu_term": "دعا", + "arabic_original": "دعاء", + "transliteration": "dua", + "english_equivalent": "supplication", + "category": "ibadat", + "definition_ur": "اللہ تعالیٰ سے حاجت اور مدد طلب کرنا۔", + "variants": ["دعاء", "دُعا", "dua", "du'a"] + }, + { + "id": "ur-family-nikah", + "urdu_term": "نکاح", + "arabic_original": "نكاح", + "transliteration": "nikah", + "english_equivalent": "Islamic marriage contract", + "category": "family", + "definition_ur": "مرد اور عورت کے درمیان شرعی عقد ازدواج۔", + "variants": ["نكاح", "nikah", "nikaah"] + }, + { + "id": "ur-family-talaq", + "urdu_term": "طلاق", + "arabic_original": "طلاق", + "transliteration": "talaq", + "english_equivalent": "divorce", + "category": "family", + "definition_ur": "شرعی طریقے سے نکاح ختم کرنے کا عمل، جس کے احکام تفصیلی ہیں۔", + "variants": ["talaq", "divorce"] + }, + { + "id": "ur-honorific-allah", + "urdu_term": "اللہ تعالیٰ", + "arabic_original": "الله تعالى", + "transliteration": "Allah Ta'ala", + "english_equivalent": "Allah, the Exalted", + "category": "honorific", + "definition_ur": "اللہ کے نام کے ساتھ ادب و تعظیم کا معروف اسلوب۔", + "variants": ["اللہ", "الله", "اللہ تعالٰی", "Allah"] + }, + { + "id": "ur-honorific-prophet", + "urdu_term": "رسول اللہ ﷺ", + "arabic_original": "رسول الله صلى الله عليه وسلم", + "transliteration": "Rasul Allah, sallallahu alayhi wa sallam", + "english_equivalent": "Messenger of Allah, peace and blessings be upon him", + "category": "honorific", + "definition_ur": "نبی کریم محمد ﷺ کے ذکر کا باادب اسلوب۔", + "variants": ["رسول اللہ", "نبی کریم ﷺ", "حضرت محمد ﷺ", "صلی اللہ علیہ وسلم", "PBUH"] + } + ] +} diff --git a/tests/test_urdu_processor.py b/tests/test_urdu_processor.py new file mode 100644 index 0000000..fdd2445 --- /dev/null +++ b/tests/test_urdu_processor.py @@ -0,0 +1,75 @@ +"""Offline tests for Urdu normalization, terminology, and mixed-script processing.""" + +from fastapi.testclient import TestClient +from fastapi import FastAPI + +from urdu_processor import ( + analyze_script, + extract_islamic_terms, + normalize_urdu, + process_urdu, + router, + tokenize_urdu, + transliterate_urdu, +) + + +def test_normalizes_arabic_keyboard_variants_and_nastaliq_spacing() -> None: + assert normalize_urdu(" زكاة\u200c كي حكم؟ ") == "زکاۃ کی حکم؟" + + +def test_diacritics_are_preserved_by_default_and_optionally_removed() -> None: + text = "قُرْآن" + assert "ُ" in normalize_urdu(text) + assert normalize_urdu(text, preserve_diacritics=False) == "قرآن" + + +def test_multiword_islamic_terms_are_single_tokens() -> None: + tokens = tokenize_urdu("فقہ حنفی میں زکوٰۃ کا حکم") + assert tokens[0].text == "فقہ حنفی" + assert tokens[0].kind == "islamic_term" + assert tokens[0].term_id == "ur-fiqh-hanafi" + + +def test_recognizes_urdu_arabic_and_latin_variants_without_duplicates() -> None: + terms = extract_islamic_terms("وضو اور wudu کے بعد نماز salah") + ids = [term.id for term in terms] + assert ids.count("ur-ibadat-wudu") == 1 + assert ids.count("ur-ibadat-salah") == 1 + + +def test_mixed_script_profile() -> None: + profile = analyze_script("زکوٰۃ nisab 2.5%") + assert profile.mixed_script is True + assert profile.dominant_script == "mixed" + assert profile.urdu_arabic_characters > 0 + assert profile.latin_characters > 0 + + +def test_curated_transliteration_is_preferred_for_terms() -> None: + result = transliterate_urdu("زکوٰۃ اور نماز") + assert "zakat" in result + assert "namaz / salah" in result + + +def test_full_analysis_supplies_urdu_generation_and_citation_guidance() -> None: + result = process_urdu("Quran میں صبر کے متعلق کیا حکم ہے؟") + assert result.normalized_text + assert result.script_profile.mixed_script is True + assert any(term.id == "ur-quran-quran" for term in result.recognized_terms) + assert "قرآنی آیات" in result.generation_guidance + assert "حوالہ" in result.generation_guidance + + +def test_router_process_and_term_search() -> None: + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + processed = client.post("/urdu/process", json={"text": "نماز اور وضو"}) + assert processed.status_code == 200 + assert len(processed.json()["recognized_terms"]) == 2 + + searched = client.get("/urdu/terms", params={"query": "tawhid"}) + assert searched.status_code == 200 + assert searched.json()[0]["id"] == "ur-belief-tawhid" diff --git a/urdu_processor.py b/urdu_processor.py new file mode 100644 index 0000000..f70b6fc --- /dev/null +++ b/urdu_processor.py @@ -0,0 +1,360 @@ +"""Urdu Islamic-language processing utilities. + +The processor is deterministic and offline. It normalizes Urdu, Arabic, and +Persian Unicode variants without changing Quranic Arabic by default, recognizes +curated Islamic terminology, preserves multi-word terms during tokenization, +and supplies prompt guidance for an existing multilingual generation pipeline. +""" + +from __future__ import annotations + +import json +import re +import unicodedata +from functools import lru_cache +from pathlib import Path +from typing import Literal + +from fastapi import APIRouter +from pydantic import BaseModel, Field + +router = APIRouter(prefix="/urdu", tags=["urdu"]) + +DATA_PATH = Path(__file__).resolve().parent / "data" / "urdu_islamic_terms.json" + +# Presentation forms are compatibility-normalized by NFKC. These mappings +# then fold Arabic/Persian keyboard variants to the standard Urdu code points. +_URDU_CHAR_MAP = str.maketrans( + { + "ي": "ی", + "ى": "ی", + "ئ": "ئ", + "ك": "ک", + "ة": "ۃ", + "ۀ": "ہ", + "ه": "ہ", + "ھ": "ھ", + "ؤ": "ؤ", + "ـ": "", + "\u200b": "", + "\u200c": "", + "\u200d": "", + "\ufeff": "", + } +) + +# Arabic combining marks, Quranic annotation signs, and superscript alif. +_DIACRITICS_RE = re.compile("[\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06ed]") +_WHITESPACE_RE = re.compile(r"[\t\r\f\v ]+") +_URDU_CHAR_RE = re.compile(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff]") +_LATIN_CHAR_RE = re.compile(r"[A-Za-z]") +_DEVANAGARI_CHAR_RE = re.compile(r"[\u0900-\u097f]") +_TOKEN_RE = re.compile( + r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff]+(?:['’][\u0600-\u06ff]+)*" + r"|[A-Za-z]+(?:['’-][A-Za-z]+)*|\d+(?:[.,:/-]\d+)*|[^\s]" +) + +# Conservative character transliteration intended for discoverability and +# mixed-script search, not for replacing a scholarly transliteration standard. +_TRANSLITERATION_MAP = { + "ا": "a", + "آ": "aa", + "أ": "a", + "إ": "i", + "ء": "'", + "ؤ": "u", + "ئ": "y", + "ب": "b", + "پ": "p", + "ت": "t", + "ٹ": "t", + "ث": "th", + "ج": "j", + "چ": "ch", + "ح": "h", + "خ": "kh", + "د": "d", + "ڈ": "d", + "ذ": "dh", + "ر": "r", + "ڑ": "r", + "ز": "z", + "ژ": "zh", + "س": "s", + "ش": "sh", + "ص": "s", + "ض": "d", + "ط": "t", + "ظ": "z", + "ع": "'", + "غ": "gh", + "ف": "f", + "ق": "q", + "ک": "k", + "گ": "g", + "ل": "l", + "م": "m", + "ن": "n", + "ں": "n", + "و": "w", + "ہ": "h", + "ۃ": "h", + "ھ": "h", + "ی": "y", + "ے": "e", +} + + +class UrduIslamicTerm(BaseModel): + """A curated Islamic term used in Urdu scholarly and everyday writing.""" + + id: str + urdu_term: str + arabic_original: str + transliteration: str + english_equivalent: str + category: str + definition_ur: str + variants: list[str] = Field(default_factory=list) + + +class UrduToken(BaseModel): + """A token with optional terminology metadata.""" + + text: str + normalized: str + kind: Literal["islamic_term", "urdu", "latin", "number", "punctuation"] + term_id: str | None = None + + +class ScriptProfile(BaseModel): + """Character-level script statistics for code-switched input.""" + + urdu_arabic_characters: int + latin_characters: int + devanagari_characters: int + dominant_script: Literal["urdu_arabic", "latin", "devanagari", "mixed", "none"] + mixed_script: bool + + +class UrduAnalysis(BaseModel): + """Normalized text and linguistic signals for downstream retrieval/generation.""" + + original_text: str + normalized_text: str + tokens: list[UrduToken] + recognized_terms: list[UrduIslamicTerm] + script_profile: ScriptProfile + transliteration: str + generation_guidance: str + + +class UrduProcessRequest(BaseModel): + text: str = Field(min_length=1) + preserve_diacritics: bool = True + + +class UrduTerminology: + """Load and search the bundled Urdu Islamic terminology database.""" + + def __init__(self, data_file: Path = DATA_PATH) -> None: + self.terms: list[UrduIslamicTerm] = [] + self._lookup: dict[str, UrduIslamicTerm] = {} + if data_file.exists(): + with data_file.open(encoding="utf-8") as source: + payload = json.load(source) + for raw in payload.get("terms", []): + term = UrduIslamicTerm.model_validate(raw) + self.terms.append(term) + for form in (term.urdu_term, term.arabic_original, term.transliteration, *term.variants): + key = normalize_urdu(form, preserve_diacritics=False).casefold() + if key: + self._lookup[key] = term + + def lookup(self, text: str) -> UrduIslamicTerm | None: + key = normalize_urdu(text, preserve_diacritics=False).casefold() + return self._lookup.get(key) + + def search(self, query: str, limit: int = 20) -> list[UrduIslamicTerm]: + key = normalize_urdu(query, preserve_diacritics=False).casefold() + if not key: + return self.terms[:limit] + exact = self._lookup.get(key) + matches = [ + term + for term in self.terms + if key in normalize_urdu( + " ".join( + [term.urdu_term, term.arabic_original, term.transliteration, term.english_equivalent, *term.variants] + ), + preserve_diacritics=False, + ).casefold() + ] + if exact is not None: + matches = [exact, *(term for term in matches if term.id != exact.id)] + return matches[:limit] + + @property + def phrases(self) -> list[str]: + forms = [form for form in self._lookup if " " in form] + return sorted(forms, key=len, reverse=True) + + +@lru_cache(maxsize=1) +def get_terminology() -> UrduTerminology: + return UrduTerminology() + + +def normalize_urdu(text: str, preserve_diacritics: bool = True) -> str: + """Normalize presentation forms, keyboard variants, spacing, and punctuation. + + Diacritics are preserved by default because they can be semantically + important in Arabic quotations. Callers may remove them for retrieval keys. + """ + normalized = unicodedata.normalize("NFKC", text).translate(_URDU_CHAR_MAP) + if not preserve_diacritics: + normalized = _DIACRITICS_RE.sub("", normalized) + normalized = normalized.replace("?", "؟").replace(",", "،") + lines = [_WHITESPACE_RE.sub(" ", line).strip() for line in normalized.split("\n")] + return "\n".join(line for line in lines if line).strip() + + +def analyze_script(text: str) -> ScriptProfile: + urdu_count = len(_URDU_CHAR_RE.findall(text)) + latin_count = len(_LATIN_CHAR_RE.findall(text)) + devanagari_count = len(_DEVANAGARI_CHAR_RE.findall(text)) + populated = sum(count > 0 for count in (urdu_count, latin_count, devanagari_count)) + if populated > 1: + dominant = "mixed" + elif urdu_count: + dominant = "urdu_arabic" + elif latin_count: + dominant = "latin" + elif devanagari_count: + dominant = "devanagari" + else: + dominant = "none" + return ScriptProfile( + urdu_arabic_characters=urdu_count, + latin_characters=latin_count, + devanagari_characters=devanagari_count, + dominant_script=dominant, + mixed_script=populated > 1, + ) + + +def _protect_phrases(text: str, terminology: UrduTerminology) -> tuple[str, dict[str, str]]: + protected = text + replacements: dict[str, str] = {} + for index, phrase in enumerate(terminology.phrases): + pattern = re.compile(r"(? list[UrduToken]: + """Tokenize mixed Urdu text while keeping known multi-word terms intact.""" + terminology = get_terminology() + normalized_text = normalize_urdu(text) + protected, replacements = _protect_phrases(normalized_text, terminology) + tokens: list[UrduToken] = [] + for raw in _TOKEN_RE.findall(protected): + token_text = replacements.get(raw, raw) + normalized = normalize_urdu(token_text, preserve_diacritics=False) + term = terminology.lookup(normalized) + if term is not None: + kind: Literal["islamic_term", "urdu", "latin", "number", "punctuation"] = "islamic_term" + elif normalized.replace(".", "").replace(",", "").isdigit(): + kind = "number" + elif _URDU_CHAR_RE.search(normalized): + kind = "urdu" + elif _LATIN_CHAR_RE.search(normalized): + kind = "latin" + else: + kind = "punctuation" + tokens.append( + UrduToken( + text=token_text, + normalized=normalized, + kind=kind, + term_id=term.id if term else None, + ) + ) + return tokens + + +def transliterate_urdu(text: str) -> str: + """Return a stable Latin transliteration, preferring curated term forms.""" + terminology = get_terminology() + tokens = tokenize_urdu(text) + rendered: list[str] = [] + for token in tokens: + term = terminology.lookup(token.normalized) + if term is not None: + rendered.append(term.transliteration) + continue + value = "".join(_TRANSLITERATION_MAP.get(char, char) for char in token.normalized) + rendered.append(value) + + result = " ".join(rendered) + result = re.sub(r"\s+([،۔؟!,:;])", r"\1", result) + result = result.replace("،", ",").replace("۔", ".").replace("؟", "?") + return re.sub(r"\s+", " ", result).strip() + + +def extract_islamic_terms(text: str) -> list[UrduIslamicTerm]: + """Recognize terms once, in textual order, across Urdu and Latin forms.""" + terminology = get_terminology() + found: dict[str, UrduIslamicTerm] = {} + for token in tokenize_urdu(text): + term = terminology.lookup(token.normalized) + if term is not None: + found.setdefault(term.id, term) + return list(found.values()) + + +def build_generation_guidance(terms: list[UrduIslamicTerm], profile: ScriptProfile) -> str: + """Create concise instructions for the existing multilingual model pipeline.""" + guidance = [ + "جواب واضح، باادب اور فطری اردو میں دیں۔", + "قرآنی آیات اور احادیث کے اصل عربی متن کو تبدیل نہ کریں اور حوالہ واضح لکھیں۔", + "فقہی اختلاف میں کسی ایک رائے کو بلا وضاحت قطعی یا واحد رائے نہ کہیں۔", + ] + if terms: + labels = "، ".join(f"{term.urdu_term} ({term.transliteration})" for term in terms) + guidance.append(f"سوال میں شناخت شدہ اسلامی اصطلاحات: {labels}۔ انہی مستند املا اور معانی کو ملحوظ رکھیں۔") + if profile.mixed_script: + guidance.append("مخلوط رسم الخط کو سمجھیں، مگر جواب کا بنیادی رسم الخط اردو رکھیں؛ ضروری اصل عربی محفوظ رکھیں۔") + return " ".join(guidance) + + +def process_urdu(text: str, preserve_diacritics: bool = True) -> UrduAnalysis: + """Run the complete Urdu preprocessing and query-understanding pipeline.""" + normalized = normalize_urdu(text, preserve_diacritics=preserve_diacritics) + profile = analyze_script(normalized) + terms = extract_islamic_terms(normalized) + return UrduAnalysis( + original_text=text, + normalized_text=normalized, + tokens=tokenize_urdu(normalized), + recognized_terms=terms, + script_profile=profile, + transliteration=transliterate_urdu(normalized), + generation_guidance=build_generation_guidance(terms, profile), + ) + + +@router.post("/process", response_model=UrduAnalysis) +def process_endpoint(request: UrduProcessRequest) -> UrduAnalysis: + """Normalize and analyze an Urdu or Urdu-code-switched query.""" + return process_urdu(request.text, preserve_diacritics=request.preserve_diacritics) + + +@router.get("/terms", response_model=list[UrduIslamicTerm]) +def search_terms(query: str = "", limit: int = 20) -> list[UrduIslamicTerm]: + """Search curated terminology by Urdu, Arabic, transliteration, or English.""" + safe_limit = min(max(limit, 1), 100) + return get_terminology().search(query, safe_limit)