diff --git a/docs/source/learners/llms4ol.rst b/docs/source/learners/llms4ol.rst index 820171a8..3e1d0363 100644 --- a/docs/source/learners/llms4ol.rst +++ b/docs/source/learners/llms4ol.rst @@ -79,3 +79,4 @@ LLMs4OL is a community development initiative collocated with the International llms4ol_challenge/skhnlp_learner llms4ol_challenge/alexbek_learner llms4ol_challenge/sbunlp_learner + llms4ol_challenge/semanticswingers_learner diff --git a/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst new file mode 100644 index 00000000..421919d6 --- /dev/null +++ b/docs/source/learners/llms4ol_challenge/semanticswingers_learner.rst @@ -0,0 +1,269 @@ +Semantic-Swingers Learner +========================== + + +.. sidebar:: Semantic-Swingers Learner Examples + + * Term Typing: `llm_learner_semanticswingers_term_typing.py `_ + * Taxonomy Discovery: `llm_learner_semanticswingers_taxonomy_discovery.py `_ + * Text2Onto (Task A, flagship): `llm_learner_semanticswingers_text2onto.py `_ + +The Semantic-Swingers team participated in the LLMs4OL 2026 Shared Task. This page documents +the term-typing learner (Task B), the taxonomy-discovery learner (Task C), and the flagship +text2onto + taxonomy-discovery learner (Task A). Tasks B/C share the same design: a strong +sentence-embedding encoder plus a swappable selection step with three interchangeable +backends — an offline embedding heuristic (default, no API key), the OpenAI competition +champion, and a free local Ollama reproduction of the champion pipeline. Task A is different: +its champion is not a prompted API model but the team's own LoRA-fine-tuned open model, so its +learner is a generative retrieval-augmented-generation pipeline instead. + +Term Typing (Task B) +--------------------------------- + +Closed-vocabulary term typing: ``fit`` learns the inventory of allowed type labels from the +train split, and at inference the selector assigns types to each term from that inventory only. + +- ``"embedding"`` (default) — each term gets its nearest type label by sentence-embedding + cosine similarity. Fully offline and deterministic. +- ``"openai"`` — the champion. An OpenAI chat model (default ``gpt-4.1-mini``) classifies + term batches against the closed vocabulary with a precision-biased prompt (multi-type + allowed, abstains when nothing fits, labels copied exactly). +- ``"ollama"`` — the same classification prompt served by a local Ollama model (default + ``llama3.1:8b``). No API key required. + +.. code-block:: python + + from ontolearner import Wine, train_test_split, LearnerPipeline + from ontolearner.learner.term_typing import SemanticSwingersTermTypingLearner + + ontology = Wine() + ontology.load() + train_data, test_data = train_test_split(ontology.extract(), test_size=0.2, random_state=42) + + learner = SemanticSwingersTermTypingLearner(selector="embedding", device="cpu") + + pipeline = LearnerPipeline(llm=learner, llm_id="semanticswingers-term-typing") + outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="term-typing", + evaluate=True, + ) + print(outputs["metrics"]) + +Taxonomy Discovery (Task C) +--------------------------------- + +The learner treats taxonomy discovery as *retrieve-then-select*: + +1. **Retrieve** — a sentence-embedding encoder embeds the type vocabulary; for every child + type, the ``top_k`` nearest neighbours become candidate parents. The team's finding is + that the *encoder* is the main lever for this stage, so the default encoder is + ``mixedbread-ai/mxbai-embed-large-v1``. +2. **Select** — a selection step picks the parent for each child from its candidates. + Three selectors are provided: + + - ``"embedding"`` (default) — fully offline and deterministic. The most *general* + candidate (highest mean similarity to the whole vocabulary) is chosen as parent. + No API key or LLM required; intended as a fast, reproducible baseline. + - ``"openai"`` — the competition champion. An OpenAI chat model (default + ``gpt-4.1-mini``) picks the parent from the retrieved candidates. Requires an + API key (via the ``api_key`` argument or the ``OPENAI_API_KEY`` environment + variable — never hard-coded); without a key the learner silently degrades to + the embedding selector. + - ``"ollama"`` — a free, local reproduction of the champion *pipeline*. The same + selection prompt is served by a local `Ollama `_ model + (default ``llama3.1:8b``) through its OpenAI-compatible endpoint. No API key + required. Prefer direct-answering models here: thinking models (e.g. Qwen3.5) + spend their completion budget on reasoning tokens and need ``max_tokens=1024`` + or more to produce an answer at all. + +The learner requires no training: ``fit`` is a no-op and all edges are induced at +inference time, so it works on unseen ontologies without any target-vocabulary +assumptions. + +Loading Ontological Data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ontolearner import Wine, train_test_split + + ontology = Wine() + ontology.load() + data = ontology.extract() + + train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +Initialize Learner +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ontolearner.learner.taxonomy_discovery import SemanticSwingersTaxonomyLearner + + # Offline baseline (no API key, deterministic) + learner = SemanticSwingersTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + top_k=30, + selector="embedding", + device="cpu", + ) + + # Champion configuration (OpenAI LLM selection) + # learner = SemanticSwingersTaxonomyLearner( + # top_k=30, selector="openai", api_key="", + # ) + + # Local champion-reproduction (no API key; requires a running Ollama server) + # learner = SemanticSwingersTaxonomyLearner( + # top_k=30, selector="ollama", + # ) + +Run the Pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The learner runs on raw ontology objects, so pass ``ontologizer_data=False``. + +.. code-block:: python + + from ontolearner import LearnerPipeline + + pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-taxonomy", + ontologizer_data=False, + ) + + outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, + ) + + print(outputs["metrics"]) + +Scale-aware structural-matrix variant +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``SemanticSwingersMatrixTaxonomyLearner`` is a scalable alternative to the primary +LLM pipeline above, aimed at very large ontologies. On +top of the retrieve-then-select base it adds a trained bilinear structural matrix +``W`` (1024-D, over ``mxbai-embed-large-v1`` embeddings) that scores candidate is-a +edges by direction, plus a DAG cleanup pass (cycle breaking + transitive reduction) +and a matrix-only high-speed bypass for very large type vocabularies +(``llm_threshold``). The matrix weights download automatically from the public +Hugging Face registry ``datagero/taxonomy-structural-matrix-1024-mxbai`` on first +``load()`` when no local copy is present, so it runs from a fresh clone. + +.. code-block:: python + + from ontolearner.learner.taxonomy_discovery import SemanticSwingersMatrixTaxonomyLearner + + learner = SemanticSwingersMatrixTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + top_k=10, + llm_threshold=1000, # matrix-only bypass above this many types + selector="openai", # champion; "embedding"/"ollama" run offline + ) + +Text2Onto + Taxonomy Discovery, joint (Task A, flagship) +--------------------------------------------------------- + +``SemanticSwingersText2OntoLearner`` is ONE class implementing TWO hooks, dispatched via the +``task`` string ``AutoLearner.fit``/``predict`` already receive: + +- ``_text2onto`` — the team's competition champion: retrieval-augmented generation (RAG, + top-``k`` document exemplars) with a LoRA fine-tuned ``Qwen/Qwen3.5-9B`` (RA-FT), extracting + ``[subject, relation, object]`` triples per document and projecting them onto the native + ``{"terms": [...], "types": [...]}`` shape. +- ``_taxonomy_discovery`` — delegates to :class:`SemanticSwingersTaxonomyLearner` (Task C, + documented above) **by composition, not a rewrite**. The native taxonomy-discovery harness + hands the learner a bare type vocabulary with no source document text, so the RAG+FT + generator — which needs text to extract triples from — cannot serve that path; the team's + proven embedding-retrieval taxonomy inducer is the right tool there instead. Expect the + native taxonomy F1 this hook reports to differ from the team's own joint + ``graph_similarity`` figure (RA-FT k10, val_20: ``0.6688``) — that score is a different, + combined metric (term + type + edge overlap together) computed on the team's own document + corpus, not OntoLearner's standalone taxonomy metric on a vocabulary-only benchmark ontology. + A gap here is an expected apples-to-oranges artifact, not a regression. + +``_text2onto``'s returned dict also carries the raw, unprojected triples under an extra +``"triples"`` key (``[[doc_id, subject, relation, object], ...]``). The native +``text2onto_metrics`` scorer reads only ``"terms"``/``"types"`` and silently ignores unknown +keys, so this is purely additive — native scoring is unchanged, while the ``is-a``-dominant +signal the ``{terms, types}`` projection would otherwise discard survives in +``run_report['predictions']`` for downstream inspection. This is the "retained signal" +demonstration referenced in this PR's Future-work section (ADR-0018 addendum §4, team's main +repo ``llms4ol-2026``): a document-grounded, triple-scored harness variant was proposed but +deliberately not built here, since this additive key already preserves the richer signal at +zero core-code cost. + +Model portability (why this needs an unusual install) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``Qwen/Qwen3.5-9B`` uses the ``qwen3_5``/``qwen3_next`` hybrid (dense + linear-attention) +architecture. As of 2026-07-09 no *released* ``transformers`` version registers it — only +``transformers`` installed from git source does: + +.. code-block:: bash + + pip install "transformers @ git+https://github.com/huggingface/transformers.git@1f2fd05824a7ef71a767a122ebd7526ca4e55e40" \ + "peft>=0.19" "accelerate>=1.0" + +This exact commit was verified (2026-07-09) to both register the architecture *and* produce +coherent, on-topic triple extraction when loading the base model plus the team's PEFT adapter — +**never a manually merged/fused checkpoint**: that route was tried and abandoned after producing +a byte-identical-but-semantically-garbage state dict load (see +``docs/ontolearner-native-integration-poc.md`` in the team's main repo, ``llms4ol-2026``, for +the full investigation). This requirement is deliberately **not** added to OntoLearner's core +``pyproject.toml`` — it is heavy (a from-source build) and a moving target that only this one +learner needs. Calling ``learner.load()`` without it raises a clear ``ImportError`` naming the +exact command above. + +.. code-block:: python + + from ontolearner import LearnerPipeline + from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + + train_data = { + "documents": [{"doc_id": "d1", "text": "A poodle is a dog. A dog is a mammal."}], + "triples": {"d1": [["poodle", "is-a", "dog"], ["dog", "is-a", "mammal"]]}, + } + test_data = {"documents": [{"doc_id": "d2", "text": "A tabby is a cat."}]} + + # RA-FT (champion): trained WITH exemplars baked in, wants top_k > 0. + # adapter="baseft", top_k=0 selects the retrieval-free standard fine-tune instead. + learner = SemanticSwingersText2OntoLearner(adapter="raft", top_k=1, device="cpu") + + pipeline = LearnerPipeline(llm=learner, llm_id="semanticswingers-text2onto", ontologizer_data=False) + outputs = pipeline( + train_data=train_data, test_data=test_data, + task="text2onto", evaluate=False, ontologizer_data=False, + ) + print(outputs["predictions"]) + +Reproducibility +--------------------------------- + +Which selector reproduces which reported number, and what is required to run it: + +- **Term typing (Task B)** — the offline ``"embedding"`` selector alone gets close to the + competition champion on Wine (local ``≈0.687`` vs. the champion's ``0.690``). No API key + is needed to reproduce this figure. +- **Taxonomy discovery (Task C)** — the gap is much larger: the paid champion selector + (``"openai"``) reaches ``0.21``, while the offline ``"embedding"`` heuristic reaches only + ``0.07``. Reproducing the champion number for this task requires an OpenAI API key (or the + local ``"ollama"`` selector as a free, unverified approximation of the same prompting + strategy). +- **Determinism** — both offline ``"embedding"`` selectors are fully deterministic: same + encoder, same inputs, same outputs, every run (no sampling, ``temperature`` is irrelevant + since no LLM is called). The ``"openai"``/``"ollama"`` selectors call ``temperature=0`` + but LLM outputs are not guaranteed bit-for-bit reproducible across provider versions. +- **API key handling** — ``selector="openai"`` reads ``api_key`` if passed explicitly, + otherwise falls back to the ``OPENAI_API_KEY`` environment variable; if neither is set, + the learner silently degrades to the offline ``"embedding"`` selector rather than raising, + so pipelines never hard-fail for lack of a key. ``selector="ollama"`` never reads + ``OPENAI_API_KEY`` and needs no key at all — only a local Ollama server. diff --git a/examples/llm_learner_semanticswingers_extend.py b/examples/llm_learner_semanticswingers_extend.py new file mode 100644 index 00000000..98070415 --- /dev/null +++ b/examples/llm_learner_semanticswingers_extend.py @@ -0,0 +1,75 @@ +"""Extending the Semantic-Swingers Task A learner for *your own* experiments. + +The learner is deliberately configuration-driven: base model, adapter, generation backend, +retriever, training regime, the extraction **prompt**, and the relation set that projects to +`types` are all constructor arguments. So most adaptations need **no subclassing at all** — you +pass different arguments. This file shows the three levels of customization, cheapest first. + +Run with a local Ollama (`ollama serve` + a small model) so it needs no API key or GPU. +""" + +from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + + +# --------------------------------------------------------------------------------------------- +# Level 1 — same method, YOUR model / backend / retriever (no code, just arguments) +# --------------------------------------------------------------------------------------------- +# Swap the generator (any OpenAI-compatible endpoint via backend="ollama"/"openai", any local +# checkpoint via backend="peft"/"mlx"), the retriever encoder, and how many exemplars to use. +learner = SemanticSwingersText2OntoLearner( + backend="ollama", + llm_model="llama3.1:8b", # <- your generator + retriever_model_id="sentence-transformers/all-mpnet-base-v2", # <- your retriever + top_k=5, +) + + +# --------------------------------------------------------------------------------------------- +# Level 2 — YOUR domain: a different extraction prompt and a different relation vocabulary +# --------------------------------------------------------------------------------------------- +# `system_prompt` replaces the extraction instructions; `typing_relations` controls which +# relations count as term→type edges when projecting to OntoLearner's {terms, types} shape. +# Neither requires touching the package. +MY_PROMPT = ( + "You are a biomedical ontology engineer. Extract triples [subject, relation, object] using " + "ONLY these relations: rdfs:subClassOf, rdf:type, part_of. " + 'Output ONLY JSON {"triples": [[s, r, o], ...]}.' +) +domain_learner = SemanticSwingersText2OntoLearner( + backend="ollama", + llm_model="llama3.1:8b", + system_prompt=MY_PROMPT, # <- your instructions + typing_relations={"rdf:type", "rdfs:subClassOf"}, # <- your typing relations +) + + +# --------------------------------------------------------------------------------------------- +# Level 3 — YOUR pipeline: subclass to change one step, reuse the rest +# --------------------------------------------------------------------------------------------- +# When a *behaviour* needs to change (not just a value), override a single method. Everything +# else — retrieval, backend dispatch, training, the {terms, types} projection — is inherited. +class MyText2OntoLearner(SemanticSwingersText2OntoLearner): + """Example: post-filter generated triples to a relation allow-list of your choosing.""" + + ALLOW = {"rdf:type", "rdfs:subClassOf", "part_of"} + + def _generate_triples(self, text, exemplars): + triples = super()._generate_triples(text, exemplars) # reuse the whole generation path + return [(s, r, o) for (s, r, o) in triples if r in self.ALLOW] + + +# --------------------------------------------------------------------------------------------- +# Training your own adapter is one more argument, not a separate script (train_mode + fit()). +# --------------------------------------------------------------------------------------------- +# trainer = SemanticSwingersText2OntoLearner( +# train_mode="raft", # or "baseft" +# train_backend="mlx", # or "peft" (CUDA) +# output_dir="my_adapter", +# system_prompt=MY_PROMPT, # trains against YOUR prompt +# ) +# trainer.fit(train_docs, task="text2onto") # builds pairs (leave-one-out for raft) -> trains -> loads + +if __name__ == "__main__": + print("Level 1 learner:", learner.llm_model, "top_k", learner.top_k) + print("Level 2 typing relations:", sorted(domain_learner.typing_relations)) + print("Level 3 subclass:", MyText2OntoLearner().__class__.__name__) diff --git a/examples/llm_learner_semanticswingers_taxonomy_discovery.py b/examples/llm_learner_semanticswingers_taxonomy_discovery.py new file mode 100644 index 00000000..4895dc1e --- /dev/null +++ b/examples/llm_learner_semanticswingers_taxonomy_discovery.py @@ -0,0 +1,71 @@ +from ontolearner import Wine, train_test_split, LearnerPipeline +from ontolearner.learner.taxonomy_discovery import ( + SemanticSwingersTaxonomyLearner, + SemanticSwingersMatrixTaxonomyLearner +) + +# 1) Load & split +ontology = Wine() +ontology.load() +data = ontology.extract() +train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +# ===================================================================== +# APPROACH 1 +# ===================================================================== +# 2) Configure the semantic-swingers taxonomy learner. +# Offline default (no API key): the "embedding" selector. +# Champion: selector="openai" + api_key=... for gpt-4.1-mini parent selection. +# Local champion-reproduction (no API key): selector="ollama" (local Ollama server). +learner = SemanticSwingersTaxonomyLearner( + embedding_model="sentence-transformers/all-MiniLM-L6-v2", # champion: mxbai-embed-large-v1 + top_k=10, + selector="embedding", + device="cpu", +) + +# 3) Build pipeline (pass our learner as `llm`, raw ontology objects) +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-taxonomy", + ontologizer_data=False, +) + +# 4) Train (no-op) + predict + evaluate +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, +) + +print("Metrics:", outputs.get("metrics")) +print("Elapsed time:", outputs["elapsed_time"]) + +# ===================================================================== +# APPROACH 2: 1024-D Structural Matrix + DAG Reduction +# ===================================================================== + +learner2 = SemanticSwingersMatrixTaxonomyLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + matrix_weights_path="structural_matrix_w_1024_mxbai.pt", + top_k=10, + llm_threshold=1000, # Automatically skips LLM for massive datasets + selector="embedding", # change to 'openai' for full competition pipeline + device="cpu", +) + +pipeline2 = LearnerPipeline( + llm=learner2, + llm_id="semanticswingers-taxonomy-champion", + ontologizer_data=False, +) + +outputs2 = pipeline2( + train_data=train_data, + test_data=test_data, + task="taxonomy-discovery", + evaluate=True, + ontologizer_data=False, +) \ No newline at end of file diff --git a/examples/llm_learner_semanticswingers_term_typing.py b/examples/llm_learner_semanticswingers_term_typing.py new file mode 100644 index 00000000..d7c22979 --- /dev/null +++ b/examples/llm_learner_semanticswingers_term_typing.py @@ -0,0 +1,35 @@ +from ontolearner import Wine, train_test_split, LearnerPipeline +from ontolearner.learner.term_typing import SemanticSwingersTermTypingLearner + +# 1) Load & split +ontology = Wine() +ontology.load() +data = ontology.extract() +train_data, test_data = train_test_split(data, test_size=0.2, random_state=42) + +# 2) Configure the semantic-swingers term-typing learner. +# Offline default (no API key): the "embedding" selector (nearest type label). +# Champion: selector="openai" + api_key=... for gpt-4.1-mini classification. +# Local champion-reproduction (no API key): selector="ollama" (local Ollama server). +learner = SemanticSwingersTermTypingLearner( + embedding_model="mixedbread-ai/mxbai-embed-large-v1", + selector="embedding", + device="cpu", +) + +# 3) Build pipeline (pass our learner as `llm`) +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-term-typing", +) + +# 4) Train (learn the type inventory) + predict + evaluate +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="term-typing", + evaluate=True, +) + +print("Metrics:", outputs.get("metrics")) +print("Elapsed time:", outputs["elapsed_time"]) diff --git a/examples/llm_learner_semanticswingers_text2onto.py b/examples/llm_learner_semanticswingers_text2onto.py new file mode 100644 index 00000000..c76a9e98 --- /dev/null +++ b/examples/llm_learner_semanticswingers_text2onto.py @@ -0,0 +1,85 @@ +"""Semantic-Swingers Task A (flagship): text2onto with the RA-FT Qwen3.5-9B generator. + +Requires transformers installed from git source (no released version registers the qwen3_5 +architecture yet) plus peft. See the ImportError raised by `learner.load()` for the exact +pinned pip install command, or docs/ontolearner-native-integration-poc.md in the team's main +repo (llms4ol-2026) for the full model-portability investigation: + + pip install "transformers @ git+https://github.com/huggingface/transformers.git@\ +" \ +"peft>=0.19" "accelerate>=1.0" + +The `adapter` argument accepts a local filesystem path (fastest way to try this without HF +access — point it at the team's `data/ft/_runners/raft_adapter/final` or `baseft_adapter/final`) +or a HuggingFace repo id once the adapters are published (see yp4). +""" + +from ontolearner import LearnerPipeline +from ontolearner.learner.text2onto import SemanticSwingersText2OntoLearner + +# ---- Toy document set (self-contained; swap in the competition's real train/test docs) ---- +train_data = { + "documents": [ + { + "doc_id": "d1", + "text": "A poodle is a dog. A dog is a mammal. A mammal is an animal.", + }, + ], + "triples": { + "d1": [ + ["poodle", "is-a", "dog"], + ["dog", "is-a", "mammal"], + ["mammal", "is-a", "animal"], + ], + }, +} +test_data = { + "documents": [ + { + "doc_id": "d2", + "text": "A tabby is a cat. A cat is a mammal.", + }, + ], +} + +# ---- Configure the learner ---- +# RA-FT (competition champion): trained WITH exemplars baked into the prompt, wants top_k > 0. +# Swap adapter="baseft", top_k=0 for the retrieval-free standard fine-tune instead. +learner = SemanticSwingersText2OntoLearner( + adapter="raft", # or a local path, e.g. "../llms4ol-2026/data/ft/_runners/raft_adapter/final" + top_k=1, + device="cpu", # "cuda" strongly recommended — see the class docstring on CPU speed +) + +pipeline = LearnerPipeline( + llm=learner, + llm_id="semanticswingers-text2onto", + ontologizer_data=False, +) + +outputs = pipeline( + train_data=train_data, + test_data=test_data, + task="text2onto", + evaluate=False, # no gold terms2docs/terms2types provided for this toy example + ontologizer_data=False, +) + +print(outputs["predictions"]) +print("Elapsed time:", outputs["elapsed_time"]) + +# ---- The same learner also serves taxonomy-discovery (composed, not a rewrite) ---- +# from ontolearner import Wine, train_test_split +# +# wine = Wine() +# wine.load() +# wine_train, wine_test = train_test_split(wine.extract(), test_size=0.2, random_state=42) +# +# taxonomy_outputs = pipeline( +# train_data=wine_train, +# test_data=wine_test, +# task="taxonomy-discovery", +# evaluate=True, +# ontologizer_data=False, +# ) +# print(taxonomy_outputs["metrics"]) diff --git a/ontolearner/learner/taxonomy_discovery/__init__.py b/ontolearner/learner/taxonomy_discovery/__init__.py index ec6f2f4f..361c7bef 100644 --- a/ontolearner/learner/taxonomy_discovery/__init__.py +++ b/ontolearner/learner/taxonomy_discovery/__init__.py @@ -15,4 +15,8 @@ from .alexbek import AlexbekCrossAttnLearner from .rwthdbis import RWTHDBISSFTLearner from .sbunlp import SBUNLPFewShotLearner +from .semanticswingers import ( + SemanticSwingersTaxonomyLearner, + SemanticSwingersMatrixTaxonomyLearner, +) from .skhnlp import SKHNLPSequentialFTLearner, SKHNLPZSLearner diff --git a/ontolearner/learner/taxonomy_discovery/semanticswingers.py b/ontolearner/learner/taxonomy_discovery/semanticswingers.py new file mode 100644 index 00000000..2adf34c1 --- /dev/null +++ b/ontolearner/learner/taxonomy_discovery/semanticswingers.py @@ -0,0 +1,392 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers taxonomy-discovery learner (LLMs4OL 2026, Task C). + +A retrieval-first taxonomy inducer: a sentence-embedding encoder embeds the type +vocabulary, nearest neighbours become candidate parents, and a selection step picks +the parent for each child. Three selectors are provided: + +* ``"embedding"`` (default) — fully offline/deterministic; the most *general* candidate + (highest mean similarity to the vocabulary) is chosen as parent. No API key needed. +* ``"openai"`` — the competition champion; an OpenAI chat model picks the parent from + the retrieved candidates. Enabled only when an ``api_key`` is supplied (never hard-coded). +* ``"ollama"`` — free, local reproduction of the champion: the same selection prompt is + served by a local Ollama model through its OpenAI-compatible endpoint. No API key needed. + +The team's finding is that the *encoder* is the bottleneck for this task, so the default +encoder is ``mixedbread-ai/mxbai-embed-large-v1`` (a clean, no-fine-tuning +0.03 over MiniLM). +""" +from __future__ import annotations + +import os +from typing import Any, List, Optional + +import numpy as np +from sentence_transformers import SentenceTransformer, util + +import torch +import torch.nn as nn +import networkx as nx + +from ...base import AutoLearner + +#: Default parent-selection instruction for the LLM selectors. A template with ``{child}`` and +#: ``{candidates}`` placeholders; override per-instance via the ``selection_prompt`` argument. +_SELECTION_PROMPT = ( + "Which of the following is the most likely direct parent (superclass) of '{child}'? " + "Answer with exactly one option or 'NONE'.\n{candidates}" +) + + +def _reasoning_off(model: Optional[str]) -> dict: + """Extra request kwargs that disable a thinking model's hidden reasoning pass. + + Qwen3.x served over an OpenAI-compatible endpoint (e.g. Ollama) is a *thinking* + model: left on, it spends the whole ``max_tokens`` budget on reasoning and returns + empty content (``finish_reason="length"``, ``content=""``), so the selector parses + nothing and scores 0. Passing ``reasoning_effort="none"`` via ``extra_body`` turns + that off — the same fix the competition pipeline applies for the qwen+RAG=0 failure. + A no-op for non-qwen3 models, so it is always safe to include. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + +class SemanticSwingersTaxonomyLearner(AutoLearner): + """Embedding-retrieval taxonomy induction with a swappable parent selector. + + Args: + embedding_model: SentenceTransformer id used to embed type labels. Defaults to + ``mixedbread-ai/mxbai-embed-large-v1`` (the team's champion encoder). + top_k: Number of candidate parents retrieved per child before selection. + selector: ``"embedding"`` (offline heuristic), ``"openai"`` (champion LLM + selection), or ``"ollama"`` (local LLM selection, no API key). + llm_model: Chat model id used by the LLM selectors. Defaults to + ``gpt-4.1-mini`` for ``"openai"`` and ``llama3.1:8b`` for ``"ollama"``. + api_key: OpenAI API key for ``selector="openai"``. If ``None``, falls back to + the ``OPENAI_API_KEY`` env var; if still unset, the learner silently + degrades to the embedding selector. Ignored by ``"ollama"``. + base_url: OpenAI-compatible endpoint for the LLM selector. Defaults to the + local Ollama server (``http://localhost:11434/v1``) when + ``selector="ollama"``, and to the OpenAI API otherwise. + max_tokens: Completion budget for the LLM selector. Direct-answering models + need very little; thinking models (e.g. Qwen3.5) spend reasoning tokens + before answering and need a much larger budget (512+). + device: Torch device for the encoder. + """ + + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "llama3.1:8b"} + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + top_k: int = 30, + selector: str = "embedding", + llm_model: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + max_tokens: int = 64, + device: str = "cpu", + selection_prompt: Optional[str] = None, + ) -> None: + """Initialise the learner and record configuration (no I/O yet). + + ``selection_prompt`` is the parent-selection instruction for the LLM selectors — a template + with ``{child}`` and ``{candidates}`` placeholders. Defaults to :data:`_SELECTION_PROMPT`; + override to change the phrasing / task framing without subclassing. + """ + super().__init__() + self.embedding_model = embedding_model + self.top_k = top_k + self.selector = selector + self.llm_model = llm_model or self._DEFAULT_LLM.get(selector) + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and selector == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url + self.max_tokens = max_tokens + self.device = device + self.selection_prompt = selection_prompt or _SELECTION_PROMPT + self._encoder: Optional[SentenceTransformer] = None + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the sentence-embedding encoder. + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``. A model id that looks + like a SentenceTransformer path (contains ``/``) overrides the default; a plain + bookkeeping label is ignored. + """ + if model_id and "/" in model_id: + self.embedding_model = model_id + self._encoder = SentenceTransformer(self.embedding_model, device=self.device) + + def tasks_data_former(self, data: Any, task: str, test: bool = False) -> Any: + """Pass the raw ``OntologyData`` through (run with ``ontologizer_data=False``).""" + return data + + @staticmethod + def _nodes(data: Any) -> List[str]: + """Return the deduped type vocabulary (declared types + edge endpoints).""" + tt = data.type_taxonomies + nodes: List[str] = list(tt.types) + for rel in tt.taxonomies: + nodes.extend([rel.parent, rel.child]) + seen: set = set() + out: List[str] = [] + for node in nodes: + if node not in seen: + seen.add(node) + out.append(node) + return out + + def _select_embedding(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + """Offline selector: the most general retrieved candidate is the parent. + + Generality is proxied by mean similarity to the whole vocabulary (a central, + broadly-similar type tends to be a superclass). For each child we keep the top + candidate that is more general than the child itself. + """ + generality = sim.mean(axis=1) + preds: List[dict] = [] + for child_idx, child in enumerate(nodes): + candidates = np.argsort(-sim[child_idx])[: self.top_k] + for parent_idx in candidates: + if parent_idx == child_idx: + continue + if generality[parent_idx] >= generality[child_idx]: + preds.append({"parent": nodes[int(parent_idx)], "child": child}) + break + return preds + + def _select_llm(self, nodes: List[str], sim: np.ndarray) -> List[dict]: + """LLM selector: a chat model picks the parent from the retrieved candidates. + + Serves both the ``"openai"`` champion and the local ``"ollama"`` fallback — + the latter is just an OpenAI-compatible endpoint with a placeholder key. + """ + from openai import OpenAI + + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) + preds: List[dict] = [] + for child_idx, child in enumerate(nodes): + candidates = [ + nodes[int(i)] + for i in np.argsort(-sim[child_idx])[: self.top_k] + if int(i) != child_idx + ] + if not candidates: + continue + prompt = self.selection_prompt.format( + child=child, candidates="\n".join(f"- {c}" for c in candidates) + ) + resp = client.chat.completions.create( + model=self.llm_model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model), + ) + answer = (resp.choices[0].message.content or "").strip() + for cand in candidates: + if cand.lower() in answer.lower(): + preds.append({"parent": cand, "child": child}) + break + return preds + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: + """Retrieval-only method: no training; induce edges at inference.""" + if not test: + return None + if self._encoder is None: + self.load() + nodes = self._nodes(data) + emb = np.asarray( + self._encoder.encode(nodes, normalize_embeddings=True, show_progress_bar=False) + ) + sim = emb @ emb.T + use_llm = self.selector == "ollama" or (self.selector == "openai" and self.api_key) + return self._select_llm(nodes, sim) if use_llm else self._select_embedding(nodes, sim) + + +class BilinearAdjacencyLayer(nn.Module): + """1024-D Structural Matrix Layer mapping Child -> Parent asymmetric space.""" + def __init__(self, embedding_dim: int = 1024): + super().__init__() + self.W = nn.Linear(embedding_dim, embedding_dim, bias=False) + + def forward(self, child: torch.Tensor, parent: torch.Tensor) -> torch.Tensor: + transformed_child = self.W(child) + return torch.sum(transformed_child * parent, dim=-1) + + +class SemanticSwingersMatrixTaxonomyLearner(SemanticSwingersTaxonomyLearner): + """Hybrid Structural Matrix + LLM Taxonomy Discovery Learner. + + A scalable alternative to the primary LLM pipeline + (:class:`SemanticSwingersTaxonomyLearner`): a trained bilinear structural matrix + scores candidate is-a edges so per-node LLM inference can be bypassed on very + large ontologies (see ``llm_threshold``), with DAG cleanup on the output. + """ + + #: Public HuggingFace repo holding the trained structural matrix, so the + #: weights auto-download when no local copy is present (reviewers can run this + #: learner straight from a fresh clone). + _HF_REPO_ID = "datagero/taxonomy-structural-matrix-1024-mxbai" + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + matrix_weights_path: str = "structural_matrix_w_1024_mxbai.pt", + top_k: int = 10, + llm_threshold: int = 1000, + hf_repo_id: Optional[str] = None, + **kwargs + ) -> None: + # Pass backend/LLM args up to his constructor + super().__init__(embedding_model=embedding_model, top_k=top_k, **kwargs) + self.matrix_weights_path = matrix_weights_path + self.hf_repo_id = hf_repo_id or self._HF_REPO_ID + self.llm_threshold = llm_threshold + # Force PyTorch device detection + self.device = "mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu") + self._structural_layer = None + + def _resolve_matrix_weights(self) -> str: + """Return a local path to the matrix weights, downloading from the HF + registry on first use when no local file exists.""" + if os.path.exists(self.matrix_weights_path): + return self.matrix_weights_path + from huggingface_hub import hf_hub_download + return hf_hub_download( + repo_id=self.hf_repo_id, + filename=os.path.basename(self.matrix_weights_path), + ) + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Override load to instantiate PyTorch tensors and the Matrix layer.""" + super().load(model_id, **kwargs) # Loads the standard SentenceTransformer + + embedding_dim = self._encoder.get_sentence_embedding_dimension() + self._structural_layer = BilinearAdjacencyLayer(embedding_dim=embedding_dim).to(self.device) + + weights_path = self._resolve_matrix_weights() + self._structural_layer.load_state_dict(torch.load(weights_path, map_location=self.device)) + self._structural_layer.eval() + + def _cleanup_graph(self, raw_predictions: List[dict]) -> List[dict]: + """Enforces DAG rules via NetworkX cycle breaking and Transitive Reduction.""" + if not raw_predictions: + return [] + G = nx.DiGraph() + for p in raw_predictions: + G.add_edge(p["parent"], p["child"]) + + while not nx.is_directed_acyclic_graph(G): + try: + cycle = nx.find_cycle(G) + G.remove_edge(cycle[0][0], cycle[0][1]) + except nx.NetworkXNoCycle: + break + + G_reduced = nx.transitive_reduction(G) + return [{"parent": str(u), "child": str(v)} for u, v in G_reduced.edges()] + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[Any]: + """Override the core logic to use the 1024-D Matrix and hybrid fallback.""" + if not test: + return None + if self._encoder is None or self._structural_layer is None: + self.load() + + nodes = self._nodes(data) # Reusing his method! + if not nodes: + return [] + + # 1. Generate Embeddings for the entire vocabulary + import logging + logger = logging.getLogger(__name__) + logger.info(f"Encoding {len(nodes)} concept nodes...") + + embeddings = self._encoder.encode(nodes, convert_to_tensor=True, show_progress_bar=False) + + raw_preds = [] + # Fallback to Pure Matrix if dataset > llm_threshold to prevent timeout + use_llm = (len(nodes) <= self.llm_threshold) and (self.selector in ["openai", "ollama"]) + + for child_idx, child in enumerate(nodes): + child_vec = embeddings[child_idx] + + # Step 1: Semantic Candidate Retrieval (Top 41 to safely exclude self) + cos_scores = util.cos_sim(child_vec, embeddings)[0] + top_semantic = torch.topk(cos_scores, k=min(41, len(nodes))) + + candidate_indices = [idx for idx in top_semantic[1].tolist() if idx != child_idx] + if not candidate_indices: + continue + + candidate_vecs = torch.stack([embeddings[idx] for idx in candidate_indices]) + candidates = [nodes[idx] for idx in candidate_indices] + + child_vec_expanded = child_vec.unsqueeze(0).repeat(candidate_vecs.size(0), 1) + + # Step 2: 1024-D Structural Matrix Scoring + with torch.no_grad(): + structural_scores = self._structural_layer(child_vec_expanded, candidate_vecs) + + # Step 3: Routing (LLM Verification vs. High-Speed Matrix Bypass) + if use_llm: + # Send top K structural candidates to LLM selector + ranked_indices = torch.argsort(structural_scores, descending=True)[:min(self.top_k, len(candidates))] + top_candidates = [candidates[idx] for idx in ranked_indices] + parent = self._select_llm_parent(child, top_candidates) + if parent: + raw_preds.append({"parent": parent, "child": child}) + else: + # Dataset is massive: take the #1 matrix candidate instantly + best_idx = torch.argmax(structural_scores).item() + raw_preds.append({"parent": candidates[best_idx], "child": child}) + + # Step 4: Clean the graph (Cycle-Breaking + Transitive Reduction) + return self._cleanup_graph(raw_preds) + + def _select_llm_parent(self, child: str, candidates: List[str]) -> Optional[str]: + """Helper method to ask the LLM to pick the best parent from the matrix's short-list.""" + from openai import OpenAI + from ontolearner.learner.taxonomy_discovery.semanticswingers import _reasoning_off + + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) + + prompt = self.selection_prompt.format( + child=child, candidates="\n".join(f"- {c}" for c in candidates) + ) + try: + resp = client.chat.completions.create( + model=self.llm_model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model) # Reuses his Qwen fix! + ) + answer = (resp.choices[0].message.content or "").strip() + for cand in candidates: + if cand.lower() in answer.lower(): + return cand + except Exception as e: + import logging + logging.getLogger(__name__).error(f"LLM selection failed for {child}: {e}") + return None diff --git a/ontolearner/learner/term_typing/__init__.py b/ontolearner/learner/term_typing/__init__.py index dec8b9f9..f70f3f1c 100644 --- a/ontolearner/learner/term_typing/__init__.py +++ b/ontolearner/learner/term_typing/__init__.py @@ -15,3 +15,4 @@ from .alexbek import AlexbekRAGLearner, AlexbekRFLearner from .rwthdbis import RWTHDBISSFTLearner from .sbunlp import SBUNLPZSLearner +from .semanticswingers import SemanticSwingersTermTypingLearner diff --git a/ontolearner/learner/term_typing/semanticswingers.py b/ontolearner/learner/term_typing/semanticswingers.py new file mode 100644 index 00000000..628dfa23 --- /dev/null +++ b/ontolearner/learner/term_typing/semanticswingers.py @@ -0,0 +1,228 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers term-typing learner (LLMs4OL 2026, Task B). + +Closed-vocabulary term typing: ``fit`` learns the inventory of allowed type labels +from the train split, and at inference a selector assigns types to each term from +that inventory only. Three selectors are provided (mirroring the team's Task C +taxonomy-discovery learner): + +* ``"embedding"`` (default) — fully offline/deterministic; each term gets its + nearest type label by sentence-embedding cosine similarity. No API key needed. +* ``"openai"`` — the competition champion; an OpenAI chat model classifies terms + against the closed vocabulary with a precision-biased prompt. Enabled only when + an ``api_key`` is supplied (never hard-coded). +* ``"ollama"`` — free, local reproduction of the champion: the same classification + prompt is served by a local Ollama model through its OpenAI-compatible endpoint. +""" +from __future__ import annotations + +import json +import os +import re +from typing import Any, Dict, List, Optional + +import numpy as np +from sentence_transformers import SentenceTransformer + +from ...base import AutoLearner + +_SYSTEM = ( + "You are an expert ontology engineer specialising in term-typing. " + "You are given terms and a closed vocabulary of allowed type labels. " + "Rules: (1) assign a type only when confident — precision over recall; " + "(2) a term may have multiple types; " + "(3) if no allowed type fits, return an empty list for that term; " + "(4) copy type labels EXACTLY as given, case-sensitive. " + 'Answer as JSON: {"typings": [{"term": "...", "types": ["..."]}]}' +) + + +def _reasoning_off(model: Optional[str]) -> Dict[str, Any]: + """Extra request kwargs that disable a thinking model's hidden reasoning pass. + + Qwen3.x served over an OpenAI-compatible endpoint (e.g. Ollama) is a *thinking* + model: left on, it spends the whole ``max_tokens`` budget on reasoning and returns + empty content (``finish_reason="length"``, ``content=""``), so the selector parses + nothing and scores 0. Passing ``reasoning_effort="none"`` via ``extra_body`` turns + that off — the same fix the competition pipeline applies for the qwen+RAG=0 failure. + A no-op for non-qwen3 models, so it is always safe to include. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + +class SemanticSwingersTermTypingLearner(AutoLearner): + """Closed-vocabulary term typing with a swappable type selector. + + Args: + embedding_model: SentenceTransformer id used by the ``"embedding"`` selector. + Defaults to ``mixedbread-ai/mxbai-embed-large-v1`` (the team's champion encoder). + selector: ``"embedding"`` (offline nearest-type), ``"openai"`` (champion LLM + classification), or ``"ollama"`` (local LLM classification, no API key). + llm_model: Chat model id used by the LLM selectors. Defaults to + ``gpt-4.1-mini`` for ``"openai"`` and ``llama3.1:8b`` for ``"ollama"``. + api_key: OpenAI API key for ``selector="openai"``. If ``None``, falls back to + the ``OPENAI_API_KEY`` env var; if still unset, the learner silently + degrades to the embedding selector. Ignored by ``"ollama"``. + base_url: OpenAI-compatible endpoint for the LLM selector. Defaults to the + local Ollama server (``http://localhost:11434/v1``) when + ``selector="ollama"``, and to the OpenAI API otherwise. + max_tokens: Completion budget per LLM request. Thinking models (e.g. Qwen3.5) + spend reasoning tokens before answering and need a much larger budget. + batch_size: Number of terms classified per LLM request. + device: Torch device for the encoder. + """ + + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "llama3.1:8b"} + + def __init__( + self, + embedding_model: str = "mixedbread-ai/mxbai-embed-large-v1", + selector: str = "embedding", + llm_model: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + max_tokens: int = 1024, + batch_size: int = 20, + device: str = "cpu", + system_prompt: Optional[str] = None, + ) -> None: + """Initialise the learner and record configuration (no I/O yet).""" + super().__init__() + self.embedding_model = embedding_model + self.selector = selector + self.llm_model = llm_model or self._DEFAULT_LLM.get(selector) + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and selector == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url + self.max_tokens = max_tokens + self.batch_size = batch_size + self.device = device + # Extension point: override the classification instructions without subclassing. + self.system_prompt = system_prompt or _SYSTEM + self._encoder: Optional[SentenceTransformer] = None + self._allowed_types: List[str] = [] + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the sentence-embedding encoder. + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``. A model id that + looks like a SentenceTransformer path (contains ``/``) overrides the default; + a plain bookkeeping label is ignored. + """ + if model_id and "/" in model_id: + self.embedding_model = model_id + self._encoder = SentenceTransformer(self.embedding_model, device=self.device) + + def _select_embedding(self, terms: List[str]) -> List[Dict[str, Any]]: + """Offline selector: each term gets its nearest type label by cosine similarity.""" + if self._encoder is None: + self.load() + type_emb = np.asarray( + self._encoder.encode( + self._allowed_types, normalize_embeddings=True, show_progress_bar=False + ) + ) + term_emb = np.asarray( + self._encoder.encode(terms, normalize_embeddings=True, show_progress_bar=False) + ) + sim = term_emb @ type_emb.T + best = np.argmax(sim, axis=1) + return [ + {"term": term, "types": [self._allowed_types[int(best[i])]]} + for i, term in enumerate(terms) + ] + + def _parse_typings(self, content: str, allowed: set) -> Dict[str, List[str]]: + """Parse an LLM answer into ``term -> types``, enforcing the closed vocabulary.""" + text = content.strip() + try: + payload = json.loads(text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if not match: + return {} + try: + payload = json.loads(match.group(0)) + except json.JSONDecodeError: + return {} + items = payload.get("typings", payload) if isinstance(payload, dict) else payload + out: Dict[str, List[str]] = {} + if not isinstance(items, list): + return out + for item in items: + if not isinstance(item, dict): + continue + term = item.get("term") + types = [t for t in item.get("types", []) if t in allowed] + if term: + out[str(term)] = types + return out + + def _select_llm(self, terms: List[str]) -> List[Dict[str, Any]]: + """LLM selector: a chat model classifies term batches against the vocabulary. + + Serves both the ``"openai"`` champion and the local ``"ollama"`` fallback — + the latter is just an OpenAI-compatible endpoint with a placeholder key. + """ + from openai import OpenAI + + api_key = self.api_key if self.selector == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=api_key, base_url=self.base_url) + allowed = set(self._allowed_types) + vocab_block = "\n".join(f"- {t}" for t in self._allowed_types) + preds: List[Dict[str, Any]] = [] + for start in range(0, len(terms), self.batch_size): + batch = terms[start : start + self.batch_size] + user = ( + "Terms to classify:\n" + + "\n".join(f"- {t}" for t in batch) + + "\n\nAllowed type vocabulary (use ONLY these exact labels):\n" + + vocab_block + + "\n\nAssign types to each term. Answer as JSON." + ) + params: Dict[str, Any] = dict( + model=self.llm_model, + messages=[ + {"role": "system", "content": self.system_prompt}, + {"role": "user", "content": user}, + ], + temperature=0, + max_tokens=self.max_tokens, + **_reasoning_off(self.llm_model), + ) + try: + resp = client.chat.completions.create( + response_format={"type": "json_object"}, **params + ) + except Exception: + resp = client.chat.completions.create(**params) + typed = self._parse_typings(resp.choices[0].message.content or "", allowed) + preds.extend({"term": t, "types": typed.get(t, [])} for t in batch) + return preds + + def _term_typing(self, data: Any, test: bool = False) -> Optional[List[Dict[str, Any]]]: + """Train mode: learn the type inventory. Test mode: type each term.""" + if not test: + self._allowed_types = sorted(set(data)) + return None + terms = [str(t) for t in data] + if not terms or not self._allowed_types: + return [{"term": t, "types": []} for t in terms] + use_llm = self.selector == "ollama" or (self.selector == "openai" and self.api_key) + return self._select_llm(terms) if use_llm else self._select_embedding(terms) diff --git a/ontolearner/learner/text2onto/__init__.py b/ontolearner/learner/text2onto/__init__.py index af31523c..83651169 100644 --- a/ontolearner/learner/text2onto/__init__.py +++ b/ontolearner/learner/text2onto/__init__.py @@ -14,3 +14,4 @@ from .alexbek import AlexbekRAGFewShotLearner from .sbunlp import SBUNLPFewShotLearner +from .semanticswingers import SemanticSwingersText2OntoLearner diff --git a/ontolearner/learner/text2onto/semanticswingers.py b/ontolearner/learner/text2onto/semanticswingers.py new file mode 100644 index 00000000..da27feff --- /dev/null +++ b/ontolearner/learner/text2onto/semanticswingers.py @@ -0,0 +1,596 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Semantic-Swingers Task A (flagship) learner (LLMs4OL 2026): joint text2onto + taxonomy-discovery. + +ONE learner class, TWO hooks, dispatched via ``AutoLearner.fit``/``predict``'s ``task`` string: + +* ``_text2onto`` — the team's competition champion: retrieval-augmented generation (RAG, top-k + document exemplars) with a LoRA fine-tuned ``Qwen/Qwen3.5-9B`` (RA-FT), extracting + ``[subject, relation, object]`` triples per document and projecting them onto OntoLearner's + native ``{"terms": [...], "types": [...]}`` shape. This is a document-driven generative method + (needs source text), unlike Task B/C's classification-style selectors. +* ``_taxonomy_discovery`` — reuses :class:`~ontolearner.learner.taxonomy_discovery.semanticswingers. + SemanticSwingersTaxonomyLearner` **by composition** (delegation, not a rewrite). The native + OntoLearner taxonomy-discovery harness hands the learner a bare type *vocabulary* with no source + text (see ``AutoLearner.tasks_data_former``'s default reshaping for that task), so the RAG+FT + generator — which needs document text to extract triples from — cannot serve that invocation + path; the team's proven embedding-retrieval taxonomy inducer is the right tool there instead. + Expect the native taxonomy-discovery F1 reported by this hook to differ from the team's joint + ``graph_similarity`` score (val_20 RA-FT k10 = 0.6688) — that figure scores a *different*, + combined metric (term/type/edge overlap together) on the team's own document corpus, not + OntoLearner's standalone taxonomy metric on a vocabulary-only benchmark ontology. A gap here is + an expected apples-to-oranges artifact, not a regression. + +Model portability background (``docs/ontolearner-native-integration-poc.md`` in the team's main +repo, section "Model portability"): ``Qwen/Qwen3.5-9B`` uses the ``qwen3_5``/``qwen3_next`` hybrid +(dense + linear-attention) architecture, which no *released* ``transformers`` version registers as +of 2026-07-09 — only ``transformers`` installed from git source does. Loading the base model plus +the team's PEFT LoRA adapter **unmerged** (never a manually fused/merged checkpoint — that route +was tried and abandoned after producing incoherent output despite a byte-identical state dict) is +the exact recipe already used to produce every graded Task A result the team has, and is the one +implemented here. +""" +from __future__ import annotations + +import ast +import json +import os +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from ...base import AutoLearner, AutoRetriever +from ..taxonomy_discovery.semanticswingers import SemanticSwingersTaxonomyLearner + +#: Exact transformers commit verified (2026-07-09) to (a) register `qwen3_5`/`qwen3_next` in +#: `CONFIG_MAPPING` and (b) load base `Qwen/Qwen3.5-9B` + the team's PEFT adapter producing +#: coherent, on-topic triple extraction on a real held-out document (not just a token count). +#: Never float `@main` — the qwen3_5 modeling code is young enough that semantics have already +#: drifted once between dev snapshots (see the POC doc's fused-checkpoint post-mortem). +_VERIFIED_TRANSFORMERS_COMMIT = "1f2fd05824a7ef71a767a122ebd7526ca4e55e40" +_PIP_INSTALL_LINE = ( + f'pip install "transformers @ git+https://github.com/huggingface/transformers.git' + f'@{_VERIFIED_TRANSFORMERS_COMMIT}" "peft>=0.19" "accelerate>=1.0"' +) + +_SYSTEM_PROMPT = ( + "You are an expert ontology engineer. Extract a primitive ontology from the document as " + "triples [subject, relation, object].\n" + "RULES:\n" + "1. Extract Terms (instances) and Types (classes).\n" + "2. Dominant relations: 'is-a' (subclassing), 'instance-of' or 'type' (term typing), " + "'equivalent class' (synonyms), 'disjoint with', 'part_of'/'has part'. Use these exact " + "labels; reuse consistent labels.\n" + "3. WARNING: Most texts only contain taxonomic definitions. Do NOT extract semantic " + "relationships unless explicitly demanded by strict logical constraints. If there are no " + "relations other than 'is-a', do not invent them.\n" + "4. Direction matters: emit Child -> is-a -> Parent.\n" + 'Output ONLY JSON {"triples": [[subject, relation, object], ...]}; entities grounded in ' + "the text." +) +_NOTHINK_SUFFIX = "<|im_start|>assistant\n\n\n\n\n" +_TYPING_RELATIONS = {"is-a", "instance-of", "type"} + +# HuggingFace repo ids for the two published PEFT adapters (private until competition submission +# 2026-07-26; override via the `adapter` constructor arg with a local path in the meantime, e.g. +# the team's `data/ft/_runners/raft_adapter/final`). +_ADAPTER_REPOS = { + "raft": "datagero/qwen3.5-9b-ontology-extraction-raft", + "baseft": "datagero/qwen3.5-9b-ontology-extraction-baseft", + # Apple-Silicon 4-bit MLX variant (base-FT regime); pair with backend="mlx". + "baseft-mlx": "datagero/qwen3.5-9b-ontology-extraction-baseft-mlx", +} + + +def _require_qwen35_transformers() -> None: + """Fail loudly and actionably if the installed transformers doesn't know qwen3_5. + + Lazy import guard (RWTH/g4f optional-dep precedent): the git-source transformers requirement + is NOT added to OntoLearner's core ``pyproject.toml`` — it is heavy (build-from-source), a + moving target, and only this one learner needs it. Callers see a clear, copy-pasteable fix. + """ + try: + from transformers.models.auto.configuration_auto import CONFIG_MAPPING + except ImportError as e: # pragma: no cover - transformers itself missing + raise ImportError( + "SemanticSwingersText2OntoLearner requires transformers with qwen3_5/qwen3_next " + f"support, but transformers is not importable at all. Install it with:\n" + f" {_PIP_INSTALL_LINE}" + ) from e + if "qwen3_5" not in CONFIG_MAPPING: + raise ImportError( + "SemanticSwingersText2OntoLearner requires a transformers build that registers the " + "'qwen3_5' architecture (Qwen/Qwen3.5-9B's model type). No released transformers " + "version ships this yet (verified 2026-07-09) — install from git source, pinned to " + "the commit this learner was validated against:\n" + f" {_PIP_INSTALL_LINE}\n" + "(see docs/ontolearner-native-integration-poc.md in the team's main repo for the " + "full portability investigation)." + ) + + +def _parse_triples_json(text: str) -> List[Any]: + """Robust JSON/markdown-fence extraction of a ``{"triples": [...]}`` payload. + + Ported (self-contained, no cross-repo import) from the team's ``src/experiments/ + triple_utils.py::_parse`` — kept intentionally small so the fork carries no dependency on the + private llms4ol-2026 competition repo. + """ + if not isinstance(text, str) or not text: + return [] + m = re.search(r"`{3}(?:json)?\s*([\s\S]*?)\s*`{3}", text, re.IGNORECASE) + body = m.group(1).strip() if m else text + if not m: + a, b = body.find("["), body.rfind("]") + if a != -1 and b != -1: + body = body[a : b + 1] + for loader in (json.loads, ast.literal_eval): + try: + d = loader(body) + if isinstance(d, dict): + d = d.get("triples", []) + if isinstance(d, list): + return d + except Exception: + continue + return [] + + +def _to_triples(items: List[Any]) -> List[Tuple[str, str, str]]: + """Coerce parsed triple items (list-of-3 or dict shapes) into clean ``(s, r, o)`` tuples.""" + out: List[Tuple[str, str, str]] = [] + for t in items or []: + if isinstance(t, dict): + s = t.get("subject", t.get("head", t.get("source", ""))) + r = t.get("relation", t.get("predicate", t.get("type", ""))) + o = t.get("object", t.get("tail", t.get("target", ""))) + elif isinstance(t, (list, tuple)) and len(t) == 3: + s, r, o = t + else: + continue + s, r, o = str(s).strip(), str(r).strip().lower(), str(o).strip() + if s and r and o: + out.append((s, r, o)) + return out + + +def _reasoning_off(model: Optional[str]) -> Dict[str, Any]: + """Disable a thinking model's hidden reasoning pass (qwen3.x over an OpenAI-compatible API). + + Left on, qwen3.x spends the whole ``max_tokens`` budget reasoning and returns empty content, + so nothing parses. No-op for other models. Mirrors the Task B/C learners. + """ + if str(model or "").startswith("qwen3"): + return {"extra_body": {"reasoning_effort": "none"}} + return {} + + +class SemanticSwingersText2OntoLearner(AutoLearner): + """RAG + LoRA-fine-tuned Qwen3.5-9B triple extractor, serving text2onto and taxonomy-discovery. + + Args: + adapter: ``"raft"`` (retrieval-aware fine-tune, the competition champion — trained with + exemplars baked into the prompt, wants ``top_k > 0`` at inference) or ``"baseft"`` + (standard fine-tune, trained without exemplars — best used retrieval-free, + ``top_k=0``). Either may also be a local filesystem path or a HuggingFace repo id to + a PEFT adapter directory, overriding the published default for that name. + base_model_id: HuggingFace id of the frozen base model the adapter was trained on. + top_k: Number of retrieved training-document exemplars baked into the generation prompt + (RA-FT's regime; ``0`` degrades to zero-shot, base-FT's regime). + retriever_model_id: SentenceTransformer id used to embed documents for exemplar retrieval. + max_new_tokens: Generation budget per document. + device: Torch device string (``"cpu"``, ``"cuda"``, ``"mps"``, ...). CPU works but is + slow without the ``fla``/``causal-conv1d`` fast kernels (GPU-only); see the class + docstring's model-portability note. + system_prompt: Extraction instructions handed to the generator. Defaults to the team's + (:data:`_SYSTEM_PROMPT`). **Override to adapt to your domain / relation vocabulary** + without subclassing. + typing_relations: The relations that project a triple ``(s, r, o)`` to a ``type`` when + reshaping to OntoLearner's ``{terms, types}`` output. Defaults to + ``{"is-a", "instance-of", "type"}``; override for a different schema (e.g. + ``{"rdf:type", "rdfs:subClassOf"}``). + taxonomy_kwargs: Extra keyword arguments forwarded to the composed + :class:`SemanticSwingersTaxonomyLearner` used for the ``_taxonomy_discovery`` hook. + + Extending this learner (see ``examples/llm_learner_semanticswingers_extend.py``): + * **Different model / backend / retriever** — pass ``backend``, ``llm_model``, + ``base_model_id``, ``adapter``, ``retriever_model_id``. No code. + * **Different domain** — pass ``system_prompt`` and ``typing_relations``. No code. + * **Different behaviour** — subclass and override one method (e.g. ``_generate_triples``); + retrieval, backend dispatch, training and the projection are all inherited. + * **Your own adapter** — ``train_mode`` + ``fit()`` train against whatever ``system_prompt`` + you set. + """ + + _OLLAMA_BASE_URL = "http://localhost:11434/v1" + _DEFAULT_LLM = {"openai": "gpt-4.1-mini", "ollama": "qwen3.5-nothink:9b"} + + def __init__( + self, + adapter: str = "raft", + base_model_id: str = "Qwen/Qwen3.5-9B", + backend: str = "peft", + llm_model: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + top_k: int = 10, + retriever_model_id: str = "sentence-transformers/all-MiniLM-L6-v2", + max_new_tokens: int = 1500, + device: str = "cpu", + system_prompt: Optional[str] = None, + typing_relations: Optional[set] = None, + train_mode: Optional[str] = None, + train_backend: str = "peft", + output_dir: Optional[str] = None, + train_kwargs: Optional[Dict[str, Any]] = None, + taxonomy_kwargs: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialise the learner and record configuration (no I/O, no model load yet). + + Training (optional): set ``train_mode`` to ``"raft"`` (retrieval-aware, needs ``top_k>0``) + or ``"baseft"`` (no exemplars, ``top_k=0``) and ``fit()`` will *train* a LoRA adapter into + ``output_dir`` and then load it, instead of only indexing exemplars. ``train_backend`` picks + the trainer: ``"peft"`` (CUDA) or ``"mlx"`` (Apple Silicon). ``train_mode=None`` (default) + keeps the pure-inference behaviour unchanged. + """ + super().__init__() + self.adapter = _ADAPTER_REPOS.get(adapter, adapter) + self.base_model_id = base_model_id + self.backend = backend + # Extension points: bring your own extraction instructions / typing relations without + # subclassing. Default to the team's, so existing behaviour is unchanged. + self.system_prompt = system_prompt or _SYSTEM_PROMPT + self.typing_relations = set(typing_relations) if typing_relations else _TYPING_RELATIONS + self.train_mode = train_mode + self.train_backend = train_backend + self.output_dir = output_dir + self.train_kwargs = train_kwargs or {} + self.llm_model = llm_model or self._DEFAULT_LLM.get(backend) + self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + if base_url is None and backend == "ollama": + base_url = self._OLLAMA_BASE_URL + self.base_url = base_url + self.top_k = top_k + self.retriever_model_id = retriever_model_id + self.max_new_tokens = max_new_tokens + self.device = device + + self._model: Optional[Any] = None + self._tokenizer: Optional[Any] = None + self._retriever = AutoRetriever() + self._train_docs: List[Dict[str, Any]] = [] # {"doc_id", "text", "triples"} + + # Composition, not rewrite: the native taxonomy-discovery invocation gives us a bare + # type vocabulary (no document text), so it gets the team's proven embedding-retrieval + # inducer verbatim rather than a second copy of the generative pipeline. + self._taxonomy_learner = SemanticSwingersTaxonomyLearner(**(taxonomy_kwargs or {})) + + def tasks_data_former(self, data: Any, task: str, test: bool = False) -> Any: + """Pass the raw data through unchanged for both hooks (run with ``ontologizer_data=False``). + + ``_text2onto`` expects ``{"documents": [...], "terms2docs": {...}, "terms2types": {...}}`` + (mirrors the ``alexbek`` text2onto learner's contract); ``_taxonomy_discovery`` expects + the native ``OntologyData`` object the composed :class:`SemanticSwingersTaxonomyLearner` + already knows how to read. + """ + return data + + def load(self, model_id: Optional[str] = None, **kwargs: Any) -> None: + """Load the base model + PEFT adapter (lazily; only once). + + Called by ``LearnerPipeline`` as ``load(model_id=llm_id)``; a ``model_id`` containing + ``"/"`` overrides ``base_model_id``, a plain bookkeeping label is ignored (mirrors the + team's Task B/C learners). + """ + if self._model is not None: + return + if model_id and "/" in model_id and model_id != self.adapter: + self.base_model_id = model_id + + if self.backend in ("ollama", "openai"): + # API-served backends need no local weights: the generator is remote. + self._model = self # sentinel so `_generate_triples` sees an initialised learner + self._taxonomy_learner.load(model_id=self.retriever_model_id) + return + + if self.backend == "mlx": + # Apple-Silicon path: MLX base + (optional) MLX-format LoRA adapter. Runs the + # fine-tuned champions on a Mac, where the transformers/PEFT path falls back to CPU. + try: + from mlx_lm import load as _mlx_load + except ImportError as e: + raise RuntimeError( + "backend='mlx' needs mlx-lm (Apple Silicon). `pip install mlx-lm`. " + "Original error: " + str(e) + ) from e + mlx_base = self.base_model_id + if mlx_base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling + mlx_base = "mlx-community/Qwen3.5-9B-4bit" + adapter_path = self.adapter if self.adapter and self.adapter != "none" else None + # mlx_lm.load(adapter_path=...) only accepts a LOCAL dir — unlike PEFT it does not + # fetch a HF repo id. If the adapter is a repo id (not an on-disk path), pull it first. + if adapter_path and not Path(adapter_path).exists(): + from huggingface_hub import snapshot_download + adapter_path = snapshot_download( + repo_id=adapter_path, + allow_patterns=["adapters.safetensors", "adapter_config.json", + "adapter_model.safetensors"], + ) + self._mlx_model, self._tokenizer = _mlx_load(mlx_base, adapter_path=adapter_path) + self._model = self # sentinel + self._taxonomy_learner.load(model_id=self.retriever_model_id) + return + + _require_qwen35_transformers() + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + from peft import PeftModel + + self._tokenizer = AutoTokenizer.from_pretrained(self.base_model_id, trust_remote_code=True) + # Place the model on the requested device. Without device_map the base loads on CPU + # regardless of `device=`, so a "cuda" run would silently execute the 9B on CPU + # (minutes/doc). device_map loads weights straight onto the accelerator (also avoids a + # CPU-RAM spike vs. load-then-.to). CPU stays the default when device is cpu/unset. + _device_map = self.device if self.device and self.device != "cpu" else None + base = AutoModelForCausalLM.from_pretrained( + self.base_model_id, + dtype=torch.bfloat16, + trust_remote_code=True, + low_cpu_mem_usage=True, + device_map=_device_map, + ) + self._model = PeftModel.from_pretrained(base, self.adapter) + self._model.eval() + + self._taxonomy_learner.load(model_id=self.retriever_model_id) + + # -- shared generation -------------------------------------------------------------------- + + def _exemplar_block(self, ex: Dict[str, Any], max_ctx: int = 1000, max_triples: int = 20) -> str: + """One retrieved training exemplar rendered as a user/assistant turn pair.""" + trips = [list(t) for t in ex["triples"][:max_triples]] + payload = json.dumps({"triples": trips}, ensure_ascii=False) + return ( + f"<|im_start|>user\nDocument:\n{ex['text'][:max_ctx]}\n\n" + f"Extract the ontology triples as JSON.<|im_end|>\n" + f"<|im_start|>assistant\n{payload}<|im_end|>\n" + ) + + def _build_prompt(self, text: str, exemplars: List[Dict[str, Any]]) -> str: + """Render the generation prompt for the local (peft/mlx) backends. + + Uses the loaded tokenizer's **own chat template** so the learner works with any model + family, not just Qwen's ChatML. Verified byte-identical to the hand-built ChatML for + ``Qwen/Qwen3.5-9B`` (``enable_thinking=False`` reproduces the ```` no-think + suffix exactly), so this does not perturb the team's trained adapters. Falls back to the + Qwen ChatML string only if no tokenizer / chat template is available. + """ + msgs = self._chat_messages(text, exemplars) + tok = getattr(self, "_tokenizer", None) + if tok is not None and getattr(tok, "chat_template", None): + try: + return tok.apply_chat_template( + msgs, add_generation_prompt=True, tokenize=False, enable_thinking=False + ) + except TypeError: # tokenizer's template doesn't take enable_thinking + return tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) + # Fallback: the original hand-built Qwen ChatML (no tokenizer available). + ex_block = "".join(self._exemplar_block(e) for e in exemplars) + user = (f"<|im_start|>user\nDocument:\n{text[:4000]}\n\n" + "Extract the ontology triples as JSON.<|im_end|>\n") + return ( + f"<|im_start|>system\n{self.system_prompt}<|im_end|>\n" + ex_block + user + _NOTHINK_SUFFIX + ) + + @staticmethod + def _strip_think(text: str) -> str: + if "" in text: + text = text.split("", 1)[1] + return text.strip() + + def _chat_messages(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Dict[str, str]]: + """Same prompt content as ``_build_prompt``, in OpenAI chat-message form. + + The PEFT path renders exemplars into a single ChatML string; API backends want + structured turns. Content is identical so the two backends stay comparable. + """ + msgs: List[Dict[str, str]] = [{"role": "system", "content": self.system_prompt}] + for ex in exemplars: + payload = json.dumps({"triples": [list(t) for t in ex["triples"][:20]]}, + ensure_ascii=False) + msgs.append({"role": "user", + "content": f"Document:\n{ex['text'][:1000]}\n\n" + "Extract the ontology triples as JSON."}) + msgs.append({"role": "assistant", "content": payload}) + msgs.append({"role": "user", + "content": f"Document:\n{text[:4000]}\n\n" + "Extract the ontology triples as JSON."}) + return msgs + + def _generate_api(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Generate via an OpenAI-compatible endpoint (``ollama`` or ``openai`` backend).""" + from openai import OpenAI + + key = self.api_key if self.backend == "openai" else (self.api_key or "ollama") + client = OpenAI(api_key=key, base_url=self.base_url) + params: Dict[str, Any] = dict( + model=self.llm_model, + messages=self._chat_messages(text, exemplars), + temperature=0, + max_tokens=self.max_new_tokens, + **_reasoning_off(self.llm_model), + ) + try: + resp = client.chat.completions.create(response_format={"type": "json_object"}, **params) + except Exception: + resp = client.chat.completions.create(**params) + return _to_triples(_parse_triples_json( + self._strip_think(resp.choices[0].message.content or ""))) + + def _generate_mlx(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Generate via MLX (Apple Silicon) using the same ChatML prompt as the PEFT path.""" + from mlx_lm import generate as _mlx_generate + from mlx_lm.sample_utils import make_sampler + + if self._model is None: + self.load() + prompt = self._build_prompt(text, exemplars) + out = _mlx_generate(self._mlx_model, self._tokenizer, prompt=prompt, + max_tokens=self.max_new_tokens, + sampler=make_sampler(temp=0.0), verbose=False) + return _to_triples(_parse_triples_json(self._strip_think(out))) + + def _generate_triples(self, text: str, exemplars: List[Dict[str, Any]]) -> List[Tuple[str, str, str]]: + """Run the generator on one document and return parsed ``(s, r, o)`` triples.""" + if self.backend in ("ollama", "openai"): + return self._generate_api(text, exemplars) + if self.backend == "mlx": + return self._generate_mlx(text, exemplars) + + import torch + + if self._model is None: + self.load() + prompt = self._build_prompt(text, exemplars) + ids = self._tokenizer(prompt, return_tensors="pt").to(self._model.device) + with torch.no_grad(): + out = self._model.generate( + **ids, + max_new_tokens=self.max_new_tokens, + do_sample=False, + pad_token_id=self._tokenizer.eos_token_id, + ) + decoded = self._tokenizer.decode(out[0][ids["input_ids"].shape[1] :], skip_special_tokens=True) + return _to_triples(_parse_triples_json(self._strip_think(decoded))) + + def _retrieve_exemplars(self, text: str) -> List[Dict[str, Any]]: + if self.top_k <= 0 or not self._train_docs: + return [] + hits = self._retriever.retrieve([text], top_k=self.top_k)[0] + by_text = {d["text"]: d for d in self._train_docs} + return [by_text[h] for h in hits if h in by_text] + + def _retrieve_exemplars_loo(self, text: str, exclude_id: str) -> List[Dict[str, Any]]: + """Leave-one-out retrieval for TRAINING: never return the document's own gold. + + Over-fetch by one then drop the self-match, so a training prompt for doc X is built from + *other* documents' exemplars. Dropping this is silent leakage that inflates the score. + """ + if self.top_k <= 0 or not self._train_docs: + return [] + hits = self._retriever.retrieve([text], top_k=self.top_k + 1)[0] + by_text = {d["text"]: d for d in self._train_docs} + out = [by_text[h] for h in hits if h in by_text and by_text[h].get("doc_id") != exclude_id] + return out[: self.top_k] + + def _fit_adapter(self) -> None: + """Train a LoRA adapter from ``self._train_docs`` and load it for inference. + + Delegates to :mod:`semanticswingers_train`: build ``{prompt, completion}`` pairs (with + leave-one-out exemplars when ``train_mode="raft"``), run the chosen trainer, then point + ``self.adapter`` at the produced adapter and reload. Keeps this class free of the heavy + training stack unless training is actually requested. + """ + from .semanticswingers_train import TrainConfig, build_training_pairs, train_adapter + + if not self.output_dir: + raise ValueError("train_mode is set but output_dir is None — nowhere to write the adapter.") + pairs = build_training_pairs( + self._train_docs, self._build_prompt, self._retrieve_exemplars_loo, self.train_mode + ) + cfg = TrainConfig( + output_dir=self.output_dir, base_model_id=self.base_model_id, + train_mode=self.train_mode, top_k=self.top_k, **self.train_kwargs, + ) + produced = train_adapter(pairs, cfg, self.train_backend) + # switch to inference on the freshly trained adapter + self.adapter = produced + self.backend = "mlx" if self.train_backend == "mlx" else "peft" + self._model = None + self.load() + + # -- _text2onto ----------------------------------------------------------------------------- + + @staticmethod + def _doc_id(d: Dict[str, Any]) -> str: + return str(d.get("doc_id") or d.get("id") or d.get("docid") or "") + + @staticmethod + def _doc_text(d: Dict[str, Any]) -> str: + return str(d.get("text") or d.get("context") or "") + + def _text2onto(self, data: Any, test: bool = False) -> Optional[Dict[str, List[Any]]]: + """Train mode: index document exemplars. Test mode: extract triples -> {terms, types}. + + The returned dict also carries the raw, unprojected ``"triples"`` under an extra key + (``[[doc_id, subject, relation, object], ...]``). ``text2onto_metrics`` (OntoLearner's + native scorer, ``evaluation/metrics.py:81-90``) reads only ``"terms"``/``"types"`` and + silently ignores unknown keys, and the full dict passes through unchanged to + ``run_report['predictions']`` (``_learner.py:120``) — so this is purely additive: native + scoring is identical, and the ~90% ``is-a`` signal the ``{terms, types}`` projection + would otherwise discard survives in the returned predictions for downstream inspection + (see ADR-0018 addendum §4 in the team's main repo, ``llms4ol-2026``). + """ + if not isinstance(data, dict): + raise ValueError( + "text2onto expects {'documents': [...], 'terms2docs': {...}, 'terms2types': " + "{...}} (see AlexbekRAGFewShotLearner's contract)." + ) + docs = data.get("documents") or data.get("docs") or [] + gold_triples = data.get("triples") or {} # optional: doc_id -> [[s, r, o], ...] + + if not test: + self._train_docs = [] + for d in docs: + did = self._doc_id(d) + text = self._doc_text(d) + triples = gold_triples.get(did, []) + self._train_docs.append({"doc_id": did, "text": text, "triples": triples}) + if self._train_docs: + self._retriever.load(self.retriever_model_id) + self._retriever.index([d["text"] for d in self._train_docs]) + if self.train_mode: + self._fit_adapter() # train a LoRA adapter, then load it for inference + return None + + if self._model is None: + self.load() + + pred_terms: List[Dict[str, str]] = [] + pred_types: List[Dict[str, str]] = [] + pred_triples: List[List[str]] = [] + for d in docs: + did = self._doc_id(d) + text = self._doc_text(d) + exemplars = self._retrieve_exemplars(text) + triples = self._generate_triples(text, exemplars) + + terms_seen: set = set() + types_seen: set = set() + for s, r, o in triples: + pred_triples.append([did, s, r, o]) + if s not in terms_seen: + terms_seen.add(s) + pred_terms.append({"doc_id": did, "term": s}) + if r in self.typing_relations and o not in types_seen: + types_seen.add(o) + pred_types.append({"doc_id": did, "type": o}) + + return {"terms": pred_terms, "types": pred_types, "triples": pred_triples} + + # -- _taxonomy_discovery (composed, not reimplemented) --------------------------------------- + + def _taxonomy_discovery(self, data: Any, test: bool = False) -> Optional[List[Dict[str, str]]]: + """Delegate to the team's proven embedding-retrieval taxonomy inducer (Task C hook).""" + return self._taxonomy_learner._taxonomy_discovery(data, test=test) diff --git a/ontolearner/learner/text2onto/semanticswingers_train.py b/ontolearner/learner/text2onto/semanticswingers_train.py new file mode 100644 index 00000000..f4e2db62 --- /dev/null +++ b/ontolearner/learner/text2onto/semanticswingers_train.py @@ -0,0 +1,282 @@ +# Copyright (c) 2025 SciKnowOrg +# +# Licensed under the MIT License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/MIT +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fine-tuning for the Semantic-Swingers Task A learner — the training counterpart of inference. + +The competition champion is a **LoRA fine-tune** of ``Qwen/Qwen3.5-9B``. Until now the fork shipped +only inference: ``fit()`` built the retrieval index and the adapter arrived pre-trained. This module +adds the code that *produces* the adapter, in two flavours that reconcile behind one interface: + +* ``train_backend="peft"`` — transformers + PEFT LoRA SFT (the reported bf16/8-bit champions). + Needs a CUDA GPU; the ``qwen3_5`` hybrid's fast-attention kernels do not exist on Apple Silicon. +* ``train_backend="mlx"`` — ``mlx_lm.lora`` on ``mlx-community/Qwen3.5-9B-4bit``. Runs on Apple + Silicon; produces a *separate 4-bit artifact*, so its scores are not the bf16 champion numbers. + +Two training regimes, matching the inference ``top_k``: + +* ``train_mode="raft"`` — retrieval-aware: top-k exemplars are baked into each training prompt with + **leave-one-out** retrieval (a doc never sees its own gold). Pairs with ``top_k > 0`` at inference. +* ``train_mode="baseft"`` — no exemplars in the prompt (k=0). Pairs with ``top_k = 0`` at inference. + +The data-building and prompt-masking below are **pure Python** and unit-testable with no GPU and no +model download; only the two ``_train_*`` backends import the heavy stack, lazily, at call time. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + + +@dataclass +class TrainConfig: + """Hyper-parameters for a LoRA SFT run. Defaults mirror the team's champion runs.""" + + output_dir: str + base_model_id: str = "Qwen/Qwen3.5-9B" + train_mode: str = "raft" # "raft" | "baseft" + top_k: int = 10 # exemplars baked into each prompt (raft only) + seq_len: int = 3584 # raft prompts are long; baseft can use 1024 + lora_r: int = 16 + lora_alpha: int = 32 + lora_dropout: float = 0.05 + learning_rate: float = 1e-5 + target_steps: int = 2000 + batch_size: int = 1 + grad_accum: int = 4 + save_every: int = 50 + steps_per_report: int = 25 + seed: int = 42 + # PEFT hybrid-arch LoRA targets (dense + linear-attention projections). Mirrors the team's + # ft_nvidia.py default; the reported adapters were trained on exactly these modules. + target_modules: Sequence[str] = field(default_factory=lambda: [ + "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", + "in_proj_qkvz", "in_proj_ba", + ]) + + +# ---- data building (pure python, unit-testable) ------------------------------------------------ + + +def build_training_pairs( + train_docs: List[Dict[str, Any]], + build_prompt: Callable[[str, List[Dict[str, Any]]], str], + retrieve_exemplars: Callable[[str, str], List[Dict[str, Any]]], + train_mode: str, +) -> List[Dict[str, str]]: + """Turn ``[{doc_id, text, triples}]`` into ``[{prompt, completion}]`` SFT pairs. + + * ``build_prompt(text, exemplars)`` is the learner's own prompt builder, so training prompts are + byte-identical to what inference sends — the single most important correctness property here. + * ``retrieve_exemplars(text, exclude_id)`` must exclude the document's own id (**leave-one-out**) + so a training doc never sees its own gold. For ``baseft`` we pass no exemplars at all. + * ``completion`` is the gold triples as ``{"triples": [[s, r, o], ...]}`` — the exact surface + form the inference parser expects, so we teach the model to emit what we later read back. + """ + pairs: List[Dict[str, str]] = [] + for d in train_docs: + text = d["text"] + if train_mode == "raft": + exemplars = retrieve_exemplars(text, d.get("doc_id", "")) # leave-one-out + else: + exemplars = [] + prompt = build_prompt(text, exemplars) + completion = json.dumps( + {"triples": [list(t) for t in (d.get("triples") or [])]}, ensure_ascii=False + ) + pairs.append({"prompt": prompt, "completion": completion}) + return pairs + + +def encode_example( + tokenize_fn: Callable[[str], List[int]], + eos_id: Optional[int], + prompt: str, + completion: str, + max_len: int, +) -> Tuple[List[int], List[int]]: + """Tokenize prompt+completion, **masking prompt tokens** (label ``-100``). + + Loss is computed only on completion tokens — the same ``--mask-prompt`` semantics ``mlx_lm`` + uses. Without this the model trains to reproduce the *document*, not to extract triples. On + overflow, truncate from the LEFT of the prompt so the assistant-turn suffix and the full + completion (the target) survive. + """ + prompt_ids = list(tokenize_fn(prompt)) + completion_ids = list(tokenize_fn(completion)) + if eos_id is not None: + completion_ids = completion_ids + [eos_id] + if len(prompt_ids) + len(completion_ids) > max_len: + keep = max(0, max_len - len(completion_ids)) + prompt_ids = prompt_ids[-keep:] if keep > 0 else [] + input_ids = prompt_ids + completion_ids + labels = [-100] * len(prompt_ids) + completion_ids + return input_ids[:max_len], labels[:max_len] + + +def write_jsonl(pairs: List[Dict[str, str]], path: Path) -> Path: + """Persist SFT pairs; both backends read this file, and it is a useful artefact on its own.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for p in pairs: + f.write(json.dumps(p, ensure_ascii=False) + "\n") + return path + + +# ---- backend dispatch -------------------------------------------------------------------------- + + +def train_adapter(pairs: List[Dict[str, str]], cfg: TrainConfig, backend: str) -> str: + """Train a LoRA adapter from SFT pairs and return the path to the finished adapter dir. + + Dispatches to the CUDA (``peft``) or Apple-Silicon (``mlx``) backend. The heavy imports live + inside each backend so this module imports cleanly with neither stack installed. + """ + out = Path(cfg.output_dir) + out.mkdir(parents=True, exist_ok=True) + write_jsonl(pairs, out / "train.jsonl") + if backend == "peft": + return _train_peft(pairs, cfg) + if backend == "mlx": + return _train_mlx(pairs, cfg) + raise ValueError(f"train_backend must be 'peft' or 'mlx', got {backend!r}") + + +def _pad_batch(examples: Sequence[Tuple[List[int], List[int]]], pad_id: int) -> Dict[str, list]: + max_len = max((len(ids) for ids, _ in examples), default=0) + input_ids, attn, labels = [], [], [] + for ids, labs in examples: + pad_n = max_len - len(ids) + input_ids.append(ids + [pad_id] * pad_n) + attn.append([1] * len(ids) + [0] * pad_n) + labels.append(labs + [-100] * pad_n) + return {"input_ids": input_ids, "attention_mask": attn, "labels": labels} + + +def _train_peft(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: + """transformers + PEFT LoRA SFT with prompt masking. CUDA-only in practice. + + Ported from the team's ``data/ft/_runners/ft_nvidia.py`` — a manual loop (not ``Trainer``) so + prompt masking and non-finite-loss skipping stay explicit and inspectable. + """ + try: + import random + + import torch + from peft import LoraConfig, get_peft_model + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as e: + raise RuntimeError( + "train_backend='peft' needs torch + transformers + peft (a CUDA GPU in practice; the " + "qwen3_5 kernels fall back to CPU on Apple Silicon). Install them on a GPU box, or use " + "train_backend='mlx' on Apple Silicon. Original error: " + str(e) + ) from e + + torch.manual_seed(cfg.seed) + tok = AutoTokenizer.from_pretrained(cfg.base_model_id, trust_remote_code=True) + if tok.pad_token_id is None: + tok.pad_token = tok.eos_token + model = AutoModelForCausalLM.from_pretrained( + cfg.base_model_id, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto" + ) + model = get_peft_model(model, LoraConfig( + r=cfg.lora_r, lora_alpha=cfg.lora_alpha, lora_dropout=cfg.lora_dropout, + target_modules=list(cfg.target_modules), task_type="CAUSAL_LM", bias="none", + )) + + def tokenize_fn(t: str) -> List[int]: + return tok(t, add_special_tokens=False)["input_ids"] + + encoded = [encode_example(tokenize_fn, tok.eos_token_id, p["prompt"], p["completion"], cfg.seq_len) + for p in pairs] + device = next(model.parameters()).device + optim = torch.optim.AdamW((p for p in model.parameters() if p.requires_grad), lr=cfg.learning_rate) + model.train() + rng = random.Random(cfg.seed) + step = micro = 0 + optim.zero_grad() + while step < cfg.target_steps: + batch = _pad_batch([encoded[rng.randrange(len(encoded))] for _ in range(cfg.batch_size)], + tok.pad_token_id) + out = model(input_ids=torch.tensor(batch["input_ids"], device=device), + attention_mask=torch.tensor(batch["attention_mask"], device=device), + labels=torch.tensor(batch["labels"], device=device)) + if not torch.isfinite(out.loss): + optim.zero_grad() + micro += 1 + continue + (out.loss / cfg.grad_accum).backward() + micro += 1 + if micro % cfg.grad_accum: + continue + torch.nn.utils.clip_grad_norm_((p for p in model.parameters() if p.requires_grad), 1.0) + optim.step() + optim.zero_grad() + step += 1 + if step % cfg.steps_per_report == 0: + print(f"[peft] step {step}/{cfg.target_steps} loss={out.loss.item():.4f}", flush=True) + if step % cfg.save_every == 0: + model.save_pretrained(Path(cfg.output_dir) / f"checkpoint-{step}") + final = Path(cfg.output_dir) / "final" + model.save_pretrained(final) + return str(final) + + +def _train_mlx(pairs: List[Dict[str, str]], cfg: TrainConfig) -> str: + """LoRA SFT via the ``mlx_lm lora`` **CLI** on Apple Silicon. + + Invokes ``python -m mlx_lm lora --train`` — the stable, version-robust entry point, and the exact + command the team's own ``ft35_overnight.py`` used to produce the real MLX adapters. We do **not** + bind ``mlx_lm``'s low-level ``tuner`` API: it churns across releases (the dataset ``__getitem__`` + contract and the LoRA-config keys have both shifted), so re-implementing the loop is fragile for + no benefit. The CLI reads ``{prompt, completion}`` jsonl, applies prompt masking itself, and writes + ``adapters.safetensors`` + ``adapter_config.json`` — the format ``backend="mlx"`` inference loads. + + The base defaults to the 4-bit MLX build, so the adapter is a *separate 4-bit artifact* from the + bf16 PEFT champions (a Mac-native FT variant, not a reproduction of the reported scores). + """ + import subprocess + import sys + + out = Path(cfg.output_dir) + write_jsonl(pairs, out / "train.jsonl") + write_jsonl(pairs[: max(1, len(pairs) // 10)], out / "valid.jsonl") + + base = cfg.base_model_id + if base == "Qwen/Qwen3.5-9B": # bf16 id -> its MLX-quantized sibling + base = "mlx-community/Qwen3.5-9B-4bit" + + cmd = [ + sys.executable, "-m", "mlx_lm", "lora", + "--model", base, + "--train", + "--data", str(out), + "--fine-tune-type", "lora", + "--num-layers", "8", + "--batch-size", str(cfg.batch_size), + "--iters", str(cfg.target_steps), + "--max-seq-length", str(cfg.seq_len), + "--learning-rate", str(cfg.learning_rate), + "--steps-per-report", str(cfg.steps_per_report), + "--save-every", str(cfg.save_every), + "--adapter-path", str(out), + "--grad-checkpoint", + ] + result = subprocess.run(cmd) + if result.returncode != 0: + raise RuntimeError( + f"`mlx_lm lora` training failed (exit {result.returncode}). Command:\n " + + " ".join(cmd) + ) + return str(out) diff --git a/tests/test_semanticswingers_taxonomy_discovery.py b/tests/test_semanticswingers_taxonomy_discovery.py new file mode 100644 index 00000000..bbf8291e --- /dev/null +++ b/tests/test_semanticswingers_taxonomy_discovery.py @@ -0,0 +1,259 @@ +from types import SimpleNamespace + +import numpy as np +import pytest +from unittest.mock import MagicMock, patch + +from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersTaxonomyLearner, +) + +MODULE = "ontolearner.learner.taxonomy_discovery.semanticswingers" + + +def _relation(parent, child): + return SimpleNamespace(parent=parent, child=child) + + +def _ontology_data(types, taxonomies): + return SimpleNamespace( + type_taxonomies=SimpleNamespace( + types=types, + taxonomies=[_relation(p, c) for p, c in taxonomies], + ) + ) + + +def _make_encoder(vectors): + """A fake SentenceTransformer whose ``encode`` looks vectors up by text.""" + def encode(texts, normalize_embeddings=True, show_progress_bar=False): + return np.array([vectors[t] for t in texts]) + + encoder = MagicMock() + encoder.encode.side_effect = encode + return encoder + + +# Unit vectors (the real encoder is called with normalize_embeddings=True). +# "Animal" sits at 45 degrees, equidistant from "Dog" (10 degrees) and "Cat" +# (80 degrees), so it has the highest mean similarity to the vocabulary and +# wins the generality tie-break; "Dog" and "Cat" are far apart from each other. +VECTORS = { + "Animal": [0.70711, 0.70711], + "Dog": [0.98481, 0.17365], + "Cat": [0.17365, 0.98481], +} + + +@pytest.fixture +def mock_encoder(): + with patch(f"{MODULE}.SentenceTransformer") as MockST: + MockST.return_value = _make_encoder(VECTORS) + yield MockST + + +@pytest.fixture(autouse=True) +def no_openai_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + +def test_nodes_dedups_declared_types_and_edge_endpoints(): + data = _ontology_data( + types=["Animal", "Dog"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + nodes = SemanticSwingersTaxonomyLearner._nodes(data) + + assert nodes == ["Animal", "Dog", "Cat"] + + +def test_tasks_data_former_passes_data_through_unchanged(): + data = _ontology_data(types=["Animal"], taxonomies=[]) + learner = SemanticSwingersTaxonomyLearner() + + assert learner.tasks_data_former(data, task="taxonomy-discovery", test=True) is data + + +def test_fit_is_a_noop(): + learner = SemanticSwingersTaxonomyLearner() + data = _ontology_data(types=["Animal"], taxonomies=[]) + + assert learner._taxonomy_discovery(data, test=False) is None + + +def test_predict_returns_parent_child_dicts(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + preds = learner._taxonomy_discovery(data, test=True) + + assert len(preds) > 0 + assert all(set(p.keys()) == {"parent", "child"} for p in preds) + + +def test_embedding_selector_picks_most_general_candidate(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu", top_k=10) + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + preds = learner._taxonomy_discovery(data, test=True) + by_child = {p["child"]: p["parent"] for p in preds} + + assert by_child["Dog"] == "Animal" + assert by_child["Cat"] == "Animal" + # "Animal" is the most general node overall, so it never gets a parent. + assert "Animal" not in by_child + + +def test_embedding_selector_is_deterministic(mock_encoder): + learner = SemanticSwingersTaxonomyLearner(device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + first = learner._taxonomy_discovery(data, test=True) + second = learner._taxonomy_discovery(data, test=True) + + assert first == second + + +def test_openai_selector_without_api_key_falls_back_to_embedding(mock_encoder): + with patch("openai.OpenAI") as MockOpenAI: + learner = SemanticSwingersTaxonomyLearner(selector="openai", device="cpu") + data = _ontology_data( + types=["Animal", "Dog", "Cat"], + taxonomies=[("Animal", "Dog"), ("Animal", "Cat")], + ) + + assert learner.api_key is None + preds = learner._taxonomy_discovery(data, test=True) + + by_child = {p["child"]: p["parent"] for p in preds} + assert by_child["Dog"] == "Animal" + MockOpenAI.assert_not_called() + + +def test_ollama_selector_defaults(): + learner = SemanticSwingersTaxonomyLearner(selector="ollama") + assert learner.llm_model == "llama3.1:8b" + assert learner.base_url == "http://localhost:11434/v1" + + +def test_openai_selector_defaults(): + learner = SemanticSwingersTaxonomyLearner(selector="openai") + assert learner.llm_model == "gpt-4.1-mini" + assert learner.base_url is None + + +def test_explicit_llm_model_overrides_default(): + learner = SemanticSwingersTaxonomyLearner(selector="ollama", llm_model="qwen3.5") + assert learner.llm_model == "qwen3.5" + + +def test_reasoning_off_only_for_qwen3(): + """qwen3.x is a thinking model: without this the selector gets empty output (F1=0).""" + from ontolearner.learner.taxonomy_discovery.semanticswingers import _reasoning_off + + assert _reasoning_off("qwen3.5-nothink:9b") == { + "extra_body": {"reasoning_effort": "none"} + } + assert _reasoning_off("qwen3.5:9b") == {"extra_body": {"reasoning_effort": "none"}} + assert _reasoning_off("gpt-4.1-mini") == {} + assert _reasoning_off("llama3.1:8b") == {} + assert _reasoning_off(None) == {} + + +def test_selection_prompt_is_injectable_and_defaults(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersTaxonomyLearner, _SELECTION_PROMPT, + ) + assert SemanticSwingersTaxonomyLearner().selection_prompt == _SELECTION_PROMPT + custom = SemanticSwingersTaxonomyLearner(selection_prompt="parent of {child}: {candidates}") + out = custom.selection_prompt.format(child="Chianti", candidates="- Wine") + assert out == "parent of Chianti: - Wine" + + +# --- Structural-matrix champion (SemanticSwingersMatrixTaxonomyLearner) -------- + +def test_bilinear_layer_scores_child_parent_pairs(): + import torch + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + BilinearAdjacencyLayer, + ) + + layer = BilinearAdjacencyLayer(embedding_dim=4) + child = torch.randn(3, 4) + parent = torch.randn(3, 4) + scores = layer(child, parent) + + # one score per (child, parent) row, and it equals + assert scores.shape == (3,) + expected = torch.sum(layer.W(child) * parent, dim=-1) + assert torch.allclose(scores, expected) + + +def test_cleanup_graph_breaks_cycles_and_transitively_reduces(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner() + + # A->B, B->C, A->C : the redundant A->C must be dropped by transitive reduction. + reduced = learner._cleanup_graph([ + {"parent": "A", "child": "B"}, + {"parent": "B", "child": "C"}, + {"parent": "A", "child": "C"}, + ]) + edges = {(e["parent"], e["child"]) for e in reduced} + assert ("A", "C") not in edges + assert ("A", "B") in edges and ("B", "C") in edges + + # A->B, B->A : a cycle must be broken so the result is a DAG. + import networkx as nx + deacycled = learner._cleanup_graph([ + {"parent": "A", "child": "B"}, + {"parent": "B", "child": "A"}, + ]) + G = nx.DiGraph((e["parent"], e["child"]) for e in deacycled) + assert nx.is_directed_acyclic_graph(G) + + assert learner._cleanup_graph([]) == [] + + +def test_matrix_learner_defaults_to_public_hf_registry(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner() + assert learner.hf_repo_id == "datagero/taxonomy-structural-matrix-1024-mxbai" + + +def test_resolve_matrix_weights_prefers_local_then_falls_back_to_hf(): + from ontolearner.learner.taxonomy_discovery.semanticswingers import ( + SemanticSwingersMatrixTaxonomyLearner, + ) + + learner = SemanticSwingersMatrixTaxonomyLearner( + matrix_weights_path="structural_matrix_w_1024_mxbai.pt", + ) + + # Local file present -> used as-is, no download. + with patch(f"{MODULE}.os.path.exists", return_value=True): + assert learner._resolve_matrix_weights() == "structural_matrix_w_1024_mxbai.pt" + + # Local file absent -> fetched from the HF registry by basename. + with patch(f"{MODULE}.os.path.exists", return_value=False), \ + patch("huggingface_hub.hf_hub_download", return_value="/cache/x.pt") as dl: + assert learner._resolve_matrix_weights() == "/cache/x.pt" + dl.assert_called_once_with( + repo_id="datagero/taxonomy-structural-matrix-1024-mxbai", + filename="structural_matrix_w_1024_mxbai.pt", + ) diff --git a/tests/test_semanticswingers_term_typing.py b/tests/test_semanticswingers_term_typing.py new file mode 100644 index 00000000..69b4064c --- /dev/null +++ b/tests/test_semanticswingers_term_typing.py @@ -0,0 +1,174 @@ +import numpy as np +import pytest +from unittest.mock import MagicMock, patch + +from ontolearner.learner.term_typing.semanticswingers import ( + SemanticSwingersTermTypingLearner, +) + +MODULE = "ontolearner.learner.term_typing.semanticswingers" + + +def _make_encoder(vectors): + """A fake SentenceTransformer whose ``encode`` looks vectors up by text.""" + def encode(texts, normalize_embeddings=True, show_progress_bar=False): + return np.array([vectors[t] for t in texts]) + + encoder = MagicMock() + encoder.encode.side_effect = encode + return encoder + + +VECTORS = { + "Animal": [1.0, 0.0], + "Plant": [0.0, 1.0], + "dog": [0.9, 0.1], + "rose": [0.1, 0.9], +} + + +@pytest.fixture +def mock_encoder(): + with patch(f"{MODULE}.SentenceTransformer") as MockST: + MockST.return_value = _make_encoder(VECTORS) + yield MockST + + +@pytest.fixture(autouse=True) +def no_openai_key(monkeypatch): + """Hermetic default: no OPENAI_API_KEY leaks in from the environment.""" + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + +def test_fit_learns_type_inventory_from_list(): + learner = SemanticSwingersTermTypingLearner() + learner._term_typing(["Animal", "Plant", "Animal"], test=False) + + assert learner._allowed_types == ["Animal", "Plant"] + + +def test_predict_returns_entry_per_term(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + preds = learner._term_typing(["dog", "rose"], test=True) + + assert len(preds) == 2 + assert {p["term"] for p in preds} == {"dog", "rose"} + assert all(set(p.keys()) == {"term", "types"} for p in preds) + + +def test_embedding_selector_picks_nearest_type(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + preds = learner._term_typing(["dog", "rose"], test=True) + by_term = {p["term"]: p["types"] for p in preds} + + assert by_term["dog"] == ["Animal"] + assert by_term["rose"] == ["Plant"] + + +def test_predict_empty_terms_and_empty_vocab_short_circuit(): + learner = SemanticSwingersTermTypingLearner() + learner._allowed_types = ["Animal"] + assert learner._term_typing([], test=True) == [] + + learner2 = SemanticSwingersTermTypingLearner() + learner2._allowed_types = [] + assert learner2._term_typing(["dog"], test=True) == [{"term": "dog", "types": []}] + + +def test_embedding_selector_is_deterministic(mock_encoder): + learner = SemanticSwingersTermTypingLearner(device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + first = learner._term_typing(["dog", "rose"], test=True) + second = learner._term_typing(["dog", "rose"], test=True) + + assert first == second + + +class TestParseTypings: + def setup_method(self): + self.learner = SemanticSwingersTermTypingLearner() + self.allowed = {"Animal", "Plant"} + + def test_parses_plain_json(self): + content = '{"typings": [{"term": "dog", "types": ["Animal"]}]}' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + def test_drops_types_outside_closed_vocabulary(self): + content = '{"typings": [{"term": "dog", "types": ["Animal", "Robot"]}]}' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + def test_malformed_json_returns_empty_dict(self): + result = self.learner._parse_typings("not json at all", self.allowed) + assert result == {} + + def test_recovers_fenced_json(self): + content = ( + "Here is the answer:\n```json\n" + '{"typings": [{"term": "rose", "types": ["Plant"]}]}\n```' + ) + result = self.learner._parse_typings(content, self.allowed) + assert result == {"rose": ["Plant"]} + + def test_top_level_list_payload(self): + content = '[{"term": "dog", "types": ["Animal"]}]' + result = self.learner._parse_typings(content, self.allowed) + assert result == {"dog": ["Animal"]} + + +def test_openai_selector_without_api_key_falls_back_to_embedding(mock_encoder): + with patch("openai.OpenAI") as MockOpenAI: + learner = SemanticSwingersTermTypingLearner(selector="openai", device="cpu") + learner._allowed_types = ["Animal", "Plant"] + + assert learner.api_key is None + preds = learner._term_typing(["dog", "rose"], test=True) + + by_term = {p["term"]: p["types"] for p in preds} + assert by_term["dog"] == ["Animal"] + assert by_term["rose"] == ["Plant"] + MockOpenAI.assert_not_called() + + +def test_ollama_selector_defaults(): + learner = SemanticSwingersTermTypingLearner(selector="ollama") + assert learner.llm_model == "llama3.1:8b" + assert learner.base_url == "http://localhost:11434/v1" + + +def test_openai_selector_defaults(): + learner = SemanticSwingersTermTypingLearner(selector="openai") + assert learner.llm_model == "gpt-4.1-mini" + assert learner.base_url is None + + +def test_explicit_llm_model_overrides_default(): + learner = SemanticSwingersTermTypingLearner(selector="ollama", llm_model="qwen3.5") + assert learner.llm_model == "qwen3.5" + + +def test_reasoning_off_only_for_qwen3(): + """qwen3.x is a thinking model: without this the selector gets empty output (F1=0).""" + from ontolearner.learner.term_typing.semanticswingers import _reasoning_off + + assert _reasoning_off("qwen3.5-nothink:9b") == { + "extra_body": {"reasoning_effort": "none"} + } + assert _reasoning_off("qwen3.5:9b") == {"extra_body": {"reasoning_effort": "none"}} + assert _reasoning_off("gpt-4.1-mini") == {} + assert _reasoning_off("llama3.1:8b") == {} + assert _reasoning_off(None) == {} + + +def test_system_prompt_is_injectable_and_defaults(): + from ontolearner.learner.term_typing.semanticswingers import ( + SemanticSwingersTermTypingLearner, _SYSTEM, + ) + assert SemanticSwingersTermTypingLearner().system_prompt == _SYSTEM + assert SemanticSwingersTermTypingLearner(system_prompt="CUSTOM").system_prompt == "CUSTOM" diff --git a/tests/test_semanticswingers_text2onto.py b/tests/test_semanticswingers_text2onto.py new file mode 100644 index 00000000..ae983daf --- /dev/null +++ b/tests/test_semanticswingers_text2onto.py @@ -0,0 +1,405 @@ +import json +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + _parse_triples_json, + _require_qwen35_transformers, + _to_triples, + _VERIFIED_TRANSFORMERS_COMMIT, +) + +MODULE = "ontolearner.learner.text2onto.semanticswingers" + + +# -- lazy import guard ----------------------------------------------------------------------- + + +def test_lazy_import_guard_raises_actionable_error_without_qwen35(): + """The project's pinned transformers (4.57.6, no git-source install) lacks qwen3_5 — this + exercises the real guard against the real installed library, no mocking needed.""" + with pytest.raises(ImportError) as exc_info: + _require_qwen35_transformers() + + message = str(exc_info.value) + assert "qwen3_5" in message + assert _VERIFIED_TRANSFORMERS_COMMIT in message + assert "pip install" in message + + +def test_load_raises_before_touching_the_network(monkeypatch): + """load() must fail fast on the lazy-import guard, never reaching AutoModelForCausalLM.""" + learner = SemanticSwingersText2OntoLearner() + with patch(f"{MODULE}.AutoRetriever"): + with pytest.raises(ImportError): + learner.load() + + +# -- adapter name resolution ------------------------------------------------------------------- + + +def test_known_adapter_alias_resolves_to_hf_repo(): + learner = SemanticSwingersText2OntoLearner(adapter="raft") + assert learner.adapter == "datagero/qwen3.5-9b-ontology-extraction-raft" + assert (SemanticSwingersText2OntoLearner(adapter="baseft-mlx").adapter + == "datagero/qwen3.5-9b-ontology-extraction-baseft-mlx") + + +def test_unknown_adapter_value_passed_through_unchanged(): + learner = SemanticSwingersText2OntoLearner(adapter="/local/path/to/adapter") + assert learner.adapter == "/local/path/to/adapter" + + +# -- load() with the guard mocked out ------------------------------------------------------ + + +@pytest.fixture +def mock_model_stack(): + """Mock transformers/peft so load() never touches the network or a real model.""" + with patch(f"{MODULE}._require_qwen35_transformers"), \ + patch("transformers.AutoTokenizer") as MockTok, \ + patch("transformers.AutoModelForCausalLM") as MockModel, \ + patch("peft.PeftModel") as MockPeft: + MockTok.from_pretrained.return_value = MagicMock() + MockModel.from_pretrained.return_value = MagicMock() + MockPeft.from_pretrained.return_value = MagicMock() + yield SimpleNamespace(tokenizer=MockTok, model=MockModel, peft=MockPeft) + + +def test_load_applies_peft_adapter_on_top_of_base(mock_model_stack): + learner = SemanticSwingersText2OntoLearner(adapter="/some/adapter/dir", device="cpu") + with patch(f"{MODULE}.SemanticSwingersTaxonomyLearner.load"): + learner.load() + + mock_model_stack.model.from_pretrained.assert_called_once() + assert mock_model_stack.model.from_pretrained.call_args[0][0] == "Qwen/Qwen3.5-9B" + mock_model_stack.peft.from_pretrained.assert_called_once_with( + mock_model_stack.model.from_pretrained.return_value, "/some/adapter/dir" + ) + assert learner._model is not None + + +def test_load_is_idempotent(mock_model_stack): + learner = SemanticSwingersText2OntoLearner() + with patch(f"{MODULE}.SemanticSwingersTaxonomyLearner.load"): + learner.load() + learner.load() + + mock_model_stack.model.from_pretrained.assert_called_once() + + +# -- triple parsing helpers ------------------------------------------------------------------ + + +def test_parse_triples_json_plain_array(): + text = '{"triples": [["poodle", "is-a", "dog"]]}' + assert _parse_triples_json(text) == [["poodle", "is-a", "dog"]] + + +def test_parse_triples_json_markdown_fence(): + text = '```json\n{"triples": [["cat", "is-a", "mammal"]]}\n```' + assert _parse_triples_json(text) == [["cat", "is-a", "mammal"]] + + +def test_parse_triples_json_garbage_returns_empty(): + assert _parse_triples_json("not json at all") == [] + + +def test_to_triples_normalizes_relation_case_and_strips(): + raw = [[" Dog ", " IS-A ", " Animal "], {"subject": "Cat", "relation": "type", "object": "Mammal"}] + triples = _to_triples(raw) + assert triples == [("Dog", "is-a", "Animal"), ("Cat", "type", "Mammal")] + + +def test_to_triples_drops_incomplete_entries(): + raw = [["only", "two"], ["", "is-a", "animal"], ["dog", "is-a", "animal"]] + assert _to_triples(raw) == [("dog", "is-a", "animal")] + + +# -- _text2onto: fit indexes exemplars, predict projects triples ------------------------------- + + +def test_tasks_data_former_passes_data_through_unchanged(): + learner = SemanticSwingersText2OntoLearner() + data = {"documents": []} + assert learner.tasks_data_former(data, task="text2onto", test=True) is data + assert learner.tasks_data_former(data, task="taxonomy-discovery", test=False) is data + + +def test_text2onto_fit_indexes_train_docs_and_returns_none(): + learner = SemanticSwingersText2OntoLearner() + with patch.object(learner._retriever, "load"), patch.object(learner._retriever, "index") as mock_index: + data = { + "documents": [{"doc_id": "d1", "text": "A dog is a mammal."}], + "triples": {"d1": [["dog", "is-a", "mammal"]]}, + } + result = learner._text2onto(data, test=False) + + assert result is None + assert learner._train_docs == [ + {"doc_id": "d1", "text": "A dog is a mammal.", "triples": [["dog", "is-a", "mammal"]]} + ] + mock_index.assert_called_once_with(["A dog is a mammal."]) + + +def test_text2onto_fit_with_no_documents_skips_indexing(): + learner = SemanticSwingersText2OntoLearner() + with patch.object(learner._retriever, "load") as mock_load: + result = learner._text2onto({"documents": []}, test=False) + + assert result is None + assert learner._train_docs == [] + mock_load.assert_not_called() + + +def test_text2onto_rejects_non_dict_input(): + learner = SemanticSwingersText2OntoLearner() + with pytest.raises(ValueError): + learner._text2onto(["not", "a", "dict"], test=True) + + +def test_text2onto_predict_projects_triples_to_terms_and_types(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() # short-circuits load() + + canned_triples = [ + ("poodle", "is-a", "dog"), + ("dog", "is-a", "mammal"), + ("mammal", "instance-of", "animal-class"), + ] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d2", "text": "..."}]}, test=True) + + assert result["terms"] == [ + {"doc_id": "d2", "term": "poodle"}, + {"doc_id": "d2", "term": "dog"}, + {"doc_id": "d2", "term": "mammal"}, + ] + assert result["types"] == [ + {"doc_id": "d2", "type": "dog"}, + {"doc_id": "d2", "type": "mammal"}, + {"doc_id": "d2", "type": "animal-class"}, + ] + + +def test_text2onto_predict_dedupes_repeated_terms_and_types(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + canned_triples = [("dog", "is-a", "mammal"), ("dog", "is-a", "mammal")] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d3", "text": "..."}]}, test=True) + + assert result["terms"] == [{"doc_id": "d3", "term": "dog"}] + assert result["types"] == [{"doc_id": "d3", "type": "mammal"}] + + +# -- _text2onto: raw triples pass through under an extra key (harness-ignored) ----------------- + + +def test_text2onto_predict_includes_raw_triples_alongside_terms_and_types(): + """The 'triples' key is additive: terms/types shape is unchanged, and the raw (unprojected) + triples — including non-typing relations text2onto_metrics never sees — survive alongside.""" + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + canned_triples = [ + ("poodle", "is-a", "dog"), + ("dog", "part_of", "canine-family"), # non-typing relation: absent from terms/types logic + ] + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=canned_triples): + result = learner._text2onto({"documents": [{"doc_id": "d4", "text": "..."}]}, test=True) + + assert set(result.keys()) == {"terms", "types", "triples"} + assert result["triples"] == [ + ["d4", "poodle", "is-a", "dog"], + ["d4", "dog", "part_of", "canine-family"], + ] + # terms/types content is unaffected by the extra key: still exactly what the projection produces + assert result["terms"] == [ + {"doc_id": "d4", "term": "poodle"}, + {"doc_id": "d4", "term": "dog"}, + ] + assert result["types"] == [{"doc_id": "d4", "type": "dog"}] + + +def test_text2onto_predict_triples_empty_list_when_no_triples_generated(): + learner = SemanticSwingersText2OntoLearner() + learner._model = MagicMock() + + with patch.object(learner, "_retrieve_exemplars", return_value=[]), \ + patch.object(learner, "_generate_triples", return_value=[]): + result = learner._text2onto({"documents": [{"doc_id": "d5", "text": "..."}]}, test=True) + + assert result == {"terms": [], "types": [], "triples": []} + + +# -- _taxonomy_discovery: composed delegation, not a rewrite ----------------------------------- + + +def test_taxonomy_discovery_delegates_to_composed_taxonomy_learner(): + learner = SemanticSwingersText2OntoLearner() + sentinel_data = SimpleNamespace(type_taxonomies=SimpleNamespace(types=[], taxonomies=[])) + + with patch.object( + learner._taxonomy_learner, "_taxonomy_discovery", return_value=[{"parent": "a", "child": "b"}] + ) as mock_delegate: + result = learner._taxonomy_discovery(sentinel_data, test=True) + + mock_delegate.assert_called_once_with(sentinel_data, test=True) + assert result == [{"parent": "a", "child": "b"}] + + +def test_taxonomy_discovery_fit_is_a_noop_via_delegation(): + learner = SemanticSwingersText2OntoLearner() + sentinel_data = SimpleNamespace(type_taxonomies=SimpleNamespace(types=[], taxonomies=[])) + + assert learner._taxonomy_discovery(sentinel_data, test=False) is None + + +def test_api_backends_need_no_local_weights(): + """backend='ollama'/'openai' must not touch transformers/peft — that is the whole point.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + learner = SemanticSwingersText2OntoLearner(backend="ollama") + assert learner.llm_model == "qwen3.5-nothink:9b" + assert learner.base_url == "http://localhost:11434/v1" + + learner_oai = SemanticSwingersText2OntoLearner(backend="openai") + assert learner_oai.llm_model == "gpt-4.1-mini" + assert learner_oai.base_url is None + + # explicit override wins + custom = SemanticSwingersText2OntoLearner(backend="ollama", llm_model="llama3.1:8b") + assert custom.llm_model == "llama3.1:8b" + + +def test_peft_remains_the_default_backend(): + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + assert SemanticSwingersText2OntoLearner().backend == "peft" + + +def test_chat_messages_carry_exemplars_as_turns(): + """API backends need structured turns with the same content as the ChatML prompt.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, + ) + + learner = SemanticSwingersText2OntoLearner(backend="ollama") + msgs = learner._chat_messages( + "target doc", [{"text": "ex doc", "triples": [("a", "is-a", "b")]}] + ) + assert msgs[0]["role"] == "system" + assert [m["role"] for m in msgs[1:]] == ["user", "assistant", "user"] + assert "ex doc" in msgs[1]["content"] + assert '"triples"' in msgs[2]["content"] + assert "target doc" in msgs[3]["content"] + + +# -- training (semanticswingers_train) ---------------------------------------------------------- + +def test_encode_example_masks_prompt_tokens(): + """Loss must be computed only on completion tokens (prompt labels = -100).""" + from ontolearner.learner.text2onto.semanticswingers_train import encode_example + + ids = {"the prompt": [1, 2, 3], "the completion": [4, 5]} + def tok(s): return ids[s] + input_ids, labels = encode_example(tok, eos_id=9, prompt="the prompt", + completion="the completion", max_len=100) + assert input_ids == [1, 2, 3, 4, 5, 9] + assert labels == [-100, -100, -100, 4, 5, 9] # prompt masked, completion+eos trained + + +def test_encode_example_left_truncates_prompt_on_overflow(): + """On overflow keep the completion (the target) intact; drop prompt from the left.""" + from ontolearner.learner.text2onto.semanticswingers_train import encode_example + + def tok(s): return list(range(10)) if s == "P" else [100, 101] + input_ids, labels = encode_example(tok, eos_id=None, prompt="P", completion="C", max_len=4) + assert input_ids[-2:] == [100, 101] # completion survives + assert labels[-2:] == [100, 101] + assert len(input_ids) == 4 + + +def test_build_training_pairs_raft_is_leave_one_out(): + """A RAFT training prompt for doc X must never contain X's own gold (no leakage).""" + from ontolearner.learner.text2onto.semanticswingers_train import build_training_pairs + + docs = [{"doc_id": "x", "text": "doc x", "triples": [("a", "is-a", "b")]}, + {"doc_id": "y", "text": "doc y", "triples": [("c", "is-a", "d")]}] + + def fake_retrieve(text, exclude_id): + # would return self if not excluded; assert the learner passes the right id + return [d for d in docs if d["doc_id"] != exclude_id] + + seen = {} + + def fake_build_prompt(text, exemplars): + seen[text] = [e["doc_id"] for e in exemplars] + return "PROMPT:" + text + + pairs = build_training_pairs(docs, fake_build_prompt, fake_retrieve, "raft") + assert seen["doc x"] == ["y"] # x trained with y's exemplar, not its own + assert seen["doc y"] == ["x"] + assert json.loads(pairs[0]["completion"]) == {"triples": [["a", "is-a", "b"]]} + + +def test_build_training_pairs_baseft_has_no_exemplars(): + from ontolearner.learner.text2onto.semanticswingers_train import build_training_pairs + + docs = [{"doc_id": "x", "text": "doc x", "triples": [("a", "is-a", "b")]}] + calls = [] + pairs = build_training_pairs( + docs, lambda t, ex: (calls.append(ex) or "P"), + lambda t, xid: [{"doc_id": "should-not-be-used"}], "baseft") + assert calls == [[]] # baseft prompts carry no exemplars + assert len(pairs) == 1 + + +def test_train_adapter_rejects_unknown_backend(): + from ontolearner.learner.text2onto.semanticswingers_train import TrainConfig, train_adapter + import tempfile + + with tempfile.TemporaryDirectory() as d: + cfg = TrainConfig(output_dir=d) + try: + train_adapter([{"prompt": "p", "completion": "c"}], cfg, "nonsense") + assert False, "expected ValueError" + except ValueError as e: + assert "peft" in str(e) and "mlx" in str(e) + + +# -- extension points ------------------------------------------------------------------------- + +def test_system_prompt_is_injectable_and_defaults(): + """Others can bring their own extraction instructions without subclassing.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, _SYSTEM_PROMPT, + ) + assert SemanticSwingersText2OntoLearner().system_prompt == _SYSTEM_PROMPT + custom = SemanticSwingersText2OntoLearner(system_prompt="MY DOMAIN RULES") + assert custom.system_prompt == "MY DOMAIN RULES" + # the custom prompt actually reaches the built prompt + chat messages + assert "MY DOMAIN RULES" in custom._build_prompt("doc", []) + assert custom._chat_messages("doc", [])[0]["content"] == "MY DOMAIN RULES" + + +def test_typing_relations_is_injectable_and_defaults(): + """The relation set that projects to `types` is domain-specific and overridable.""" + from ontolearner.learner.text2onto.semanticswingers import ( + SemanticSwingersText2OntoLearner, _TYPING_RELATIONS, + ) + assert SemanticSwingersText2OntoLearner().typing_relations == _TYPING_RELATIONS + custom = SemanticSwingersText2OntoLearner(typing_relations={"rdf:type", "subclass_of"}) + assert custom.typing_relations == {"rdf:type", "subclass_of"}