✏️ Added 23 new model properties and harden JSON extraction [#15]

src/model_contribution_selector.py

@@ -16,6 +16,12 @@
 # Generic auxiliary terms that are usually not standalone model contributions
 # in comparison-style extractions.
 _AUXILIARY_KEYWORDS = {
+    "vision", 
+    "speech", 
+    "audio", 
+    "image", 
+    "multimodal", 
+    "visual",
     "adapter",
     "chat",
     "classifier",

src/model_variant_merger.py

@@ -21,6 +21,7 @@
 import logging
 import re
 from typing import Any, Dict, List, Optional
+from src.template_mapper import _MULTI_VALUED_FIELDS
 
 logger = logging.getLogger(__name__)
 
@@ -483,6 +484,39 @@ def _merge_group(models: List[Dict[str, Any]], canonical_name: str) -> Dict[str,
         "research_problem",
         "application",
         "paper_title",
+        # Conditional fields added to the extraction schema — must be listed
+        # here or they are dropped when variants are merged.
+        "training_corpus_size",
+        "finetuning_data",
+        "tokenizer",
+        "supported_language",
+        "hardware_description",
+        "carbon_emitted",
+        "source_code",
+        "context_length",
+        "activated_parameters",
+        "attention_mechanism",
+        "context_length_max",
+        "context_extension_method",
+        "training_pipeline",
+        "reasoning_mode",
+        "moe_configuration",
+        "quantization_precision",
+        "synthetic_data_generation_method",
+        "rl_algorithm",
+        "reward_mechanism",
+        "tool_calling_format",
+        "training_environment_scale",
+        "safety_evaluation_protocol",
+        "safety_defect_rate",
+        "fusion_architecture",
+        "vision_encoder",
+        "base_model",
+        "optimizer_innovation",
+        "benchmark_result",
+        "weight_clipping_mechanism",
+        "number_of_attention_heads",
+        "post_training_infrastructure",
     ]
 
     for field in fields_to_merge:
@@ -567,14 +601,20 @@ def _merge_field(models: List[Dict[str, Any]], field: str) -> Any:
     ]:
         return max(non_null, key=lambda v: len(str(v)))
 
-    # List/multi-value fields (e.g., blog_post): merge and deduplicate
-    if field == "blog_post":
-        all_links = []
+    # Multi-value fields: union and deduplicate values captured across chunks,
+      # using each field's own separator (from _MULTI_VALUED_FIELDS, the single
+      # source of truth). blog_post/source_code are URL lists that aren't in that
+      # map, so handle them explicitly with a comma.
+    sep = _MULTI_VALUED_FIELDS.get(field) or (
+          "," if field in ("blog_post", "source_code") else None
+      )
+    if sep:
+        all_parts = []
         for val in non_null:
-            links = str(val).split(",")
-            all_links.extend(link.strip() for link in links if link.strip())
-        unique_links = sorted(set(all_links))
-        return ", ".join(unique_links) if unique_links else None
+            parts = str(val).split(sep)
+            all_parts.extend(part.strip() for part in parts if part.strip())
+        unique_parts = sorted(set(all_parts))
+        return f"{sep} ".join(unique_parts) if unique_parts else None
 
     # Default: first non-null
     return non_null[0]

src/orkg_client.py

@@ -407,8 +407,18 @@ def create_paper_with_contributions(
                     elif datatype == "URI" or (
                         isinstance(value, str) and value.startswith("http")
                     ):
-                        # HTTP URL → reference an existing resource directly
-                        statements[prop_id].append({"id": str(value)})
+                        # HTTP URL → xsd:anyURI literal so ORKG renders it as a
+                          # clickable link (URL chip) instead of a plain text chip.
+                          # Do NOT emit {"id": url}: ORKG treats "id" as a
+                          # reference to an existing resource, and a URL is not a
+                          # resolvable resource id — that makes papers.add 500.
+                        literal_id = f"#literal_{literal_counter}"
+                        orkg_literals[literal_id] = {
+                            "label": str(value),
+                            "data_type": "xsd:anyURI",
+                        }
+                        statements[prop_id].append({"id": literal_id})
+                        literal_counter += 1
                     elif datatype in ("date", "Date"):
                         literal_id = f"#literal_{literal_counter}"
                         orkg_literals[literal_id] = {
@@ -559,7 +569,8 @@ def _convert_properties_to_statements(self, properties: List[Dict[str, Any]]) ->
                 if datatype == "resource":
                     statements[prop_id].append({"label": str(value)})
                 elif datatype == "URI" or (isinstance(value, str) and value.startswith("http")):
-                    statements[prop_id].append({"id": value})
+                    # URL as a literal, not {"id": url} (that is a resource ref).
+                    statements[prop_id].append({"label": str(value)})
                 elif datatype in ("date", "Date"):
                     statements[prop_id].append({"label": str(value), "datatype": "Date"})
                 elif datatype in ("integer", "Integer") or isinstance(value, int):
@@ -670,7 +681,8 @@ def add_contribution_to_paper(
                     # Format value object based on type
                     value_obj = {}
                     if datatype == "URI" or (isinstance(value, str) and value.startswith("http")):
-                        value_obj["@id"] = str(value)
+                        # URL as a literal, not {"@id": url} (that is a resource ref).
+                        value_obj["text"] = str(value)
                     elif datatype in ["Date", "date"]:
                         value_obj["text"] = str(value)
                         value_obj["datatype"] = "xsd:date"

src/orkg_manager.py

@@ -12,9 +12,26 @@
 class ORKGPaperManager:
     """Manages the creation of papers in ORKG with extracted LLM data."""
 
-    def __init__(self, orkg_client: ORKGClient, template_mapper: TemplateMapper):
+    def __init__(
+        self,
+        orkg_client: ORKGClient,
+        template_mapper: TemplateMapper,
+        comparison_id: str = "R1364660",
+        comparison_title: Optional[str] = None,
+        comparison_description: Optional[str] = None,
+    ):
         self.client = orkg_client
         self.mapper = template_mapper
+        # Comparison to attach contributions to — id AND title must match the
+        # target ORKG instance (sandbox vs live), so they come from config.
+        # Updating a comparison writes a NEW VERSION with this title, so a wrong
+        # title would rename the live comparison. Defaults are a fallback only.
+        self.comparison_id = comparison_id
+        self.comparison_title = comparison_title or "Generative AI Model Landscape"
+        self.comparison_description = (
+            comparison_description
+            or "A landscape of Generative AI Models extracted from research papers."
+        )
 
     def process_and_upload(
         self, extraction_data: Dict[str, Any], paper_metadata: Optional[Dict[str, Any]] = None
@@ -87,52 +104,66 @@ def process_and_upload(
                 logger.error("Mapping failed - no valid contributions to upload")
                 return None
 
-            # 4. Check for existing paper to avoid duplicates
-            # TEMPORARILY DISABLED FOR TESTING - Always create new papers
+            # 4. On live, reuse an existing paper (dedup) and add only the
+            #    contributions that aren't already there. On sandbox/incubating we
+            #    intentionally skip the search and always create a fresh test paper
+            #    (its title carries a unique [TEST-...] suffix, so a search by the
+            #    real title wouldn't match one anyway).
             paper_id = None
             contribution_ids = []
-
-            # DISABLED: Paper search logic (for testing - always create new papers)
-            # existing_papers = self.client.search_papers(paper_title)
-            # for paper in existing_papers:
-            #     if paper.get("title", "").strip().lower() == paper_title.strip().lower():
-            #         paper_id = paper.get("id")
-            #         logger.info(f"Found existing paper in ORKG: {paper_id}")
-            #
-            #         # Fetch its contributions to check for duplicates
-            #         paper_data = self.client.get_paper(paper_id)
-            #         existing_contribs = paper_data.get('contributions', []) if paper_data else []  # noqa: E501
-            #         existing_labels = {c.get('label', '').strip().lower() for c in existing_contribs if isinstance(c, dict)}  # noqa: E501
-            #
-            #         contribution_ids = [c.get('id') for c in existing_contribs if isinstance(c, dict)]  # noqa: E501
-            #
-            #         # Add new contributions that don't exist yet
-            #         for contrib_data in mapped_data["contributions"]:
-            #             label = contrib_data.get("label", "").strip()
-            #             if label.lower() not in existing_labels:
-            #                 logger.info(f"Adding contribution '{label}' to paper {paper_id}")
-            #                 new_cid = self.client.add_contribution_to_paper(
-            #                     paper_id, contrib_data)
-            #                 if new_cid:
-            #                     contribution_ids.append(new_cid)
-            #                     existing_labels.add(label.lower())
-            #             else:
-            #                 logger.info(f"Contribution '{label}' exists, skipping")
-            #         break
-
-            # Always create new paper (for testing)
+            is_live = getattr(self.client, "host", "sandbox") == "production"
+
+            if is_live:
+                existing_papers = self.client.search_papers(paper_title) or []
+                for paper in existing_papers:
+                    if paper.get("title", "").strip().lower() == paper_title.strip().lower():
+                        paper_id = paper.get("id")
+                        logger.info(f"Found existing paper in ORKG: {paper_id}")
+
+                        # Fetch its existing contributions so we don't re-add them
+                        paper_data = self.client.get_paper(paper_id)
+                        existing_contribs = (
+                            paper_data.get("contributions", []) if paper_data else []
+                        )
+                        existing_labels = {
+                            c.get("label", "").strip().lower()
+                            for c in existing_contribs
+                            if isinstance(c, dict)
+                        }
+                        contribution_ids = [
+                            c.get("id")
+                            for c in existing_contribs
+                            if isinstance(c, dict) and c.get("id")
+                        ]
+
+                        # Add only the models not already on this paper
+                        for contrib_data in mapped_data["contributions"]:
+                            label = contrib_data.get("label", "").strip()
+                            if label.lower() in existing_labels:
+                                logger.info(f"Contribution '{label}' already exists, skipping")
+                                continue
+                            logger.info(f"Adding contribution '{label}' to paper {paper_id}")
+                            new_cid = self.client.add_contribution_to_paper(paper_id, contrib_data)
+                            if new_cid:
+                                contribution_ids.append(new_cid)
+                                existing_labels.add(label.lower())
+                        break
+
+            # Create a new paper when none matched (always, on sandbox)
             if not paper_id:
                 # 5. Create Paper with all Contributions (Step A)
-                # Add unique timestamp suffix to avoid ORKG API duplicate title rejection
-                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-                unique_paper_title = f"{paper_title} [TEST-{timestamp}]"
-                logger.info(
-                    "TESTING MODE: Paper search disabled. Creating new paper "
-                    "in ORKG (duplicates allowed)..."
-                )
-                logger.info(f"Using unique title: {unique_paper_title}")
+                # Live: real title. Sandbox/incubating: unique [TEST-...] suffix so
+                # repeated test runs don't collide on duplicate-title rejection.
+                if is_live:
+                    paper_title_to_use = paper_title
+                    logger.info("Creating new paper in ORKG (live)...")
+                else:
+                    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+                    paper_title_to_use = f"{paper_title} [TEST-{timestamp}]"
+                    logger.info("Creating new paper in ORKG (sandbox test mode, unique title)...")
+                logger.info(f"Using title: {paper_title_to_use}")
                 result = self.client.create_paper_with_contributions(
-                    title=unique_paper_title,
+                    title=paper_title_to_use,
                     authors=[
                         {"name": a} if isinstance(a, str) else a
                         for a in paper_metadata.get("authors", [])
@@ -152,17 +183,15 @@ def process_and_upload(
                 paper_id = result["paper_id"]
                 contribution_ids = result.get("contribution_ids", [])
 
-            # 6. Link to Comparison Table (Step B)
-            # Use the sandbox comparison ID
-            comparison_id = "R1364660"
-
+            # 6. Link to Comparison Table (Step B) — comparison is env-specific,
+            # supplied from config (sandbox vs live).
             logger.info(
-                f"Linking {len(contribution_ids)} contributions to comparison {comparison_id}"
+                f"Linking {len(contribution_ids)} contributions to comparison {self.comparison_id}"
             )
             self.client.update_comparison_with_contributions(
-                comparison_id=comparison_id,
-                title="Generative AI Model Landscape",
-                description="A landscape of Generative AI Models extracted from research papers.",
+                comparison_id=self.comparison_id,
+                title=self.comparison_title,
+                description=self.comparison_description,
                 new_contribution_ids=contribution_ids,
                 research_fields=["R133"],
                 authors=[{"name": "Alaa Kefi"}],

src/pipeline.py

@@ -219,7 +219,7 @@ def _initialize_components(self):
               logger.warning("PaperClassifier not initialized (no LLM model available)")
 
         # Initialize template mapper
-        self.template_mapper = TemplateMapper(template_id=self.config["orkg"]["template_id"])
+        self.template_mapper = TemplateMapper(template_id=self.config["orkg"]["template_id"], host=self.config["orkg"].get("host", "sandbox"))
 
     def _get_orkg_client(self):
         """Lazily initialize ORKG client (only when needed)."""
@@ -238,7 +238,11 @@ def _get_orkg_manager(self):
         if self._orkg_manager is None:
             logger.info("Initializing ORKG paper manager (lazy initialization)...")
             self._orkg_manager = ORKGPaperManager(
-                orkg_client=self._get_orkg_client(), template_mapper=self.template_mapper
+                orkg_client=self._get_orkg_client(),
+                template_mapper=self.template_mapper,
+                comparison_id=self.config["orkg"]["comparison_id"],
+                comparison_title=self.config["orkg"].get("comparison_title"),
+                comparison_description=self.config["orkg"].get("comparison_description"),
             )
         return self._orkg_manager