Found uncomitted: Adding catalog config we need to tune etc

note: was conflict in docker-compose.dev.local.override.yml -- copied
invocation of datalad-catalog create

Author: yarikoptic

datalad_registry/overview.py

@@ -3,10 +3,25 @@
 
 import logging
 
-from flask import Blueprint, render_template, request
-from sqlalchemy import Text, nullslast, or_
-
-from datalad_registry.models import RepoUrl, URLMetadata, db
+import datalad.api as dl
+from flask import (
+    Blueprint,
+    render_template,
+    request,
+    send_from_directory,
+)
+from sqlalchemy import (
+    Text,
+    nullslast,
+    or_,
+)
+
+from datalad_registry.blueprints.api.url_metadata import URLMetadataModel
+from datalad_registry.models import (
+    RepoUrl,
+    URLMetadata,
+    db,
+)
 
 lgr = logging.getLogger(__name__)
 bp = Blueprint("overview", __name__, url_prefix="/overview")
@@ -85,16 +100,12 @@ def overview():  # No type hints due to mypy#7187.
         url_filter=filter,
     )
 
-import json
-from flask import send_from_directory
-import datalad.api as dl
-
 
 # @bp.route('/catalog/', defaults={'path': ''})
 # TODO: move from placing dataset identifier within path -- place into query
-# TODO: do not use ID may be but use URL, or allow for both -- that would make it possible to make those URLs
-#  pointing to datasets easier to create/digest for humans
-@bp.route('/catalog/<int:id_>/<path:path>')
+# TODO: do not use ID may be but use URL, or allow for both -- that would make it
+#  possible to make those URLs pointing to datasets easier to create/digest for humans
+@bp.route("/catalog/<int:id_>/<path:path>")
 def send_report(id_, path):
     # ds_id = request.args.get("id", None, type=int)
     if not path:
@@ -109,17 +120,31 @@ def send_report(id_, path):
             repo_url_row = repo_url_row[0]
             metadatas = {}
             for mr in repo_url_row.metadata_:
-                if mr.extractor_name not in {'metalad_core', 'bids_dataset', 'metalad_studyminimeta'}:
+                if mr.extractor_name not in {
+                    "metalad_core",
+                    "bids_dataset",
+                    "metalad_studyminimeta",
+                }:
                     continue
-                # TODO: here metadta record had only @context and @graph and no other fields
-                # figure out if enough....
-                m = mr.extracted_metadata
-                m['type'] = 'dataset'
-                m['dataset_id'] = repo_url_row.ds_id
+                # TODO: here metadta record had only @context and @graph and no other
+                #  fields figure out if enough....
+                m = URLMetadataModel.from_orm(mr).dict()
+                # lgr.warning(f"ROW: {m}")
+                m["type"] = "dataset"
+                m["dataset_id"] = repo_url_row.ds_id
                 # Didn't want to translate yet
                 lgr.warning(f"Translating record with keys {m.keys()}")
-                metadatas[mr.extractor_name] = dl.catalog_translate(m)
-                # metadatas[mr.extractor_name] = m
-            lgr.warning(f"ROW: {metadatas}")
+                m_translated = dl.catalog_translate(m)[0]["translated_metadata"]
+                metadatas[mr.extractor_name] = m_translated
+
+            if "metalad_studyminimeta" not in metadatas:
+                metadatas["metalad_core"]["name"] = repo_url_row.url
+
+            for m in metadatas.values():
+                m["name"] = repo_url_row.url
+                lgr.warning(f"URL: {repo_url_row.url!r} {type(repo_url_row.url)}")
+                dl.catalog_add("/app-catalog", metadata=m)
     # TODO: figure out how to pass all the metadata goodness to the catalog
-    return send_from_directory('/app-catalog', path)
+    # f'/app-catalog/dataset/{repo_url_row.ds_id}/'
+    #                     f'{metadatas['metalad_core']['dataset_version']}'
+    return send_from_directory("/app-catalog", path)

datalad_registry/resources/catalog-config.json

@@ -0,0 +1,80 @@
+{
+  "catalog_name": "DataCat of Registry",
+  "logo_path": "",
+  "link_color": "#fba304",
+  "link_hover_color": "#af7714",
+  "social_links": {
+    "about": null,
+    "documentation": "https://docs.datalad.org/projects/catalog/en/latest/",
+    "github": "https://github.com/datalad/datalad-catalog",
+    "mastodon": "https://fosstodon.org/@datalad",
+    "x": "https://x.com/datalad"
+  },
+  "dataset_options": {
+    "include_metadata_export": true
+  },
+  "property_sources": {
+    "dataset": {
+      "dataset_id": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "dataset_version": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "type": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "children": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "short_name": {},
+      "description": {
+        "rule": "priority",
+        "source": [
+          "catalog_readme",
+          "metalad_studyminimeta",
+          "datacite_gin",
+          "bids_dataset"
+        ]
+      },
+      "doi": {},
+      "url": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "authors": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "keywords": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "license": {},
+      "funding": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "publications": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "subdatasets": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "additional_display": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "top_display": {
+        "rule": "merge",
+        "source": "any"
+      }
+    }
+  }
+}

docker-compose.dev.local.override.yml

@@ -7,7 +7,7 @@ services:
     command: [
       "/sbin/my_init", "--",
       "bash", "-c",
-      "git config --global --add safe.directory /app && pip3 install -U -e . && pip install -e /metalad && pip install -e /catalog && flask init-db && exec flask run --host=0.0.0.0 --debug"
+      "git config --global --add safe.directory /app && pip3 install -U -e . && pip install -e /metalad && pip install -e /catalog && flask init-db && datalad catalog-create -c /app-catalog -F /app/datalad_registry/resources/catalog-config.json && exec flask run --host=0.0.0.0 --debug"
     ]
     volumes:
       - ./:/app