Add script/notebook-friendly config interface

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.4]
+
+Feature - Provide interface for using Tabulator as a package, allowing Jupyter notebooks to use Tabulator without handling a `config.json` file
+
 ## [0.1.3]
 
 Bug fix - building a table with entity_table now updates the tabulator object's

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rocrate-tabular"
-version = "0.1.2"
+version = "0.1.4"
 description = "A Python library to turn RO-Crates into tables"
 readme = "README.md"
 requires-python = ">=3.11"

src/rocrate_tabular/tabulator.py

@@ -5,6 +5,8 @@
 from pathlib import Path
 from sqlite_utils import Database
 from tqdm import tqdm
+import difflib
+import collections
 import csv
 import json
 import re
@@ -106,10 +108,10 @@ def build(self, properties):
         """Takes the properties of this entity and builds a dictionary to
         be inserted into the database, plus any junction records required"""
         self.data["entity_id"] = self.entity_id
-        self.cf = self.tabulator.cf["tables"][self.table]
+        self.config = self.tabulator.config["tables"][self.table]
         self.text_prop = self.tabulator.text_prop
-        self.expand_props = self.cf.get("expand_props", [])
-        self.ignore_props = self.cf.get("ignore_props", [])
+        self.expand_props = self.config.get("expand_props", [])
+        self.ignore_props = self.config.get("ignore_props", [])
         for prop_row in properties:
             prop = prop_row["property_label"]
             value = prop_row["value"]
@@ -144,7 +146,7 @@ def add_expanded_property(self, prop, target):
 
     def set_property(self, prop, value, target_id):
         """Add a property to entity_data, and add the target_id if defined"""
-        if prop in self.cf["junctions"]:
+        if prop in self.config["junctions"]:
             self.set_property_relational(prop, value, target_id)
         else:
             self.set_property_numbered(prop, value)
@@ -172,27 +174,166 @@ def set_property_relational(self, prop, value, target_id):
             self.junctions[prop].append(target_id)
 
 
+class Config(collections.UserDict):
+    """
+    Helper class to provide a default empty config and for pretty display in notebooks.
+    """
+
+    def __init__(self, _dict=None):
+        if not _dict:
+            self.data = {"export_queries": {}, "tables": {}, "potential_tables": {}}
+        else:
+            self.data = _dict
+
+    def _display_(self):
+        return self.data
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        self.data[key] = value
+
+
+class OutputList(collections.UserList):
+    """
+    Helper class for pretty output in notebooks.
+    """
+
+    def __init__(self, initlist, message=None):
+        self.message = message
+        super().__init__(initlist)
+
+    def _repr_markdown_(self):
+        if self.message:
+            return self.message
+        else:
+            return self[:]
+
+
+class OutputDict(collections.UserDict):
+    """
+    Helper class for pretty output in notebooks.
+    """
+
+    def __init__(self, *args, **kwargs):
+        return dict.__init__(self, *args, **kwargs)
+
+    def _repr_markdown_(self):
+        if self.message:
+            return self.message
+        else:
+            return self[:]
+
+
 class ROCrateTabulator:
     def __init__(self):
         self.crate_dir = None
         self.db_file = None
         self.db = None
         self.crate = None
-        self.cf = None
+        self.config = Config()
         self.text_prop = None
         self.schemaCrate = minimal_crate()
         self.encodedProps = {}
 
+    def use_tables(self, table_names):
+        if isinstance(table_names, str):
+            table_names = [table_names]
+        for table_name in table_names:
+            if table_name in self.config["tables"]:
+                raise ROCrateTabulatorException(
+                    f"already generated the `{table_name}` table"
+                )
+            if table_name not in self.config["potential_tables"]:
+                close_matches = difflib.get_close_matches(
+                    table_name, self.config["potential_tables"]
+                )
+                raise ROCrateTabulatorException(
+                    f"`{table_name}` is not recognised as a potential table. Did you mean: `{close_matches[0]}`?"
+                )
+            self.config["tables"][table_name] = self.config["potential_tables"][
+                table_name
+            ]
+            del self.config["potential_tables"][table_name]
+
+        message = "### Properties\n"
+        for table in self.config["tables"]:
+            props = self.entity_table(table)
+            self.config["tables"][table]["all_props"] = list(props)
+
+            message += f"<details><summary>{table}</summary>"
+            message += "<ul>"
+            for prop in props:
+                message += f"<li><code>{prop}</code></li>"
+            message += "</ul></details>\n\n"
+
+        message += """
+To attempt to expand references from a particular property
+and bring the values from linked entities into the primary
+table as columns: (example code, replace with your desired table and properties)
+
+```python
+tb.expand_properties("CreativeWork", ["author"])
+```
+
+To exlude a property from the table:
+
+```python
+tb.ignore_properties("CreativeWork", ["datePublished"])
+```
+        """
+
+        return OutputList(self.config["tables"], message)
+
+    def ignore_properties(self, table_name, props):
+        if isinstance(props, str):
+            props = [props]
+        for prop in props:
+            if table_name not in self.config["tables"]:
+                if table_name in self.config["potential_tables"]:
+                    raise ROCrateTabulatorException(
+                        f'please run `use_tables(["{table_name}"])` before ignoring a property on this table'
+                    )
+                else:
+                    close_matches = difflib.get_close_matches(
+                        table_name, self.config["tables"]
+                    )
+                    raise ROCrateTabulatorException(
+                        f"`{table_name}` is not recognised as a table name. Did you mean `{close_matches[0]}`?"
+                    )
+            self.config["tables"][table_name]["ignore_props"].append(prop)
+            self.config["tables"][table_name]["all_props"].remove(prop)
+
+    def expand_properties(self, table_name, props):
+        if isinstance(props, str):
+            props = [props]
+        for prop in props:
+            if table_name not in self.config["tables"]:
+                if table_name in self.config["potential_tables"]:
+                    raise ROCrateTabulatorException(
+                        f'please run `use_tables(["{table_name}"])` before ignoring a property on this table'
+                    )
+                else:
+                    close_matches = difflib.get_close_matches(
+                        table_name, self.config["tables"]
+                    )
+                    raise ROCrateTabulatorException(
+                        f"`{table_name}` is not recognised as a table name. Did you mean `{close_matches[0]}`?"
+                    )
+            self.config["tables"][table_name]["expand_props"].append(prop)
+            self.config["tables"][table_name]["all_props"].remove(prop)
+
     def load_config(self, config_file):
         """Load config from file"""
         close_file = False
         if isinstance(config_file, (str, PathLike)):
-            config_file = open(config_file, "r")
+            config_file = open(config_file, "r", encoding="utf-8")
             close_file = True
         else:
             config_file.seek(0)
 
-        self.cf = json.load(config_file)
+        self.config = json.load(config_file)
 
         if close_file:
             config_file.close()
@@ -205,25 +346,39 @@ def infer_config(self):
             raise ROCrateTabulatorException(
                 "Need to run crate_to_db before infer_config"
             )
-        self.cf = {"export_queries": {}, "tables": {}, "potential_tables": {}}
+        self.config = Config()
 
         for attype in self.fetch_types():
-            self.cf["potential_tables"][attype] = {
+            self.config["potential_tables"][attype] = {
                 "all_props": [],
                 "ignore_props": [],
                 "expand_props": [],
             }
 
+        return OutputList(
+            self.fetch_types(),
+            f"""
+Potential tables:
+{", ".join(self.config["potential_tables"].keys())}
+
+To create your tables run: (example code, replace with your desired tables)
+```python
+tb.use_tables(["CreativeWork", "Person"])
+```
+""",
+        )
+
     def write_config(self, config_file):
         """Write the config file with any changes made"""
         close_file = False
         if isinstance(config_file, (str, PathLike)):
-            config_file = open(config_file, "w")
+            config_file = open(config_file, "w", encoding="utf-8")
             close_file = True
         else:
             config_file.seek(0)
 
-        json.dump(self.cf, config_file, indent=4)
+        # `default=dict` makes json.dump pass our Config subclass of UserDict into dict(), making it serialisable
+        json.dump(self.config, config_file, indent=4, default=dict)
 
         if close_file:
             config_file.close()
@@ -366,19 +521,19 @@ def entity_table(self, table, text_prop=None):
                     )
                     seq += 1
         self.db[table].insert_all(entities, pk="entity_id", replace=True, alter=True)
-        self.cf["tables"][table]["all_props"] = list(allprops)
+        self.config["tables"][table]["all_props"] = list(allprops)
         return list(allprops)
 
     def entity_table_plan(self, table):
         """Check entity relations to see if any need to be done as a junction
         table to avoid huge numbers of expanded columns"""
-        if "junctions" not in self.cf["tables"][table]:
-            self.cf["tables"][table]["junctions"] = []
+        if "junctions" not in self.config["tables"][table]:
+            self.config["tables"][table]["junctions"] = []
         for prop_counts in self.fetch_relation_counts(table):
             if prop_counts["n_links"] > MAX_NUMBERED_COLS:
                 label = prop_counts["property_label"]
                 print(f"{table}.{label} > {MAX_NUMBERED_COLS} relations")
-                self.cf["tables"][table]["junctions"].append(label)
+                self.config["tables"][table]["junctions"].append(label)
 
     # Some helper methods for wrapping SQLite statements
 
@@ -435,9 +590,9 @@ def fetch_relation_counts(self, t):
     def export_csv(self, rocrate_dir):
         """Export csvs as configured"""
 
-        queries = self.cf["export_queries"]
+        queries = self.config["export_queries"]
         # print("Global props", self.global_props)
-        # self.cf["global_props"] = list(self.global_props)
+        # self.config["global_props"] = list(self.global_props)
 
         # Ensure rocrate_dir exists if it's provided
         if rocrate_dir is not None:
@@ -451,7 +606,7 @@ def export_csv(self, rocrate_dir):
             csv_path = csv_filename
             if rocrate_dir is not None:
                 csv_path = Path(rocrate_dir) / csv_filename
-            with open(csv_path, "w", newline="") as csvfile:
+            with open(csv_path, "w", newline="", encoding="utf-8") as csvfile:
                 writer = csv.DictWriter(
                     csvfile, fieldnames=result[0].keys(), quoting=csv.QUOTE_MINIMAL
                 )
@@ -600,9 +755,10 @@ def main(args):
         tb.infer_config()
 
     tb.text_prop = args.text
-    for table in tb.cf["tables"]:
+    for table in tb.config["tables"]:
         print(f"Building entity table for {table}")
-        tb.entity_table(table)
+        allprops = tb.entity_table(table)
+        tb.config["tables"][table]["all_props"] = list(allprops)
 
     tb.write_config(args.config)
     print(f"""

tests/test_package_interface.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+from rocrate_tabular.tabulator import ROCrateTabulator
+
+
+def test_config_interface(crates, tmp_path):
+    """Test that the first-pass config can be read"""
+    cwd = Path(tmp_path)
+    dbfile = cwd / "sqlite.db"
+    # conffile = cwd / "config.json"
+    tb = ROCrateTabulator()
+    tb.crate_to_db(crates["minimal"], dbfile)
+    tb.infer_config()
+    tb.use_tables(["Dataset"])
+    # TODO: finish this (check output)
+    tb.close()  # for Windows
+
+
+# TODO: test that passes string to use_tables