Rc 0.1.3 - bugfix for all_props

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.3]
+
+Bug fix - building a table with entity_table now updates the tabulator object's
+cf config object again
+
 ## [0.1.2]
 
 Bug fix release - made indexing text work again and added a test for it

README.md

@@ -8,12 +8,68 @@ Install [uv](https://docs.astral.sh/uv/), then
 
     > git clone git@github.com:Sydney-Informatics-Hub/rocrate-tabular.git
     > cd rocrate-tabular
-    > uv run src/rocrate_tabular/rocrate_tabular.py -h
+    > uv run tabulator --help
 
 `uv run` should create a local venv and install the dependencies
 
 ## Usage
 
-    > uv run src/rocrate_tabular/rocrate_tabular.py path/to/crate crate.db 
+First pass: this will scan an RO-Crate directory, build a `properties` table
+in the sqlite database crate.db, and generate a config file with a list of
+all available tables in the `potential_tables` section
 
+    > uv run tabulator -c config.json ./path/to/crate crate.db
 
+You can then edit the config file and move the tables you want to create in
+the database and/or csv to the `tables` section, and re-run the tabulator
+
+    > uv run tabulator -c config.json ./path/to/crate crate.db
+
+To export a CSV version of any of the tables, you can define a filename / query
+pair in the "export queries" section of the config file, for example:
+
+    {
+        "export_queries": {
+            "repo_objects.csv": "SELECT * FROM RepositoryObject"
+        },
+        "tables": {
+            "RepositoryObject": {
+                "all_props": [ ... ],
+                "ignore_props": [ ... ],
+                "expand_props": [ ... ]                
+            }
+        }
+    }
+
+## Expanded properties
+
+The "expand_props" field in a table's config can be used to tell the tabulator
+to try to follow references from a particular property and bring the values
+from linked entities into the primary table as columns. For example, here is
+a fragment of an RO-Crate with a CreativeWork and a Person who is its author:
+
+    {
+        "@id": "#a_creative_work",
+        "@type": "CreativeWork",
+        "name": "A Creative Work",
+        "description": "A creative work", 
+        "author": { "@id": "#jane_smith" }
+    },
+    {   
+        "@id": "#jane_smith",
+        "@type": "Person",
+        "name": "Jane Smith"
+    }
+
+If "author" is in "expand_props" on the "CreativeWork" entry in the "tables"
+section of the config, the following additional columns will be created in the
+CreativeWork table:
+
+    author_@id
+    author_@type
+    author_name
+
+## Ignored properties
+
+Properties in the "ignore_props" section will not be added to the tables, and
+will also be ignored when looking up expanded properties. 

src/rocrate_tabular/tabulator.py

@@ -366,7 +366,8 @@ def entity_table(self, table, text_prop=None):
                     )
                     seq += 1
         self.db[table].insert_all(entities, pk="entity_id", replace=True, alter=True)
-        return allprops
+        self.cf["tables"][table]["all_props"] = list(allprops)
+        return list(allprops)
 
     def entity_table_plan(self, table):
         """Check entity relations to see if any need to be done as a junction
@@ -601,8 +602,7 @@ def main(args):
     tb.text_prop = args.text
     for table in tb.cf["tables"]:
         print(f"Building entity table for {table}")
-        allprops = tb.entity_table(table)
-        tb.cf["tables"][table]["all_props"] = list(allprops)
+        tb.entity_table(table)
 
     tb.write_config(args.config)
     print(f"""

tests/test_basic.py

@@ -98,4 +98,12 @@ def test_all_props(crates, tmp_path):
 
     for table in cf["tables"]:
         all_props = tb.entity_table(table)
-        assert all_props == props[table]
+        assert set(all_props) == props[table]
+
+    # check that the properties are also now in the config file
+    tb.write_config(conffile)
+
+    cf_out = read_config(conffile)
+
+    for table in cf_out["tables"]:
+        assert set(cf_out["tables"][table]["all_props"]) == props[table]

tests/test_expanded_properties.py

@@ -1,6 +1,31 @@
-import pytest
+from util import tabulator, read_config, write_config
+from pathlib import Path
+from tinycrate.tinycrate import TinyCrate
 
 
-@pytest.mark.skip("Not implemented")
-def test_expanded_properties(tmp_path):
-    assert True
+# FIXME this is very basic
+def test_expanded_properties(crates, tmp_path):
+    expand_props = ["author", "publisher"]
+    tb = tabulator(tmp_path, crates["languageFamily"])
+    cffile = Path(tmp_path) / "config.json"
+    cf = read_config(cffile)
+    cf["tables"]["Dataset"]["expand_props"] = expand_props
+
+    write_config(cf, cffile)
+    tb.load_config(cffile)
+    tb.entity_table("Dataset")
+    rows = list(tb.db.query("SELECT * FROM Dataset"))
+    assert len(rows) == 1
+    row = rows[0]
+
+    tc = TinyCrate(crates["languageFamily"])
+    te = tc.get(row["entity_id"])
+    assert te
+    for prop in expand_props:
+        rele = tc.deref(te, prop)
+        assert rele
+        for relprop in rele.props:
+            if not relprop == "@id":
+                eprop = f"{prop}_{relprop}"
+                assert eprop in row
+                assert row[eprop] == rele[relprop]