diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c17a0..868696a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.3] + +Bug fix - building a table with entity_table now updates the tabulator object's +cf config object again + ## [0.1.2] Bug fix release - made indexing text work again and added a test for it diff --git a/README.md b/README.md index e61384a..9473664 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,68 @@ Install [uv](https://docs.astral.sh/uv/), then > git clone git@github.com:Sydney-Informatics-Hub/rocrate-tabular.git > cd rocrate-tabular - > uv run src/rocrate_tabular/rocrate_tabular.py -h + > uv run tabulator --help `uv run` should create a local venv and install the dependencies ## Usage - > uv run src/rocrate_tabular/rocrate_tabular.py path/to/crate crate.db +First pass: this will scan an RO-Crate directory, build a `properties` table +in the sqlite database crate.db, and generate a config file with a list of +all available tables in the `potential_tables` section + > uv run tabulator -c config.json ./path/to/crate crate.db +You can then edit the config file and move the tables you want to create in +the database and/or csv to the `tables` section, and re-run the tabulator + + > uv run tabulator -c config.json ./path/to/crate crate.db + +To export a CSV version of any of the tables, you can define a filename / query +pair in the "export queries" section of the config file, for example: + + { + "export_queries": { + "repo_objects.csv": "SELECT * FROM RepositoryObject" + }, + "tables": { + "RepositoryObject": { + "all_props": [ ... ], + "ignore_props": [ ... ], + "expand_props": [ ... ] + } + } + } + +## Expanded properties + +The "expand_props" field in a table's config can be used to tell the tabulator +to try to follow references from a particular property and bring the values +from linked entities into the primary table as columns. For example, here is +a fragment of an RO-Crate with a CreativeWork and a Person who is its author: + + { + "@id": "#a_creative_work", + "@type": "CreativeWork", + "name": "A Creative Work", + "description": "A creative work", + "author": { "@id": "#jane_smith" } + }, + { + "@id": "#jane_smith", + "@type": "Person", + "name": "Jane Smith" + } + +If "author" is in "expand_props" on the "CreativeWork" entry in the "tables" +section of the config, the following additional columns will be created in the +CreativeWork table: + + author_@id + author_@type + author_name + +## Ignored properties + +Properties in the "ignore_props" section will not be added to the tables, and +will also be ignored when looking up expanded properties. \ No newline at end of file diff --git a/src/rocrate_tabular/tabulator.py b/src/rocrate_tabular/tabulator.py index 00fbd5a..609f266 100644 --- a/src/rocrate_tabular/tabulator.py +++ b/src/rocrate_tabular/tabulator.py @@ -366,7 +366,8 @@ def entity_table(self, table, text_prop=None): ) seq += 1 self.db[table].insert_all(entities, pk="entity_id", replace=True, alter=True) - return allprops + self.cf["tables"][table]["all_props"] = list(allprops) + return list(allprops) def entity_table_plan(self, table): """Check entity relations to see if any need to be done as a junction @@ -601,8 +602,7 @@ def main(args): tb.text_prop = args.text for table in tb.cf["tables"]: print(f"Building entity table for {table}") - allprops = tb.entity_table(table) - tb.cf["tables"][table]["all_props"] = list(allprops) + tb.entity_table(table) tb.write_config(args.config) print(f""" diff --git a/tests/test_basic.py b/tests/test_basic.py index a5dc0c8..561a9ba 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -98,4 +98,12 @@ def test_all_props(crates, tmp_path): for table in cf["tables"]: all_props = tb.entity_table(table) - assert all_props == props[table] + assert set(all_props) == props[table] + + # check that the properties are also now in the config file + tb.write_config(conffile) + + cf_out = read_config(conffile) + + for table in cf_out["tables"]: + assert set(cf_out["tables"][table]["all_props"]) == props[table] diff --git a/tests/test_expanded_properties.py b/tests/test_expanded_properties.py index 580a4a1..a411a68 100644 --- a/tests/test_expanded_properties.py +++ b/tests/test_expanded_properties.py @@ -1,6 +1,31 @@ -import pytest +from util import tabulator, read_config, write_config +from pathlib import Path +from tinycrate.tinycrate import TinyCrate -@pytest.mark.skip("Not implemented") -def test_expanded_properties(tmp_path): - assert True +# FIXME this is very basic +def test_expanded_properties(crates, tmp_path): + expand_props = ["author", "publisher"] + tb = tabulator(tmp_path, crates["languageFamily"]) + cffile = Path(tmp_path) / "config.json" + cf = read_config(cffile) + cf["tables"]["Dataset"]["expand_props"] = expand_props + + write_config(cf, cffile) + tb.load_config(cffile) + tb.entity_table("Dataset") + rows = list(tb.db.query("SELECT * FROM Dataset")) + assert len(rows) == 1 + row = rows[0] + + tc = TinyCrate(crates["languageFamily"]) + te = tc.get(row["entity_id"]) + assert te + for prop in expand_props: + rele = tc.deref(te, prop) + assert rele + for relprop in rele.props: + if not relprop == "@id": + eprop = f"{prop}_{relprop}" + assert eprop in row + assert row[eprop] == rele[relprop]