diff --git a/.gitignore b/.gitignore index 4b669d1..8c80a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ *-config.json +.python-version .idea .DS_Store diff --git a/.python-version b/.python-version deleted file mode 100644 index 2c07333..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.11 diff --git a/pyproject.toml b/pyproject.toml index a1a190f..91cdf79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,14 @@ [project] name = "rocrate-tabular" -version = "0.1.0" -description = "Add your description here" +version = "0.1.1" +description = "A Python library to turn RO-Crates into tables" readme = "README.md" requires-python = ">=3.11" dependencies = [ "requests>=2.32.3", "sqlite-utils>=3.37", + "tinycrate>=0.1.2", + "tqdm>=4.67.1", ] [project.scripts] diff --git a/src/rocrate_tabular/find_bad_bytes.py b/src/rocrate_tabular/find_bad_bytes.py new file mode 100644 index 0000000..6e66031 --- /dev/null +++ b/src/rocrate_tabular/find_bad_bytes.py @@ -0,0 +1,13 @@ +# quick script to find wide characters in COOEE text files for the Windoes +# encoding bug + +from pathlib import Path + +d = Path("./cooee/data") +print(d) +for fn in sorted(d.glob("*.txt")): + try: + with open(fn, "r", encoding="cp1252") as fh: + dummy = fh.readlines() + except Exception as e: + print(f"read failed {fn} {e}") diff --git a/src/rocrate_tabular/tabulator.py b/src/rocrate_tabular/tabulator.py index 240f351..616f508 100644 --- a/src/rocrate_tabular/tabulator.py +++ b/src/rocrate_tabular/tabulator.py @@ -1,12 +1,57 @@ from os import PathLike -from rocrate_tabular.tinycrate import TinyCrate, TinyCrateException +from tinycrate.tinycrate import TinyCrate, TinyCrateException, minimal_crate from argparse import ArgumentParser from pathlib import Path from sqlite_utils import Database +from tqdm import tqdm import csv import json +import re import requests +import sys +from dataclasses import dataclass, field + +# FIXME: add real logging + +# TERMINOLOGY + +# a 'relation' is an entity which refers to another entity by some property +# whose value is like { '@id': '#foo' } + +# a 'junction' is a many-to-many relation through its own table which is +# used to capture relations in the database like: +# +# [ +# { +# "@id": "#adocument" +# "@type": "CreativeWork" +# "name": "Title of Document" +# "author": [ +# { "@id": "#jdoe" }, +# { "@id": "#jroe" } +# ] +# }, +# { +# "@id": "#jdoe", +# "@type": "Person", +# "name": "John Doe" +# }, +# { +# "@id": "#jroe", +# "@type": "Person", +# "name": "Jane Roe" +# } +# ] +# +# this is modeled in the database as +# Document [ id, name ] +# Document_hasPart [ Document_id, Person_id ] +# Person [ id, name ] +# +# Note that the junction table is on Type_property which means that the +# target @ids could be of different @types - for example, Datasets could have +# Datesets and Files as hasParts PROPERTIES = { "row_id": str, @@ -17,7 +62,8 @@ "value": str, } -MAX_NUMBERED_COLS = 999 # sqllite limit +MAX_NUMBERED_COLS = 10 +# MAX_NUMBERED_COLS = 999 # sqllite limit def get_as_list(v): @@ -42,6 +88,90 @@ class ROCrateTabulatorException(Exception): pass +@dataclass +class EntityRecord: + """Class which represents an entity as mapped to a database row, + plus any records which are used in junction tables""" + + table: str + tabulator: object + entity_id: str + expand_props: list = field(default_factory=list) + ignore_props: list = field(default_factory=list) + props: set = field(default_factory=set) + data: dict = field(default_factory=dict) + junctions: dict = field(default_factory=dict) + + def build(self, properties): + """Takes the properties of this entity and builds a dictionary to + be inserted into the database, plus any junction records required""" + self.data["entity_id"] = self.entity_id + self.cf = self.tabulator.cf["tables"][self.table] + self.text_prop = self.tabulator.text_prop + self.expand_props = self.cf.get("expand_props", []) + self.ignore_props = self.cf.get("ignore_props", []) + for prop_row in properties: + prop = prop_row["property_label"] + value = prop_row["value"] + target = prop_row["target_id"] + self.props.add(prop) + if prop == self.text_prop: + try: + self.data[prop] = self.crate.get(target).fetch() + except TinyCrateException as e: + self.data[prop] = f"load failed: {e}" + else: + if prop in self.expand_props and target: + self.add_expanded_property(prop, target) + else: + if prop not in self.ignore_props: + self.set_property(prop, value, target) + return self.props + + def add_expanded_property(self, prop, target): + """Do a subquery on a target ID to make expanded properties like + author_name author_id""" + for ep_row in self.tabulator.fetch_properties(target): + expanded_prop = f"{prop}_{ep_row['property_label']}" + # Special case - if this is indexable text then we want to read t + self.props.add(expanded_prop) + if expanded_prop not in self.ignore_props: + self.set_property( + expanded_prop, + ep_row["value"], + ep_row["target_id"], + ) + + def set_property(self, prop, value, target_id): + """Add a property to entity_data, and add the target_id if defined""" + if prop in self.cf["junctions"]: + self.set_property_relational(prop, value, target_id) + else: + self.set_property_numbered(prop, value) + if target_id: + self.set_property_numbered(f"{prop}_id", target_id) + + # TODO: we should only call this if there are more than one + def set_property_numbered(self, prop, value): + if prop in self.data: + # Find the first available integer to append to property_name + i = 1 + while f"{prop}_{i}" in self.data: + i += 1 + prop = f"{prop}_{i}" + if i > MAX_NUMBERED_COLS: + raise ROCrateTabulatorException(f"Too many columns for {prop}") + self.data[prop] = value + + def set_property_relational(self, prop, value, target_id): + """Add junctions between an entity and related entities""" + # FIXME what happens to value here? + if prop not in self.junctions: + self.junctions[prop] = [target_id] + else: + self.junctions[prop].append(target_id) + + class ROCrateTabulator: def __init__(self): self.crate_dir = None @@ -49,8 +179,11 @@ def __init__(self): self.db = None self.crate = None self.cf = None + self.text_prop = None + self.schemaCrate = minimal_crate() + self.encodedProps = {} - def load_config(self, config_file): + def read_config(self, config_file): """Load config from file""" close_file = False if isinstance(config_file, (str, PathLike)): @@ -73,14 +206,8 @@ def infer_config(self): "Need to run crate_to_db before infer_config" ) self.cf = {"export_queries": {}, "tables": {}, "potential_tables": {}} - query = """ - SELECT DISTINCT(p.value) - FROM property p - WHERE p.property_label = '@type' - """ - types = self.db.query(query) - for attype in [row["value"] for row in types]: + for attype in self.fetch_types(): self.cf["potential_tables"][attype] = { "all_props": [], "ignore_props": [], @@ -103,20 +230,25 @@ def write_config(self, config_file): else: config_file.seek(0) - def crate_to_db(self, crate_uri, db_file): - """Load the crate and build the properties table""" + def crate_to_db(self, crate_uri, db_file, rebuild=True): + """Load the crate and build the properties and relations tables""" self.crate_dir = crate_uri try: jsonld = self._load_crate(crate_uri) - self.crate = TinyCrate(jsonld=jsonld, directory=crate_uri) + self.crate = TinyCrate(jsonld) except Exception as e: raise ROCrateTabulatorException(f"Crate load failed: {e}") self.db_file = db_file + if not rebuild: + if not Path(db_file).is_file(): + raise ROCrateTabulatorException(f"db file {db_file} not found") + self.db = Database(self.db_file) + return self.db = Database(self.db_file, recreate=True) properties = self.db["property"].create(PROPERTIES) seq = 0 propList = [] - for e in self.crate.all(): + for e in tqdm(self.crate.all()): for row in self.entity_properties(e): row["row_id"] = seq seq += 1 @@ -128,6 +260,35 @@ def close(self): """Close the connection to the SQLite database - for Windows users""" self.db.close() + def dump_structure(self): + """Testing getting metadata about relations from the database""" + # get all types + + for t in self.fetch_types(): + print(f"@type: {t}") + query = """ + SELECT p.source_id, p.property_label, count(p.target_id) as n_links + FROM property as p + WHERE p.source_id IN ( + SELECT p.source_id + FROM property p + WHERE p.property_label = '@type' AND p.value = ? + ) + GROUP BY p.source_id, p.property_label + ORDER BY n_links desc + LIMIT 1 + """ + summary = self.db.query(query, [t]) + for row in summary: + if row["n_links"] > 0: + print( + row["source_id"] + + "." + + row["property_label"] + + ": " + + str(row["n_links"]) + ) + def _load_crate(self, crate_uri): if crate_uri[:4] == "http": response = requests.get(crate_uri) @@ -175,16 +336,58 @@ def property_row(self, eid, ename, prop, value): "value": value, } - def entity_table(self, table, text): - """Build a db table for one type of entity""" - self.text_prop = text - - for entity_id in self.fetch_ids(table): - properties = list(self.fetch_entity(entity_id)) - self.flatten_one_entity(table, entity_id, properties) + def entity_table(self, table): + """Build a db table for one type of entity. Returns a set() of all + the properties found during the build""" + self.entity_table_plan(table) + entities = [] + allprops = set() + for entity_id in tqdm(list(self.fetch_ids(table))): + entity = EntityRecord(tabulator=self, table=table, entity_id=entity_id) + props = entity.build(self.fetch_properties(entity_id)) + allprops.update(props) + entities.append(entity.data) + for prop, target_ids in entity.junctions.items(): + jtable = f"{table}_{prop}" + seq = 0 + for target_id in target_ids: + self.db[jtable].insert( + { + "seq": seq, + "entity_id": entity_id, + "target_id": target_id, + }, + pk=("entity_id", "target_id"), + replace=True, + alter=True, + ) + seq += 1 + self.db[table].insert_all(entities, pk="entity_id", replace=True, alter=True) + return allprops + + def entity_table_plan(self, table): + """Check entity relations to see if any need to be done as a junction + table to avoid huge numbers of expanded columns""" + if "junctions" not in self.cf["tables"][table]: + self.cf["tables"][table]["junctions"] = [] + for prop_counts in self.fetch_relation_counts(table): + if prop_counts["n_links"] > MAX_NUMBERED_COLS: + label = prop_counts["property_label"] + print(f"{table}.{label} > {MAX_NUMBERED_COLS} relations") + self.cf["tables"][table]["junctions"].append(label) # Some helper methods for wrapping SQLite statements + def fetch_types(self): + """return all types in the database""" + rows = self.db.query(""" + SELECT DISTINCT(p.value) + FROM property p + WHERE p.property_label = '@type' + """) + for t in [row["value"] for row in rows]: + yield t + def fetch_ids(self, entity_type): """return a generator which yields all ids of this type""" rows = self.db.query( @@ -198,7 +401,7 @@ def fetch_ids(self, entity_type): for entity_id in [row["source_id"] for row in rows]: yield entity_id - def fetch_entity(self, entity_id): + def fetch_properties(self, entity_id): """return a generator which yields all properties for an entity""" properties = self.db.query( """ @@ -211,99 +414,96 @@ def fetch_entity(self, entity_id): for prop in properties: yield prop - def flatten_one_entity(self, table, entity_id, properties): - """Add a single entity's properties to its table""" - # FIXME - analyse properties and find multiples - # Create a dictionary to hold the properties for this entity - entity_data = {"entity_id": entity_id} - config = self.cf["tables"][table] - expand_props = config.get("expand_props") - ignore_props = config.get("ignore_props") - # Loop through properties and add them to entity_data - props = set() - for prop in properties: - name = prop["property_label"] - value = prop["value"] - target = prop["target_id"] - props.add(name) - - if self.text_prop and name == self.text_prop: - try: - print(f"looking for target: {target}") - entity_data[name] = self.crate.get(target).fetch() - except TinyCrateException as e: - entity_data[name] = f"load failed: {e}" - - else: - if name in expand_props and target: - props.update( - self.add_expanded_property( - entity_data, ignore_props, name, target - ) - ) - else: - # If it's a normal property, just add it to the entity_data dictionary - if name not in ignore_props: - self.set_property(entity_data, name, value, target) - self.db[table].insert(entity_data, pk="entity_id", replace=True, alter=True) - props.update(config["all_props"]) - config["all_props"] = list(props) - - def add_expanded_property(self, entity_data, ignore_props, name, target): - """Do a subquery on a target ID to make expanded properties like - author_name author_id""" - props = set() - for expanded_prop in self.fetch_entity(target): - expanded_property_name = f"{name}_{expanded_prop['property_label']}" - # Special case - if this is indexable text then we want to read t - props.add(expanded_property_name) - if expanded_property_name not in ignore_props: - self.set_property( - entity_data, - expanded_property_name, - expanded_prop["value"], - expanded_prop["target_id"], - ) - return props + def fetch_relation_counts(self, t): + query = """ + SELECT p.source_id, p.property_label, count(p.target_id) as n_links + FROM property as p + WHERE p.source_id IN ( + SELECT p.source_id + FROM property p + WHERE p.property_label = '@type' AND p.value = ? + ) + GROUP BY p.source_id, p.property_label + ORDER BY n_links desc + """ + return self.db.query(query, [t]) - # Both of the following mutate entity_data + def export_csv(self, rocrate_dir): + """Export csvs as configured""" - def set_property(self, entity_data, name, value, target_id): - """Add a property to entity_data, and add the target_id if defined""" - self.set_property_name(entity_data, name, value) - if target_id: - self.set_property_name(entity_data, f"{name}_id", target_id) + queries = self.cf["export_queries"] + # print("Global props", self.global_props) + # self.cf["global_props"] = list(self.global_props) - def set_property_name(self, entity_data, name, value): - """FIXME - strategy by property goes here""" - if name in entity_data: - # Find the first available integer to append to property_name - i = 1 - while f"{name}_{i}" in entity_data: - i += 1 - name = f"{name}_{i}" - if i > MAX_NUMBERED_COLS: - raise ROCrateTabulatorException(f"Too many columns for {name}") - entity_data[name] = value + # Ensure rocrate_dir exists if it's provided + if rocrate_dir is not None: + Path(rocrate_dir).mkdir(parents=True, exist_ok=True) + files = [] - def export_csv(self): - """Export csvs as configured""" - queries = self.cf["export_queries"] - for csv_file, query in queries.items(): + for csv_filename, query in queries.items(): + files.append({"@id": csv_filename}) result = list(self.db.query(query)) # Convert result into a CSV file using csv writer - with open(csv_file, "w", newline="", encoding="utf-8") as csvfile: + csv_path = csv_filename + if rocrate_dir is not None: + csv_path = Path(rocrate_dir) / csv_filename + with open(csv_path, "w", newline="") as csvfile: writer = csv.DictWriter( csvfile, fieldnames=result[0].keys(), quoting=csv.QUOTE_MINIMAL ) writer.writeheader() + # Check if the key is a string and replace newlines + + keys = set() + for row in result: for key, value in row.items(): if isinstance(value, str): row[key] = value.replace("\n", "\\n").replace("\r", "\\r") + keys.add(key) writer.writerow(row) - # print(f"Exported data to {csv_file}") + # add the schema to the CSV + schema_id = "#SCHEMA_" + csv_filename + schema_props = { + "name": "CSVW Table schema for: " + csv_filename, + "columns": [], + } + for key in keys: + base_prop = re.sub(r".*_", "", key) + column_props = { + "name": key, + "label": base_prop, + } + uri = self.crate.resolve_term(base_prop) + + if uri: + column_props["propertyUrl"] = uri + definition = self.crate.get(uri) + if definition: + print("definition", definition["rdfs:comment"]) + column_props["description"] = definition["rdfs:comment"] + # TODO -- look up local definitions and add a description + col_id = "#COLUMN_" + csv_filename + "_" + key + self.schemaCrate.add("csvw:Column", col_id, column_props) + schema_props["columns"].append({"@id": col_id}) + + self.schemaCrate.add( + ["File", "csvw:Table"], + csv_filename, + { + "tableSchema": {"@id": schema_id}, + "name": "Generated export from RO-Crate: " + csv_filename, + }, + ) + self.schemaCrate.add("csvw:Schema", schema_id, schema_props) + + print(f"Exported {csv_filename} to {csv_path}") + + root_entity = self.schemaCrate.root() + root_entity["hasPart"] = files + root_entity["name"] = "CSV exported from RO-Crate" + self.schemaCrate.write_json(rocrate_dir) def find_csv(self): files = self.db.query(""" @@ -317,9 +517,7 @@ def find_csv(self): def add_csv(self, csv_path, table_name): with open(csv_path, newline="", encoding="utf-8") as f: - reader = csv.DictReader( - f - ) # Use DictReader to read each row as a dictionary + reader = csv.DictReader(f) rows = list(reader) if rows: # Insert rows into the table (the table will be created if it doesn't exist) @@ -332,12 +530,12 @@ def add_csv(self, csv_path, table_name): # write an sqlite database to stdout -def cli(): +def parse_args(arg_list=None): ap = ArgumentParser("RO-Crate to tables") ap.add_argument( "crate", type=str, - help="RO-Crate URL or directory", + help="Input RO-Crate URL or directory", ) ap.add_argument( "output", @@ -345,6 +543,9 @@ def cli(): type=Path, help="SQLite database file", ) + ap.add_argument( + "--csv", default="csv", type=Path, help="Output directory for CSV files" + ) ap.add_argument( "-c", "--config", default="config.json", type=Path, help="Configuration file" ) @@ -356,37 +557,64 @@ def cli(): help="Entities of this type will be loaded as text into the database", ) ap.add_argument( - "--csv", + "--concat", action="store_true", help="Find any CSV files and concatenate them into a table", ) - args = ap.parse_args() + ap.add_argument( + "--rebuild", + action="store_true", + help="Force rebuild of the database", + ) + ap.add_argument( + "--structure", + action="store_true", + help="Report on the database structure", + ) + return ap.parse_args(arg_list) + +def main(args): tb = ROCrateTabulator() - print("Building properties table") - tb.crate_to_db(args.crate, args.output) + if Path(args.output).is_file() and not args.rebuild: + print("Loading properties table") + tb.crate_to_db(args.crate, args.output, rebuild=False) + else: + print("Building properties table") + tb.crate_to_db(args.crate, args.output) + + if args.structure: + tb.dump_structure() + sys.exit() if args.config.is_file(): print(f"Loading config from {args.config}") - tb.load_config(args.config) + tb.read_config(args.config) else: print(f"Config {args.config} not found - generating default") tb.infer_config() + tb.text_prop = args.text for table in tb.cf["tables"]: print(f"Building entity table for {table}") - tb.entity_table(table, args.text) # should args.text be config? + allprops = tb.entity_table(table) + tb.cf["tables"][table]["all_props"] = list(allprops) tb.write_config(args.config) print(f""" Updated config file: {args.config}, edit this file to change the flattening configuration or deleted it to start over """) - if args.csv: + if args.concat: tb.find_csv_contents() - tb.export_csv() + tb.export_csv(args.csv) + + +def cli(): + args = parse_args() + main(args) if __name__ == "__main__": diff --git a/src/rocrate_tabular/tinycrate.py b/src/rocrate_tabular/tinycrate.py deleted file mode 100644 index 225b726..0000000 --- a/src/rocrate_tabular/tinycrate.py +++ /dev/null @@ -1,142 +0,0 @@ -# a new, tiny python RO-Crate library with emphasis on manipulating -# the json-ld metadata - -from pathlib import Path -import json -import requests - -LICENSE_ID = ("https://creativecommons.org/licenses/by-nc-sa/3.0/au/",) -LICENSE_DESCRIPTION = """ -This work is licensed under the Creative Commons -Attribution-NonCommercial-ShareAlike 3.0 Australia License. -To view a copy of this license, visit -http://creativecommons.org/licenses/by-nc-sa/3.0/au/ or send a letter to -Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. -""" -LICENSE_IDENTIFIER = "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" -LICENSE_NAME = """ -Attribution-NonCommercial-ShareAlike 3.0 Australia (CC BY-NC-SA 3.0 AU)" -""" - - -class TinyCrateException(Exception): - pass - - -class TinyCrate: - def __init__(self, jsonld=None, directory=None): - if jsonld is not None: - self.context = jsonld["@context"] - self.graph = jsonld["@graph"] - else: - self.context = "https://w3id.org/ro/crate/1.1/context" - self.graph = [] - self.directory = directory - - def add(self, t, i, props): - json_props = dict(props) - json_props["@id"] = i - json_props["@type"] = t - self.graph.append(json_props) - - def all(self): - """dunno about this""" - return [TinyEntity(self, e) for e in self.graph] - - def get(self, i): - es = [e for e in self.graph if e["@id"] == i] - if es: - return TinyEntity(self, es[0]) - else: - return None - - def root(self): - metadata = self.get("ro-crate-metadata.json") - if metadata is None: - raise TinyCrateException("no ro-crate-metadata.json entity") - root = self.get(metadata["about"]["@id"]) - if root is None: - raise TinyCrateException("malformed or missing root entity") - return root - - def json(self): - return json.dumps({"@context": self.context, "@graph": self.graph}, indent=2) - - def write_json(self, crate_dir): - with open(Path(crate_dir) / "ro-crate-metadata.json", "w") as jfh: - json.dump({"@context": self.context, "@graph": self.graph}, jfh, indent=2) - - -class TinyEntity: - def __init__(self, crate, ejsonld): - self.crate = crate - self.type = ejsonld["@type"] - self.id = ejsonld["@id"] - self.props = dict(ejsonld) - - def __getitem__(self, prop): - return self.props.get(prop, None) - - def __setitem__(self, prop, val): - self.props[prop] = val - - def fetch(self): - """Get this entity's content""" - if self.id[:4] == "http": - return self._fetch_http() - else: - return self._fetch_file() - - def _fetch_http(self): - response = requests.get(self.id) - if response.ok: - return response.text - raise TinyCrateException( - f"http request to {self.id} failed with status {response.status_code}" - ) - - def _fetch_file(self): - if self.crate.directory is None: - # maybe use pwd for this? - raise TinyCrateException("Can't load file: no crate directory") - fn = Path(self.crate.directory) / self.id - try: - with open(fn, "r", encoding="utf-8") as fh: - content = fh.read() - return content - except Exception as e: - raise TinyCrateException(f"File read failed: {e}") - - -def minimal_crate(name="Minimal crate", description="Minimal crate"): - """Create ROCrate json with the minimal structure""" - crate = TinyCrate() - license_id = "https://creativecommons.org/licenses/by-nc-sa/3.0/au/" - crate.add( - "Dataset", - "./", - { - "name": name, - "description": description, - "license": {"@id": license_id}, - "datePublished": "2024", - }, - ) - crate.add( - "CreativeWork", - license_id, - { - "name": LICENSE_NAME, - "description": "LICENSE_DESCRIPTION", - "identifier": "LICENSE_IDENTIFIER", - }, - ) - crate.add( - "CreativeWork", - "ro-crate-metadata.json", - { - "about": {"@id": "./"}, - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}, - }, - ) - return crate diff --git a/tests/conftest.py b/tests/conftest.py index fa3ed1a..deeb085 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,4 +8,30 @@ def crates(): "wide": "./tests/crates/wide", "textfiles": "./tests/crates/textfiles", "utf8": "./tests/crates/utf8", + "languageFamily": "./tests/crates/languageFamily", } + + +@pytest.fixture +def csv_headers(): + return [ + "entity_id", + "@type", + "name", + "description", + "datePublished", + "pcdm:memberOf", + "pcdm:memberOf_id", + "license", + "license_id", + "inLanguage", + "inLanguage_id", + "ldac:subjectLanguage", + "ldac:subjectLanguage_id", + "arcp://name,custom/terms#languageSubFamily", + "arcp://name,custom/terms#languageSubFamily_id", + "hasPart", + "hasPart_id", + "hasPart_1", + "hasPart_id_1", + ] diff --git a/tests/crates/languageFamily/Audio/UDHR_Danish.mp3 b/tests/crates/languageFamily/Audio/UDHR_Danish.mp3 new file mode 100644 index 0000000..700450d Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Danish.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Finnish.mp3 b/tests/crates/languageFamily/Audio/UDHR_Finnish.mp3 new file mode 100644 index 0000000..e16a301 Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Finnish.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_German.mp3 b/tests/crates/languageFamily/Audio/UDHR_German.mp3 new file mode 100644 index 0000000..58292cd Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_German.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Icelandic.mp3 b/tests/crates/languageFamily/Audio/UDHR_Icelandic.mp3 new file mode 100644 index 0000000..8e6602b Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Icelandic.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Latin.mp3 b/tests/crates/languageFamily/Audio/UDHR_Latin.mp3 new file mode 100644 index 0000000..4ab323e Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Latin.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Lithuanian.mp3 b/tests/crates/languageFamily/Audio/UDHR_Lithuanian.mp3 new file mode 100644 index 0000000..7cc10ff Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Lithuanian.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Mongolian.mp3 b/tests/crates/languageFamily/Audio/UDHR_Mongolian.mp3 new file mode 100644 index 0000000..eec7b66 Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Mongolian.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_ScottishGaelic.mp3 b/tests/crates/languageFamily/Audio/UDHR_ScottishGaelic.mp3 new file mode 100644 index 0000000..b55a723 Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_ScottishGaelic.mp3 differ diff --git a/tests/crates/languageFamily/Audio/UDHR_Welsh.mp3 b/tests/crates/languageFamily/Audio/UDHR_Welsh.mp3 new file mode 100644 index 0000000..69747a8 Binary files /dev/null and b/tests/crates/languageFamily/Audio/UDHR_Welsh.mp3 differ diff --git a/tests/crates/languageFamily/Images/UDHR_AncientEgyptian.png b/tests/crates/languageFamily/Images/UDHR_AncientEgyptian.png new file mode 100644 index 0000000..5edeffe Binary files /dev/null and b/tests/crates/languageFamily/Images/UDHR_AncientEgyptian.png differ diff --git a/tests/crates/languageFamily/Images/UDHR_OldEnglish.png b/tests/crates/languageFamily/Images/UDHR_OldEnglish.png new file mode 100644 index 0000000..41fd2e9 Binary files /dev/null and b/tests/crates/languageFamily/Images/UDHR_OldEnglish.png differ diff --git a/tests/crates/languageFamily/Licenses/Example.txt b/tests/crates/languageFamily/Licenses/Example.txt new file mode 100644 index 0000000..cfd7359 --- /dev/null +++ b/tests/crates/languageFamily/Licenses/Example.txt @@ -0,0 +1 @@ +This is an example license file. diff --git a/tests/crates/languageFamily/Text/UDHR_Danish.txt b/tests/crates/languageFamily/Text/UDHR_Danish.txt new file mode 100644 index 0000000..4d0ff76 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Danish.txt @@ -0,0 +1 @@ +Alle mennesker er født frie og lige i værdighed og rettigheder. De er udstyret med fornuft og samvittighed, og de bør handle mod hverandre i en broderskabets ånd. diff --git a/tests/crates/languageFamily/Text/UDHR_English.txt b/tests/crates/languageFamily/Text/UDHR_English.txt new file mode 100644 index 0000000..2c6a340 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_English.txt @@ -0,0 +1 @@ +All human beings are born free and equal in dignity and rights. They are endowed with reason and conscience and should act towards one another in a spirit of brotherhood. diff --git a/tests/crates/languageFamily/Text/UDHR_Finnish.txt b/tests/crates/languageFamily/Text/UDHR_Finnish.txt new file mode 100644 index 0000000..f14b531 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Finnish.txt @@ -0,0 +1 @@ +Kaikki ihmiset syntyvät vapaina ja tasavertaisina arvoltaan ja oikeuksiltaan. Heille on annettu järki ja omatunto, ja heidän on toimittava toisiaan kohtaan veljeyden hengessä. diff --git a/tests/crates/languageFamily/Text/UDHR_German.txt b/tests/crates/languageFamily/Text/UDHR_German.txt new file mode 100644 index 0000000..f5949f3 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_German.txt @@ -0,0 +1 @@ +Alle Menschen sind frei und gleich an Würde und Rechten geboren. Sie sind mit Vernunft und Gewissen begabt und sollen einander im Geist der Brüderlichkeit begegnen. diff --git a/tests/crates/languageFamily/Text/UDHR_Icelandic.txt b/tests/crates/languageFamily/Text/UDHR_Icelandic.txt new file mode 100644 index 0000000..cd62118 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Icelandic.txt @@ -0,0 +1 @@ +Hver maður er borinn frjáls og jafn öðrum að virðingu og réttindum. Menn eru gæddir vitsmunum og samvisku, og ber þeim að breyta bróðurlega hverjum við annan. diff --git a/tests/crates/languageFamily/Text/UDHR_Latin.txt b/tests/crates/languageFamily/Text/UDHR_Latin.txt new file mode 100644 index 0000000..29b4ae0 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Latin.txt @@ -0,0 +1 @@ +Omnes homines dignitate et iure liberi et pares nascuntur, rationis et conscientiae participes sunt, quibus inter se concordiae studio est agendum. diff --git a/tests/crates/languageFamily/Text/UDHR_Lithuanian.txt b/tests/crates/languageFamily/Text/UDHR_Lithuanian.txt new file mode 100644 index 0000000..fc3537e --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Lithuanian.txt @@ -0,0 +1 @@ +Visi žmonės gimsta laisvi ir lygūs savo orumu ir teisėmis. Jiems suteiktas protas ir sąžinė ir jie turi elgtis vienas kito atžvilgiu kaip broliai. diff --git a/tests/crates/languageFamily/Text/UDHR_Mongolian.txt b/tests/crates/languageFamily/Text/UDHR_Mongolian.txt new file mode 100644 index 0000000..b704e1c --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Mongolian.txt @@ -0,0 +1 @@ +Хүн бүр төрж мэндлэхдээ эрх чөлөөтэй, адилхан нэр төртэй, ижил эрхтэй байдаг. Оюун ухаан нандин чанар заяасан хүн гэгч өөр хоорондоо ахан дүүгийн үзэл санаагаар харьцах учиртай. diff --git a/tests/crates/languageFamily/Text/UDHR_OldEnglish.txt b/tests/crates/languageFamily/Text/UDHR_OldEnglish.txt new file mode 100644 index 0000000..6dedc8b --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_OldEnglish.txt @@ -0,0 +1 @@ +Eall folc weorþaþ frēo and efne bē āre and rihtum ġeboren. Ġerād and inġehyġd sind heom ġifeþu, and hīe þurfon tō ōþrum ōn fēore brōþorsċipes dōn. diff --git a/tests/crates/languageFamily/Text/UDHR_ScottishGaelic.txt b/tests/crates/languageFamily/Text/UDHR_ScottishGaelic.txt new file mode 100644 index 0000000..65d520d --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_ScottishGaelic.txt @@ -0,0 +1 @@ +Rugadh na h-uile duine saor agus co-ionnan nan urram 's nan còirichean. Tha iad reusanta is cogaiseach, agus bu chòir dhaibh a ghiùlain ris a chèile ann an spiorad bràthaireil. diff --git a/tests/crates/languageFamily/Text/UDHR_Welsh.txt b/tests/crates/languageFamily/Text/UDHR_Welsh.txt new file mode 100644 index 0000000..07f8429 --- /dev/null +++ b/tests/crates/languageFamily/Text/UDHR_Welsh.txt @@ -0,0 +1 @@ +Genir pawb yn rhydd ac yn gydradd â'i gilydd mewn urddas a hawliau. Fe'u cynysgaeddir â rheswm a chydwybod, a dylai pawb ymddwyn y naill at y llall mewn ysbryd cymodlon. diff --git a/tests/crates/languageFamily/ro-crate-metadata-w-subcollections_20250610.xlsx b/tests/crates/languageFamily/ro-crate-metadata-w-subcollections_20250610.xlsx new file mode 100644 index 0000000..8e67b91 Binary files /dev/null and b/tests/crates/languageFamily/ro-crate-metadata-w-subcollections_20250610.xlsx differ diff --git a/tests/crates/languageFamily/ro-crate-metadata.json b/tests/crates/languageFamily/ro-crate-metadata.json new file mode 100644 index 0000000..979e3d4 --- /dev/null +++ b/tests/crates/languageFamily/ro-crate-metadata.json @@ -0,0 +1,1174 @@ +{ + "@context": [ + "https://w3id.org/ro/crate/1.1/context", + { + "@vocab": "http://schema.org/", + "ldac": "https://w3id.org/ldac/terms#", + "custom": "arcp://name,custom/terms#" + } + ], + "@graph": [ + { + "@id": "#Omniglot", + "@type": "Organization", + "name": "Omniglot" + }, + { + "@id": "https://ror.org/00rqy9422", + "@type": "Organization", + "name": "University of Queensland" + }, + { + "@id": "https://creativecommons.org/licenses/by/4.0/", + "@type": "ldac:DataReuseLicense", + "name": "Attribution 4.0 International (CC BY 4.0)", + "description": "You are free to: Share — copy and redistribute the material in any medium or format. Adapt — remix, transform, and build upon the material for any purpose, even commercially. This license is acceptable for Free Cultural Works. The licensor cannot revoke these freedoms as long as you follow the license terms.", + "ldac:allowTextIndex": "true" + }, + { + "@id": "collection.txt", + "@type": [ + "File", + "ldac:CollectionProtocol" + ], + "name": "Collection Notes", + "datePublished": "2025-05-28", + "author": { + "@id": "#LDaCA" + } + }, + { + "@id": "#LDaCA", + "@type": "Organization", + "name": "Language Data Commons of Australia" + }, + { + "@id": "arcp://name,custom/terms#LanguageStatusTerms", + "@type": "DefinedTermSet", + "name": "Language Status Terms", + "description": "The set of expected values for language status." + }, + { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms", + "@type": "DefinedTermSet", + "name": "Language Sub-Family Terms", + "description": "The set of expected values for language sub-family." + }, + { + "@id": "UDHR_w_subcollections", + "@type": [ + "Dataset", + "RepositoryCollection" + ], + "name": "Test Dataset: UDHR Translations with SubCollections", + "description": "Translations of Article 1 of the Universal Declaration of Human Rights from Omniglot.", + "ldac:doi": "https://doi.org/10.1000/182", + "datePublished": "2025-05-28", + "ldac:metadataIsPublic": "true", + "author": { + "@id": "#Omniglot" + }, + "publisher": { + "@id": "https://ror.org/00rqy9422" + }, + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "dct:rightsHolder": { + "@id": "#Omniglot" + }, + "accountablePerson": { + "@id": "#Omniglot" + }, + "ldac:hasCollectionProtocol": { + "@id": "collection.txt" + }, + "pcdm:hasMember": [ + { + "@id": "#Afro-Asiatic" + }, + { + "@id": "#Indo-European" + }, + { + "@id": "#Uralic" + }, + { + "@id": "#Mongolic" + } + ], + "hasPart": [ + { + "@id": "Licenses/Example.txt" + }, + { + "@id": "Images/UDHR_OldEnglish.png" + }, + { + "@id": "Images/UDHR_AncientEgyptian.png" + }, + { + "@id": "Audio/UDHR_Latin.mp3" + }, + { + "@id": "Audio/UDHR_Danish.mp3" + }, + { + "@id": "Audio/UDHR_ScottishGaelic.mp3" + }, + { + "@id": "Audio/UDHR_Mongolian.mp3" + }, + { + "@id": "Audio/UDHR_Welsh.mp3" + }, + { + "@id": "Audio/UDHR_Finnish.mp3" + }, + { + "@id": "Audio/UDHR_German.mp3" + }, + { + "@id": "Audio/UDHR_Icelandic.mp3" + }, + { + "@id": "Audio/UDHR_Lithuanian.mp3" + }, + { + "@id": "Text/UDHR_ScottishGaelic.txt" + }, + { + "@id": "Text/UDHR_OldEnglish.txt" + }, + { + "@id": "Text/UDHR_Icelandic.txt" + }, + { + "@id": "Text/UDHR_Lithuanian.txt" + }, + { + "@id": "Text/UDHR_English.txt" + }, + { + "@id": "Text/UDHR_Latin.txt" + }, + { + "@id": "Text/UDHR_Finnish.txt" + }, + { + "@id": "Text/UDHR_German.txt" + }, + { + "@id": "Text/UDHR_Welsh.txt" + }, + { + "@id": "Text/UDHR_Mongolian.txt" + }, + { + "@id": "Text/UDHR_Danish.txt" + }, + { + "@id": "ro-crate-metadata-w-subcollections_20250610.xlsx" + }, + { + "@id": "ro-crate-preview.html" + } + ], + "conformsTo": { + "@id": "https://w3id.org/ldac/profile#Collection" + } + }, + { + "@id": "#Afro-Asiatic", + "@type": "RepositoryCollection", + "name": "Afro-Asiatic", + "description": "Afro-Asiatic languages.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "UDHR_w_subcollections" + }, + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "accountablePerson": { + "@id": "#Omniglot" + }, + "pcdm:hasMember": { + "@id": "#UDHR_AncientEgyptian" + } + }, + { + "@id": "#Indo-European", + "@type": "RepositoryCollection", + "name": "Indo-European", + "description": "Indo-European languages.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "UDHR_w_subcollections" + }, + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "accountablePerson": { + "@id": "#Omniglot" + }, + "pcdm:hasMember": [ + { + "@id": "#UDHR_Danish" + }, + { + "@id": "#UDHR_English" + }, + { + "@id": "#UDHR_German" + }, + { + "@id": "#UDHR_Icelandic" + }, + { + "@id": "#UDHR_Latin" + }, + { + "@id": "#UDHR_Lithuanian" + }, + { + "@id": "#UDHR_OldEnglish" + }, + { + "@id": "#UDHR_ScottishGaelic" + }, + { + "@id": "#UDHR_Welsh" + } + ] + }, + { + "@id": "#Uralic", + "@type": "RepositoryCollection", + "name": "Uralic", + "description": "Uralic languages.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "UDHR_w_subcollections" + }, + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "accountablePerson": { + "@id": "#Omniglot" + }, + "pcdm:hasMember": { + "@id": "#UDHR_Finnish" + } + }, + { + "@id": "#Mongolic", + "@type": "RepositoryCollection", + "name": "Mongolic", + "description": "Mongolic languages.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "UDHR_w_subcollections" + }, + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "accountablePerson": { + "@id": "#Omniglot" + }, + "pcdm:hasMember": { + "@id": "#UDHR_Mongolian" + } + }, + { + "@id": "Licenses/Example.txt", + "@type": [ + "ldac:DataReuseLicense", + "File" + ], + "name": "Example Custom License", + "description": "This license explains who is allowed to use and possibly redistribute this data, and for what purpose.", + "ldac:allowTextIndex": "true", + "isPartOf": { + "@id": "UDHR_w_subcollections" + } + }, + { + "@id": "#UDHR_AncientEgyptian", + "@type": "RepositoryObject", + "name": "UDHR_AncientEgyptian", + "description": "Article 1 UDHR Ancient Egyptian.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Afro-Asiatic" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#AncientEgyptian" + }, + "ldac:subjectLanguage": { + "@id": "#AncientEgyptian" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Egyptian" + }, + "hasPart": { + "@id": "Images/UDHR_AncientEgyptian.png" + } + }, + { + "@id": "#UDHR_Danish", + "@type": "RepositoryObject", + "name": "UDHR_Danish", + "description": "Article 1 UDHR Danish.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Danish" + }, + "ldac:subjectLanguage": { + "@id": "#Danish" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Germanic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Danish.mp3" + }, + { + "@id": "Text/UDHR_Danish.txt" + } + ] + }, + { + "@id": "#UDHR_English", + "@type": "RepositoryObject", + "name": "UDHR_English", + "description": "Article 1 UDHR English.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#English" + }, + "ldac:subjectLanguage": { + "@id": "#English" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Germanic" + }, + "hasPart": { + "@id": "Text/UDHR_English.txt" + } + }, + { + "@id": "#UDHR_German", + "@type": "RepositoryObject", + "name": "UDHR_German", + "description": "Article 1 UDHR German.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#German" + }, + "ldac:subjectLanguage": { + "@id": "#German" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Germanic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_German.mp3" + }, + { + "@id": "Text/UDHR_German.txt" + } + ] + }, + { + "@id": "#UDHR_Icelandic", + "@type": "RepositoryObject", + "name": "UDHR_Icelandic", + "description": "Article 1 UDHR Icelandic.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Icelandic" + }, + "ldac:subjectLanguage": { + "@id": "#Icelandic" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Germanic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Icelandic.mp3" + }, + { + "@id": "Text/UDHR_Icelandic.txt" + } + ] + }, + { + "@id": "#UDHR_Latin", + "@type": "RepositoryObject", + "name": "UDHR_Latin", + "description": "Article 1 UDHR Latin.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Latin" + }, + "ldac:subjectLanguage": { + "@id": "#Latin" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Italic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Latin.mp3" + }, + { + "@id": "Text/UDHR_Latin.txt" + } + ] + }, + { + "@id": "#UDHR_Lithuanian", + "@type": "RepositoryObject", + "name": "UDHR_Lithuanian", + "description": "Article 1 UDHR Lithuanian.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Lithuanian" + }, + "ldac:subjectLanguage": { + "@id": "#Lithuanian" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#BaltoSlavic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Lithuanian.mp3" + }, + { + "@id": "Text/UDHR_Lithuanian.txt" + } + ] + }, + { + "@id": "#UDHR_OldEnglish", + "@type": "RepositoryObject", + "name": "UDHR_OldEnglish", + "description": "Article 1 UDHR Old English.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#OldEnglish" + }, + "ldac:subjectLanguage": { + "@id": "#OldEnglish" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Germanic" + }, + "hasPart": [ + { + "@id": "Images/UDHR_OldEnglish.png" + }, + { + "@id": "Text/UDHR_OldEnglish.txt" + } + ] + }, + { + "@id": "#UDHR_ScottishGaelic", + "@type": "RepositoryObject", + "name": "UDHR_ScottishGaelic", + "description": "Article 1 UDHR Scottish Gaelic.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#ScottishGaelic" + }, + "ldac:subjectLanguage": { + "@id": "#ScottishGaelic" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Celtic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_ScottishGaelic.mp3" + }, + { + "@id": "Text/UDHR_ScottishGaelic.txt" + } + ] + }, + { + "@id": "#UDHR_Welsh", + "@type": "RepositoryObject", + "name": "UDHR_Welsh", + "description": "Article 1 UDHR Welsh.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Indo-European" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Welsh" + }, + "ldac:subjectLanguage": { + "@id": "#Welsh" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Celtic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Welsh.mp3" + }, + { + "@id": "Text/UDHR_Welsh.txt" + } + ] + }, + { + "@id": "#UDHR_Finnish", + "@type": "RepositoryObject", + "name": "UDHR_Finnish", + "description": "Article 1 UDHR Finnish.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Uralic" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Finnish" + }, + "ldac:subjectLanguage": { + "@id": "#Finnish" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#Finnic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Finnish.mp3" + }, + { + "@id": "Text/UDHR_Finnish.txt" + } + ] + }, + { + "@id": "#UDHR_Mongolian", + "@type": "RepositoryObject", + "name": "UDHR_Mongolian", + "description": "Article 1 UDHR Mongolian.", + "datePublished": "2025-05-28", + "pcdm:memberOf": { + "@id": "#Mongolic" + }, + "license": { + "@id": "Licenses/Example.txt" + }, + "inLanguage": { + "@id": "#Mongolian" + }, + "ldac:subjectLanguage": { + "@id": "#Mongolian" + }, + "arcp://name,custom/terms#languageSubFamily": { + "@id": "arcp://name,custom/terms#CentralMongolic" + }, + "hasPart": [ + { + "@id": "Audio/UDHR_Mongolian.mp3" + }, + { + "@id": "Text/UDHR_Mongolian.txt" + } + ] + }, + { + "@id": "#AncientEgyptian", + "@type": "Language", + "name": "Ancient Egyptian" + }, + { + "@id": "arcp://name,custom/terms#Egyptian", + "@type": "DefinedTerm", + "name": "Egyptian", + "description": "Egyptian languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Images/UDHR_AncientEgyptian.png", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_AncientEgyptian" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Extinct" + } + }, + { + "@id": "#Danish", + "@type": "Language", + "name": "Danish" + }, + { + "@id": "arcp://name,custom/terms#Germanic", + "@type": "DefinedTerm", + "name": "Germanic", + "description": "Germanic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_Danish.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Danish" + }, + "ldac:speaker": { + "@id": "#Danish_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Danish.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Danish" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#English", + "@type": "Language", + "name": "English" + }, + { + "@id": "Text/UDHR_English.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_English" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#German", + "@type": "Language", + "name": "German" + }, + { + "@id": "Audio/UDHR_German.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_German" + }, + "ldac:speaker": { + "@id": "#German_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_German.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_German" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#Icelandic", + "@type": "Language", + "name": "Icelandic" + }, + { + "@id": "Audio/UDHR_Icelandic.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Icelandic" + }, + "ldac:speaker": { + "@id": "#Icelandic_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Icelandic.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Icelandic" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#Latin", + "@type": "Language", + "name": "Latin" + }, + { + "@id": "arcp://name,custom/terms#Italic", + "@type": "DefinedTerm", + "name": "Italic", + "description": "Italic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_Latin.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Latin" + }, + "ldac:speaker": { + "@id": "#Latin_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Extinct" + } + }, + { + "@id": "Text/UDHR_Latin.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Latin" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Extinct" + } + }, + { + "@id": "#Lithuanian", + "@type": "Language", + "name": "Lithuanian" + }, + { + "@id": "arcp://name,custom/terms#BaltoSlavic", + "@type": "DefinedTerm", + "name": "Balto-Slavic", + "description": "Balto-Slavic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_Lithuanian.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Lithuanian" + }, + "ldac:speaker": { + "@id": "#Lithuanian_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Lithuanian.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Lithuanian" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#OldEnglish", + "@type": "Language", + "name": "Old English" + }, + { + "@id": "Images/UDHR_OldEnglish.png", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_OldEnglish" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Extinct" + } + }, + { + "@id": "Text/UDHR_OldEnglish.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_OldEnglish" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Extinct" + } + }, + { + "@id": "#ScottishGaelic", + "@type": "Language", + "name": "Scottish Gaelic" + }, + { + "@id": "arcp://name,custom/terms#Celtic", + "@type": "DefinedTerm", + "name": "Celtic", + "description": "Celtic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_ScottishGaelic.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_ScottishGaelic" + }, + "ldac:speaker": { + "@id": "#ScottishGaelic_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_ScottishGaelic.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_ScottishGaelic" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#Welsh", + "@type": "Language", + "name": "Welsh" + }, + { + "@id": "Audio/UDHR_Welsh.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Welsh" + }, + "ldac:speaker": { + "@id": "#Welsh_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Welsh.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Welsh" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#Finnish", + "@type": "Language", + "name": "Finnish" + }, + { + "@id": "arcp://name,custom/terms#Finnic", + "@type": "DefinedTerm", + "name": "Finnic", + "description": "Finnic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_Finnish.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Finnish" + }, + "ldac:speaker": { + "@id": "#Finnish_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Finnish.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Finnish" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "#Mongolian", + "@type": "Language", + "name": "Mongolian" + }, + { + "@id": "arcp://name,custom/terms#CentralMongolic", + "@type": "DefinedTerm", + "name": "Central Mongolic", + "description": "Central Mongolic languages.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageSubFamilyTerms" + } + }, + { + "@id": "Audio/UDHR_Mongolian.mp3", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Mongolian" + }, + "ldac:speaker": { + "@id": "#Mongolian_Speaker" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "Text/UDHR_Mongolian.txt", + "@type": [ + "File", + "ldac:PrimaryMaterial" + ], + "isPartOf": { + "@id": "#UDHR_Mongolian" + }, + "arcp://name,custom/terms#languageStatus": { + "@id": "arcp://name,custom/terms#Living" + } + }, + { + "@id": "arcp://name,custom/terms#Extinct", + "@type": "DefinedTerm", + "name": "Extinct", + "description": "The language is no longer spoken as a first language.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageStatusTerms" + } + }, + { + "@id": "#Danish_Speaker", + "@type": "Person", + "name": "Unknown" + }, + { + "@id": "arcp://name,custom/terms#Living", + "@type": "DefinedTerm", + "name": "Living", + "description": "The language is spoken as a first language.", + "inDefinedTermSet": { + "@id": "arcp://name,custom/terms#LanguageStatusTerms" + } + }, + { + "@id": "#Finnish_Speaker", + "@type": "Person", + "name": "Unknown" + }, + { + "@id": "#German_Speaker", + "@type": "Person", + "name": "Unknown" + }, + { + "@id": "#Icelandic_Speaker", + "@type": "Person", + "name": "Alexander Jarl" + }, + { + "@id": "#Latin_Speaker", + "@type": "Person", + "name": "Connor Ferguson" + }, + { + "@id": "#Lithuanian_Speaker", + "@type": "Person", + "name": "Augustinas Žemaitis" + }, + { + "@id": "#Mongolian_Speaker", + "@type": "Person", + "name": "Bat-Orgil Myagmardorj" + }, + { + "@id": "#ScottishGaelic_Speaker", + "@type": "Person", + "name": "Frederic (Calum) Bayer" + }, + { + "@id": "#Welsh_Speaker", + "@type": "Person", + "name": "Stephen Turner" + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "identifier": "ro-crate-metadata.json", + "about": { + "@id": "UDHR_w_subcollections" + } + }, + { + "@id": "arcp://name,custom/terms#languageStatus", + "@type": "rdf:Property", + "name": "Language Status", + "description": "The status of the language, whether living or extinct." + }, + { + "@id": "arcp://name,custom/terms#languageSubFamily", + "@type": "rdf:Property", + "name": "Language Sub-Family", + "description": "A group of languages related through descent from a common ancestor." + }, + { + "@id": "ro-crate-metadata-w-subcollections_20250610.xlsx", + "@type": "File" + }, + { + "@id": "ro-crate-preview.html", + "@type": "File" + } + ] +} \ No newline at end of file diff --git a/tests/crates/languageFamily/ro-crate-preview.html b/tests/crates/languageFamily/ro-crate-preview.html new file mode 100644 index 0000000..f23d524 --- /dev/null +++ b/tests/crates/languageFamily/ro-crate-preview.html @@ -0,0 +1,2700 @@ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

+

Test Dataset: UDHR Translations with SubCollections

+ + +⬇️🏷️ Download all the metadata for Test Dataset: UDHR Translations with SubCollections in JSON-LD format + + +
+ + +
+ + + +
+
+
+

Test Dataset: UDHR Translations with SubCollections

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Omniglot

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Go to: University of Queensland

+ + + + +
+ + + + + + + + + + + + + + + + +
+
+


+
+

Go to: Attribution 4.0 International (CC BY 4.0)

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Collection Notes

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Afro-Asiatic

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Indo-European

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Uralic

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

Mongolic

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Example Custom License

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Images/UDHR_OldEnglish.png

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Images/UDHR_AncientEgyptian.png

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Latin.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Danish.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_ScottishGaelic.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Mongolian.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Welsh.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Finnish.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_German.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Icelandic.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Audio/UDHR_Lithuanian.mp3

+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_ScottishGaelic.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_OldEnglish.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Icelandic.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Lithuanian.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_English.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Latin.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Finnish.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_German.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Welsh.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Mongolian.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: Text/UDHR_Danish.txt

+ + + + +
+ + + + + + + + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: ro-crate-metadata-w-subcollections_20250610.xlsx

+ + + + +
+ + + + + + + + + + + + + +
+
+


+
+

⬇️ Download: ro-crate-preview.html

+ + + + +
+ + + + + + + + + + + + + +
+
+





+
+ + + + + + + diff --git a/tests/crates/languageFamily/~$ro-crate-metadata-w-subcollections_20250610.xlsx b/tests/crates/languageFamily/~$ro-crate-metadata-w-subcollections_20250610.xlsx new file mode 100644 index 0000000..f6a81f5 Binary files /dev/null and b/tests/crates/languageFamily/~$ro-crate-metadata-w-subcollections_20250610.xlsx differ diff --git a/tests/test_basic.py b/tests/test_basic.py index cb06135..802e661 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,20 +1,27 @@ from pathlib import Path -from rocrate_tabular.tabulator import ROCrateTabulator +from rocrate_tabular.tabulator import ROCrateTabulator, parse_args, main +from tinycrate.tinycrate import TinyCrate +from collections import defaultdict +from util import read_config, write_config -def test_minimal(crates, tmp_path): - dbfile = Path(tmp_path) / "sqlite.db" - tb = ROCrateTabulator() - tb.crate_to_db(crates["minimal"], dbfile) +def test_smoke_cli(crates, tmp_path): + cwd = Path(tmp_path) + dbfile = cwd / "sqlite.db" + conffile = cwd / "config.json" + args = parse_args(["-c", str(conffile), crates["minimal"], str(dbfile)]) + main(args) -def test_wide(crates, tmp_path): +def test_minimal(crates, tmp_path): + """Basically tests whether imports work""" dbfile = Path(tmp_path) / "sqlite.db" tb = ROCrateTabulator() - tb.crate_to_db(crates["wide"], dbfile) + tb.crate_to_db(crates["minimal"], dbfile) def test_config(crates, tmp_path): + """Test that the first-pass config can be read""" cwd = Path(tmp_path) dbfile = cwd / "sqlite.db" conffile = cwd / "config.json" @@ -26,4 +33,69 @@ def test_config(crates, tmp_path): # smoke test to make sure another tabulator can read the config tb2 = ROCrateTabulator() tb2.crate_to_db(crates["minimal"], dbfile) - tb2.load_config(conffile) + tb2.read_config(conffile) + + +def test_one_to_lots(crates, tmp_path): + cwd = Path(tmp_path) + dbfile = cwd / "sqlite.db" + conffile = cwd / "config.json" + tb = ROCrateTabulator() + tb.crate_to_db(crates["wide"], dbfile) + tb.infer_config() + tb.write_config(conffile) + tb.close() + + # load the config and move the potential tables to tables + cf = read_config(conffile) + cf["tables"] = cf["potential_tables"] + cf["potential_tables"] = [] + write_config(cf, conffile) + + # this will raise an error for too many columns + tb = ROCrateTabulator() + tb.read_config(conffile) + + tb.crate_to_db(crates["wide"], dbfile) + + for table in cf["tables"]: + tb.entity_table(table) + + +def test_all_props(crates, tmp_path): + cwd = Path(tmp_path) + dbfile = cwd / "sqlite.db" + conffile = cwd / "config.json" + tb = ROCrateTabulator() + tb.crate_to_db(crates["languageFamily"], dbfile) + tb.infer_config() + tb.write_config(conffile) + tb.close() + + # load the config and move the potential tables to tables + cf = read_config(conffile) + cf["tables"] = cf["potential_tables"] + cf["potential_tables"] = [] + write_config(cf, conffile) + + tb = ROCrateTabulator() + tb.read_config(conffile) + + tb.crate_to_db(crates["languageFamily"], dbfile) + + # build our own list of all properties (excluding @ids) + + tc = TinyCrate(crates["languageFamily"]) + props = defaultdict(set) + for e in tc.all(): + for prop, val in e.items(): + if prop != "@id": + if type(e.type) is list: + for t in e.type: + props[t].add(prop) + else: + props[e.type].add(prop) + + for table in cf["tables"]: + all_props = tb.entity_table(table) + assert all_props == props[table] diff --git a/tests/test_crates.py b/tests/test_crates.py deleted file mode 100644 index 27a7c53..0000000 --- a/tests/test_crates.py +++ /dev/null @@ -1,84 +0,0 @@ -# from pathlib import Path - -import json -from pathlib import Path -from fuzz import random_text, random_property -from pytest_httpserver import HTTPServer - -from rocrate_tabular.tinycrate import TinyCrate, minimal_crate - - -# https://pytest-httpserver.readthedocs.io/en/latest/ for testing fetch - - -def test_crate(tmp_path): - """Build and save and reload a crate""" - jcrate = minimal_crate() - for i in range(1000): - props = {"name": random_text(4), "description": random_text(11)} - for j in range(30): - prop, val = random_property() - props[prop] = val - jcrate.add("Dataset", f"#ds{i:05d}", props) - jsonf = Path(tmp_path) / "ro-crate-metadata.json" - jcrate.write_json(Path(tmp_path)) - with open(jsonf, "r") as jfh: - jsonld = json.load(jfh) - jcrate2 = TinyCrate(jsonld=jsonld) - for i in range(1000): - eid = f"#ds{i:05d}" - ea = jcrate.get(eid) - eb = jcrate2.get(eid) - assert ea.props == eb.props - - -def test_load_file(crates): - cratedir = crates["textfiles"] - with open(Path(cratedir) / "ro-crate-metadata.json", "r") as jfh: - jsonld = json.load(jfh) - crate = TinyCrate(jsonld=jsonld, directory=cratedir) - tfile = crate.get("doc001/textfile.txt") - contents = tfile.fetch() - with open(Path(cratedir) / "doc001" / "textfile.txt", "r") as tfh: - contents2 = tfh.read() - assert contents == contents2 - - -def test_load_utf8(crates): - """Reads a textfile known to have utf-8 characters which cause - encoding bugs on a Jupyter notebook on Windows""" - cratedir = crates["utf8"] - with open(Path(cratedir) / "ro-crate-metadata.json", "r") as jfh: - jsonld = json.load(jfh) - crate = TinyCrate(jsonld=jsonld, directory=cratedir) - tfile = crate.get("data/2-035-plain.txt") - contents = tfile.fetch() - # note: have to also explicitly set the encoding on this read so that - # it doesn't also break the tests - with open( - Path(cratedir) / "data" / "2-035-plain.txt", "r", encoding="utf-8" - ) as tfh: - contents2 = tfh.read() - assert contents == contents2 - - -def test_load_url(crates, httpserver: HTTPServer): - # test http endpoint with some content - contents = """ -Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. -""" - httpserver.expect_request("/textfileonurl.txt").respond_with_data( - contents, content_type="text/plain" - ) - - cratedir = crates["textfiles"] - with open(Path(cratedir) / "ro-crate-metadata.json", "r") as jfh: - jsonld = json.load(jfh) - crate = TinyCrate(jsonld=jsonld, directory=cratedir) - # add an entity to the crate with the endpoint URL as the id - urlid = httpserver.url_for("/textfileonurl.txt") - crate.add("File", urlid, {"name": "textfileonurl.txt"}) - # get the entity and try to fetch - efile = crate.get(urlid) - contents2 = efile.fetch() - assert contents == contents2 diff --git a/tests/test_expanded_properties.py b/tests/test_expanded_properties.py new file mode 100644 index 0000000..580a4a1 --- /dev/null +++ b/tests/test_expanded_properties.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.mark.skip("Not implemented") +def test_expanded_properties(tmp_path): + assert True diff --git a/tests/test_export_csv.py b/tests/test_export_csv.py new file mode 100644 index 0000000..fd42908 --- /dev/null +++ b/tests/test_export_csv.py @@ -0,0 +1,75 @@ +from pathlib import Path +from rocrate_tabular.tabulator import ROCrateTabulator +from tinycrate.tinycrate import TinyCrate +import json +import sys +import csv + + +def read_config(cffile): + with open(cffile, "r") as cfh: + return json.load(cfh) + + +def write_config(cf, cffile): + with open(cffile, "w") as cfh: + json.dump(cf, cfh) + + +def test_export(crates, csv_headers, tmp_path): + cwd = Path(tmp_path) + dbfile = Path(tmp_path) / "lf.db" + conffile = cwd / "config.json" + tb = ROCrateTabulator() + tb.crate_to_db(crates["languageFamily"], dbfile) + tb.infer_config() + tb.write_config(conffile) + tb.close() + + # load the config and move the potential tables to tables + cf = read_config(conffile) + cf["tables"]["RepositoryObject"] = cf["potential_tables"]["RepositoryObject"] + write_config(cf, conffile) + + tb = ROCrateTabulator() + tb.read_config(conffile) + tb.crate_to_db(crates["languageFamily"], dbfile) + + tb.entity_table("RepositoryObject") + + tb.close() + + # add the export_query to build csv + cf = read_config(conffile) + cf["export_queries"] = {"lf.csv": "SELECT * FROM RepositoryObject"} + write_config(cf, conffile) + + tb2 = ROCrateTabulator() + tb2.read_config(conffile) + tb2.crate_to_db(crates["languageFamily"], dbfile) + tb2.entity_table("RepositoryObject") # shouldn't need to rebuild it + print(f"Tables: {tb2.db.tables}", file=sys.stderr) + csvout = cwd / "csv" + tb2.export_csv(csvout) + assert csvout.is_dir() + csvfile = csvout / "lf.csv" + assert csvfile.is_file() + assert (csvout / "ro-crate-metadata.json").is_file() + csv_crate = TinyCrate(csvout) + assert csv_crate + + csv_data = {} + with open(csvfile, "r") as cfh: + head = True + for row in csv.reader(cfh): + if head: + assert row == csv_headers + head = False + else: + csv_data[row[0]] = row + + orig_crate = TinyCrate(crates["languageFamily"]) + objects = [e for e in orig_crate.all() if e.type == "RepositoryObject"] + + for ro in objects: + assert ro["@id"] in csv_data diff --git a/tests/test_fuzz.py b/tests/test_fuzz.py index 0758e6a..480b33b 100644 --- a/tests/test_fuzz.py +++ b/tests/test_fuzz.py @@ -1,7 +1,6 @@ from pathlib import Path - from rocrate_tabular.tabulator import ROCrateTabulator -from rocrate_tabular.tinycrate import minimal_crate +from tinycrate.tinycrate import minimal_crate from fuzz import random_text, random_property @@ -22,7 +21,7 @@ def test_random(tmp_path): # loop through the crate's graph and try to find every entity and check # the properties are all there for entity in jcrate.graph: - db_props = list(tb.fetch_entity(entity["@id"])) + db_props = list(tb.fetch_properties(entity["@id"])) assert db_props # build a dict from the props // this should get promoted to the lib db_entity = {"@id": entity["@id"]} @@ -32,7 +31,3 @@ def test_random(tmp_path): else: db_entity[db_prop["property_label"]] = db_prop["value"] assert db_entity == entity - - -def test_expanded_properties(tmp_path): - assert True diff --git a/tests/test_relational.py b/tests/test_relational.py new file mode 100644 index 0000000..0a6a693 --- /dev/null +++ b/tests/test_relational.py @@ -0,0 +1,47 @@ +from pathlib import Path +from rocrate_tabular.tabulator import ROCrateTabulator +from tinycrate.tinycrate import TinyCrate +from util import read_config, write_config + + +def test_wide(crates, tmp_path): + cwd = Path(tmp_path) + dbfile = Path(tmp_path) / "wide.db" + conffile = cwd / "wide.json" + tb = ROCrateTabulator() + tb.crate_to_db(crates["wide"], dbfile) + tb.infer_config() + tb.write_config(conffile) + tb.close() + + # load the config and move the potential tables to tables + cf = read_config(conffile) + cf["tables"]["Dataset"] = cf["potential_tables"]["Dataset"] + cf["tables"]["File"] = cf["potential_tables"]["File"] + write_config(cf, conffile) + + tb = ROCrateTabulator() + tb.read_config(conffile) + tb.crate_to_db(crates["wide"], dbfile) + + tb.entity_table("Dataset") + tb.entity_table("File") + + rows = tb.db.query(""" + SELECT d.entity_id as dataset_id, + d.name as dataset, + f.entity_id as file_id, + f.name as file + FROM Dataset as d + JOIN Dataset_hasPart as dh on d.entity_id = dh.entity_id + JOIN File as f on dh.target_id = f.entity_id + """) + + files = {} + + for row in rows: + files[row["file_id"]] = row["file"] + + orig_crate = TinyCrate(crates["wide"]) + dataset = orig_crate.get("./") + assert dataset diff --git a/tests/util.py b/tests/util.py new file mode 100644 index 0000000..53176a9 --- /dev/null +++ b/tests/util.py @@ -0,0 +1,11 @@ +import json + + +def read_config(cffile): + with open(cffile, "r") as cfh: + return json.load(cfh) + + +def write_config(cf, cffile): + with open(cffile, "w") as cfh: + json.dump(cf, cfh) diff --git a/uv.lock b/uv.lock index c5786fd..b351885 100644 --- a/uv.lock +++ b/uv.lock @@ -342,11 +342,13 @@ wheels = [ [[package]] name = "rocrate-tabular" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "requests" }, { name = "sqlite-utils" }, + { name = "tinycrate" }, + { name = "tqdm" }, ] [package.dev-dependencies] @@ -361,6 +363,8 @@ dev = [ requires-dist = [ { name = "requests", specifier = ">=2.32.3" }, { name = "sqlite-utils", specifier = ">=3.37" }, + { name = "tinycrate", specifier = ">=0.1.2" }, + { name = "tqdm", specifier = ">=4.67.1" }, ] [package.metadata.requires-dev] @@ -440,6 +444,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "tinycrate" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "types-requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/40/ede4d6566efc7fbc6b87d3d8a43fd8f4cfe50befeb9053425e19b49ff4bc/tinycrate-0.1.2.tar.gz", hash = "sha256:7ed3c720643f1bd967c6ded8658496ba8319669215a10a7eb1515a596320474c", size = 60307 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/7b/d6ba447136d8c9e0a43c2e95e903245c61cb427748cc897f8c304c68711a/tinycrate-0.1.2-py3-none-any.whl", hash = "sha256:28c74950c9759c99d49aa3e8f3d6e12ca347c7b313cb234ae3c988e04857ab0c", size = 6998 }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + +[[package]] +name = "types-requests" +version = "2.32.0.20250602" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/b0/5321e6eeba5d59e4347fcf9bf06a5052f085c3aa0f4876230566d6a4dc97/types_requests-2.32.0.20250602.tar.gz", hash = "sha256:ee603aeefec42051195ae62ca7667cd909a2f8128fdf8aad9e8a5219ecfab3bf", size = 23042 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/18/9b782980e575c6581d5c0c1c99f4c6f89a1d7173dad072ee96b2756c02e6/types_requests-2.32.0.20250602-py3-none-any.whl", hash = "sha256:f4f335f87779b47ce10b8b8597b409130299f6971ead27fead4fe7ba6ea3e726", size = 20638 }, +] + [[package]] name = "urllib3" version = "2.2.3"