WIP: Galaxy workflow-run export to ro-crate including provenance

README.md

@@ -315,6 +315,27 @@ rocrate add test-instance test1 http://example.com -r jobs -i test1_1
 rocrate add test-definition test1 test/test1/sort-and-change-case-test.yml -e planemo -v '>=0.70'
 ```
 
+Exporting a galaxy workflo-run export to an ro-crate.
+
+```python
+from rocrate import rocrate_api
+
+wf_path = base_path + "example-history-export3.ga"
+dataset_path = base_path + "example-history-export3/datasets/"
+wfr_metadata_path = base_path + "example-history-export3"
+files_list = os.listdir(dataset_path)
+files_list = [dataset_path + f for f in files_list]
+
+# Create base package
+wf_crate = rocrate_api.make_workflow_run_rocrate(workflow_path=wf_path,
+                        wfr_metadata_path=wfr_metadata_path, author=None, orcid=None,
+                        wf_type="Galaxy",include_files=files_list, prov_name="test_prov")
+
+# write crate to disk
+out_path = base_path + "example-history-export3-crate-new"
+wf_crate.write(out_path)
+```
+
 
 ## License
 

notebooks/ROcrate-example.ipynb

@@ -0,0 +1,160 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "sys.path.append('/home/padge/Elixir/workflow-export/ro-crate-py/')\n",
+    "import os\n",
+    "from rocrate.rocrate import ROCrate\n",
+    "base_path = '/home/padge/Elixir/workflow-export/'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/home/padge/Elixir/workflow-export/ro-crate-py/notebooks'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# adding a Dataset\n",
+    "crate = ROCrate()\n",
+    "os.makedirs(\"tmp3\", exist_ok=True)\n",
+    "open('tmp3/empty_file', 'w').close()\n",
+    "dataset_entity = crate.add_directory(source=\"tmpd3\", dest_path=\"new_tmp\")\n",
+    "crate.write(\"./new_crate4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from IPython.core.debugger import set_trace;set_trace()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SRR10903401\n",
+      "FooBar\n"
+     ]
+    }
+   ],
+   "source": [
+    "shutil.rmtree(\"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate\")\n",
+    "shutil.copytree(\"/home/padge/Downloads/Elixir/ro-crate-py/test/test-data/read_crate\", \"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate\")\n",
+    "crate = ROCrate(\"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate\")\n",
+    "readme = crate.dereference(\"test_file_galaxy.txt\")\n",
+    "with open(readme.source, \"rt\") as f:\n",
+    "    print(f.readline().strip())\n",
+    "new_source = \"/home/padge/Downloads/Elixir/ro-crate-py/tmp/foobar.txt\"\n",
+    "with open(new_source, \"wt\") as f:\n",
+    "    f.write(\"FooBar\\n\")\n",
+    "crate.delete(readme)\n",
+    "crate.add_file(new_source, \"test_file_galaxy.txt\")\n",
+    "crate.write(\"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate\")\n",
+    "with open(readme.source, \"rt\") as f:\n",
+    "    print(f.readline().strip())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import pdb\n",
+    "from IPython.core.debugger import set_trace\n",
+    "set_trace()\n",
+    "\n",
+    "# adding a Dataset\n",
+    "crate = ROCrate(\"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate/\")\n",
+    "sample_dir = '/home/padge/Downloads/Elixir/ro-crate-py/tmp/examples2/'\n",
+    "dataset_entity = crate.add_directory(sample_dir, \"new_dir\")\n",
+    "crate.write(\"/home/padge/Downloads/Elixir/ro-crate-py/tmp/crate2\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from rocrate import rocrate_api\n",
+    "\n",
+    "wf_path = base_path + \"example-history-export3.ga\"\n",
+    "dataset_path = base_path + \"example-history-export3/datasets/\"\n",
+    "files_list = os.listdir(dataset_path)\n",
+    "files_list = [dataset_path + f for f in files_list]\n",
+    "\n",
+    "# Create base package\n",
+    "wf_crate = rocrate_api.make_workflow_rocrate(workflow_path=wf_path,wf_type=\"Galaxy\",include_files=files_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write crate to disk\n",
+    "out_path = base_path + \"example-history-export3-crate\"\n",
+    "wf_crate.write(out_path)"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "bcc799c17010608b26cede4750872a7cf236af662c7ba3d883e539f7db8da28c"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.5 64-bit ('base': conda)",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -4,3 +4,6 @@ galaxy2cwl
 jinja2
 python-dateutil
 click
+prov
+typing-extensions
+pydot

rocrate/provenance_constants.py

@@ -0,0 +1,55 @@
+import hashlib
+import os
+import uuid
+
+from prov.identifier import Namespace
+
+__citation__ = "https://doi.org/10.5281/zenodo.1208477"
+
+# NOTE: Semantic versioning of the CWLProv Research Object
+# **and** the cwlprov files
+#
+# Rough guide (major.minor.patch):
+# 1. Bump major number if removing/"breaking" resources or PROV statements
+# 2. Bump minor number if adding resources or PROV statements
+# 3. Bump patch number for non-breaking non-adding changes,
+#    e.g. fixing broken relative paths
+CWLPROV_VERSION = "https://w3id.org/cwl/prov/0.6.0"
+
+# Research Object folders
+METADATA = "metadata"
+DATA = "data"
+WORKFLOW = "workflow"
+SNAPSHOT = "snapshot"
+# sub-folders
+MAIN = os.path.join(WORKFLOW, "main")
+PROVENANCE = os.path.join(METADATA, "provenance")
+LOGS = os.path.join(METADATA, "logs")
+WFDESC = Namespace("wfdesc", "http://purl.org/wf4ever/wfdesc#")
+WFPROV = Namespace("wfprov", "http://purl.org/wf4ever/wfprov#")
+WF4EVER = Namespace("wf4ever", "http://purl.org/wf4ever/wf4ever#")
+RO = Namespace("ro", "http://purl.org/wf4ever/ro#")
+ORE = Namespace("ore", "http://www.openarchives.org/ore/terms/")
+FOAF = Namespace("foaf", "http://xmlns.com/foaf/0.1/")
+SCHEMA = Namespace("schema", "http://schema.org/")
+CWLPROV = Namespace("cwlprov", "https://w3id.org/cwl/prov#")
+ORCID = Namespace("orcid", "https://orcid.org/")
+UUID = Namespace("id", "urn:uuid:")
+
+# BagIt and YAML always use UTF-8
+ENCODING = "UTF-8"
+TEXT_PLAIN = 'text/plain; charset="%s"' % ENCODING
+
+# sha1, compatible with the File type's "checksum" field
+# e.g. "checksum" = "sha1$47a013e660d408619d894b20806b1d5086aab03b"
+# See ./cwltool/schemas/v1.0/Process.yml
+Hasher = hashlib.sha1
+SHA1 = "sha1"
+SHA256 = "sha256"
+SHA512 = "sha512"
+
+# TODO: Better identifiers for user, at least
+# these should be preserved in ~/.config/cwl for every execution
+# on this host
+USER_UUID = uuid.uuid4().urn
+ACCOUNT_UUID = uuid.uuid4().urn