Handling pyiron tables

pyiron_rdm/classic.py

@@ -31,6 +31,16 @@ def classic_general_job(job, export_env_file: bool):
     job_cdict = process_general_job(job)
     return job_cdict
 
+def classic_table(table_job, export_env_file: bool):
+    from ob.concept_dict import process_table_job
+
+    if export_env_file:
+        from ob.concept_dict import export_env
+        export_env(table_job.path)
+
+    table_cdict = process_table_job(table_job)
+    return table_cdict
+
 def classic_lammps(lammps_job, export_env_file):
     '''export_env_file: Bool'''
     from ob.concept_dict import process_lammps_job
@@ -111,14 +121,6 @@ def validate_upload_options(options, allowed_keys, allowed_defects=None):
                 raise ValueError(
                     f'Invalid defect(s) in "options[\'defects\']": {sorted(invalid_defects)}. \
                     Allowed defects are: {sorted(allowed_defects)}')
-    materials = options.get('materials')
-    if materials:
-        if not isinstance(materials, list):
-            options['materials'] = [options['materials']]
-    pseudopots = options.get('pseudopotentials')
-    if pseudopots:
-        if not isinstance(pseudopots, list):
-            options['pseudopotentials'] = [options['pseudopotentials']]
 
 def openbis_login(url, username, instance='bam', s3_config_path = None):
     #instance = get_datamodel(o)
@@ -155,11 +157,15 @@ def upload_classic_pyiron(job, o, space, project, collection=None, export_env_fi
     else:
         options = {}
 
-    structure = job.structure
-    if not structure:
-        print('This job does not contain a structure and will not be uploaded. \
-                Please add structure before trying to upload.')
-        return
+    job_type = job.to_dict()['TYPE'].lower()
+    structure = None
+
+    if 'tablejob' not in job_type:
+        structure = job.structure
+        if not structure:
+            print('This job does not contain a structure and will not be uploaded. \
+                    Please add structure before trying to upload.')
+            return
 
     # Project env file - TODO what is this for??
     pr = job.project
@@ -169,18 +175,15 @@ def upload_classic_pyiron(job, o, space, project, collection=None, export_env_fi
 
     if not collection:
         collection = pr.name
-    space = space.upper()
-    project = project.upper()
-    collection = collection.upper()
 
 # ------------------------------------VALIDATION----------------------------------------------
     cdicts_to_validate = []
 
-    struct_dict = classic_structure(pr, structure, structure_name=job.name + '_structure', options=options,
-                                    is_init_struct=is_init_struct, init_structure=init_structure)
-    cdicts_to_validate.append(struct_dict)
+    if structure:
+        struct_dict = classic_structure(pr, structure, structure_name=job.name + '_structure', options=options,
+                                        is_init_struct=is_init_struct, init_structure=init_structure)
+        cdicts_to_validate.append(struct_dict)
 
-    job_type = job.to_dict()['TYPE']
     proceed = True    
     if 'lammps' in job_type:
         job_cdict = classic_lammps(job, export_env_file=export_env_file)
@@ -197,6 +200,10 @@ def upload_classic_pyiron(job, o, space, project, collection=None, export_env_fi
         cdicts_to_validate.append(equil_struct_dict)
         cdicts_to_validate += [child_cdict for child_cdict in child_jobs_cdict]
 
+    elif 'tablejob' in job_type:
+        table_cdict = classic_table(job, export_env_file=export_env_file)
+        cdicts_to_validate.append(table_cdict)
+
     else:
         print(f'The {job_type} job type is not implemented for OpenBIS upload yet.')
         proceed = input("Type 'yes' to proceed with an upload to general pyiron job type.")
@@ -213,7 +220,7 @@ def upload_classic_pyiron(job, o, space, project, collection=None, export_env_fi
     elif datamodel == 'bam':
         upload_final_struct = False
 
-    if upload_final_struct and (not 'murn' in job.to_dict()['TYPE']):
+    if upload_final_struct and all(x not in job_type for x in ('murn', 'tablejob')):
         if is_init_struct:
             init_structure = structure
         final_structure = job.get_structure()
@@ -231,42 +238,49 @@ def upload_classic_pyiron(job, o, space, project, collection=None, export_env_fi
 #--------------------------------------UPLOAD-------------------------------------------------
     from ob.ob_upload import openbis_upload_validated
 
-    # Structure
-    cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[0]
-    ob_structure_id = openbis_upload_validated(o, space, project, collection, object_name, 
-                             object_type, ob_parents, props_dict, ds_types, cdict)
-
-    if datamodel == 'sfb1394':
-        job_parents = None            # job does not have init structure as parent
-        str_parent = ob_structure_id  # equil structure has init as parent
-    elif datamodel == 'bam':
-        job_parents = ob_structure_id # job has init structure as parent
-        str_parent = None             # equil structure does not have init as parent
-
-    # (Main) job
-    cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[1]
-    ob_job_id = openbis_upload_validated(o, space, project, collection, object_name, 
-                             object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=job_parents)
-
-    if 'murn' in job_type:
-        ob_children_ids = []
-        # Murn equilibrium structure
-        cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[2]
-        ob_equil_struct_id = openbis_upload_validated(o, space, project, collection, object_name, 
-                                object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=str_parent)
-        ob_children_ids.append(ob_equil_struct_id)
-        # Murn children jobs
-        for validated_child in validated_to_upload[3:]:
-            cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_child
-            ob_child_id = openbis_upload_validated(o, space, project, collection, object_name, 
-                                object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=str_parent)
-            ob_children_ids.append(ob_child_id)
-        from ob.ob_upload import link_children
-        link_children(o, ob_job_id, ob_children_ids)
-
-    # Final structure upload (already included as equilibrium for murn)
-    elif upload_final_struct:
-        cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[-1]
-        ob_final_structure_id = openbis_upload_validated(o, space, project, collection, object_name, 
-                             object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=[ob_structure_id, ob_job_id])
+    # Pyiron table job - no structure
+    if 'tablejob' in job_type:
+        cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[0]
+        ob_job_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                object_type, ob_parents, props_dict, ds_types, cdict)
+
+    else:
+        # Structure
+        cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[0]
+        ob_structure_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                object_type, ob_parents, props_dict, ds_types, cdict)
+
+        if datamodel == 'sfb1394':
+            job_parents = None            # job does not have init structure as parent
+            str_parent = ob_structure_id  # equil structure has init as parent
+        elif datamodel == 'bam':
+            job_parents = ob_structure_id # job has init structure as parent
+            str_parent = None             # equil structure does not have init as parent
+
+        # (Main) job
+        cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[1]
+        ob_job_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=job_parents)
+
+        if 'murn' in job_type:
+            ob_children_ids = []
+            # Murn equilibrium structure
+            cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[2]
+            ob_equil_struct_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                    object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=str_parent)
+            ob_children_ids.append(ob_equil_struct_id)
+            # Murn children jobs
+            for validated_child in validated_to_upload[3:]:
+                cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_child
+                ob_child_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                    object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=str_parent)
+                ob_children_ids.append(ob_child_id)
+            from ob.ob_upload import link_children
+            link_children(o, ob_job_id, ob_children_ids)
+
+        # Final structure upload (already included as equilibrium for murn)
+        elif upload_final_struct:
+            cdict, props_dict, object_type, ds_types, ob_parents, object_name = validated_to_upload[-1]
+            ob_final_structure_id = openbis_upload_validated(o, space, project, collection, object_name, 
+                                object_type, ob_parents, props_dict, ds_types, cdict, parent_ids=[ob_structure_id, ob_job_id])
 #---------------------------------------------------------------------------------------------

pyiron_rdm/concept_dict.py

@@ -21,12 +21,24 @@
 def process_general_job(job):
     method_dict = {}
     add_simulation_software(job, method_dict)
+    add_general_contexts(method_dict)
     get_simulation_folder(job, method_dict)
     file_name = job.path + '_concept_dict.json'
     with open(file_name, 'w') as f:
         json.dump(method_dict, f, indent=2)
     return method_dict
 
+def process_table_job(table_job):
+    method_dict = {}
+    add_simulation_software(table_job, method_dict)
+    add_general_contexts(method_dict)
+    extract_table_info(table_job, method_dict)
+    get_simulation_folder(table_job, method_dict)
+    file_name = table_job.path + '_concept_dict.json'
+    with open(file_name, 'w') as f:
+        json.dump(method_dict, f, indent=2)
+    return method_dict
+
 def process_lammps_job(job):
     method_dict = {}
     add_lammps_contexts(method_dict)
@@ -81,6 +93,15 @@ def process_vasp_job(job):
         json.dump(method_dict, f, indent=2)
     return method_dict
 
+def add_general_contexts(method_dict):
+    method_dict['@context'] = {}
+    method_dict['@context']['path'] = 'http://purls.helmholtz-metadaten.de/cmso/hasPath'
+    method_dict['@context']['inputs'] = 'http://purls.helmholtz-metadaten.de/asmo/hasInputParameter'
+    method_dict['@context']['label'] = 'http://www.w3.org/2000/01/rdf-schema#label'
+    method_dict['@context']['value'] = 'http://purls.helmholtz-metadaten.de/asmo/hasValue'
+    method_dict['@context']['outputs'] = 'http://purls.helmholtz-metadaten.de/cmso/hasCalculatedProperty'
+    method_dict['@context']['workflow_manager'] = 'http://demo.fiz-karlsruhe.de/matwerk/E457491'
+
 def add_lammps_contexts(method_dict):
     method_dict['@context'] = {}
     method_dict['@context']['sample'] = 'http://purls.helmholtz-metadaten.de/cmso/AtomicScaleSample'
@@ -1181,6 +1202,47 @@ def extract_vasp_calculated_quantities(job, method_dict):
         )
     method_dict['outputs'] = outputs
 
+def extract_table_info(table_job, method_dict):
+    import base64
+    import json
+
+    table_df = table_job.get_dataframe()
+    inputs = []
+    inputs.append(
+        {
+            "label": "columns",
+            "value": ", ".join(table_df.columns)
+        }
+    )
+    inputs.append(
+        {
+            "label": "number_of_jobs",
+            "value": len(table_df)
+        }
+    )
+
+    table_df_cl = table_df.head(100).astype(str).replace("nan", "NaN")
+    spreadsheet = {
+        "headers": table_df_cl.columns.to_list(),
+        "data": table_df_cl.to_numpy().tolist(),
+        "width": [150] * (len(table_df_cl.columns)),
+    }
+    preview = (
+        "<DATA>"
+        + str(
+            base64.b64encode(bytes(json.dumps(spreadsheet), encoding="utf-8")),
+            encoding="utf-8",
+        )
+        + "</DATA>"
+)
+    inputs.append(
+        {
+            "label": "table_preview",
+            "value": preview
+        }
+    )
+    method_dict['inputs'] = inputs
+
 def export_env(path):
     """Exports to path+_environment.yml"""
     import os

pyiron_rdm/ob_OT_bam.py

@@ -3,15 +3,15 @@
 def material_par(props_dict, options):
     object_type = "MATERIAL_V1"
     if options.get("materials"):
-        permid_materials = options["materials"]
+        parent_materials = options["materials"]
         where_clause = {}
         requested_attrs = []
     else:
         mat_dict_pct_str = species_by_num_to_pct(props_dict)
         where_clause = {"chem_species_by_comp_in_pct": mat_dict_pct_str}
         requested_attrs = ["chem_species_by_comp_in_pct"]
-        permid_materials = ""
-    return object_type, permid_materials, where_clause, requested_attrs
+        parent_materials = ""
+    return object_type, parent_materials, where_clause, requested_attrs
 
 def intpot_par(cdict):
     object_type = "INTERATOMIC_POTENTIAL"
@@ -21,8 +21,8 @@ def intpot_par(cdict):
 
 def pseudopot_par(options):
     object_type = "PSEUDOPOTENTIAL"
-    permid_pseudopots = options.get("pseudopotentials", "")
-    return object_type, permid_pseudopots
+    parent_materials = options.get("pseudopotentials", "")
+    return object_type, parent_materials
 
 def sw_par(cdict):
     import re
@@ -97,9 +97,9 @@ def get_ot_info(cdict):
         )
 
 def get_inv_parent(parent_name, cdict, props_dict, options):
-    ob_type, permids, where_clause, requested_attrs, ob_code = "", "", {}, [], ""
+    ob_type, parents, where_clause, requested_attrs, ob_code = "", "", {}, [], ""
     if parent_name == "material":
-        ob_type, permids, where_clause, requested_attrs = material_par(props_dict, options)
+        ob_type, parents, where_clause, requested_attrs = material_par(props_dict, options)
     elif parent_name == "compute_resource":
         ob_type, ob_code = compresource_par(cdict)
     elif parent_name == "software":
@@ -111,7 +111,7 @@ def get_inv_parent(parent_name, cdict, props_dict, options):
     elif parent_name == "wf_reference":
         ob_type, ob_code = wfref_par(cdict)
 
-    return ob_type, permids, where_clause, requested_attrs, ob_code
+    return ob_type, parents, where_clause, requested_attrs, ob_code
 
 # upload options ______________________________________________