Fix file loading

Author: blythed

CHANGELOG.md

@@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 #### Bug Fixes
 
 - Fix the random silent failure bug when ibis creates tables.
-- Remove a bunch of dead code with vulture
+- Remove a bunch of dead code with vulture 
 
 ## [0.5.0](https://github.com/superduper-io/superduper/compare/0.5.0...0.4.0])    (2024-Nov-02)
 

superduper/base/base.py

@@ -213,6 +213,7 @@ def decode(r, db: t.Optional['Datalayer'] = None):
             from superduper.misc.importing import import_object
 
             cls = import_object(r['_path'])
+
         r = Document.decode(r, schema=cls.class_schema, db=db)
         return cls.from_dict(r, db=None)
 
@@ -234,7 +235,7 @@ def encode(
         for k, v in kwargs.items():
             setattr(context, k, v)
 
-        r = self.dict()  # metadata=context.metadata, defaults=context.defaults)
+        r = self.dict()
         r = self.class_schema.encode_data(r, context=context)
 
         def _replace_loads_with_references(record, lookup):
@@ -273,7 +274,6 @@ def _replace_uuids_with_keys(record):
                 del r['uuid']
             r = _replace_uuids_with_keys(r)
 
-        # TODO deprecate this wrapper (not needed)
         return {
             **r,
             KEY_BUILDS: context.builds,

superduper/base/datatype.py

@@ -211,7 +211,9 @@ def _decode_leaf(r, builds, db: t.Optional['Datalayer'] = None):
             db=db,
         )
     else:
-        assert issubclass(cls, Base)
+        mro = [f'{x.__module__}.{x.__name__}' for x in cls.__mro__]
+
+        assert issubclass(cls, Base) or 'superduper.base.base.Base' in mro
         dict_ = cls.class_schema.decode_data(dict_, builds=builds, db=db)
         out = cls.from_dict(dict_, db=db)
 
@@ -272,7 +274,7 @@ def encode_data(self, item, context):
         :param context: A context object containing caches.
         """
         assert isinstance(item, list)
-        return [
+        out = [
             (
                 ComponentType().encode_data(
                     r,
@@ -283,6 +285,7 @@ def encode_data(self, item, context):
             )
             for r in item
         ]
+        return out
 
     def decode_data(self, item, builds, db):
         """Decode the item from `bytes`.
@@ -310,7 +313,8 @@ def encode_data(self, item, context):
         :param context: A context object containing caches.
         """
         assert isinstance(item, dict)
-        return {k: File().encode_data(v, context) for k, v in item.items()}
+        out = {k: File().encode_data(v, context) for k, v in item.items()}
+        return out
 
     def decode_data(self, item, builds, db):
         """Decode the item from `bytes`.
@@ -633,9 +637,11 @@ def encode_data(self, item, context):
         :param context: A context object containing caches.
         """
         if isinstance(item, FileItem):
-            return item.reference
-        assert os.path.exists(item)
-        file = FileItem(identifier=self.hash(item), path=item)
+            file = item
+        else:
+            assert os.path.exists(item)
+            file = FileItem(identifier=self.hash(item), path=item)
+
         context.files[file.identifier] = file.path
         return file.reference
 
@@ -646,7 +652,9 @@ def decode_data(self, item, builds, db):
         :param builds: The build cache.
         :param db: Datalayer.
         """
-        return FileItem(identifier=item.split(':')[-1], db=db)
+        file_id = item.split(':')[-1]
+        path = db.artifact_store.get_file(file_id)
+        return FileItem(identifier=file_id, path=path, db=db)
 
     @classmethod
     def hash(cls, item):

superduper/base/schema.py

@@ -3,11 +3,11 @@
 import json
 import re
 import typing as t
-from functools import cached_property
 
 from superduper import CFG, logging
 from superduper.base.datatype import BaseDataType
 from superduper.base.encoding import EncodeContext
+from superduper.misc.importing import isreallyinstance
 from superduper.misc.special_dicts import dict_to_ascii_table
 
 if t.TYPE_CHECKING:
@@ -44,7 +44,9 @@ def __add__(self, other: 'Schema'):
     @property
     def trivial(self):
         """Determine if the schema contains only trivial fields."""
-        return not any([isinstance(v, BaseDataType) for v in self.fields.values()])
+        return not any(
+            [isreallyinstance(v, BaseDataType) for v in self.fields.values()]
+        )
 
     def __repr__(self):
         return dict_to_ascii_table(self.fields)
@@ -138,6 +140,7 @@ def is_ref(x):
                 ):
                     assert isinstance(value, str), msg
                     value = json.loads(value)
+
                 decoded[k] = field.decode_data(value, builds=builds, db=db)
 
         return decoded

superduper/misc/importing.py

@@ -23,3 +23,14 @@ def import_object(path):
     module = '.'.join(path.split('.')[:-1])
     cls = path.split('.')[-1]
     return getattr(importlib.import_module(module), cls)
+
+
+def isreallyinstance(this, cls):
+    """Check if the component is an instance of a class.
+
+    :param this: The component to check.
+    :param cls: The class to check.
+    """
+    # no idea why this is sometimes necessary - may be IPython autoreload related
+    mro = [f'{o.__module__}.{o.__name__}' for o in this.__class__.__mro__]
+    return isinstance(this, cls) or f'{cls.__module__}.{cls.__name__}' in mro

test/unittest/component/test_schema.py

@@ -96,9 +96,6 @@ def test_schema_with_file(db, tmp_file):
     # loaded document contains a pointer to the file
     assert isinstance(r['my_file'], FileItem)
 
-    # however the path has not been populated
-    assert not r['my_file'].path
-
     # unpacking the document copies the file to the artifact-store
     rr = r.unpack()