Preserve relative links within the upload directory.

This matches a corresponding update to the gobbler backend and allows users to
explicitly deduplicate their uploads when they need to store the same file
multiple times, e.g., for delayed arrays with shared seeds.

Author: LTLA

src/pygobbler/upload_directory.py

@@ -48,14 +48,13 @@ def upload_directory(project: str, asset: str, version: str, directory: str, sta
                 if os.path.islink(src):
                     slink = os.readlink(src)
 
-                if slink == "" or not os.path.isabs(slink):
-                    try:
-                        os.link(src, dest)
-                    except:
-                        import shutil
-                        shutil.copy(src, dest)
-                else:
+                if slink == "":
+                    _link_or_copy(src, dest)
+                elif _is_absolute_or_local_link(slink, f):
                     os.symlink(slink, dest)
+                else:
+                    full_src = os.path.normpath(os.path.join(os.path.dirname(src), slink))
+                    _link_or_copy(full_src, dest)
 
         directory = newdir
 
@@ -68,3 +67,33 @@ def upload_directory(project: str, asset: str, version: str, directory: str, sta
     }
     ut.dump_request(staging, url, "upload", req)
     return
+
+
+def _is_absolute_or_local_link(target: str, link_path: str) -> bool:
+    if os.path.isabs(target):
+        return True
+
+    pre_length = len(link_path.split("/"))
+    post_fragments = target.split("/")[:-1]
+
+    # link_path should be relative, so the idea is to check whether the
+    # 'target' file is still a child of 'dirname(link_path)'.
+    for x in post_fragments:
+        if x == ".":
+            continue
+        elif x == "..":
+            pre_length -= 1
+            if pre_length < 0:
+                return False
+        else:
+            pre_length += 1
+
+    return True
+
+
+def _link_or_copy(src: str, dest: str):
+    try:
+        os.link(src, dest)
+    except:
+        import shutil
+        shutil.copy(src, dest)

tests/test_upload_directory.py

@@ -83,6 +83,41 @@ def test_upload_directory_links(setup):
     assert "link" not in man["whee"]
 
 
+def test_upload_directory_relative_links(setup):
+    _, staging, registry, url = pyg.start_gobbler()
+
+    dest = tempfile.mkdtemp()
+    with open(os.path.join(dest, "blah.txt"), "w") as handle:
+        handle.write("motto motto")
+    os.symlink("blah.txt", os.path.join(dest, "whee.txt")) # relative links inside the directory are retained.
+    os.mkdir(os.path.join(dest, "foo"))
+    os.symlink("../whee.txt", os.path.join(dest, "foo/bar.txt")) 
+
+    fid, outside = tempfile.mkstemp(dir=os.path.dirname(dest))
+    with open(outside, "w") as handle:
+        handle.write("toki wa kizuna uta")
+    os.symlink(os.path.join("../../", os.path.basename(outside)), os.path.join(dest, "foo/outer.txt")) # relative links outside the directory are lost.
+
+    pyg.upload_directory(
+        project="test-more-upload", 
+        asset="nicole", 
+        version="1", 
+        directory=dest,
+        staging=staging,
+        url=url
+    )
+
+    man = pyg.fetch_manifest("test-more-upload", "nicole", "1", registry=registry, url=url)
+    assert sorted(man.keys()) == [ "blah.txt", "foo/bar.txt", "foo/outer.txt", "whee.txt" ]
+    assert "link" in man["whee.txt"]
+    assert "link" not in man["foo/outer.txt"]
+    assert "link" in man["foo/bar.txt"]
+    assert "link" not in man["blah.txt"]
+    assert man["foo/outer.txt"]["size"] == 18
+    assert man["whee.txt"]["size"] == man["foo/bar.txt"]["size"]
+    assert man["whee.txt"]["size"] == man["blah.txt"]["size"]
+
+
 def test_upload_directory_staging(setup):
     _, staging, registry, url = pyg.start_gobbler()