From ffe6e4d7a611e7e45dfed3acad84f32e2baa6554 Mon Sep 17 00:00:00 2001 From: Martin Simonovsky Date: Tue, 2 Sep 2025 22:59:45 +0000 Subject: [PATCH 1/4] Use filenames with relative paths to root Signed-off-by: Martin Simonovsky --- flytekit/remote/remote_fs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flytekit/remote/remote_fs.py b/flytekit/remote/remote_fs.py index ace6ebd852..f88bf6ad34 100644 --- a/flytekit/remote/remote_fs.py +++ b/flytekit/remote/remote_fs.py @@ -24,6 +24,7 @@ _DEFAULT_CALLBACK = NoOpCallback() _PREFIX_KEY = "upload_prefix" _HASHES_KEY = "hashes" +_LPATH_ROOT_KEY = "local_path_root" # This file system is not really a filesystem, so users aren't really able to specify the remote path, # at least not yet. REMOTE_PLACEHOLDER = "flyte://data" @@ -143,8 +144,12 @@ async def _put_file( Make the request and upload, but then how do we get the s3 paths back to the user? """ prefix = kwargs.pop(_PREFIX_KEY) + lpath = pathlib.Path(lpath) + relative_subdirectory = str(lpath.parent)[len(kwargs.pop(_LPATH_ROOT_KEY)):] + if relative_subdirectory: + prefix += relative_subdirectory _, native_url = self._remote.upload_file( - pathlib.Path(lpath), self._remote.default_project, self._remote.default_domain, prefix + lpath, self._remote.default_project, self._remote.default_domain, prefix ) return native_url @@ -234,6 +239,7 @@ async def _put( kwargs[_PREFIX_KEY] = prefix kwargs[_HASHES_KEY] = file_info + kwargs[_LPATH_ROOT_KEY] = lpath.rstrip(os.path.sep) res = await super()._put(lpath, REMOTE_PLACEHOLDER, recursive, callback, batch_size, **kwargs) if isinstance(res, list): res = self.extract_common(res) From de5c0566e8618c6fc765edf10e7b85e80fb230f1 Mon Sep 17 00:00:00 2001 From: Martin Simonovsky Date: Tue, 2 Sep 2025 23:06:28 +0000 Subject: [PATCH 2/4] more pythonic Signed-off-by: Martin Simonovsky --- flytekit/remote/remote_fs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flytekit/remote/remote_fs.py b/flytekit/remote/remote_fs.py index f88bf6ad34..7b4a7c5d36 100644 --- a/flytekit/remote/remote_fs.py +++ b/flytekit/remote/remote_fs.py @@ -145,9 +145,9 @@ async def _put_file( """ prefix = kwargs.pop(_PREFIX_KEY) lpath = pathlib.Path(lpath) - relative_subdirectory = str(lpath.parent)[len(kwargs.pop(_LPATH_ROOT_KEY)):] - if relative_subdirectory: - prefix += relative_subdirectory + relative_subdirectory = str(lpath.parent.relative_to(pathlib.Path(kwargs.pop(_LPATH_ROOT_KEY)))) + if relative_subdirectory != ".": + prefix += "/" + relative_subdirectory _, native_url = self._remote.upload_file( lpath, self._remote.default_project, self._remote.default_domain, prefix ) @@ -239,7 +239,7 @@ async def _put( kwargs[_PREFIX_KEY] = prefix kwargs[_HASHES_KEY] = file_info - kwargs[_LPATH_ROOT_KEY] = lpath.rstrip(os.path.sep) + kwargs[_LPATH_ROOT_KEY] = lpath res = await super()._put(lpath, REMOTE_PLACEHOLDER, recursive, callback, batch_size, **kwargs) if isinstance(res, list): res = self.extract_common(res) From a90ea5b032f1e821526dc5a40ecd7596da558637 Mon Sep 17 00:00:00 2001 From: Martin Simonovsky Date: Wed, 3 Sep 2025 07:28:16 +0000 Subject: [PATCH 3/4] safer popping Signed-off-by: Martin Simonovsky --- flytekit/remote/remote_fs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flytekit/remote/remote_fs.py b/flytekit/remote/remote_fs.py index 7b4a7c5d36..fcd5797a0d 100644 --- a/flytekit/remote/remote_fs.py +++ b/flytekit/remote/remote_fs.py @@ -145,9 +145,11 @@ async def _put_file( """ prefix = kwargs.pop(_PREFIX_KEY) lpath = pathlib.Path(lpath) - relative_subdirectory = str(lpath.parent.relative_to(pathlib.Path(kwargs.pop(_LPATH_ROOT_KEY)))) - if relative_subdirectory != ".": - prefix += "/" + relative_subdirectory + if _LPATH_ROOT_KEY in kwargs: + lpath_root = pathlib.Path(kwargs.pop(_LPATH_ROOT_KEY)) + relative_subdirectory = str(lpath.parent.relative_to(lpath_root)) + if relative_subdirectory != ".": + prefix += "/" + relative_subdirectory _, native_url = self._remote.upload_file( lpath, self._remote.default_project, self._remote.default_domain, prefix ) From 3d4e56443a85473e55db929ddeebfbc3c8785277 Mon Sep 17 00:00:00 2001 From: Martin Simonovsky Date: Wed, 3 Sep 2025 22:13:46 +0000 Subject: [PATCH 4/4] fix 'is not in the subpath of' errors in integration tests Signed-off-by: Martin Simonovsky --- flytekit/remote/remote_fs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flytekit/remote/remote_fs.py b/flytekit/remote/remote_fs.py index fcd5797a0d..67bfd14a15 100644 --- a/flytekit/remote/remote_fs.py +++ b/flytekit/remote/remote_fs.py @@ -147,9 +147,10 @@ async def _put_file( lpath = pathlib.Path(lpath) if _LPATH_ROOT_KEY in kwargs: lpath_root = pathlib.Path(kwargs.pop(_LPATH_ROOT_KEY)) - relative_subdirectory = str(lpath.parent.relative_to(lpath_root)) - if relative_subdirectory != ".": - prefix += "/" + relative_subdirectory + if lpath.parent.is_relative_to(lpath_root): + relative_subdirectory = str(lpath.parent.relative_to(lpath_root)) + if relative_subdirectory != ".": + prefix += "/" + relative_subdirectory _, native_url = self._remote.upload_file( lpath, self._remote.default_project, self._remote.default_domain, prefix )