Skip to content

Commit 0870c45

Browse files
committed
Fix bug from prior change
1 parent d2cf350 commit 0870c45

File tree

1 file changed

+10
-4
lines changed

1 file changed

+10
-4
lines changed

src/pyremotedata/implicit_mount.py

+10-4
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,15 @@ def download(self, remote_path: Union[str, List[str]], local_destination: Union[
726726
# If multiple remote paths are specified, use multi_download instead of download,
727727
# this function is more flexible than mirror (works for files from different directories) and much faster than executing multiple pget commands
728728
if not isinstance(remote_path, str) and len(remote_path) > 1:
729+
if local_destination is not None:
730+
local_destination_dirs = os.path.dirname(local_destination) if isinstance(local_destination, str) else [os.path.dirname(l) for l in local_destination]
731+
if len(set(local_destination_dirs)) == 1:
732+
local_destination = local_destination_dirs[0]
733+
else:
734+
result = []
735+
for this_local_dir in set(local_destination_dirs):
736+
result.extend(self.multi_download([r for r, ld in zip(remote_path, local_destination_dirs) if ld == this_local_dir], this_local_dir, **kwargs))
737+
return result
729738
return self.multi_download(remote_path, local_destination, **kwargs)
730739
if len(remote_path) == 1:
731740
remote_path = remote_path[0]
@@ -792,10 +801,6 @@ def multi_download(self, remote_paths: List[str], local_destination: str, blocki
792801
elif local_destination is None:
793802
local_destination = self.local_dir
794803
local_files = [os.path.join(local_destination, os.path.basename(r)) for r in remote_paths]
795-
if len(remote_paths) != len(local_destination):
796-
raise ValueError("remote_paths and local_destination must have the same length.")
797-
if any([os.path.splitext(l)[1] != os.path.splitext(r)[1] for l, r in zip(local_destination, remote_paths)]):
798-
raise ValueError("Local and remote file extensions must match.")
799804

800805
# Assemble the mget command, options and arguments
801806
multi_command = f'mget -O "{local_destination}" -P {n} ' + ' '.join([f'"{r}"' for r in remote_paths])
@@ -1174,6 +1179,7 @@ def _cleanup(self, force=False) -> None:
11741179
if self.download_thread is not None:
11751180
self.download_thread.join(timeout=1)
11761181
self.download_thread = None
1182+
time.sleep(0.01) # Wait a little to make sure the side-effects of the download thread are processed (race-condition)
11771183
# Clean up the temporary directory
11781184
while not self.download_queue.empty():
11791185
self.delete_queue.put(self.download_queue.get()[0])

0 commit comments

Comments
 (0)