@@ -726,6 +726,15 @@ def download(self, remote_path: Union[str, List[str]], local_destination: Union[
726
726
# If multiple remote paths are specified, use multi_download instead of download,
727
727
# this function is more flexible than mirror (works for files from different directories) and much faster than executing multiple pget commands
728
728
if not isinstance (remote_path , str ) and len (remote_path ) > 1 :
729
+ if local_destination is not None :
730
+ local_destination_dirs = os .path .dirname (local_destination ) if isinstance (local_destination , str ) else [os .path .dirname (l ) for l in local_destination ]
731
+ if len (set (local_destination_dirs )) == 1 :
732
+ local_destination = local_destination_dirs [0 ]
733
+ else :
734
+ result = []
735
+ for this_local_dir in set (local_destination_dirs ):
736
+ result .extend (self .multi_download ([r for r , ld in zip (remote_path , local_destination_dirs ) if ld == this_local_dir ], this_local_dir , ** kwargs ))
737
+ return result
729
738
return self .multi_download (remote_path , local_destination , ** kwargs )
730
739
if len (remote_path ) == 1 :
731
740
remote_path = remote_path [0 ]
@@ -792,10 +801,6 @@ def multi_download(self, remote_paths: List[str], local_destination: str, blocki
792
801
elif local_destination is None :
793
802
local_destination = self .local_dir
794
803
local_files = [os .path .join (local_destination , os .path .basename (r )) for r in remote_paths ]
795
- if len (remote_paths ) != len (local_destination ):
796
- raise ValueError ("remote_paths and local_destination must have the same length." )
797
- if any ([os .path .splitext (l )[1 ] != os .path .splitext (r )[1 ] for l , r in zip (local_destination , remote_paths )]):
798
- raise ValueError ("Local and remote file extensions must match." )
799
804
800
805
# Assemble the mget command, options and arguments
801
806
multi_command = f'mget -O "{ local_destination } " -P { n } ' + ' ' .join ([f'"{ r } "' for r in remote_paths ])
@@ -1174,6 +1179,7 @@ def _cleanup(self, force=False) -> None:
1174
1179
if self .download_thread is not None :
1175
1180
self .download_thread .join (timeout = 1 )
1176
1181
self .download_thread = None
1182
+ time .sleep (0.01 ) # Wait a little to make sure the side-effects of the download thread are processed (race-condition)
1177
1183
# Clean up the temporary directory
1178
1184
while not self .download_queue .empty ():
1179
1185
self .delete_queue .put (self .download_queue .get ()[0 ])
0 commit comments