Skip to content

Commit c31ea4d

Browse files
committed
Fixed invalid type hint and updated docs slightly.
1 parent 194ef4b commit c31ea4d

File tree

4 files changed

+22
-17
lines changed

4 files changed

+22
-17
lines changed

docs/RemotePathDataLoader.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
RemotePathDataLoader
2-
=============
2+
====================
33

44
.. autoclass:: pyremotedata.dataloader.RemotePathDataLoader
55
:show-inheritance:

docs/RemotePathDataset.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
RemotePathDataset
2-
=============
2+
=================
33

44
.. autoclass:: pyremotedata.dataloader.RemotePathDataset
55
:show-inheritance:

docs/dataloader.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PyTorch Integration
2-
=============================
2+
===================
33

44
Includes a PyTorch IterableDataset and IterableDataLoader for loading data from a remote SFTP server through the pyremotedata.RemotePathIterator class.
55

src/pyremotedata/dataloader.py

+19-14
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import warnings
77
from queue import Queue
88
from threading import Thread
9-
from typing import Union, Optional, Tuple, Callable
9+
from typing import Union, Optional, Tuple, List, Callable
1010

1111
# Dependency imports
1212
import torch
@@ -19,21 +19,19 @@
1919
from .implicit_mount import RemotePathIterator
2020

2121
class RemotePathDataset(IterableDataset):
22-
"""
22+
'''
2323
Creates a PyTorch dataset from a RemotePathIterator.
2424
2525
By default the dataset will return the image as a tensor and the remote path as a string.
26-
27-
### Hierarchical mode
28-
If `hierarchical` >= 1, the dataset is in "Hierarchical mode" and will return the image as a tensor and the label as a list of integers (class indices for each level in the hierarchy).
29-
30-
The `class_handles` property can be used to get the class-idx mappings for the dataset.
31-
32-
By default the dataset will use a parser which assumes that the hierarchical levels are encoded in the remote path as directories like so:
3326
34-
`.../level_n/.../level_1/level_0/image.jpg`
35-
36-
Where `n = (hierarchical - 1)` and `level_0` is the leaf level.
27+
"""""""""""""""""""""
28+
**Hierarchical mode**
29+
"""""""""""""""""""""
30+
| If `hierarchical` >= 1, the dataset is in "Hierarchical mode" and will return the image as a tensor and the label as a list of integers (class indices for each level in the hierarchy).
31+
| The `class_handles` property can be used to get the class-idx mappings for the dataset.
32+
| By default the dataset will use a parser which assumes that the hierarchical levels are encoded in the remote path as directories like so:
33+
| `.../level_n/.../level_1/level_0/image.jpg`
34+
| Where `n = (hierarchical - 1)` and `level_0` is the leaf level.
3735
3836
Args:
3937
remote_path_iterator (RemotePathIterator): The remote path iterator to create the dataset from.
@@ -47,7 +45,14 @@ class RemotePathDataset(IterableDataset):
4745
return_remote_path (bool, optional): Whether to return the remote path. Default: False.
4846
return_local_path (bool, optional): Whether to return the local path. Default: False.
4947
verbose (bool, optional): Whether to print verbose output. Default: False.
50-
"""
48+
49+
Yields:
50+
Tuple[torch.Tensor, Union[str, List[int]]]: A tuple containing the image as a tensor and the label as the remote path or class indices.
51+
or
52+
Tuple[torch.Tensor, Union[str, List[int]], str]: A tuple containing the image as a tensor, the label as the remote path or class indices, and the local or remote path.
53+
or
54+
Tuple[torch.Tensor, Union[str, List[int]], str, str]: A tuple containing the image as a tensor, the label as the remote path or class indices, the local path, and the remote path.
55+
'''
5156
def __init__(
5257
self,
5358
remote_path_iterator : "RemotePathIterator",
@@ -339,7 +344,7 @@ def __next__(self):
339344
def __len__(self):
340345
return len(self.remote_path_iterator)
341346

342-
def parse_item(self, local_path : str, remote_path : str) -> Union[Tuple[Union[str, torch.Tensor], str], Tuple[Union[str, torch.Tensor], str, str], Tuple[Union[str, torch.Tensor], str, str, str]]:
347+
def parse_item(self, local_path : str, remote_path : str) -> Union[Tuple[torch.Tensor, Union[str, List[int]]], Tuple[torch.Tensor, Union[str, List[int]], str], Tuple[torch.Tensor, Union[str, List[int]], str, str]]:
343348
## Image processing
344349
# Check if image format is supported (jpeg/jpg/png)
345350
image_type = os.path.splitext(local_path)[-1]

0 commit comments

Comments
 (0)