Merge branch 'fix/pagination' into rogues/python3.8

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 88
+max-complexity = 18
+# E999 Ignore syntax errors bc we use py3.11
+# C901 Ignore complexity errors bc I don't feel like rewriting several files
+# A005 Shadowing built-in modules
+ignore = E203, E266, E501, W503, RST304, C901, E999, A005 

.github/workflows/feature_branch.yaml

@@ -0,0 +1,11 @@
+name: Feature branch
+on:
+  push:
+    branches-ignore:
+      [master]
+
+jobs:
+  lint:
+    name: Lint
+    uses: ./.github/workflows/lint.yaml
+    secrets: inherit

.github/workflows/lint.yaml

@@ -0,0 +1,33 @@
+name: "Lint"
+on: workflow_call
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Install Python
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        with:
+          python-version: "3.9"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@bafa4d4adfc54e5fba0da6c1eb34001a4fdbaa4c # v1.0
+        with:
+          version: 1.8.3
+
+      - name: Install Dependencies
+        run: poetry install
+
+      - name: Pre-Commit
+        run: poetry run pre-commit run --all-files
+

.github/workflows/main_branch.yaml

@@ -0,0 +1,86 @@
+# Action for merging PRs into the main branch
+name: Main Branch
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [closed]
+    branches:
+      - master
+
+jobs:
+  lint:
+    name: Lint
+    uses: ./.github/workflows/lint.yaml
+    secrets: inherit
+
+  release:
+    name: Create GitHub Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Check if PR was merged
+        run: |
+          if [ ${{ github.event_name }} == 'pull_request' ]; then
+            if [ ${{ github.event.pull_request.merged }} == 'false' ]; then
+              echo "This PR was closed without merging. Exiting..."
+              exit 0
+            fi
+          fi
+
+      - name: Checkout code
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: TOML Reader
+        id: read_toml
+        uses: SebRollen/toml-action@2bd04b06a3ebc3e6a3eb6060de115710cad16cd6 #v1.0.2
+        with:
+          file: 'pyproject.toml'
+          field: 'tool.poetry.version'
+
+      - name: Define TAG
+        run: |
+          export VERSION="${{ steps.read_toml.outputs.value }}"
+          echo "TAG=v$VERSION" >> $GITHUB_ENV
+
+      - name: Check if tag exists
+        id: check_tag
+        uses: actions/github-script@v6
+        with:
+          result-encoding: string
+          script: |
+            const tag = process.env.TAG;
+            const { owner, repo } = context.repo;
+            try {
+              await github.rest.git.getRef({
+                owner,
+                repo,
+                ref: `tags/${tag}`,
+              });
+              // If the API call doesn't throw an error, the tag exists
+              return true;
+            } catch (error) {
+              // If the API call throws an error, the tag doesn't exist
+              return false;
+            }
+        env:
+          TAG: ${{ env.TAG }}
+
+      - name: Create Release and Tag
+        uses: actions/github-script@v6
+        with:
+          result-encoding: string
+          retries: 3
+          script: |
+            github.rest.repos.createRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: process.env.TAG,
+              target_commitish: context.sha,
+              name: process.env.TAG,
+              generate_release_notes: true
+            })

.github/workflows/pr-labeler.yml → .github/workflows/pr-labeler.yaml

@@ -19,4 +19,4 @@ jobs:
       - uses: TimonVS/pr-labeler-action@v4
         with: 
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          congiruation-path: .github/pr-labeler.yml
+          configuration-path: .github/pr-labeler.yaml

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+repos:
+- repo: https://github.com/pycqa/isort
+  rev: 5.12.0
+  hooks:
+    - id: isort
+      args: ["--profile", "black"]
+- repo: https://github.com/ambv/black
+  rev: 23.3.0
+  hooks:
+    - id: black
+- repo: https://github.com/pycqa/flake8
+  rev: '6.1.0'
+  hooks:
+    - id: flake8
+      exclude: (tests|doc)
+      additional_dependencies: [
+          'flake8-docstrings',
+          'flake8-builtins',
+          'flake8-logging-format',
+          'flake8-rst-docstrings',
+          'pygments',
+          'pep8-naming'
+      ]
+default_language_version:
+    python: python3

.python-version

@@ -0,0 +1 @@
+3.9

S3MP/__init__.py

@@ -1,2 +1,4 @@
 """S3 MirrorPath package."""
-from ._version import __version__
+from ._version import __version__
+
+__all__ = ["__version__"]

S3MP/_version.py

@@ -1,3 +1,3 @@
 """Semantic versioning for S3MP."""
 
-__version__ = "0.5.2"
+__version__ = "0.5.3"

S3MP/async_utils.py

@@ -1,10 +1,13 @@
 """Asynchronous transfer utilities."""
-from S3MP.global_config import S3MPConfig
-import aioboto3
 import asyncio
 from typing import Coroutine, List
+
+import aioboto3
+
+from S3MP.global_config import S3MPConfig
 from S3MP.mirror_path import MirrorPath
 
+
 async def async_upload_from_mirror(mirror_path: MirrorPath):
     """Asynchronously upload a file from a MirrorPath."""
     session = aioboto3.Session()

S3MP/callbacks.py

@@ -1,13 +1,11 @@
-"""
-S3 callbacks to be used for boto3 transfers (uploads, downloads, and copies).
-"""
+"""S3 callbacks to be used for boto3 transfers (uploads, downloads, and copies)."""
 from pathlib import Path
-from S3MP.global_config import S3MPConfig
-import os
+
 import tqdm
 
+from S3MP.global_config import S3MPConfig
 from S3MP.mirror_path import MirrorPath
-from S3MP.types import SList, S3Resource
+from S3MP.types import S3Resource, SList
 
 
 class FileSizeTQDMCallback(tqdm.tqdm):
@@ -29,25 +27,31 @@ def __init__(
         :param is_download: Marker for upload/download transfer.
         """
         if transfer_objs is None:
-            return 
+            return
         if resource is None:
             resource = S3MPConfig.s3_resource
         if bucket_key is None:
             bucket_key = S3MPConfig.default_bucket_key
         if not isinstance(transfer_objs, list):
             transfer_objs = [transfer_objs]
-
 
         self._total_bytes = 0
         for transfer_mapping in transfer_objs:
             if is_download:
-                s3_key = transfer_mapping.s3_key if isinstance(transfer_mapping, MirrorPath) else transfer_mapping
+                s3_key = (
+                    transfer_mapping.s3_key
+                    if isinstance(transfer_mapping, MirrorPath)
+                    else transfer_mapping
+                )
                 self._total_bytes += resource.Object(bucket_key, s3_key).content_length
             else:
-                local_path = transfer_mapping.local_path if isinstance(transfer_mapping, MirrorPath) else transfer_mapping
+                local_path = (
+                    transfer_mapping.local_path
+                    if isinstance(transfer_mapping, MirrorPath)
+                    else transfer_mapping
+                )
                 self._total_bytes += local_path.stat().st_size
 
-
         transfer_str = "Download" if is_download else "Upload"
         super().__init__(
             self,

S3MP/global_config.py

@@ -1,11 +1,14 @@
 """Set global values for S3MP module."""
+import tempfile
 from configparser import ConfigParser
 from dataclasses import dataclass
 from pathlib import Path
-import tempfile
 from typing import Callable
+
 import boto3
-from S3MP.types import S3Client, S3Resource, S3Bucket, S3TransferConfig
+
+from S3MP.types import S3Bucket, S3Client, S3Resource, S3TransferConfig
+
 
 def get_config_file_path() -> Path:
     """Get the location of the config file."""
@@ -14,17 +17,21 @@ def get_config_file_path() -> Path:
 
 
 class Singleton(type):
-    # Singleton metaclass 
+    """Singleton metaclass."""
+
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
+        """Get instance of class."""
         if cls not in cls._instances:
             cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
+
 @dataclass
 class S3MPConfig(metaclass=Singleton):
     """Singleton class for S3MP globals."""
+
     # Boto3 Objects
     _s3_client: S3Client = None
     _s3_resource: S3Resource = None
@@ -45,14 +52,14 @@ def s3_client(self) -> S3Client:
         if not self._s3_client:
             self._s3_client = boto3.client("s3")
         return self._s3_client
-    
+
     @property
     def s3_resource(self) -> S3Resource:
         """Get S3 resource."""
         if not self._s3_resource:
             self._s3_resource = boto3.resource("s3")
         return self._s3_resource
-    
+
     @property
     def bucket(self, bucket_key: str = None) -> S3Bucket:
         """Get bucket."""
@@ -63,42 +70,44 @@ def bucket(self, bucket_key: str = None) -> S3Bucket:
                 raise ValueError("No default bucket key set.")
             self._bucket = self.s3_resource.Bucket(self.default_bucket_key)
         return self._bucket
-    
+
     @property
     def mirror_root(self) -> Path:
         """Get mirror root."""
         if self._mirror_root is None:
-            print("Mirror Root not set, a temporary directory will be used as the mirror root.")
+            print(
+                "Mirror Root not set, a temporary directory will be used as the mirror root."
+            )
             self._mirror_root = Path(tempfile.gettempdir())
         return self._mirror_root
-    
+
     def load_config(self, config_file_path: Path = None):
         """Load the config file."""
         config_file_path = config_file_path or get_config_file_path()
         config = ConfigParser()
         config.read(config_file_path)
-        
+
         if "DEFAULT" not in config:
-            return 
-        
+            return
+
         if "default_bucket_key" in config["DEFAULT"]:
             self.default_bucket_key = config["DEFAULT"]["default_bucket_key"]
-        
+
         if "mirror_root" in config["DEFAULT"]:
             self._mirror_root = Path(config["DEFAULT"]["mirror_root"])
-    
+
     def save_config(self, config_file_path: Path = None):
         """Write config file."""
         config_file_path = config_file_path or get_config_file_path()
         config = ConfigParser()
-        config['DEFAULT'] = {}
+        config["DEFAULT"] = {}
         if self.default_bucket_key:
-            config['DEFAULT']['default_bucket_key'] = self.default_bucket_key
+            config["DEFAULT"]["default_bucket_key"] = self.default_bucket_key
         if self._mirror_root:
-            config['DEFAULT']['mirror_root'] = str(self._mirror_root)
-        with open(config_file_path, 'w') as configfile:
+            config["DEFAULT"]["mirror_root"] = str(self._mirror_root)
+        with open(config_file_path, "w") as configfile:
             config.write(configfile)
 
 
-S3MPConfig = S3MPConfig() 
+S3MPConfig = S3MPConfig()
 S3MPConfig.load_config()