Support explicitly specifying the boto3utils3.s3 object (#92)

* Task.upload_item_assets_to_s3 and asset_io.upload_item_assets_to_s3 support explicitly specifying the boto3utils3.s3 object

Author: Phil Varner

Diff for: CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [unreleased] - TBD
+
+### Added
+
+- ([#92](https://github.com/stac-utils/stac-task/pull/92)) Task.upload_item_assets_to_s3 and asset_io.upload_item_assets_to_s3 support explicitly specifying the boto3utils3.s3 object.
+
 ## [v0.4.1] - 2024-03-06
 
 ### Fixed
@@ -65,7 +71,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Initial release.
 
-<!-- [unreleased]: <https://github.com/stac-utils/stac-task/compare/v0.4.1...main>-->
+[unreleased]: <https://github.com/stac-utils/stac-task/compare/v0.4.1...main>
 [v0.4.1]: <https://github.com/stac-utils/stac-task/compare/v0.4.0...v0.4.1>
 [v0.4.0]: <https://github.com/stac-utils/stac-task/compare/v0.3.0...v0.4.0>
 [v0.3.0]: <https://github.com/stac-utils/stac-task/compare/v0.2.0...v0.3.0>

Diff for: pyproject.toml

@@ -37,7 +37,7 @@ dev = [
     "pre-commit~=3.5.0",
     "pytest-cov~=4.1.0",
     "pytest~=8.0",
-    "ruff==0.3.0",
+    "ruff~=0.3.1",
     "types-setuptools~=69.0",
 ]
 
@@ -53,5 +53,5 @@ strict = true
 module = ["boto3utils", "jsonpath_ng.ext", "fsspec"]
 ignore_missing_imports = true
 
-[tool.ruff]
+[tool.ruff.lint]
 select = ["F", "E", "W", "I", "ERA", "RUF"]

Diff for: stactask/asset_io.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 
 # global dictionary of sessions per bucket
-s3_client = s3()
+global_s3_client = s3()
 
 SIMULTANEOUS_DOWNLOADS = int(os.getenv("STAC_SIMULTANEOUS_DOWNLOADS", 3))
 sem = asyncio.Semaphore(SIMULTANEOUS_DOWNLOADS)
@@ -104,6 +104,7 @@ def upload_item_assets_to_s3(
     path_template: str = "${collection}/${id}",
     s3_urls: bool = False,
     headers: Optional[Dict[str, Any]] = None,
+    s3_client: Optional[s3] = None,
     **kwargs: Any,
 ) -> Item:
     """Upload Item assets to s3 bucket
@@ -117,10 +118,16 @@ def upload_item_assets_to_s3(
         s3_urls (bool, optional): Return s3 URLs instead of http URLs. Defaults
             to False.
         headers (Dict, optional): Dictionary of headers to set on uploaded
-            assets. Defaults to {},
+            assets. Defaults to {}.
+        s3_client (boto3utils.s3, optional): Use this s3 object instead of the default
+            global one. Defaults to None.
     Returns:
         Dict: A new STAC Item with uploaded assets pointing to newly uploaded file URLs
     """
+
+    if s3_client is None:
+        s3_client = global_s3_client
+
     if headers is None:
         headers = {}
 
@@ -160,4 +167,5 @@ def upload_item_assets_to_s3(
             filename, url, public=public, extra=_headers, http_url=not s3_urls
         )
         _item["assets"][key]["href"] = url_out
+
     return Item.from_dict(_item)

Diff for: stactask/task.py

@@ -15,6 +15,7 @@
 from typing import Any, Callable, Dict, Iterable, List, Optional, Union
 
 import fsspec
+from boto3utils import s3
 from pystac import Item, ItemCollection
 
 from .asset_io import (
@@ -279,12 +280,17 @@ def download_items_assets(
         return list(items)
 
     def upload_item_assets_to_s3(
-        self, item: Item, assets: Optional[List[str]] = None
+        self,
+        item: Item,
+        assets: Optional[List[str]] = None,
+        s3_client: Optional[s3] = None,
     ) -> Item:
         if self._skip_upload:
             self.logger.warning("Skipping upload of new and modified assets")
             return item
-        item = upload_item_assets_to_s3(item, assets=assets, **self.upload_options)
+        item = upload_item_assets_to_s3(
+            item=item, assets=assets, s3_client=s3_client, **self.upload_options
+        )
         return item
 
     # this should be in PySTAC