Use moto-server to create a local S3 service. (#39)

* Use moto-server to create a local S3 service.

* Use local_s3 for github ci

* Disable warnings - this code is better.

.github/workflows/smoke-test.yml

@@ -37,12 +37,15 @@ jobs:
     - name: List dependencies
       run: |
         pip list
-    - name: Run ABFS unit tests with pytest
+    - name: Run local s3 with pytest
       run: |
-        python -m pytest tests --cloud abfs
-      env:
-        ABFS_LINCCDATA_ACCOUNT_NAME: ${{ secrets.LINCC_ABFS_ACCOUNT_NAME }}
-        ABFS_LINCCDATA_ACCOUNT_KEY: ${{ secrets.LINCC_ABFS_ACCOUNT_KEY }}
+        python -m pytest tests --cloud local_s3
+    # - name: Run ABFS unit tests with pytest
+    #   run: |
+    #     python -m pytest tests --cloud abfs
+    #   env:
+    #     ABFS_LINCCDATA_ACCOUNT_NAME: ${{ secrets.LINCC_ABFS_ACCOUNT_NAME }}
+    #     ABFS_LINCCDATA_ACCOUNT_KEY: ${{ secrets.LINCC_ABFS_ACCOUNT_KEY }}
     - name: Send status to Slack app (LSDB CI Reporter)
       if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
       id: slack

.github/workflows/testing-and-coverage.yml

@@ -29,9 +29,12 @@ jobs:
         python -m pip install --upgrade pip
         pip install -e .[dev]
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Run ABFS unit tests with pytest
+    - name: Run local s3 with pytest
       run: |
-        python -m pytest tests --cloud abfs
-      env:
-        ABFS_LINCCDATA_ACCOUNT_NAME: ${{ secrets.LINCC_ABFS_ACCOUNT_NAME }}
-        ABFS_LINCCDATA_ACCOUNT_KEY: ${{ secrets.LINCC_ABFS_ACCOUNT_KEY }}
+        python -m pytest tests --cloud local_s3
+    # - name: Run ABFS unit tests with pytest
+    #   run: |
+    #     python -m pytest tests --cloud abfs
+    #   env:
+    #     ABFS_LINCCDATA_ACCOUNT_NAME: ${{ secrets.LINCC_ABFS_ACCOUNT_NAME }}
+    #     ABFS_LINCCDATA_ACCOUNT_KEY: ${{ secrets.LINCC_ABFS_ACCOUNT_KEY }}

pyproject.toml

@@ -18,6 +18,8 @@ requires-python = ">=3.9"
 dependencies = [
     # lsdb, hipscat, and hipscat-import are built from source via requirements.txt
     "adlfs",
+    "s3fs",
+    "moto[server]",
     "shortuuid",
 ]
 

src/hipscat_cloudtests/temp_cloud_directory.py

@@ -13,18 +13,21 @@ class TempCloudDirectory:
 
     On exit, we will recursively remove the created directory."""
 
-    def __init__(self, prefix_path, method_name=""):
+    def __init__(self, prefix_path, method_name="", real_directories=True):
         """Create a new context manager.
 
         This will NOT create the new temp path - that happens when we enter the context.
 
         Args:
             prefix_path (UPath): base path to the cloud resource
             method_name (str): optional token to indicate the method under test
+            real_directories (bool): are directories in this file system real, and
+                should be deleted along with other temp content?
         """
         self.prefix_path = prefix_path
         self.method_name = method_name
         self.temp_path = ""
+        self.real_directories = real_directories
 
     def __enter__(self):
         """Create a new temporary path
@@ -63,7 +66,10 @@ def close(self, num_retries=4):
             for attempt_number in range(1, num_retries + 1):
                 ## Try
                 try:
-                    file_io.remove_directory(self.temp_path)
+                    if self.real_directories:
+                        file_io.remove_directory(self.temp_path)
+                    else:
+                        _try_remove_contents(self.temp_path)
                     return
                 except RuntimeError:
                     if attempt_number == num_retries:
@@ -72,3 +78,11 @@ def close(self, num_retries=4):
                 print(f"Failed to remove directory {self.temp_path}. Trying again.")
                 time.sleep(sleep_time)
                 sleep_time *= 2
+
+
+def _try_remove_contents(directory):
+    for item in directory.iterdir():
+        if item.is_dir():
+            _try_remove_contents(item)
+        else:
+            item.unlink()

tests/cloud/data/small_sky/catalog_info.json

@@ -0,0 +1,8 @@
+{
+    "catalog_name": "small_sky",
+    "catalog_type": "object",
+    "total_rows": 131,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec"
+}

tests/cloud/data/small_sky/partition_info.csv

@@ -0,0 +1,2 @@
+Norder,Npix,Dir
+0,11,0

tests/cloud/data/small_sky/provenance_info.json

@@ -0,0 +1,57 @@
+{
+    "catalog_name": "small_sky",
+    "catalog_type": "object",
+    "total_rows": 131,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec",
+    "version": "0.3.10.dev5+g4357f0c",
+    "generation_date": "2024.10.03",
+    "tool_args": {
+        "tool_name": "hipscat_import",
+        "version": "0.3.6.dev16+g829fe47",
+        "runtime_args": {
+            "catalog_name": "small_sky",
+            "output_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data",
+            "output_artifact_name": "small_sky",
+            "tmp_dir": "/tmp/tmpghnadxlo",
+            "dask_tmp": null,
+            "dask_n_workers": 1,
+            "dask_threads_per_worker": 1,
+            "catalog_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data/small_sky",
+            "tmp_path": "/tmp/tmpghnadxlo/small_sky/intermediate",
+            "epoch": "J2000",
+            "catalog_type": "object",
+            "input_path": "small_sky_parts",
+            "input_paths": [
+                "small_sky_parts/catalog_00_of_05.csv",
+                "small_sky_parts/catalog_01_of_05.csv",
+                "small_sky_parts/catalog_02_of_05.csv",
+                "small_sky_parts/catalog_03_of_05.csv",
+                "small_sky_parts/catalog_04_of_05.csv"
+            ],
+            "input_file_list": [],
+            "ra_column": "ra",
+            "dec_column": "dec",
+            "use_hipscat_index": false,
+            "sort_columns": null,
+            "constant_healpix_order": -1,
+            "lowest_healpix_order": 0,
+            "highest_healpix_order": 1,
+            "pixel_threshold": 1000000,
+            "mapping_healpix_order": 1,
+            "debug_stats_only": false,
+            "file_reader_info": {
+                "input_reader_type": "CsvReader",
+                "chunksize": 500000,
+                "header": "infer",
+                "schema_file": null,
+                "column_names": null,
+                "type_map": null,
+                "parquet_kwargs": null,
+                "upath_kwargs": null,
+                "kwargs": {}
+            }
+        }
+    }
+}

tests/cloud/data/small_sky_object_index/catalog_info.json

@@ -0,0 +1,8 @@
+{
+    "catalog_name": "small_sky_object_index",
+    "catalog_type": "index",
+    "total_rows": 131,
+    "primary_catalog": "small_sky_order1",
+    "indexing_column": "id",
+    "extra_columns": []
+}

tests/cloud/data/small_sky_object_index/provenance_info.json

@@ -0,0 +1,31 @@
+{
+    "catalog_name": "small_sky_object_index",
+    "catalog_type": "index",
+    "total_rows": 131,
+    "primary_catalog": "small_sky_order1",
+    "indexing_column": "id",
+    "extra_columns": [],
+    "version": "0.3.10.dev5+g4357f0c",
+    "generation_date": "2024.10.03",
+    "tool_args": {
+        "tool_name": "hipscat_import",
+        "version": "0.3.6.dev16+g829fe47",
+        "runtime_args": {
+            "catalog_name": "small_sky_object_index",
+            "output_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data",
+            "output_artifact_name": "small_sky_object_index",
+            "tmp_dir": "/tmp/tmp52o7hhz0",
+            "dask_tmp": null,
+            "dask_n_workers": 1,
+            "dask_threads_per_worker": 1,
+            "catalog_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data/small_sky_object_index",
+            "tmp_path": "/tmp/tmp52o7hhz0/small_sky_object_index/intermediate",
+            "input_catalog_path": "small_sky_order1",
+            "indexing_column": "id",
+            "extra_columns": [],
+            "include_hipscat_index": true,
+            "include_order_pixel": true,
+            "include_radec": false
+        }
+    }
+}

tests/cloud/data/small_sky_order1/catalog_info.json

@@ -0,0 +1,8 @@
+{
+    "catalog_name": "small_sky_order1",
+    "catalog_type": "object",
+    "total_rows": 131,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec"
+}

tests/cloud/data/small_sky_order1/partition_info.csv

@@ -0,0 +1,5 @@
+Norder,Npix,Dir
+1,44,0
+1,45,0
+1,46,0
+1,47,0

tests/cloud/data/small_sky_order1/provenance_info.json

@@ -0,0 +1,57 @@
+{
+    "catalog_name": "small_sky_order1",
+    "catalog_type": "object",
+    "total_rows": 131,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec",
+    "version": "0.3.10.dev5+g4357f0c",
+    "generation_date": "2024.10.03",
+    "tool_args": {
+        "tool_name": "hipscat_import",
+        "version": "0.3.6.dev16+g829fe47",
+        "runtime_args": {
+            "catalog_name": "small_sky_order1",
+            "output_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data",
+            "output_artifact_name": "small_sky_order1",
+            "tmp_dir": "/tmp/tmpv5r7yuuc",
+            "dask_tmp": null,
+            "dask_n_workers": 1,
+            "dask_threads_per_worker": 1,
+            "catalog_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data/small_sky_order1",
+            "tmp_path": "/tmp/tmpv5r7yuuc/small_sky_order1/intermediate",
+            "epoch": "J2000",
+            "catalog_type": "object",
+            "input_path": "small_sky_parts",
+            "input_paths": [
+                "small_sky_parts/catalog_00_of_05.csv",
+                "small_sky_parts/catalog_01_of_05.csv",
+                "small_sky_parts/catalog_02_of_05.csv",
+                "small_sky_parts/catalog_03_of_05.csv",
+                "small_sky_parts/catalog_04_of_05.csv"
+            ],
+            "input_file_list": [],
+            "ra_column": "ra",
+            "dec_column": "dec",
+            "use_hipscat_index": false,
+            "sort_columns": null,
+            "constant_healpix_order": 1,
+            "lowest_healpix_order": 0,
+            "highest_healpix_order": 10,
+            "pixel_threshold": 1000000,
+            "mapping_healpix_order": 1,
+            "debug_stats_only": false,
+            "file_reader_info": {
+                "input_reader_type": "CsvReader",
+                "chunksize": 500000,
+                "header": "infer",
+                "schema_file": null,
+                "column_names": null,
+                "type_map": null,
+                "parquet_kwargs": null,
+                "upath_kwargs": null,
+                "kwargs": {}
+            }
+        }
+    }
+}

tests/cloud/data/small_sky_order1_margin/catalog_info.json

@@ -0,0 +1,10 @@
+{
+    "catalog_name": "small_sky_order1_margin",
+    "catalog_type": "margin",
+    "total_rows": 28,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec",
+    "primary_catalog": "small_sky_order1",
+    "margin_threshold": 7200
+}

tests/cloud/data/small_sky_order1_margin/partition_info.csv

@@ -0,0 +1,6 @@
+Norder,Npix,Dir
+0,4,0
+1,44,0
+1,45,0
+1,46,0
+1,47,0

tests/cloud/data/small_sky_order1_margin/provenance_info.json

@@ -0,0 +1,31 @@
+{
+    "catalog_name": "small_sky_order1_margin",
+    "catalog_type": "margin",
+    "total_rows": 28,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec",
+    "primary_catalog": "small_sky_order1",
+    "margin_threshold": 7200,
+    "version": "0.3.10.dev5+g4357f0c",
+    "generation_date": "2024.10.03",
+    "tool_args": {
+        "tool_name": "hipscat_import",
+        "version": "0.3.6.dev16+g829fe47",
+        "runtime_args": {
+            "catalog_name": "small_sky_order1_margin",
+            "output_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data",
+            "output_artifact_name": "small_sky_order1_margin",
+            "tmp_dir": "/tmp/tmpsr8rjc0i",
+            "dask_tmp": null,
+            "dask_n_workers": 1,
+            "dask_threads_per_worker": 1,
+            "catalog_path": "/home/delucchi/git/s3/hipscat-cloudtests/tests/cloud/data/small_sky_order1_margin",
+            "tmp_path": "/tmp/tmpsr8rjc0i/small_sky_order1_margin/intermediate",
+            "input_catalog_path": "small_sky_order1",
+            "margin_threshold": 7200,
+            "margin_order": 4,
+            "debug_filter_pixel_list": []
+        }
+    }
+}

tests/cloud/data/small_sky_xmatch/catalog_info.json

@@ -0,0 +1,8 @@
+{
+    "catalog_name": "small_sky_xmatch",
+    "catalog_type": "object",
+    "total_rows": 111,
+    "epoch": "J2000",
+    "ra_column": "ra",
+    "dec_column": "dec"
+}

tests/cloud/data/small_sky_xmatch/partition_info.csv

@@ -0,0 +1,4 @@
+Norder,Npix,Dir
+1,44,0
+1,45,0
+1,46,0