Merge pull request #35 from dClimate/add-authentication

Add authenticated IPFS RPC connections to IPFSStore

Author: abidsikder

.github/workflows/run-checks.yaml

@@ -25,6 +25,71 @@ jobs:
           run_daemon: true
         id: ipfs_setup
 
+      - name: Install and configure Nginx
+        run: |
+          # Install Nginx
+          sudo apt-get update
+          sudo apt-get install -y nginx
+
+          # Create Nginx config for reverse proxy with auth
+          cat <<EOF | sudo tee /etc/nginx/sites-available/ipfs
+          server {
+              listen 5002;
+              server_name localhost;
+
+              # Default deny unless authenticated
+              set \$auth_valid 0;
+
+              location /api/v0/ {
+                  # Enforce X-API-Key for API key auth
+                  if (\$http_x_api_key = "test") {
+                      set \$auth_valid 1;
+                  }
+
+                  # Check Bearer token
+                  if (\$http_authorization = "Bearer test") {
+                      set \$auth_valid 1;
+                  }
+
+                  # Check Basic Auth (test:test = dGVzdDp0ZXN0)
+                  if (\$http_authorization = "Basic dGVzdDp0ZXN0") {
+                      set \$auth_valid 1;
+                  }
+
+                  # Deny if no valid auth method
+                  if (\$auth_valid = 0) {
+                      return 401 "Unauthorized: Invalid or missing authentication";
+                  }
+
+                  # Proxy to IPFS RPC API
+                  proxy_pass http://127.0.0.1:5001;
+                  proxy_set_header Host \$host;
+                  proxy_set_header X-Real-IP \$remote_addr;
+                  proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+                  proxy_set_header X-Forwarded-Proto \$scheme;
+              }
+          }
+          EOF
+
+          # Enable the site and remove default
+          sudo ln -s /etc/nginx/sites-available/ipfs /etc/nginx/sites-enabled/
+          sudo rm -f /etc/nginx/sites-enabled/default
+
+          # Test Nginx config
+          sudo nginx -t
+     
+      - name: Start Nginx and restart IPFS daemon
+        run: |
+          # Start Nginx
+          sudo systemctl start nginx
+
+          # Restart IPFS daemon to ensure it’s running
+          ipfs shutdown
+          ipfs daemon &
+
+          # Wait for IPFS and Nginx to be ready
+          sleep 5
+
       - name: Run pytest with coverage
         run: uv run pytest --cov=py_hamt tests/ --cov-report=xml
 

py_hamt/store.py

@@ -66,6 +66,11 @@ class IPFSStore(Store):
     Use IPFS as a backing store for a HAMT. The IDs returned from save and used by load are IPFS CIDs.
 
     Save methods use the RPC API but `load` uses the HTTP Gateway, so read-only HAMTs will only access the HTTP Gateway. This allows for connection to remote gateways as well.
+
+    You can write to an authenticated IPFS node by providing credentials in the constructor. The following authentication methods are supported:
+    - Basic Authentication: Provide a tuple of (username, password) to the `basic_auth` parameter.
+    - Bearer Token: Provide a bearer token to the `bearer_token` parameter.
+    - API Key: Provide an API key to the `api_key` parameter. You can customize the header name for the API key by setting the `api_key_header` parameter.
     """
 
     def __init__(
@@ -76,6 +81,11 @@ def __init__(
         hasher: str = "blake3",
         pin_on_add: bool = False,
         debug: bool = False,
+        # Authentication parameters
+        basic_auth: tuple[str, str] | None = None,  # (username, password)
+        bearer_token: str | None = None,
+        api_key: str | None = None,
+        api_key_header: str = "X-API-Key",  # Customizable API key header
     ):
         self.timeout_seconds = timeout_seconds
         """
@@ -100,6 +110,16 @@ def __init__(
         self.total_received: None | int = 0 if debug else None
         """Total bytes in responses from IPFS for blocks. Used for debugging purposes."""
 
+        # Authentication settings
+        self.basic_auth = basic_auth
+        """Tuple of (username, password) for Basic Authentication"""
+        self.bearer_token = bearer_token
+        """Bearer token for token-based authentication"""
+        self.api_key = api_key
+        """API key for API key-based authentication"""
+        self.api_key_header = api_key_header
+        """Header name to use for API key authentication"""
+
     def save(self, data: bytes, cid_codec: str) -> CID:
         """
         This saves the data to an ipfs daemon by calling the RPC API, and then returns the CID, with a multicodec set by the input cid_codec. We need to do this since the API always returns either a multicodec of raw or dag-pb if it had to shard the input data.
@@ -116,9 +136,23 @@ def save(self, data: bytes, cid_codec: str) -> CID:
         """
         pin_string: str = "true" if self.pin_on_add else "false"
 
+        # Apply authentication based on provided credentials
+        headers = {}
+        if self.bearer_token:
+            headers["Authorization"] = f"Bearer {self.bearer_token}"
+        elif self.api_key:
+            headers[self.api_key_header] = self.api_key
+
+        # Prepare request parameters
+        url = f"{self.rpc_uri_stem}/api/v0/add?hash={self.hasher}&pin={pin_string}"
+
+        # Make the request with appropriate authentication
         response = requests.post(
-            f"{self.rpc_uri_stem}/api/v0/add?hash={self.hasher}&pin={pin_string}",
+            url,
             files={"file": data},
+            headers=headers,
+            auth=self.basic_auth,
+            timeout=self.timeout_seconds,
         )
         response.raise_for_status()
 

tests/test_zarr_ipfs.py

@@ -12,7 +12,7 @@
 from py_hamt import HAMT, IPFSStore, create_zarr_encryption_transformers
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def random_zarr_dataset():
     """Creates a random xarray Dataset and saves it to a temporary zarr store.
 
@@ -137,3 +137,43 @@ def test_encryption(random_zarr_dataset: tuple[str, xr.Dataset]):
     # We should be unable to read precipitation values which are still encrypted
     with pytest.raises(Exception):
         ds.precip.sum()
+
+
+# This test assumes the other IPFSStore zarr ipfs tests are working fine, so if other things are breaking check those first
+def test_authenticated_gateway(random_zarr_dataset: tuple[str, xr.Dataset]):
+    zarr_path, test_ds = random_zarr_dataset
+
+    def write_and_check(store: IPFSStore) -> bool:
+        try:
+            store.rpc_uri_stem = "http://127.0.0.1:5002"  # 5002 is the port configured in the run-checks.yaml actions file for nginx to serve the proxy on
+            hamt = HAMT(store=store)
+            test_ds.to_zarr(store=hamt, mode="w")
+            loaded_ds = xr.open_zarr(store=hamt)
+            xr.testing.assert_identical(test_ds, loaded_ds)
+            return True
+        except Exception as _:
+            return False
+
+    # Test with API Key
+    api_key_store = IPFSStore(api_key="test")
+    assert write_and_check(api_key_store)
+
+    # Test that wrong API Key fails
+    bad_api_key_store = IPFSStore(api_key="badKey")
+    assert not write_and_check(bad_api_key_store)
+
+    # Test just bearer token
+    bearer_ipfs_store = IPFSStore(bearer_token="test")
+    assert write_and_check(bearer_ipfs_store)
+
+    # Test with wrong bearer
+    bad_bearer_store = IPFSStore(bearer_token="wrongBearer")
+    assert not write_and_check(bad_bearer_store)
+
+    # Test with just basic auth
+    basic_auth_store = IPFSStore(basic_auth=("test", "test"))
+    assert write_and_check(basic_auth_store)
+
+    # Test with wrong basic auth
+    bad_basic_auth_store = IPFSStore(basic_auth=("wrong", "wrong"))
+    assert not write_and_check(bad_basic_auth_store)