add storage block

.github/workflows/docker.yml

@@ -0,0 +1,121 @@
+name: Build additional barge modules
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+
+env:
+  DOCKERHUB_IMAGE: ${{ 'oceanprotocol/barge' }}
+  GHCR_IMAGE: ${{ 'ghcr.io/oceanprotocol/barge' }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # Only run when not from dependabot and when the Dockerfile for this matrix component exists
+    if: ${{ github.actor != 'dependabot[bot]'}}
+    strategy:
+      fail-fast: false
+      matrix:
+        # we keep this just in case we need to change
+        platform: ${{ github.event_name == 'pull_request' && fromJSON('["linux/amd64"]') || fromJSON('["linux/amd64"]') }}
+        component:
+          - folder: "./additional-modules/storage/"
+            tag: "-storage"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: ${{ matrix.platform }}
+          #image: tonistiigi/binfmt:qemu-v8.0.4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: ${{ matrix.platform }}
+      - name: Login to Docker Hub
+        id: dockerhub_login
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKER_PUSH_TOKEN }}
+        if: env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != ''
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
+          password: ${{ secrets.DOCKER_PUSH_TOKEN }}
+      - name: Login to GitHub Container Registry
+        id: ghcr_login
+        env:
+          GHCR_PUSH_TOKEN: ${{ secrets.GHCR_PUSH_TOKEN }}
+        if: env.GHCR_PUSH_TOKEN != ''
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GHCR_PUSH_TOKEN }}
+      - name: Process tag
+        id: process_tag
+        run: |
+          TAG="${{ matrix.component.tag }}"
+          PROCESSED_TAG="${TAG:1}"
+          echo "processed_tag=${PROCESSED_TAG}" >> $GITHUB_OUTPUT
+      - name: Set Docker metadata
+        id: ocean_node_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
+          # generate Docker tags based on the following events/attributes
+          # we only build main branch and PRs
+          tags: |
+            type=ref,event=pr,suffix=${{ matrix.component.tag }},
+            type=raw,value=${{ steps.process_tag.outputs.processed_tag }}
+
+          # type=semver,pattern={{major}}.{{minor}}
+          # type=semver,pattern={{major}}
+          # type=sha
+      - name: Prepare image tags
+        id: image_tags
+        run: |
+          TAGS="${{ steps.ocean_node_meta.outputs.tags }}"
+          echo "dockerhub<<EOF" >> $GITHUB_OUTPUT
+          echo "$TAGS" | grep "^${{ env.DOCKERHUB_IMAGE }}:" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          echo "ghcr<<EOF" >> $GITHUB_OUTPUT
+          echo "$TAGS" | grep "^${{ env.GHCR_IMAGE }}:" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+      - name: Build and push to Docker Hub
+        if: steps.dockerhub_login.outcome == 'success'
+        id: build_dockerhub
+        uses: docker/build-push-action@v5
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: ${{ matrix.component.folder }}
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ${{ steps.image_tags.outputs.dockerhub }}
+          labels: ${{ steps.ocean_node_meta.outputs.labels }}
+      - name: Build and push to GHCR
+        if: steps.ghcr_login.outcome == 'success'
+        id: build_ghcr
+        uses: docker/build-push-action@v5
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: ${{ matrix.component.folder }}
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ${{ steps.image_tags.outputs.ghcr }}
+          labels: ${{ steps.ocean_node_meta.outputs.labels }}

additional-modules/storage/Dockerfile

@@ -0,0 +1,69 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# ----------------------------------------------------
+# Install packages
+# ----------------------------------------------------
+RUN apt-get update && \
+    apt-get install -y \
+        apache2 \
+        vsftpd \
+        ceph \
+        ceph-mds \
+        ceph-mgr \
+        ceph-mon \
+        ceph-osd \
+        radosgw \
+        supervisor \
+        uuid-runtime \
+        dnsutils \
+        curl \
+        jq && \
+    apt-get clean
+
+# ----------------------------------------------------
+# Apache config
+# ----------------------------------------------------
+RUN a2enmod rewrite
+
+# ----------------------------------------------------
+# FTP config
+# ----------------------------------------------------
+RUN sed -i 's/anonymous_enable=NO/anonymous_enable=YES/' /etc/vsftpd.conf && \
+    sed -i 's/#write_enable=YES/write_enable=YES/' /etc/vsftpd.conf
+
+# ----------------------------------------------------
+# Ceph directories
+# ----------------------------------------------------
+RUN mkdir -p /etc/ceph /var/lib/ceph/mon/ceph-a /var/lib/ceph/osd/ceph-0 /var/lib/ceph/mgr/ceph-a
+
+# ----------------------------------------------------
+# Copy bootstrap script
+# ----------------------------------------------------
+COPY bootstrap-ceph.sh /usr/local/bin/bootstrap-ceph.sh
+RUN chmod +x /usr/local/bin/bootstrap-ceph.sh
+
+# ----------------------------------------------------
+# Supervisor config
+# ----------------------------------------------------
+RUN mkdir -p /var/log/supervisor
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# ----------------------------------------------------
+# Expose ports
+# ----------------------------------------------------
+# Apache
+EXPOSE 80       
+# FTP data
+EXPOSE 20        
+# FTP control
+EXPOSE 21
+# Ceph MON
+EXPOSE 6789
+# Ceph OSD/MGR
+EXPOSE 6800-6900
+# RGW S3 + Swift
+EXPOSE 7480 
+
+CMD ["/usr/local/bin/bootstrap-ceph.sh"]

additional-modules/storage/bootstrap-ceph.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+set -e
+
+BOOTSTRAP_MARKER=/var/lib/ceph/.bootstrapped
+
+if [ -f "$BOOTSTRAP_MARKER" ]; then
+  echo "Ceph already bootstrapped, starting supervisor..."
+  exec /usr/bin/supervisord -n
+fi
+
+echo "Bootstrapping Ceph..."
+
+FSID=$(uuidgen)
+HOST_IP=127.0.0.1
+
+# ----------------------------------------------------
+# Required runtime directories
+# ----------------------------------------------------
+mkdir -p /var/run/ceph
+mkdir -p /var/lib/ceph/tmp
+chown -R ceph:ceph /var/run/ceph /var/lib/ceph/tmp
+
+# ----------------------------------------------------
+# Create ceph.conf
+# ----------------------------------------------------
+cat <<EOF >/etc/ceph/ceph.conf
+[global]
+fsid = $FSID
+mon initial members = a
+mon host = $HOST_IP
+public network = 0.0.0.0/0
+cluster network = 0.0.0.0/0
+osd objectstore = bluestore
+EOF
+
+# ----------------------------------------------------
+# Create monmap
+# ----------------------------------------------------
+echo "Creating monmap..."
+monmaptool --create --add a $HOST_IP --fsid $FSID --clobber /etc/ceph/monmap
+
+# ----------------------------------------------------
+# Create keyrings
+# ----------------------------------------------------
+echo "Creating keyrings..."
+ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin
+ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon.
+ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+
+# ----------------------------------------------------
+# Initialize MON
+# ----------------------------------------------------
+echo "Initializing mon..."
+ceph-mon --mkfs -i a --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
+chown -R ceph:ceph /var/lib/ceph/mon/ceph-a
+
+# ----------------------------------------------------
+# Initialize OSD
+# ----------------------------------------------------
+echo "Initializing OSD..."
+OSD_UUID=$(uuidgen)
+
+ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-0/keyring --gen-key -n osd.0
+ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /var/lib/ceph/osd/ceph-0/keyring
+
+cp /etc/ceph/ceph.conf /var/lib/ceph/osd/ceph-0/
+cp /etc/ceph/monmap /var/lib/ceph/osd/ceph-0/
+
+chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
+
+echo "Mkfs OSD..."
+sudo -u ceph ceph-osd -i 0 --mkfs --osd-uuid $OSD_UUID --no-mon-config
+
+# ----------------------------------------------------
+# Initialize MGR
+# ----------------------------------------------------
+echo "Initialize MGR..."
+ceph-authtool --create-keyring /var/lib/ceph/mgr/ceph-a/keyring --gen-key -n mgr.a
+ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /var/lib/ceph/mgr/ceph-a/keyring
+chown -R ceph:ceph /var/lib/ceph/mgr/ceph-a
+
+# ----------------------------------------------------
+# Start MON + MGR + OSD temporarily to create pools
+# ----------------------------------------------------
+echo "Starting MON/MGR/OSD temporarily for pool creation..."
+ceph-mon -i a --foreground &
+sleep 3
+ceph-mgr -i a &
+sleep 3
+ceph-osd -i 0 &
+sleep 5
+
+# ----------------------------------------------------
+# Create RGW pools
+# ----------------------------------------------------
+echo "Creating RGW pools..."
+ceph osd pool create default.rgw.meta 1
+ceph osd pool create default.rgw.log 1
+ceph osd pool create default.rgw.control 1
+ceph osd pool create default.rgw.buckets.data 1
+ceph osd pool create default.rgw.buckets.index 1
+
+# ----------------------------------------------------
+# Create default S3 user
+# ----------------------------------------------------
+echo "Creating default S3 user..."
+
+ACCESS_KEY="ocean123"
+SECRET_KEY="ocean123secret"
+
+radosgw-admin user create \
+  --uid="ocean" \
+  --display-name="Ocean Test User" \
+  --access-key="$ACCESS_KEY" \
+  --secret-key="$SECRET_KEY" \
+  >/tmp/s3-user.json
+
+# ----------------------------------------------------
+# Create test bucket
+# ----------------------------------------------------
+echo "Creating test bucket..."
+radosgw-admin bucket create --bucket="test-bucket" --uid="ocean"
+
+# ----------------------------------------------------
+# Print credentials
+# ----------------------------------------------------
+echo "==============================================="
+echo " Default S3 Credentials"
+echo "-----------------------------------------------"
+echo " Access Key: $ACCESS_KEY"
+echo " Secret Key: $SECRET_KEY"
+echo " Endpoint:   http://localhost:7480"
+echo " Bucket:     test-bucket"
+echo "==============================================="
+
+# ----------------------------------------------------
+# Stop temporary daemons
+# ----------------------------------------------------
+killall ceph-mon || true
+killall ceph-mgr || true
+killall ceph-osd || true
+sleep 2
+
+# ----------------------------------------------------
+# Mark bootstrap complete and start supervisor
+# ----------------------------------------------------
+touch "$BOOTSTRAP_MARKER"
+echo "Starting supervisor..."
+exec /usr/bin/supervisord -n

additional-modules/storage/supervisord.conf

@@ -0,0 +1,22 @@
+[supervisord]
+nodaemon=true
+
+[program:apache2]
+command=/usr/sbin/apachectl -D FOREGROUND
+autorestart=true
+
+[program:vsftpd]
+command=/usr/sbin/vsftpd /etc/vsftpd.conf
+autorestart=true
+
+[program:ceph-mon]
+command=/usr/bin/ceph-mon -i a --foreground
+autorestart=true
+
+[program:ceph-mgr]
+command=/usr/bin/ceph-mgr -i a
+autorestart=true
+
+[program:ceph-osd]
+command=/usr/bin/ceph-osd -i 0
+autorestart=true

cleanup.sh

@@ -8,6 +8,7 @@ docker container stop ocean-ganache-1
 docker container stop ocean-faucet-1
 docker container stop ocean-dashboard-1
 docker container stop docker-registry
+docker container stop ocean-storage-1
 
 docker container rm ocean-node-1
 docker container rm ocean-ipfs-1
@@ -18,11 +19,15 @@ docker container rm ocean-ganache-1
 docker container rm ocean-faucet-1
 docker container rm ocean-dashboard-1
 docker container rm docker-registry
+docker container rm ocean-storage-1
 
 docker volume rm ocean-graphipfs
 docker volume rm ocean-graphpgsql
 docker volume rm ocean-provider1db
 docker volume rm ocean-provider2db
+docker volume rm ocean_ceph_data
+docker volume rm ocean_ftp
+docker volume rm ocean_www
 
 docker network rm ocean_backend
 docker volume rm $(docker volume ls -q)