Initial commit

Author: Kahtaf

.github/workflows/build-and-release.yml

@@ -0,0 +1,72 @@
+name: Build and Release
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          load: true
+          tags: |
+            refiner:${{ github.run_number }}
+            refiner:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Export image to file
+        run: |
+          docker save refiner:latest | gzip > refiner-${{ github.run_number }}.tar.gz
+
+      - name: Generate release body
+        run: |
+          echo "Image SHA256: $(sha256sum refiner-${{ github.run_number }}.tar.gz | cut -d' ' -f1)" >> release_body.txt
+
+      - name: Upload image
+        uses: actions/upload-artifact@v4
+        with:
+          name: refiner-image
+          path: refiner-${{ github.run_number }}.tar.gz
+
+      - name: Create Release and Upload Assets
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ github.run_number }}
+          name: Release v${{ github.run_number }}
+          body_path: release_body.txt
+          draft: false
+          prerelease: false
+          files: |
+            ./refiner-${{ github.run_number }}.tar.gz
+
+      - name: Log build result
+        if: always()
+        run: |
+          if [ ${{ job.status }} == "success" ]; then
+            echo "Build and release completed successfully"
+          else
+            echo "Build and release failed"
+          fi

.gitignore

@@ -0,0 +1,11 @@
+*.pem
+*.tar.gz
+__pycache__/
+*.pyc
+
+/input/user.json
+
+/output/db.libsql
+
+.idea/
+.DS_Store

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY . /app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+CMD ["python", "-m", "refiner"]

LICENSE

@@ -0,0 +1,8 @@
+The MIT License (MIT)
+Copyright © 2024 Corsali, Inc. dba Vana
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,127 @@
+# Vana Satya Proof of Contribution - Python Template
+
+This repository serves as a template for creating a [proof of contribution](https://docs.vana.org/docs/proof-of-contribution/) tasks using Python. It is executed on Vana's Satya Network, a group of highly confidential and secure compute nodes that can validate data without revealing its contents to the node operator.
+
+## Overview
+
+This template provides a basic structure for building proof tasks that:
+
+1. Read input files from the `/input` directory.
+2. Process the data securely, running any necessary validations to prove the data authentic, unique, high quality, etc.
+3. Write proof results to the `/output/results.json` file in the following format:
+
+```json
+{
+  "dlp_id": 1234, // DLP ID is found in the Root Network contract after the DLP is registered
+  "valid": false, // A single boolean to summarize if the file is considered valid in this DLP
+  "score": 0.7614457831325301, // A score between 0 and 1 for the file, used to determine how valuable the file is. This can be an aggregation of the individual scores below.
+  "authenticity": 1.0, // A score between 0 and 1 to rate if the file has been tampered with
+  "ownership": 1.0, // A score between 0 and 1 to verify the ownership of the file
+  "quality": 0.6024096385542169, // A score between 0 and 1 to show the quality of the file
+  "uniqueness": 0, // A score between 0 and 1 to show unique the file is, compared to others in the DLP
+  "attributes": { // Custom attributes added to the proof to provide extra context, written offchain in IPFS
+    "total_score": 0.5,
+    "score_threshold": 0.83,
+    "email_verified": true
+  },
+  "metadata": { // Custom attributes added to the proof to provide extra context, written onchain
+    "dlp_id": 1234
+  }
+}
+```
+
+The project is designed to work with Intel TDX (Trust Domain Extensions), providing hardware-level isolation and security guarantees for confidential computing workloads.
+
+## Project Structure
+
+- `my_proof/`: Contains the main proof logic
+    - `proof.py`: Implements the proof generation logic
+    - `__main__.py`: Entry point for the proof execution
+    - `models/`: Data models for the proof system
+- `input/`: Contains a sample input file for testing. In production, the Satya node will mount the decrypted data point to the /input directory of the docker container
+- `output/`: Results output from the container is written here, this is passed to the Satya node after execution
+- `Dockerfile`: Defines the container image for the proof task
+- `requirements.txt`: Python package dependencies
+
+## Getting Started
+
+To use this template:
+
+1. Fork this repository
+2. Modify the `my_proof/proof.py` file to implement your specific proof logic
+3. Update the project dependencies in `requirements.txt` if needed
+4. Commit your changes and push to your repository
+
+## Customizing the Proof Logic
+
+The main proof logic is implemented in `my_proof/proof.py`. To customize it, update the `Proof.generate()` function to change how input files are processed.
+
+The proof can be configured using environment variables, eg:
+
+- `USER_EMAIL`: The email address of the data contributor, to verify data ownership
+
+If you want to use a language other than Python, you can modify the Dockerfile to install the necessary dependencies and build the proof task in the desired language.
+
+## Local Development
+
+To run the proof locally for testing, you can use Docker:
+
+```bash
+docker build -t refiner .
+docker run \
+  --rm \
+  --volume $(pwd)/input:/input \
+  --volume $(pwd)/output:/output \
+  refiner
+```
+
+## Running with Intel TDX
+
+Intel TDX (Trust Domain Extensions) provides hardware-based memory encryption and integrity protection for virtual machines. To run this container in a TDX-enabled environment, follow your infrastructure provider's specific instructions for deploying confidential containers.
+
+Common volume mounts and environment variables:
+
+```bash
+docker run \
+  --rm \
+  --volume /path/to/input:/input \
+  --volume /path/to/output:/output \
+  --env [email protected] \
+  my-proof
+```
+
+Remember to populate the `/input` directory with the files you want to process.
+
+## Security Features
+
+This template leverages several security features:
+
+1. **Hardware-based Isolation**: The proof runs inside a TDX-protected environment, isolating it from the rest of the system
+2. **Input/Output Isolation**: Input and output directories are mounted separately, ensuring clear data flow boundaries
+3. **Minimal Container**: Uses a minimal Python base image to reduce attack surface
+
+## Customization
+
+Feel free to modify any part of this template to fit your specific needs. The goal is to provide a starting point that can be easily adapted to various proof tasks.
+
+## Contributing
+
+If you have suggestions for improving this template, please open an issue or submit a pull request.
+
+## License
+
+[MIT License](LICENSE)
+
+
+
+- raw data in /input directory
+- encrypt with DLP public key
+- 
+- upload refined data to Pinata
+- /output
+  - schema.json
+  - ipfs URL
+  - data.db
+  - data.db.encrypted -> encrypted w/ derived encryption key
+
+

input/user.zip

@@ -0,0 +1,60 @@
+import json
+import logging
+import os
+import sys
+import traceback
+import zipfile
+from typing import Dict, Any
+
+from refiner.refine import Refiner
+
+INPUT_DIR, OUTPUT_DIR = '/input', '/output'
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+
+def load_config() -> Dict[str, Any]:
+    """Load refinement configuration from environment variables."""
+    config = {
+        'input_dir': INPUT_DIR,
+        'output_dir': OUTPUT_DIR,
+    }
+    logging.info(f"Using config: {json.dumps(config, indent=2)}")
+    return config
+
+
+def run() -> None:
+    """Transform all input files into the database."""
+    config = load_config()
+    input_files_exist = os.path.isdir(INPUT_DIR) and bool(os.listdir(INPUT_DIR))
+
+    if not input_files_exist:
+        raise FileNotFoundError(f"No input files found in {INPUT_DIR}")
+    extract_input()
+
+    refiner = Refiner(config)
+    refiner.transform()
+    
+    logging.info("Data transformation complete")
+
+
+def extract_input() -> None:
+    """
+    If the input directory contains any zip files, extract them
+    :return:
+    """
+    for input_filename in os.listdir(INPUT_DIR):
+        input_file = os.path.join(INPUT_DIR, input_filename)
+
+        if zipfile.is_zipfile(input_file):
+            with zipfile.ZipFile(input_file, 'r') as zip_ref:
+                zip_ref.extractall(INPUT_DIR)
+
+
+if __name__ == "__main__":
+    try:
+        run()
+    except Exception as e:
+        logging.error(f"Error during data transformation: {e}")
+        traceback.print_exc()
+        sys.exit(1)

refiner/__init__.py

@@ -0,0 +1,41 @@
+from datetime import datetime
+from sqlalchemy import Column, String, Integer, Float, ForeignKey, DateTime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+# Base model for SQLAlchemy
+Base = declarative_base()
+
+# Define database models - the schema is generated using these
+class UserRefined(Base):
+    __tablename__ = 'users'
+    
+    user_id = Column(String, primary_key=True)
+    email = Column(String, nullable=False, unique=True)
+    name = Column(String, nullable=False)
+    locale = Column(String, nullable=False)
+    created_at = Column(DateTime, nullable=False)
+    
+    storage_metrics = relationship("StorageMetric", back_populates="user")
+    auth_sources = relationship("AuthSource", back_populates="user")
+
+class StorageMetric(Base):
+    __tablename__ = 'storage_metrics'
+    
+    metric_id = Column(Integer, primary_key=True, autoincrement=True)
+    user_id = Column(String, ForeignKey('users.user_id'), nullable=False)
+    percent_used = Column(Float, nullable=False)
+    recorded_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    
+    user = relationship("UserRefined", back_populates="storage_metrics")
+
+class AuthSource(Base):
+    __tablename__ = 'auth_sources'
+    
+    auth_id = Column(Integer, primary_key=True, autoincrement=True)
+    user_id = Column(String, ForeignKey('users.user_id'), nullable=False)
+    source = Column(String, nullable=False)
+    collection_date = Column(DateTime, nullable=False)
+    data_type = Column(String, nullable=False)
+    
+    user = relationship("UserRefined", back_populates="auth_sources")

refiner/__main__.py

@@ -0,0 +1,23 @@
+from typing import Optional
+from pydantic import BaseModel
+
+
+class Profile(BaseModel):
+    name: str
+    locale: str
+
+class Storage(BaseModel):
+    percentUsed: float
+
+class Metadata(BaseModel):
+    source: str
+    collectionDate: str
+    dataType: str
+
+class User(BaseModel):
+    userId: str
+    email: str
+    timestamp: int
+    profile: Profile
+    storage: Optional[Storage] = None
+    metadata: Optional[Metadata] = None