feat: upload encrypted results to IPFS

Author: Kahtaf

.gitignore

@@ -4,7 +4,8 @@ __pycache__/
 *.pyc
 
 /input/user.json
-/output
+/output/*
+!/output/.gitkeep
 .env
 
 .idea/

output/.gitkeep

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import sys
@@ -19,9 +20,12 @@ def run() -> None:
     extract_input()
 
     refiner = Refiner()
-    refiner.transform()
+    output = refiner.transform()
 
-    logging.info("Data transformation complete")
+    output_path = os.path.join(settings.OUTPUT_DIR, "output.json")
+    with open(output_path, 'w') as f:
+        json.dump(output.model_dump(), f, indent=2)    
+    logging.info(f"Data transformation complete: {output}")
 
 
 def extract_input() -> None:

refiner/__main__.py

@@ -1,7 +1,6 @@
-from typing import Optional
 from pydantic import BaseModel
 
-class Schema(BaseModel):
+class OffChainSchema(BaseModel):
     name: str
     version: str
     description: str

refiner/models/schema.py refiner/models/offchain_schema.pyrefiner/models/schema.py renamed to refiner/models/offchain_schema.py

@@ -0,0 +1,8 @@
+from typing import Optional
+from pydantic import BaseModel
+
+from refiner.models.offchain_schema import OffChainSchema
+
+class Output(BaseModel):
+    refinement_url: Optional[str] = None
+    schema: Optional[OffChainSchema] = None

refiner/models/output.py

@@ -2,18 +2,21 @@
 import logging
 import os
 
-from refiner.models.schema import Schema
+from refiner.models.offchain_schema import OffChainSchema
+from refiner.models.output import Output
 from refiner.transformer.user_transformer import UserTransformer
 from refiner.config import settings
-from refiner.utils.ipfs import upload_json_to_ipfs
+from refiner.utils.encrypt import encrypt_file
+from refiner.utils.ipfs import upload_file_to_ipfs, upload_json_to_ipfs
 
 class Refiner:
     def __init__(self):
         self.db_path = os.path.join(settings.OUTPUT_DIR, 'db.libsql')
 
-    def transform(self) -> None:
+    def transform(self) -> Output:
         """Transform all input files into the database."""
         logging.info("Starting data transformation")
+        output = Output()
 
         # Iterate through files and transform data
         for input_filename in os.listdir(settings.INPUT_DIR):
@@ -28,20 +31,28 @@ def transform(self) -> None:
                         transformer.process(input_data)
                         logging.info(f"Transformed {input_filename}")
 
-                        # Create a schema file
+                        # Create a schema based on the SQLAlchemy schema
+                        schema = OffChainSchema(
+                            name=settings.SCHEMA_NAME,
+                            version=settings.SCHEMA_VERSION,
+                            description=settings.SCHEMA_DESCRIPTION,
+                            dialect=settings.SCHEMA_DIALECT,
+                            schema=transformer.get_schema()
+                        )
+                        output.schema = schema
+                            
+                        # Upload the schema to IPFS
                         schema_file = os.path.join(settings.OUTPUT_DIR, 'schema.json')
                         with open(schema_file, 'w') as f:
-                            schema = Schema(
-                                name=settings.SCHEMA_NAME,
-                                version=settings.SCHEMA_VERSION,
-                                description=settings.SCHEMA_DESCRIPTION,
-                                dialect=settings.SCHEMA_DIALECT,
-                                schema=transformer.get_schema()
-                            )
                             json.dump(schema.model_dump(), f, indent=4)
-                            logging.info(f"Schema saved to {schema_file}")
-                            # ipfs_hash = upload_json_to_ipfs(schema.model_dump())
-                            # logging.info(f"Schema uploaded to IPFS with hash: {ipfs_hash}")
+                            schema_ipfs_hash = upload_json_to_ipfs(schema.model_dump())
+                            logging.info(f"Schema uploaded to IPFS with hash: {schema_ipfs_hash}")
+                        
+                        # Encrypt and upload the database to IPFS
+                        encrypted_path = encrypt_file(settings.REFINEMENT_ENCRYPTION_KEY, self.db_path)
+                        ipfs_hash = upload_file_to_ipfs(encrypted_path)
+                        output.refinement_url = f"https://ipfs.vana.org/ipfs/{ipfs_hash}"
                         continue
 
         logging.info("Data transformation completed successfully")
+        return output

refiner/refine.py

@@ -1,4 +1,5 @@
 pgpy
 pydantic
 pydantic_settings
+requests
 sqlalchemy