DSperce integration

.devcontainer/devcontainer.json

@@ -33,7 +33,7 @@
     }
   },
   // Install ezkl cli, done here instead of the Dockerfile to test new versions without rebuilding the image.
-  "onCreateCommand": "curl https://raw.githubusercontent.com/zkonduit/ezkl/main/install_ezkl_cli.sh | bash -s v19.0.7",
+  "onCreateCommand": "curl https://raw.githubusercontent.com/zkonduit/ezkl/main/install_ezkl_cli.sh | bash -s v22.2.1",
   "postCreateCommand": "uv tool install bittensor-cli",
   "remoteEnv": {
     "PATH": "${containerEnv:PATH}:/home/vscode/.ezkl"

cspell.json

@@ -12,6 +12,9 @@
     "CIRCOM",
     "circuitized",
     "coldkey",
+    "dslice",
+    "dslices",
+    "dsperse",
     "dtype",
     "ezkl",
     "fastapi",
@@ -41,6 +44,7 @@
     "uvicorn",
     "venv",
     "wandb",
+    "winddown",
     "zkproof"
   ],
   "ignoreWords": [],

neurons/_miner/miner_session.py

@@ -23,10 +23,12 @@
     SINGLE_PROOF_OF_WEIGHTS_MODEL_ID,
 )
 from deployment_layer.circuit_store import circuit_store
+from execution_layer.dsperse_manager import DSperseManager
 from execution_layer.generic_input import GenericInput
 from execution_layer.verified_model_session import VerifiedModelSession
 from protocol import (
     Competition,
+    DSliceProofGenerationDataModel,
     ProofOfWeightsDataModel,
     QueryForCapacities,
     QueryZkProof,
@@ -48,6 +50,7 @@ def __init__(self):
         self.configure()
         self.check_register(should_exit=True)
         self.auto_update = AutoUpdate()
+        self.dsperse_manager = DSperseManager()
         self.log_batch = []
         self.shuffled_uids = None
         self.last_shuffle_epoch = -1
@@ -80,6 +83,10 @@ def start_server(self) -> bool:
         self.server.register_route(
             path=f"/{QueryForCapacities.name}", endpoint=self.handleCapacityRequest
         )
+        self.server.register_route(
+            path=f"/{DSliceProofGenerationDataModel.name}",
+            endpoint=self.handleDSliceRequest,
+        )
         self.server.start()
 
         existing_miner = self.metagraph.axons[self.subnet_uid]
@@ -466,6 +473,24 @@ def handleCompetitionRequest(self, data: Competition) -> JSONResponse:
                 status_code=500,
             )
 
+    def handleDSliceRequest(self, data: DSliceProofGenerationDataModel) -> JSONResponse:
+        """
+        Handle DSlice proof generation requests from validators.
+        """
+        bt.logging.info(
+            f"Handling DSlice proof generation request for slice_num={data.slice_num} run_uid={data.run_uid}"
+        )
+
+        result = self.dsperse_manager.prove_slice(
+            circuit_id=data.circuit,
+            slice_num=data.slice_num,
+            inputs=data.inputs,
+            outputs=data.outputs,
+        )
+
+        # Implementation for handling DSlice slice requests goes here
+        return JSONResponse(content=result, status_code=200)
+
     def queryZkProof(self, data: QueryZkProof) -> JSONResponse:
         """
         This function run proof generation of the model (with its output as well)

neurons/_validator/api/__init__.py

@@ -23,6 +23,7 @@
 import bittensor as bt
 from _validator.models.poc_rpc_request import ProofOfComputationRPCRequest
 from _validator.models.pow_rpc_request import ProofOfWeightsRPCRequest
+from _validator.models.base_rpc_request import QueuedRequestDataModel
 import hashlib
 from constants import (
     MAX_SIGNATURE_LIFESPAN,
@@ -70,9 +71,10 @@ def _should_rate_limit(ip: str):
 class ValidatorAPI:
     def __init__(self, config: ValidatorConfig):
         self.config = config
-        self.external_requests_queue: list[
-            ProofOfWeightsRPCRequest | ProofOfComputationRPCRequest
-        ] = []
+        # a Queue of requests to be sent to miners
+        # consists of "real world requests" ProofOfWeightsRPCRequest and ProofOfComputationRPCRequest
+        # and a Request with one slice of a DSperse model (DSlice)
+        self.stacked_requests_queue: list[QueuedRequestDataModel] = []
         self.ws_manager = WebSocketManager()
         self.recent_requests: dict[str, int] = {}
         self.validator_keys_cache = ValidatorKeysCache(config)
@@ -282,7 +284,7 @@ async def handle_proof_of_weights(
                 return InvalidParams(str(e))
 
             self.pending_requests[external_request.hash] = asyncio.Event()
-            self.external_requests_queue.insert(0, external_request)
+            self.stacked_requests_queue.insert(0, external_request)
             bt.logging.success(
                 f"External request with hash {external_request.hash} added to queue"
             )
@@ -341,7 +343,7 @@ async def handle_proof_of_computation(
                 return InvalidParams(str(e))
 
             self.pending_requests[external_request.hash] = asyncio.Event()
-            self.external_requests_queue.insert(0, external_request)
+            self.stacked_requests_queue.insert(0, external_request)
             bt.logging.success(
                 f"External request with hash {external_request.hash} added to queue"
             )

neurons/_validator/api/client.py

@@ -1,11 +1,11 @@
 import hashlib
 import json
 import time
-import traceback
 
 import bittensor as bt
 import httpx
 
+from _validator.models.miner_response import MinerResponse
 from _validator.core.request import Request
 from utils.signatures import Headers
 
@@ -14,44 +14,31 @@ async def query_miner(
     httpx_client: httpx.AsyncClient,
     request: Request,
     wallet: bt.wallet,
-) -> Request | None:
-    try:
-        # Use httpx.URL for safer URL construction
-        url = httpx.URL(
-            scheme="http",
-            host=request.ip,
-            port=request.port,
-            path=f"/{request.url_path.lstrip('/')}",
-        )
-        content = json.dumps(request.data)
-
-        headers = get_headers(request, content, wallet)
-
-        start_time = time.perf_counter()
-        response = await httpx_client.post(
-            url=url,
-            content=content,
-            timeout=request.circuit.timeout if request.circuit else None,
-            headers=headers,
-        )
-        response.raise_for_status()
-        end_time = time.perf_counter()
-
-        result = response.json()
-        request.response_time = end_time - start_time
-        request.deserialized = result
-        return request
-
-    except httpx.InvalidURL:
-        bt.logging.warning(
-            f"Ignoring UID as there is not a valid URL: {request.uid}. {request.ip}:{request.port}"
-        )
-        return None
-
-    except httpx.HTTPError as e:
-        bt.logging.warning(f"Failed to query miner for UID: {request.uid}. Error: {e}")
-        traceback.print_exc()
-        return None
+) -> MinerResponse:
+    # Use httpx.URL for safer URL construction
+    url = httpx.URL(
+        scheme="http",
+        host=request.ip,
+        port=request.port,
+        path=f"/{request.url_path.lstrip('/')}",
+    )
+    content = json.dumps(request.data)
+
+    headers = get_headers(request, content, wallet)
+
+    start_time = time.perf_counter()
+    response = await httpx_client.post(
+        url=url,
+        content=content,
+        timeout=request.circuit.timeout if request.circuit else None,
+        headers=headers,
+    )
+    response.raise_for_status()
+    end_time = time.perf_counter()
+
+    request.response_time = end_time - start_time
+
+    return MinerResponse.from_raw_response(request, response.json())
 
 
 def get_headers(request: Request, content: str, wallet: bt.wallet) -> dict:

neurons/_validator/core/exceptions.py

@@ -0,0 +1,28 @@
+"""Custom exceptions for validator core operations."""
+
+
+class ProofException(Exception):
+    """Base exception for proof-related errors."""
+
+    def __init__(self, uid: int, circuit: str, message: str = ""):
+        self.uid = uid
+        self.circuit = circuit
+        self.message = message
+        super().__init__(self.message)
+
+
+class EmptyProofException(ProofException):
+    """Raised when miner fails to provide a proof."""
+
+    def __init__(self, uid: int, circuit: str, raw_response: str | None = None):
+        self.raw_response = raw_response
+        message = f"Miner at UID {uid} failed to provide a valid proof for {circuit}."
+        super().__init__(uid, circuit, message)
+
+
+class IncorrectProofException(ProofException):
+    """Raised when proof verification fails."""
+
+    def __init__(self, uid: int, circuit: str):
+        message = f"Miner at UID {uid} provided an incorrect proof for {circuit}."
+        super().__init__(uid, circuit, message)

neurons/_validator/core/request.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Any
 
+from _validator.models.base_rpc_request import QueuedRequestDataModel
 from _validator.models.request_type import RequestType
 from execution_layer.circuit import Circuit
 from execution_layer.generic_input import GenericInput
@@ -22,7 +23,15 @@ class Request:
     circuit: Circuit | None = None
     data: dict[str, Any] | None = None
     inputs: GenericInput | None = None
-    request_hash: str | None = None
+    dsperse_slice_num: int | None = None
+    dsperse_run_uid: str | None = None
+    # next one is used only for rescheduling DSlice and RWR requests in case of failure
+    queued_request: QueuedRequestDataModel | None = None
+    # `external_request_hash` is the hash of the original request from external API user
+    # we use it to report back results to `ValidatorAPI`` class. It sends the results to the user.
+    external_request_hash: str | None = None
+    # `guard_hash` is the hash used by HashGuard to prevent duplicate requests
+    # It's calculated from the inputs of the request.
+    guard_hash: str | None = None
     response_time: float | None = None
-    deserialized: dict[str, object] | None = None
     save: bool = False