Add alternative provider retrieval check

deps.ts

@@ -34,3 +34,6 @@ export {
 export { assertOkResponse } from 'https://cdn.skypack.dev/[email protected]/?dts'
 import pRetry from 'https://cdn.skypack.dev/[email protected]/?dts'
 export { pRetry }
+
+import Prando from 'https://cdn.jsdelivr.net/npm/[email protected]/+esm'
+export { Prando }

lib/ipni-client.js

@@ -1,12 +1,15 @@
 import { decodeBase64, decodeVarint, pRetry, assertOkResponse } from '../vendor/deno-deps.js'
 
+/** @typedef {{ address: string; protocol: string; contextId: string; }} Provider  */
+
 /**
  *
  * @param {string} cid
  * @param {string} providerId
  * @returns {Promise<{
  *  indexerResult: string;
- *  provider?: { address: string; protocol: string };
+ *  provider?: Provider;
+ *  alternativeProviders?: Provider[];
  * }>}
  */
 export async function queryTheIndex(cid, providerId) {
@@ -31,9 +34,8 @@ export async function queryTheIndex(cid, providerId) {
   }
 
   let graphsyncProvider
+  const alternativeProviders = []
   for (const p of providerResults) {
-    if (p.Provider.ID !== providerId) continue
-
     const [protocolCode] = decodeVarint(decodeBase64(p.Metadata))
     const protocol = {
       0x900: 'bitswap',
@@ -45,22 +47,31 @@ export async function queryTheIndex(cid, providerId) {
     const address = p.Provider.Addrs[0]
     if (!address) continue
 
+    const provider = {
+      address: formatProviderAddress(p.Provider.ID, address, protocol),
+      contextId: p.ContextID,
+      protocol,
+    }
+
+    if (p.Provider.ID !== providerId) {
+      alternativeProviders.push(provider)
+      continue
+    }
+
     switch (protocol) {
       case 'http':
         return {
           indexerResult: 'OK',
-          provider: { address, protocol },
+          provider,
         }
 
       case 'graphsync':
         if (!graphsyncProvider) {
-          graphsyncProvider = {
-            address: `${address}/p2p/${p.Provider.ID}`,
-            protocol,
-          }
+          graphsyncProvider = provider
         }
     }
   }
+
   if (graphsyncProvider) {
     console.log('HTTP protocol is not advertised, falling back to Graphsync.')
     return {
@@ -70,7 +81,7 @@ export async function queryTheIndex(cid, providerId) {
   }
 
   console.log('All advertisements are from other miners or for unsupported protocols.')
-  return { indexerResult: 'NO_VALID_ADVERTISEMENT' }
+  return { indexerResult: 'NO_VALID_ADVERTISEMENT', alternativeProviders }
 }
 
 async function getRetrievalProviders(cid) {
@@ -81,3 +92,7 @@ async function getRetrievalProviders(cid) {
   const result = await res.json()
   return result.MultihashResults.flatMap((r) => r.ProviderResults)
 }
+
+function formatProviderAddress(id, address, protocol) {
+  return protocol === 'http' ? address : `${address}/p2p/${id}`
+}

lib/spark.js

@@ -1,5 +1,6 @@
 /* global Zinnia */
 
+/** @import { Provider } from './ipni-client.js' */
 import { ActivityState } from './activity-state.js'
 import {
   SPARK_VERSION,
@@ -18,6 +19,7 @@ import {
   CarBlockIterator,
   encodeHex,
   HashMismatchError,
+  Prando,
   UnsupportedHashError,
   validateBlock,
 } from '../vendor/deno-deps.js'
@@ -41,16 +43,17 @@ export default class Spark {
 
   async getRetrieval() {
     const retrieval = await this.#tasker.next()
-    if (retrieval) {
+    if (retrieval.retrievalTask) {
       console.log({ retrieval })
     }
+
     return retrieval
   }
 
-  async executeRetrievalCheck(retrieval, stats) {
-    console.log(`Calling Filecoin JSON-RPC to get PeerId of miner ${retrieval.minerId}`)
+  async executeRetrievalCheck({ retrievalTask, stats, randomness }) {
+    console.log(`Calling Filecoin JSON-RPC to get PeerId of miner ${retrievalTask.minerId}`)
     try {
-      const peerId = await this.#getIndexProviderPeerId(retrieval.minerId)
+      const peerId = await this.#getIndexProviderPeerId(retrievalTask.minerId)
       console.log(`Found peer id: ${peerId}`)
       stats.providerId = peerId
     } catch (err) {
@@ -70,19 +73,39 @@ export default class Spark {
       throw err
     }
 
-    console.log(`Querying IPNI to find retrieval providers for ${retrieval.cid}`)
-    const { indexerResult, provider } = await queryTheIndex(retrieval.cid, stats.providerId)
+    console.log(`Querying IPNI to find retrieval providers for ${retrievalTask.cid}`)
+    const { indexerResult, provider, alternativeProviders } = await queryTheIndex(
+      retrievalTask.cid,
+      stats.providerId,
+    )
     stats.indexerResult = indexerResult
 
     const providerFound = indexerResult === 'OK' || indexerResult === 'HTTP_NOT_ADVERTISED'
-    if (!providerFound) return
+    const noValidAdvertisement = indexerResult === 'NO_VALID_ADVERTISEMENT'
+
+    // In case index lookup failed we will not perform any retrieval
+    if (!providerFound && !noValidAdvertisement) return
+
+    // In case we fail to find a valid advertisement for the provider
+    // we will try to perform network wide retrieval from other providers
+    if (noValidAdvertisement) {
+      console.log(
+        'No valid advertisement found. Trying to retrieve from an alternative provider...',
+      )
+      stats.alternativeProviderCheck = await this.checkRetrievalFromAlternativeProvider({
+        alternativeProviders,
+        randomness,
+        cid: retrievalTask.cid,
+      })
+      return
+    }
 
     stats.protocol = provider.protocol
     stats.providerAddress = provider.address
 
-    await this.fetchCAR(provider.protocol, provider.address, retrieval.cid, stats)
+    await this.fetchCAR(provider.protocol, provider.address, retrievalTask.cid, stats)
     if (stats.protocol === 'http') {
-      await this.testHeadRequest(provider.address, retrieval.cid, stats)
+      await this.testHeadRequest(provider.address, retrievalTask.cid, stats)
     }
   }
 
@@ -202,6 +225,31 @@ export default class Spark {
     }
   }
 
+  async checkRetrievalFromAlternativeProvider({ alternativeProviders, randomness, cid }) {
+    if (!alternativeProviders.length) {
+      console.info('No alternative providers found for this CID.')
+      return
+    }
+
+    const randomProvider = pickRandomProvider(alternativeProviders, randomness)
+    if (!randomProvider) {
+      console.warn(
+        'No providers serving the content via HTTP or Graphsync found. Skipping network-wide retrieval check.',
+      )
+      return
+    }
+
+    const alternativeProviderRetrievalStats = newAlternativeProviderCheckStats()
+    await this.fetchCAR(
+      randomProvider.protocol,
+      randomProvider.address,
+      cid,
+      alternativeProviderRetrievalStats,
+    )
+
+    return alternativeProviderRetrievalStats
+  }
+
   async submitMeasurement(task, stats) {
     console.log('Submitting measurement...')
     const payload = {
@@ -228,17 +276,17 @@ export default class Spark {
   }
 
   async nextRetrieval() {
-    const retrieval = await this.getRetrieval()
-    if (!retrieval) {
+    const { retrievalTask, randomness } = await this.getRetrieval()
+    if (!retrievalTask) {
       console.log('Completed all tasks for the current round. Waiting for the next round to start.')
       return
     }
 
     const stats = newStats()
 
-    await this.executeRetrievalCheck(retrieval, stats)
+    await this.executeRetrievalCheck({ retrievalTask, randomness, stats })
 
-    const measurementId = await this.submitMeasurement(retrieval, { ...stats })
+    const measurementId = await this.submitMeasurement(retrievalTask, { ...stats })
     Zinnia.jobCompleted()
     return measurementId
   }
@@ -315,6 +363,17 @@ export function newStats() {
     carChecksum: null,
     statusCode: null,
     headStatusCode: null,
+    alternativeProviderCheck: null,
+  }
+}
+
+function newAlternativeProviderCheckStats() {
+  return {
+    statusCode: null,
+    timeout: false,
+    endAt: null,
+    carTooLarge: false,
+    providerId: null,
   }
 }
 
@@ -395,3 +454,62 @@ function mapErrorToStatusCode(err) {
   // Fallback code for unknown errors
   return 600
 }
+
+/**
+ * Picks a random provider based on their priority and supplied randomness.
+ *
+ * Providers are prioritized in the following order:
+ *
+ * 1. HTTP Providers with Piece Info advertised in their ContextID.
+ * 2. Graphsync Providers with Piece Info advertised in their ContextID.
+ * 3. HTTP Providers.
+ * 4. Graphsync Providers.
+ *
+ * @param {Provider[]} providers
+ * @param {number} randomness
+ * @returns {Provider | undefined}
+ */
+export function pickRandomProvider(providers, randomness) {
+  const rng = new Prando(randomness)
+
+  const filterByProtocol = (items, protocol) =>
+    items.filter((provider) => provider.protocol === protocol)
+
+  const pickRandomItem = (items) => {
+    if (!items.length) return undefined
+    return items[Math.floor(rng.next() * items.length)]
+  }
+
+  const providersWithPieceInfoContextID = providers.filter(
+    (p) => p.contextId.startsWith('ghsA') && p.protocol !== 'bitswap',
+  )
+
+  // Priority 1: HTTP providers with ContextID containing PieceCID
+  const httpProvidersWithPieceInfoContextID = filterByProtocol(
+    providersWithPieceInfoContextID,
+    'http',
+  )
+  if (httpProvidersWithPieceInfoContextID.length) {
+    return pickRandomItem(httpProvidersWithPieceInfoContextID, randomness)
+  }
+
+  // Priority 2: Graphsync providers with ContextID containing PieceCID
+  const graphsyncProvidersWithPieceInfoContextID = filterByProtocol(
+    providersWithPieceInfoContextID,
+    'graphsync',
+  )
+  if (graphsyncProvidersWithPieceInfoContextID.length) {
+    return pickRandomItem(graphsyncProvidersWithPieceInfoContextID, randomness)
+  }
+
+  // Priority 3: HTTP providers
+  const httpProviders = filterByProtocol(providers, 'http')
+  if (httpProviders.length) return pickRandomItem(httpProviders, randomness)
+
+  // Priority 4: Graphsync providers
+  const graphsyncProviders = filterByProtocol(providers, 'graphsync')
+  if (graphsyncProviders.length) return pickRandomItem(graphsyncProviders, randomness)
+
+  // No valid providers found
+  return undefined
+}

lib/tasker.js

@@ -15,6 +15,7 @@ export class Tasker {
   #remainingRoundTasks
   #fetch
   #activity
+  #randomness
 
   /**
    * @param {object} args
@@ -35,11 +36,12 @@ export class Tasker {
   }
 
   /**
-   * @returns {Task | undefined}
+   * @returns {{retrievalTask?: RetrievalTask; randomness: number; }}
    */
   async next() {
     await this.#updateCurrentRound()
-    return this.#remainingRoundTasks.pop()
+    const retrievalTask = this.#remainingRoundTasks.pop()
+    return { retrievalTask, randomness: this.#randomness }
   }
 
   async #updateCurrentRound() {
@@ -72,13 +74,13 @@ export class Tasker {
     console.log('  %s retrieval tasks', retrievalTasks.length)
     this.maxTasksPerRound = maxTasksPerNode
 
-    const randomness = await getRandomnessForSparkRound(round.startEpoch)
-    console.log('  randomness: %s', randomness)
+    this.#randomness = await getRandomnessForSparkRound(round.startEpoch)
+    console.log('  randomness: %s', this.#randomness)
 
     this.#remainingRoundTasks = await pickTasksForNode({
       tasks: retrievalTasks,
       maxTasksPerRound: this.maxTasksPerRound,
-      randomness,
+      randomness: this.#randomness,
       stationId: Zinnia.stationId,
     })
 

manual-check.js

@@ -7,7 +7,7 @@ import Spark, { getRetrievalUrl } from './lib/spark.js'
 import { getIndexProviderPeerId as defaultGetIndexProvider } from './lib/miner-info.js'
 
 // The task to check, replace with your own values
-const task = {
+const retrievalTask = {
   cid: 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq',
   minerId: 'f0frisbii',
 }
@@ -19,8 +19,8 @@ const getIndexProviderPeerId = (minerId) =>
 
 // Run the check
 const spark = new Spark({ getIndexProviderPeerId })
-const stats = { ...task, indexerResult: null, statusCode: null, byteLength: 0 }
-await spark.executeRetrievalCheck(task, stats)
+const stats = { ...retrievalTask, indexerResult: null, statusCode: null, byteLength: 0 }
+await spark.executeRetrievalCheck({ retrievalTask, stats })
 console.log('Measurement: %o', stats)
 
 if (stats.providerAddress && stats.statusCode !== 200) {
@@ -31,15 +31,15 @@ if (stats.providerAddress && stats.statusCode !== 200) {
       console.log(
         '  lassie fetch -o /dev/null -vv --dag-scope block --protocols graphsync --providers %s %s',
         JSON.stringify(stats.providerAddress),
-        task.cid,
+        retrievalTask.cid,
       )
       console.log(
         '\nHow to install Lassie: https://github.com/filecoin-project/lassie?tab=readme-ov-file#installation',
       )
       break
     case 'http':
       try {
-        const url = getRetrievalUrl(stats.protocol, stats.providerAddress, task.cid)
+        const url = getRetrievalUrl(stats.protocol, stats.providerAddress, retrievalTask.cid)
         console.log('You can get more details by requesting the following URL yourself:\n')
         console.log('  %s', url)
         console.log('\nE.g. using `curl`:')
@@ -48,7 +48,7 @@ if (stats.providerAddress && stats.statusCode !== 200) {
         console.log(
           '  lassie fetch -o /dev/null -vv --dag-scope block --protocols http --providers %s %s',
           JSON.stringify(stats.providerAddress),
-          task.cid,
+          retrievalTask.cid,
         )
         console.log(
           '\nHow to install Lassie: https://github.com/filecoin-project/lassie?tab=readme-ov-file#installation',