feat: estimated shiny probability for pokemon (#1133)

* feat: estimated shiny probability for pokemon
* feat: shiny since date
* feat: lazy load shiny rates

NB The database holding the spawnpoint table will be used to query pokemon_shiny_stats.

Author: Mygod

packages/locales/lib/human/en.json

@@ -377,7 +377,8 @@
   "with_ar": "With AR",
   "both": "Both",
   "without_ar": "Without AR",
-  "shiny_probability": "Shiny probability: <0/>",
+  "shiny_probability": "Shiny rate: <0/>",
+  "shiny_sample": "{{percentage}}%: {{shiny}} shiny/{{checks}} checks since {{date}}",
   "exclude_quest_multi": "Exclude {{reward}}",
   "cluster_limit_0": "{{variable_0}} limit ({{variable_1}}) has been hit",
   "cluster_limit_1": "Please zoom in or narrow your filters",

packages/types/lib/scanner.d.ts

@@ -41,6 +41,12 @@ export interface PokemonDisplay {
   location_card: number
 }
 
+export interface PokemonShinyStats {
+  shiny_seen: number
+  encounters_seen: number
+  since_date?: string
+}
+
 export interface Defender extends PokemonDisplay {
   pokemon_id: number
   deployed_ms: number

packages/types/lib/server.d.ts

@@ -46,6 +46,8 @@ export interface DbContext {
   hasShowcaseForm: boolean
   hasShowcaseType: boolean
   hasStationedGmax: boolean
+  hasPokemonShinyStats?: boolean
+  connection?: number
 }
 
 export interface ExpressUser extends User {

server/src/graphql/resolvers.js

@@ -24,6 +24,14 @@ const resolvers = {
   JSON: GraphQLJSON,
   Query: {
     available: (_, _args, { Event, Db, perms }) => {
+      const supportsShinyStats = Array.isArray(Db.models?.Pokemon)
+        ? Db.models.Pokemon.some(({ SubModel, ...ctx }) =>
+            typeof SubModel.supportsShinyStats === 'function'
+              ? SubModel.supportsShinyStats(ctx)
+              : false,
+          )
+        : false
+
       const data = {
         questConditions: perms.quests ? Db.questConditions : {},
         masterfile: { ...Event.masterfile, invasions: Event.invasions },
@@ -36,6 +44,7 @@ const resolvers = {
           ...config.getSafe('icons'),
           styles: Event.uicons,
         },
+        supportsShinyStats,
       }
       return data
     },
@@ -269,6 +278,23 @@ const resolvers = {
       }
       return {}
     },
+    pokemonShinyStats: async (_, args, { perms, Db }) => {
+      if (!perms?.pokemon) {
+        return null
+      }
+      const sources = Db.models?.Pokemon
+      if (!Array.isArray(sources)) {
+        return null
+      }
+      const results = await Promise.all(
+        sources.map(({ SubModel, ...ctx }) =>
+          typeof SubModel.getShinyStats === 'function'
+            ? SubModel.getShinyStats(perms, args, ctx)
+            : Promise.resolve(null),
+        ),
+      )
+      return results.find(Boolean) || null
+    },
     portals: (_, args, { perms, Db }) => {
       if (perms?.portals) {
         return Db.query('Portal', 'getAll', perms, args)

server/src/graphql/typeDefs/index.graphql

@@ -50,6 +50,7 @@ type Query {
     filters: JSON
   ): [Pokemon]
   pokemonSingle(id: ID, perm: String): Pokemon
+  pokemonShinyStats(pokemon_id: Int!, form: Int): PokemonShinyStats
   portals(
     minLat: Float
     maxLat: Float

server/src/graphql/typeDefs/map.graphql

@@ -4,6 +4,7 @@ type MapData {
   questConditions: JSON
   icons: JSON
   audio: JSON
+  supportsShinyStats: Boolean
 }
 
 type Badge {

server/src/graphql/typeDefs/scanner.graphql

@@ -148,6 +148,12 @@ type Pokestop {
   hasShowcase: Boolean
 }
 
+type PokemonShinyStats {
+  shiny_seen: Int
+  encounters_seen: Int
+  since_date: String
+}
+
 type Pokemon {
   id: ID
   encounter_id: Int

server/src/models/Pokemon.js

@@ -380,6 +380,7 @@ class Pokemon extends Model {
         finalResults.push(result)
       }
     }
+
     return finalResults
   }
 
@@ -436,6 +437,210 @@ class Pokemon extends Model {
     return results || []
   }
 
+  /**
+   * @param {number | null | undefined} preferredConnection
+   * @returns {import('knex').Knex | null}
+   */
+  static getStatsKnex(preferredConnection = null) {
+    return this.getStatsHandle(preferredConnection)?.knex ?? null
+  }
+
+  /**
+   * @param {number | null | undefined} preferredConnection
+   * @returns {{
+   *   knex: import('knex').Knex,
+   *   connection: number,
+   *   spawnSource?: import('@rm/types').DbContext & { connection: number },
+   * } | null}
+   */
+  static getStatsHandle(preferredConnection = null) {
+    const dbManager = state.db
+    if (!dbManager) return null
+    const { connections } = dbManager
+    if (!connections?.length) return null
+
+    const spawnSources = dbManager.models?.Spawnpoint
+
+    const getSpawnByConnection = (connection) =>
+      Array.isArray(spawnSources)
+        ? spawnSources.find((source) => source.connection === connection)
+        : undefined
+
+    const hasConnection = (connection) =>
+      typeof connection === 'number' && Boolean(connections?.[connection])
+
+    if (!Array.isArray(spawnSources) || !spawnSources.length) {
+      return null
+    }
+
+    let candidate = null
+
+    if (typeof preferredConnection === 'number') {
+      candidate = spawnSources.find(
+        (source) =>
+          source.connection === preferredConnection &&
+          source.hasPokemonShinyStats &&
+          hasConnection(source.connection),
+      )
+    }
+
+    if (!candidate) {
+      candidate = spawnSources.find(
+        (source) =>
+          source.hasPokemonShinyStats && hasConnection(source.connection),
+      )
+    }
+
+    if (!candidate) {
+      return null
+    }
+
+    const knexInstance = connections?.[candidate.connection]
+    if (!knexInstance) {
+      return null
+    }
+
+    return {
+      knex: knexInstance,
+      connection: candidate.connection,
+      spawnSource: getSpawnByConnection(candidate.connection),
+    }
+  }
+
+  /**
+   * @param {import("@rm/types").DbContext} ctx
+   * @returns {boolean}
+   */
+  static supportsShinyStats(ctx) {
+    const statsHandle = this.getStatsHandle(ctx?.connection)
+    if (!statsHandle?.knex) {
+      return false
+    }
+    const flag =
+      typeof statsHandle.spawnSource?.hasPokemonShinyStats === 'boolean'
+        ? statsHandle.spawnSource.hasPokemonShinyStats
+        : typeof ctx?.hasPokemonShinyStats === 'boolean'
+          ? ctx.hasPokemonShinyStats
+          : false
+    return flag
+  }
+
+  /**
+   * @param {string[]} keys
+   * @param {import('knex').Knex | null | undefined} [statsKnex]
+   * @param {number | null | undefined} [preferredConnection]
+   * @returns {Promise<Map<string, { shiny_seen: number, encounters_seen: number, since_date: string | null }>>}
+   */
+  static async fetchShinyStats(
+    keys,
+    statsKnex = null,
+    preferredConnection = null,
+  ) {
+    if (!keys.length) return new Map()
+
+    let knexInstance = statsKnex || null
+    if (!knexInstance) {
+      const statsHandle = this.getStatsHandle(preferredConnection)
+      knexInstance = statsHandle?.knex ?? null
+    }
+    if (!knexInstance) {
+      try {
+        knexInstance = this.knex()
+      } catch (e) {
+        knexInstance = null
+      }
+    }
+    if (!knexInstance) return new Map()
+
+    const pairs = keys
+      .map((key) => key.split('-'))
+      .map(([pokemonId, formId]) => {
+        const parsedPokemon = Number.parseInt(pokemonId, 10)
+        if (Number.isNaN(parsedPokemon)) return null
+        const parsedForm = Number.parseInt(formId, 10)
+        return [parsedPokemon, Number.isNaN(parsedForm) ? 0 : parsedForm]
+      })
+      .filter(Boolean)
+
+    if (!pairs.length) return new Map()
+
+    const whereClause = pairs
+      .map(() => '(pokemon_id = ? AND COALESCE(form_id, 0) = ?)')
+      .join(' OR ')
+    const bindings = pairs.flatMap(([pokemonId, formId]) => [pokemonId, formId])
+    const query = `
+      SELECT
+        pokemon_id,
+        COALESCE(form_id, 0) AS form_id,
+        date,
+        SUM(count) AS shiny,
+        SUM(total) AS checks
+      FROM pokemon_shiny_stats
+      WHERE area = 'world'
+        AND fence = 'world'
+        AND (${whereClause})
+        AND date >= CURRENT_DATE - INTERVAL 7 DAY
+      GROUP BY pokemon_id, form_id, date
+      ORDER BY pokemon_id, form_id, date DESC
+    `
+
+    const [rows] = await knexInstance.raw(query, bindings)
+
+    const grouped = new Map()
+    for (let i = 0; i < rows.length; i += 1) {
+      const row = rows[i]
+      const key = `${row.pokemon_id}-${row.form_id ?? 0}`
+      const entry = grouped.get(key)
+      const rowDate =
+        row.date instanceof Date
+          ? row.date.toISOString().slice(0, 10)
+          : `${row.date}`
+      const payload = {
+        shiny: Number(row.shiny) || 0,
+        checks: Number(row.checks) || 0,
+        date: rowDate,
+      }
+      if (entry) {
+        entry.push(payload)
+      } else {
+        grouped.set(key, [payload])
+      }
+    }
+
+    const statsMap = new Map()
+    const today = new Date()
+    today.setHours(0, 0, 0, 0)
+    const cutoff = new Date(today)
+    cutoff.setDate(cutoff.getDate() - 1)
+    const cutoffStr = cutoff.toISOString().slice(0, 10)
+
+    grouped.forEach((entries, key) => {
+      let shinySum = 0
+      let checkSum = 0
+      let sinceDate = null
+      for (let i = 0; i < entries.length; i += 1) {
+        const { shiny, checks, date } = entries[i]
+        const includeRecent = date >= cutoffStr
+        // 20000 checks would give >99% of distinguishing even 1/512 from 1/256
+        if (!includeRecent && checkSum >= 20000) {
+          break
+        }
+        shinySum += shiny
+        checkSum += checks
+        if (!sinceDate || date < sinceDate) {
+          sinceDate = date
+        }
+      }
+      statsMap.set(key, {
+        shiny_seen: shinySum,
+        encounters_seen: checkSum,
+        since_date: sinceDate,
+      })
+    })
+
+    return statsMap
+  }
+
   /**
    * @param {import("@rm/types").Permissions} perms
    * @param {object} args
@@ -512,12 +717,13 @@ class Pokemon extends Model {
       secret,
       httpAuth,
     )
-    return results
-      .filter(
-        (item) =>
-          !mem ||
-          filterRTree(item, perms.areaRestrictions, args.filters.onlyAreas),
-      )
+    const filtered = results.filter(
+      (item) =>
+        !mem ||
+        filterRTree(item, perms.areaRestrictions, args.filters.onlyAreas),
+    )
+
+    const built = filtered
       .map((item) => {
         const filter =
           filterMap[
@@ -536,6 +742,52 @@ class Pokemon extends Model {
           ] || globalFilter
         return filter.valid(pkmn)
       })
+
+    return built
+  }
+
+  /**
+   * @param {import("@rm/types").Permissions} _perms
+   * @param {{ pokemon_id: number, form?: number | null }} args
+   * @param {import("@rm/types").DbContext} ctx
+   * @returns {Promise<import("@rm/types").PokemonShinyStats | null>}
+   */
+  static async getShinyStats(_perms, args, ctx) {
+    const statsHandle = this.getStatsHandle(ctx?.connection)
+    if (!statsHandle?.knex) {
+      return null
+    }
+    const hasStats =
+      typeof statsHandle.spawnSource?.hasPokemonShinyStats === 'boolean'
+        ? statsHandle.spawnSource.hasPokemonShinyStats
+        : typeof ctx?.hasPokemonShinyStats === 'boolean'
+          ? ctx.hasPokemonShinyStats
+          : false
+    if (!hasStats) {
+      return null
+    }
+    const pokemonId = Number.parseInt(`${args.pokemon_id}`, 10)
+    if (Number.isNaN(pokemonId)) {
+      return null
+    }
+    const formId = Number.parseInt(`${args.form ?? 0}`, 10)
+    const key = `${pokemonId}-${Number.isNaN(formId) ? 0 : formId}`
+    try {
+      const stats = await this.fetchShinyStats(
+        [key],
+        statsHandle.knex,
+        statsHandle.connection,
+      )
+      return stats.get(key) || null
+    } catch (e) {
+      log.error(TAGS.pokemon, 'Failed to fetch shiny stats', e)
+      if (e?.code === 'ER_NO_SUCH_TABLE') {
+        if (statsHandle.spawnSource) {
+          statsHandle.spawnSource.hasPokemonShinyStats = false
+        }
+      }
+      return null
+    }
   }
 
   /**