fix: refactor to use gdoc api (#257)

Author: zekth

.github/workflows/build.yml

@@ -31,10 +31,13 @@ jobs:
       - name: install
         run: |
           yarn
+      - name: create-json secret
+        uses: jsdaniell/[email protected]
+        with:
+          name: "key.json"
+          json: ${{ secrets.GDOC_KEY }}
       - name: Generate db
         id: dbgen
-        env:
-          G_API_KEY: ${{ secrets.G_API_KEY }}
         run: |
           yarn gen
           yarn validate

.github/workflows/pr.yml

@@ -29,8 +29,11 @@ jobs:
       - name: lint
         run: |
           yarn lint
+      - name: create-json secret
+        uses: jsdaniell/[email protected]
+        with:
+          name: "key.json"
+          json: ${{ secrets.GDOC_KEY }}
       - name: Test generation
-        env:
-          G_API_KEY: ${{ secrets.G_API_KEY }}
         run: |
           node src/generate.js test

.github/workflows/pr2.yml

@@ -0,0 +1,37 @@
+name: Database Lint 2
+
+on:
+  pull_request:
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - id: nodeversion
+        run: echo "NODE=$(cat .nvmrc)" >> $GITHUB_OUTPUT
+      - name: restore Cache
+        uses: actions/cache@master
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+      - name: Use Node.js ${{steps.nodeversion.outputs.NODE}}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ steps.nodeversion.outputs.NODE }}
+      - name: install
+        run: |
+          yarn
+      - name: lint
+        run: |
+          yarn lint
+      - name: create-json secret
+        uses: jsdaniell/[email protected]
+        with:
+          name: "key.json"
+          json: ${{ secrets.GDOC_KEY }}
+      - name: Test generation
+        run: |
+          node src/generate.js test

.gitignore

@@ -4,3 +4,4 @@ package-lock.json
 SAVE_IMG/
 catalog_old.json
 dump
+key.json

package.json

@@ -20,13 +20,16 @@
     "standard": "^17.0.0"
   },
   "dependencies": {
+    "@googleapis/docs": "^2.0.1",
     "@mixmaxhq/promise-pool": "^2.0.0",
     "ajv": "^6.12.6",
     "axios": "^1.4.0",
     "crc": "^4.3.2",
     "csv-stringify": "^6.3.3",
     "googleapis": "118",
     "he": "^1.2.0",
+    "jsonpath": "^1.1.1",
+    "lodash": "^4.17.21",
     "node-html-parser": "^6.1.5",
     "rimraf": "^5.0.1",
     "sharp": "^0.32.1",

src/generate.js

@@ -25,7 +25,7 @@ async function moduleScrap (catalog, moduleName, isTest = false) {
   const destFile = path.join(DEST, formattedName)
   if (moduleCatalog.hasError !== true) {
     catalog.push(moduleCatalog)
-    fs.writeFileSync(destFile, JSON.stringify(moduleCatalog))
+    fs.writeFileSync(destFile, JSON.stringify(moduleCatalog, null, ' '))
   } else {
     // using the previous version of the file
     console.warn(`ERRORS: ${formattedName}`)
@@ -49,7 +49,7 @@ async function jsonScrap (catalog, filename, isTest = false) {
   const destFile = path.join(DEST, formattedName)
   if (moduleCatalog.hasError !== true) {
     catalog.push(moduleCatalog)
-    fs.writeFileSync(destFile, JSON.stringify(moduleCatalog))
+    fs.writeFileSync(destFile, JSON.stringify(moduleCatalog, null, ' '))
   } else {
     // using the previous version of the file
     console.warn(`ERRORS: ${formattedName}`)
@@ -96,13 +96,16 @@ async function generate (isTest = false, targetCat = undefined) {
   const pool = new PromisePool({ numConcurrent: 2 })
   const customScraps = fs.readdirSync(customImporterPath)
   const jsonScraps = fs.readdirSync(jsonImporterPath)
+  const total = customScraps.length + jsonScraps.length
+  let idx = 1
   for (const s of customScraps) {
     if (targetCat && !path.join(customImporterPath, s).endsWith(targetCat)) {
       continue
     }
     await pool.start(
       async (cat, filename) => {
         await moduleScrap(cat, path.join(customImporterPath, filename), isTest)
+        console.log(`${idx++}/${total}`)
       },
       catalog,
       s
@@ -115,6 +118,7 @@ async function generate (isTest = false, targetCat = undefined) {
     await pool.start(
       async (cat, filename) => {
         await jsonScrap(cat, path.join(jsonImporterPath, filename), isTest)
+        console.log(`${idx++}/${total}`)
       },
       catalog,
       s

src/google/index.js

@@ -0,0 +1,141 @@
+const docs = require('@googleapis/docs')
+const path = require('path')
+const jp = require('jsonpath')
+const { genId, getAttributes, attributes, getNationality, isSelfOrdered } = require('../utils')
+const _ = require('lodash')
+
+// get the credential client with the proper scopes
+async function getCredentials () {
+  const auth = new docs.auth.GoogleAuth({
+    keyFilename: path.join(__dirname, '..', '..', 'key.json'),
+    scopes: ['https://www.googleapis.com/auth/documents']
+  })
+  return auth.getClient()
+}
+
+// download the json google Doc representation
+async function downloadJsonDoc (id, credentials) {
+  const client = docs.docs({ version: 'v1', auth: credentials })
+  return client.documents.get({ documentId: id })
+}
+
+// get the google doc image URI from the kixId within the jsonDoc
+function getImgUrl (kixId, jsonDoc) {
+  const inlineObjectNode = jsonDoc.data.inlineObjects[kixId]
+  return inlineObjectNode.inlineObjectProperties.embeddedObject.imageProperties.contentUri
+}
+
+// parse the jsonDocument and hydrate the catalog object
+function gDocParse (catalog, jsonDoc) {
+  catalog.selfOrder = false
+  catalog.nationality = undefined
+
+  jp.query(jsonDoc.data.body.content, '$..textRun').forEach(x => {
+    const nat = getNationality(x.content)
+    if (nat) { catalog.nationality = nat }
+    if (isSelfOrdered(x.content)) { catalog.isSelfOrdered = true }
+  })
+
+  // parse all the tables
+  const tables = jp.query(jsonDoc, '$.data.body.content..table')
+  for (let i = 0; i < tables.length; i++) {
+    const odd = !!(i % 2)
+    // odd == colorway table
+    // even == sculpt title table
+    if (odd) {
+      const colorwayCells = jp.query(tables[i], '$..tableCells')
+      const flattenedCells = _.flatten(colorwayCells)
+      const currentSculptObj = catalog.sculpts[catalog.sculpts.length - 1]
+      for (const cell of flattenedCells) {
+        const imgId = jp.query(cell, '$..inlineObjectId')
+        if (!imgId.length) { continue }
+        const contentNodes = _.flatten(jp.query(cell, '$..content'))
+        let cellString = contentNodes.map(x => {
+          if (typeof x !== 'string') { return undefined }
+          return x.trim()
+        }).filter(Boolean).join(' ')
+        let isCover = false
+        const reCover = new RegExp(`\\(${attributes.cover}\\)`, 'gim')
+        if (reCover.test(cellString)) {
+          isCover = true
+          cellString = cellString.replace(reCover, '')
+        }
+
+        const regDate = /\(([a-zA-Z ]*\d{4})\)/gim
+        const dateMatch = regDate.exec(cellString)
+
+        const regCount = /\(count (\d+)\)/gim
+        const countMatch = regCount.exec(cellString)
+
+        const regComm = /\(\*\)/gim
+        const commMatch = regComm.exec(cellString)
+
+        const regGiveaway = /\(giveaway\)|\(give-away\)/gim
+        const giveawayMatch = regGiveaway.exec(cellString)
+        let releaseDate
+        let totalCount
+        let commissioned
+        let giveaway
+        if (dateMatch) {
+          // eslint-disable-next-line prefer-destructuring
+          releaseDate = dateMatch[1]
+          cellString = cellString.replace(regDate, '')
+        }
+        if (countMatch) {
+          // eslint-disable-next-line prefer-destructuring
+          totalCount = countMatch[1]
+          cellString = cellString.replace(regCount, '')
+        }
+        if (commMatch) {
+          commissioned = true
+          cellString = cellString.replace(regComm, '')
+        }
+        if (giveawayMatch) {
+          giveaway = true
+          cellString = cellString.replace(regGiveaway, '')
+        }
+        const sanitizedName = cellString.trim()
+        const imgKixId = imgId[0]
+        catalog.sculpts[catalog.sculpts.length - 1].colorways.push(
+          {
+            id: genId(`${catalog.name}-${currentSculptObj.name}-${sanitizedName}-${imgKixId}`),
+            img: getImgUrl(imgKixId, jsonDoc),
+            name: sanitizedName,
+            isCover,
+            releaseDate,
+            totalCount,
+            commissioned,
+            giveaway,
+            note: ''
+          }
+        )
+      }
+    } else {
+      const contentSculpt = jp.query(tables[i], '$..textRun..content')
+      if (contentSculpt.length !== 0) {
+        const s = contentSculpt[0].trim()
+        const fullContentSculpt = contentSculpt.join(' ')
+        const sculptName = s
+        const sculptDate = undefined
+        const attributes = getAttributes(fullContentSculpt)
+        catalog.sculpts.push({
+          id: genId(`${catalog.name}-${s}`),
+          name: sculptName,
+          releaseDate: sculptDate,
+          colorways: [],
+          ...attributes
+        })
+      } else {
+        break
+      }
+    }
+  }
+  catalog.sculpts = catalog.sculpts.filter(x => x.colorways.length)
+  return catalog
+}
+
+module.exports = {
+  getCredentials,
+  downloadJsonDoc,
+  gDocParse
+}

src/scraper/gdoc.js

@@ -1,5 +1,5 @@
-const htmlparser = require('node-html-parser')
-const { downloadFile, genId, gDriveParse, gDocUrl, isSelfOrdered, getNationality } = require('../utils')
+const { genId, gDocUrl } = require('../utils')
+const { gDocParse, getCredentials, downloadJsonDoc } = require('../google')
 
 function scrapFrom (gdocID, pMeta = {}, tabsOperations = []) {
   const meta = { ...pMeta }
@@ -15,30 +15,20 @@ function scrapFrom (gdocID, pMeta = {}, tabsOperations = []) {
 
   return async function scrap () {
     try {
-      const index = await downloadFile(gdocID)
-      const rootNode = htmlparser.parse(index)
-      const tabs = rootNode.querySelectorAll('table')
-      tabsOperations.forEach((tabOperation) => {
-        if (typeof tabOperation === 'function') {
-          tabOperation(tabs)
-        } else if (typeof tabOperation === 'string' && Array.prototype[tabOperation]) {
-          Array.prototype[tabOperation].call(tabs)
-        }
-      })
+      const creds = await getCredentials()
+      const jsonDoc = await downloadJsonDoc(gdocID, creds)
       const catalog = {
         src: gDocUrl(gdocID),
         id: '',
         name: '',
         instagram: '',
         website: '',
         discord: '',
-        nationality: getNationality(index),
-        selfOrder: isSelfOrdered(index),
         sculpts: [],
         ...meta
       }
       catalog.id = genId(meta.id || meta.name)
-      return gDriveParse(catalog, tabs)
+      return gDocParse(catalog, jsonDoc)
     } catch (e) {
       return {
         hasError: true,

src/utils.js

@@ -326,6 +326,7 @@ module.exports = {
   isSelfOrdered,
   getNationality,
   attributes,
+  getAttributes,
   sortBy,
   launcher,
   resize,