Merge pull request #13 from mbelsky/playwright-draft

Implement Playwright scraper

Author: mbelsky

.dockerignore

@@ -33,3 +33,4 @@ Makefile
 **/cert.json
 **/.env*
 **/.nvmrc
+seccomp_profile.json

.github/workflows/build-push-docker.yml

@@ -9,10 +9,18 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      - name: Push to Docker Hub
+      - name: Push hltv-featured to Docker Hub
         uses: docker/[email protected]
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: ${{ secrets.DOCKER_USERNAME }}/hltv-featured
           tag_with_ref: true
+      - name: Push hltv-featured-playwright to Docker Hub
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: ${{ secrets.DOCKER_USERNAME }}/hltv-featured-playwright
+          build_args: NODEIMG=mcr.microsoft.com/playwright:v1.31.0-focal
+          tag_with_ref: true

.vscode/launch.json

@@ -22,6 +22,12 @@
       "name": "Launch Scraper",
       "program": "${workspaceFolder}/src/scraper/scraper.js"
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Playwright Scraper",
+      "program": "${workspaceFolder}/src/playwright-scraper/playwright-scraper.js"
+    },
     {
       "type": "node",
       "request": "launch",

Makefile

@@ -2,6 +2,7 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 COMMON_RUN_ARGS:=-it --env-file ./.env --env FIREBASE_CONFIG=/data/cert.json --mount type=bind,src=$(ROOT_DIR)/cert.json,dst=/data/cert.json,ro
 IMGV?=latest
 IMG:=mbelsky/hltv-featured:$(IMGV)
+PLAYWRIGHT_IMG:=mbelsky/hltv-featured-playwright:$(IMGV)
 
 .PHONY: build
 build:
@@ -22,3 +23,11 @@ bot:
 .PHONY: cron
 cron:
 		docker run --name=hltv-featured-cron $(COMMON_RUN_ARGS) --env CRON=true -d --restart on-failure:3 $(IMG)
+
+.PHONY: build-playwright
+build-playwright:
+		docker build --build-arg NODEIMG=mcr.microsoft.com/playwright:v1.31.0-focal -t $(PLAYWRIGHT_IMG) .
+
+.PHONY: playwright-scraper
+playwright-scraper:
+		docker run $(COMMON_RUN_ARGS) --rm --ipc=host --user pwuser --security-opt seccomp=seccomp_profile.json $(PLAYWRIGHT_IMG) node /app/src/playwright-scraper/playwright-scraper.js

README.md

@@ -7,6 +7,7 @@ This is a set of nodejs applications to get data about upcoming CSGO matches and
 - `src/backend` handles subscribers requests
 - `src/notifier` deliveries data to subscribers
 - `src/scraper` parses HLTV.org/matches
+- `src/playwright-scraper` same as scraper but uses playwright
 
 ## Configure server
 

crond/crontab

@@ -1,2 +1,2 @@
 0 5 * * * docker start hltv-featured-notifier
-55 4-21 * * * docker start hltv-featured-scraper
+55 4-21 * * * docker start hltv-featured-playwright-scraper

scripts/create-playwright-scraper-container.sh

@@ -0,0 +1,10 @@
+#! /bin/sh
+
+docker create \
+  --name=hltv-featured-playwright-scraper \
+  --env-file /???/.env \
+  --env FIREBASE_CONFIG=/data/cert.json \
+  --mount type=bind,src=/???/cert.json,dst=/data/cert.json,ro \
+  --ipc=host --user pwuser --security-opt seccomp=/???/seccomp_profile.json \
+  mbelsky/hltv-featured-playwright:v??? \
+  node /app/src/playwright-scraper/playwright-scraper.js

seccomp_profile.json

@@ -0,0 +1,8 @@
+{
+  "comment": "Allow create user namespaces",
+  "names": ["clone", "setns", "unshare"],
+  "action": "SCMP_ACT_ALLOW",
+  "args": [],
+  "includes": {},
+  "excludes": {}
+}

src/playwright-scraper/get-html.js

@@ -0,0 +1,78 @@
+const alerter = require('@hltvf/monitoring/alerter')
+const playwright = require('playwright-chromium')
+
+async function launchChromium() {
+  return await playwright.chromium.launch()
+}
+
+async function scrap(browser, url) {
+  const context = await browser.newContext({
+    acceptDownloads: false,
+    javaScriptEnabled: false,
+    locale: 'en-GB',
+  })
+
+  const page = await context.newPage()
+
+  page.setExtraHTTPHeaders({
+    accept:
+      'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'accept-language': 'en-US,en;q=0.9',
+    'cache-control': 'no-cache',
+    pragma: 'no-cache',
+    'sec-ch-ua':
+      '"Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"macOS"',
+    'sec-fetch-dest': 'document',
+    'sec-fetch-mode': 'navigate',
+    'sec-fetch-site': 'none',
+    'sec-fetch-user': '?1',
+    'upgrade-insecure-requests': '1',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36',
+  })
+
+  // Too lazy to remove these comments
+  // const requestPromise = page.waitForRequest(url, {
+  //   timeout: 5000,
+  // })
+  // const responsePromise = page.waitForResponse(url, {
+  //   timeout: 5000,
+  // })
+
+  // const [req, res] = await Promise.all([
+  //   requestPromise,
+  //   responsePromise,
+  //   page.goto(url),
+  // ])
+
+  // const resH = await res.allHeaders()
+  // const reqH = await req.allHeaders()
+  // const text = await res.text()
+
+  await page.goto(url)
+
+  const innerHtml = await page.content()
+  return innerHtml
+}
+
+async function getHtml(url) {
+  const browser = await launchChromium()
+
+  try {
+    const result = await scrap(browser, url)
+
+    return result
+  } finally {
+    try {
+      await browser.close()
+    } catch (e) {
+      alerter.error('Failed to close browser', e)
+    }
+  }
+}
+
+module.exports = {
+  getHtml,
+}

src/playwright-scraper/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "playwright-scraper",
+  "version": "0.1.0",
+  "main": "playwright-scraper.js",
+  "license": "GNU GPLv3",
+  "private": true,
+  "optionalDependencies": {
+    "playwright-chromium": "1.18.0"
+  }
+}

src/playwright-scraper/playwright-scraper.js

@@ -0,0 +1,69 @@
+const alerter = require('@hltvf/monitoring/alerter')
+const log = require('@hltvf/monitoring/logger').logFabric(
+  'requests-playwright-scraper',
+)
+const htmlToMatches = require('@hltvf/scraper/parseHtml')
+
+const { getTeams } = require('common/getTeams')
+const {
+  removeOutdatedMatches,
+  saveFeaturedMatches,
+} = require('common/manageMatches')
+const { addTeams } = require('common/manageTeams')
+const { getHtml } = require('./get-html')
+
+const root = process.env.ROOT_URL || 'https://www.hltv.org'
+const url = process.env.PAGE_URL || 'https://www.hltv.org/matches'
+
+const logResult = (data, matches) => {
+  if (!matches.length) {
+    alerter.warn('Zero matches scraped. HTML:\n\n' + data)
+  }
+
+  const message = `Scraped & saved ${matches.length} matches`
+  console.log(new Date().toUTCString(), message)
+
+  if ('production' === process.env.NODE_ENV) {
+    log({
+      level: 'info',
+      data: {
+        message,
+      },
+    })
+  }
+}
+
+function scrap() {
+  log({
+    level: 'info',
+    data: {
+      message: 'Playwright-scraper has started',
+    },
+  })
+
+  return removeOutdatedMatches()
+    .catch(alerter.error)
+    .then(() => getHtml(url))
+    .then(
+      (data) => {
+        const matches = htmlToMatches(data, { root })
+        const teams = getTeams({ alerter, matches })
+
+        const addTeamsPromise = addTeams(teams).catch(alerter.error)
+        const saveMatchesPromise = saveFeaturedMatches(matches)
+          .then(() => logResult(data, matches))
+          .catch(alerter.error)
+
+        return Promise.all([addTeamsPromise, saveMatchesPromise])
+      },
+      (error) => {
+        alerter.error('Failed to get html', error)
+      },
+    )
+}
+
+if (require.main === module) {
+  scrap()
+} else {
+  module.exports = scrap
+}

src/scraper/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "scraper",
+  "name": "@hltvf/scraper",
   "version": "0.1.0",
   "main": "scraper.js",
   "license": "GNU GPLv3",