-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcluster.js
35 lines (29 loc) · 1.08 KB
/
cluster.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
const { addExtra } = require('puppeteer-extra')
const puppeteerVanilla = require('puppeteer')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
const { Cluster } = require('puppeteer-cluster')
const puppeteer = addExtra(puppeteerVanilla)
puppeteer.use(StealthPlugin)
const clusterSearch = async () => {
const cluster = await Cluster.launch({
puppeteer,
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 1,
puppeteerOptions: {
headless: true
}
})
await cluster.task(async ({ page, data: url }) => {
await page.goto(url, { waitUntil: 'networkidle2' });
const version = await page.evaluate(() => { pbjs.version });
console.log(url, version);
});
cluster.queue('https://www.techcrunch.com/');
cluster.queue('https://www.google.com/');
cluster.queue('https://www.wikipedia.org/');
cluster.queue('https://www.abcnews.go.com/');
cluster.queue('https://www.foxnews.com/');
await cluster.idle();
await cluster.close();
}
clusterSearch()