add cheerio support

Author: Granitosaurus

.eslintrc.json

@@ -4,7 +4,7 @@
         "es6": true,
         "node": true
     },
-    "ignorePatterns": ["**/*.d.ts"],
+    "ignorePatterns": ["**/*.d.ts", "examples/**/*"],
     "parser": "@typescript-eslint/parser",
     "parserOptions": {
         "project": "tsconfig.json",

README.md

@@ -3,17 +3,18 @@
 `npm install scrapfly-sdk`
 
 Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
-- Scrape the web without being blocked.
-- Use headless browsers to access Javascript-powered page data.
-- Scale up web scraping.
-- ... and [much more](https://scrapfly.io/docs/scrape-api/getting-started)!
+
+-   Scrape the web without being blocked.
+-   Use headless browsers to access Javascript-powered page data.
+-   Scale up web scraping.
+-   ... and [much more](https://scrapfly.io/docs/scrape-api/getting-started)!
 
 For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapeguide](https://scrapfly.io/blog/tag/scrapeguide/) tag for how to scrape specific targets.
 
 ## Quick Intro
 
 1. Register a [Scrapfly account for free](https://scrapfly.io/register)
-2. Get your API Key on [scrapfly.io/dashboard](https://scrapfly.io/dashboard) 
+2. Get your API Key on [scrapfly.io/dashboard](https://scrapfly.io/dashboard)
 3. Start scraping: 🚀
 
 ```javascript
@@ -43,6 +44,24 @@ For more see [/examples](/examples/) directory.
 For more on Scrapfly API see our [getting started documentation](https://scrapfly.io/docs/scrape-api/getting-started)
 For Python see [Scrapfly Python SDK](https://github.com/scrapfly/python-scrapfly)
 
+## Debugging
+
+To enable debug logs set Scrapfly's log level to `"DEBUG"`:
+
+```javascript
+log.setLevel('DEBUG');
+```
+
+Additionally, set `debug=true` in `ScrapeConfig` to access debug information in [Scrapfly web dashboard](https://scrapfly.io/dashboard):
+
+```typescript
+new ScrapeConfig({
+    url: 'https://web-scraping.dev/product/1',
+    debug: true,
+    // ^ enable debug information
+});
+```
+
 ## Development
 
 Install and setup environment:

__tests__/client.test.ts

@@ -1,7 +1,9 @@
 import axios from 'axios';
+import { AxiosRequestConfig } from 'axios';
 import { ScrapflyClient } from '../src/client.js';
 import * as errors from '../src/errors.js';
 import { ScrapeConfig } from '../src/scrapeconfig.js';
+import { describe, it, expect, jest, beforeEach } from '@jest/globals';
 
 jest.mock('axios');
 
@@ -37,7 +39,7 @@ describe('concurrent scrape', () => {
     // mock axios to return /account data and 2 types of results:
     // - success for /success endpoints
     // - ASP failure for /failure endpoints
-    mockedAxios.request.mockImplementation(async (config) => {
+    mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
         if (config.url.includes('/account')) {
             return {
                 status: 200,
@@ -123,7 +125,7 @@ describe('scrape', () => {
 
     it('GET success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('GET');
@@ -147,7 +149,7 @@ describe('scrape', () => {
 
     it('GET complex urls', async () => {
         const url = 'https://httpbin.dev/anything/?website=https://httpbin.dev/anything';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('GET');
@@ -171,7 +173,7 @@ describe('scrape', () => {
 
     it('POST success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('POST');
@@ -233,7 +235,7 @@ describe('client errors', () => {
 
     it('raises ApiHttpServerError on 500 and success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 500,
@@ -247,7 +249,7 @@ describe('client errors', () => {
 
     it('raises BadApiKeyError on 401', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 401,
@@ -260,7 +262,7 @@ describe('client errors', () => {
     });
     it('raises TooManyRequests on 429 and success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 429,
@@ -271,7 +273,7 @@ describe('client errors', () => {
         await expect(client.scrape(new ScrapeConfig({ url }))).rejects.toThrow(errors.TooManyRequests);
     });
     it('raises ScrapflyScrapeError on ::SCRAPE:: resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SCRAPE::BAD_PROTOCOL',
@@ -284,7 +286,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyWebhookError on ::WEBHOOK:: resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::WEBHOOK::DISABLED ',
@@ -296,7 +298,7 @@ describe('client errors', () => {
         );
     });
     it('raises ScrapflyProxyError on ERR::PROXY::POOL_NOT_FOUND  resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::PROXY::POOL_NOT_FOUND ',
@@ -309,7 +311,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyScheduleError on ERR::SCHEDULE::DISABLED resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SCHEDULE::DISABLED',
@@ -322,7 +324,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyAspError on ERR::ASP::SHIELD_ERROR resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::ASP::SHIELD_ERROR',
@@ -335,7 +337,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflySessionError on ERR::SESSION::CONCURRENT_ACCESS resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SESSION::CONCURRENT_ACCESS',
@@ -348,7 +350,7 @@ describe('client errors', () => {
     });
 
     it('raises ApiHttpClientError on success and unknown status', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::NEW',
@@ -360,7 +362,7 @@ describe('client errors', () => {
         );
     });
     it('raises UpstreamHttpServerError on failure, ERR::SCRAPE::BAD_UPSTREAM_RESPONSE and >=500', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -373,7 +375,7 @@ describe('client errors', () => {
         );
     });
     it('raises UpstreamHttpClientError on failure, ERR::SCRAPE::BAD_UPSTREAM_RESPONSE and 4xx status', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -396,7 +398,7 @@ describe('client errors', () => {
             SESSION: errors.ScrapflySessionError,
         };
         for (const [resource, err] of Object.entries(resourceErrMap)) {
-            mockedAxios.request.mockImplementation(async (config) => {
+            mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
                 return resultFactory({
                     url: config.url,
                     success: false,
@@ -408,7 +410,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyError on unhandled failure', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -421,7 +423,7 @@ describe('client errors', () => {
         );
     });
     it('raises  on unhandled failure', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,

__tests__/result.test.ts

@@ -0,0 +1,21 @@
+import * as fs from 'fs';
+import { ScrapeResult } from '../src/result.js';
+import * as errors from '../src/errors.js';
+import { describe, it, expect } from '@jest/globals';
+
+describe('cheerio selector', () => {
+    it('lazy loads and caches itself', () => {
+        const response = JSON.parse(fs.readFileSync('__tests__/data/response_html_success.json', 'utf8'));
+        const result = new ScrapeResult(response);
+        expect(result.selector('h1').text()).toEqual('Herman Melville - Moby-Dick');
+        // make sure calling it twice performs the same
+        expect(result.selector('h1').text()).toEqual('Herman Melville - Moby-Dick');
+    });
+    it('throws ContentTypeError when accessing .selector on JSON data', () => {
+        const response = JSON.parse(fs.readFileSync('__tests__/data/response_json_success.json', 'utf8'));
+        const result = new ScrapeResult(response);
+        expect(() => {
+            result.selector('h1').text();
+        }).toThrow(errors.ContentTypeError);
+    });
+});

__tests__/scrapeconfig.test.ts

@@ -1,6 +1,7 @@
 import { ScrapeConfig } from '../src/scrapeconfig.js';
 import { HttpMethod } from '../src/types.js';
 import { ScrapeConfigError } from '../src/errors.js';
+import { describe, it, expect } from '@jest/globals';
 
 describe('scrapeconfig', () => {
     it('loads', () => {

__tests__/utils.test.ts

@@ -1,4 +1,5 @@
 import { urlsafe_b64encode } from '../src/utils.js';
+import { describe, it, expect } from '@jest/globals';
 
 describe('urlsafe_b64encode', () => {
     it('should encode a string to base64', () => {