diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1fa79d7e65786e..70c70260916fb0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,14 +18,15 @@ updates: # https://github.com/yoimiya-kokomi/Miao-Yunzai/pull/515 https://github.com/zhangfisher/flex-tools/commit/09b565dfe6e2932bb829613ddbe09f6d0acbccd4 - dependency-name: art-template versions: ['>=4.13.3'] - # no longer includes KJUR.crypto.Cipher for RSA - - dependency-name: jsrsasign - versions: ['>=11.0.0'] # pin jsdom to avoid `Error: require() of ES Module` issue caused by parse5 v8 # https://github.com/jsdom/jsdom/issues/3959 - dependency-name: jsdom versions: ['>=27.0.1'] groups: + cloudflare: + patterns: + - '@cloudflare/*' + - 'wrangler' eslint: patterns: - 'eslint' @@ -33,6 +34,17 @@ updates: opentelemetry: patterns: - '@opentelemetry/*' + oxc: + patterns: + - '@oxlint/*' + - 'oxfmt' + - 'oxlint' + - 'oxlint-tsgolint' + proxy-agent: + patterns: + - 'https-proxy-agent' + - 'pac-proxy-agent' + - 'socks-proxy-agent' typescript-eslint: patterns: - '@typescript-eslint/*' diff --git a/.github/prompts/pr_review_rules.md b/.github/prompts/pr_review_rules.md new file mode 100644 index 00000000000000..a162ccec99dfd1 --- /dev/null +++ b/.github/prompts/pr_review_rules.md @@ -0,0 +1,50 @@ +# PR Review Rules for RSSHub + +You are reviewing pull requests for RSSHub. + +Only report **clear and actionable** violations in changed lines/files. Do not report speculative or uncertain issues. + +## Route Metadata and Docs + +1. `example` must start with `/` and be a working RSSHub route path. +2. Route name must not repeat namespace name. +3. In radar rules, `source` must be a relative host/path (no `https://`, no hash/query matching). +4. In radar rules, `target` may be empty; if present, it must match the route path and its parameters. +5. Namespace `url` should not include protocol prefix. +6. Use a single category in `categories`. +7. `parameters` keys must match real path parameters. +8. Keep route/docs lists in alphabetical order when touching sorted files. +9. Do not modify default values or working examples unless they are broken. + +## Data Handling and Feed Quality + +10. Use `cache.tryGet()` (from `@/utils/cache`) for detail fetching in loops; cache processed result instead of raw HTML. +11. `description` should contain article content only; do not duplicate `title`, `author`, `pubDate`, or tags. +12. Extract tags/categories into `category` field. +13. Use `parseDate()` for date fields when source provides time. +14. Do not set fake dates (`new Date()` fallback) when source has no valid time. +15. Keep each item `link` unique; feed-level `link` should be human-readable (not raw API endpoint). +16. Do not trim/truncate title/content manually. + +## API and Requesting + +17. Prefer official API endpoints over scraping when available. +18. Fetch first page only; do not add custom pagination behavior. +19. Use common parameter `limit` instead of custom limit/query filtering. +20. Prefer path parameters over custom query parameters for route config. +21. Use RSSHub built-in UA behavior; when browser-like headers are needed, use `config.trueUA` instead of hardcoded UA strings. + +## Code Style and Maintainability + +22. Use `camelCase` naming. +23. Use `import type { ... }` for type-only imports. +24. Keep imports sorted. +25. Use JSX-based rendering (`renderToString` and template components) for custom HTML rendering patterns used by RSSHub. +26. Avoid unnecessary changes outside PR scope. + +## Reporting Format + +- Report only violated rules. +- Each bullet should include: file path, problem, and concrete fix. +- Group repeated issues across files into one concise bullet when possible. +- If no rule is clearly violated, do not comment. diff --git a/.github/prompts/similar_issues.prompt.yml b/.github/prompts/similar_issues.prompt.yml deleted file mode 100644 index c1bf40ea58cc3f..00000000000000 --- a/.github/prompts/similar_issues.prompt.yml +++ /dev/null @@ -1,46 +0,0 @@ -messages: - - role: system - content: |- - You are a GitHub assistant with access to GitHub Model Context Protocol (MCP) - tools in read-only mode. Your task is to search this repository's issues to find - previously filed issues similar to the provided issue title and body. Use the - GitHub tools via MCP to perform the search and retrieve real issue data (do not - fabricate results). Consider semantic similarity across title and body. Exclude - the current issue {{issue_number}}. Return up to 3 of the most similar past issues. If none are - reasonably similar, return an empty list. Output must follow the response schema - exactly and include only data you actually retrieved from GitHub tools. - The current GitHub repository is: "{{repository}}". - - role: user - content: |- - Find similar issues for issue {{issue_number}}: - Title: {{issue_title}} - Body: - {{issue_body}} -model: openai/gpt-4.1-mini -responseFormat: json_schema -jsonSchema: |- - { - "name": "similar_issues_result", - "strict": true, - "schema": { - "type": "object", - "properties": { - "matches": { - "type": "array", - "items": { - "type": "object", - "properties": { - "number": { "type": "integer" }, - "title": { "type": "string" }, - "url": { "type": "string" }, - "similarity_score": { "type": "number", "minimum": 0, "maximum": 1 } - }, - "required": ["number", "title", "url", "similarity_score"], - "additionalProperties": false - } - } - }, - "required": ["matches"], - "additionalProperties": false - } - } diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index af541b4c50c07d..50f8024fd5816e 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -20,9 +20,9 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 - name: Use Node.js Active LTS - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3c83d0301ef8b8..a870789efa99b3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,13 +55,13 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: - languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality + languages: ${{ matrix.language }} # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) diff --git a/.github/workflows/comment-on-issue.yml b/.github/workflows/comment-on-issue.yml index 7a3fc050da6050..78371364a5471f 100644 --- a/.github/workflows/comment-on-issue.yml +++ b/.github/workflows/comment-on-issue.yml @@ -14,8 +14,8 @@ jobs: if: github.event.sender.login != 'issuehunt-oss[bot]' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index e5b4b8ff444742..0f2789c862fdba 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -71,16 +71,16 @@ jobs: echo "repo-name=$REPO_NAME_LOWER" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to Docker Hub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -88,7 +88,7 @@ jobs: - name: Extract Docker metadata (ordinary version) id: meta-ordinary - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: | ${{ vars.DOCKER_USERNAME }}/${{ steps.repo-name.outputs.repo-name }} @@ -107,7 +107,7 @@ jobs: - name: Build and push Docker image (ordinary version) id: build-and-push - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . tags: ${{ steps.image-name-ordinary.outputs.tags }} @@ -118,7 +118,7 @@ jobs: outputs: type=image,compression=zstd,force-compression=true,push-by-digest=true,name-canonical=true,push=true - name: Attest (ordinary version) - uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-name: | ${{ vars.DOCKER_USERNAME }}/${{ steps.repo-name.outputs.repo-name }} @@ -132,7 +132,7 @@ jobs: touch "${{ runner.temp }}/digests/ordinary/${digest#sha256:}" - name: Upload digest (ordinary version) - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: digests-ordinary-${{ env.PLATFORM_PAIR }} path: ${{ runner.temp }}/digests/ordinary/* @@ -141,7 +141,7 @@ jobs: - name: Extract Docker metadata (Chromium-bundled version) id: meta-chromium-bundled - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: | ${{ vars.DOCKER_USERNAME }}/${{ steps.repo-name.outputs.repo-name }} @@ -160,7 +160,7 @@ jobs: - name: Build and push Docker image (Chromium-bundled version) id: build-and-push-chromium - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . build-args: PUPPETEER_SKIP_DOWNLOAD=0 @@ -173,7 +173,7 @@ jobs: outputs: type=image,compression=zstd,force-compression=true,push-by-digest=true,name-canonical=true,push=true - name: Attest (Chromium-bundled version) - uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-name: | ${{ vars.DOCKER_USERNAME }}/${{ steps.repo-name.outputs.repo-name }} @@ -187,7 +187,7 @@ jobs: touch "${{ runner.temp }}/digests/chromium/${digest#sha256:}" - name: Upload digest (Chromium-bundled version) - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: digests-chromium-${{ env.PLATFORM_PAIR }} path: ${{ runner.temp }}/digests/chromium/* @@ -204,23 +204,23 @@ jobs: id-token: write steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to Docker Hub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Download digests (ordinary version) - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/digests/ordinary pattern: digests-ordinary-* @@ -228,7 +228,7 @@ jobs: - name: Extract Docker metadata (ordinary version) id: meta-ordinary-merge - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: | ${{ vars.DOCKER_USERNAME }}/${{ needs.release.outputs.repo-name }} @@ -246,7 +246,7 @@ jobs: $(printf '${{ vars.DOCKER_USERNAME }}/${{ needs.release.outputs.repo-name }}@sha256:%s ' *) - name: Download digests (Chromium-bundled version) - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/digests/chromium pattern: digests-chromium-* @@ -254,7 +254,7 @@ jobs: - name: Extract Docker metadata (Chromium-bundled version) id: meta-chromium-bundled-merge - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: | ${{ vars.DOCKER_USERNAME }}/${{ needs.release.outputs.repo-name }} diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml index 4f6412aff20a7c..6032b0cb961753 100644 --- a/.github/workflows/docker-test-cont.yml +++ b/.github/workflows/docker-test-cont.yml @@ -46,6 +46,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: lts/* + cache: 'pnpm' + + - name: Install dependencies (pnpm) # require js-beautify + run: pnpm i + - name: Fetch affected routes id: fetch-route uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 @@ -64,7 +74,7 @@ jobs: - name: Fetch Docker image if: (env.TEST_CONTINUE) - uses: dawidd6/action-download-artifact@5c98f0b039f36ef966fdb7dfa9779262785ecb05 # v14 + uses: dawidd6/action-download-artifact@1f8785ff7a5130826f848e7f72725c85d241860f # v18 with: workflow: ${{ github.event.workflow_run.workflow_id }} run_id: ${{ github.event.workflow_run.id }} @@ -87,17 +97,31 @@ jobs: -p 1200:1200 \ rsshub:latest - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - name: Wait for RSSHub startup if: (env.TEST_CONTINUE) - with: - node-version: lts/* - cache: 'pnpm' + env: + TEST_BASEURL: http://localhost:1200 + run: | + set -euo pipefail + healthcheck_url="$TEST_BASEURL/healthz" - - name: Install dependencies (pnpm) # require js-beautify - if: (env.TEST_CONTINUE) - run: pnpm i + for _ in $(seq 1 30); do + if curl --silent --show-error --fail "$healthcheck_url" >/dev/null; then + exit 0 + fi + + if [ "$(docker inspect -f '{{.State.Status}}' rsshub 2>/dev/null)" != 'running' ]; then + docker logs rsshub || true + echo "rsshub container exited before becoming ready" + exit 1 + fi + + sleep 1 + done + + docker logs rsshub || true + echo "Timed out waiting for RSSHub health check: $healthcheck_url" + exit 1 - name: Generate feedback if: (env.TEST_CONTINUE) @@ -125,7 +149,7 @@ jobs: with: actions: 'add-labels' token: ${{ secrets.GITHUB_TOKEN }} - issue-number: ${{ steps.source-run-info.outputs.pullRequestNumber }} + issue-number: ${{ steps.source-run-info.outputs.number }} labels: 'auto: DO NOT merge' - name: Print Docker container logs diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index a45571b8180ec3..53c794a5ce8c4c 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -30,17 +30,17 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Docker Buildx # needed by `cache-from` - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Extract Docker metadata id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: rsshub flavor: latest=true - name: Build Docker image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . build-args: PUPPETEER_SKIP_DOWNLOAD=0 # also test bundling Chromium @@ -69,7 +69,7 @@ jobs: run: docker save rsshub:latest | zstdmt -o rsshub.tar.zst - name: Upload Docker image - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: docker-image path: rsshub.tar.zst diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 6e87cddec3927c..3c7e08335747ca 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -15,8 +15,8 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index 62b1c29391f039..f1aed08f6d6438 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -68,10 +68,10 @@ jobs: ref: ${{ fromJson(steps.pr-data.outputs.data).head.ref }} - name: Install pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 - name: Use Node.js Active LTS - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' @@ -129,7 +129,7 @@ jobs: run: cat ${{ github.workspace }}/logs/combined.log - name: Upload Artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: logs path: logs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 42a5d5305e3e07..b2e41d32d459f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,33 +15,33 @@ jobs: eslint-warning: name: Lint if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} - runs-on: ubuntu-slim + runs-on: ubuntu-latest timeout-minutes: 15 permissions: security-events: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' - run: pnpm i - - name: Install ESLint SARIF formatter - # https://github.com/microsoft/sarif-js-sdk/issues/91 unncessary deps on eslint@8 - run: | - wget https://raw.githubusercontent.com/microsoft/sarif-js-sdk/refs/heads/main/packages/eslint-formatter-sarif/sarif.js -O node_modules/sarif.cjs - pnpm i -D utf8 lodash jschardet + - name: Install oxlint to SARIF converter + run: pnpm i -g oxlint-json-to-sarif - name: Lint - run: pnpm run lint - --format node_modules/sarif.cjs - --output-file eslint-results.sarif + run: pnpm exec oxlint --type-aware + --format=json | oxlint-json-to-sarif > oxlint-results.sarif continue-on-error: true - name: Upload analysis results to GitHub uses: github/codeql-action/upload-sarif@v4 with: - sarif_file: eslint-results.sarif + sarif_file: oxlint-results.sarif wait-for-processing: true + - name: Upload Artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + path: oxlint-results.sarif # https://github.com/amannn/action-semantic-pull-request title-lint: diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 7dc81876edddfc..bfbf5fa31852ff 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -23,8 +23,8 @@ jobs: HUSKY: 0 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml new file mode 100644 index 00000000000000..1bf60cacc649cf --- /dev/null +++ b/.github/workflows/pr-review.yml @@ -0,0 +1,147 @@ +name: pr-review + +on: + workflow_run: + workflows: [PR - Docker build test] + types: [completed] + workflow_dispatch: + inputs: + pr_number: + description: Pull request number to review manually + required: true + type: number + +jobs: + review-pr: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # https://github.com/orgs/community/discussions/25220#discussioncomment-11316244 + - name: Search the PR that triggered this workflow + if: github.event_name != 'workflow_dispatch' + id: source-run-info + env: + GH_TOKEN: ${{ github.token }} + PR_TARGET_REPO: ${{ github.repository }} + PR_BRANCH: |- + ${{ + (github.event.workflow_run.head_repository.owner.login != github.event.workflow_run.repository.owner.login) + && format('{0}:{1}', github.event.workflow_run.head_repository.owner.login, github.event.workflow_run.head_branch) + || github.event.workflow_run.head_branch + }} + run: | + gh pr view --repo "${PR_TARGET_REPO}" "${PR_BRANCH}" \ + --json 'number' --jq '"number=\(.number)"' \ + >> "${GITHUB_OUTPUT}" + + - name: Check PR author + if: github.event_name != 'workflow_dispatch' + id: check-pr-author + env: + GH_TOKEN: ${{ github.token }} + PR_TARGET_REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.source-run-info.outputs.number }} + run: | + AUTHOR=$(gh pr view --repo "${PR_TARGET_REPO}" "${PR_NUMBER}" --json author --jq '.author.login') + echo "author=${AUTHOR}" >> "${GITHUB_OUTPUT}" + if [[ "${AUTHOR}" == "dependabot[bot]" || "${AUTHOR}" == "app/dependabot" ]]; then + echo "skip=true" >> "${GITHUB_OUTPUT}" + else + echo "skip=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Comment on failed Docker build + if: github.event_name != 'workflow_dispatch' && github.event.workflow_run.conclusion == 'failure' + env: + GH_TOKEN: ${{ github.token }} + PR_TARGET_REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.source-run-info.outputs.number }} + run: | + gh pr comment --repo "${PR_TARGET_REPO}" "${PR_NUMBER}" --body " + ## Auto Review + ⚠️ PR review will proceed after the Docker image can be successfully built. + + Please fix the Docker build test issues first." + + - name: Set up Bun + if: github.event_name == 'workflow_dispatch' || (steps.check-pr-author.outputs.skip != 'true' && github.event.workflow_run.conclusion != 'failure') + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + + - name: Install opencode + if: github.event_name == 'workflow_dispatch' || (steps.check-pr-author.outputs.skip != 'true' && github.event.workflow_run.conclusion != 'failure') + run: curl -fsSL https://opencode.ai/install | bash + + - name: Review PR with rules + if: github.event_name == 'workflow_dispatch' || (steps.check-pr-author.outputs.skip != 'true' && github.event.workflow_run.conclusion != 'failure') + env: + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.source-run-info.outputs.number || inputs.pr_number }} + OPENCODE_PERMISSION: | + { + "bash": { + "*": "deny", + "gh auth*": "allow", + "gh pr*": "allow", + "gh api*": "allow" + }, + "webfetch": "deny" + } + run: | + if [ -z "$PR_NUMBER" ]; then + echo "pr_number is required" + exit 1 + fi + + RULES=$(cat .github/prompts/pr_review_rules.md) + + opencode run -m ${{ vars.OPENCODE_MODEL }} "A pull request has been created or updated in this repository. + + Pull request number: + $PR_NUMBER + + Repository: + $GITHUB_REPOSITORY + + Your task: + 1. Use GitHub CLI commands to inspect this PR's metadata and code changes. + 2. Review only based on the following rules. + 3. Report only clear and actionable violations from changed files. + 4. If no clear violations are found, do not comment. + + Review rules: + $RULES + + Required behavior: + - Keep feedback concise and grouped by rule. + - Include file path and a concrete fix suggestion for each issue. + - Ignore uncertain or low-confidence findings. + - Avoid duplicate comments. + + Comment protocol: + - Use marker: + - Check existing comments in issue comments for this marker. + - If a marker comment exists, update it with latest findings. + - Otherwise create a new PR comment. + - If there are no findings and marker comment exists, edit marker comment to a short pass status. + + Suggested comment format: + + ## Auto Review + - [Rule] file: issue + suggestion + + For no findings: + + ## Auto Review + No clear rule violations found in the current diff. + + Use only gh commands and repository data." diff --git a/.github/workflows/similar-issues.yml b/.github/workflows/similar-issues.yml index c3f1da19feb461..5af8b404b1d104 100644 --- a/.github/workflows/similar-issues.yml +++ b/.github/workflows/similar-issues.yml @@ -1,91 +1,71 @@ -name: Similar Issues via AI MCP +name: duplicate-issues on: issues: types: [opened] + workflow_dispatch: + inputs: + issue_number: + description: Issue number to check manually + required: true + type: number jobs: - find-similar: + check-duplicates: + runs-on: ubuntu-latest permissions: contents: read issues: write - models: read - runs-on: ubuntu-slim steps: - - name: Check out repository + - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Prepare prompt variables - id: prepare_input - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - script: | - const issue = context.payload.issue || {}; - const title = issue.title || ''; - const body = issue.body || ''; - const indent = ' '; - // Indent subsequent lines so YAML block scalar indentation remains valid - const bodyIndented = body.replace(/\n/g, '\n' + indent); - core.setOutput('issue_title_json', JSON.stringify(title)); - core.setOutput('issue_body_indented_json', JSON.stringify(bodyIndented)); - core.setOutput('issue_number', issue.number); + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - - name: Find similar issues with AI (MCP) - id: inference - uses: actions/ai-inference@a380166897b5408b8fb7dddd148142794cb5624a # v2.0.6 - with: - prompt-file: ./.github/prompts/similar_issues.prompt.yml - input: | - issue_title: ${{ steps.prepare_input.outputs.issue_title_json }} - issue_body: ${{ steps.prepare_input.outputs.issue_body_indented_json }} - issue_number: ${{ steps.prepare_input.outputs.issue_number }} - repository: ${{ github.repository }} - enable-github-mcp: true - # Inference token can use GITHUB_TOKEN. MCP specifically requires a PAT. - token: ${{ secrets.GITHUB_TOKEN }} - github-mcp-token: ${{ secrets.USER_PAT }} - max-tokens: 8000 + - name: Install opencode + run: curl -fsSL https://opencode.ai/install | bash - - name: Prepare comment body - id: prepare - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - name: Check for duplicate issues env: - AI_RESPONSE: ${{ steps.inference.outputs.response }} - with: - script: | - let data; - try { - data = JSON.parse(process.env.AI_RESPONSE || '{}'); - } catch (e) { - core.setOutput('has_matches', 'false'); - return; + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }} + OPENCODE_PERMISSION: | + { + "bash": { + "*": "deny", + "gh issue*": "allow" + }, + "webfetch": "deny" } - const matches = Array.isArray(data.matches) ? data.matches.filter(m => m.number !== context.payload.issue.number) : []; - if (!matches.length) { - core.setOutput('has_matches', 'false'); - return; - } - const lines = []; - lines.push('I found similar issues that might help:'); - for (const m of matches.slice(0, 3)) { - const num = m.number != null ? `#${m.number}` : ''; - const title = m.title || 'Untitled'; - const url = m.url || ''; - const score = typeof m.similarity_score === 'number' ? ` (similarity: ${m.similarity_score.toFixed(2)})` : ''; - lines.push(`- ${url}${score}`.trim()); - } - core.setOutput('has_matches', 'true'); - core.setOutput('comment_body', lines.join('\n')); + run: | + if [ -z "$ISSUE_NUMBER" ]; then + echo "issue_number is required" + exit 1 + fi + + opencode run -m ${{ vars.OPENCODE_MODEL }} "A new issue has been created:' + + Issue number: + $ISSUE_NUMBER + + Lookup this issue and search through existing issues (excluding #$ISSUE_NUMBER) in this repository to find any potential duplicates of this new issue. + Consider: + 1. Similar titles or descriptions + 2. Same error messages or symptoms + 3. Related functionality or components + 4. Similar feature requests + + If you find any potential duplicates, please comment on the new issue with: + - A brief explanation of why it might be a duplicate + - Links to the potentially duplicate issues + - A suggestion to check those issues first + + Use this format for the comment: + 'This issue might be a duplicate of existing issues. Please check: + - #[issue_number]: [brief description of similarity] + + Feel free to ignore if none of these address your specific case.' - - name: Comment similar issues - if: steps.prepare.outputs.has_matches == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - script: | - const body = ${{ toJson(steps.prepare.outputs.comment_body) }}; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body - }); + If no clear duplicates are found, do not comment." diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 983edb0203f3e6..dbb88ca6ff90b7 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,7 +12,7 @@ jobs: stale: runs-on: ubuntu-slim steps: - - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: # Don't stale issues days-before-issue-stale: -1 @@ -33,5 +33,3 @@ jobs: github-token: ${{ github.token }} process-only: 'prs' pr-inactive-days: '30' - pr-comment: > - This pull request has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs. diff --git a/.github/workflows/test-full-routes.yml b/.github/workflows/test-full-routes.yml index f3d77e1bec9ce1..6427d4c16c2093 100644 --- a/.github/workflows/test-full-routes.yml +++ b/.github/workflows/test-full-routes.yml @@ -16,9 +16,9 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 - name: Use Node.js Active LTS - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b954f85c74f7fe..3a948ca457131e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,8 +31,8 @@ jobs: name: Vitest on Node ${{ matrix.node-version }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' @@ -49,7 +49,7 @@ jobs: - name: Test all and generate coverage run: pnpm run vitest:coverage --reporter=github-actions env: - REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}/ + REDIS_URL: redis://localhost:${{ job.services.redis.ports['6379'] }}/ - name: Upload coverage to Codecov if: ${{ matrix.node-version == 'lts/*' }} uses: codecov/codecov-action@v5 @@ -76,8 +76,8 @@ jobs: name: Vitest puppeteer on Node ${{ matrix.node-version }} with ${{ matrix.chromium.name }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' @@ -123,8 +123,8 @@ jobs: name: Build radar and maintainer on Node ${{ matrix.node-version }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 - - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' @@ -132,7 +132,7 @@ jobs: - name: Build radar and maintainer run: npm run build - name: Upload assets - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: generated-assets-${{ matrix.node-version }} path: assets/build/ diff --git a/.github/workflows/update-nix-hash.yml b/.github/workflows/update-nix-hash.yml new file mode 100644 index 00000000000000..f225c781e9da82 --- /dev/null +++ b/.github/workflows/update-nix-hash.yml @@ -0,0 +1,68 @@ +name: Update Nix Hash + +on: + push: + branches: + - master + paths: + - 'pnpm-lock.yaml' + +permissions: + contents: write + +jobs: + update-hash: + # Only run on the upstream repo, not forks + if: github.repository_owner == 'DIYgod' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Install Nix + uses: cachix/install-nix-action@1ca7d21a94afc7c957383a2d217460d980de4934 # v31.10.1 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Update Nix flake hash + id: update-hash + run: | + set -e + + # Extract current hash + CURRENT_HASH=$(grep -oP 'hash = "sha256-\K[^"]+' flake.nix || echo "") + echo "Current hash: sha256-$CURRENT_HASH" + + # Set temporary invalid hash to trigger error + sed -i 's/hash = "sha256-[^"]*";/hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";/' flake.nix + + # Build and capture the correct hash from error message + NEW_HASH=$(nix build .# 2>&1 | grep "got:" | awk '{print $2}' | sed 's/sha256-//' || echo "") + + if [ -z "$NEW_HASH" ]; then + echo "Failed to get new hash, hash may already be correct" + git checkout flake.nix + echo "hash_changed=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # Update with correct hash + sed -i "s/hash = \"sha256-[^\"]*\";/hash = \"sha256-$NEW_HASH\";/" flake.nix + + if [ "$CURRENT_HASH" = "$NEW_HASH" ]; then + echo "Hash unchanged" + echo "hash_changed=false" >> $GITHUB_OUTPUT + else + echo "Hash updated from sha256-$CURRENT_HASH to sha256-$NEW_HASH" + echo "hash_changed=true" >> $GITHUB_OUTPUT + echo "new_hash=sha256-$NEW_HASH" >> $GITHUB_OUTPUT + fi + + - name: Commit and push if changed + if: steps.update-hash.outputs.hash_changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add flake.nix + git commit -m "chore(nix): update dependencies hash to ${{ steps.update-hash.outputs.new_hash }}" + git push diff --git a/.oxfmtrc.json b/.oxfmtrc.json index c25043a2b47308..0e90a74943fefc 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -5,5 +5,12 @@ "singleQuote": true, "trailingComma": "es5", "arrowParens": "always", - "ignorePatterns": ["lib/routes-deprecated", "lib/router.js", "babel.config.js", "scripts/docker/minify-docker.js", "dist", "pnpm-lock.yaml"] + "ignorePatterns": ["lib/routes-deprecated", "lib/router.js", "babel.config.js", "scripts/docker/minify-docker.js", "dist", "pnpm-lock.yaml"], + "sortPackageJson": { + "sortScripts": true + }, + "sortImports": { + "groups": ["side_effect", "builtin", "external", ["internal", "subpath"], ["parent", "sibling", "index"], "style", "unknown"], + "order": "asc" + } } diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 00000000000000..62b6dd4b485588 --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,526 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "categories": { + "correctness": "off" + }, + "ignorePatterns": ["**/coverage", "**/.vscode", "**/docker-compose.yml", "!.github", "assets/build", "lib/routes-deprecated", "lib/router.js", "dist", "dist-lib", "dist-worker"], + "env": { + "builtin": true, + "browser": true, + "es2026": true, + "node": true + }, + "plugins": ["eslint", "typescript", "node", "unicorn", "import"], + "jsPlugins": [ + "@stylistic/eslint-plugin", + { "name": "import-x-js", "specifier": "eslint-plugin-import-x" }, + { "name": "n", "specifier": "eslint-plugin-n" }, + "eslint-plugin-simple-import-sort", + "./eslint-plugins/no-then.js", + "./eslint-plugins/nsfw-flag.js" + ], + "rules": { + // #region --- ESLint js/recommended possible problems --- + "constructor-super": "error", + "for-direction": "error", + "getter-return": "error", + "no-async-promise-executor": "error", + "no-class-assign": "error", + "no-compare-neg-zero": "error", + "no-cond-assign": "error", + "no-const-assign": "error", + "no-constant-binary-expression": "error", + "no-constant-condition": "error", + // "no-control-regex": "error", -> off + "no-debugger": "error", + "no-dupe-class-members": "error", + "no-dupe-else-if": "error", + "no-dupe-keys": "error", + "no-duplicate-case": "error", + "no-empty-character-class": "error", + "no-empty-pattern": "error", + "no-ex-assign": "error", + "no-fallthrough": "error", + "no-func-assign": "error", + "no-import-assign": "error", + "no-invalid-regexp": "error", + "no-irregular-whitespace": "error", + "no-loss-of-precision": "error", + "no-misleading-character-class": "error", + "no-new-native-nonconstructor": "error", + "no-nonoctal-decimal-escape": "error", + "no-obj-calls": "error", + // "no-prototype-builtins": "error", -> off + "no-self-assign": "error", + "no-setter-return": "error", + "no-sparse-arrays": "error", + "no-this-before-super": "error", + "no-unassigned-vars": "error", + // "no-undef": "error", + "no-unexpected-multiline": "error", + "no-unreachable": "error", + "no-unsafe-finally": "error", + "no-unsafe-negation": "error", + "no-unsafe-optional-chaining": "error", + "no-unused-private-class-members": "error", + // "no-unused-vars": "error", -> off for @typescript-eslint/no-unused-vars + "no-useless-backreference": "error", + "use-isnan": "error", + "valid-typeof": "error", + // #endregion + + // #region --- ESLint js/recommended suggestions --- + "no-case-declarations": "error", + "no-delete-var": "error", + "no-empty": "error", + "no-empty-static-block": "error", + "no-extra-boolean-cast": "error", + "no-global-assign": "error", + "no-redeclare": "error", + "no-regex-spaces": "error", + "no-shadow-restricted-names": "error", + "no-unused-labels": "error", + "no-useless-catch": "error", + "no-useless-escape": "error", + "no-with": "error", + "preserve-caught-error": "error", + "require-yield": "error", + // #endregion + + // #region --- TypeScript flat/recommended --- + // "@typescript-eslint/ban-ts-comment": "error", + "no-array-constructor": "error", // equivalent to @typescript-eslint/no-array-constructor in oxlint + "@typescript-eslint/no-duplicate-enum-values": "error", + "@typescript-eslint/no-empty-object-type": "error", + // "@typescript-eslint/no-explicit-any": "error", + "@typescript-eslint/no-extra-non-null-assertion": "error", + "@typescript-eslint/no-misused-new": "error", + "@typescript-eslint/no-namespace": "error", + "@typescript-eslint/no-non-null-asserted-optional-chain": "error", + "@typescript-eslint/no-require-imports": "error", + "@typescript-eslint/no-this-alias": "error", + "@typescript-eslint/no-unnecessary-type-constraint": "error", + "@typescript-eslint/no-unsafe-declaration-merging": "error", + "@typescript-eslint/no-unsafe-function-type": "error", + // "no-unused-expressions": "off", + // "@typescript-eslint/no-unused-expressions": "error", + // "no-unused-vars": "off", + // "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-wrapper-object-types": "error", + "@typescript-eslint/prefer-as-const": "error", + "@typescript-eslint/prefer-namespace-keyword": "error", + "@typescript-eslint/triple-slash-reference": "error", + // #endregion + + // #region --- TypeScript flat/stylistic --- + "@typescript-eslint/adjacent-overload-signatures": "error", + // "@typescript-eslint/array-type": "error", + "@typescript-eslint/ban-tslint-comment": "error", + "@typescript-eslint/class-literal-property-style": "error", + "@typescript-eslint/consistent-generic-constructors": "error", + // "@typescript-eslint/consistent-indexed-object-style": "error", + "@typescript-eslint/consistent-type-assertions": "error", + // "@typescript-eslint/consistent-type-definitions": "error", + "@typescript-eslint/no-confusing-non-null-assertion": "error", + // "no-empty-function": "off", + // "@typescript-eslint/no-empty-function": "error", + // "@typescript-eslint/no-inferrable-types": "error", + "@typescript-eslint/prefer-for-of": "error", + "@typescript-eslint/prefer-function-type": "error", + // #endregion + + // #region --- n flat/recommended-script --- + "n/hashbang": "error", + // "n/no-deprecated-api": "error", + "node/no-exports-assign": "error", + "n/no-extraneous-import": "error", + // "n/no-extraneous-require": "error", + // "n/no-missing-import": "error", + // "n/no-missing-require": "error", + // "n/no-process-exit": "error", + "n/no-unpublished-bin": "error", + // "n/no-unpublished-import": "error", + // "n/no-unpublished-require": "error", + "n/no-unsupported-features/es-builtins": "error", + "n/no-unsupported-features/es-syntax": "error", + // "n/no-unsupported-features/node-builtins": "error", + "n/process-exit-as-throw": "error", + // #endregion + + // #region --- unicorn --- + "unicorn/catch-error-name": "error", + "unicorn/consistent-assert": "error", + "unicorn/consistent-date-clone": "error", + "unicorn/consistent-empty-array-spread": "error", + "unicorn/consistent-existence-index-check": "error", + // "unicorn/consistent-function-scoping": "error", + "unicorn/empty-brace-spaces": "error", + "unicorn/error-message": "error", + "unicorn/escape-case": "error", + // "unicorn/expiring-todo-comments": "error", // not yet implemented + // "unicorn/explicit-length-check": "error", + // "unicorn/filename-case": "error", + // "unicorn/import-style": "error", // not yet implemented + // "unicorn/isolated-functions": "error", // not yet implemented + "unicorn/new-for-builtins": "error", + "unicorn/no-abusive-eslint-disable": "error", + "unicorn/no-accessor-recursion": "error", + "unicorn/no-anonymous-default-export": "error", + // "unicorn/no-array-callback-reference": "error", + "unicorn/no-array-for-each": "error", + "unicorn/no-array-method-this-argument": "error", + // "unicorn/no-array-reduce": "error", + "unicorn/no-array-reverse": "error", + // "unicorn/no-array-sort": "error", + // "unicorn/no-await-expression-member": "error", + "unicorn/no-await-in-promise-methods": "error", + "unicorn/no-console-spaces": "error", + "unicorn/no-document-cookie": "error", + // "unicorn/no-empty-file": "error", + // "unicorn/no-for-loop": "error", // won't be implemented + // "unicorn/no-hex-escape": "error", + "unicorn/no-immediate-mutation": "error", + "unicorn/no-instanceof-builtins": "error", + "unicorn/no-invalid-fetch-options": "error", + "unicorn/no-invalid-remove-event-listener": "error", + "unicorn/no-lonely-if": "error", + "unicorn/no-magic-array-flat-depth": "error", + // "unicorn/no-named-default": "error", -> use import/no-named-default + "no-negated-condition": "off", + "unicorn/no-negated-condition": "error", + "unicorn/no-negation-in-equality-check": "error", + "no-nested-ternary": "off", + // "unicorn/no-nested-ternary": "error", + "unicorn/no-new-array": "error", + "unicorn/no-new-buffer": "error", + // "unicorn/no-null": "error", + // "unicorn/no-object-as-default-parameter": "error", + // "unicorn/no-process-exit": "error", + "unicorn/no-single-promise-in-promise-methods": "error", + "unicorn/no-static-only-class": "error", + "unicorn/no-thenable": "error", + "unicorn/no-this-assignment": "error", + "unicorn/no-typeof-undefined": "error", + "unicorn/no-unnecessary-array-flat-depth": "error", + "unicorn/no-unnecessary-array-splice-count": "error", + "unicorn/no-unnecessary-await": "error", + // "unicorn/no-unnecessary-polyfills": "error", // not yet implemented + "unicorn/no-unnecessary-slice-end": "error", + "unicorn/no-unreadable-array-destructuring": "error", + "unicorn/no-unreadable-iife": "error", + "unicorn/no-useless-collection-argument": "error", + "unicorn/no-useless-error-capture-stack-trace": "error", + "unicorn/no-useless-fallback-in-spread": "error", + "unicorn/no-useless-length-check": "error", + "unicorn/no-useless-promise-resolve-reject": "error", + "unicorn/no-useless-spread": "error", + // "unicorn/no-useless-switch-case": "error", + // "unicorn/no-useless-undefined": "error", + "unicorn/no-zero-fractions": "error", + // "unicorn/number-literal-case": "error", + // "unicorn/numeric-separators-style": "error", + "unicorn/prefer-add-event-listener": "error", + "unicorn/prefer-array-find": "error", + "unicorn/prefer-array-flat": "error", + "unicorn/prefer-array-flat-map": "error", + "unicorn/prefer-array-index-of": "error", + "unicorn/prefer-array-some": "error", + "unicorn/prefer-at": "error", + "unicorn/prefer-bigint-literals": "error", + "unicorn/prefer-blob-reading-methods": "error", + "unicorn/prefer-class-fields": "error", + "unicorn/prefer-classlist-toggle": "error", + // "unicorn/prefer-code-point": "error", + "unicorn/prefer-date-now": "error", + "unicorn/prefer-default-parameters": "error", + "unicorn/prefer-dom-node-append": "error", + "unicorn/prefer-dom-node-dataset": "error", + "unicorn/prefer-dom-node-remove": "error", + "unicorn/prefer-dom-node-text-content": "error", + "unicorn/prefer-event-target": "error", + // "unicorn/prefer-export-from": "error", // not yet implemented + // "unicorn/prefer-global-this": "error", + "unicorn/prefer-includes": "error", + "unicorn/prefer-keyboard-event-key": "error", + "unicorn/prefer-logical-operator-over-ternary": "error", + "unicorn/prefer-math-min-max": "error", + "unicorn/prefer-math-trunc": "error", + "unicorn/prefer-modern-dom-apis": "error", + "unicorn/prefer-modern-math-apis": "error", + // "unicorn/prefer-module": "error", + "unicorn/prefer-native-coercion-functions": "error", + "unicorn/prefer-negative-index": "error", + "unicorn/prefer-node-protocol": "error", + // "unicorn/prefer-number-properties": "error", + "unicorn/prefer-object-from-entries": "error", + "unicorn/prefer-optional-catch-binding": "error", + "unicorn/prefer-prototype-methods": "error", + "unicorn/prefer-query-selector": "error", + "unicorn/prefer-reflect-apply": "error", + "unicorn/prefer-regexp-test": "error", + "unicorn/prefer-response-static-json": "error", + "unicorn/prefer-set-has": "error", + "unicorn/prefer-set-size": "error", + // "unicorn/prefer-single-call": "error", // not yet implemented + // "unicorn/prefer-spread": "error", + "unicorn/prefer-string-raw": "error", + "unicorn/prefer-string-replace-all": "error", + // "unicorn/prefer-string-slice": "error", + "unicorn/prefer-string-starts-ends-with": "error", + "unicorn/prefer-string-trim-start-end": "error", + "unicorn/prefer-structured-clone": "error", + // "unicorn/prefer-switch": "error", // not yet implemented + "unicorn/prefer-ternary": "error", + // "unicorn/prefer-top-level-await": "error", + "unicorn/prefer-type-error": "error", + // "unicorn/prevent-abbreviations": "error", // not yet implemented + "unicorn/relative-url-style": "error", + "unicorn/require-array-join-separator": "error", + "unicorn/require-module-attributes": "error", + "unicorn/require-module-specifiers": "error", + "unicorn/require-number-to-fixed-digits-argument": "error", + // "unicorn/switch-case-braces": "error", + // "unicorn/template-indent": "error", // not yet implemented + // "unicorn/text-encoding-identifier-case": "error", + "unicorn/throw-new-error": "error", + // #endregion + + // --- custom rules --- + // #region --- possible problems --- + "array-callback-return": ["error", { "allowImplicit": true }], + + "no-await-in-loop": "error", + "no-control-regex": "off", + "no-prototype-builtins": "off", + "no-undef": "off", // typescript/eslint-recommended, ts(2552) + // #endregion + + // #region --- suggestions --- + "arrow-body-style": "error", + "block-scoped-var": "error", + "curly": "error", + // "dot-notation": "error", -> use @typescript-eslint/dot-notation + "eqeqeq": "error", + + "default-case": ["warn", { "commentPattern": "^no default$" }], + + "default-case-last": "error", + "no-console": "error", + "no-eval": "error", + "no-extend-native": "error", + "no-extra-label": "error", + + "no-implicit-coercion": [ + "error", + { + "boolean": false, + "number": false, + "string": false, + "disallowTemplateShorthand": true + } + ], + + // "no-implicit-globals": "error", // not yet implemented + "no-labels": "error", + "no-lonely-if": "error", + "no-multi-str": "error", + "no-new-func": "error", + "no-unneeded-ternary": "error", + "no-useless-computed-key": "error", + "no-useless-concat": "warn", + "no-useless-rename": "error", + "no-var": "error", + // "object-shorthand": "error", // not yet implemented + // "prefer-arrow-callback'": "error", // not yet implemented + "prefer-const": "error", + "prefer-object-has-own": "error", + "require-await": "error", + // "prefer-regex-literals": [ // not yet implemented + // "error", + // { + // "disallowRedundantWrapping": true + // } + // ], + // #endregion + + // #region --- TypeScript --- + "@typescript-eslint/array-type": ["error", { "default": "array-simple" }], + + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/consistent-indexed-object-style": "off", // stylistic + "@typescript-eslint/consistent-type-definitions": "off", // stylistic + "@typescript-eslint/no-empty-function": "off", // stylistic && tests + "@typescript-eslint/no-explicit-any": "off", + + "@typescript-eslint/no-inferrable-types": ["error", { "ignoreParameters": true, "ignoreProperties": true }], + + "@typescript-eslint/no-unnecessary-template-expression": "error", // type-aware + + "@typescript-eslint/no-unused-expressions": ["error", { "allowShortCircuit": true, "allowTernary": true }], + "@typescript-eslint/no-unused-vars": ["error", { "args": "after-used", "argsIgnorePattern": "^_" }], + + // type-aware + // "@typescript-eslint/await-thenable": "off", + // "@typescript-eslint/no-base-to-string": "off", + // "@typescript-eslint/no-floating-promises": "off", + // "@typescript-eslint/no-misused-spread": "off", + // "@typescript-eslint/no-redundant-type-constituents": "off", + // "@typescript-eslint/unbound-method": "off", + // "@typescript-eslint/restrict-template-expressions": "off", + // #endregion + + // #region --- unicorn --- + "unicorn/consistent-function-scoping": "warn", + "unicorn/explicit-length-check": "off", + + "unicorn/filename-case": [ + "error", + { + "case": "kebabCase", + "ignore": [".*\\.(yaml|yml)$", "RequestInProgress\\.js$"] + } + ], + + "unicorn/no-array-callback-reference": "warn", + "unicorn/no-array-reduce": "warn", + "unicorn/no-array-sort": "warn", + "unicorn/no-await-expression-member": "off", + "unicorn/no-empty-file": "warn", + // "unicorn/no-for-loop": "off", // won't be implemented + "unicorn/no-hex-escape": "warn", + "unicorn/no-nested-ternary": "off", + "unicorn/no-null": "off", + "unicorn/no-object-as-default-parameter": "warn", + "unicorn/no-process-exit": "off", + "unicorn/no-useless-switch-case": "off", + + "unicorn/no-useless-undefined": ["error", { "checkArguments": false }], + + "unicorn/number-literal-case": "off", + + "unicorn/numeric-separators-style": [ + "warn", + { + "onlyIfContainsSeparator": false, + "number": { + "minimumDigits": 7, + "groupLength": 3 + }, + "binary": { + "minimumDigits": 9, + "groupLength": 4 + }, + "octal": { + "minimumDigits": 9, + "groupLength": 4 + }, + "hexadecimal": { + "minimumDigits": 5, + "groupLength": 2 + } + } + ], + + "unicorn/prefer-code-point": "warn", + "unicorn/prefer-global-this": "off", + "unicorn/prefer-import-meta-properties": "warn", + "unicorn/prefer-module": "off", + + "unicorn/prefer-number-properties": ["error", { "checkInfinity": false, "checkNaN": false }], + + "unicorn/prefer-spread": "warn", + "unicorn/prefer-string-slice": "warn", + + // "unicorn/prefer-switch": [ // not yet implemented + // "warn", + // { + // "emptyDefaultCase": "do-nothing-comment" + // } + // ], + + "unicorn/prefer-top-level-await": "off", + "unicorn/prevent-abbreviations": "off", + "unicorn/switch-case-braces": ["error", "avoid"], + "unicorn/text-encoding-identifier-case": "off", + // #endregion + + // #region --- stylistic --- + "@stylistic/arrow-parens": "error", + "@stylistic/arrow-spacing": "error", + "@stylistic/comma-spacing": "error", + "@stylistic/comma-style": "error", + "@stylistic/function-call-spacing": "error", + "@stylistic/keyword-spacing": "off", + "@stylistic/linebreak-style": "error", + + "@stylistic/lines-around-comment": ["error", { "beforeBlockComment": false }], + + "@stylistic/no-multiple-empty-lines": "error", + "@stylistic/no-trailing-spaces": "error", + "@stylistic/rest-spread-spacing": "error", + "@stylistic/semi": "error", + "@stylistic/space-before-blocks": "error", + "@stylistic/space-in-parens": "error", + "@stylistic/space-infix-ops": "error", + "@stylistic/space-unary-ops": "error", + "@stylistic/spaced-comment": "error", + // #endregion + + // #region --- import sorting --- + // oxfmt also handles import sorting + "sort-imports": "off", + "import-x/order": "off", + "simple-import-sort/imports": "error", // oxc doesn't sort named imports yet https://github.com/oxc-project/oxc/issues/13610 + "simple-import-sort/exports": "error", + + "import-x/first": "error", + "import-x-js/newline-after-import": "error", // oxc native not yet implemented + "no-duplicate-imports": "off", + "import-x/no-duplicates": "error", + + "@typescript-eslint/consistent-type-imports": "error", + // #endregion + + // #region --- n --- + "n/no-extraneous-require": "error", + "n/no-deprecated-api": "warn", + "n/no-missing-import": "off", + "n/no-missing-require": "off", + "n/no-process-exit": "off", + "n/no-unpublished-import": "off", + + "n/no-unpublished-require": ["error", { "allowModules": ["tosource"] }], + + "n/no-unsupported-features/node-builtins": [ + "error", + { + "version": "^22.20.0 || ^24", + "allowExperimental": true, + "ignores": [] + } + ], + // #endregion + + // github + "github/no-then": "warn", + + // rsshub + "@rsshub/nsfw-flag/add-nsfw-flag": "error" + }, + "overrides": [ + { + "files": [".puppeteerrc.cjs"], + "plugins": ["typescript"], + "rules": { + "@typescript-eslint/no-require-imports": "off" + } + }, + { + "files": ["**/*.test.ts"], + "plugins": ["typescript"], + "rules": { + "@typescript-eslint/no-unnecessary-template-expression": "off" + } + } + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index 33f9172b1abcac..3f118736609a03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,6 @@ services: CACHE_TYPE: redis REDIS_URL: 'redis://redis:6379/' PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000' # marked - PUPPETEER_REAL_BROWSER_SERVICE: 'http://real-browser:3000' # marked healthcheck: test: ['CMD', 'curl', '-f', 'http://localhost:1200/healthz'] interval: 30s @@ -22,17 +21,6 @@ services: - redis - browserless # marked - real-browser: - image: ghcr.io/hyoban/puppeteer-real-browser-hono - restart: always - ports: - - '3001:3000' - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:3000'] - interval: 30s - timeout: 10s - retries: 3 - browserless: # marked image: browserless/chrome # marked restart: always # marked diff --git a/eslint-plugins/no-then.js b/eslint-plugins/no-then.js new file mode 100644 index 00000000000000..eef945cc34c94d --- /dev/null +++ b/eslint-plugins/no-then.js @@ -0,0 +1,45 @@ +import { eslintCompatPlugin } from '@oxlint/plugins'; + +const rule = { + meta: { + type: 'suggestion', + docs: { + description: 'enforce using `async/await` syntax over Promises', + url: 'https://github.com/github/eslint-plugin-github/blob/main/docs/rules/no-then.md', + recommended: true, + }, + schema: [], + messages: { + preferAsyncAwait: 'Prefer async/await to Promise.{{method}}()', + }, + }, + + createOnce(context) { + return { + MemberExpression(node) { + if (node.property && node.property.name === 'then') { + context.report({ + node: node.property, + messageId: 'preferAsyncAwait', + data: { method: 'then' }, + }); + } else if (node.property && node.property.name === 'catch') { + context.report({ + node: node.property, + messageId: 'preferAsyncAwait', + data: { method: 'catch' }, + }); + } + }, + }; + }, +}; + +export default eslintCompatPlugin({ + meta: { + name: 'github', + }, + rules: { + 'no-then': rule, + }, +}); diff --git a/eslint-plugins/nsfw-flag.js b/eslint-plugins/nsfw-flag.js index 7eff8f79d9bbb9..32c9397cd69046 100644 --- a/eslint-plugins/nsfw-flag.js +++ b/eslint-plugins/nsfw-flag.js @@ -1,6 +1,7 @@ /** * ESLint 9 plugin to automatically mark NSFW routes with the nsfw flag */ +import { eslintCompatPlugin } from '@oxlint/plugins'; const nsfwRoutes = [ '141jav', @@ -88,7 +89,7 @@ function isNsfwRoute(filePath) { }); } -export default { +export default eslintCompatPlugin({ meta: { name: '@rsshub/nsfw-flag', version: '1.0.0', @@ -118,15 +119,14 @@ export default { missingNsfwFlag: 'NSFW route is missing the nsfw flag in features', }, }, - create(context) { - const filename = context.filename || context.getFilename(); - - // 如果不是 NSFW 路由，跳过检查 - if (!isNsfwRoute(filename)) { - return {}; - } - + createOnce(context) { return { + before() { + // 如果不是 NSFW 路由，跳过检查 + if (!isNsfwRoute(context.filename)) { + return false; + } + }, ExportNamedDeclaration(node) { // 查找 export const route: Route = {...} if ( @@ -211,4 +211,4 @@ export default { }, }, }, -}; +}); diff --git a/eslint.config.mjs b/eslint.config.mjs index 3560e7b6df8093..76ce19cbefddc7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,23 +1,21 @@ -import { FlatCompat } from '@eslint/eslintrc'; import js from '@eslint/js'; import stylistic from '@stylistic/eslint-plugin'; import typescriptEslint from '@typescript-eslint/eslint-plugin'; import tsParser from '@typescript-eslint/parser'; -import { importX } from 'eslint-plugin-import-x'; +// import { importX } from 'eslint-plugin-import-x'; import n from 'eslint-plugin-n'; -import simpleImportSort from 'eslint-plugin-simple-import-sort'; +// import simpleImportSort from 'eslint-plugin-simple-import-sort'; import unicorn from 'eslint-plugin-unicorn'; import eslintPluginYml from 'eslint-plugin-yml'; +import { defineConfig } from 'eslint/config'; import globals from 'globals'; + +// import github from './eslint-plugins/no-then.js'; // import nsfwFlagPlugin from './eslint-plugins/nsfw-flag.js'; -const __dirname = import.meta.dirname; -const compat = new FlatCompat({ - baseDirectory: __dirname, - recommendedConfig: js.configs.recommended, -}); +const SOURCE_FILES_GLOB = '**/*.?([cm])[jt]s?(x)'; -export default [ +export default defineConfig([ // { // plugins: { // '@rsshub/nsfw-flag': nsfwFlagPlugin, @@ -27,17 +25,19 @@ export default [ // }, // }, { - ignores: ['**/coverage', '**/.vscode', '**/docker-compose.yml', '!.github', 'assets/build', 'lib/routes-deprecated', 'lib/router.js', '**/babel.config.js', 'scripts/docker/minify-docker.js', 'dist', 'dist-lib'], + ignores: ['**/coverage', '**/.vscode', '**/docker-compose.yml', '!.github', 'assets/build', 'lib/routes-deprecated', 'lib/router.js', 'dist', 'dist-lib', 'dist-worker'], }, - ...compat.extends('eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:@typescript-eslint/stylistic'), - n.configs['flat/recommended-script'], - unicorn.configs.recommended, - ...eslintPluginYml.configs.recommended, { + files: [SOURCE_FILES_GLOB], plugins: { '@stylistic': stylistic, '@typescript-eslint': typescriptEslint, + // github, + js, + n, + unicorn, }, + // extends: [js.configs.recommended, typescriptEslint.configs['flat/recommended'], typescriptEslint.configs['flat/stylistic'], n.configs['flat/recommended-script'], unicorn.configs.recommended], languageOptions: { globals: { @@ -50,32 +50,30 @@ export default [ sourceType: 'module', }, + linterOptions: { + reportUnusedDisableDirectives: false, + }, + rules: { - // possible problems - 'array-callback-return': [ - 'error', - { - allowImplicit: true, - }, - ], + // #region possible problems + /* + 'array-callback-return': ['error', { allowImplicit: true }], 'no-await-in-loop': 'error', 'no-control-regex': 'off', 'no-prototype-builtins': 'off', + */ + // #endregion - // suggestions + // #region suggestions + /* 'arrow-body-style': 'error', 'block-scoped-var': 'error', curly: 'error', 'dot-notation': 'error', eqeqeq: 'error', - 'default-case': [ - 'warn', - { - commentPattern: '^no default$', - }, - ], + 'default-case': ['warn', { commentPattern: '^no default$' }], 'default-case-last': 'error', 'no-console': 'error', @@ -95,10 +93,10 @@ export default [ 'no-implicit-globals': 'error', 'no-labels': 'error', + 'no-lonely-if': 'error', 'no-multi-str': 'error', 'no-new-func': 'error', - 'no-restricted-imports': 'error', - + */ 'no-restricted-syntax': [ 'error', { @@ -125,8 +123,12 @@ export default [ selector: 'CallExpression[callee.property.name="catch"] > ArrowFunctionExpression[params.length=0] > BlockStatement[body.length=0]', message: 'Usage of .catch(() => {}) is not allowed. Please handle the error appropriately.', }, + { + selector: 'CallExpression[callee.name="load"] AwaitExpression > CallExpression', + message: 'Do not use await in call expressions. Extract the result into a variable first.', + }, ], - + /* 'no-unneeded-ternary': 'error', 'no-useless-computed-key': 'error', 'no-useless-concat': 'warn', @@ -136,7 +138,6 @@ export default [ 'prefer-arrow-callback': 'error', 'prefer-const': 'error', 'prefer-object-has-own': 'error', - 'no-useless-escape': 'warn', 'prefer-regex-literals': [ 'error', @@ -146,8 +147,11 @@ export default [ ], 'require-await': 'error', + */ + // #endregion - // typescript + // #region typescript + /* '@typescript-eslint/array-type': ['error', { default: 'array-simple' }], '@typescript-eslint/ban-ts-comment': 'off', @@ -157,28 +161,13 @@ export default [ '@typescript-eslint/no-explicit-any': 'off', '@typescript-eslint/no-inferrable-types': ['error', { ignoreParameters: true, ignoreProperties: true }], + '@typescript-eslint/no-unused-expressions': ['error', { allowShortCircuit: true, allowTernary: true }], + '@typescript-eslint/no-unused-vars': ['error', { args: 'after-used', argsIgnorePattern: '^_' }], + */ + // #endregion - '@typescript-eslint/no-var-requires': 'off', - - '@typescript-eslint/no-unused-expressions': [ - 'error', - { - allowShortCircuit: true, - allowTernary: true, - }, - ], - - '@typescript-eslint/no-unused-vars': [ - 'error', - { - args: 'after-used', - argsIgnorePattern: '^_', - }, - ], - - '@typescript-eslint/prefer-for-of': 'error', - - // unicorn + // #region unicorn + /* 'unicorn/consistent-function-scoping': 'warn', 'unicorn/explicit-length-check': 'off', @@ -197,18 +186,15 @@ export default [ 'unicorn/no-empty-file': 'warn', 'unicorn/no-for-loop': 'off', 'unicorn/no-hex-escape': 'warn', + 'unicorn/no-nested-ternary': 'off', 'unicorn/no-null': 'off', 'unicorn/no-object-as-default-parameter': 'warn', - 'unicorn/no-nested-ternary': 'off', 'unicorn/no-process-exit': 'off', 'unicorn/no-useless-switch-case': 'off', - 'unicorn/no-useless-undefined': [ - 'error', - { - checkArguments: false, - }, - ], + 'unicorn/no-useless-undefined': ['error', { checkArguments: false }], + + 'unicorn/number-literal-case': 'off', 'unicorn/numeric-separators-style': [ 'warn', @@ -242,13 +228,7 @@ export default [ 'unicorn/prefer-import-meta-properties': 'warn', 'unicorn/prefer-module': 'off', - 'unicorn/prefer-number-properties': [ - 'error', - { - checkInfinity: false, - checkNaN: false, - }, - ], + 'unicorn/prefer-number-properties': ['error', { checkInfinity: false, checkNaN: false }], 'unicorn/prefer-spread': 'warn', 'unicorn/prefer-string-slice': 'warn', @@ -264,9 +244,11 @@ export default [ 'unicorn/prevent-abbreviations': 'off', 'unicorn/switch-case-braces': ['error', 'avoid'], 'unicorn/text-encoding-identifier-case': 'off', - 'unicorn/number-literal-case': 'off', + */ + // #endregion - // formatting rules + // #region stylistic + /* '@stylistic/arrow-parens': 'error', '@stylistic/arrow-spacing': 'error', '@stylistic/comma-spacing': 'error', @@ -275,12 +257,7 @@ export default [ '@stylistic/keyword-spacing': 'off', '@stylistic/linebreak-style': 'error', - '@stylistic/lines-around-comment': [ - 'error', - { - beforeBlockComment: false, - }, - ], + '@stylistic/lines-around-comment': ['error', { beforeBlockComment: false }], '@stylistic/no-multiple-empty-lines': 'error', '@stylistic/no-trailing-spaces': 'error', @@ -291,23 +268,19 @@ export default [ '@stylistic/space-infix-ops': 'error', '@stylistic/space-unary-ops': 'error', '@stylistic/spaced-comment': 'error', + */ + // #endregion - // https://github.com/eslint-community/eslint-plugin-n - // node specific rules + // #region node specific rules + /* 'n/no-extraneous-require': 'error', - 'n/no-deprecated-api': 'warn', 'n/no-missing-import': 'off', 'n/no-missing-require': 'off', 'n/no-process-exit': 'off', 'n/no-unpublished-import': 'off', - 'n/no-unpublished-require': [ - 'error', - { - allowModules: ['tosource'], - }, - ], + 'n/no-unpublished-require': ['error', { allowModules: ['tosource'] }], 'n/no-unsupported-features/node-builtins': [ 'error', @@ -317,6 +290,11 @@ export default [ ignores: [], }, ], + */ + // #endregion + + // github + // 'github/no-then': 'warn', }, }, { @@ -326,37 +304,8 @@ export default [ }, }, { - files: ['**/*.yaml', '**/*.yml'], - ignores: ['pnpm-lock.yaml'], - language: 'yml/yaml', - rules: { - 'lines-around-comment': [ - 'error', - { - beforeBlockComment: false, - }, - ], - - 'yml/indent': [ - 'error', - 4, - { - indicatorValueIndent: 2, - }, - ], - - 'yml/no-empty-mapping-value': 'off', - - 'yml/quotes': [ - 'error', - { - prefer: 'single', - }, - ], - }, - }, - { - files: ['**/*.?([cm])[jt]s?(x)'], + /* + files: [SOURCE_FILES_GLOB], plugins: { 'simple-import-sort': simpleImportSort, 'import-x': importX, @@ -374,6 +323,23 @@ export default [ '@typescript-eslint/consistent-type-imports': 'error', 'import-x/consistent-type-specifier-style': ['error', 'prefer-top-level'], + },*/ + }, + { + files: ['**/*.yaml', '**/*.yml'], + ignores: ['pnpm-lock.yaml'], + plugins: { + yml: eslintPluginYml, + }, + language: 'yml/yaml', + rules: { + 'lines-around-comment': ['error', { beforeBlockComment: false }], + + 'yml/indent': ['error', 4, { indicatorValueIndent: 2 }], + + 'yml/no-empty-mapping-value': 'off', + + 'yml/quotes': ['error', { prefer: 'single' }], }, }, -]; +]); diff --git a/flake.nix b/flake.nix index 0ecedb3e031d72..563a42c913326a 100644 --- a/flake.nix +++ b/flake.nix @@ -12,11 +12,11 @@ # Helper to define the RSSHub package makeRSSHub = pkgs: let - pnpm = pkgs.pnpm_9; - deps = pnpm.fetchDeps { + pnpm = pkgs.pnpm_10; + deps = pkgs.fetchPnpmDeps { pname = "rsshub"; src = ./.; - hash = "sha256-ErMPvlOIDqn03s2P+tzbQbYPZFEax5P61O1DJputvo4="; + hash = "sha256-QG1cIkZh+qBA5Dipt0iDLuQpEOI45wdFhuG/CTcRVU8="; fetcherVersion = 2; }; in @@ -28,7 +28,8 @@ nativeBuildInputs = with pkgs; [ nodejs_22 - pnpm.configHook + pnpm + pnpmConfigHook git ]; @@ -166,6 +167,19 @@ ''; }; + environmentFiles = mkOption { + type = types.listOf types.path; + default = [ ]; + example = literalExpression '' + [ config.sops.secrets.rsshub.path ] + ''; + description = '' + Environment variables stored in files for RSSHub. + It can be used for secrets like agenix, sops-nix, etc. + See https://docs.rsshub.app/deploy/config for available options. + ''; + }; + redis = { enable = mkOption { type = types.bool; @@ -238,7 +252,7 @@ User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.dataDir; - EnvironmentFile = environmentFile; + EnvironmentFile = [environmentFile] ++ cfg.environmentFiles; ExecStart = "${cfg.package}/bin/rsshub"; Restart = "on-failure"; RestartSec = "5s"; diff --git a/lib/api/category/one.ts b/lib/api/category/one.ts index b5021b545f668c..a8c8726cb9d2b1 100644 --- a/lib/api/category/one.ts +++ b/lib/api/category/one.ts @@ -45,6 +45,7 @@ const QuerySchema = z.object({ const route = createRoute({ method: 'get', path: '/category/{category}', + description: 'Namespace list filtered by category', tags: ['Category'], request: { query: QuerySchema, @@ -52,7 +53,7 @@ const route = createRoute({ }, responses: { 200: { - description: 'Namespace list by categories and language', + description: 'Namespaces matching the requested category', }, }, }); diff --git a/lib/api/follow/config.ts b/lib/api/follow/config.ts index dc002e1fecc093..4ed8a289c61654 100644 --- a/lib/api/follow/config.ts +++ b/lib/api/follow/config.ts @@ -7,10 +7,11 @@ import { gitDate, gitHash } from '@/utils/git-hash'; const route = createRoute({ method: 'get', path: '/follow/config', + description: 'Follow configuration for the current instance', tags: ['Follow'], responses: { 200: { - description: 'Follow config', + description: 'Follow configuration for the current instance', }, }, }); diff --git a/lib/api/index.ts b/lib/api/index.ts index a3f54b9713fcaf..76ba26e49dddb4 100644 --- a/lib/api/index.ts +++ b/lib/api/index.ts @@ -8,6 +8,7 @@ import { handler as namespaceAllHandler, route as namespaceAllRoute } from '@/ap import { handler as namespaceOneHandler, route as namespaceOneRoute } from '@/api/namespace/one'; import { handler as radarRulesAllHandler, route as radarRulesAllRoute } from '@/api/radar/rules/all'; import { handler as radarRulesOneHandler, route as radarRulesOneRoute } from '@/api/radar/rules/one'; +import { handler as routeStatusHandler, route as routeStatusRoute } from '@/api/route/status'; const app = new OpenAPIHono(); @@ -16,6 +17,7 @@ app.openapi(namespaceOneRoute, namespaceOneHandler); app.openapi(radarRulesAllRoute, radarRulesAllHandler); app.openapi(radarRulesOneRoute, radarRulesOneHandler); app.openapi(categoryOneRoute, categoryOneHandler); +app.openapi(routeStatusRoute, routeStatusHandler); app.openapi(followConfigRoute, followConfigHandler); const docs = app.getOpenAPI31Document({ @@ -30,6 +32,34 @@ for (const path in docs.paths) { delete docs.paths[path]; } app.get('/openapi.json', (ctx) => ctx.json(docs)); -app.get('/reference', Scalar({ content: docs })); +app.get( + '/reference', + Scalar({ + content: docs, + hiddenClients: { + c: true, + clojure: true, + csharp: true, + dart: true, + fsharp: true, + go: false, + http: true, + java: true, + js: true, + kotlin: true, + node: ['axios'], // allow fetch, ofetch, undici + objc: true, + ocaml: true, + php: false, + powershell: true, + python: false, + r: true, + ruby: true, + rust: true, + shell: ['httpie', 'wget'], // allow curl + swift: true, + }, + }) +); export default app; diff --git a/lib/api/namespace/all.ts b/lib/api/namespace/all.ts index 9c08d2775366c9..ac104b3f5f42d8 100644 --- a/lib/api/namespace/all.ts +++ b/lib/api/namespace/all.ts @@ -6,10 +6,11 @@ import { namespaces } from '@/registry'; const route = createRoute({ method: 'get', path: '/namespace', + description: 'Information about all namespaces', tags: ['Namespace'], responses: { 200: { - description: 'Information about all namespaces', + description: 'Namespace registry data for all namespaces', }, }, }); diff --git a/lib/api/namespace/one.ts b/lib/api/namespace/one.ts index 843a72a12eac71..d447e580580273 100644 --- a/lib/api/namespace/one.ts +++ b/lib/api/namespace/one.ts @@ -16,13 +16,14 @@ const ParamsSchema = z.object({ const route = createRoute({ method: 'get', path: '/namespace/{namespace}', + description: 'Information about a namespace', tags: ['Namespace'], request: { params: ParamsSchema, }, responses: { 200: { - description: 'Information about a namespace', + description: 'Namespace registry data for a namespace', }, }, }); diff --git a/lib/api/radar/rules/all.ts b/lib/api/radar/rules/all.ts index ada23fbc12cf3f..d94e925619dbb2 100644 --- a/lib/api/radar/rules/all.ts +++ b/lib/api/radar/rules/all.ts @@ -45,10 +45,11 @@ for (const namespace in namespaces) { const route = createRoute({ method: 'get', path: '/radar/rules', + description: 'All Radar rules grouped by domain', tags: ['Radar'], responses: { 200: { - description: 'All Radar rules', + description: 'Radar rules grouped by domain', }, }, }); diff --git a/lib/api/radar/rules/one.ts b/lib/api/radar/rules/one.ts index ee17cfeeef26be..551f3a696ec0c1 100644 --- a/lib/api/radar/rules/one.ts +++ b/lib/api/radar/rules/one.ts @@ -55,13 +55,14 @@ const ParamsSchema = z.object({ const route = createRoute({ method: 'get', path: '/radar/rules/{domain}', + description: 'Radar rules for a domain name', tags: ['Radar'], request: { params: ParamsSchema, }, responses: { 200: { - description: 'Radar rules for a domain name (does not support subdomains)', + description: 'Radar rules for a domain name (no subdomains)', }, }, }); diff --git a/lib/api/route/status.test.ts b/lib/api/route/status.test.ts new file mode 100644 index 00000000000000..ef582e4fed34a2 --- /dev/null +++ b/lib/api/route/status.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import api from '@/api'; + +const mockHas = vi.hoisted(() => vi.fn()); +const mockGet = vi.hoisted(() => vi.fn()); + +vi.mock('@/utils/cache/index', () => ({ + default: { + status: { available: true }, + globalCache: { + has: mockHas, + get: mockGet, + set: vi.fn(), + }, + tryGet: vi.fn(), + }, +})); + +describe('GET /api/route/status', () => { + beforeEach(() => { + mockHas.mockReset(); + mockGet.mockReset(); + }); + + it('returns 404 when cache is cold', async () => { + mockHas.mockResolvedValue(false); + + const response = await api.request('/route/status?requestPath=/github/comments/DIYgod/RSSHub/20768'); + expect(response.status).toBe(404); + + const data = await response.json(); + expect(data.cached).toBe(false); + expect(data.lastBuildDate).toBeNull(); + }); + + it('returns cached: true with lastBuildDate when cache is warm', async () => { + const mockBuildDate = 'Mon, 1 Jan 2026 10:00:00 GMT'; + mockHas.mockResolvedValue(true); + mockGet.mockResolvedValue( + JSON.stringify({ + lastBuildDate: mockBuildDate, + items: [], + }) + ); + + const response = await api.request('/route/status?requestPath=/github/comments/DIYgod/RSSHub/20768'); + expect(response.status).toBe(200); + + const data = await response.json(); + expect(data.cached).toBe(true); + expect(data.lastBuildDate).toBe(mockBuildDate); + }); + + it('returns 503 when cache is unavailable', async () => { + const { default: cacheModule } = await import('@/utils/cache/index'); + (cacheModule.status as { available: boolean }).available = false; + + try { + const response = await api.request('/route/status?requestPath=/github/comments/DIYgod/RSSHub/20768'); + expect(response.status).toBe(503); + + const data = await response.json(); + expect(data.cached).toBe(false); + } finally { + (cacheModule.status as { available: boolean }).available = true; + } + }); +}); diff --git a/lib/api/route/status.ts b/lib/api/route/status.ts new file mode 100644 index 00000000000000..f7b2e1ca0c083d --- /dev/null +++ b/lib/api/route/status.ts @@ -0,0 +1,88 @@ +import type { RouteHandler } from '@hono/zod-openapi'; +import { createRoute, z } from '@hono/zod-openapi'; +import xxhash from 'xxhash-wasm'; + +import cacheModule from '@/utils/cache/index'; + +const QuerySchema = z.object({ + requestPath: z.string().openapi({ + param: { + name: 'requestPath', + in: 'query', + }, + example: '/github/comments/DIYgod/RSSHub/20768', + description: 'The route path to check cache status for', + }), +}); + +const ResponseSchema = z.object({ + cached: z.boolean(), + lastBuildDate: z.string().nullable(), +}); + +const route = createRoute({ + method: 'get', + path: '/route/status', + description: 'Check if a route path is cached', + tags: ['Route'], + request: { + query: QuerySchema, + }, + responses: { + 200: { + content: { + 'application/json': { + schema: ResponseSchema, + }, + }, + description: 'Cache found', + }, + 404: { + content: { + 'application/json': { + schema: ResponseSchema, + }, + }, + description: 'Cache not found', + }, + 503: { + content: { + 'application/json': { + schema: ResponseSchema, + }, + }, + description: 'Cache module unavailable', + }, + }, +}); + +const handler: RouteHandler = async (ctx) => { + if (!cacheModule.status.available) { + return ctx.json({ cached: false, lastBuildDate: null }, 503); + } + + const { requestPath } = ctx.req.valid('query'); + const { h64ToString } = await xxhash(); + const key = 'rsshub:koa-redis-cache:' + h64ToString(requestPath + ':rss'); + const cached = await cacheModule.globalCache.has(key); + + if (!cached) { + return ctx.json({ cached: false, lastBuildDate: null }, 404); + } + + let lastBuildDate: string | null = null; + + try { + const cachedData = await cacheModule.globalCache.get(key); + if (cachedData) { + const parsed = JSON.parse(cachedData); + lastBuildDate = parsed.lastBuildDate || null; + } + } catch { + // + } + + return ctx.json({ cached, lastBuildDate }, 200); +}; + +export { handler, route }; diff --git a/lib/app.worker.tsx b/lib/app.worker.tsx index 6e8563fab5bdfb..d8c96848d56f55 100644 --- a/lib/app.worker.tsx +++ b/lib/app.worker.tsx @@ -54,7 +54,7 @@ app.use(trace); // Heavy middleware excluded in Worker build: // - sentry: @sentry/node // - antiHotlink: cheerio -// - parameter: cheerio, sanitize-html, @postlight/parser +// - parameter: cheerio, sanitize-html, @jocmp/mercury-parser app.use(cache); app.use(accessControl); diff --git a/lib/config.ts b/lib/config.ts index 7403d58f4bc5d2..61e5204f887269 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -120,9 +120,7 @@ type ConfigEnvKeys = | 'HEFENG_API_HOST' | 'HUITUN_COOKIE' | 'INFZM_COOKIE' - | 'INITIUM_USERNAME' - | 'INITIUM_PASSWORD' - | 'INITIUM_BEARER_TOKEN' + | 'INITIUM_MEMBER_COOKIE' | 'IG_USERNAME' | 'IG_PASSWORD' | 'IG_PROXY' @@ -206,6 +204,8 @@ type ConfigEnvKeys = | 'TUMBLR_REFRESH_TOKEN' | 'TWITTER_CONSUMER_KEY' | 'TWITTER_CONSUMER_SECRET' + | 'TWITTER_ACCESS_TOKEN' + | 'TWITTER_ACCESS_SECRET' // | 'TWITTER_USERNAME' // | 'TWITTER_PASSWORD' // | 'TWITTER_AUTHENTICATION_SECRET' @@ -234,6 +234,7 @@ type ConfigEnvKeys = | 'YOUTUBE_CLIENT_ID' | 'YOUTUBE_CLIENT_SECRET' | 'YOUTUBE_REFRESH_TOKEN' + | 'YOUTUBE_VIDEO_EMBED_URL' | 'ZHIHU_COOKIES' | 'ZODGAME_COOKIE' | 'ZSXQ_ACCESS_TOKEN' @@ -441,9 +442,7 @@ export type Config = { cookie?: string; }; initium: { - username?: string; - password?: string; - bearertoken?: string; + memberCookie?: string; }; instagram: { username?: string; @@ -621,6 +620,8 @@ export type Config = { twitter: { consumerKey?: string; consumerSecret?: string; + accessToken?: string; + accessSecret?: string; // username?: string[]; // password?: string[]; // authenticationSecret?: string[]; @@ -671,6 +672,7 @@ export type Config = { clientId?: string; clientSecret?: string; refreshToken?: string; + videoEmbedUrl?: string; }; zhihu: { cookies?: string; @@ -928,9 +930,7 @@ const calculateValue = () => { cookie: envs.INFZM_COOKIE, }, initium: { - username: envs.INITIUM_USERNAME, - password: envs.INITIUM_PASSWORD, - bearertoken: envs.INITIUM_BEARER_TOKEN, + memberCookie: envs.INITIUM_MEMBER_COOKIE, }, instagram: { username: envs.IG_USERNAME, @@ -1108,6 +1108,8 @@ const calculateValue = () => { twitter: { consumerKey: envs.TWITTER_CONSUMER_KEY, consumerSecret: envs.TWITTER_CONSUMER_SECRET, + accessToken: envs.TWITTER_ACCESS_TOKEN, + accessSecret: envs.TWITTER_ACCESS_SECRET, // username: envs.TWITTER_USERNAME?.split(','), // password: envs.TWITTER_PASSWORD?.split(','), // authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET?.split(','), @@ -1158,6 +1160,7 @@ const calculateValue = () => { clientId: envs.YOUTUBE_CLIENT_ID, clientSecret: envs.YOUTUBE_CLIENT_SECRET, refreshToken: envs.YOUTUBE_REFRESH_TOKEN, + videoEmbedUrl: envs.YOUTUBE_VIDEO_EMBED_URL || 'https://www.youtube-nocookie.com/embed/', }, zhihu: { cookies: envs.ZHIHU_COOKIES, diff --git a/lib/middleware/debug.test.ts b/lib/middleware/debug.test.ts index 8c798260382401..5c2b9e75065f13 100644 --- a/lib/middleware/debug.test.ts +++ b/lib/middleware/debug.test.ts @@ -20,7 +20,8 @@ describe('debug', () => { const response = await app.request('/'); - const $ = load(await response.text()); + const html = await response.text(); + const $ = load(html); $('.debug-item').each((index, item) => { const key = $(item).find('.debug-key').html()?.trim(); const value = $(item).find('.debug-value').html()?.trim(); diff --git a/lib/middleware/parameter.ts b/lib/middleware/parameter.ts index 51478c88f5fc6a..d3b31692c94f02 100644 --- a/lib/middleware/parameter.ts +++ b/lib/middleware/parameter.ts @@ -1,4 +1,4 @@ -import Parser from '@postlight/parser'; +import Parser from '@jocmp/mercury-parser'; import type { CheerioAPI } from 'cheerio'; import { load } from 'cheerio'; import type { Element } from 'domhandler'; @@ -97,7 +97,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => { if (item.link) { let baseUrl = data.link; if (baseUrl && !/^https?:\/\//.test(baseUrl)) { - baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl; + baseUrl = baseUrl.startsWith('//') ? 'http:' + baseUrl : 'http://' + baseUrl; } item.link = new URL(item.link, baseUrl).href; @@ -109,7 +109,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => { let baseUrl = item.link || data.link; if (baseUrl && !/^https?:\/\//.test(baseUrl)) { - baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl; + baseUrl = baseUrl.startsWith('//') ? 'http:' + baseUrl : 'http://' + baseUrl; } $('script').remove(); diff --git a/lib/routes/0x80/index.ts b/lib/routes/0x80/index.ts index 2c5ed27a0260b1..c59a1cf0785c70 100644 --- a/lib/routes/0x80/index.ts +++ b/lib/routes/0x80/index.ts @@ -18,7 +18,6 @@ export const route: Route = { function extractDateFromURL(url: string) { const regex = /\d{4}-\d{2}-\d{2}/; const match = url.match(regex); - return match ? match[0] : null; } diff --git a/lib/routes/0xxx/index.ts b/lib/routes/0xxx/index.ts index 47d7be9a7f230a..f8d319b236b8cf 100644 --- a/lib/routes/0xxx/index.ts +++ b/lib/routes/0xxx/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('table#home-table tr:not(.gore)') + let items: DataItem[] = $('table#home-table tr:not(.gore)') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/10000link/info.ts b/lib/routes/10000link/info.ts index 6fa93593bfa99e..bdffd3c32e1fe2 100644 --- a/lib/routes/10000link/info.ts +++ b/lib/routes/10000link/info.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.l_newshot li dl.lhotnew2') + let items: DataItem[] = $('ul.l_newshot li dl.lhotnew2') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/163/renjian.ts b/lib/routes/163/renjian.ts index 50d78180c91690..c1f92c510a8d6e 100644 --- a/lib/routes/163/renjian.ts +++ b/lib/routes/163/renjian.ts @@ -1,7 +1,7 @@ import { load } from 'cheerio'; import iconv from 'iconv-lite'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -55,7 +55,7 @@ async function handler(ctx) { const data = iconv.decode(response.data, 'gbk'); - let items = {}; + let items: DataItem[]; const urls = data.match(/url:"(.*)",/g); diff --git a/lib/routes/18comic/album.ts b/lib/routes/18comic/album.ts index c2ecd580a14417..575c71d7254808 100644 --- a/lib/routes/18comic/album.ts +++ b/lib/routes/18comic/album.ts @@ -72,7 +72,7 @@ async function handler(ctx) { const chapterResult = await processApiItems(chapterApiUrl); const result = {}; const chapterNum = index + 1; - result.title = `第${String(chapterNum)}話 ${item.name === '' ? `${String(chapterNum)}` : item.name}`; + result.title = `第${chapterNum}話 ${item.name === '' ? chapterNum : item.name}`; result.link = `${rootUrl}/photo/${item.id}`; result.guid = `${rootUrl}/photo/${item.id}`; result.updated = new Date(chapterResult.addtime * 1000); diff --git a/lib/routes/199it/index.tsx b/lib/routes/199it/index.tsx index 478c221c1aaead..de4747a76d2353 100644 --- a/lib/routes/199it/index.tsx +++ b/lib/routes/199it/index.tsx @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('article.newsplus') + let items: DataItem[] = $('article.newsplus') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/2048/index.tsx b/lib/routes/2048/index.tsx index be0410e22a4a2c..2c95599aaaa9af 100644 --- a/lib/routes/2048/index.tsx +++ b/lib/routes/2048/index.tsx @@ -169,7 +169,7 @@ async function handler(ctx) { const downloadLink = content('#read_tpc').first().find('a').last(); const copyLink = content('#copytext')?.first()?.text(); - if (downloadLink?.text()?.startsWith('http') && /bt\.azvmw\.com$/.test(new URL(downloadLink.text()).hostname)) { + if (new URL(downloadLink.text()).hostname === 'bt.azvmw.com') { const torrentResponse = await ofetch(downloadLink.text()); const torrent = load(torrentResponse); diff --git a/lib/routes/21caijing/channel.ts b/lib/routes/21caijing/channel.ts index 4b06de223d8453..1678d0a89f60b4 100644 --- a/lib/routes/21caijing/channel.ts +++ b/lib/routes/21caijing/channel.ts @@ -85,9 +85,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = JSON.parse(response) + let items: DataItem[] = JSON.parse(response) .slice(0, limit) .map((item): DataItem => { const title: string = item.title; diff --git a/lib/routes/423down/index.ts b/lib/routes/423down/index.ts index 51bec746302fbb..810d2740892ec8 100644 --- a/lib/routes/423down/index.ts +++ b/lib/routes/423down/index.ts @@ -27,8 +27,9 @@ export const handler = async (ctx) => { item = $(item); const link = item.find('h2 a').prop('href'); + const isAdItem = item.find('span.cat').text().includes('423Down'); - return new RegExp(domain).test(link); + return new RegExp(domain).test(link) && !isAdItem; }) .slice(0, limit) .map((item) => { diff --git a/lib/routes/4chan/catalog.tsx b/lib/routes/4chan/catalog.tsx new file mode 100644 index 00000000000000..ba04087e76feec --- /dev/null +++ b/lib/routes/4chan/catalog.tsx @@ -0,0 +1,51 @@ +import type { Context } from 'hono'; + +import type { Route } from '@/types'; +import got from '@/utils/got'; + +import type { CatalogApiReturn } from './utils'; +import { parseParams, processCatalog } from './utils'; + +const handler = async (ctx: Context) => { + const { board } = ctx.req.param(); + const viewOptions = parseParams(ctx.req.param('routeParams')); + const { data }: { data: CatalogApiReturn } = await got(`https://a.4cdn.org/${board}/catalog.json`); + + return { + title: `4chan's /${board}/`, + link: `https://boards.4chan.org/${board}/catalog`, + item: processCatalog({ data, board, viewOptions }), + }; +}; + +export const route: Route = { + path: '/:board/catalog/:routeParams?', + categories: ['bbs'], + example: '/4chan/g/catalog', + parameters: { + board: '4chan board', + routeParams: 'extra parameters, see the table above', + }, + features: { + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: "Board's catalog", + maintainers: ['heisenshark'], + radar: [ + { + source: ['boards.4chan.org/:board/'], + target: '/:board/catalog', + }, + ], + description: `Specify options (in the format of query string) in parameter \`routeParams\` to control some extra features for Tweets +| Key | Description | Accepts | Defaults to | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------- | ----------------------------------------- | +| \`showReplyCount\` | Show number of replies of each thread in catalog | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` | +| \`showLastReplies\` | Show last 5 replies of each thread | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` | +| \`revealSpoilers\` | Don't wrap images tagged as spoilers | \`0\`/\`1\`/\`true\`/\`false\` | \`false\` |`, + handler, +}; diff --git a/lib/routes/4chan/namespace.ts b/lib/routes/4chan/namespace.ts new file mode 100644 index 00000000000000..1c7e8172eddd6b --- /dev/null +++ b/lib/routes/4chan/namespace.ts @@ -0,0 +1,9 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: '4chan', + url: '4chan.org', + categories: ['bbs'], + lang: 'en', + description: '', +}; diff --git a/lib/routes/4chan/utils.tsx b/lib/routes/4chan/utils.tsx new file mode 100644 index 00000000000000..bc5b3631df82d5 --- /dev/null +++ b/lib/routes/4chan/utils.tsx @@ -0,0 +1,124 @@ +import { raw } from 'hono/html'; +import { renderToString } from 'hono/jsx/dom/server'; +import sanitizeHtml from 'sanitize-html'; + +import { parseDate } from '@/utils/parse-date'; +import { queryToBoolean } from '@/utils/readable-social'; + +const parseParams = (routeParams: string) => { + const parsed = new URLSearchParams(routeParams); + const viewOptions = { + includeLastReplies: !!queryToBoolean(parsed.get('showLastReplies')), + includeReplyCount: !!queryToBoolean(parsed.get('showReplyCount')), + revealSpoilers: !!queryToBoolean(parsed.get('revealSpoilers')), + }; + return viewOptions; +}; + +const processCatalog = ({ data, board, viewOptions }: { data: CatalogApiReturn; board: string; viewOptions: ViewOptions }) => { + const transformedData = data.flatMap((page) => page.threads); + return transformedData.map((thread) => ({ + author: `${thread.name} ${thread.trip ?? thread.no}`, + description: renderToString(renderPost({ post: thread, board, viewOptions })), + link: `https://boards.4chan.org/${board}/thread/${thread.no}`, + pubDate: parseDate(thread.time * 1000), + title: thread.sub ?? sanitizeHtml(thread.com?.split('
')[0] ?? '', { allowedTags: [] }), + })); +}; + +const renderPost = ({ post, board, viewOptions }: { post: ChanPost; board: string; viewOptions: ViewOptions }) => { + let media = <>; + switch (post.ext) { + case '.jpg': + case '.png': + case '.gif': + media = ; + break; + case '.pdf': + media = ; + break; + case '.swf': + media = ; + break; + case '.webm': + media = ; + break; + default: + break; + } + if (post.spoiler) { + media = ( +
+ Spoiler + {media} +
+ ); + } + media = ( + <> +
{media}
+ + ); + const renderedPost = ( + <> + {post.last_replies && viewOptions.includeReplyCount && ( + <> + {post.replies} 💬 +
+ + )} + {raw(post.com ?? '')} + {media} + {viewOptions.includeLastReplies && post.last_replies?.map((n) =>
{renderPost({ post: n, board, viewOptions })}
)} + + ); + return renderedPost; +}; + +type CatalogApiReturn = Array<{ page: number; threads: ChanPost[] }>; +type ViewOptions = ReturnType; + +interface ChanPost { + no: number; + resto: number; + sticky?: number; + closed?: number; + now: string; + time: number; + name: string; + trip?: string; + id?: string; + capcode?: string; + country?: string; + country_name?: string; + sub?: string; + com?: string; + tim?: number; + filename: string; + ext: '.jpg' | '.png' | '.gif' | '.pdf' | '.swf' | '.webm'; + fsize: number; + md5: string; + w?: number; + h?: number; + tn_w?: number; + tn_h?: number; + filedeleted?: 1; + spoiler?: 1; + custom_spoiler?: 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10; + omitted_posts?: number; + omitted_images?: number; + replies?: number; + images?: number; + bumplimit?: 1; + imagelimit?: 1; + last_modified?: number; + tag?: string; + semantic_url?: string; + since4pass?: number; + unique_ips?: number; + m_img?: 1; + last_replies?: ChanPost[]; +} + +export type { CatalogApiReturn, ChanPost, ViewOptions }; +export { parseParams, processCatalog, renderPost }; diff --git a/lib/routes/4gamers/category.ts b/lib/routes/4gamers/category.ts index 076131dc16e06e..e2584f6994020b 100644 --- a/lib/routes/4gamers/category.ts +++ b/lib/routes/4gamers/category.ts @@ -33,10 +33,9 @@ async function handler(ctx) { const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseItem(item)))); - let categories = []; let categoryName = '最新消息'; if (!isLatest) { - categories = await getCategories(cache.tryGet); + const categories = await getCategories(cache.tryGet); categoryName = categories.find((c) => c.id === Number.parseInt(category)).name; } diff --git a/lib/routes/50forum/zhuanjia.ts b/lib/routes/50forum/zhuanjia.ts index db5c80dad20f95..580a506836ef23 100644 --- a/lib/routes/50forum/zhuanjia.ts +++ b/lib/routes/50forum/zhuanjia.ts @@ -10,7 +10,11 @@ export const route: Route = { path: '/', radar: [ { - source: ['www.50forum.org.cn/portal/list/index.html?id=6', '50forum.org.cn/'], + source: ['www.50forum.org.cn/portal/list/index.html?id=6'], + target: '', + }, + { + source: ['50forum.org.cn/'], target: '', }, ], diff --git a/lib/routes/51cto/recommend.ts b/lib/routes/51cto/recommend.ts index d18858624eda3d..1a5c218713510a 100644 --- a/lib/routes/51cto/recommend.ts +++ b/lib/routes/51cto/recommend.ts @@ -27,7 +27,6 @@ export const route: Route = { const pattern = /'(WTKkN|bOYDu|wyeCN)':\s*(\d+)/g; async function getFullcontent(item, cookie = '') { - let fullContent: null | string = null; const articleResponse = await ofetch(item.url, { headers: { cookie, @@ -35,7 +34,7 @@ async function getFullcontent(item, cookie = '') { }); const $ = load(articleResponse); - fullContent = new URL(item.url).host === 'ost.51cto.com' ? $('.posts-content').html() : $('article').html(); + const fullContent = new URL(item.url).host === 'ost.51cto.com' ? $('.posts-content').html() : $('article').html(); if (!fullContent && cookie === '') { // If fullContent is null and haven't tried to request with cookie, try to get fullContent with cookie diff --git a/lib/routes/51read/article.ts b/lib/routes/51read/article.ts index 90018f53eaffe2..35e7ffa4eab7f6 100644 --- a/lib/routes/51read/article.ts +++ b/lib/routes/51read/article.ts @@ -36,10 +36,12 @@ export const route: Route = { async function handler(ctx) { const { id } = ctx.req.param(); const link = `https://m.51read.org/xiaoshuo/${id}`; - const $book = load(await ofetch(link)); + const bookHtml = await ofetch(link); + const $book = load(bookHtml); const chapter = `https://m.51read.org/zhangjiemulu/${id}`; - const $chapter = load(await ofetch(chapter)); + const chapterHtml = await ofetch(chapter); + const $chapter = load(chapterHtml); const pageLength = $chapter('.ml-page select') .find('option') @@ -61,7 +63,8 @@ async function handler(ctx) { const createItem = async (baseUrl: string, page: number) => { const url = `${baseUrl}/${page}`; - const $latest = load(await ofetch(url)); + const html = await ofetch(url); + const $latest = load(html); const item = await Promise.all( $latest('.kb-jp li>a') .toArray() @@ -73,7 +76,8 @@ const createItem = async (baseUrl: string, page: number) => { const buildItem = (url: string) => cache.tryGet(url, async () => { - const $ = load(await ofetch(url)); + const html = await ofetch(url); + const $ = load(html); return { title: $('h1').text(), diff --git a/lib/routes/69shu/article.ts b/lib/routes/69shu/article.ts index 329038e67adbae..bc13b6c30ce7b6 100644 --- a/lib/routes/69shu/article.ts +++ b/lib/routes/69shu/article.ts @@ -29,7 +29,8 @@ export const route: Route = { handler: async (ctx) => { const { id } = ctx.req.param(); const link = `https://www.69shuba.cx/book/${id}.htm`; - const $ = load(await get(link)); + const html = await get(link); + const $ = load(html); const item = await Promise.all( $('.qustime li>a') @@ -51,7 +52,8 @@ export const route: Route = { const createItem = (url: string) => cache.tryGet(url, async () => { - const $ = load(await get(url)); + const html = await get(url); + const $ = load(html); const { articleid, chapterid, chaptername } = parseObject(/bookinfo\s?=\s?{[\S\s]+?}/, $('head>script:not([src])').text()); const decryptionMap = parseObject(/_\d+\s?=\s?{[\S\s]+?}/, $('.txtnav+script').text()); diff --git a/lib/routes/6v123/index.ts b/lib/routes/6v123/index.ts index a6d3ec9e5af328..ef0a69c95dff9c 100644 --- a/lib/routes/6v123/index.ts +++ b/lib/routes/6v123/index.ts @@ -25,9 +25,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(iconv.decode(Buffer.from(response), encoding)); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.list li') + let items: DataItem[] = $('ul.list li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/8kcos/article.ts b/lib/routes/8kcos/article.ts deleted file mode 100644 index f6c2bdb1165c0b..00000000000000 --- a/lib/routes/8kcos/article.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { load } from 'cheerio'; - -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; - -async function loadArticle(link) { - const resp = await got(link); - const article = load(resp.body); - const entryChildren = article('div.entry-content').children(); - const imgs = entryChildren - .find('noscript') - .toArray() - .map((e) => e.children[0].data); - const txt = entryChildren - .slice(2) - .toArray() - .map((e) => load(e).html()); - return { - title: article('.entry-title').text(), - description: [...imgs, ...txt].join(''), - pubDate: parseDate(article('time')[0].attribs.datetime), - link, - }; -} -export default loadArticle; diff --git a/lib/routes/8kcos/cat.ts b/lib/routes/8kcos/cat.ts index 816c3525e89220..f758aa1e5bbe43 100644 --- a/lib/routes/8kcos/cat.ts +++ b/lib/routes/8kcos/cat.ts @@ -1,22 +1,21 @@ -import { load } from 'cheerio'; - import type { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import loadArticle from './article'; -import { SUB_NAME_PREFIX, SUB_URL } from './const'; +import { getCategoryInfo, getPosts } from './utils'; export const route: Route = { - path: '/cat/:cat{.+}?', + path: '/cat/:cat?', + parameters: { + cat: '默认值为 `8kasianidol`，将目录页面url中 /category/ 后面的部分填入。如：https://www.8kcosplay.com/category/8kchineseidol/%e9%a3%8e%e4%b9%8b%e9%a2%86%e5%9f%9f/ 对应的RSS页面为 /8kcos/cat/%e9%a3%8e%e4%b9%8b%e9%a2%86%e5%9f%9f。', + }, + example: '/8kcos/cat/8kasianidol', radar: [ { - source: ['8kcosplay.com/'], - target: '', + source: ['8kcosplay.com/category/:mainCategory/:cat/', '8kcosplay.com/category/:cat/'], + target: '/cat/:cat', }, ], - name: 'Unknown', - maintainers: [], + name: '分类', + maintainers: ['KotoriK'], handler, url: '8kcosplay.com/', features: { @@ -25,20 +24,15 @@ export const route: Route = { }; async function handler(ctx) { - const limit = Number.parseInt(ctx.req.query('limit')); + const limit = Number.parseInt(ctx.req.query('limit') ?? 10, 10); const { cat = '8kasianidol' } = ctx.req.param(); - const url = `${SUB_URL}category/${cat}/`; - const resp = await got(url); - const $ = load(resp.body); - const itemRaw = $('li.item').toArray(); + const categoryInfo = await getCategoryInfo(cat); + const items = await getPosts(limit, { categories: categoryInfo.id }); + return { - title: `${SUB_NAME_PREFIX}-${$('span[property=name]:not(.hide)').text()}`, - link: url, - item: await Promise.all( - (limit ? itemRaw.slice(0, limit) : itemRaw).map((e) => { - const { href } = load(e)('h2 > a')[0].attribs; - return cache.tryGet(href, () => loadArticle(href)); - }) - ), + title: categoryInfo.title, + description: categoryInfo.description, + link: categoryInfo.link, + item: items, }; } diff --git a/lib/routes/8kcos/const.ts b/lib/routes/8kcos/const.ts deleted file mode 100644 index b3d599140873d8..00000000000000 --- a/lib/routes/8kcos/const.ts +++ /dev/null @@ -1,4 +0,0 @@ -const SUB_NAME_PREFIX = '8KCosplay'; -const SUB_URL = 'https://www.8kcosplay.com/'; - -export { SUB_NAME_PREFIX, SUB_URL }; diff --git a/lib/routes/8kcos/latest.ts b/lib/routes/8kcos/latest.ts index 8a9860e8d957f4..3fb94126373cac 100644 --- a/lib/routes/8kcos/latest.ts +++ b/lib/routes/8kcos/latest.ts @@ -1,18 +1,11 @@ -import { load } from 'cheerio'; - import type { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; - -import loadArticle from './article'; -import { SUB_NAME_PREFIX, SUB_URL } from './const'; -const url = SUB_URL; +import { getPosts, SUB_NAME_PREFIX, SUB_URL } from './utils'; export const route: Route = { path: '/', categories: ['picture'], - example: '/8kcos/', + example: '/8kcos', parameters: {}, features: { requireConfig: false, @@ -26,7 +19,6 @@ export const route: Route = { radar: [ { source: ['8kcosplay.com/'], - target: '', }, ], name: '最新', @@ -36,19 +28,11 @@ export const route: Route = { }; async function handler(ctx) { - const limit = Number.parseInt(ctx.req.query('limit')); - const response = await got(url); - const itemRaw = load(response.body)('ul.post-loop li.item').toArray(); + const limit = Number.parseInt(ctx.req.query('limit') ?? 10, 10); + const items = await getPosts(limit); return { title: `${SUB_NAME_PREFIX}-最新`, - link: url, - item: - response.body && - (await Promise.all( - (limit ? itemRaw.slice(0, limit) : itemRaw).map((e) => { - const { href } = load(e)('h2 > a')[0].attribs; - return cache.tryGet(href, () => loadArticle(href)); - }) - )), + link: SUB_URL, + item: items, }; } diff --git a/lib/routes/8kcos/tag.ts b/lib/routes/8kcos/tag.ts index c86040dfa91199..3ffbad1fb87834 100644 --- a/lib/routes/8kcos/tag.ts +++ b/lib/routes/8kcos/tag.ts @@ -1,11 +1,6 @@ -import { load } from 'cheerio'; - import type { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import loadArticle from './article'; -import { SUB_NAME_PREFIX, SUB_URL } from './const'; +import { getPosts, getTagInfo, SUB_URL } from './utils'; export const route: Route = { path: '/tag/:tag', @@ -33,21 +28,14 @@ export const route: Route = { }; async function handler(ctx) { - const limit = Number.parseInt(ctx.req.query('limit')); + const limit = Number.parseInt(ctx.req.query('limit') ?? 10, 10); const tag = ctx.req.param('tag'); - const url = `${SUB_URL}tag/${tag}/`; - const resp = await got(url); - const $ = load(resp.body); - const itemRaw = $('li.item').toArray(); + const tagInfo = await getTagInfo(tag); + const items = await getPosts(limit, { tags: tagInfo.id }); return { - title: `${SUB_NAME_PREFIX}-${$('span[property=name]:not(.hide)').text()}`, - link: url, - item: await Promise.all( - (limit ? itemRaw.slice(0, limit) : itemRaw).map((e) => { - const { href } = load(e)('h2 > a')[0].attribs; - return cache.tryGet(href, () => loadArticle(href)); - }) - ), + title: `${tagInfo.title}`, + link: `${SUB_URL}/tag/${tag}/`, + item: items, }; } diff --git a/lib/routes/8kcos/utils.ts b/lib/routes/8kcos/utils.ts new file mode 100644 index 00000000000000..9bb08bc5b04cda --- /dev/null +++ b/lib/routes/8kcos/utils.ts @@ -0,0 +1,62 @@ +import type { DataItem } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const SUB_NAME_PREFIX = '8KCosplay'; +export const SUB_URL = 'https://www.8kcosplay.com'; + +export const getPosts = async (limit: number, options?: { categories?: number; tags?: number }) => { + const data = await ofetch(`https://www.8kcosplay.com/wp-json/wp/v2/posts`, { + query: { + per_page: limit, + _embed: '', + ...options, + }, + }); + return data.map((item) => ({ + title: item.title.rendered, + description: item.content.rendered, + link: item.link, + pubDate: parseDate(item.date_gmt), + author: item._embedded?.['author']?.map((a) => a.name).join(', '), + category: item._embedded?.['wp:term']?.flatMap((terms) => terms.map((t) => t.name)), + })) satisfies DataItem[]; +}; + +export const getCategoryInfo = (category: string) => + cache.tryGet(`8kcosplay:category:${category}`, async () => { + const data = await ofetch(`https://www.8kcosplay.com/wp-json/wp/v2/categories`, { + query: { + slug: category, + }, + }); + const categoryInfo = data[0]; + if (!categoryInfo) { + throw new Error('Category not found'); + } + return { + id: categoryInfo.id, + title: categoryInfo.yoast_head_json.title, + description: categoryInfo.description, + link: categoryInfo.link, + }; + }); + +export const getTagInfo = (tag: string) => + cache.tryGet(`8kcosplay:tag:${tag}`, async () => { + const data = await ofetch(`https://www.8kcosplay.com/wp-json/wp/v2/tags`, { + query: { + slug: tag, + }, + }); + const tagInfo = data[0]; + if (!tagInfo) { + throw new Error('Tag not found'); + } + return { + id: tagInfo.id, + title: tagInfo.yoast_head_json.title, + description: tagInfo.description, + }; + }); diff --git a/lib/routes/aa1/60s.ts b/lib/routes/aa1/60s.ts index 597e9990358284..80ddbc69207b12 100644 --- a/lib/routes/aa1/60s.ts +++ b/lib/routes/aa1/60s.ts @@ -41,9 +41,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = response.slice(0, limit).map((item): DataItem => { + const items: DataItem[] = response.slice(0, limit).map((item): DataItem => { const title: string = item.title?.rendered ?? item.title; const description: string | undefined = item.content.rendered; const pubDate: number | string = item.date_gmt; diff --git a/lib/routes/abc/index.ts b/lib/routes/abc/index.ts index 255367b0db6fa9..8ac465c92f7fd5 100644 --- a/lib/routes/abc/index.ts +++ b/lib/routes/abc/index.ts @@ -40,7 +40,7 @@ async function handler(ctx) { const rootUrl = 'https://www.abc.net.au'; const apiUrl = new URL('news-web/api/loader/channelrefetch', rootUrl).href; - let currentUrl = ''; + let currentUrl: string; let documentId; if (Number.isNaN(category)) { diff --git a/lib/routes/abc/templates/description.tsx b/lib/routes/abc/templates/description.tsx index ab5f1a14702556..334ec2e93c36f4 100644 --- a/lib/routes/abc/templates/description.tsx +++ b/lib/routes/abc/templates/description.tsx @@ -1,5 +1,6 @@ import { raw } from 'hono/html'; import { renderToString } from 'hono/jsx/dom/server'; +import type { JSX } from 'hono/jsx/jsx-runtime'; type DescriptionData = { image?: { diff --git a/lib/routes/adquan/case-library.ts b/lib/routes/adquan/case-library.ts index b604c4a74372af..49bc2804798f5c 100644 --- a/lib/routes/adquan/case-library.ts +++ b/lib/routes/adquan/case-library.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.article_1') + let items: DataItem[] = $('div.article_1') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/adquan/index.ts b/lib/routes/adquan/index.ts index 706f904a9d8072..5a11ff58e430e1 100644 --- a/lib/routes/adquan/index.ts +++ b/lib/routes/adquan/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.article_1') + let items: DataItem[] = $('div.article_1') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/aflcio/blog.ts b/lib/routes/aflcio/blog.ts index ba9d30a3de4c3e..55fe585424896d 100644 --- a/lib/routes/aflcio/blog.ts +++ b/lib/routes/aflcio/blog.ts @@ -19,9 +19,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('article.article') + let items: DataItem[] = $('article.article') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/ainvest/article.ts b/lib/routes/ainvest/article.ts index 649e2bb327223e..77f827f4b0d043 100644 --- a/lib/routes/ainvest/article.ts +++ b/lib/routes/ainvest/article.ts @@ -1,8 +1,5 @@ +import { fetchContentItems } from '@/routes/ainvest/utils'; import type { Route } from '@/types'; -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; - -import { decryptAES, encryptAES, getHeaders, randomString } from './utils'; export const route: Route = { path: '/article', @@ -19,49 +16,22 @@ export const route: Route = { }, radar: [ { - source: ['ainvest.com/news'], + source: ['www.ainvest.com/news/articles-latest/', 'www.ainvest.com'], }, ], name: 'Latest Article', maintainers: ['TonyRL'], handler, - url: 'ainvest.com/news', + url: 'www.ainvest.com/news/articles-latest/', }; async function handler(ctx) { - const key = randomString(16); - - const { data: response } = await got.post('https://api.ainvest.com/gw/socialcenter/v1/edu/article/listArticle', { - headers: getHeaders(key), - searchParams: { - timestamp: Date.now(), - }, - data: encryptAES( - JSON.stringify({ - batch: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30, - startId: null, - tags: { - in: ['markettrends', 'premarket', 'companyinsights', 'macro'], - and: ['web', 'creationplatform'], - }, - }), - key - ), - }); - - const { data } = JSON.parse(decryptAES(response, key)); - - const items = data.map((item) => ({ - title: item.title, - description: item.content, - link: item.sourceUrl, - pubDate: parseDate(item.postDate, 'x'), - category: [item.nickName, ...item.tags.map((tag) => tag.code)], - })); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 5; + const items = await fetchContentItems([109], limit); return { title: 'AInvest - Latest Articles', - link: 'https://www.ainvest.com/news', + link: 'https://www.ainvest.com/news/articles-latest/', language: 'en', item: items, }; diff --git a/lib/routes/ainvest/news.ts b/lib/routes/ainvest/news.ts index ab639f9584167a..cf45cb879db539 100644 --- a/lib/routes/ainvest/news.ts +++ b/lib/routes/ainvest/news.ts @@ -1,9 +1,6 @@ +import { fetchContentItems } from '@/routes/ainvest/utils'; import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; - -import { decryptAES, getHeaders, randomString } from './utils'; export const route: Route = { path: '/news', @@ -21,46 +18,23 @@ export const route: Route = { }, radar: [ { - source: ['ainvest.com/news'], + source: ['www.ainvest.com/news/'], }, ], name: 'Latest News', maintainers: ['TonyRL'], handler, - url: 'ainvest.com/news', + url: 'www.ainvest.com/news/', }; async function handler(ctx) { - const key = randomString(16); - - const { data: response } = await got('https://api.ainvest.com/gw/news_f10/v1/newsFlash/getNewsData', { - headers: getHeaders(key), - searchParams: { - terminal: 'web', - tab: 'all', - page: 1, - size: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50, - lastId: '', - timestamp: Date.now(), - }, - }); - - const { data } = JSON.parse(decryptAES(response, key)); - - const items = data.content.map((item) => ({ - title: item.title, - description: item.content, - link: item.sourceUrl, - pubDate: parseDate(item.publishTime, 'x'), - category: item.tagList.map((tag) => tag.nameEn), - author: item.userInfo.nickname, - upvotes: item.likeCount, - comments: item.commentCount, - })); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 5; + const streamIds = [109, 416, 438, 529, 721, 834, 835]; + const items = await fetchContentItems(streamIds, limit); return { title: 'AInvest - Latest News', - link: 'https://www.ainvest.com/news', + link: 'https://www.ainvest.com/news/', language: 'en', item: items, }; diff --git a/lib/routes/ainvest/utils.ts b/lib/routes/ainvest/utils.ts index 49d15799aa606e..9c196820a0f0cb 100644 --- a/lib/routes/ainvest/utils.ts +++ b/lib/routes/ainvest/utils.ts @@ -1,69 +1,54 @@ -import crypto from 'node:crypto'; - -import CryptoJS from 'crypto-js'; -import { hextob64, KEYUTIL, KJUR } from 'jsrsasign'; - -const publicKey = - 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCARnxLlrhTK28bEV7s2IROjT73KLSjfqpKIvV8L+Yhe4BrF0Ut4oOH728HZlbSF0C3N0vXZjLAFesoS4v1pYOjVCPXl920Lh2seCv82m0cK78WMGuqZTfA44Nv7JsQMHC3+J6IZm8YD53ft2d8mYBFgKektduucjx8sObe7eRyoQIDAQAB'; - -const randomString = (length: number) => { - if (length > 32) { - throw new Error('Max length is 32.'); - } - return uuidv4().replaceAll('-', '').slice(0, length); -}; - -const uuidv4 = () => crypto.randomUUID(); - -/** - * @param {string} str - * @returns {CryptoJS.lib.WordArray} - */ -const MD5 = (str) => CryptoJS.MD5(str); - -const encryptAES = (data, key) => { - if (typeof key === 'string') { - key = MD5(key); - } - return CryptoJS.AES.encrypt(CryptoJS.enc.Utf8.parse(data), key, { - mode: CryptoJS.mode.ECB, - padding: CryptoJS.pad.Pkcs7, - }).toString(); +import { config } from '@/config'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +const contentStreamUrl = 'https://news.ainvest.com/news-w-ds-hxcmp-content-stream/content_stream/api/stream_item/v1/query_content_stream'; +const contentPageUrl = 'https://news.ainvest.com/content-page/v1/page'; + +const normalizeItem = (item) => ({ + title: item.title, + link: item.h5_url, + pubDate: parseDate(item.ctime, 'X'), + category: item.content_tags.map((tag) => tag.name), + author: item.author_name, + image: item.cover_image, + seoKey: item.seo_key, +}); + +export const fetchContentStream = (streamId, limit) => + cache.tryGet( + `ainvest:news:${streamId}:${limit}`, + async () => { + const response = await ofetch(contentStreamUrl, { + query: { + lang: 'en', + pageSize: limit, + stream_id: streamId, + page: 1, + }, + }); + return response.data.list; + }, + config.cache.routeExpire, + false + ); + +export const fetchContentItems = async (streamIds, limit) => { + const streams = await Promise.all(streamIds.map((streamId) => fetchContentStream(streamId, limit))); + const list = streams.flat().map((item) => normalizeItem(item)); + + return Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const response = await ofetch(`${contentPageUrl}/${item.seoKey}`); + const { data } = response; + + item.description = data.pageInfo.structuredContent.map((c) => c.content).join(''); + item.image = data.contentInfo.coverImage; + + return item; + }) + ) + ); }; - -const decryptAES = (data, key) => { - if (typeof key === 'string') { - key = MD5(key); - } - return CryptoJS.AES.decrypt(data, key, { - mode: CryptoJS.mode.ECB, - padding: CryptoJS.pad.Pkcs7, - }).toString(CryptoJS.enc.Utf8); -}; - -const encryptRSA = (data) => { - // Original code: - // var n = new JSEncrypt(); - // n.setPublicKey(pubKey); - // return n.encrypt(message); - // Note: Server will reject the public key if it's encrypted using crypto.publicEncrypt(). - let pubKey = `-----BEGIN PUBLIC KEY-----${publicKey}-----END PUBLIC KEY-----`; - pubKey = KEYUTIL.getKey(pubKey); - return hextob64(KJUR.crypto.Cipher.encrypt(data, pubKey)); -}; - -const getHeaders = (key) => { - const fingerPrint = uuidv4(); - - return { - 'content-type': 'application/json', - 'ovse-trace': uuidv4(), - callertype: 'USER', - fingerprint: encryptAES(fingerPrint, MD5(key)), - onetimeskey: encryptRSA(key), - timestamp: encryptAES(Date.now(), key), - referer: 'https://www.ainvest.com/', - }; -}; - -export { decryptAES, encryptAES, getHeaders, randomString }; diff --git a/lib/routes/amazfitwatchfaces/index.ts b/lib/routes/amazfitwatchfaces/index.ts index 1198f2fb83ce59..d6c10fb11a7a56 100644 --- a/lib/routes/amazfitwatchfaces/index.ts +++ b/lib/routes/amazfitwatchfaces/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.wf-panel') + let items: DataItem[] = $('div.wf-panel') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/anthropic/news.ts b/lib/routes/anthropic/news.ts index e827f9c10d59ff..9708029ef63779 100644 --- a/lib/routes/anthropic/news.ts +++ b/lib/routes/anthropic/news.ts @@ -25,20 +25,20 @@ async function handler(ctx) { const link = 'https://www.anthropic.com/news'; const response = await ofetch(link); const $ = load(response); - const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20; + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 10; - const list: DataItem[] = $('.contentFadeUp a') + const list: DataItem[] = $('[class^="PublicationList-module-scss-module__"][class$="__list"] a') .toArray() .slice(0, limit) .map((el) => { const $el = $(el); - const title = $el.find('h3').text().trim(); + const title = $el.find('[class*="__title"]').text().trim(); const href = $el.attr('href') ?? ''; - const pubDate = $el.find('p.detail-m.agate').text().trim() || $el.find('div[class^="PostList_post-date__"]').text().trim(); // legacy selector used roughly before Jan 2025 - const fullLink = href.startsWith('http') ? href : `https://www.anthropic.com${href}`; + const pubDate = $el.find('time').text().trim(); + const link = href.startsWith('http') ? href : `https://www.anthropic.com${href}`; return { title, - link: fullLink, + link, pubDate, }; }); @@ -52,15 +52,7 @@ async function handler(ctx) { const content = $('#main-content'); - // Remove meaningless information (heading, sidebar, quote carousel, footer and codeblock controls) - $(` - [class^="PostDetail_post-heading"], - [class^="ArticleDetail_sidebar-container"], - [class^="QuoteCarousel_carousel-controls"], - [class^="PostDetail_b-social-share"], - [class^="LandingPageSection_root"], - [class^="CodeBlock_controls"] - `).remove(); + $('[class$="__header"], [class$="__socialShare"], [class*="__carousel-controls"], [class^="LandingPageSection-module-scss-module__"]').remove(); content.find('img').each((_, e) => { const $e = $(e); diff --git a/lib/routes/apple/apps.ts b/lib/routes/apple/apps.ts index fc74125f9f7690..c9fd61023c3cfa 100644 --- a/lib/routes/apple/apps.ts +++ b/lib/routes/apple/apps.ts @@ -1,4 +1,4 @@ -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import { ViewType } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; @@ -116,8 +116,8 @@ async function handler(ctx) { const artistName = attributes.artistName; const platformAttributes = attributes.platformAttributes; - let items = []; - let title = ''; + let items: DataItem[] = []; + let title: string; let description = ''; let image = ''; diff --git a/lib/routes/apple/podcast.ts b/lib/routes/apple/podcast.ts index d73d90fe341353..ecca6a797c5551 100644 --- a/lib/routes/apple/podcast.ts +++ b/lib/routes/apple/podcast.ts @@ -43,8 +43,9 @@ async function handler(ctx) { const $ = load(response); - const serializedServerData = JSON.parse($('#serialized-server-data').text()); - const header = serializedServerData[0].data.shelves.find((item) => item.contentType === 'showHeaderRegular').items[0]; + const rawServerData = JSON.parse($('#serialized-server-data').text()); + const serverData = (Array.isArray(rawServerData) ? rawServerData : rawServerData.data)[0].data; + const header = serverData.shelves.find((item) => item.contentType === 'showHeaderRegular').items[0]; const bearerToken = await cache.tryGet( 'apple:podcast:bearer', @@ -93,7 +94,7 @@ async function handler(ctx) { }; }); - const channel = episodeReponse.data.find((d) => d.type === 'podcast-episodes').relationships.channel.data.find((d) => d.type === 'podcast-channels')?.attributes; + const channel = episodeReponse.data.find((d) => d.type === 'podcast-episodes')?.relationships?.channel?.data?.find((d) => d.type === 'podcast-channels')?.attributes; return { title: channel?.name ?? header.title, diff --git a/lib/routes/apple/security-releases.ts b/lib/routes/apple/security-releases.ts index f3d3c7a27a4854..ff1a9d3f7dc034 100644 --- a/lib/routes/apple/security-releases.ts +++ b/lib/routes/apple/security-releases.ts @@ -27,9 +27,7 @@ export const handler = async (ctx: Context): Promise => { .toArray() .map((el) => $(el).text()); - let items: DataItem[] = []; - - items = $trEls + let items: DataItem[] = $trEls .slice(1, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/arcteryx/regear-new-arrivals.tsx b/lib/routes/arcteryx/regear-new-arrivals.tsx index 14219e87507af7..6785aa96c410e2 100644 --- a/lib/routes/arcteryx/regear-new-arrivals.tsx +++ b/lib/routes/arcteryx/regear-new-arrivals.tsx @@ -47,34 +47,31 @@ async function handler() { items = items.filter((item) => item.availableSizes.length !== 0); const list = items.map((item) => { + const imgUrl = JSON.parse(item.imageUrls).front; + const originalPrice = getUSDPrice(item.originalPrice); + const regearPrice = item.priceRange[0] === item.priceRange[1] ? getUSDPrice(item.priceRange[0]) : `${getUSDPrice(item.priceRange[0])} - ${getUSDPrice(item.priceRange[1])}`; const data = { title: item.displayTitle, link: item.pdpLink.url, - imgUrl: JSON.parse(item.imageUrls).front, - availableSizes: item.availableSizes, - color: item.color, - originalPrice: getUSDPrice(item.originalPrice), - regearPrice: item.priceRange[0] === item.priceRange[1] ? getUSDPrice(item.priceRange[0]) : `${getUSDPrice(item.priceRange[0])} - ${getUSDPrice(item.priceRange[1])}`, - description: '', + description: renderToString( +
+ Available Sizes:  + {item.availableSizes.map((size) => ( + <>{size}  + ))} +
+ Color: {item.color} +
+ Original Price: {originalPrice} +
+ Regear Price: {regearPrice} +
+ +
+
+
+ ), }; - data.description = renderToString( -
- Available Sizes:  - {data.availableSizes.map((size) => ( - <>{size}  - ))} -
- Color: {data.color} -
- Original Price: {data.originalPrice} -
- Regear Price: {data.regearPrice} -
- -
-
-
- ); return data; }); @@ -82,10 +79,6 @@ async function handler() { title: 'Arcteryx - Regear - New Arrivals', link: url, description: 'Arcteryx - Regear - New Arrivals', - item: list.map((item) => ({ - title: item.title, - link: item.link, - description: item.description, - })), + item: list, }; } diff --git a/lib/routes/aschmelyun/blog.ts b/lib/routes/aschmelyun/blog.ts new file mode 100644 index 00000000000000..0a2b43cbbe4076 --- /dev/null +++ b/lib/routes/aschmelyun/blog.ts @@ -0,0 +1,48 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + name: 'Blog', + categories: ['blog'], + maintainers: ['raxod502'], + path: '/blog', + example: '/aschmelyun/blog', + handler, + radar: [ + { + source: ['aschmelyun.com'], + target: '/blog', + }, + ], +}; + +async function handler() { + const response = await ofetch('https://aschmelyun.com/blog/'); + const $ = load(response); + + const items = $('div.rounded-lg') + .toArray() + .map((item) => { + item = $(item); + const a = item.find('a.text-xl').first(); + return { + title: a.text(), + link: new URL(a.attr('href'), 'https://aschmelyun.com/blog/').href, + pubDate: parseDate(item.find('span.text-sm').text()), + category: item + .find('a.rounded-full') + .toArray() + .map((cat) => $(cat).text().trim()), + description: item.find('p').first().text(), + }; + }); + + return { + title: 'Andrew Schmelyun Blog', + link: 'https://aschmelyun.com/', + item: items, + }; +} diff --git a/lib/routes/aschmelyun/namespace.ts b/lib/routes/aschmelyun/namespace.ts new file mode 100644 index 00000000000000..9d62f14e038f68 --- /dev/null +++ b/lib/routes/aschmelyun/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Andrew Schmelyun', + url: 'aschmelyun.com', +}; diff --git a/lib/routes/asiafruitchina/categories.ts b/lib/routes/asiafruitchina/categories.ts index 42db49cdfb55af..157570857cc5e0 100644 --- a/lib/routes/asiafruitchina/categories.ts +++ b/lib/routes/asiafruitchina/categories.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.listBlocks ul li') + let items: DataItem[] = $('div.listBlocks ul li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/asiafruitchina/news.ts b/lib/routes/asiafruitchina/news.ts index ec109a2ccbebc9..9ae47525121dd0 100644 --- a/lib/routes/asiafruitchina/news.ts +++ b/lib/routes/asiafruitchina/news.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.listBlocks ul li') + let items: DataItem[] = $('div.listBlocks ul li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/asmr-200/type.ts b/lib/routes/asmr-200/type.ts index a9ebf6bc0ce56f..eda1a23be1bd2a 100644 --- a/lib/routes/asmr-200/type.ts +++ b/lib/routes/asmr-200/type.ts @@ -49,7 +49,7 @@ export interface Work { rank_date: string; term: string; }> | null; - rate_average_2dp: number | number; + rate_average_2dp: number; rate_count: number; rate_count_detail: Array<{ count: number; diff --git a/lib/routes/augmentcode/blog.tsx b/lib/routes/augmentcode/blog.tsx index 1836d6598f232e..f8ad180f0bef67 100644 --- a/lib/routes/augmentcode/blog.tsx +++ b/lib/routes/augmentcode/blog.tsx @@ -40,9 +40,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div[data-slot="card"]') + let items: DataItem[] = $('div[data-slot="card"]') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/bangumi.tv/user/collections.tsx b/lib/routes/bangumi.tv/user/collections.tsx index e285d576a10e0e..172ff8fa02fd46 100644 --- a/lib/routes/bangumi.tv/user/collections.tsx +++ b/lib/routes/bangumi.tv/user/collections.tsx @@ -154,7 +154,7 @@ async function handler(ctx) { const typeName = typeNames[type] || ''; const subjectTypeName = subjectTypeNames[subjectType] || ''; - let descriptionFields = ''; + let descriptionFields: string; if (typeName && subjectTypeName) { descriptionFields = `${typeName}的${subjectTypeName}列表`; diff --git a/lib/routes/banshujiang/index.ts b/lib/routes/banshujiang/index.ts index dc983fe3943e78..3393ad2027ef3a 100644 --- a/lib/routes/banshujiang/index.ts +++ b/lib/routes/banshujiang/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.small-list li.row') + let items: DataItem[] = $('ul.small-list li.row') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/banyuetan/index.ts b/lib/routes/banyuetan/index.ts index 25e7a94ca77c06..efcad0dd949b67 100644 --- a/lib/routes/banyuetan/index.ts +++ b/lib/routes/banyuetan/index.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.bty_tbtj_list ul.clearFix li') + let items: DataItem[] = $('div.bty_tbtj_list ul.clearFix li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/bbc/index.ts b/lib/routes/bbc/index.ts index bc48c45df81d89..0068b4d5199303 100644 --- a/lib/routes/bbc/index.ts +++ b/lib/routes/bbc/index.ts @@ -1,11 +1,8 @@ -import { load } from 'cheerio'; - import type { Route } from '@/types'; import cache from '@/utils/cache'; -import ofetch from '@/utils/ofetch'; import parser from '@/utils/rss-parser'; -import utils from './utils'; +import { fetchBbcContent } from './utils'; export const route: Route = { path: '/:site?/:channel?', @@ -36,13 +33,11 @@ async function handler(ctx) { switch (site.toLowerCase()) { case 'chinese': title = 'BBC News 中文网'; - feed = await (channel ? parser.parseURL(`https://www.bbc.co.uk/zhongwen/simp/${channel}/index.xml`) : parser.parseURL('https://www.bbc.co.uk/zhongwen/simp/index.xml')); break; case 'traditionalchinese': title = 'BBC News 中文網'; - feed = await (channel ? parser.parseURL(`https://www.bbc.co.uk/zhongwen/trad/${channel}/index.xml`) : parser.parseURL('https://www.bbc.co.uk/zhongwen/trad/index.xml')); link = 'https://www.bbc.com/zhongwen/trad'; break; @@ -63,47 +58,31 @@ async function handler(ctx) { const items = await Promise.all( feed.items .filter((item) => item && item.link) + .map((item) => { + const link = item.link.split('?')[0]; + return { + ...item, + // https://www.bbc.co.uk/zhongwen/simp/index.xml returns trad regardless of lang parameter + // which requires manual fixing + link: site === 'chinese' ? item.link.replace('/trad', '/simp') : link, + }; + }) .map((item) => cache.tryGet(item.link, async () => { - try { - const linkURL = new URL(item.link); - if (linkURL.hostname === 'www.bbc.com') { - linkURL.hostname = 'www.bbc.co.uk'; - } - - const response = await ofetch(linkURL.href, { - retryStatusCodes: [403], - }); - - const $ = load(response); - - const path = linkURL.pathname; - - let description; + const linkURL = new URL(item.link); + if (linkURL.hostname === 'www.bbc.com') { + linkURL.hostname = 'www.bbc.co.uk'; + } - switch (true) { - case path.startsWith('/sport'): - description = item.content; - break; - case path.startsWith('/sounds/play'): - description = item.content; - break; - case path.startsWith('/news/live'): - description = item.content; - break; - default: - description = utils.ProcessFeed($); - } + const { category, description } = await fetchBbcContent(linkURL.href, item); - return { - title: item.title || '', - description: description || '', - pubDate: item.pubDate || new Date().toUTCString(), - link: item.link, - }; - } catch { - return {} as Record; - } + return { + title: item.title || '', + description: description || '', + pubDate: item.pubDate, + link: item.link, + category: category ?? item.categories ?? [], + }; }) ) ); @@ -113,6 +92,6 @@ async function handler(ctx) { link, image: 'https://www.bbc.com/favicon.ico', description: title, - item: items.filter((item) => Object.keys(item).length > 0), + item: items, }; } diff --git a/lib/routes/bbc/sport.ts b/lib/routes/bbc/sport.ts new file mode 100644 index 00000000000000..92ecda00eb84c2 --- /dev/null +++ b/lib/routes/bbc/sport.ts @@ -0,0 +1,68 @@ +import { load } from 'cheerio'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { extractInitialData, fetchBbcContent } from './utils'; + +export const route: Route = { + path: '/sport/:sport', + name: 'Sport', + maintainers: ['TonyRL'], + handler, + example: '/bbc/sport/formula1', + parameters: { + sport: 'The sport to fetch news for, can be found in the URL.', + }, + radar: [ + { + source: ['www.bbc.com/sport/:sport'], + }, + ], + categories: ['traditional-media'], +}; + +async function handler(ctx) { + const { sport } = ctx.req.param(); + const link = `https://www.bbc.com/sport/${sport}`; + + const response = await ofetch(link); + const $ = load(response); + + const initialData = extractInitialData($); + const { page } = initialData.stores.metadata; + + const list: DataItem[] = Object.values(initialData.data) + .filter((d) => d.name === 'hierarchical-promo-collection' && d.props.title !== 'Elsewhere on the BBC') + .flatMap((d) => d.data.promos) + .map((item) => ({ + title: item.headline, + description: item.description, + link: `https://www.bbc.com${item.url}`, + pubDate: item.lastPublished ? parseDate(item.lastPublished) : undefined, + image: item.image?.src.replace('/480/', '/1536/'), + })); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link!, async () => { + const { category, description } = await fetchBbcContent(item.link!, item); + + item.category = category; + item.description = description; + + return item; + }) + ) + ); + + return { + title: page.title, + description: page.description, + link, + image: 'https://www.bbc.com/favicon.ico', + item: items, + }; +} diff --git a/lib/routes/bbc/topic-zhongwen.ts b/lib/routes/bbc/topic-zhongwen.ts new file mode 100644 index 00000000000000..6e03a1ec152cf2 --- /dev/null +++ b/lib/routes/bbc/topic-zhongwen.ts @@ -0,0 +1,72 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { fetchBbcContent } from './utils'; + +export const route: Route = { + path: '/zhongwen/topics/:topic/:variant?', + name: 'Topics - BBC News 中文', + maintainers: ['TonyRL'], + handler, + example: '/bbc/zhongwen/topics/ckr7mn6r003t', + parameters: { + topic: 'The topic ID to fetch news for, can be found in the URL.', + variant: { + description: 'The language variant.', + default: 'trad', + options: [ + { label: '简', value: 'simp' }, + { label: '繁', value: 'trad' }, + ], + }, + }, + radar: [ + { + source: ['www.bbc.com/zhongwen/topics/:topic/:variant'], + }, + ], + categories: ['traditional-media'], +}; + +async function handler(ctx) { + const { topic, variant = 'trad' } = ctx.req.param(); + const link = `https://www.bbc.com/zhongwen/topics/${topic}/${variant}`; + + const response = await ofetch(link); + const $ = load(response); + + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const pageData = nextData.props.pageProps.pageData; + + const list = pageData.curations[0].summaries.map((item) => ({ + title: item.title, + link: item.link, + pubDate: parseDate(item.firstPublished), + image: item.imageUrl ? item.imageUrl.replace('/{width}/', '/1536/') : undefined, + })); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const { category, description } = await fetchBbcContent(item.link, item); + + item.category = category; + item.description = description; + + return item; + }) + ) + ); + + return { + title: `${pageData.title} - BBC News 中文`, + description: pageData.description, + link, + image: 'https://www.bbc.com/favicon.ico', + item: items, + }; +} diff --git a/lib/routes/bbc/topic.ts b/lib/routes/bbc/topic.ts new file mode 100644 index 00000000000000..3e3f32f76ace52 --- /dev/null +++ b/lib/routes/bbc/topic.ts @@ -0,0 +1,66 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { fetchBbcContent } from './utils'; + +export const route: Route = { + path: '/topics/:topic', + name: 'Topics', + maintainers: ['TonyRL'], + handler, + example: '/bbc/topics/c77jz3md4rwt', + parameters: { + topic: 'The topic ID to fetch news for, can be found in the URL.', + }, + radar: [ + { + source: ['www.bbc.com/news/topics/:topic'], + }, + ], + categories: ['traditional-media'], +}; + +async function handler(ctx) { + const { topic } = ctx.req.param(); + const link = `https://www.bbc.com/news/topics/${topic}`; + + const response = await ofetch(link); + const $ = load(response); + + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const pageProps = nextData.props.pageProps; + const topicData = pageProps.page[pageProps.pageKey]; + + const list = topicData.sections[0].content.map((item) => ({ + title: item.title, + link: `https://www.bbc.com${item.href}`, + description: item.description, + pubDate: parseDate(item.metadata.firstUpdated), + category: item.metadata.topics, + image: item.image ? item.image.model.blocks.src : undefined, + })); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const { category, description } = await fetchBbcContent(item.link, item); + item.category = category; + item.description = description; + + return item; + }) + ) + ); + + return { + title: topicData.seo.title, + description: topicData.seo.description, + link, + image: 'https://www.bbc.com/favicon.ico', + item: items, + }; +} diff --git a/lib/routes/bbc/utils.tsx b/lib/routes/bbc/utils.tsx index 164ba2aeb42112..abfd5d3bc27246 100644 --- a/lib/routes/bbc/utils.tsx +++ b/lib/routes/bbc/utils.tsx @@ -1,129 +1,436 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; +import { raw } from 'hono/html'; import { renderToString } from 'hono/jsx/dom/server'; +import type { JSX } from 'hono/jsx/jsx-runtime'; -const processImageAttributes = ($img) => { - if (!$img.attr('src') && $img.attr('srcSet')) { - const srcs = $img.attr('srcSet').split(', '); - const lastSrc = srcs.at(-1); - if (lastSrc) { - $img.attr('src', lastSrc.split(' ')[0]); +import ofetch from '@/utils/ofetch'; + +type BlockAttribute = 'bold' | 'italic'; + +type BlockModel = { + blocks?: Block[]; + text?: string; + timestamp?: number; + listType?: string; + listStyle?: string; + locator?: string; + originCode?: string; + width?: number; + height?: number; + copyrightHolder?: string; + attributes?: BlockAttribute[]; + imageUrl?: string; + imageCopyright?: string; + caption?: Block; + image?: { + alt?: string; + copyright?: string; + height?: number; + width?: number; + src?: string; + }; + media?: { + items?: Array<{ + holdingImageUrl?: string; + title?: string; + }>; + }; + oembed?: { + html: string; + }; +}; + +type Block = { + type?: string; + model?: BlockModel; + blocks?: Block[]; + items?: Block[]; +}; + +const applyAttributes = (content: JSX.Element | string, attributes?: BlockAttribute[]): JSX.Element | string => { + let result: JSX.Element | string = content; + for (const attribute of attributes ?? []) { + switch (attribute) { + case 'bold': + result = {result}; + break; + + case 'italic': + result = {result}; + break; + + default: + throw new Error(`Unhandled attribute: ${attribute}`); } } - $img.removeAttr('srcset').removeAttr('sizes'); + return result; }; -const buildCleanFigure = (src, alt, figcaptionContent) => - renderToString( -
- {alt} - {figcaptionContent &&
{figcaptionContent}
} -
- ); +const extractText = (blocks?: Block[]): string => { + if (!blocks?.length) { + return ''; + } -const cleanFigureElement = ($, figure) => { - const $figure = $(figure); - const $img = $figure.find('img'); + return blocks + .map((block) => { + if (block.type === 'fragment') { + return block.model?.text ?? ''; + } + + if (block.model?.text) { + return block.model.text; + } - if ($img.length === 0) { - return; + return extractText(block.model?.blocks ?? block.blocks ?? block.items); + }) + .join(''); +}; + +const renderInlineBlocks = (blocks?: Block[]): Array => { + if (!blocks?.length) { + return []; } - processImageAttributes($img); + return blocks.flatMap((block) => { + if (block.type === 'fragment') { + const text = block.model?.text ?? ''; + if (!text) { + return []; + } + + return [applyAttributes(text, block.model?.attributes)]; + } - let sourceText = ''; - let captionText = ''; + if (block.model?.blocks || block.blocks || block.items) { + return renderInlineBlocks(block.model?.blocks ?? block.blocks ?? block.items); + } - // extract image source: (simp chinese/trad chinese) - const $sourceP = $figure.find('p[class*="css-"]').first(); - if ($sourceP.length > 0) { - const sourceSpans = $sourceP.find('span'); - if (sourceSpans.length >= 2) { - sourceText = sourceSpans.eq(1).text().trim(); + if (block.model?.text) { + return [block.model.text]; } + + return []; + }); +}; + +const renderList = (block: Block, key: string): JSX.Element | null => { + const items = block.model?.blocks ?? block.blocks ?? block.items; + if (!items?.length) { + return null; + } + + const listItems = items.map((item, index) => { + const content = renderInlineBlocks(item.model?.blocks ?? item.blocks ?? item.items); + const fallback = item.model?.text ? [item.model.text] : []; + return
  • {content.length ? content : fallback}
    • ; + }); + + const listType = block.model?.listType ?? block.model?.listStyle ?? block.type; + if (listType === 'ordered' || listType === 'orderedList') { + return
      {listItems}
    ; } - let $figcaption = $figure.find('figcaption'); + return
      {listItems}
    ; +}; + +const renderParagraph = (blocks?: Block[], keyPrefix = 'paragraph'): JSX.Element[] => { + if (!blocks?.length) { + return []; + } + + return blocks.flatMap((block, index) => { + const key = `${keyPrefix}-${index}`; + switch (block.type) { + case 'paragraph': { + const content = renderInlineBlocks(block.model?.blocks ?? block.blocks ?? block.items); + const fallbackText = block.model?.text; + if (!content.length && !fallbackText) { + return []; + } - // english version - if ($figcaption.length === 0) { - const $next = $figure.next(); - if ($next.length > 0) { - $figcaption = $next.find('figcaption'); - // if found, remove the sibling div after extracting caption - if ($figcaption.length > 0) { - $next.remove(); + return [

    {content.length ? content : fallbackText}

    ]; } + case 'subheading': + case 'heading': + return [

    {extractText(block.model?.blocks ?? block.blocks ?? block.items)}

    ]; + case 'list': + case 'unorderedList': + case 'orderedList': { + const list = renderList(block, key); + return list ? [list] : []; + } + default: + return renderParagraph(block.model?.blocks ?? block.blocks ?? block.items, key); } + }); +}; + +const findBlocksByType = (blocks: Block[] | undefined, type: string): Block[] => { + if (!blocks?.length) { + return []; + } + + const matches: Block[] = []; + for (const block of blocks) { + if (block.type === type) { + matches.push(block); + } + + matches.push(...findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, type)); + } + + return matches; +}; + +const buildImageUrl = (model?: BlockModel): string | undefined => { + if (!model?.locator || !model.originCode) { + return undefined; + } + + return `https://ichef.bbci.co.uk/news/1536/${model.originCode}/${model.locator}`; +}; + +const renderFigure = (key: string, src: string, altText: string, width?: number, height?: number, caption?: string, copyrightHolder?: string): JSX.Element => ( +
    + {altText} + {caption || copyrightHolder ?
    {[copyrightHolder, caption].filter(Boolean).join(' / ')}
    : null} +
    +); + +const renderImage = (block: Block, index: number): JSX.Element | null => { + const imagePayload = block.model?.image; + const captionPayload = block.model?.caption; + if (imagePayload?.src) { + const caption = extractText(captionPayload?.model?.blocks ?? captionPayload?.blocks ?? captionPayload?.items).trim(); + const altText = imagePayload.alt?.trim() ?? ''; + const copyrightHolder = imagePayload.copyright?.trim(); + + return renderFigure(`image-${index}`, imagePayload.src, altText, imagePayload.width, imagePayload.height, caption, copyrightHolder); } - if ($figcaption.length > 0) { - // try to find caption in specific elements, excluding visually-hidden labels - const $captionParagraph = $figcaption.find('[data-testid="caption-paragraph"]'); + const altTextBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'altText')[0]; + const captionBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'caption')[0]; + const rawImageBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'rawImage')[0]; - if ($captionParagraph.length > 0) { - captionText = $captionParagraph.text().trim(); - } else { - // remove visually-hidden elements (like "Image caption, " labels) - const $figcaptionClone = $figcaption.clone(); - $figcaptionClone.find('.visually-hidden, [class*="VisuallyHidden"]').remove(); - captionText = $figcaptionClone.text().trim(); + const altText = extractText(altTextBlock?.model?.blocks ?? altTextBlock?.blocks ?? altTextBlock?.items).trim(); + const caption = extractText(captionBlock?.model?.blocks ?? captionBlock?.blocks ?? captionBlock?.items).trim(); + const imageModel = rawImageBlock?.model ?? block.model; + const src = buildImageUrl(imageModel); + + if (!src) { + return null; + } + + const copyrightHolder = imageModel?.copyrightHolder?.trim(); + + return renderFigure(`image-${index}`, src, altText, imageModel?.width, imageModel?.height, caption, copyrightHolder); +}; + +const renderVideo = (block: Block, index: number): JSX.Element | null => { + const altTextBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'altText')[0]; + const captionBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'caption')[0]; + const mediaMetadataBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'mediaMetadata')[0]; + const aresMediaBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'aresMedia')[0]; + const aresMediaMetadata = aresMediaBlock ? (aresMediaBlock.model?.blocks ?? aresMediaBlock.blocks ?? aresMediaBlock.items)?.[0] : undefined; + + const altText = extractText(altTextBlock?.model?.blocks ?? altTextBlock?.blocks ?? altTextBlock?.items).trim(); + const caption = extractText(captionBlock?.model?.blocks ?? captionBlock?.blocks ?? captionBlock?.items).trim(); + + let src: string | undefined; + let sourceText: string | undefined; + + if (mediaMetadataBlock?.model?.imageUrl) { + src = mediaMetadataBlock.model.imageUrl; + sourceText = mediaMetadataBlock.model.imageCopyright; + } else if (aresMediaMetadata?.model?.imageUrl) { + src = `https://${aresMediaMetadata.model.imageUrl.replace('/$recipe/', '/800xn/')}`; + sourceText = aresMediaMetadata.model.imageCopyright; + } + + if (!src) { + return null; + } + + return renderFigure(`video-${index}`, src, altText, undefined, undefined, caption, sourceText); +}; + +const renderMedia = (block: Block, index: number): JSX.Element | null => { + const mediaItem = block.model?.media?.items?.[0]; + const holdingImageUrl = mediaItem?.holdingImageUrl; + + if (!holdingImageUrl) { + return null; + } + + const caption = extractText(block.model?.caption?.model?.blocks ?? block.model?.caption?.blocks ?? block.model?.caption?.items).trim(); + const altText = mediaItem?.title?.trim() ?? ''; + + return renderFigure(`media-${index}`, holdingImageUrl.replace('/$recipe/', '/800xn/'), altText, undefined, undefined, caption); +}; + +const renderOEmbed = (block: Block, index: number, keyPrefix = 'oembed'): JSX.Element | null => { + const aresOEmbedBlock = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'aresOEmbed')[0]; + const oembedHtml = aresOEmbedBlock?.model?.oembed?.html ?? block.model?.oembed?.html; + + if (!oembedHtml) { + return null; + } + + return
    {raw(oembedHtml)}
    ; +}; + +const renderEmbedImages = (block: Block, index: number): JSX.Element | null => { + const imageBlocks = findBlocksByType(block.model?.blocks ?? block.blocks ?? block.items, 'image'); + + if (!imageBlocks?.length) { + return null; + } + + let largestImage: Block | null = null; + let maxWidth = 0; + + for (const imageBlock of imageBlocks) { + const rawImageBlock = findBlocksByType(imageBlock.model?.blocks ?? imageBlock.blocks ?? imageBlock.items, 'rawImage')[0]; + const width = rawImageBlock?.model?.width; + + if (width && width > maxWidth) { + maxWidth = width; + largestImage = imageBlock; } } - const parts = [sourceText, captionText].filter(Boolean); - const figcaptionContent = parts.join(' / '); + if (!largestImage) { + return null; + } + + return renderImage(largestImage, index); +}; - $figure.replaceWith(buildCleanFigure($img.attr('src'), $img.attr('alt') || '', figcaptionContent)); +const renderBlock = (block: Block, index: number): JSX.Element | JSX.Element[] | null => { + switch (block.type) { + case 'image': + return renderImage(block, index); + case 'video': + return renderVideo(block, index); + case 'text': + return renderParagraph(block.model?.blocks ?? block.blocks ?? block.items, `text-${index}`); + case 'media': + return renderMedia(block, index); + case 'subheadline': { + const text = extractText(block.model?.blocks ?? block.blocks ?? block.items).trim(); + return text ?

    {text}

    : null; + } + case 'social': + return renderOEmbed(block, index, 'social'); + case 'oEmbed': + return renderOEmbed(block, index); + case 'embedImages': + return renderEmbedImages(block, index); + case 'headline': + case 'timestamp': + case 'byline': + case 'advertisement': + case 'embed': // #region /zhongwen/topics + case 'disclaimer': + case 'continueReading': + case 'mpu': + case 'wsoj': + case 'relatedContent': + case 'links': + case 'visuallyHiddenHeadline': + case 'fauxHeadline': // #endregion + case 'metadata': // #region /sports + case 'topicList': + case 'promoList': // #endregion + return null; + default: + return renderParagraph(block.model?.blocks ?? block.blocks ?? block.items, `block-${index}`); + } }; -const ProcessFeed = ($) => { - // by default treat it as a hybrid news with video and story-body__inner - let content = $('#main-content article'); +const renderArticleContent = (content: Block[]): string => renderToString(<>{content.flatMap((block, index) => renderBlock(block, index)).filter((node) => node !== null)}); + +export const extractInitialData = ($: CheerioAPI): any => { + const initialDataText = JSON.parse( + $('script:contains("window.__INITIAL_DATA__")') + .text() + .match(/window\.__INITIAL_DATA__\s*=\s*(.*);/)?.[1] ?? '"{}"' + ); + + return JSON.parse(initialDataText); +}; - if (content.length === 0) { - // it's a video news with video and story-body - content = $('div.story-body'); +const extractArticleWithInitialData = ($: CheerioAPI, item) => { + if (item.link.includes('/live/') || item.link.includes('/videos/') || item.link.includes('/extra/') || item.link.includes('/sounds/play/')) { + return { + description: item.content, + }; } - if (content.length === 0) { - // chinese version has different structure - content = $('main[role="main"]'); + const initialData = extractInitialData($); + if (!initialData || !initialData.data) { + return { + description: item.content, + }; } - // remove useless DOMs - content.find('header, section, [data-testid="bbc-logo-wrapper"]').remove(); + const article = Object.values(initialData.data).find((d) => d.name === 'article')?.data; + const topics = Array.isArray(article?.topics) ? article.topics : []; + const blocks = article?.content?.model?.blocks; - // remove article title as it's already in RSS item title - content.find('h1').remove(); + return { + category: [...new Set([...(item.category || []), ...topics.map((t) => t.title)])], + description: blocks ? renderArticleContent(blocks) : item.content, + }; +}; - content.find('noscript').each((i, e) => { - $(e).parent().html($(e).html()); +export const fetchBbcContent = async (link: string, item) => { + const response = await ofetch.raw(link, { + retryStatusCodes: [403], }); + const $ = load(response._data); + const { hostname, pathname } = new URL(response.url); - // clean up figure elements with images - content.find('figure').each((i, figure) => cleanFigureElement($, figure)); - - // handle standalone images with figcaption siblings (English version) - content - .find('img') - .not('figure img') - .each((i, img) => { - const $img = $(img); - processImageAttributes($img); - - // check for figcaption sibling - const $next = $img.next(); - if ($next.length > 0 && $next.find('figcaption').length > 0) { - const captionText = $next.find('figcaption').first().text().trim(); - if (captionText) { - $img.replaceWith(buildCleanFigure($img.attr('src'), $img.attr('alt') || '', captionText)); - $next.remove(); - } - } - }); + switch (true) { + case hostname === 'www.bbc.co.uk': + case pathname.startsWith('/sport'): + return extractArticleWithInitialData($, item); + case pathname.startsWith('/sounds/play') || pathname.startsWith('/news/live'): + return { + description: item.content ?? item.description ?? '', + }; + case pathname.startsWith('/zhongwen/articles/'): { + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const pageData = nextData.props.pageProps.pageData; + const metadata = pageData.metadata; + const aboutLabels = metadata.tags?.about?.map((t) => t.thingLabel) ?? []; + const topicNames = metadata.topics?.map((t) => t.topicName) ?? []; - content.find('[data-component="media-block"] figcaption').prepend('View video in browser: '); + return { + category: [...new Set([...aboutLabels, ...topicNames])], + description: renderArticleContent(pageData.content.model.blocks), + }; + } + case pathname.startsWith('/news/'): { + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const pageProps = nextData.props.pageProps; + const articleData = pageProps.page[pageProps.pageKey]; - return content.html(); -}; + return { + category: [...new Set([...(item.category || []), ...(articleData.topics?.map((t) => t.title) ?? [])])], + description: renderArticleContent(articleData.contents), + }; + } + default: + break; + } + + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const pageProps = nextData.props.pageProps; + const articleData = pageProps.page[pageProps.pageKey]; -export default { ProcessFeed }; + return { + description: renderArticleContent(articleData.contents), + }; +}; diff --git a/lib/routes/bestblogs/newsletter.ts b/lib/routes/bestblogs/newsletter.ts new file mode 100644 index 00000000000000..6ed9e2d7f50e38 --- /dev/null +++ b/lib/routes/bestblogs/newsletter.ts @@ -0,0 +1,49 @@ +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/newsletter', + categories: ['programming'], + example: '/bestblogs/newsletter', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: '精选推送', + maintainers: ['occam-7'], + handler, +}; + +async function handler(ctx) { + const pageSize = Number.parseInt(ctx.req.query('limit') ?? '10', 10); + + const response = await ofetch('https://api.bestblogs.dev/api/newsletter/list', { + method: 'POST', + body: { + currentPage: 1, + pageSize, + userLanguage: 'zh', + }, + }); + + const list = response.data?.dataList ?? []; + + const items = list.map((item) => ({ + title: item.title, + link: `https://www.bestblogs.dev/newsletter/${item.id}`, + description: item.summary, + pubDate: parseDate(item.createdTimestamp), + })); + + return { + title: 'Bestblogs.dev - 精选推送', + link: 'https://www.bestblogs.dev/newsletter', + item: items, + }; +} diff --git a/lib/routes/bilibili/api-interface.d.ts b/lib/routes/bilibili/api-interface.d.ts index 89b2038ee55741..40adbbc6425fa4 100644 --- a/lib/routes/bilibili/api-interface.d.ts +++ b/lib/routes/bilibili/api-interface.d.ts @@ -474,8 +474,7 @@ export type DynamicType = * 更多类型请参考：https://github.com/SocialSisterYi/bilibili-API-collect/blob/master/docs/dynamic/dynamic_enum.md#%E5%8A%A8%E6%80%81%E4%B8%BB%E4%BD%93%E7%B1%BB%E5%9E%8B */ export type MajorType = - | 'MAJOR_TYPE_NONE' // 动态失效, 示例: 716510857084796964 - | 'MAJOR_TYPE_NONE' // 转发动态, 示例: 866756840240709701 + | 'MAJOR_TYPE_NONE' // 动态失效, 示例: 716510857084796964 转发动态, 示例: 866756840240709701 | 'MAJOR_TYPE_OPUS' // 图文动态, 示例: 870176712256651305 | 'MAJOR_TYPE_ARCHIVE' // 视频, 示例: 716526237365829703 | 'MAJOR_TYPE_PGC' // 剧集更新, 示例: 645981661420322824 diff --git a/lib/routes/bilibili/hot-search.ts b/lib/routes/bilibili/hot-search.ts index ecfaff915bf775..a644a3dc7d1a59 100644 --- a/lib/routes/bilibili/hot-search.ts +++ b/lib/routes/bilibili/hot-search.ts @@ -19,7 +19,10 @@ export const route: Route = { }, radar: [ { - source: ['www.bilibili.com/', 'm.bilibili.com/'], + source: ['www.bilibili.com/'], + }, + { + source: ['m.bilibili.com/'], }, ], name: '热搜', diff --git a/lib/routes/bilibili/link-news.ts b/lib/routes/bilibili/link-news.ts index e3846aeda68b1e..bd15493e8dc5d8 100644 --- a/lib/routes/bilibili/link-news.ts +++ b/lib/routes/bilibili/link-news.ts @@ -22,7 +22,7 @@ export const route: Route = { async function handler(ctx) { const product = ctx.req.param('product'); - let productTitle = ''; + let productTitle: string; switch (product) { case 'live': diff --git a/lib/routes/bilibili/live-area.ts b/lib/routes/bilibili/live-area.ts index ed5b0791e689e4..4fa7861c0bb3df 100644 --- a/lib/routes/bilibili/live-area.ts +++ b/lib/routes/bilibili/live-area.ts @@ -26,7 +26,7 @@ async function handler(ctx) { const areaID = ctx.req.param('areaID'); const order = ctx.req.param('order'); - let orderTitle = ''; + let orderTitle: string; switch (order) { case 'live_time': orderTitle = '最新开播'; @@ -47,7 +47,7 @@ async function handler(ctx) { }); let parentTitle = ''; - let parentID = ''; + let parentID: string; let areaTitle = ''; let areaLink = ''; diff --git a/lib/routes/bilibili/live-search.ts b/lib/routes/bilibili/live-search.ts index ef7e7480e6c46d..82b544fe71c6c3 100644 --- a/lib/routes/bilibili/live-search.ts +++ b/lib/routes/bilibili/live-search.ts @@ -27,7 +27,7 @@ async function handler(ctx) { const order = ctx.req.param('order'); const urlEncodedKey = encodeURIComponent(key); - let orderTitle = ''; + let orderTitle: string; switch (order) { case 'live_time': diff --git a/lib/routes/bilibili/partion-ranking.ts b/lib/routes/bilibili/partion-ranking.ts index 1b160d167367eb..3401482443f2fa 100644 --- a/lib/routes/bilibili/partion-ranking.ts +++ b/lib/routes/bilibili/partion-ranking.ts @@ -1,4 +1,4 @@ -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import got from '@/utils/got'; import utils from './utils'; @@ -43,11 +43,8 @@ async function handler(ctx) { const responseApi = `https://api.bilibili.com/x/web-interface/newlist?ps=15&rid=${tid}&_=${Date.now()}`; const response = await got_ins.get(responseApi); - const items = []; let name = '未知'; - let list = {}; - - list = response.data.data.archives; + const list = response.data.data.archives; if (list && list[0] && list[0].tname) { name = list[0].tname; } @@ -58,16 +55,13 @@ async function handler(ctx) { const HotRankResponse = await got_ins.get(HotRankResponseApi); const hotlist = HotRankResponse.data.result; - for (let item of hotlist) { - item = { - title: item.title, - description: utils.renderUGCDescription(embed, item.pic, `${item.description} - ${item.tag}`, item.id, undefined, item.bvid), - pubDate: new Date(item.pubdate).toUTCString(), - link: item.pubdate > utils.bvidTime && item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : `https://www.bilibili.com/video/av${item.id}`, - author: item.author, - }; - items.push(item); - } + const items: DataItem[] = hotlist.map((item) => ({ + title: item.title, + description: utils.renderUGCDescription(embed, item.pic, `${item.description} - ${item.tag}`, item.id, undefined, item.bvid), + pubDate: new Date(item.pubdate).toUTCString(), + link: item.pubdate > utils.bvidTime && item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : `https://www.bilibili.com/video/av${item.id}`, + author: item.author, + })); return { title: `bilibili ${name} 最热视频`, diff --git a/lib/routes/bilibili/ranking.ts b/lib/routes/bilibili/ranking.ts index 20f4aec24d04b2..7c9f5d3452aec6 100644 --- a/lib/routes/bilibili/ranking.ts +++ b/lib/routes/bilibili/ranking.ts @@ -193,7 +193,7 @@ function getAPI(isNumericRid: boolean, rid: string | number) { const ridEnglish = zone[1].english; let apiBase = 'https://api.bilibili.com/x/web-interface/ranking/v2'; - let apiParams = ''; + let apiParams: string; switch (ridType) { case 'x/rid': diff --git a/lib/routes/bilibili/utils.ts b/lib/routes/bilibili/utils.ts index 65bf087866b11f..ae3e7581519f67 100644 --- a/lib/routes/bilibili/utils.ts +++ b/lib/routes/bilibili/utils.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ import CryptoJS from 'crypto-js'; import { config } from '@/config'; diff --git a/lib/routes/bilibili/wasm-exec.ts b/lib/routes/bilibili/wasm-exec.ts index 7cfb8a02f44e31..b4359a3da0fc7f 100644 --- a/lib/routes/bilibili/wasm-exec.ts +++ b/lib/routes/bilibili/wasm-exec.ts @@ -1,3 +1,5 @@ +// oxlint-disable unicorn/prefer-math-trunc +// oxlint-disable no-unused-vars /* eslint-disable prefer-rest-params */ /* eslint-disable default-case */ /* eslint-disable unicorn/consistent-function-scoping */ @@ -319,7 +321,7 @@ // func resetMemoryDataView() 'runtime.resetMemoryDataView': (sp) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars + // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-useless-assignment sp >>>= 0; this.mem = new DataView(this._inst.exports.mem.buffer); }, diff --git a/lib/routes/bilibili/weekly-recommend.ts b/lib/routes/bilibili/weekly-recommend.ts index ad30fca8cf4141..104fd9079f8c20 100644 --- a/lib/routes/bilibili/weekly-recommend.ts +++ b/lib/routes/bilibili/weekly-recommend.ts @@ -51,23 +51,25 @@ async function handler(ctx) { title: 'B站每周必看', link: 'https://www.bilibili.com/h5/weekly-recommend', description: 'B站每周必看', - item: data.map(async (item) => { - const subtitles = isJsonFeed && !config.bilibili.excludeSubtitles && item.bvid ? await cache.getVideoSubtitleAttachment(item.bvid) : []; - return { - title: item.title, - description: utils.renderUGCDescription(embed, item.cover, `${weekly_name} ${item.title} - ${item.rcmd_reason}`, item.param, undefined, item.bvid), - link: weekly_number > 60 && item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : `https://www.bilibili.com/video/av${item.param}`, - attachments: item.bvid - ? [ - { - url: getVideoUrl(item.bvid), - mime_type: 'text/html', - duration_in_seconds: parseDuration(item.cover_right_text_1), - }, - ...subtitles, - ] - : undefined, - }; - }), + item: await Promise.all( + data.map(async (item) => { + const subtitles = isJsonFeed && !config.bilibili.excludeSubtitles && item.bvid ? await cache.getVideoSubtitleAttachment(item.bvid) : []; + return { + title: item.title, + description: utils.renderUGCDescription(embed, item.cover, `${weekly_name} ${item.title} - ${item.rcmd_reason}`, item.param, undefined, item.bvid), + link: weekly_number > 60 && item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : `https://www.bilibili.com/video/av${item.param}`, + attachments: item.bvid + ? [ + { + url: getVideoUrl(item.bvid), + mime_type: 'text/html', + duration_in_seconds: parseDuration(item.cover_right_text_1), + }, + ...subtitles, + ] + : undefined, + }; + }) + ), }; } diff --git a/lib/routes/biquge/index.ts b/lib/routes/biquge/index.ts index 1160970c9dad95..4da457d8e16400 100644 --- a/lib/routes/biquge/index.ts +++ b/lib/routes/biquge/index.ts @@ -60,11 +60,11 @@ async function handler(ctx) { .map((item) => { item = $(item); - let link = ''; + let link: string; const url = item.attr('href'); if (url.startsWith('http')) { link = url; - } else if (/^\//.test(url)) { + } else if (url.startsWith('/')) { link = `${rootUrl}${url}`; } else { link = `${currentUrl}/${url}`; diff --git a/lib/routes/blizzard/news-cn.ts b/lib/routes/blizzard/news-cn.ts index 7580c1fc9315af..2337527eb5a934 100644 --- a/lib/routes/blizzard/news-cn.ts +++ b/lib/routes/blizzard/news-cn.ts @@ -20,7 +20,15 @@ export const route: Route = { }, radar: [ { - source: ['ow.blizzard.cn', 'wow.blizzard.cn', 'hs.blizzard.cn'], + source: ['ow.blizzard.cn'], + target: '/news-cn/', + }, + { + source: ['wow.blizzard.cn'], + target: '/news-cn/', + }, + { + source: ['hs.blizzard.cn'], target: '/news-cn/', }, ], diff --git a/lib/routes/bse/index.ts b/lib/routes/bse/index.ts index 77e0b4421712a3..64435b249c4e40 100644 --- a/lib/routes/bse/index.ts +++ b/lib/routes/bse/index.ts @@ -1,4 +1,4 @@ -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; @@ -185,7 +185,7 @@ async function handler(ctx) { const data = JSON.parse(response.data.match(/null\(\[({.*})]\)/)[1]); - let items = []; + let items: DataItem[]; switch (type) { case '/info/listse': diff --git a/lib/routes/bt0/util.ts b/lib/routes/bt0/util.ts index dfcd8d72038ec8..9892befcb4605c 100644 --- a/lib/routes/bt0/util.ts +++ b/lib/routes/bt0/util.ts @@ -19,7 +19,7 @@ async function doGot(num, host, link) { throw new Error('api error'); } cookieJar.setCookieSync(match[1], host); - return doGot(++num, host, link); + return doGot(num + 1, host, link); } return data; } diff --git a/lib/routes/bugzilla/bug.ts b/lib/routes/bugzilla/bug.ts index da5530d1c49ab3..830944edb0ac05 100644 --- a/lib/routes/bugzilla/bug.ts +++ b/lib/routes/bugzilla/bug.ts @@ -21,7 +21,8 @@ async function handler(ctx: Context): Promise { throw new InvalidParameterError(`unknown site: ${site}`); } const link = `https://${INSTANCES.get(site)}/show_bug.cgi?id=${bugId}`; - const $ = load(await ofetch(`${link}&ctype=xml`)); + const xml = await ofetch(`${link}&ctype=xml`); + const $ = load(xml); const items = $('long_desc').map((index, rawItem) => { const $ = load(rawItem, null, false); return { diff --git a/lib/routes/bullionvault/gold-news.ts b/lib/routes/bullionvault/gold-news.ts index f4986b4c246c1b..86a624f798f491 100644 --- a/lib/routes/bullionvault/gold-news.ts +++ b/lib/routes/bullionvault/gold-news.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('section#block-bootstrap-views-block-latest-articles-block div.media, div.gold-news-content table tr') + let items: DataItem[] = $('section#block-bootstrap-views-block-latest-articles-block div.media, div.gold-news-content table tr') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/caixin/utils.ts b/lib/routes/caixin/utils.ts index 4fb6f13d11cc84..fd9be6dee05c4c 100644 --- a/lib/routes/caixin/utils.ts +++ b/lib/routes/caixin/utils.ts @@ -5,7 +5,7 @@ import got from '@/utils/got'; import { renderArticle } from './templates/article'; const parseArticle = async (item) => { - if (/\.blog\.caixin\.com$/.test(new URL(item.link).hostname)) { + if (new URL(item.link).hostname.endsWith('.blog.caixin.com')) { return parseBlogArticle(item); } else { const { data: response } = await got(item.link); diff --git a/lib/routes/canada.ca/news.ts b/lib/routes/canada.ca/news.ts index 5892399e2d4233..5502c2b1d0bc1c 100644 --- a/lib/routes/canada.ca/news.ts +++ b/lib/routes/canada.ca/news.ts @@ -25,12 +25,11 @@ export const route: Route = { }, // Innovation, Science and Economic Development Canada { - source: [ - 'ised-isde.canada.ca/site/ised/:lang', - 'ised-isde.canada.ca/site/isde/:lang', - 'www.canada.ca/:lang/innovation-science-economic-development/news/*', - 'www.canada.ca/:lang/innovation-sciences-developpement-economique/nouvelles/*', - ], + source: ['ised-isde.canada.ca/site/ised/:lang', 'ised-isde.canada.ca/site/isde/:lang'], + target: '/news/:lang/departmentofindustry', + }, + { + source: ['www.canada.ca/:lang/innovation-science-economic-development/news/*', 'www.canada.ca/:lang/innovation-sciences-developpement-economique/nouvelles/*'], target: '/news/:lang/departmentofindustry', }, // All news diff --git a/lib/routes/capitalmind/utils.ts b/lib/routes/capitalmind/utils.ts index 5f92b42769115a..dc04f8b14e288a 100644 --- a/lib/routes/capitalmind/utils.ts +++ b/lib/routes/capitalmind/utils.ts @@ -2,6 +2,7 @@ import { load } from 'cheerio'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; +import logger from '@/utils/logger'; import ofetch from '@/utils/ofetch'; export const baseUrl = 'https://www.capitalmind.in'; diff --git a/lib/routes/cast/index.ts b/lib/routes/cast/index.ts index 887283ca4d1408..04119145473941 100644 --- a/lib/routes/cast/index.ts +++ b/lib/routes/cast/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -88,7 +88,7 @@ async function handler(ctx) { const $ = load(indexData); - let items: any[] = []; + let items: DataItem[]; // 新闻-视频首页特殊处理 if (column === 'xw' && subColumn === 'SP' && !category) { diff --git a/lib/routes/cbndata/information.ts b/lib/routes/cbndata/information.ts index 08bc5b5b507e41..aad1a06eb853d5 100644 --- a/lib/routes/cbndata/information.ts +++ b/lib/routes/cbndata/information.ts @@ -29,9 +29,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = response.data.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.slice(0, limit).map((item): DataItem => { const title: string = item.title; const image: string | undefined = item.image; const description: string | undefined = renderDescription({ diff --git a/lib/routes/ccagm/index.ts b/lib/routes/ccagm/index.ts index 2a95b397fa99d7..1eaf3d35ecbc88 100644 --- a/lib/routes/ccagm/index.ts +++ b/lib/routes/ccagm/index.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.news_list li a') + let items: DataItem[] = $('ul.news_list li a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/cccfna/index.ts b/lib/routes/cccfna/index.ts index de6152dd93cc2e..3d2b33d1d9bc8f 100644 --- a/lib/routes/cccfna/index.ts +++ b/lib/routes/cccfna/index.ts @@ -62,7 +62,8 @@ export const route: Route = { const items = await Promise.all( list.map((item) => cache.tryGet(item.link!, async () => { - const $ = load(await ofetch(item.link!)); + const html = await ofetch(item.link!); + const $ = load(html); const content = $('.list_cont'); item.title = content.find('.title').text(); diff --git a/lib/routes/cccmc/index.ts b/lib/routes/cccmc/index.ts index b134289e43b9ba..92441a0bb57d90 100644 --- a/lib/routes/cccmc/index.ts +++ b/lib/routes/cccmc/index.ts @@ -20,11 +20,9 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - const regex = /\{url:'(.*)',title:'(.*)',time:'(.*)'\},/g; - items = + let items: DataItem[] = response .match(regex) ?.slice(0, limit) diff --git a/lib/routes/ccg/index.ts b/lib/routes/ccg/index.ts index 4fa5559455bb60..0594be88795ffa 100644 --- a/lib/routes/ccg/index.ts +++ b/lib/routes/ccg/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.huodong-list li') + let items: DataItem[] = $('ul.huodong-list li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/cde/xxgk.tsx b/lib/routes/cde/xxgk.tsx index 0d858d0ceca796..c770309252cfc2 100644 --- a/lib/routes/cde/xxgk.tsx +++ b/lib/routes/cde/xxgk.tsx @@ -81,7 +81,7 @@ async function handler(ctx) { }); const items = data.data.records.map((item) => { - let description = ''; + let description: string; switch (category) { case 'priorityApproval': description = renderToString(); diff --git a/lib/routes/chinadaily/language.ts b/lib/routes/chinadaily/language.ts index b3f256fd62714c..47dc17388be09d 100644 --- a/lib/routes/chinadaily/language.ts +++ b/lib/routes/chinadaily/language.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.gy_box, ul.content_list li') + let items: DataItem[] = $('div.gy_box, ul.content_list li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/chinaratings/credit-research.ts b/lib/routes/chinaratings/credit-research.ts index 9f18b236767d25..4ccb5ae9ff251f 100644 --- a/lib/routes/chinaratings/credit-research.ts +++ b/lib/routes/chinaratings/credit-research.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.contRight ul.list li') + let items: DataItem[] = $('div.contRight ul.list li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/chongdiantou/index.ts b/lib/routes/chongdiantou/index.ts index 36c91c288d8b09..ec3f636f7816b7 100644 --- a/lib/routes/chongdiantou/index.ts +++ b/lib/routes/chongdiantou/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; @@ -23,7 +23,7 @@ export const route: Route = { async function handler() { const response = await ofetch('https://www.chongdiantou.com/nice-json/front-end/home-load-more'); - let items = []; + let items: DataItem[]; items = response.data.map((item) => ({ title: item.title, diff --git a/lib/routes/chsi/hotnews.ts b/lib/routes/chsi/hotnews.ts index 316ab02af29494..e0d54ea8c19bbf 100644 --- a/lib/routes/chsi/hotnews.ts +++ b/lib/routes/chsi/hotnews.ts @@ -41,7 +41,7 @@ async function handler() { let itemUrl = ''; itemUrl = path.startsWith('http') ? path : host + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; let itemDate; if (path) { const result = await got(itemUrl); diff --git a/lib/routes/chsi/kydt.ts b/lib/routes/chsi/kydt.ts index b590fdaa26d4ff..830fbee1b05fc8 100644 --- a/lib/routes/chsi/kydt.ts +++ b/lib/routes/chsi/kydt.ts @@ -46,7 +46,7 @@ async function handler() { const path = item.find('a').attr('href'); const itemUrl = path.startsWith('http') ? path : host + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; if (!path.startsWith('https://') && !path.startsWith('http://')) { const result = await got(itemUrl); const $ = load(result.data); diff --git a/lib/routes/chsi/kyzx.ts b/lib/routes/chsi/kyzx.ts index 824ceb163dce09..8b359069797b4f 100644 --- a/lib/routes/chsi/kyzx.ts +++ b/lib/routes/chsi/kyzx.ts @@ -53,7 +53,7 @@ async function handler(ctx) { let itemUrl = ''; itemUrl = path.startsWith('http') ? path : host + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; if (itemUrl) { const result = await got(itemUrl); const $ = load(result.data); diff --git a/lib/routes/chub/characters.ts b/lib/routes/chub/characters.ts index 0ed1d8a44502c1..ae237a85f88111 100644 --- a/lib/routes/chub/characters.ts +++ b/lib/routes/chub/characters.ts @@ -1,7 +1,16 @@ +import MarkdownIt from 'markdown-it'; + import type { Route } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +const md = MarkdownIt({ + breaks: true, + html: true, + linkify: true, + typographer: true, +}); + export const route: Route = { path: '/characters', categories: ['new-media'], @@ -14,6 +23,10 @@ export const route: Route = { }, }; +function convertUndefinedToString(value: any): string { + return value === undefined ? '' : value.toString(); +} + async function handler() { const hostURL = 'https://www.chub.ai/characters'; const apiURL = 'https://api.chub.ai/search'; @@ -52,13 +65,13 @@ async function handler() { link: hostURL, item: nodes.map((item) => ({ title: item.name, - description: `${item.tagline}

    ${item.description}`, + description: `
    ${item.tagline}
    ${md.render(convertUndefinedToString(item.description))}
    `, pubDate: parseDate(item.createdAt), updated: parseDate(item.lastActivityAt), link: `${hostURL}/${item.fullPath}`, author: String(item.fullPath.split('/', 1)), - enclosure_url: item.avatar_url, - enclosure_type: `image/webp`, + enclosure_url: item.max_res_url, + enclosure_type: `image/png`, category: item.topics, })), }; diff --git a/lib/routes/claude/blog.ts b/lib/routes/claude/blog.ts new file mode 100644 index 00000000000000..9cc243c6baeab6 --- /dev/null +++ b/lib/routes/claude/blog.ts @@ -0,0 +1,89 @@ +import { load } from 'cheerio'; +import pMap from 'p-map'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +const baseUrl = 'https://claude.com'; + +export const route: Route = { + path: '/blog', + categories: ['programming'], + example: '/claude/blog', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['claude.com/blog'], + target: '/blog', + }, + ], + name: 'Blog', + maintainers: ['zhenlohuang'], + handler, + url: 'claude.com/blog', +}; + +async function handler(ctx) { + const link = `${baseUrl}/blog`; + const response = await ofetch(link); + const $ = load(response); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 15; + + const list: DataItem[] = $('.blog_cms_list article.card_blog_list_wrap') + .toArray() + .slice(0, limit) + .map((el) => { + const $el = $(el); + const title = $el.find('.card_blog_list_title').text().trim(); + const href = $el.find('a.clickable_link').attr('href') ?? ''; + const pubDateText = $el.find('[fs-list-fieldtype="date"][fs-list-field="date"]').text().trim(); + const category = $el + .find('[fs-list-field="category"]') + .toArray() + .map((c) => $(c).text().trim()) + .filter(Boolean); + + return { + title, + link: href.startsWith('http') ? href : `${baseUrl}${href}`, + pubDate: pubDateText ? parseDate(pubDateText) : undefined, + category, + }; + }); + + const items = await pMap( + list, + (item) => + cache.tryGet(item.link!, async () => { + const response = await ofetch(item.link!); + const $ = load(response); + + const content = $('.blog_post_content_wrap'); + + content.find('style, script').remove(); + + item.description = content.html() ?? undefined; + + return item; + }), + { concurrency: 3 } + ); + + return { + title: 'Claude Blog', + link, + description: 'Product news and best practices for teams building with Claude.', + language: 'en', + item: items, + }; +} diff --git a/lib/routes/claude/code-changelog.ts b/lib/routes/claude/code-changelog.ts new file mode 100644 index 00000000000000..c5697ac52bc9d2 --- /dev/null +++ b/lib/routes/claude/code-changelog.ts @@ -0,0 +1,78 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; +import type { Context } from 'hono'; + +import type { Data, DataItem, Route } from '@/types'; +import ofetch from '@/utils/ofetch'; + +const handler = async (ctx: Context): Promise => { + const limit = Number.parseInt(ctx.req.query('limit') ?? '25', 10); + + const baseUrl = 'https://code.claude.com'; + const targetUrl = `${baseUrl}/docs/en/changelog`; + + const response = await ofetch(targetUrl); + const $: CheerioAPI = load(response); + + const items: DataItem[] = $('div.markdown-heading') + .slice(0, limit) + .toArray() + .map((el): DataItem => { + const $heading = $(el); + const version = $heading.find('h2.heading-element').text().trim(); + if (!version) { + return null as unknown as DataItem; + } + + const descriptionParts: string[] = []; + $heading.nextUntil('div.markdown-heading').each((_, sibling) => { + descriptionParts.push($(sibling).prop('outerHTML') ?? ''); + }); + const description = descriptionParts.join(''); + + const anchor = $heading.find('a.anchor').attr('href') ?? `#${version.replaceAll('.', '')}`; + const link = `${targetUrl}${anchor}`; + + return { + title: version, + description, + link, + guid: `claude-code-${version}`, + id: `claude-code-${version}`, + }; + }) + .filter(Boolean); + + return { + title: 'Claude Code Changelog', + description: 'Changelog for Claude Code CLI', + link: targetUrl, + item: items, + allowEmpty: true, + }; +}; + +export const route: Route = { + path: '/code/changelog', + name: 'Code Changelog', + url: 'code.claude.com', + maintainers: ['rmaced0'], + handler, + example: '/claude/code/changelog', + categories: ['program-update'], + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['code.claude.com/docs/en/changelog'], + target: '/code/changelog', + }, + ], +}; diff --git a/lib/routes/claude/namespace.ts b/lib/routes/claude/namespace.ts new file mode 100644 index 00000000000000..f2d0c96299d5ed --- /dev/null +++ b/lib/routes/claude/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Claude', + url: 'claude.com', + lang: 'en', +}; diff --git a/lib/routes/cma/channel.tsx b/lib/routes/cma/channel.tsx index a41b0784b3f564..59c9484bef6f37 100644 --- a/lib/routes/cma/channel.tsx +++ b/lib/routes/cma/channel.tsx @@ -104,8 +104,7 @@ async function handler(ctx) { $( $('div.col-xs-8 span') .toArray() - .filter((a) => $(a).text().startsWith('来源')) - ?.pop() + .findLast((a) => $(a).text().startsWith('来源')) ) ?.text() ?.split(/：/) diff --git a/lib/routes/cnljxh/index.ts b/lib/routes/cnljxh/index.ts index 24930df3d76931..6cc695cd3fb869 100644 --- a/lib/routes/cnljxh/index.ts +++ b/lib/routes/cnljxh/index.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.main_left ul li') + let items: DataItem[] = $('div.main_left ul li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/cognition/blog.ts b/lib/routes/cognition/blog.ts index f1a53b09ce7e0e..4e52bd107f965e 100644 --- a/lib/routes/cognition/blog.ts +++ b/lib/routes/cognition/blog.ts @@ -1,15 +1,15 @@ import { load } from 'cheerio'; -import type { Data, DataItem, Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import { ViewType } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { - path: '/blog', + path: '/blog/:category?', name: 'Blog', url: 'cognition.ai/blog', - maintainers: ['Loongphy'], + maintainers: ['Loongphy', 'ttttmr'], example: '/cognition/blog', categories: ['programming'], features: { @@ -23,12 +23,15 @@ export const route: Route = { }, radar: [ { - source: ['cognition.ai/blog/1'], - target: '/blog', + source: ['cognition.ai/blog/1', 'cognition.ai/blog/:category/1'], + target: '/blog/:category?', }, ], view: ViewType.Articles, handler, + parameters: { + category: 'Category name, e.g., Research, Tutorials', + }, }; const splitAuthors = (text: string | undefined): DataItem['author'] => { @@ -50,9 +53,10 @@ const splitAuthors = (text: string | undefined): DataItem['author'] => { })); }; -export async function handler(): Promise { +export async function handler(ctx) { const baseUrl = 'https://cognition.ai'; - const listPath = '/blog/1'; + const { category } = ctx.req.param(); + const listPath = category ? `/blog/${category}/1` : '/blog/1'; const targetUrl = new URL(listPath, baseUrl).href; const html = await ofetch(targetUrl); const $ = load(html); diff --git a/lib/routes/comic-fuz/magazine.ts b/lib/routes/comic-fuz/magazine.ts new file mode 100644 index 00000000000000..0d6c09072ac73e --- /dev/null +++ b/lib/routes/comic-fuz/magazine.ts @@ -0,0 +1,97 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/magazine/:id', + categories: ['anime'], + example: '/comic-fuz/magazine/27860', + parameters: { id: 'ComicFuz中对应的杂志id' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['comic-fuz.com/magazine/:id'], + target: '/magazine/:id', + }, + ], + name: '杂志详情', + maintainers: ['xiaobailoves'], + + handler: async (ctx) => { + const { id } = ctx.req.param(); + const baseUrl = 'https://comic-fuz.com'; + const openUrl = `${baseUrl}/magazine/${id}`; + const imgUrl = `https://img.comic-fuz.com`; + + const response = await ofetch(openUrl, { + headers: { + Referer: 'https://comic-fuz.com/', + 'Accept-Language': 'ja,en-US;q=0.9,en;q=0.8', + }, + }); + + const $ = load(response); + const nextDataText = $('#__NEXT_DATA__').text(); + + if (!nextDataText) { + throw new Error('无法解析页面数据，请检查杂志 ID 是否正确或页面结构是否变动'); + } + + const nextData = JSON.parse(nextDataText); + const pageProps = nextData.props?.pageProps; + + if (!pageProps) { + throw new Error('无法解析页面 Props 数据'); + } + + const magazineTitle = pageProps.magazineName || ''; + + const magazineDescription = pageProps.pickupMagazineIssue?.longDescription || ''; + + const issues = pageProps.magazineIssues || []; + + const items = issues.map((item: any) => { + const amount = item.paidPoint || 0; + const statusText = amount > 0 ? '付费' : '无料'; + + let thumb = item.thumbnailUrl; + if (thumb && thumb.startsWith('/')) { + thumb = `${imgUrl}${thumb}`; + } + + const rawDate = item.updatedDate ? item.updatedDate.replace(/\s*発売/, '').trim() : ''; + + return { + title: `${magazineTitle} - ${item.magazineIssueName}`, + link: `${baseUrl}/magazine/viewer/${item.magazineIssueId}`, + description: ` + ${thumb ? `
    ` : ''} +

    价格: ${amount} 金币/银币

    +

    发售日期: ${item.updatedDate}

    +

    截止日期: ${item.endDate || '无'}

    + ${item.longDescription ? `

    ${item.longDescription}

    ` : ''} + `, + pubDate: rawDate ? parseDate(rawDate, 'YYYY/MM/DD') : undefined, + guid: `comicfuz-magazine-id-${item.magazineIssueId}`, + category: [statusText], + }; + }); + + return { + title: `COMIC FUZ - ${magazineTitle}`, + link: openUrl, + description: magazineDescription, + item: items, + language: 'ja', + }; + }, +}; diff --git a/lib/routes/comic-fuz/manga.ts b/lib/routes/comic-fuz/manga.ts new file mode 100644 index 00000000000000..341e85f9543749 --- /dev/null +++ b/lib/routes/comic-fuz/manga.ts @@ -0,0 +1,106 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/manga/:id', + categories: ['anime'], + example: '/comic-fuz/manga/218', + parameters: { id: 'ComicFuz中对应的漫画id' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['comic-fuz.com/manga/:id'], + target: '/manga/:id', + }, + ], + name: '漫画详情', + maintainers: ['xiaobailoves'], + + handler: async (ctx) => { + const { id } = ctx.req.param(); + const baseUrl = 'https://comic-fuz.com'; + const openUrl = `${baseUrl}/manga/${id}`; + const imgUrl = `https://img.comic-fuz.com`; + + const response = await ofetch(openUrl, { + headers: { + Referer: 'https://comic-fuz.com/', + 'Accept-Language': 'ja,en-US;q=0.9,en;q=0.8', + }, + }); + + const $ = load(response); + const nextDataText = $('#__NEXT_DATA__').text(); + + if (!nextDataText) { + throw new Error('无法解析页面数据，请检查漫画 ID 是否正确或页面结构是否变动'); + } + + const nextData = JSON.parse(nextDataText); + const pageProps = nextData.props?.pageProps; + + if (!pageProps) { + throw new Error('无法解析页面 Props 数据'); + } + + const mangaTitle = $('title').text().trim(); + const mangaAuthor = pageProps.authorships?.map((item: any) => item.author?.authorName).join(', ') || ''; + const mangaDescription = pageProps.manga?.longDescription || ''; + + const chapterGroups = pageProps.chapters || []; + + const allChapters = chapterGroups.flatMap((group: any) => group.chapters || []); + + const items = allChapters.map((chapter: any) => { + const pointInfo = chapter.pointConsumption; + const amount = pointInfo?.amount || 0; + + let statusText = ''; + if (pointInfo && Object.keys(pointInfo).length === 0) { + statusText = '无料'; + } else if (amount > 0) { + statusText = '付费'; + } + + let thumb = chapter.thumbnailUrl; + if (thumb && thumb.startsWith('/')) { + thumb = `${imgUrl}${thumb}`; + } + + const fullTitle = `${chapter.chapterMainName}${chapter.chapterSubName ? ` - ${chapter.chapterSubName}` : ''}`; + + return { + title: fullTitle, + link: `${baseUrl}/manga/viewer/${chapter.chapterId}`, + description: ` + ${thumb ? `
    ` : ''} + ${amount > 0 ? `

    价格: ${amount} 金币/铜币

    ` : ''} + `, + guid: `comicfuz-comic-id-${chapter.chapterId}`, + category: statusText, + author: mangaAuthor, + pubDate: chapter.updatedDate ? parseDate(chapter.updatedDate, 'YYYY/MM/DD') : undefined, + upvotes: chapter.numberOfLikes || 0, + comments: chapter.numberOfComments || 0, + }; + }); + + return { + title: `COMIC FUZ - ${mangaTitle}`, + link: openUrl, + description: mangaDescription, + item: items, + language: 'ja', + }; + }, +}; diff --git a/lib/routes/comic-fuz/namespace.ts b/lib/routes/comic-fuz/namespace.ts new file mode 100644 index 00000000000000..bd74414f19e23e --- /dev/null +++ b/lib/routes/comic-fuz/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'COMIC FUZ', + url: 'comic-fuz.com', + lang: 'ja', +}; diff --git a/lib/routes/comic-walker/manga.ts b/lib/routes/comic-walker/manga.ts new file mode 100644 index 00000000000000..77bdf53afcdd5f --- /dev/null +++ b/lib/routes/comic-walker/manga.ts @@ -0,0 +1,122 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +const getEpisodes = (obj: any) => obj?.result || obj?.items || (Array.isArray(obj) ? obj : []); + +export const route: Route = { + path: '/manga/:id', + categories: ['anime'], + example: '/comic-walker/manga/KC_006778_S', + parameters: { id: 'カドコミ(Kadocomi)中对应的作品workCode，例如 KC_006778_S' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['comic-walker.com/detail/:id'], + target: '/manga/:id', + }, + ], + name: '漫画详情', + maintainers: ['xiaobailoves'], + + handler: async (ctx) => { + const { id } = ctx.req.param(); + const baseUrl = 'https://comic-walker.com'; + + const fetchUrl = `${baseUrl}/detail/${id}?episodeType=first`; + const openUrl = `${baseUrl}/detail/${id}`; + + const response = await ofetch(fetchUrl, { + headers: { + Referer: baseUrl, + 'Accept-Language': 'ja,en-US;q=0.9,en;q=0.8', + }, + }); + + const $ = load(response); + const nextDataText = $('#__NEXT_DATA__').text(); + + if (!nextDataText) { + throw new Error('无法解析页面 HTML 数据，可能触发了反爬策略或页面结构巨变'); + } + + const nextData = JSON.parse(nextDataText); + const queries = nextData.props?.pageProps?.dehydratedState?.queries || []; + + const workQuery = queries.find((q: any) => q.queryKey?.includes('/api/contents/details/work') || (Array.isArray(q.queryKey) && q.queryKey.some((k: any) => typeof k === 'string' && k.includes('work')))); + + if (!workQuery || !workQuery.state?.data) { + throw new Error('无法在 HTML 缓存中提取核心数据对象'); + } + + const data = workQuery.state.data; + const work = data.work; + + if (!work) { + throw new Error('成功获取数据对象，但未找到作品基本信息'); + } + + const mangaTitle = work.title || $('title').text().trim(); + const mangaAuthor = work.authors?.map((author: any) => author.name).join(', ') || ''; + const mangaDescription = work.summary || ''; + const coverImage = work.bookCover || work.thumbnail; + + const firstEpisodes = getEpisodes(data.firstEpisodes); + const latestEpisodes = getEpisodes(data.latestEpisodes); + const extraEpisodes = getEpisodes(data.episodes); + + const seenCodes = new Set(); + const allChapters = [...firstEpisodes, ...latestEpisodes, ...extraEpisodes] + .filter((ep) => { + if (ep?.code && !seenCodes.has(ep.code)) { + seenCodes.add(ep.code); + return true; + } + return false; + }) + .toSorted((a: any, b: any) => (b.internal?.episodeNo || 0) - (a.internal?.episodeNo || 0)); + + if (allChapters.length === 0) { + throw new Error('HTML 缓存中无章节!'); + } + + const items = allChapters.map((chapter: any) => { + const epType = chapter.type === 'normal' ? '正篇' : '特别篇/PR'; + const isReadStatus = chapter.isActive ? '' : ' (未解锁/仍需等待)'; + const fullTitle = `${chapter.title}${chapter.subTitle ? ` - ${chapter.subTitle}` : ''}${isReadStatus}`; + const thumb = chapter.originalThumbnail || chapter.thumbnail; + + const currentPubDate = chapter.updateDate ? parseDate(chapter.updateDate) : undefined; + + return { + title: fullTitle, + link: `${baseUrl}/detail/${id}/episodes/${chapter.code}`, + description: ` + ${thumb ? `
    ` : ''} + `, + guid: `Kadocomi-manga-${chapter.code}`, + category: epType, + author: mangaAuthor, + pubDate: currentPubDate, + }; + }); + + return { + title: `Kadocomi - ${mangaTitle}`, + link: openUrl, + description: mangaDescription, + image: coverImage, + item: items, + language: 'ja', + }; + }, +}; diff --git a/lib/routes/comic-walker/namespace.ts b/lib/routes/comic-walker/namespace.ts new file mode 100644 index 00000000000000..d98db15372c0fd --- /dev/null +++ b/lib/routes/comic-walker/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'カドコミ(Kadocomi)', + url: 'comic-walker.com', + lang: 'ja', +}; diff --git a/lib/routes/copymanga/comic.tsx b/lib/routes/copymanga/comic.tsx index 67321cc65a1ee5..d57e16e15528b3 100644 --- a/lib/routes/copymanga/comic.tsx +++ b/lib/routes/copymanga/comic.tsx @@ -41,7 +41,7 @@ async function handler(ctx) { const chapterArray = await cache.tryGet( strBaseUrl, async () => { - let bHasNextPage = false; + let bHasNextPage: boolean; let chapters = []; let iReqOffSet = 0; diff --git a/lib/routes/costar/press-releases.ts b/lib/routes/costar/press-releases.ts index adb7154ea9262c..7afabf862fbf4c 100644 --- a/lib/routes/costar/press-releases.ts +++ b/lib/routes/costar/press-releases.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.views-row article') + let items: DataItem[] = $('div.views-row article') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/csu/utils.ts b/lib/routes/csu/utils.ts index aefb50aee7ff73..7283a8d26a6bf9 100644 --- a/lib/routes/csu/utils.ts +++ b/lib/routes/csu/utils.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ import { inflateSync } from 'node:zlib'; const unzip = (b64Data) => { diff --git a/lib/routes/curius/links.tsx b/lib/routes/curius/links.tsx index 56d08d6108e787..d2bccb9f33f7d4 100644 --- a/lib/routes/curius/links.tsx +++ b/lib/routes/curius/links.tsx @@ -66,7 +66,7 @@ async function handler(ctx) { } const renderDescription = (item): string => { - const fullText = item.metadata?.full_text ? item.metadata.full_text.replaceAll(/\n/gm, '
    ') : ''; + const fullText = item.metadata?.full_text ? item.metadata.full_text.replaceAll('\n', '
    ') : ''; const firstComment = item.comments?.length ? item.comments[0].text.slice(0, 100) : ''; return renderToString( diff --git a/lib/routes/cursor/changelog.ts b/lib/routes/cursor/changelog.ts index c9b9e03639a3e5..cacbc9f095b3ce 100644 --- a/lib/routes/cursor/changelog.ts +++ b/lib/routes/cursor/changelog.ts @@ -22,42 +22,29 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = ($('html').attr('lang') ?? 'en') as Language; - const items: DataItem[] = $('article.relative') + const items: DataItem[] = $('main') + .first() + .find('article') .slice(0, limit) .toArray() .map((el): DataItem => { const $el: Cheerio = $(el); - let version = ''; - let pubDateStr: string | undefined; + const timeEl = $el.find('time').first(); + const pubDateStr = timeEl.attr('datetime') || timeEl.text().trim(); + const versionLabel = timeEl.closest('a').find('.label').text().trim(); - $el.find('div').each((_, div) => { - const text = $(div).text().trim(); - const dateVersionMatch = text.match(/^(\w+\s+\d{1,2},\s+\d{4})(\d+\.\d+)$/); - if (dateVersionMatch) { - pubDateStr = dateVersionMatch[1]; - version = dateVersionMatch[2]; - return false; // Stop after finding first match - } - }); - - const linkEl = $el.find('a[href^="/changelog/"]').first(); - const titleText = linkEl.length ? linkEl.text().trim() : $el.find('h2').first().text().trim(); - - const title: string = version ? `[${version}] ${titleText}` : titleText; + const linkEl = $el.find('h1 a').first(); + const titleText = linkEl.length ? linkEl.text().trim() : $el.find('h1').first().text().trim(); + const title: string = versionLabel ? `[${versionLabel}] ${titleText}` : titleText; const linkUrl: string | undefined = linkEl.attr('href'); - const guid = `cursor-changelog-${version || 'unknown'}`; - const upDatedStr: string | undefined = pubDateStr; - - const $h2El = $el.find('h2').first(); - - if ($h2El.length) { - $h2El.prevAll().remove(); - $h2El.remove(); + let guid = linkUrl ? linkUrl.split('/').pop() : 'unknown'; + if (versionLabel) { + guid = `cursor-changelog-${versionLabel}`; } - const description: string = $el.html() || ''; + const description: string = $el.find('.prose').html() || ''; const processedItem: DataItem = { title, @@ -70,7 +57,6 @@ export const handler = async (ctx: Context): Promise => { html: description, text: description, }, - updated: upDatedStr ? parseDate(upDatedStr) : undefined, language, }; @@ -79,7 +65,7 @@ export const handler = async (ctx: Context): Promise => { return { title: $('title').text(), - description: $('meta[property="og:description"]').attr('content'), + description: $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content'), link: targetUrl, item: items, allowEmpty: true, diff --git a/lib/routes/cyzone/util.ts b/lib/routes/cyzone/util.ts index 6f1cc8f9e9c74f..95f4b83b2b65b2 100644 --- a/lib/routes/cyzone/util.ts +++ b/lib/routes/cyzone/util.ts @@ -65,7 +65,7 @@ const processItems = async (apiUrl, limit, tryGet, ...params) => { return { title: item.title, - link: /^\/\//.test(item.url) ? `https:${item.url}` : item.url, + link: item.url.startsWith('//') ? `https:${item.url}` : item.url, description: item.description, category: [item.category_name, ...(item.tags?.split(',') ?? [])], guid: item.content_id, diff --git a/lib/routes/dailypush/utils.ts b/lib/routes/dailypush/utils.ts index 2198cbf992ef9b..041eaa600ed077 100644 --- a/lib/routes/dailypush/utils.ts +++ b/lib/routes/dailypush/utils.ts @@ -4,14 +4,14 @@ import { load } from 'cheerio'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; -import { parseDate } from '@/utils/parse-date'; +import { parseRelativeDate } from '@/utils/parse-date'; export const BASE_URL = 'https://www.dailypush.dev'; export interface ArticleItem { title: string; link: string; - author?: string; + author: DataItem['author']; pubDate?: Date; category?: string[]; description?: string; @@ -19,22 +19,81 @@ export interface ArticleItem { dailyPushUrl?: string; } +/** + * Try to parse text as a date. Returns the Date if parsing succeeds and is valid, undefined otherwise. + */ +function tryParseAsDate(text: string): Date | undefined { + try { + const date = parseRelativeDate(text); + return Number.isNaN(date.getTime()) ? undefined : date; + } catch { + return undefined; + } +} + /** * Extract author from article element */ -function extractAuthor(article: ReturnType): string | undefined { +function extractAuthor(article: ReturnType): DataItem['author'] { + const container = article.find('.flex.items-center.gap-3').first(); + if (container.length === 0) { + return undefined; + } + + // Get all content spans (exclude separator spans with '•') + const allSpans = container.find('span'); + const contentSpans: string[] = []; + + for (let i = 0; i < allSpans.length; i++) { + const $span = allSpans.eq(i); + const text = $span.text().trim(); + // Skip separator spans (contain only '•' or have separator classes) + if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { + contentSpans.push(text); + } + } + + // Handle different cases based on number of content spans + switch (contentSpans.length) { + case 3: + // Structure: author, date, reading time + if (contentSpans[0].includes(',')) { + const authors: DataItem['author'] = contentSpans[0].split(',').map((author) => ({ + name: author.trim(), + })); + return authors; + } + return contentSpans[0]; + case 2: { + // Two cases: + // 1. date, reading time (no author) + // 2. author, date (no reading time) + const firstText = contentSpans[0]; + if (tryParseAsDate(firstText)) { + // First is date, so no author + break; + } + // First is author + return firstText; + } + case 1: { + // Could be date or author + const text = contentSpans[0]; + if (tryParseAsDate(text)) { + return undefined; + } + return text; + } + default: + break; + } + + // Fallback: use the post source as author const sourceSpan = article.find('span.text-xs.font-medium.uppercase').first(); if (sourceSpan.length > 0) { return sourceSpan.text().trim(); } - // Fallback: look for author name in the date section - const authorDateText = article.find('.flex.items-center.gap-3').first().text(); - const authorMatch = authorDateText.match(/^([^•]+?)(?:\s*•)/); - if (authorMatch && !/\d{4}/.test(authorMatch[1])) { - return authorMatch[1].trim(); - } - return undefined; } @@ -63,52 +122,59 @@ function extractCategories(article: ReturnType, $: CheerioAPI): stri * Extract publication date from article element */ function extractPubDate(article: ReturnType): Date | undefined { - const footer = article.find('.flex.items-center.justify-between.gap-4.flex-wrap').first(); - const authorAndDate = footer.find('.flex.items-center.gap-3.text-xs').first(); - - if (authorAndDate.length === 0) { + const container = article.find('.flex.items-center.gap-3').first(); + if (container.length === 0) { return undefined; } - const spans = authorAndDate.find('span'); - let dateText: string | undefined; + // Get all content spans (exclude separator spans with '•') + const allSpans = container.find('span'); + const contentSpans: string[] = []; - if (spans.length === 3) { - // Has author: date is in the third span (index 2) - // Structure: Author•Date - dateText = spans.eq(2).text().trim(); - } else if (spans.length === 1) { - // No author: date is in the first span (index 0) - // Structure: Date - dateText = spans.eq(0).text().trim(); + for (let i = 0; i < allSpans.length; i++) { + const $span = allSpans.eq(i); + const text = $span.text().trim(); + // Skip separator spans (contain only '•' or have separator classes) + if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) { + contentSpans.push(text); + } } - if (!dateText) { - return undefined; - } + let dateText: string | undefined; - try { - return parseDate(dateText); - } catch { - // If parsing fails, try fallback patterns - const datePattern = /((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+,\s+\d{4})/i; - const match = dateText.match(datePattern); - if (match && match[1]) { - try { - return parseDate(match[1]); - } catch { - // If parsing fails, keep undefined + // Handle different cases based on number of content spans + switch (contentSpans.length) { + case 3: + // Structure: author, date, reading time + dateText = contentSpans[1]; + break; + case 2: { + // Two cases: + // 1. date, reading time (no author) + // 2. author, date (no reading time) + const firstText = contentSpans[0]; + dateText = tryParseAsDate(firstText) ? firstText : contentSpans[1]; + break; + } + case 1: { + // Could be date or author + const text = contentSpans[0]; + if (tryParseAsDate(text)) { + dateText = text; } + break; } + default: + break; } - return undefined; + return dateText ? tryParseAsDate(dateText) : undefined; } /** * Parse a single article element into an ArticleItem */ -function parseArticle(article: ReturnType, $: CheerioAPI, baseUrl: string): ArticleItem | null { +function parseArticle(article: ReturnType, $: CheerioAPI, baseUrl: string): (DataItem & ArticleItem) | null { // Find the title link in h2 > a const titleLink = article.find('h2 a[href^="http"]').first(); if (titleLink.length === 0) { @@ -135,12 +201,13 @@ function parseArticle(article: ReturnType, $: CheerioAPI, baseUrl: s return { title, link, - author: author || undefined, + author, pubDate, category: categories.length > 0 ? categories : undefined, description, articleUrl: link, dailyPushUrl, + language: 'en', }; } @@ -162,7 +229,7 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] { */ export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined); - const itemsWithoutUrl = items.filter((item) => item.dailyPushUrl === undefined); + const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined); const enhancedItems: DataItem[] = await Promise.all( itemsWithUrl.map((item) => diff --git a/lib/routes/dbaplus/new.ts b/lib/routes/dbaplus/new.ts index d8d75f8d9cd1ba..2aa195e8a09617 100644 --- a/lib/routes/dbaplus/new.ts +++ b/lib/routes/dbaplus/new.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('ul.media-list li.media') + let items: DataItem[] = $('ul.media-list li.media') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/deepl/blog.ts b/lib/routes/deepl/blog.ts index 41419f934ffec4..0fb408d85bc389 100644 --- a/lib/routes/deepl/blog.ts +++ b/lib/routes/deepl/blog.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? lang; - let items: DataItem[] = []; - - items = $('h4, h6') + let items: DataItem[] = $('h4, h6') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/dgtle/util.ts b/lib/routes/dgtle/util.ts index a9a83f77b1773e..0eaf19610fc818 100644 --- a/lib/routes/dgtle/util.ts +++ b/lib/routes/dgtle/util.ts @@ -17,9 +17,7 @@ const md = MarkdownIt({ const baseUrl = 'https://www.dgtle.com'; const ProcessItems = async (limit: number, dataList: any): Promise => { - let items: DataItem[] = []; - - items = dataList.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = dataList.slice(0, limit).map((item): DataItem => { const title: string = item.title || item.content; const image: string | undefined = item.cover; const description: string | undefined = renderDescription({ diff --git a/lib/routes/dgtle/video.ts b/lib/routes/dgtle/video.ts index da8ce22d2eb361..1cb7c441c73c34 100644 --- a/lib/routes/dgtle/video.ts +++ b/lib/routes/dgtle/video.ts @@ -25,9 +25,7 @@ export const handler = async (ctx: Context): Promise => { const response = await ofetch(apiUrl); - let items: DataItem[] = []; - - items = response.data.list.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.list.slice(0, limit).map((item): DataItem => { const title: string = item.title; const image: string | undefined = item.cover?.split(/\?/)?.[0]; const description: string | undefined = renderDescription({ diff --git a/lib/routes/dhu/jiaowu/news.ts b/lib/routes/dhu/jiaowu/news.ts index 14b1efe76d7001..898278ac8c932d 100644 --- a/lib/routes/dhu/jiaowu/news.ts +++ b/lib/routes/dhu/jiaowu/news.ts @@ -60,7 +60,7 @@ async function handler(ctx) { return await cache.tryGet(url, async () => { // fetch article content // some contents are only available for internal network - let description = ''; + let description: string; try { const { data: response } = await got(url); const $ = load(response); diff --git a/lib/routes/dhu/xxgk/news.ts b/lib/routes/dhu/xxgk/news.ts index 36c52a931957f8..fc98fd30c6986e 100644 --- a/lib/routes/dhu/xxgk/news.ts +++ b/lib/routes/dhu/xxgk/news.ts @@ -51,7 +51,7 @@ async function handler() { return await cache.tryGet(url, async () => { // fetch article content // some contents are only available for internal network - let description = ''; + let description: string; try { const { data: response } = await got(url); const $ = load(response); diff --git a/lib/routes/dianping/user.ts b/lib/routes/dianping/user.ts index f0b77432219593..3503b66027e15a 100644 --- a/lib/routes/dianping/user.ts +++ b/lib/routes/dianping/user.ts @@ -23,7 +23,11 @@ export const route: Route = { }, radar: [ { - source: ['dianping.com/member/:id', 'm.dianping.com/userprofile/:id'], + source: ['dianping.com/member/:id'], + target: '/dianping/user/:id', + }, + { + source: ['m.dianping.com/userprofile/:id'], target: '/dianping/user/:id', }, ], diff --git a/lib/routes/diariofruticola/filtro.ts b/lib/routes/diariofruticola/filtro.ts index 79fcbdc1eb4637..b52d577946889c 100644 --- a/lib/routes/diariofruticola/filtro.ts +++ b/lib/routes/diariofruticola/filtro.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'es'; - let items: DataItem[] = []; - - items = $('div#printableArea a.text-dark') + let items: DataItem[] = $('div#printableArea a.text-dark') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/digitalpolicyalert/activity-tracker.ts b/lib/routes/digitalpolicyalert/activity-tracker.ts index 0eda4f5bddd868..ad8042148b0ddd 100644 --- a/lib/routes/digitalpolicyalert/activity-tracker.ts +++ b/lib/routes/digitalpolicyalert/activity-tracker.ts @@ -40,9 +40,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = response.results.slice(0, limit).map((item): DataItem => { + const items: DataItem[] = response.results.slice(0, limit).map((item): DataItem => { const title: string = item.title; const description: string | undefined = item.latest_event?.description ?? undefined; const pubDate: number | string = item.latest_event?.date; diff --git a/lib/routes/discuz/discuz.ts b/lib/routes/discuz/discuz.ts index b3eedadc47d5b0..359dd9bf73cce3 100644 --- a/lib/routes/discuz/discuz.ts +++ b/lib/routes/discuz/discuz.ts @@ -13,7 +13,7 @@ function fixUrl(itemLink, baseUrl) { // 处理相对链接 if (itemLink) { if (baseUrl && !/^https?:\/\//.test(baseUrl)) { - baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl; + baseUrl = baseUrl.startsWith('//') ? 'http:' + baseUrl : 'http://' + baseUrl; } itemLink = new URL(itemLink, baseUrl).href; } diff --git a/lib/routes/dlsite/utils.ts b/lib/routes/dlsite/utils.ts index 069881a797e1c2..b13e050f0c6115 100644 --- a/lib/routes/dlsite/utils.ts +++ b/lib/routes/dlsite/utils.ts @@ -133,12 +133,7 @@ const ProcessItems = async (ctx) => { images = images.length === 0 ? [detail.work_image] : images; return { - title: `${ - discountRate - ? `${discountRate}% OFF - ${` ${discountEndDate ? `${dayjs(discountEndDate).format('YYYY-MM-DD HH:mm')} まで` : ''}`}` - : ' ' - }${title}`, + title: `${discountRate ? `${discountRate}% OFF ${discountEndDate ? `${dayjs(discountEndDate).format('YYYY-MM-DD HH:mm')} まで` : ''}` : ' '}${title}`, link, pubDate, author: authors.map((a) => a.name).join(' / '), diff --git a/lib/routes/dmzj/news.ts b/lib/routes/dmzj/news.ts index 4a4cdfb1f1d341..a99b1594a00fca 100644 --- a/lib/routes/dmzj/news.ts +++ b/lib/routes/dmzj/news.ts @@ -35,7 +35,8 @@ export const route: Route = { async function handler(ctx) { const url = `https://news.dmzj.com/${ctx.req.param('category') || ''}`; - const $ = load((await got(url)).data); + const response = await got(url); + const $ = load(response.data); return { title: $('title').text(), link: url, diff --git a/lib/routes/domp4/utils.ts b/lib/routes/domp4/utils.ts index 06f5d2dace4e57..c01cc48142c66e 100644 --- a/lib/routes/domp4/utils.ts +++ b/lib/routes/domp4/utils.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; diff --git a/lib/routes/douban/book/rank.ts b/lib/routes/douban/book/rank.ts index b17915659b7742..a68a22abd10cfe 100644 --- a/lib/routes/douban/book/rank.ts +++ b/lib/routes/douban/book/rank.ts @@ -26,7 +26,7 @@ async function handler(ctx) { const { type = '' } = ctx.req.param(); const referer = `https://m.douban.com/book/${type}`; - const _ = async (type) => { + const requestItem = async (type) => { const response = await got({ url: `https://m.douban.com/rexxar/api/v2/subject_collection/book_${type}/items?start=0&count=10`, headers: { Referer: referer }, @@ -34,7 +34,7 @@ async function handler(ctx) { return response.data.subject_collection_items; }; - const items = type ? await _(type) : [...(await _('fiction')), ...(await _('nonfiction'))]; + const items = type ? await requestItem(type) : [...(await requestItem('fiction')), ...(await requestItem('nonfiction'))]; return { title: `豆瓣热门图书-${type ? (type === 'fiction' ? '虚构类' : '非虚构类') : '全部'}`, diff --git a/lib/routes/douban/channel/subject.ts b/lib/routes/douban/channel/subject.ts index 1b0ff5eaff1040..240895c8e5d0b7 100644 --- a/lib/routes/douban/channel/subject.ts +++ b/lib/routes/douban/channel/subject.ts @@ -45,7 +45,7 @@ async function handler(ctx) { const channel_name = channel_info_response.data.title; const data = response.data.modules[nav].payload.subjects; - let nav_name = ''; + let nav_name: string; switch (nav) { case '0': diff --git a/lib/routes/douban/channel/topic.ts b/lib/routes/douban/channel/topic.ts index 3b285f6cac3571..c91dbf5cbe15ea 100644 --- a/lib/routes/douban/channel/topic.ts +++ b/lib/routes/douban/channel/topic.ts @@ -45,7 +45,7 @@ async function handler(ctx) { const channel_name = channel_info_response.data.title; const data = response.data.items; - let nav_name = ''; + let nav_name: string; switch (nav) { case 'hot': diff --git a/lib/routes/douban/tv/coming.ts b/lib/routes/douban/tv/coming.ts new file mode 100644 index 00000000000000..493606ae707d7e --- /dev/null +++ b/lib/routes/douban/tv/coming.ts @@ -0,0 +1,228 @@ +import { webcrypto } from 'node:crypto'; + +import { config } from '@/config'; +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +const apiUrl = 'https://frodo.douban.com/api/v2/tv/coming_soon'; +const apiKey = '0dad551ec0f84ed02907ff5c42e8ec70'; +const apiSecret = 'bf7dddc7c9cfe6f7'; +const apiClientUa = 'api-client/1 com.douban.frodo/7.22.0.beta9(231) Android/23 product/Mate 40 vendor/HUAWEI model/Mate 40 brand/HUAWEI rom/android network/wifi platform/AndroidPad'; + +type ComingSoonSubject = { + id: string; + title: string; + url?: string; + sharing_url?: string; + pubdate?: string[]; + wish_count?: number | string; + card_subtitle?: string; + intro?: string; + genres?: string[]; + cover_url?: string; + pic?: { + large?: string; + }; +}; + +type ComingSoonResponse = { + count?: number; + start?: number; + total?: number; + subjects?: ComingSoonSubject[]; + msg?: string; + message?: string; + reason?: string; +}; + +const signRequest = async (url: string, ts: string, method = 'GET'): Promise => { + const urlPath = new URL(url).pathname; + const rawSign = `${method.toUpperCase()}&${encodeURIComponent(urlPath)}&${ts}`; + const keyData = new TextEncoder().encode(apiSecret); + const messageData = new TextEncoder().encode(rawSign); + const key = await webcrypto.subtle.importKey('raw', keyData, { name: 'HMAC', hash: 'SHA-1' }, false, ['sign']); + const signature = await webcrypto.subtle.sign('HMAC', key, messageData); + return Buffer.from(signature).toString('base64'); +}; + +const getPubDateText = (pubdate?: string[]): string | undefined => pubdate?.[0]; + +const getPubDate = (pubdate?: string[]): Date | undefined => { + const pubDateText = getPubDateText(pubdate); + if (!pubDateText) { + return undefined; + } + + const datePart = pubDateText.split('(')[0]; + return parseDate(datePart); +}; + +const getSortTimestamp = (pubdate?: string[]): number => { + const pubDateText = getPubDateText(pubdate); + if (!pubDateText) { + return Number.POSITIVE_INFINITY; + } + + const datePart = pubDateText.split('(')[0].trim(); + const match = /^(\d{4})(?:-(\d{1,2}))?(?:-(\d{1,2}))?/.exec(datePart); + if (!match) { + return Number.POSITIVE_INFINITY; + } + + const year = Number.parseInt(match[1], 10); + const month = match[2] ? Number.parseInt(match[2], 10) : 1; + const day = match[3] ? Number.parseInt(match[3], 10) : 1; + const timestamp = Date.UTC(year, month - 1, day); + return Number.isNaN(timestamp) ? Number.POSITIVE_INFINITY : timestamp; +}; + +const getWishCount = (wishCount?: number | string): number => { + if (typeof wishCount === 'number') { + return wishCount; + } + if (typeof wishCount === 'string') { + const parsed = Number.parseInt(wishCount, 10); + return Number.isNaN(parsed) ? 0 : parsed; + } + return 0; +}; + +const renderDescription = (subject: { intro?: string; wish_count?: number | string }): string => { + const wishCount = getWishCount(subject.wish_count); + const wishCountText = wishCount > 0 ? `想看人数：${wishCount}` : ''; + const introText = subject.intro ?? ''; + if (wishCountText && introText) { + return `${wishCountText}，${introText}`; + } + return wishCountText || introText; +}; + +const buildFetchError = (error: unknown): Error => { + const status = (error as { response?: { status?: number } })?.response?.status; + if (status === 429) { + return new Error('Douban 请求过于频繁（429）。请稍后重试，或降低请求频率。'); + } + if (status === 403) { + return new Error('Douban 拒绝访问（403），可能触发反爬策略。请稍后重试。'); + } + return new Error('Douban 数据请求失败，可能触发反爬或限频，请稍后重试。'); +}; + +export const route: Route = { + path: '/tv/coming/:sortBy?/:count?', + categories: ['social-media'], + example: '/douban/tv/coming', + parameters: { + sortBy: '排序方式，可选，支持 `hot` 或 `time`，默认 `hot`', + count: '请求上游返回数量，可选，正整数，默认 `10`', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: '即将播出的剧集', + maintainers: ['honue'], + handler, + description: `| 路径参数 | 含义 | 接受的值 | 默认值 | +| -------- | ---------------- | -------- | ------ | +| sortBy | 排序方式 | hot/time | hot | +| count | 请求上游返回数量 | 正整数 | 10 | + + 用例：\`/douban/tv/coming/hot/10\` + +::: tip + 服务端请求固定使用 \`sortby=hot\` 拉取数据，再按 \`sortBy\` 参数在本地重排；条目数量可通过 \`count\` 调整，仍可叠加 RSSHub 通用参数 \`limit\`。 +:::`, +}; + +async function handler(ctx) { + const sortByParam = ctx.req.param('sortBy'); + const countParam = ctx.req.param('count'); + + const sortBy = sortByParam === 'time' ? 'time' : 'hot'; + const rawCount = Number.parseInt(countParam || '', 10); + const requestCount = Number.isNaN(rawCount) || rawCount <= 0 ? 10 : rawCount; + + const ts = new Date().toISOString().slice(0, 10).replaceAll('-', ''); + const searchParams: Record = { + start: 0, + count: requestCount, + sortby: 'hot', + os_rom: 'android', + apiKey, + _ts: ts, + _sig: await signRequest(apiUrl, ts), + }; + + const cacheKey = `douban:tv:coming:${requestCount}`; + const data = (await cache.tryGet( + cacheKey, + async () => { + try { + const response = await got({ + method: 'get', + url: apiUrl, + searchParams, + headers: { + Accept: 'application/json', + 'User-Agent': apiClientUa, + }, + }); + return response.data as ComingSoonResponse; + } catch (error) { + throw buildFetchError(error); + } + }, + config.cache.routeExpire, + false + )) as ComingSoonResponse; + + if (!Array.isArray(data.subjects)) { + const details = data.msg || data.message || data.reason; + throw new Error(`Douban 返回数据结构异常，可能触发反爬或限频。${details ? `上游信息：${details}` : ''}`); + } + if (data.subjects.length === 0) { + throw new Error('Douban 返回空数据，可能触发反爬或限频。请稍后重试。'); + } + + const subscriptionCount = data.count ?? 0; + const total = data.total ?? 0; + const sortedSubjects = data.subjects.toSorted((a, b) => { + if (sortBy === 'time') { + const timeDiff = getSortTimestamp(a.pubdate) - getSortTimestamp(b.pubdate); + if (timeDiff !== 0) { + return timeDiff; + } + return getWishCount(b.wish_count) - getWishCount(a.wish_count); + } + const wishDiff = getWishCount(b.wish_count) - getWishCount(a.wish_count); + if (wishDiff !== 0) { + return wishDiff; + } + return 0; + }); + return { + title: '豆瓣剧集-即将播出', + link: 'https://movie.douban.com/tv/', + description: `即将播出的剧集，请求参数: count=${subscriptionCount}, total=${total}, sortBy=${sortBy}, requestCount=${requestCount}`, + item: sortedSubjects.map((subject) => { + const link = subject.url || subject.sharing_url || `https://movie.douban.com/subject/${subject.id}/`; + const category = subject.card_subtitle ? [subject.card_subtitle] : (subject.genres ?? []); + const pubDate = sortBy === 'time' ? getPubDate(subject.pubdate) : undefined; + return { + title: subject.title, + category: category.length > 0 ? category : undefined, + pubDate, + description: renderDescription(subject), + link, + guid: link, + }; + }), + }; +} diff --git a/lib/routes/duozhi/index.ts b/lib/routes/duozhi/index.ts index f5c1aef22044f2..82275da22952a9 100644 --- a/lib/routes/duozhi/index.ts +++ b/lib/routes/duozhi/index.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.post-item') + let items: DataItem[] = $('div.post-item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/duozhuayu/search.tsx b/lib/routes/duozhuayu/search.tsx index 35fed22ec49f53..1a977755ba04ef 100644 --- a/lib/routes/duozhuayu/search.tsx +++ b/lib/routes/duozhuayu/search.tsx @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ import aesjs from 'aes-js'; import { renderToString } from 'hono/jsx/dom/server'; diff --git a/lib/routes/dytt/index.ts b/lib/routes/dytt/index.ts index 3a58a5a5cee827..d62cccaabbe1df 100644 --- a/lib/routes/dytt/index.ts +++ b/lib/routes/dytt/index.ts @@ -26,9 +26,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(iconv.decode(Buffer.from(response), 'gb2312')); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.co_content8 ul table') + let items: DataItem[] = $('div.co_content8 ul table') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/eastmoney/utils.ts b/lib/routes/eastmoney/utils.ts index 6d040d8e3276cf..ab3078cfa64717 100644 --- a/lib/routes/eastmoney/utils.ts +++ b/lib/routes/eastmoney/utils.ts @@ -1,5 +1,5 @@ function getRatingChangeStr(ratingChange) { - let ratingChangeName = ''; + let ratingChangeName: string; switch (String(ratingChange)) { case '0': ratingChangeName = '调高'; diff --git a/lib/routes/eeo/kuaixun.ts b/lib/routes/eeo/kuaixun.ts index f6b2802b67284d..28cf67b811ba5a 100644 --- a/lib/routes/eeo/kuaixun.ts +++ b/lib/routes/eeo/kuaixun.ts @@ -34,9 +34,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = response.data.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.slice(0, limit).map((item): DataItem => { const title: string = item.title; const description: string | undefined = renderDescription({ intro: item.description, diff --git a/lib/routes/elamigos/index.ts b/lib/routes/elamigos/index.ts new file mode 100644 index 00000000000000..7ee9715444e0b6 --- /dev/null +++ b/lib/routes/elamigos/index.ts @@ -0,0 +1,171 @@ +import { load } from 'cheerio'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; + +export const route: Route = { + path: '/games', + categories: ['game'], + example: '/elamigos/games', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['elamigos.site/', 'elamigos.site/index.html'], + target: '/games', + }, + ], + name: 'Releases', + maintainers: ['Kylon92'], + handler, + description: 'Latest game releases from ElAmigos', +}; + +async function handler(ctx: any) { + const limit = Number(ctx.req.query('limit')) || 40; + const baseUrl = 'https://elamigos.site'; + + const { data: html } = await got(baseUrl); + const $ = load(html); + + const games = extractGames($, limit, baseUrl); + const items = await Promise.all(games.map((game) => processGameItem(game))); + + return { + title: 'ElAmigos - Latest Games', + link: baseUrl, + description: `Latest game releases from ElAmigos (${items.length} entries)`, + item: items, + }; +} + +function toNeutralDate(input: string, appendDay: boolean = false): Date { + const [day, month, year] = input.split('.').map(Number); + const baseDate = new Date(Date.UTC(year, month - 1, day)); + return appendDay ? new Date(baseDate.getTime() + 24 * 60 * 60 * 1000) : baseDate; +} + +function extractGames($: any, limit: number, baseUrl: string): Array<{ title: string; link: string; pubDate: string | null }> { + const dateRegex = /^\d{2}\.\d{2}\.\d{4}$/; + const games: Array<{ title: string; link: string; pubDate: string | null }> = []; + let arrivedAtGameSection = false; + + // Extract games with their publication dates based on H1 week markers + // We ignore the first H1 Date as that tells us the current Release Week of ElAmigos + // They just auto-update that Date and keep all Games released under that H1 Tag for the week + // So the best workaround to fill the Publish Date is to look for the next H1 Date Tag and append a Day to that Date. + $('h1, h3, h5').each((_, elem) => { + const $elem = $(elem); + const tagName = elem.tagName.toLowerCase(); + + if (tagName === 'h1') { + const text = $elem.text().trim(); + if (!dateRegex.test(text)) { + return; + } + arrivedAtGameSection = true; + // Found H1 date, fill all empty Games with the new Date. + for (const game of games) { + if (game.pubDate === null || game.pubDate.trim() === '') { + game.pubDate = text; + } + } + } else if ((tagName === 'h3' || tagName === 'h5') && arrivedAtGameSection) { + const link = $elem.find('a[href]').first(); + if (!link.length || games.length >= limit) { + return; + } + + const href = link.attr('href'); + const title = $elem.text().replaceAll('DOWNLOAD', '').trim(); + const fullLink = href.startsWith('http') ? href : `${baseUrl}/${href.replace(/^\//, '')}`; + games.push({ title, link: fullLink, pubDate: null }); + } + }); + + return games; +} + +function extractLatestDate(pageHtml: string): Date | null { + const dateMatches = pageHtml.match(/\b\d{2}\.\d{2}\.\d{4}\b/g) || []; + const uniqueDates = [...new Set(dateMatches)]; + + let newestDate: Date | null = null; + + for (const dateStr of uniqueDates) { + const parsedDate = toNeutralDate(dateStr); + if (newestDate === null || parsedDate > newestDate) { + newestDate = parsedDate; + } + } + + return newestDate; +} + +function sanitizeHtml(pageHtml: string): string { + const $page = load(pageHtml); + + $page('script, style, link, nav').remove(); + + $page('*').each((_: number, elem: any) => { + if (elem.attribs) { + const attributes = Object.keys(elem.attribs); + for (const attr of attributes) { + if (attr.toLowerCase().startsWith('on')) { + $page(elem).removeAttr(attr); + } + } + } + }); + + let contentHtml = $page('body').html() || ''; + contentHtml = contentHtml + .replaceAll(/^\s*[\r\n]/gm, '') + .replaceAll(/body\s*\{\s*margin-top:\s*1em;\s*\}/gi, '') + .trim(); + + return contentHtml; +} + +function processGameItem(game: { title: string; link: string; pubDate: string | null }): Promise { + const cacheKey = `elamigos:${game.link}`; + + return cache.tryGet(cacheKey, async () => { + try { + const { data: pageHtml } = await got(game.link); + + const newestDate = extractLatestDate(pageHtml); + // If the Data Page has a newer Date than what we got from the Main Page, we take it instead. + let finalPublishDate: Date | null = null; + if (game.pubDate) { + finalPublishDate = toNeutralDate(game.pubDate, true); + } + + if (finalPublishDate === null || (newestDate !== null && newestDate > finalPublishDate)) { + finalPublishDate = newestDate; + } + const contentHtml = sanitizeHtml(pageHtml); + + return { + title: game.title, + link: game.link, + pubDate: finalPublishDate === null ? undefined : finalPublishDate.toUTCString(), + description: contentHtml, + } as DataItem; + } catch { + return { + title: game.title, + link: game.link, + description: `

    View game page: ${game.link}

    `, + } as DataItem; + } + }); +} diff --git a/lib/routes/elamigos/namespace.ts b/lib/routes/elamigos/namespace.ts new file mode 100644 index 00000000000000..bbed715636d216 --- /dev/null +++ b/lib/routes/elamigos/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'ElAmigos', + url: 'elamigos.site', + description: 'Game download site with daily releases', + lang: 'en-gb', +}; diff --git a/lib/routes/esquirehk/tag.tsx b/lib/routes/esquirehk/tag.tsx index 15db4deaf13e2a..21dcda81de0bb0 100644 --- a/lib/routes/esquirehk/tag.tsx +++ b/lib/routes/esquirehk/tag.tsx @@ -2,6 +2,7 @@ import * as cheerio from 'cheerio'; import { destr } from 'destr'; import { raw } from 'hono/html'; import { renderToString } from 'hono/jsx/dom/server'; +import type { JSX } from 'hono/jsx/jsx-runtime'; import type { Route } from '@/types'; import cache from '@/utils/cache'; diff --git a/lib/routes/expats/czech-news.ts b/lib/routes/expats/czech-news.ts index 2cbcb13cdb4aaf..766d38a12269e6 100644 --- a/lib/routes/expats/czech-news.ts +++ b/lib/routes/expats/czech-news.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.main h3 a') + let items: DataItem[] = $('div.main h3 a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/fangchan/list.tsx b/lib/routes/fangchan/list.tsx index 2d61e2def3bb7d..a36ddfae4cf9d4 100644 --- a/lib/routes/fangchan/list.tsx +++ b/lib/routes/fangchan/list.tsx @@ -31,9 +31,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = response.data.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.slice(0, limit).map((item): DataItem => { const title: string = item.title; const description: string = renderToString(item.zhaiyao ?
    {item.zhaiyao}
    : null); const pubDate: number | string = item.createtime; diff --git a/lib/routes/fantia/search.ts b/lib/routes/fantia/search.ts index 945e031b23bb4e..fe28e5381450df 100644 --- a/lib/routes/fantia/search.ts +++ b/lib/routes/fantia/search.ts @@ -1,5 +1,5 @@ import { config } from '@/config'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import { ViewType } from '@/types'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -163,7 +163,7 @@ async function handler(ctx) { }, }); - let items = {}; + let items: DataItem[]; switch (type) { case 'fanclubs': diff --git a/lib/routes/ff14/ff14-zh.ts b/lib/routes/ff14/ff14-zh.ts index 2cf2f51ffcfd02..3ad1007f0f4233 100644 --- a/lib/routes/ff14/ff14-zh.ts +++ b/lib/routes/ff14/ff14-zh.ts @@ -37,17 +37,18 @@ async function handler(ctx) { const referer = 'https://ff.sdo.com/web8/index.html'; const type = ctx.req.param('type') ?? 'all'; - const type_number = { + const typeNumber: Record = { news: '5310', - announce: '5312', + announce: '5312,8324,8325,8326,8327', events: '5311', advertise: '5313', - all: '5310,5312,5311,5313,5309', }; + typeNumber.all = `5309,${Object.values(typeNumber).join(',')}`; + const response = await got({ method: 'get', - url: `http://api.act.sdo.com/UnionNews/List?gameCode=ff&category=${type_number[type]}&pageIndex=0&pageSize=50`, + url: `http://api.act.sdo.com/UnionNews/List?gameCode=ff&category=${typeNumber[type]}&pageIndex=0&pageSize=50`, headers: { Referer: referer, }, diff --git a/lib/routes/flyert/creditcard.ts b/lib/routes/flyert/creditcard.ts index 4ff98ade608c55..8ac5e9a5d6216d 100644 --- a/lib/routes/flyert/creditcard.ts +++ b/lib/routes/flyert/creditcard.ts @@ -58,7 +58,7 @@ export const route: Route = { async function handler(ctx) { const bank = ctx.req.param('bank'); const target = `${host}/forum-${bank}-1.html`; - let bankname = ''; + let bankname: string; switch (bank) { case 'creditcard': diff --git a/lib/routes/flyert/utils.ts b/lib/routes/flyert/utils.ts index 8502f150e0628a..78874f26bc53c8 100644 --- a/lib/routes/flyert/utils.ts +++ b/lib/routes/flyert/utils.ts @@ -70,7 +70,7 @@ const ProcessFeed = (list, caches) => { const other = await caches.tryGet(itemUrl, () => loadContent(itemUrl)); // 合并解析后的结果集作为该篇文章最终的输出结果 - return Object.assign({}, single, other); + return { ...single, ...other }; }, { concurrency: 2 } ); // 设置并发请求数量为 2 diff --git a/lib/routes/follow/profile.ts b/lib/routes/follow/profile.ts index 79eaeb3937e5a4..6d67374a4205c9 100644 --- a/lib/routes/follow/profile.ts +++ b/lib/routes/follow/profile.ts @@ -74,7 +74,7 @@ async function handler(ctx: Context): Promise { }; } -const getUrlIcon = (url: string, fallback?: boolean | undefined) => { +const getUrlIcon = (url: string, fallback?: boolean) => { let src: string; let fallbackUrl = ''; diff --git a/lib/routes/freecomputerbooks/index.tsx b/lib/routes/freecomputerbooks/index.tsx index d2ddeaa7261315..a062e0894bb0d9 100644 --- a/lib/routes/freecomputerbooks/index.tsx +++ b/lib/routes/freecomputerbooks/index.tsx @@ -9,7 +9,8 @@ import got from '@/utils/got'; const baseURL = 'https://freecomputerbooks.com/'; async function cheerioLoad(url) { - return load((await got(url)).data); + const response = await got(url); + return load(response.data); } export const route: Route = { diff --git a/lib/routes/ftm/index.ts b/lib/routes/ftm/index.ts index 408131eb117bee..048dfe842f5d17 100644 --- a/lib/routes/ftm/index.ts +++ b/lib/routes/ftm/index.ts @@ -35,7 +35,8 @@ async function handler(ctx) { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - const content = load(await ofetch(item.link)); + const html = await ofetch(item.link); + const content = load(html); const ldjson = JSON.parse(content('[type="application/ld+json"]:not([data-schema])').text()); item.pubDate = parseDate(ldjson.datePublished); diff --git a/lib/routes/gamer/gnn-index.ts b/lib/routes/gamer/gnn-index.ts index fcfbe7f018b66f..b17b26abaaf900 100644 --- a/lib/routes/gamer/gnn-index.ts +++ b/lib/routes/gamer/gnn-index.ts @@ -55,7 +55,7 @@ export const route: Route = { async function handler(ctx) { const category = ctx.req.param('category'); - let url = ''; + let url: string; let categoryName = ''; const categoryTable = { 1: 'PC', @@ -126,7 +126,7 @@ async function handler(ctx) { async (item) => { item.description = await cache.tryGet(item.link, async () => { const response = await got.get(item.link); - let component = ''; + let component: string; const urlReg = /window\.lazySizesConfig/g; let pubInfo; diff --git a/lib/routes/gamersky/user.ts b/lib/routes/gamersky/user.ts new file mode 100644 index 00000000000000..9b851d0be0d703 --- /dev/null +++ b/lib/routes/gamersky/user.ts @@ -0,0 +1,41 @@ +import type { Context } from 'hono'; + +import type { Route } from '@/types'; + +import { getUserArticle, getUserArticleList, parseUserArticleList } from './utils'; + +export const route: Route = { + path: '/user/:userId/:detail?', + categories: ['game'], + example: '/gamersky/user/4009731/detail', + parameters: { + userId: '用户 ID。在用户个人主页，打开“开发者工具”中的“元素”标签页，搜索 data-userid 即可找到', + detail: '是否获取文章详情。只要该参数不为空，就会获取全文内容', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: '用户动态', + maintainers: ['hualiong'], + handler, +}; + +async function handler(ctx: Context) { + const { userId, detail } = ctx.req.param(); + + const body = await getUserArticleList(userId); + const articles = parseUserArticleList(body); + if (detail) { + articles.list = await Promise.all(articles.list.map((item) => getUserArticle(item))); + } + return { + title: `${articles.uname}的动态 - 游民星空`, + link: articles.link, + item: articles.list, + }; +} diff --git a/lib/routes/gamersky/utils.ts b/lib/routes/gamersky/utils.ts index c11f631ad3af7f..403b51f53b2dfc 100644 --- a/lib/routes/gamersky/utils.ts +++ b/lib/routes/gamersky/utils.ts @@ -12,12 +12,19 @@ interface idNameMap { nodeId: string; suffix?: string; } + interface ArticleList { status: string; totalPages: number; body: string; } +interface UserArticleList { + status: string; + body: string; + total: number; +} + export const getArticleList = async (nodeId) => { const response = await ofetch( `https://db2.gamersky.com/LabelJsonpAjax.aspx?${new URLSearchParams({ @@ -88,6 +95,46 @@ export const getArticle = (item) => return item satisfies DataItem; }) as Promise; +export const getUserArticleList = async (userId: string) => { + const response = await ofetch( + `https://i.gamersky.com/u/api/v2/GetUserContent?${new URLSearchParams({ + jsondata: JSON.stringify({ pageIndex: 1, pageSize: 20, userId }), + })}`, + { + parseResponse: (txt) => JSON.parse(txt.slice(1, -2)), + } + ); + return response.body; +}; + +export const parseUserArticleList = (body: string) => { + const $ = load(body); + const list = $('.cmt-list'); + const info = list.find('.uname').first(); + return { + uname: info.text(), + link: info.attr('href'), + list: list.toArray().map((item) => { + const e = $(item); + const title = e.find('.qzcmt-content-tit a'); + return { + title: title.text(), + link: title.attr('href'), + pubDate: parseDate(e.attr('data-time')!), + description: e.find('.qzcmt-content-txt span').text(), + }; + }) as DataItem[], + }; +}; + +export const getUserArticle = (item: DataItem) => + cache.tryGet(item.link!, async () => { + const response = await ofetch(item.link!); + const $ = load(response); + item.description = $('.qzcmt-content-txt').html() || item.description; + return item; + }); + export function mdTableBuilder(data: idNameMap[]) { const table = '|' + data.map((item) => `${item.type}|`).join('') + '\n|' + Array.from({ length: data.length }).fill('---|').join('') + '\n|' + data.map((item) => `${item.name}|`).join('') + '\n'; return table; diff --git a/lib/routes/gaoyu/blog.ts b/lib/routes/gaoyu/blog.ts index 56b7cd30cc4b38..3e25b019edead5 100644 --- a/lib/routes/gaoyu/blog.ts +++ b/lib/routes/gaoyu/blog.ts @@ -29,9 +29,7 @@ export const handler = async (ctx: Context): Promise => { }, ]; - let items: DataItem[] = []; - - items = $('a.flex-col') + let items: DataItem[] = $('a.flex-col') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/gcores/radio.tsx b/lib/routes/gcores/radio.tsx index 34c451c35415f4..822acac0a4d21e 100644 --- a/lib/routes/gcores/radio.tsx +++ b/lib/routes/gcores/radio.tsx @@ -36,7 +36,8 @@ async function handler(ctx) { const limit = Number.parseInt(ctx.req.query('limit')) || 12; const link = getLink(category); - const $ = load(await get(link)); + const html = await get(link); + const $ = load(html); const title = $('head>title').text(); const description = $('head>meta[name="description"]').attr('content'); const image = $('head>link[rel="apple-touch-icon"]').attr('href'); diff --git a/lib/routes/gcores/user-radios.ts b/lib/routes/gcores/user-radios.ts new file mode 100644 index 00000000000000..a7b1238dd16758 --- /dev/null +++ b/lib/routes/gcores/user-radios.ts @@ -0,0 +1,185 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; +import type { Context } from 'hono'; + +import type { Data, DataItem, Route } from '@/types'; +import { ViewType } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { parseContent } from './parser'; +import { renderDescription } from './templates/description'; +import { baseUrl, imageBaseUrl } from './util'; + +const buildAuthors = (relationships: any, included: any[]): DataItem['author'] => { + const authorObj = relationships?.user?.data; + const authorIncluded = authorObj ? included.find((i) => i.type === authorObj.type && i.id === authorObj.id) : undefined; + + return authorIncluded + ? [ + { + name: authorIncluded.attributes?.nickname, + url: authorIncluded.id ? new URL(`users/${authorIncluded.id}`, baseUrl).href : undefined, + avatar: authorIncluded.thumb ? new URL(authorIncluded.thumb, imageBaseUrl).href : undefined, + }, + ] + : undefined; +}; + +const buildEnclosure = (attributes: any, relationships: any, included: any[], image: string | undefined, title: string | undefined) => { + const mediaAttrs = included.find((i) => i.id === relationships.media?.data?.id)?.attributes; + + let enclosureUrl: string | undefined; + let enclosureType: string | undefined; + + if (attributes['speech-path']) { + enclosureUrl = new URL(`uploads/audio/${attributes['speech-path']}`, 'https://alioss.gcores.com').href; + enclosureType = `audio/${enclosureUrl?.split(/\./).pop()}`; + } else if (mediaAttrs && mediaAttrs.audio) { + enclosureUrl = mediaAttrs.audio; + enclosureType = `audio/${enclosureUrl?.split(/\./).pop()}`; + } + + if (!enclosureUrl) { + return {}; + } + + const enclosureLength = attributes.duration ? Number(attributes.duration) : 0; + + return { + enclosure_url: enclosureUrl, + enclosure_type: enclosureType, + enclosure_title: title, + enclosure_length: enclosureLength, + itunes_duration: enclosureLength, + itunes_item_image: image, + }; +}; + +const buildDescription = (attributes: any, title: string | undefined, enclosureUrl?: string, enclosureType?: string) => + renderDescription({ + images: attributes.cover + ? [ + { + src: new URL(attributes.cover, imageBaseUrl).href, + alt: title, + }, + ] + : undefined, + audios: + enclosureType?.startsWith('audio') && enclosureUrl + ? [ + { + src: enclosureUrl, + type: enclosureType, + }, + ] + : undefined, + intro: attributes.desc || attributes.excerpt, + description: attributes.content ? parseContent(JSON.parse(attributes.content)) : undefined, + }); + +export const handler = async (ctx: Context): Promise => { + const { id } = ctx.req.param(); + const limit = Number.parseInt(ctx.req.query('limit') ?? '30', 10); + + const targetUrl = new URL(`users/${id}/content?tab=radios`, baseUrl).href; + const apiUrl = new URL(`gapi/v1/users/${id}/radios`, baseUrl).href; + + const query = { + 'page[limit]': limit, + sort: '-published-at', + include: 'user,media,albums', + }; + + const response = await ofetch(apiUrl, { query }); + const data = response.data; + + const targetResponse = await ofetch(targetUrl); + const $: CheerioAPI = load(targetResponse); + const language = $('html').attr('lang') ?? 'zh-CN'; + + const included = response.included || []; + + // 处理每个播客项目 + const items: DataItem[] = data.map((item): DataItem => { + const attributes = item.attributes; + const relationships = item.relationships; + + const title = attributes.title; + const pubDate: number | string = attributes['published-at']; + const linkUrl = new URL(`radios/${item.id}`, baseUrl).href; + const image: string | undefined = (attributes.cover ?? attributes.thumb) ? new URL(attributes.cover ?? attributes.thumb, imageBaseUrl).href : undefined; + const authors = buildAuthors(relationships, included); + const enclosure = buildEnclosure(attributes, relationships, included, image, title); + const description = buildDescription(attributes, title, enclosure.enclosure_url, enclosure.enclosure_type); + + const albumNames = (relationships?.albums?.data || []).map((album) => included.find((i) => i.type === album.type && i.id === album.id)?.attributes?.title).filter(Boolean); + + return { + title: title ?? $(description).text(), + pubDate: pubDate ? parseDate(pubDate) : undefined, + link: linkUrl, + author: authors, + category: albumNames.length > 0 ? albumNames : undefined, + guid: `gcores-${item.id}`, + id: `gcores-${item.id}`, + image, + banner: image, + updated: pubDate ? parseDate(pubDate) : undefined, + language, + description, + content: { + html: description, + text: description, + }, + ...enclosure, + }; + }); + + return { + title: $('title').text(), + description: $('meta[name="description"]').attr('content'), + link: targetUrl, + item: items, + allowEmpty: true, + author: $('title').text().split(/\|/).pop()?.trim(), + language, + id: $('meta[property="og:url"]').attr('content'), + }; +}; + +export const route: Route = { + path: '/users/:id/radios', + name: '用户播客', + url: 'www.gcores.com', + maintainers: ['DzmingLi'], + handler, + example: '/gcores/users/31418/radios', + parameters: { + id: { + description: '用户 ID，可在用户主页 URL 中找到', + }, + }, + description: `::: tip +若订阅用户 [这样重这样轻](https://www.gcores.com/users/31418) 发布的播客，网址为 \`https://www.gcores.com/users/31418\`，请截取 \`https://www.gcores.com/users/\` 之后的部分 \`31418\` 作为 \`id\` 参数填入，此时目标路由为 [\`/gcores/users/31418/radios\`](https://rsshub.app/gcores/users/31418/radios)。 +::: +`, + categories: ['game'], + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: true, + supportScihub: false, + }, + radar: [ + { + source: ['www.gcores.com/users/:id/content', 'www.gcores.com/users/:id'], + target: '/users/:id/radios', + }, + ], + view: ViewType.Audios, +}; diff --git a/lib/routes/gcores/user-talks.ts b/lib/routes/gcores/user-talks.ts new file mode 100644 index 00000000000000..4dde7cbb405605 --- /dev/null +++ b/lib/routes/gcores/user-talks.ts @@ -0,0 +1,142 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; +import type { Context } from 'hono'; + +import type { Data, DataItem, Route } from '@/types'; +import { ViewType } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { parseContent } from './parser'; +import { renderDescription } from './templates/description'; +import { baseUrl, imageBaseUrl } from './util'; + +export const handler = async (ctx: Context): Promise => { + const { id } = ctx.req.param(); + const limit = Number.parseInt(ctx.req.query('limit') ?? '30', 10); + + const targetUrl = new URL(`users/${id}/talks`, baseUrl).href; + const apiUrl = new URL(`gapi/v1/users/${id}/talks`, baseUrl).href; + + // 获取更多数据以确保过滤后仍有足够的项目 + const fetchLimit = Math.min(limit * 2, 100); + + const query = { + 'page[limit]': fetchLimit, + sort: '-created-at', + include: 'user', + }; + + const response = await ofetch(apiUrl, { query }); + const targetResponse = await ofetch(targetUrl); + const $: CheerioAPI = load(targetResponse); + const language = $('html').attr('lang') ?? 'zh-CN'; + + const included = response.included || []; + + // 过滤掉播客和视频类型的动态 + let data = response.data.filter((item) => item.type !== 'radios' && item.type !== 'videos'); + + // 限制数量 + data = data.slice(0, limit); + + const items: DataItem[] = data.map((item): DataItem => { + const attributes = item.attributes; + const relationships = item.relationships; + + const title: string = attributes.title; + const pubDate: number | string = attributes['created-at'] || attributes['published-at']; + + const authorObj = relationships?.user?.data; + const authorIncluded = authorObj ? included.find((i) => i.type === authorObj.type && i.id === authorObj.id) : undefined; + const authors: DataItem['author'] = authorIncluded + ? [ + { + name: authorIncluded.attributes?.nickname, + url: authorIncluded.id ? new URL(`users/${authorIncluded.id}`, baseUrl).href : undefined, + avatar: authorIncluded.thumb ? new URL(authorIncluded.thumb, imageBaseUrl).href : undefined, + }, + ] + : undefined; + + const guid = `gcores-${item.type}-${item.id}`; + const image: string | undefined = (attributes.cover ?? attributes.thumb) ? new URL(attributes.cover ?? attributes.thumb, imageBaseUrl).href : undefined; + + const description = renderDescription({ + images: attributes.cover + ? [ + { + src: new URL(attributes.cover, imageBaseUrl).href, + alt: title, + }, + ] + : undefined, + intro: attributes.desc || attributes.excerpt, + description: attributes.content ? parseContent(JSON.parse(attributes.content)) : undefined, + }); + + return { + title: title ?? $(description).text(), + pubDate: pubDate ? parseDate(pubDate) : undefined, + link: new URL(`${item.type}/${item.id}`, baseUrl).href, + author: authors, + guid, + id: guid, + image, + banner: image, + updated: pubDate ? parseDate(pubDate) : undefined, + language, + description, + content: { + html: description, + text: description, + }, + }; + }); + + return { + title: $('title').text(), + description: $('meta[name="description"]').attr('content'), + link: targetUrl, + item: items, + allowEmpty: true, + author: $('title').text().split(/\|/).pop()?.trim(), + language, + id: $('meta[property="og:url"]').attr('content'), + }; +}; + +export const route: Route = { + path: '/users/:id/talks', + name: '用户动态', + url: 'www.gcores.com', + maintainers: ['DzmingLi'], + handler, + example: '/gcores/users/31418/talks', + parameters: { + id: { + description: '用户 ID，可在用户主页 URL 中找到', + }, + }, + description: `::: tip +若订阅用户 [这样重这样轻](https://www.gcores.com/users/31418/talks) 的动态，网址为 \`https://www.gcores.com/users/31418/talks\`，请截取 \`https://www.gcores.com/users/\` 到 \`/talks\` 之间的部分 \`31418\` 作为 \`id\` 参数填入，此时目标路由为 [\`/gcores/users/31418/talks\`](https://rsshub.app/gcores/users/31418/talks)。 +::: +`, + categories: ['game'], + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['www.gcores.com/users/:id/talks'], + target: '/users/:id/talks', + }, + ], + view: ViewType.SocialMedia, +}; diff --git a/lib/routes/gcores/util.ts b/lib/routes/gcores/util.ts index a18b16bf3638b0..ef1b2c70457943 100644 --- a/lib/routes/gcores/util.ts +++ b/lib/routes/gcores/util.ts @@ -31,9 +31,7 @@ const processItems = async (limit: number, query: any, apiUrl: string, targetUrl const included = response.included; const data = [...response.data, ...included].filter((item) => types.has(item.type)); - let items: DataItem[] = []; - - items = data?.slice(0, limit).map((item): DataItem => { + const items: DataItem[] = data?.slice(0, limit).map((item): DataItem => { const attributes = item.attributes; const relationships = item.relationships; diff --git a/lib/routes/getitfree/util.ts b/lib/routes/getitfree/util.ts index 454a946aaa85b2..50385c17bd1f14 100644 --- a/lib/routes/getitfree/util.ts +++ b/lib/routes/getitfree/util.ts @@ -48,7 +48,7 @@ const bakeFilterSearchParams = (filterPairs, pairKey, isApi = false) => { const key = keys[0]; const pairs = filterPairs[key]; - const originalFilters = Object.assign({}, filterPairs); + const originalFilters = { ...filterPairs }; delete originalFilters[key]; filterSearchParams.append(getFilterKeyForSearchParams(key, isApi), pairs.map((pair) => (Object.hasOwn(pair, pairKey) ? pair[pairKey] : pair)).join(',')); @@ -121,7 +121,7 @@ const bakeFiltersWithPair = async (filters) => { const key = keys[0]; const keywords = filters[key]; - const originalFilters = Object.assign({}, filters); + const originalFilters = { ...filters }; delete originalFilters[key]; return bakeFilters(originalFilters, { diff --git a/lib/routes/gisreportsonline/index.ts b/lib/routes/gisreportsonline/index.ts index de27af795c6f91..e5d5fc2d658b4d 100644 --- a/lib/routes/gisreportsonline/index.ts +++ b/lib/routes/gisreportsonline/index.ts @@ -34,7 +34,8 @@ async function handler(ctx) { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - const content = load(await ofetch(item.link)); + const html = await ofetch(item.link); + const content = load(html); const ldjson = JSON.parse(content('script.rank-math-schema-pro').text())['@graph'].find((e) => e['@type'] === 'NewsArticle'); item.pubDate = parseDate(ldjson.datePublished); diff --git a/lib/routes/github/eventapi.ts b/lib/routes/github/eventapi.ts index 20a0dec6a5e7a9..4caaafdbc9cfe0 100644 --- a/lib/routes/github/eventapi.ts +++ b/lib/routes/github/eventapi.ts @@ -22,9 +22,9 @@ export const eventTypeMapping: Record = { function formatEventItem(event: any) { const { id, type, actor, repo, payload, created_at } = event; - let title = ''; - let description = ''; - let link = ''; + let title: string; + let description: string; + let link: string; switch (type) { case 'PushEvent': { diff --git a/lib/routes/google/jules.ts b/lib/routes/google/jules.ts index b732404cde3beb..835135b52e4443 100644 --- a/lib/routes/google/jules.ts +++ b/lib/routes/google/jules.ts @@ -46,6 +46,7 @@ async function handler() { // Full HTML for the item content article.find('h2').first().remove(); // remove title article.find('b').first().remove(); // remove date + article.find('.sr-only').remove(); // remove sr-only elements return { title, diff --git a/lib/routes/gov/cmse/index.ts b/lib/routes/gov/cmse/index.ts index 56c5dd59b66b36..f5f88c0506111a 100644 --- a/lib/routes/gov/cmse/index.ts +++ b/lib/routes/gov/cmse/index.ts @@ -41,7 +41,7 @@ async function handler(ctx) { return { title: item.text(), pubDate: parseDate(pubDate), - link: /\.html$/.test(link) ? link : `${link}#${pubDate}`, + link: link.endsWith('.html') ? link : `${link}#${pubDate}`, }; }); diff --git a/lib/routes/gov/cn/news/index.ts b/lib/routes/gov/cn/news/index.ts index 510ec2fd94a7d8..326d19245287cc 100644 --- a/lib/routes/gov/cn/news/index.ts +++ b/lib/routes/gov/cn/news/index.ts @@ -33,7 +33,7 @@ async function handler(ctx) { const originDomain = 'https://www.gov.cn'; let url = ''; let title = ''; - let list = ''; + let list: string; switch (uid) { case 'bm': url = `${originDomain}/lianbo/bumen/index.htm`; @@ -98,25 +98,23 @@ async function handler(ctx) { } else { description = item.find('a').text(); // 忽略获取吹风会的全文 } - } else { - if (contentUrl.includes('content')) { - fullTextGet = await got.get(contentUrl); - fullTextData = load(fullTextGet.data); - const $1 = fullTextData.html(); - pubDate = timezone(parseDate(fullTextData('meta[name="firstpublishedtime"]').attr('content'), 'YYYY-MM-DD HH:mm:ss'), 8); - author = fullTextData('meta[name="author"]').attr('content'); - category = fullTextData('meta[name="keywords"]').attr('content').split(/[,;]/); - if (/zhengceku/g.test(contentUrl)) { - // 政策文件库 - description = fullTextData('.pages_content').html(); - } else { - fullTextData('.shuzi').remove(); // 移除videobg的图片 - fullTextData('#myFlash').remove(); // 移除flash - description = /UCAP-CONTENT/g.test($1) ? fullTextData('#UCAP-CONTENT').html() : fullTextData('body').html(); - } + } else if (contentUrl.includes('content')) { + fullTextGet = await got.get(contentUrl); + fullTextData = load(fullTextGet.data); + const $1 = fullTextData.html(); + pubDate = timezone(parseDate(fullTextData('meta[name="firstpublishedtime"]').attr('content'), 'YYYY-MM-DD HH:mm:ss'), 8); + author = fullTextData('meta[name="author"]').attr('content'); + category = fullTextData('meta[name="keywords"]').attr('content').split(/[,;]/); + if (/zhengceku/g.test(contentUrl)) { + // 政策文件库 + description = fullTextData('.pages_content').html(); } else { - description = item.find('a').text(); // 忽略获取吹风会的全文 + fullTextData('.shuzi').remove(); // 移除videobg的图片 + fullTextData('#myFlash').remove(); // 移除flash + description = /UCAP-CONTENT/g.test($1) ? fullTextData('#UCAP-CONTENT').html() : fullTextData('body').html(); } + } else { + description = item.find('a').text(); // 忽略获取吹风会的全文 } return { title: item.find('a').text(), diff --git a/lib/routes/gov/csrc/news.tsx b/lib/routes/gov/csrc/news.tsx index 4e70801550e103..4bded27f63ad8c 100644 --- a/lib/routes/gov/csrc/news.tsx +++ b/lib/routes/gov/csrc/news.tsx @@ -1,7 +1,7 @@ import { load } from 'cheerio'; import { renderToString } from 'hono/jsx/dom/server'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -23,8 +23,8 @@ async function handler(ctx) { const channelId = $('meta[name="channelid"]').attr('content'); - let data, - out = []; + let data; + let out: DataItem[]; if (channelId) { data = await got(`${baseUrl}/searchList/${channelId}`, { searchParams: { diff --git a/lib/routes/gov/customs/list.ts b/lib/routes/gov/customs/list.ts index d28d20200c88d3..cf396099259ecf 100644 --- a/lib/routes/gov/customs/list.ts +++ b/lib/routes/gov/customs/list.ts @@ -39,7 +39,7 @@ export const route: Route = { async function handler(ctx) { const { gchannel = 'paimai' } = ctx.req.param(); - let channelName = ''; + let channelName: string; let link = ''; switch (gchannel) { diff --git a/lib/routes/gov/general/general.ts b/lib/routes/gov/general/general.ts index fe65e569b8eb59..7766acb8ab563b 100644 --- a/lib/routes/gov/general/general.ts +++ b/lib/routes/gov/general/general.ts @@ -105,8 +105,8 @@ const gdgov = async (info, ctx) => { const currentUrl = `${rootUrl}/${pathname}`; let $ = ''; - let name = ''; - let list = ''; + let name: string; + let list: string; // 判断是否处于特殊目录 if (pathname.startsWith('gkmlpt')) { title_element = undefined; @@ -149,7 +149,7 @@ const gdgov = async (info, ctx) => { } const lists = list.map((i, item) => { - let link = ''; + let link: string; if (pathname.startsWith('gkmlpt')) { link = i.url; @@ -198,16 +198,13 @@ const gdgov = async (info, ctx) => { const content = load(res); // 获取来源 - let author = ''; - author = author_element === undefined ? content('meta[name="ContentSource"]').attr('content') : content(author_element).text().trim().match(author_match)[1].trim().replaceAll(/(-*$)/g, ''); + const author = author_element === undefined ? content('meta[name="ContentSource"]').attr('content') : content(author_element).text().trim().match(author_match)[1].trim().replaceAll(/(-*$)/g, ''); // 获取发布时间 - let pubDate = ''; - pubDate = pubDate_element === undefined ? content('meta[name="PubDate"]').attr('content') : content(pubDate_element).text().trim().match(pubDate_match)[1].trim().replaceAll(/(-*$)/g, ''); + const pubDate = pubDate_element === undefined ? content('meta[name="PubDate"]').attr('content') : content(pubDate_element).text().trim().match(pubDate_match)[1].trim().replaceAll(/(-*$)/g, ''); // 获取标题 - let title = ''; - title = title_element === undefined ? content('meta[name="ArticleTitle"]').attr('content') : content(title_element).text().trim().match(title_match)[1]; + const title = title_element === undefined ? content('meta[name="ArticleTitle"]').attr('content') : content(title_element).text().trim().match(title_match)[1]; // 获取正文 const description_content = description_element.split(',').filter((item) => item !== ''); for (let index = 0; index < description_content.length; index++) { diff --git a/lib/routes/gov/hebei/czt.ts b/lib/routes/gov/hebei/czt.ts index e5a56a30a2c3a6..b41485c42457fd 100644 --- a/lib/routes/gov/hebei/czt.ts +++ b/lib/routes/gov/hebei/czt.ts @@ -46,7 +46,7 @@ async function handler(ctx) { return { title: item.text(), - link: `${rootUrl}${/^\.\.\/\.\./.test(item.attr('href')) ? item.attr('href').replace(/^\.\.\/\.\./, '') : `/xwdt/${category}${item.attr('href').replace(/^\./, '')}`}`, + link: `${rootUrl}${item.attr('href').startsWith('../..') ? item.attr('href').replace(/^\.\.\/\.\./, '') : `/xwdt/${category}${item.attr('href').replace(/^\./, '')}`}`, }; }); diff --git a/lib/routes/gov/maoming/maoming.ts b/lib/routes/gov/maoming/maoming.ts index bde7c1822f40ce..84a0a4c1060f44 100644 --- a/lib/routes/gov/maoming/maoming.ts +++ b/lib/routes/gov/maoming/maoming.ts @@ -16,14 +16,14 @@ async function handler(ctx) { .filter((item) => item !== ''); let pathstartat = 0; let defaultPath = ''; - let list_element = ''; + let list_element: string; let list_include = 'site'; - let title_element = ''; + let title_element: string; let title_match = '(.*)'; - let description_element = ''; - let authorisme = ''; - let pubDate_element = ''; - let pubDate_match = ''; + let description_element: string; + let authorisme: string; + let pubDate_element: string; + let pubDate_match: string; // let pubDate_format = undefined; switch (path[1]) { case 'www': diff --git a/lib/routes/gov/maonan/maonan.ts b/lib/routes/gov/maonan/maonan.ts index bbdb67c1916606..6363972180b902 100644 --- a/lib/routes/gov/maonan/maonan.ts +++ b/lib/routes/gov/maonan/maonan.ts @@ -30,8 +30,8 @@ export const route: Route = { }; async function handler(ctx) { - let id = ''; - let name = ''; + let id: string; + let name: string; switch (ctx.req.param('category')) { case 'zwgk': diff --git a/lib/routes/gov/mee/nnsa.ts b/lib/routes/gov/mee/nnsa.ts index a03e555967e475..0f1db35d87035d 100644 --- a/lib/routes/gov/mee/nnsa.ts +++ b/lib/routes/gov/mee/nnsa.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('a.cjcx_biaob, ul#div li a') + let items: DataItem[] = $('a.cjcx_biaob, ul#div li a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/gov/moa/gjs.ts b/lib/routes/gov/moa/gjs.ts index 45b7b1ad0be41e..56c65dabcc539f 100644 --- a/lib/routes/gov/moa/gjs.ts +++ b/lib/routes/gov/moa/gjs.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('ul#div li') + let items: DataItem[] = $('ul#div li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/gov/moa/moa.ts b/lib/routes/gov/moa/moa.ts index 3460037170a376..cf86e6fcfafc55 100644 --- a/lib/routes/gov/moa/moa.ts +++ b/lib/routes/gov/moa/moa.ts @@ -247,7 +247,7 @@ function dealLink(element, url) { // host 不同的是外部文章，outside // url 里带 govpublic 的都是公示文章，govpublic // 其他的都算普通文章，normal - let pageType = null; + let pageType: string; if (host === hostUrlObj.host) { pageType = href.includes('gk') || href.includes('govpublic') ? 'govpublic' : 'normal'; } else { diff --git a/lib/routes/gov/mot/index.ts b/lib/routes/gov/mot/index.ts index da2416e1648e84..7c9b58aa0c80db 100644 --- a/lib/routes/gov/mot/index.ts +++ b/lib/routes/gov/mot/index.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.tab-pane a') + let items: DataItem[] = $('div.tab-pane a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/gov/nmpa/generic.ts b/lib/routes/gov/nmpa/generic.ts index 99d06287718711..c8a6b8461593e8 100644 --- a/lib/routes/gov/nmpa/generic.ts +++ b/lib/routes/gov/nmpa/generic.ts @@ -47,7 +47,7 @@ async function handler(ctx) { const items = await Promise.all( data.items.map((item) => { - if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) { + if (item.link.startsWith('https://www.nmpa.gov.cn/')) { return cache.tryGet(item.link, async () => { const { data: html } = await got(item.link); const $ = load(html); @@ -55,7 +55,7 @@ async function handler(ctx) { item.pubDate = timezone(parseDate($('meta[name="PubDate"]').attr('content')), +8); return item; }); - } else if (/^https:\/\/mp\.weixin\.qq\.com\//.test(item.link)) { + } else if (item.link.startsWith('https://mp.weixin.qq.com/')) { return finishArticleItem(item); } else { return item; diff --git a/lib/routes/gov/nsfc/index.ts b/lib/routes/gov/nsfc/index.ts index 1c8415817b318d..92a4320430e3f6 100644 --- a/lib/routes/gov/nsfc/index.ts +++ b/lib/routes/gov/nsfc/index.ts @@ -30,7 +30,7 @@ async function handler(ctx) { } const rootUrl = 'https://www.nsfc.gov.cn'; - const currentUrl = new URL((/\/more$/.test(thePath) ? `${thePath}.htm` : thePath) || 'publish/portal0/tab442/', rootUrl).href; + const currentUrl = new URL((thePath.endsWith('/more') ? `${thePath}.htm` : thePath) || 'publish/portal0/tab442/', rootUrl).href; const { data: response } = await got(currentUrl); diff --git a/lib/routes/gov/suzhou/news.ts b/lib/routes/gov/suzhou/news.ts index e370c9f138b88c..ccab64c84ead04 100644 --- a/lib/routes/gov/suzhou/news.ts +++ b/lib/routes/gov/suzhou/news.ts @@ -58,10 +58,10 @@ export const route: Route = { async function handler(ctx) { const rootUrl = 'https://www.suzhou.gov.cn'; const uid = ctx.req.param('uid'); - let url = ''; - let title = ''; + let url: string; + let title: string; let apiUrl = ''; - let items = []; + let items: DataItem[]; switch (uid) { case 'szyw': case 'news': diff --git a/lib/routes/gov/zhengce/govall.ts b/lib/routes/gov/zhengce/govall.ts index f39868276b17fa..3afc233d72a19f 100644 --- a/lib/routes/gov/zhengce/govall.ts +++ b/lib/routes/gov/zhengce/govall.ts @@ -71,7 +71,7 @@ async function handler(ctx) { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - let description = ''; + let description: string; try { const contentData = await got(item.link); const $ = load(contentData.data); diff --git a/lib/routes/grainoil/category.ts b/lib/routes/grainoil/category.ts index 54e6be4e91a0d6..5e71abd97455b7 100644 --- a/lib/routes/grainoil/category.ts +++ b/lib/routes/grainoil/category.ts @@ -5,6 +5,7 @@ import type { Context } from 'hono'; import type { Data, DataItem, Route } from '@/types'; import { ViewType } from '@/types'; +import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; @@ -20,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.m_listpagebox ol li a') + let items: DataItem[] = $('div.m_listpagebox ol li a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/hafu/utils.tsx b/lib/routes/hafu/utils.tsx index bfe3e8376ae9df..224119be40590d 100644 --- a/lib/routes/hafu/utils.tsx +++ b/lib/routes/hafu/utils.tsx @@ -35,7 +35,7 @@ export default parseList; async function tryGetFullText(href, link, type) { let articleData = ''; - let description = ''; + let description: string; // for some unexpected href link try { const articleRes = await got(link); @@ -97,7 +97,7 @@ async function ggtzParse(ctx, $) { const result = await cache.tryGet(link, async () => { const { articleData, description } = await tryGetFullText(href, link, 'ggtz'); let author = ''; - let pubDate = ''; + let pubDate: string; if (typeof articleData === 'function') { const header = articleData('h1').next().text(); const index = header.indexOf('日期'); @@ -176,7 +176,7 @@ async function zsjycParse(ctx, $) { const result = await cache.tryGet(link, async () => { const { articleData, description } = await tryGetFullText(href, link, 'zsjyc'); - let pubDate = ''; + let pubDate: string; if (typeof articleData === 'function') { const date = articleData('span[class=timestyle127702]').text(); pubDate = parseDate(date, 'YYYY-MM-DD HH:mm'); diff --git a/lib/routes/hameln/chapter.ts b/lib/routes/hameln/chapter.ts index 0fe7ba0b0c1afc..d4ba535834d365 100644 --- a/lib/routes/hameln/chapter.ts +++ b/lib/routes/hameln/chapter.ts @@ -34,7 +34,8 @@ async function handler(ctx) { const id = ctx.req.param('id'); const limit = Number.parseInt(ctx.req.query('limit')) || 5; const link = `https://syosetu.org/novel/${id}`; - const $ = load(await get(link)); + const html = await get(link); + const $ = load(html); const title = $('span[itemprop="name"]').text(); const description = $('div.ss:nth-child(2)').text(); @@ -57,7 +58,8 @@ async function handler(ctx) { chapter_list.map((chapter) => { chapter.link = `${link}/${chapter.link}`; return cache.tryGet(chapter.link, async () => { - const content = load(await get(chapter.link)); + const html = await get(chapter.link); + const content = load(html); chapter.description = content('#honbun').html(); return chapter; }); diff --git a/lib/routes/hit/hitgs.ts b/lib/routes/hit/hitgs.ts index a38c24934d9a84..6828262d923ab1 100644 --- a/lib/routes/hit/hitgs.ts +++ b/lib/routes/hit/hitgs.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('li.news, div.tbt17') + let items: DataItem[] = $('li.news, div.tbt17') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/hkepc/index.ts b/lib/routes/hkepc/index.ts index 677e3c0d9b2873..1883c02bf6d2b9 100644 --- a/lib/routes/hkepc/index.ts +++ b/lib/routes/hkepc/index.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ import { load } from 'cheerio'; import type { Route } from '@/types'; diff --git a/lib/routes/hlju/news.ts b/lib/routes/hlju/news.ts index 82e17b54f8a817..16f667f20a0870 100644 --- a/lib/routes/hlju/news.ts +++ b/lib/routes/hlju/news.ts @@ -100,7 +100,7 @@ async function handler(ctx) { // 提取文章内容 - 只使用主要内容选择器 const content = $detail('.v_news_content'); - let description = ''; + let description: string; if (content.length > 0) { // 清理内容 diff --git a/lib/routes/hongkong/chp.ts b/lib/routes/hongkong/chp.ts index 98d4c6ab09d997..04de3eac81e6de 100644 --- a/lib/routes/hongkong/chp.ts +++ b/lib/routes/hongkong/chp.ts @@ -71,7 +71,7 @@ async function handler(ctx) { }); const list = JSON.parse(response.data.match(/"data":(\[{.*}])}/)[1]).map((item) => { - let link = ''; + let link: string; if (item.UrlPath_en) { link = item[`UrlPath_${language}`].includes('http') ? item[`UrlPath_${language}`] : `${rootUrl}${item[`UrlPath_${language}`]}`; diff --git a/lib/routes/huggingface/models.ts b/lib/routes/huggingface/models.ts index 94f8a544855172..b5e0f737801525 100644 --- a/lib/routes/huggingface/models.ts +++ b/lib/routes/huggingface/models.ts @@ -92,15 +92,13 @@ async function handler(ctx) { if (/^https?:\/\//i.test(src)) { // 已经是完整 URL $e.attr('src', src); - } else { + } else if (src.startsWith('/')) { // 处理以 / 开头的绝对路径 - if (src.startsWith('/')) { - $e.attr('src', `${item.link}/resolve/main/` + src); - } else { - // 处理相对路径（如 ./images/pic.png 或 images/pic.png） - const baseUrl = item.link + '/resolve/main/'; - $e.attr('src', baseUrl + src.replace(/^\.\//, '')); - } + $e.attr('src', `${item.link}/resolve/main/` + src); + } else { + // 处理相对路径（如 ./images/pic.png 或 images/pic.png） + const baseUrl = item.link + '/resolve/main/'; + $e.attr('src', baseUrl + src.replace(/^\.\//, '')); } }); item.description = $Out.html(); diff --git a/lib/routes/huitun/xiaohongshu.ts b/lib/routes/huitun/xiaohongshu.ts index 1c7b586b29b07c..f16ed870c641f1 100644 --- a/lib/routes/huitun/xiaohongshu.ts +++ b/lib/routes/huitun/xiaohongshu.ts @@ -55,7 +55,7 @@ async function handler(ctx) { const notes = note_data.extData.list; const items = await Promise.all( notes.map(async (item) => { - let desc = ''; + let desc: string; switch (item.type) { case 'normal': desc = `

    `; diff --git a/lib/routes/hupu/utils.ts b/lib/routes/hupu/utils.ts index d0363658d2f1e5..dadc17d8370392 100644 --- a/lib/routes/hupu/utils.ts +++ b/lib/routes/hupu/utils.ts @@ -15,7 +15,7 @@ export function extractNextData(html: string, url?: string): T { try { return JSON.parse(scriptMatch[1]) as T; } catch (error) { - throw new Error(`Failed to parse __NEXT_DATA__ JSON: ${error instanceof Error ? error.message : String(error)}`); + throw new Error(`Failed to parse __NEXT_DATA__ JSON: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); } } diff --git a/lib/routes/huxiu/util.ts b/lib/routes/huxiu/util.ts index f4a758ac4dc4fb..ccfbb9c651992c 100644 --- a/lib/routes/huxiu/util.ts +++ b/lib/routes/huxiu/util.ts @@ -262,7 +262,7 @@ const generateSignature = () => { * @param {Set} visited - Set of visited indices to prevent infinite loops. * @returns {unknown} - The resolved value. */ -const resolveNuxtData = (arr: unknown[], index: number, visited: Set = new Set()): unknown => { +const resolveNuxtData = (arr: unknown[], index: number, visited = new Set()): unknown => { if (visited.has(index)) { return arr[index]; } diff --git a/lib/routes/indianexpress/section.ts b/lib/routes/indianexpress/section.ts index d58b9a184c6bb1..8e553fda77f1b8 100644 --- a/lib/routes/indianexpress/section.ts +++ b/lib/routes/indianexpress/section.ts @@ -40,9 +40,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = response.slice(0, limit).map((item): DataItem => { + const items: DataItem[] = response.slice(0, limit).map((item): DataItem => { const title: string = item.title?.rendered ?? item.title; const description: string | undefined = item.content.rendered; const pubDate: number | string = item.date_gmt; diff --git a/lib/routes/instagram/common-utils.ts b/lib/routes/instagram/common-utils.ts index d7805006198154..25bd4ab7a261de 100644 --- a/lib/routes/instagram/common-utils.ts +++ b/lib/routes/instagram/common-utils.ts @@ -9,7 +9,7 @@ const renderItems = (items) => // Content const summary = item.caption?.text ?? ''; - let description = ''; + let description: string; switch (productType) { case 'carousel_container': { const images = item.carousel_media.map((i) => ({ diff --git a/lib/routes/instagram/web-api/utils.ts b/lib/routes/instagram/web-api/utils.ts index 560df81725e6d5..c099630a589df1 100644 --- a/lib/routes/instagram/web-api/utils.ts +++ b/lib/routes/instagram/web-api/utils.ts @@ -154,7 +154,7 @@ const renderGuestItems = (items) => { const type = node.__typename; const summary = node.edge_media_to_caption.edges[0]?.node.text ?? ''; - let description = ''; + let description: string; switch (type) { // carousel, can include GraphVideo and GraphImage case 'GraphSidecar': diff --git a/lib/routes/investor/index.ts b/lib/routes/investor/index.ts index b1b2755339d2fa..b87edcb23b137f 100644 --- a/lib/routes/investor/index.ts +++ b/lib/routes/investor/index.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.right_content_item a') + let items: DataItem[] = $('div.right_content_item a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/iqiyi/album.tsx b/lib/routes/iqiyi/album.tsx index 319f78b0aba8c1..ec0157f3658110 100644 --- a/lib/routes/iqiyi/album.tsx +++ b/lib/routes/iqiyi/album.tsx @@ -45,7 +45,7 @@ async function handler(ctx) { } let pos = 1; - let hasMore = false; + let hasMore: boolean; let epgs = []; do { const { diff --git a/lib/routes/iresearch/report.ts b/lib/routes/iresearch/report.ts index 2fc2a1075dd088..9eb20031c95b18 100644 --- a/lib/routes/iresearch/report.ts +++ b/lib/routes/iresearch/report.ts @@ -230,9 +230,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = response.List.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.List.slice(0, limit).map((item): DataItem => { const title: string = item.reportname ?? (() => { diff --git a/lib/routes/iwara/index.ts b/lib/routes/iwara/index.ts index d54718be8b3f59..13ed58f660e77b 100644 --- a/lib/routes/iwara/index.ts +++ b/lib/routes/iwara/index.ts @@ -4,38 +4,13 @@ import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -const rootUrl = 'https://www.iwara.tv'; -const apiRootUrl = 'https://api.iwara.tv'; -const imageRootUrl = 'https://i.iwara.tv'; - -const typeMap = { - video: 'Videos', - image: 'Images', -}; +import { apiRootUrl, parseThumbnail, rootUrl, typeMap } from './utils'; const apiUrlMap = { video: `${apiRootUrl}/videos`, image: `${apiRootUrl}/images`, }; -const parseThumbnail = (type, item) => { - if (type === 'image') { - return ``; - } - - if (item.embedUrl === null) { - return ``; - } - - // regex borrowed from https://stackoverflow.com/a/3726073 - const match = /https?:\/\/(?:www\.)?youtu(?:be\.com\/watch\?v=|\.be\/)([\w-]*)(&(amp;)?[\w=?]*)?/.exec(item.embedUrl); - if (match) { - return ``; - } - - return ''; -}; - export const route: Route = { path: '/users/:username/:type?', example: '/iwara/users/kelpie/video', diff --git a/lib/routes/iwara/namespace.ts b/lib/routes/iwara/namespace.ts index 01aa2f0c3fd2db..71792cb95f622b 100644 --- a/lib/routes/iwara/namespace.ts +++ b/lib/routes/iwara/namespace.ts @@ -2,6 +2,6 @@ import type { Namespace } from '@/types'; export const namespace: Namespace = { name: 'iwara', - url: 'ecchi.iwara.tv', + url: 'www.iwara.tv', lang: 'en', }; diff --git a/lib/routes/iwara/ranking.ts b/lib/routes/iwara/ranking.ts new file mode 100644 index 00000000000000..b2969e31b05eda --- /dev/null +++ b/lib/routes/iwara/ranking.ts @@ -0,0 +1,98 @@ +import { config } from '@/config'; +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import { parseDate } from '@/utils/parse-date'; +import { getPuppeteerPage } from '@/utils/puppeteer'; + +import { apiqRootUrl, parseThumbnail, rootUrl, typeMap } from './utils'; + +const sortMap = { + date: 'Latest', + trending: 'Trending', + popularity: 'Popularity', + views: 'Views', + likes: 'Likes', +}; + +const ratingMap = { + all: 'All', + general: 'General', + ecchi: 'Ecchi', +}; + +export const route: Route = { + path: '/ranking/:type?/:sort?/:rating?', + example: '/iwara/ranking/video/date/ecchi', + parameters: { + type: 'Content type, can be video or image, default is video', + sort: 'Sort type, can be date, trending, popularity, views, likes, default is date', + rating: 'Rating, can be all, general, ecchi, default is ecchi', + }, + name: 'Ranking', + maintainers: ['CaoMeiYouRen233'], + handler, + features: { + requirePuppeteer: true, + nsfw: true, + }, + radar: [ + { + source: ['www.iwara.tv/videos', 'www.iwara.tv/images'], + target: (params, url) => { + const searchParams = new URL(url).searchParams; + const type = url.includes('/videos') ? 'video' : 'image'; + const sort = searchParams.get('sort') || 'date'; + const rating = searchParams.get('rating') || 'ecchi'; + return `/iwara/ranking/${type}/${sort}/${rating}`; + }, + }, + ], +}; + +async function handler(ctx) { + const { type = 'video', sort = 'date', rating = 'ecchi' } = ctx.req.param(); + + const limit = ctx.req.query('limit') || 32; + const url = `${apiqRootUrl}/${type === 'video' ? 'videos' : 'images'}?sort=${sort}&rating=${rating}&limit=${limit}`; + + const items = await cache.tryGet( + `iwara:ranking:${type}:${sort}:${rating}`, + async () => { + const { page, destory } = await getPuppeteerPage(url, { + onBeforeLoad: async (page) => { + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'fetch' ? request.continue() : request.abort(); + }); + }, + gotoConfig: { + waitUntil: 'networkidle0', + }, + }); + + try { + const content = await page.evaluate(() => document.querySelector('pre')?.textContent || document.body.textContent); + const response = JSON.parse(content || '{}'); + + return response.results.map((item) => ({ + title: item.title, + author: item.user.name, + link: `${rootUrl}/${type === 'video' ? 'video' : 'image'}/${item.id}${item.slug ? `/${item.slug}` : ''}`, + category: item.tags?.map((i) => i.id) || [], + description: parseThumbnail(type, item), + pubDate: parseDate(item.createdAt), + })); + } finally { + await destory(); + } + }, + config.cache.routeExpire, + false + ); + + return { + title: `Iwara Ranking - ${typeMap[type]} - ${sortMap[sort]} - ${ratingMap[rating]}`, + link: `${rootUrl}/${type === 'video' ? 'videos' : 'images'}?sort=${sort}&rating=${rating}`, + item: items, + }; +} diff --git a/lib/routes/iwara/utils.ts b/lib/routes/iwara/utils.ts new file mode 100644 index 00000000000000..573e3a1ef114e1 --- /dev/null +++ b/lib/routes/iwara/utils.ts @@ -0,0 +1,28 @@ +export const rootUrl = 'https://www.iwara.tv'; +export const apiRootUrl = 'https://api.iwara.tv'; +export const apiqRootUrl = 'https://apiq.iwara.tv'; +export const imageRootUrl = 'https://i.iwara.tv'; + +export const typeMap = { + video: 'Videos', + image: 'Images', +}; + +export const parseThumbnail = (type: 'video' | 'image', item: any) => { + if (type === 'image') { + const thumbnail = item.thumbnail || item.file; + return thumbnail ? `` : ''; + } + + if (item.embedUrl === null) { + return item.file ? `` : ''; + } + + // regex borrowed from https://stackoverflow.com/a/3726073 + const match = /https?:\/\/(?:www\.)?youtu(?:be\.com\/watch\?v=|\.be\/)([\w-]*)(&(amp;)?[\w=?]*)?/.exec(item.embedUrl); + if (match) { + return ``; + } + + return ''; +}; diff --git a/lib/routes/javbus/index.tsx b/lib/routes/javbus/index.tsx index c29014a5c9ee76..f5070c88b95213 100644 --- a/lib/routes/javbus/index.tsx +++ b/lib/routes/javbus/index.tsx @@ -73,7 +73,7 @@ export const route: Route = { }; async function handler(ctx) { - const isWestern = /^\/western/.test(getSubPath(ctx)); + const isWestern = getSubPath(ctx).startsWith('/western'); const domain = ctx.req.query('domain') ?? 'javbus.com'; const westernDomain = ctx.req.query('western_domain') ?? 'javbus.org'; diff --git a/lib/routes/jike/topic-text.ts b/lib/routes/jike/topic-text.ts index 893559a9338c4e..cff99149051772 100644 --- a/lib/routes/jike/topic-text.ts +++ b/lib/routes/jike/topic-text.ts @@ -20,6 +20,11 @@ export const route: Route = { radar: [ { source: ['web.okjike.com/topic/:id'], + target: '/topic/text/:id', + }, + { + source: ['m.okjike.com/topics/:id'], + target: '/topic/text/:id', }, ], name: '圈子 - 纯文字', diff --git a/lib/routes/jike/topic.ts b/lib/routes/jike/topic.ts index 81080963412651..463f922320a598 100644 --- a/lib/routes/jike/topic.ts +++ b/lib/routes/jike/topic.ts @@ -35,6 +35,10 @@ export const route: Route = { source: ['web.okjike.com/topic/:id'], target: '/topic/:id', }, + { + source: ['m.okjike.com/topics/:id'], + target: '/topic/:id', + }, ], name: '圈子', maintainers: ['DIYgod', 'prnake'], diff --git a/lib/routes/jike/user.ts b/lib/routes/jike/user.ts index f29c472d38668d..be9308c43155bb 100644 --- a/lib/routes/jike/user.ts +++ b/lib/routes/jike/user.ts @@ -24,6 +24,10 @@ export const route: Route = { source: ['web.okjike.com/u/:uid'], target: '/user/:uid', }, + { + source: ['m.okjike.com/users/:uid'], + target: '/user/:uid', + }, ], name: '用户动态', maintainers: ['DIYgod', 'prnake'], diff --git a/lib/routes/joneslanglasalle/index.ts b/lib/routes/joneslanglasalle/index.ts index 60f1c9b21a9709..1d9ca7c9c95adc 100644 --- a/lib/routes/joneslanglasalle/index.ts +++ b/lib/routes/joneslanglasalle/index.ts @@ -136,7 +136,7 @@ export const handler = async (ctx: Context): Promise => { content_html: $$el.find('div.content-card__body').html(), }; }) - .filter((link): link is { url: string; type: string; content_html: string } => true); + .filter((_link): _link is { url: string; type: string; content_html: string } => true); const description: string = renderDescription({ description: cleanHtml($$('div.page-section').eq(1).html() ?? $$('div.copy-block').html() ?? '', ['div.richtext p', 'h3', 'h4', 'h5', 'h6', 'figure', 'img', 'ul', 'li', 'span', 'b']), diff --git a/lib/routes/jornada/index.ts b/lib/routes/jornada/index.ts index 30703d43c0b339..05644df1e2845c 100644 --- a/lib/routes/jornada/index.ts +++ b/lib/routes/jornada/index.ts @@ -1,4 +1,4 @@ -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -65,7 +65,7 @@ async function handler(ctx) { const response = await got(url); const data = response.data; - let items = {}; + let items: DataItem[]; if (category) { const newsFilteredByCategory = data.filter((item) => item.category === categories[category]); diff --git a/lib/routes/juejin/pins.ts b/lib/routes/juejin/pins.ts index dc9187a959d57c..86a0fcb407fc55 100644 --- a/lib/routes/juejin/pins.ts +++ b/lib/routes/juejin/pins.ts @@ -37,8 +37,8 @@ async function handler(ctx) { '6824710203112423437': '树洞一下', }; - let url = ''; - let json = {}; + let url: string; + let json: Record; if (/^\d+$/.test(type)) { url = `https://api.juejin.cn/recommend_api/v1/short_msg/topic`; json = { id_type: 4, sort_type: 500, cursor: '0', limit: 20, topic_id: type }; diff --git a/lib/routes/jumeili/home.ts b/lib/routes/jumeili/home.ts index 7f986194326c37..65f63df184901c 100644 --- a/lib/routes/jumeili/home.ts +++ b/lib/routes/jumeili/home.ts @@ -24,7 +24,11 @@ export const route: Route = { }, radar: [ { - source: ['www.jumeili.cn/', 'jumeili.cn/'], + source: ['www.jumeili.cn/'], + target: '/home/:column?', + }, + { + source: ['jumeili.cn/'], target: '/home/:column?', }, ], diff --git a/lib/routes/kakuyomu/works.ts b/lib/routes/kakuyomu/works.ts index 9056767cbb5a44..f9080c6915bd82 100644 --- a/lib/routes/kakuyomu/works.ts +++ b/lib/routes/kakuyomu/works.ts @@ -33,7 +33,8 @@ async function handler(ctx: Context): Promise { const id = ctx.req.param('id'); const url = `https://kakuyomu.jp/works/${id}`; const limit = Number.parseInt(ctx.req.query('limit') || '10'); - const $ = load(await ofetch(url)); + const html = await ofetch(url); + const $ = load(html); const nextData = JSON.parse($('#__NEXT_DATA__').text()); @@ -56,7 +57,8 @@ async function handler(ctx: Context): Promise { .map((item) => { const episodeUrl = `https://kakuyomu.jp/works/${id}/episodes/${item.id}`; return cache.tryGet(episodeUrl, async () => { - const $ = load(await ofetch(episodeUrl)); + const html = await ofetch(episodeUrl); + const $ = load(html); const description = $('.widget-episodeBody').html(); return { title: item.title, diff --git a/lib/routes/kanxue/topic.ts b/lib/routes/kanxue/topic.ts index a4613d536cef1a..edefb572d77df5 100644 --- a/lib/routes/kanxue/topic.ts +++ b/lib/routes/kanxue/topic.ts @@ -74,15 +74,13 @@ async function handler(ctx) { path = `forum-${categoryId[category][0]}.html`; title = `看雪论坛最新主题 - ${categoryId[category][1]}`; } - } else { + } else if (category === 'digest') { // category未知时则获取全站最新帖 - if (category === 'digest') { - path = 'new-digest.htm'; - title = '看雪论坛精华主题'; - } else { - path = 'new-tid.htm'; - title = '看雪论坛最新主题'; - } + path = 'new-digest.htm'; + title = '看雪论坛精华主题'; + } else { + path = 'new-tid.htm'; + title = '看雪论坛最新主题'; } const response = await got({ diff --git a/lib/routes/kemono/index.tsx b/lib/routes/kemono/index.tsx index e6571ad739a98c..18fec7103c5d12 100644 --- a/lib/routes/kemono/index.tsx +++ b/lib/routes/kemono/index.tsx @@ -412,6 +412,6 @@ async function handler(ctx) { item: items, }; } catch (error) { - throw new Error(`Failed to fetch data from Kemono: ${error instanceof Error ? error.message : 'Unknown error'}`); + throw new Error(`Failed to fetch data from Kemono: ${error instanceof Error ? error.message : 'Unknown error'}`, { cause: error }); } } diff --git a/lib/routes/kiro/blog.ts b/lib/routes/kiro/blog.ts index 1c17a77814be02..c7b516b6dd7500 100644 --- a/lib/routes/kiro/blog.ts +++ b/lib/routes/kiro/blog.ts @@ -19,9 +19,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('main a.group') + let items: DataItem[] = $('main a.group') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/kiro/changelog.ts b/lib/routes/kiro/changelog.ts index c76e50fd71d6b3..8565b405156ede 100644 --- a/lib/routes/kiro/changelog.ts +++ b/lib/routes/kiro/changelog.ts @@ -19,9 +19,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('a.block') + let items: DataItem[] = $('a.block') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/konachan/post.ts b/lib/routes/konachan/post.ts index e0f0201e8688e7..d542ab485b57c9 100644 --- a/lib/routes/konachan/post.ts +++ b/lib/routes/konachan/post.ts @@ -30,7 +30,10 @@ export const route: Route = { }, radar: [ { - source: ['konachan.com/post', 'konachan.net/post'], + source: ['konachan.com/post'], + }, + { + source: ['konachan.net/post'], }, ], name: 'Popular Recent Posts', diff --git a/lib/routes/koyso/index.tsx b/lib/routes/koyso/index.tsx index 5e9fd534f5dd4b..a038f6a39a8785 100644 --- a/lib/routes/koyso/index.tsx +++ b/lib/routes/koyso/index.tsx @@ -38,9 +38,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('a.game_item') + let items: DataItem[] = $('a.game_item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/kpopping/kpics.ts b/lib/routes/kpopping/kpics.ts index 328f366b1a503b..428132b0d08082 100644 --- a/lib/routes/kpopping/kpics.ts +++ b/lib/routes/kpopping/kpics.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.pics div.matrix div.cell') + let items: DataItem[] = $('div.pics div.matrix div.cell') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/kpopping/news.ts b/lib/routes/kpopping/news.ts index 770e529fe6708f..822ee81b34b2f2 100644 --- a/lib/routes/kpopping/news.ts +++ b/lib/routes/kpopping/news.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('section.news-list-item') + let items: DataItem[] = $('section.news-list-item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/lephoceen/chrono.ts b/lib/routes/lephoceen/chrono.ts new file mode 100644 index 00000000000000..17485a0bead231 --- /dev/null +++ b/lib/routes/lephoceen/chrono.ts @@ -0,0 +1,64 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const route: Route = { + path: '/chrono', + categories: ['new-media'], + example: '/lephoceen/chrono', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['lephoceen.fr/chrono'], + target: '/chrono', + }, + ], + name: 'Fil Info Le Phocéen (Chrono)', + maintainers: ['Loopy03'], + handler: async (_) => { + const response = await ofetch('https://www.lephoceen.fr/chrono'); + const $ = load(response); + + // Récupération du fichier json + const jsonRaw = $('script[id="__NEXT_DATA__"]').html(); + const jsonData = JSON.parse(jsonRaw); + + // Tableau des articles via le chemin identifié du fichier json + // Structure: props -> pageProps -> data -> datas + const articles = jsonData?.props?.pageProps?.data?.datas || []; + + const items = articles.map((item: any) => { + // Gestion du lien : le slug est relatif, on ajoute le domaine + const baseUrl = 'https://www.lephoceen.fr'; + const link = item.slug.startsWith('http') ? item.slug : `${baseUrl}${item.slug}`; + + // Gestion de la date : Le JSON fournit un timestamp UNIX (en secondes) + const pubDate = parseDate(item.date.publish_at.timestamp * 1000); + + return { + title: item.title, + link, + description: item.text || `[${item.category?.name}] ${item.title}`, + pubDate, // L'objet Date standard gère le fuseau correctement + category: [item.category?.name], + image: item.images?.['16x9']?.url, + }; + }); + + return { + title: 'Le Phocéen - Fil Info', + link: 'https://www.lephoceen.fr/chrono', + item: items, + }; + }, +}; diff --git a/lib/routes/lephoceen/namespace.ts b/lib/routes/lephoceen/namespace.ts new file mode 100644 index 00000000000000..8c179851374235 --- /dev/null +++ b/lib/routes/lephoceen/namespace.ts @@ -0,0 +1,7 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Le Phocéen', + url: 'lephoceen.fr', + description: "Actualités de l'Olympique de Marseille du site lephocéen.fr", +}; diff --git a/lib/routes/linkedin/cn/renderer.ts b/lib/routes/linkedin/cn/renderer.ts index d8fefd42fde665..6328531fb647b2 100644 --- a/lib/routes/linkedin/cn/renderer.ts +++ b/lib/routes/linkedin/cn/renderer.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ const text_tag = { LINE_BREAK: 0, INLINE_CODE: 1, @@ -68,6 +69,7 @@ class Ucs2Text { return new Ucs2Text(_start === _end ? '' : this.codePoints.slice(_start, _end)); } slice(a, b) { + // oxlint-disable-next-line unicorn/prefer-string-slice return this.substring(a, b).toString(); } toString() { diff --git a/lib/routes/linovelib/volume.ts b/lib/routes/linovelib/volume.ts index 3134b153ed6bee..bd4644dfa1372c 100644 --- a/lib/routes/linovelib/volume.ts +++ b/lib/routes/linovelib/volume.ts @@ -22,7 +22,8 @@ export const route: Route = { async function handler(ctx: Context): Promise { const { id } = ctx.req.param(); const link = `https://www.linovelib.com/novel/${id}/catalog`; - const $ = load((await got(link)).data); + const response = await got(link); + const $ = load(response.data); return { title: `${$('.book-meta h1').text()} - 哔哩轻小说`, link, diff --git a/lib/routes/liquipedia/dota2-matches.ts b/lib/routes/liquipedia/dota2-matches.ts index 90b7cc06928ce9..8e406daacfaad2 100644 --- a/lib/routes/liquipedia/dota2-matches.ts +++ b/lib/routes/liquipedia/dota2-matches.ts @@ -45,7 +45,7 @@ async function handler(ctx) { link: url, item: list?.toArray().map((item) => { item = $(item); - let message = ''; + let message: string; if (item.attr('style') === 'background:rgb(240, 255, 240)') { message = '胜'; } else if (item.attr('style') === 'background:rgb(249, 240, 242)') { diff --git a/lib/routes/ltaaa/article.ts b/lib/routes/ltaaa/article.ts index 66c2ce872b2e8b..bb2a6c67f5a94d 100644 --- a/lib/routes/ltaaa/article.ts +++ b/lib/routes/ltaaa/article.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('ul.wlist li') + let items: DataItem[] = $('ul.wlist li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/mail/imap.ts b/lib/routes/mail/imap.ts index 5ba9e0159d863e..d5b04750060555 100644 --- a/lib/routes/mail/imap.ts +++ b/lib/routes/mail/imap.ts @@ -48,7 +48,7 @@ async function handler(ctx) { try { await client.connect(); } catch (error) { - throw new Error(error.responseText); + throw new Error(error.responseText, { cause: error }); } /** diff --git a/lib/routes/mangadex/_feed.ts b/lib/routes/mangadex/_feed.ts index 02d87d080b008b..5cf42266bf4838 100644 --- a/lib/routes/mangadex/_feed.ts +++ b/lib/routes/mangadex/_feed.ts @@ -71,7 +71,7 @@ const getMangaMeta = async (id: string, needCover: boolean = true, lang?: string * @usage const mangaMetaMap = await getMangaMetaByIds(['f98660a1-d2e2-461c-960d-7bd13df8b76d']); */ export async function getMangaMetaByIds(ids: string[], needCover: boolean = true, lang?: string | string[]): Promise> { - const deDuplidatedIds = [...new Set(ids)].sort(); + const deDuplidatedIds = [...new Set(ids)].toSorted(); const includes = needCover ? ['cover_art'] : []; const rawMangaMetas = (await cache.tryGet( diff --git a/lib/routes/manus/blog.ts b/lib/routes/manus/blog.ts index 684d111531a8e6..7ed00f577ecb24 100644 --- a/lib/routes/manus/blog.ts +++ b/lib/routes/manus/blog.ts @@ -1,10 +1,15 @@ -import { load } from 'cheerio'; +import MarkdownIt from 'markdown-it'; import type { Data, DataItem, Route } from '@/types'; import { ViewType } from '@/types'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; +const md = MarkdownIt({ + html: true, + linkify: true, +}); + export const route: Route = { path: '/blog', categories: ['programming'], @@ -35,43 +40,76 @@ export const route: Route = { async function handler() { const rootUrl = 'https://manus.im/blog'; - const response = await ofetch(rootUrl); - const $ = load(response); + const renderData = await ofetch(rootUrl, { + headers: { + RSC: '1', + }, + responseType: 'text', + }); - const list: DataItem[] = $('div.mt-10.px-6 > a') - .toArray() - .map((item) => { - const element = $(item); - const link = new URL(String(element.attr('href')), rootUrl).href; - const title = String(element.find('h2').attr('title')); + let blogList; + const lines = renderData.split('\n'); + for (const line of lines) { + if (line.includes('{"blogList":{"$typeName"')) { + const jsonStr = line.slice(Math.max(0, line.indexOf('{"blogList":{"$typeName"'))); + const lastBrace = jsonStr.lastIndexOf('}'); + try { + const parsed = JSON.parse(jsonStr.slice(0, Math.max(0, lastBrace + 1))); + blogList = parsed.blogList; + break; + } catch { + // Ignore parse errors and try next line if any + } + } + } - return { - link, - title, - }; - }); + if (!blogList || !blogList.groups) { + throw new Error('Failed to parse blogList from RSC data'); + } + + const list: Array = blogList.groups.flatMap( + (group) => + group.blogs?.map((blog) => ({ + title: blog.title, + link: `https://manus.im/blog/${blog.recordUid}`, + pubDate: new Date(blog.createdAt.seconds * 1000), + description: blog.desc, + category: [group.kindName], + _contentUrl: blog.contentUrl, + })) ?? [] + ); const items: DataItem[] = await Promise.all( - list.map((item) => - cache.tryGet(String(item.link), async () => { - const response = await ofetch(String(item.link)); - const $ = load(response); - const description: string = $('div.relative:nth-child(3)').html() ?? ''; - const pubDateText: string = $('div.gap-3:nth-child(1) > span:nth-child(2)').text().trim(); - const currentYear: number = new Date().getFullYear(); - const pubDate: Date = new Date(`${pubDateText} ${currentYear}`); + list.map( + (item) => + cache.tryGet(String(item.link), async () => { + const contentUrl = item._contentUrl; + let description = String(item.description); + if (contentUrl) { + try { + let contentText = await ofetch(contentUrl, { responseType: 'text' }); + // Fix video embeds: Manus uses ![type=manus_video](url) which markdown-it renders as + contentText = contentText.replaceAll(/!\[.*?\]\((.+?\.(mp4|mov|webm))\)/gi, ''); + // Parse markdown to HTML + description = md.render(contentText); + } catch { + // Fallback to description from list if fetch fails + } + } + + // Remove the temporary property to avoid pollution + delete item._contentUrl; - return { - ...item, - description, - pubDate, - }; - }) + return { + ...item, + description, + }; + }) as Promise ) ); return { - title: 'Manus', + title: 'Manus Blog', link: rootUrl, item: items, language: 'en', diff --git a/lib/routes/mathpix/blog.tsx b/lib/routes/mathpix/blog.tsx index a43b200c7225fb..651c68074e09b3 100644 --- a/lib/routes/mathpix/blog.tsx +++ b/lib/routes/mathpix/blog.tsx @@ -33,9 +33,7 @@ export const handler = async (ctx: Context): Promise => { } }); - let items: DataItem[] = []; - - items = $('li.articles__item') + let items: DataItem[] = $('li.articles__item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/maven/central.ts b/lib/routes/maven/central.ts index b5a1717c9dfb2b..c6a1a663482496 100644 --- a/lib/routes/maven/central.ts +++ b/lib/routes/maven/central.ts @@ -72,7 +72,7 @@ async function handler(ctx) { } } catch (error: any) { if (error?.response?.status === 404) { - throw new Error(`Could not find component for ${group}:${artifact}: metadata not found`); + throw new Error(`Could not find component for ${group}:${artifact}: metadata not found`, { cause: error }); } throw error; } diff --git a/lib/routes/mercari/util.tsx b/lib/routes/mercari/util.tsx index 9de7a2000a8ec1..e8d885605a980e 100644 --- a/lib/routes/mercari/util.tsx +++ b/lib/routes/mercari/util.tsx @@ -285,7 +285,7 @@ const fetchFromMercari = async function fetchFromMercari(url: string, data: a try { return await ofetch(url, options); } catch (error) { - throw new Error(`API request failed: ${error}`); + throw new Error(`API request failed: ${error}`, { cause: error }); } }; diff --git a/lib/routes/meritalk/articles.ts b/lib/routes/meritalk/articles.ts new file mode 100644 index 00000000000000..a602e2b5056364 --- /dev/null +++ b/lib/routes/meritalk/articles.ts @@ -0,0 +1,80 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +import { renderDescription } from './templates/description'; + +export const route: Route = { + path: '/articles', + categories: ['new-media'], + example: '/meritalk/articles', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['meritalk.com/articles/'], + target: '/articles', + }, + ], + name: 'Latest Articles', + maintainers: ['superguyDiluc'], + handler, +}; + +async function handler() { + const baseUrl = 'https://www.meritalk.com/articles'; + + const { data: response } = await got(baseUrl); + const $ = load(response); + + const list = $('div.news-block-sm') + .toArray() + .map((item) => { + const $item = $(item); + const a = $item.find('.news-block-title a'); + const link = a.attr('href'); + return { + title: a.text().trim(), + link: link as string, + pubDate: parseDate($item.find('time[datetime]').attr('datetime') as string), + category: $item + .find('.category-header-name a') + .toArray() + .map((elem) => $(elem).text()), + description: '', + }; + }); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = load(response); + + const featuredImage = $('.single-featured-image').first().html() || ''; + const fullContent = $('.single-body').first().html() || ''; + item!.description = renderDescription({ + featuredImage, + fullContent, + }); + + return item; + }) + ) + ); + + return { + title: 'News – MeriTalk', + link: 'https://www.meritalk.com/articles/', + item: items, + }; +} diff --git a/lib/routes/meritalk/namespace.ts b/lib/routes/meritalk/namespace.ts new file mode 100644 index 00000000000000..793aef16332e74 --- /dev/null +++ b/lib/routes/meritalk/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'MeriTalk', + url: 'meritalk.com', +}; diff --git a/lib/routes/meritalk/templates/description.tsx b/lib/routes/meritalk/templates/description.tsx new file mode 100644 index 00000000000000..f3b6d00454c11e --- /dev/null +++ b/lib/routes/meritalk/templates/description.tsx @@ -0,0 +1,21 @@ +import { raw } from 'hono/html'; +import { renderToString } from 'hono/jsx/dom/server'; + +type DescriptionParams = { + featuredImage?: string; + fullContent?: string; +}; + +export const renderDescription = ({ featuredImage, fullContent }: DescriptionParams) => + renderToString( + <> + {featuredImage ? ( + <> + {raw(featuredImage)} +
    + + ) : null} + + {fullContent ? raw(fullContent) : null} + + ); diff --git a/lib/routes/miniflux/entry.ts b/lib/routes/miniflux/entry.ts index 0090796f86adcd..b2511d53941d40 100644 --- a/lib/routes/miniflux/entry.ts +++ b/lib/routes/miniflux/entry.ts @@ -1,6 +1,6 @@ import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; -import type { Data, Route } from '@/types'; +import type { Data, DataItem, Route } from '@/types'; import got from '@/utils/got'; export const route: Route = { @@ -131,7 +131,7 @@ async function handler(ctx) { const entriesID = []; const feedsName = []; - const articles = []; + const articles: DataItem[] = []; // MiniFlux will only preserve the *first* valid filter option // for each parameter, in order to matching the default behavior @@ -162,28 +162,16 @@ async function handler(ctx) { const feedsList = [feedsID.split('&')].flat(); if (limit && queryLimit) { - if (limit < queryLimit) { - queryLimit = limit * feedsList.length; - } else { + if (limit >= queryLimit) { const eachLimit = Number.parseInt(queryLimit / feedsList.length); - if (eachLimit) { - limit = eachLimit; - } else { - limit = 1; - queryLimit = feedsList.length; - } + limit = eachLimit || 1; } parameters += `&limit=${limit}`; } else if (limit) { parameters += `&limit=${limit}`; } else if (queryLimit) { const eachLimit = Number.parseInt(queryLimit / feedsList.length); - if (eachLimit) { - limit = eachLimit; - } else { - limit = 1; - queryLimit = feedsList.length; - } + limit = eachLimit || 1; parameters += `&limit=${limit}`; } @@ -267,7 +255,7 @@ async function handler(ctx) { }); const entries = response.data.entries; - const articles = []; + const articles: DataItem[] = []; for (const entry of entries) { entriesID.push(entry.id); let entryTitle = entry.title; diff --git a/lib/routes/mirrormedia/category.ts b/lib/routes/mirrormedia/category.ts index cb08c0bc4425ba..31b4cd1c15c07f 100644 --- a/lib/routes/mirrormedia/category.ts +++ b/lib/routes/mirrormedia/category.ts @@ -88,7 +88,7 @@ query ($take: Int, $skip: Int, $orderBy: [PostOrderByInput!]!, $filter: PostWher title: e.title, pubDate: parseDate(e.publishedDate), category: [...(e.sections ?? []).map((_) => `section:${_.name}`), ...(e.categories ?? []).map((_) => `category:${_.name}`)], - link: `${rootUrl}/${'story'}/${e.slug}`, + link: `${rootUrl}/story/${e.slug}`, })); const list = await Promise.all(items.map((item) => getArticle(item))); diff --git a/lib/routes/misskey/types.ts b/lib/routes/misskey/types.ts index 65ab43bed92ce9..efdd7948e62cc5 100644 --- a/lib/routes/misskey/types.ts +++ b/lib/routes/misskey/types.ts @@ -105,7 +105,7 @@ interface MisskeyFile { thumbnailUrl: string | null; comment: string | null; folderId: string | null; - folder?: unknown | null; + folder?: unknown; userId: string | null; user?: MisskeyUser | null; } diff --git a/lib/routes/misskey/utils.tsx b/lib/routes/misskey/utils.tsx index 11b66ae82121c3..189d7740b11313 100644 --- a/lib/routes/misskey/utils.tsx +++ b/lib/routes/misskey/utils.tsx @@ -56,7 +56,7 @@ const parseNotes = (data: MisskeyNote[], site: string, simplifyAuthor: boolean = reply: item.reply, }); - let title = ''; + let title: string; if (isReply && item.reply) { const replyToHost = item.reply.user.host ?? site; const replyToAuthor = simplifyAuthor ? item.reply.user.name : `${item.reply.user.name} (${item.reply.user.username}@${replyToHost})`; diff --git a/lib/routes/miyuki/news.ts b/lib/routes/miyuki/news.ts index 3ed4b50c130aa8..d2432d3a13a5fa 100644 --- a/lib/routes/miyuki/news.ts +++ b/lib/routes/miyuki/news.ts @@ -6,13 +6,15 @@ import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; -const ORIGIN = 'https://miyuki.jp'; +const ORIGIN = 'https://www.miyuki.jp'; const NEWS_LINK = `${ORIGIN}/s/y10/news/list`; +const DETAIL_HEADER_SELECTOR = ['.pc__news_detail__title', '.pc__news_detail__title__japanese', '.news_detail__date', '.news_detail__title', '.news_detail__ganre'].join(', '); export const route: Route = { path: '/news', example: '/miyuki/news', name: 'News', + url: 'www.miyuki.jp/s/y10/news/list', categories: ['new-media'], maintainers: ['KarasuShin'], features: { @@ -28,24 +30,23 @@ export const route: Route = { }; async function handler() { - const $ = load(await ofetch(NEWS_LINK)); + const html = await ofetch(NEWS_LINK); + const $ = load(html); const items = await Promise.all( $('.list__side_border li') .toArray() - .map(async (item) => { + .map((item) => { const $item = $(item); const link = `${ORIGIN}${$item.find('a').attr('href')!}`; - return await cache.tryGet(link, async () => { + return cache.tryGet(link, async () => { const category = $item.find('p span').last().text(); + const title = $item.find('a').text(); return { - title: `${category} - ${$item.find('a').text()}`, + title, link, pubDate: timezone(parseDate($item.find('p span').first().text()), +9), category: [category], - description: await cache.tryGet(link, async () => { - const $detail = load(await ofetch(link)); - return $detail('.contents_area__inner').html()!; - }), + description: await getDescription(link), } as DataItem; }); }) @@ -57,3 +58,57 @@ async function handler() { item: items, }; } + +async function getDescription(link: string) { + const detailHtml = await ofetch(link); + const $ = load(detailHtml); + const content = $('.contents_area__inner'); + content.children(DETAIL_HEADER_SELECTOR).remove(); + normalizePhotoLists($, content); + content.children().each((_, element) => { + const child = $(element); + if (!hasMeaningfulHtml(child.html())) { + child.remove(); + } + }); + + return content.html()?.trim() ?? ''; +} + +function hasMeaningfulHtml(html?: string | null) { + return Boolean( + html + ?.replaceAll(//g, '') + .replaceAll(' ', '') + .trim() + ); +} + +function normalizePhotoLists($, content) { + content.find('.news_detail__photo_list').each((_, element) => { + const list = $(element); + const items = list + .children('li') + .toArray() + .flatMap((item) => { + const photoItem = $(item); + photoItem.find('img.for_sp').remove(); + photoItem.find('img').each((__, image) => { + const img = $(image); + const src = img.attr('src'); + if (!src) { + img.remove(); + return; + } + + img.attr('src', new URL(src, ORIGIN).href); + img.removeAttr('class'); + }); + + const html = photoItem.html(); + return hasMeaningfulHtml(html) ? [`
    ${html!.trim()}
    `] : []; + }); + + list.replaceWith(items.join('

    ')); + }); +} diff --git a/lib/routes/modrinth/versions.tsx b/lib/routes/modrinth/versions.tsx index 15eab1f3799fe3..4ff061ace61f08 100644 --- a/lib/routes/modrinth/versions.tsx +++ b/lib/routes/modrinth/versions.tsx @@ -140,7 +140,7 @@ async function handler(ctx: Context) { }; } catch (error: any) { if (error?.response?.statusCode === 404) { - throw new Error(`${error.message}: Project ${id} not found`); + throw new Error(`${error.message}: Project ${id} not found`, { cause: error }); } throw error; } diff --git a/lib/routes/musikguru/news.ts b/lib/routes/musikguru/news.ts index a98b390e679912..4bdd53b8fbd5fd 100644 --- a/lib/routes/musikguru/news.ts +++ b/lib/routes/musikguru/news.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'de'; - let items: DataItem[] = []; - - items = $('section') + let items: DataItem[] = $('section') .eq(1) .find('div.card') .slice(0, limit) diff --git a/lib/routes/my-formosa/index.ts b/lib/routes/my-formosa/index.ts index dcddb57944a8a5..22c12684303c74 100644 --- a/lib/routes/my-formosa/index.ts +++ b/lib/routes/my-formosa/index.ts @@ -57,7 +57,7 @@ async function handler() { const res = await fetch(link); const $ = load(res); - const isTV = /^\/TV/.test(new URL(link).pathname); + const isTV = new URL(link).pathname.startsWith('/TV'); return { title, diff --git a/lib/routes/mycard520/news.ts b/lib/routes/mycard520/news.ts index 1e850e21e55528..256f32d21df6d1 100644 --- a/lib/routes/mycard520/news.ts +++ b/lib/routes/mycard520/news.ts @@ -20,11 +20,9 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-TW'; - let items: DataItem[] = []; - $('div.page_numbers').remove(); - items = $('div#tab1 ul li') + let items: DataItem[] = $('div#tab1 ul li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/natgeo/natgeo.ts b/lib/routes/natgeo/natgeo.ts index b2bd3811ab4379..5cd0bcc3fba4c8 100644 --- a/lib/routes/natgeo/natgeo.ts +++ b/lib/routes/natgeo/natgeo.ts @@ -10,12 +10,7 @@ import { parseDate } from '@/utils/parse-date'; async function loadContent(link) { const data = await ofetch(link); const $ = load(data); - const dtStr = $('.content-title-area') - .find('h6') - .first() - .text() - .replaceAll(/ /gi, ' ') - .trim(); + const dtStr = $('.content-title-area').find('h6').first().text().replaceAll(' ', ' ').trim(); $('.splide__arrows, .slide-control, [class^="ad-"], style').remove(); diff --git a/lib/routes/ncu/jwc.ts b/lib/routes/ncu/jwc.ts index 7da9318a1e90f7..6f9c36ea2dff66 100644 --- a/lib/routes/ncu/jwc.ts +++ b/lib/routes/ncu/jwc.ts @@ -1,9 +1,12 @@ -import { load } from 'cheerio'; // 可以使用类似 jQuery 的 API HTML 解析器 +import { load } from 'cheerio'; -import type { Route } from '@/types'; -import got from '@/utils/got'; // 自订的 got +import type { Data, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +const baseUrl = 'https://jwc.ncu.edu.cn'; + export const route: Route = { path: '/jwc', categories: ['university'], @@ -19,52 +22,91 @@ export const route: Route = { }, radar: [ { - source: ['jwc.ncu.edu.cn/', 'jwc.ncu.edu.cn/jwtz/index.htm'], + source: ['jwc.ncu.edu.cn'], + target: '/jwc', }, ], name: '教务通知', - maintainers: ['ywh555hhh'], + maintainers: ['ywh555hhh', 'jixiuweilan'], handler, - url: 'jwc.ncu.edu.cn/', + url: 'jwc.ncu.edu.cn/Notices.jsp', }; async function handler() { - const baseUrl = 'https://jwc.ncu.edu.cn'; - const response = await got(baseUrl); - const $ = load(response.body); - const currentDate = new Date(); - const currentYear = currentDate.getFullYear(); - const currentMonth = currentDate.getMonth() + 1; + const targetUrl = `${baseUrl}/Notices.jsp?urltype=tree.TreeTempUrl&wbtreeid=1541`; + + const response = await ofetch(targetUrl); + const $ = load(response); + + const list = $('div.space-y-2 div.group') + .toArray() + .map((item) => { + const el = $(item); + const linkEl = el.find('a').first(); + + const title = linkEl.find('span.text-gray-700').text().trim() || linkEl.text().trim(); + const rawLink = linkEl.attr('href'); + const link = rawLink ? new URL(rawLink, baseUrl).href : ''; + + const dateText = el + .find(String.raw`.font-mono span.md\:inline`) + .text() + .trim(); + + return { + title, + link, + pubDate: dateText ? parseDate(dateText, 'YYYY-MM-DD') : undefined, + }; + }) + .filter((item) => item.title && item.link); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const detailResponse = await ofetch(item.link); + const $detail = load(detailResponse); + + const contentEl = $detail('.v_news_content'); + + contentEl.find('a').each((_, el) => { + const href = $detail(el).attr('href'); + if (href && !href.startsWith('http')) { + $detail(el).attr('href', new URL(href, baseUrl).href); + } + }); + contentEl.find('img').each((_, el) => { + const src = $detail(el).attr('src'); + if (src && !src.startsWith('http')) { + $detail(el).attr('src', new URL(src, baseUrl).href); + } + }); + + let description = contentEl.html() || ''; + + const attachments = $detail('a[href*="download.jsp"]'); + if (attachments.length > 0) { + description += '
      '; + attachments.each((_, el) => { + const href = $detail(el).attr('href'); + const text = $detail(el).text().trim(); + if (href && text) { + const absoluteHref = href.startsWith('http') ? href : new URL(href, baseUrl).href; + description += `
    • ${text}
      • `; + } + }); + description += '
    '; + } - const list = $('.box3 .inner ul.img-list li'); + return { ...item, description }; + }) + ) + ); return { - title: '南昌大学教务处', - link: baseUrl, - description: '南昌大学教务处', - - item: - list && - list.toArray().map((item) => { - const el = $(item); - const linkEl = el.find('a'); - const date = el.text().split('】')[0].replace('【', '').trim(); - const title = linkEl.attr('title'); - const link = `${baseUrl}/${linkEl.attr('href')}`; - - const newsDate = parseDate(date, 'MM-DD'); - const newsMonth = newsDate.getMonth() + 1; - - // If the news month is greater than the current month, subtract 1 from the year - const year = newsMonth > currentMonth ? currentYear - 1 : currentYear; - - newsDate.setFullYear(year); - - return { - title, - link, - pubDate: newsDate, - }; - }), - }; + title: '南昌大学教务处 - 通知公告', + link: targetUrl, + description: '南昌大学教务处通知公告', + item: items, + } as Data; } diff --git a/lib/routes/neea/index.ts b/lib/routes/neea/index.ts index 5a0c95c59b1e7d..e6c3b85f2584de 100644 --- a/lib/routes/neea/index.ts +++ b/lib/routes/neea/index.ts @@ -49,7 +49,7 @@ async function handler(ctx) { pubDate: timezone(parseDate(time), +8), }; const other = await loadContent(String(itemUrl)); - return Object.assign({}, single, other); + return { ...single, ...other }; }) ); return { @@ -125,11 +125,18 @@ export const route: Route = { features: { supportRadar: true, }, - radar: Object.entries(typeDic).map(([type, value]) => ({ - title: `${value.title}动态`, - source: [`${type}.neea.edu.cn`, `${type}.neea.cn`], - target: `/local/${type}`, - })), + radar: Object.entries(typeDic).flatMap(([type, value]) => [ + { + title: `${value.title}动态`, + source: [`${type}.neea.edu.cn`], + target: `/local/${type}`, + }, + { + title: `${value.title}动态`, + source: [`${type}.neea.cn`], + target: `/local/${type}`, + }, + ]), handler, description: `| | 考试项目 | type | | ------------ | ----------------------------- | -------- | diff --git a/lib/routes/neea/jlpt.ts b/lib/routes/neea/jlpt.ts index e43520e35c4c92..ee42e625b88ecb 100644 --- a/lib/routes/neea/jlpt.ts +++ b/lib/routes/neea/jlpt.ts @@ -19,9 +19,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.indexcontent a') + let items: DataItem[] = $('div.indexcontent a') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/netflix/newsroom.ts b/lib/routes/netflix/newsroom.ts index 5799099d7730b8..be552e4cf3161f 100644 --- a/lib/routes/netflix/newsroom.ts +++ b/lib/routes/netflix/newsroom.ts @@ -49,7 +49,10 @@ export const route: Route = { }, radar: [ { - source: ['about.netflix.com/:region/newsroom', 'netflix.com'], + source: ['about.netflix.com/:region/newsroom'], + }, + { + source: ['netflix.com'], }, ], name: 'Newsroom', diff --git a/lib/routes/nikkei/cn/index.ts b/lib/routes/nikkei/cn/index.ts index 4489347f71d67b..36defa883fa112 100644 --- a/lib/routes/nikkei/cn/index.ts +++ b/lib/routes/nikkei/cn/index.ts @@ -2,7 +2,7 @@ import { load } from 'cheerio'; import Parser from 'rss-parser'; import { config } from '@/config'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import { getSubPath } from '@/utils/common-utils'; import got from '@/utils/got'; @@ -63,7 +63,7 @@ export const route: Route = { }; async function handler(ctx) { - let language = ''; + let language: string; let path = getSubPath(ctx); if (/^\/cn\/(cn|zh)/.test(path)) { @@ -81,8 +81,8 @@ async function handler(ctx) { let officialFeed; - let items = [], - $; + let items: DataItem[]; + let $; if (isOfficialRSS) { officialFeed = await parser.parseURL(currentUrl); diff --git a/lib/routes/nikkei/news.tsx b/lib/routes/nikkei/news.tsx index 701638da3b4d68..ef28bd5441c07d 100644 --- a/lib/routes/nikkei/news.tsx +++ b/lib/routes/nikkei/news.tsx @@ -29,14 +29,13 @@ export const route: Route = { async function handler(ctx) { const baseUrl = 'https://www.nikkei.com'; const { category, article_type = 'paid' } = ctx.req.param(); - let url = ''; - url = category === 'news' ? `${baseUrl}/news/category/` : `${baseUrl}/${category}/archive/`; + const url = category === 'news' ? `${baseUrl}/news/category/` : `${baseUrl}/${category}/archive/`; const response = await got(url); const data = response.data; const $ = load(data); - let categoryName = ''; + let categoryName: string; const listSelector = $('[class^="container_"] [class^="default_"]:has(article)'); const paidSelector = 'img[class^="icon_"]'; diff --git a/lib/routes/nintendo/utils.ts b/lib/routes/nintendo/utils.ts index e515fdb805083c..ef570d1e68acec 100644 --- a/lib/routes/nintendo/utils.ts +++ b/lib/routes/nintendo/utils.ts @@ -13,7 +13,7 @@ import { renderEshopCnDescription } from './templates/eshop-cn'; dayjs.extend(localizedFormat); function nuxtReader(data) { - let nuxt = {}; + let nuxt: Record; try { const dom = new JSDOM(data, { runScripts: 'dangerously', diff --git a/lib/routes/notefolio/search.tsx b/lib/routes/notefolio/search.tsx index 6cf26f560b99d8..9388ed96b3b48a 100644 --- a/lib/routes/notefolio/search.tsx +++ b/lib/routes/notefolio/search.tsx @@ -182,7 +182,7 @@ async function handler(ctx) { } // 时间范围 if (time !== 'all' && ['one-day', 'week', 'month', 'three-month'].includes(time)) { - let startTime = ''; + let startTime: string; const endTime = dayjs().endOf('d').format('YYYY-MM-DDTHH:mm:ss.SSS'); // 过去24小时-day 最近一周-week 最近一个月-month 最近三个月three-month diff --git a/lib/routes/nowcoder/hots.ts b/lib/routes/nowcoder/hots.ts index 7160f82f7c6fa5..14b1366f03c58d 100644 --- a/lib/routes/nowcoder/hots.ts +++ b/lib/routes/nowcoder/hots.ts @@ -33,7 +33,7 @@ async function handler(ctx) { const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10); const size = Number.isFinite(limit) && limit > 0 ? limit : 20; - let link = ''; + let link: string; if (type === '1') { link = `https://gw-c.nowcoder.com/api/sparta/subject/hot-subject?limit=${size}&_=${Date.now()}&t=`; const responseBody = (await got.get(link)).data; diff --git a/lib/routes/nyc/mayors-office-news.ts b/lib/routes/nyc/mayors-office-news.ts index af1b5782e946fb..159592d6535872 100644 --- a/lib/routes/nyc/mayors-office-news.ts +++ b/lib/routes/nyc/mayors-office-news.ts @@ -117,7 +117,7 @@ Categories } // Description - let description = ''; + let description: string; description = types ? toTitleCase(cleanedTypes) : 'News'; if (categories) { diff --git a/lib/routes/oncc/money18.ts b/lib/routes/oncc/money18.ts index 391506af7bb613..369add4d239a8c 100644 --- a/lib/routes/oncc/money18.ts +++ b/lib/routes/oncc/money18.ts @@ -1,7 +1,7 @@ import { load } from 'cheerio'; import dayjs from 'dayjs'; -import type { Route } from '@/types'; +import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; @@ -54,11 +54,11 @@ async function handler(ctx) { const toApiUrl = (date) => `${rootUrl}/cnt/utf8/content/${date}/articleList/list_${id}_all.js`; - let apiUrl = id === 'ipo' ? ipoApiUrl : id === 'industry' ? industryApiUrl : toApiUrl(dayjs().format('YYYYMMDD')), - hasArticle = false, - items = [], - i = 0, - response; + let apiUrl = id === 'ipo' ? ipoApiUrl : id === 'industry' ? industryApiUrl : toApiUrl(dayjs().format('YYYYMMDD')); + let hasArticle = false; + let items: DataItem[]; + let i = 0; + let response; /* eslint-disable no-await-in-loop */ diff --git a/lib/routes/openai/common.tsx b/lib/routes/openai/common.tsx index b3857d6d665aae..0246baee689f75 100644 --- a/lib/routes/openai/common.tsx +++ b/lib/routes/openai/common.tsx @@ -1,19 +1,19 @@ import { load } from 'cheerio'; -import { raw } from 'hono/html'; -import { renderToString } from 'hono/jsx/dom/server'; import { config } from '@/config'; import type { DataItem } from '@/types'; import cache from '@/utils/cache'; -import got from '@/utils/got'; import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; export const BASE_URL = new URL('https://openai.com'); /** Fetch the details of an article. */ export const fetchArticleDetails = async (url: string) => { - const page = await ofetch(url); - const $ = load(page); + // Ensure trailing slash to avoid 301 redirect + const normalizedUrl = url.endsWith('/') ? url : `${url}/`; + const html = await ofetch(normalizedUrl, { responseType: 'text' }); + const $ = load(html); const $article = $('#main article'); @@ -23,6 +23,10 @@ export const fetchArticleDetails = async (url: string) => { .toArray() .map((element) => $(element).text()); + const authors = $('[data-testid="author-list"] a') + .toArray() + .map((element) => $(element).text()); + // Article header (title, sub title and categories) $($article.find('h1').parents().get(4)).remove(); // Related articles (can be the #citations section in some cases, so the last child needs to be removed first) @@ -35,11 +39,13 @@ export const fetchArticleDetails = async (url: string) => { // Categories can be found on https://openai.com/news/ and https://openai.com/research/index/ categories, image: $('meta[property="og:image"]').attr('content'), + author: authors.join(', ') || undefined, + link: normalizedUrl, }; }; /** Fetch all articles from OpenAI's RSS feed. */ -export const fetchArticles = async (limit: number): Promise => { +export const fetchArticles = async (limit: number, category?: string): Promise => { const page = await ofetch('https://openai.com/news/rss.xml', { responseType: 'text', headers: { 'User-Agent': config.ua }, @@ -47,95 +53,32 @@ export const fetchArticles = async (limit: number): Promise => { const $ = load(page, { xml: true }); - return Promise.all( - $('item') - .toArray() - .slice(0, limit) - .map>((element) => { - const id = $(element).find('guid').text(); - - return cache.tryGet(`openai:news:${id}`, async () => { - const title = $(element).find('title').text(); - const pubDate = $(element).find('pubDate').text(); - const link = $(element).find('link').text(); - - const { content, categories } = await fetchArticleDetails(link); + let items = $('item').toArray(); + if (category) { + items = items.filter((element) => $(element).find('category').text() === category); + } - return { - guid: id, - title, - link, - pubDate, - description: content, - category: categories, - } as DataItem; - }) as Promise; - }) + return Promise.all( + items.slice(0, limit).map>((element) => { + const id = $(element).find('guid').text(); + + return cache.tryGet(`openai:news:${id}`, async () => { + const title = $(element).find('title').text(); + const pubDate = parseDate($(element).find('pubDate').text()); + const link = $(element).find('link').text(); + + const { content, categories, author, link: articleLink } = await fetchArticleDetails(link); + + return { + guid: id, + title, + link: articleLink, + pubDate, + description: content, + category: categories, + author, + } as DataItem; + }) as Promise; + }) ); }; - -const getApiUrl = async () => { - const blogRootUrl = 'https://openai.com/blog'; - - // Find API base URL - const initResponse = await got({ - method: 'get', - url: blogRootUrl, - }); - - const apiBaseUrl = initResponse.data - .toString() - .match(/(?<=TWILL_API_BASE:").+?(?=")/)[0] - .replaceAll(String.raw`\u002F`, '/'); - - return new URL(apiBaseUrl); -}; - -const parseArticle = (ctx, rootUrl, attributes) => - cache.tryGet(attributes.slug, async () => { - const textUrl = `${rootUrl}/${attributes.slug}`; - const detailResponse = await got({ - method: 'get', - url: textUrl, - }); - let content = load(detailResponse.data); - - const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span') - .toArray() - .map((entry) => content(entry).text()) - .join(', '); - - // Leave out comments - const comments = content('*') - .contents() - .filter(function () { - return this.nodeType === 8; - }); - comments.remove(); - - content = content('#content'); - - const imageSrc = attributes.seo.ogImageSrc; - const imageAlt = attributes.seo.ogImageAlt; - - const article = renderToString( - <> - {imageAlt - {raw(content.toString())} - - ); - - // Not all article has tags - attributes.tags = attributes.tags || []; - - return { - title: attributes.title, - author: authors, - description: article, - pubDate: attributes.createdAt, - category: attributes.tags.map((tag) => tag.title), - link: textUrl, - }; - }); - -export { getApiUrl, parseArticle }; diff --git a/lib/routes/openai/research.ts b/lib/routes/openai/research.ts index cedcf09982d9a0..b4a2cd43ab0b41 100644 --- a/lib/routes/openai/research.ts +++ b/lib/routes/openai/research.ts @@ -1,7 +1,8 @@ +import type { Context } from 'hono'; + import type { Route } from '@/types'; -import got from '@/utils/got'; -import { getApiUrl, parseArticle } from './common'; +import { BASE_URL, fetchArticles } from './common'; export const route: Route = { path: '/research', @@ -17,36 +18,17 @@ export const route: Route = { supportScihub: false, }, name: 'Research', - maintainers: ['yuguorui'], + maintainers: ['yuguorui', 'chesha1'], handler, }; -async function handler(ctx) { - const apiUrl = new URL('/api/v1/research-publications', await getApiUrl()); - const researchRootUrl = 'https://openai.com/research'; - - // Construct API query - apiUrl.searchParams.append('sort', '-publicationDate,-createdAt'); - apiUrl.searchParams.append('include', 'media'); - - const resp = await got({ - method: 'get', - url: apiUrl, - }); - const obj = resp.data; - - const items = await Promise.all( - obj.data.map((item) => { - const attributes = item.attributes; - return parseArticle(ctx, researchRootUrl, attributes); - }) - ); - - const title = 'OpenAI Research'; +async function handler(ctx: Context) { + const limit = Number.parseInt(ctx.req.query('limit') || '10'); + const link = new URL('/research/index', BASE_URL).href; return { - title, - link: researchRootUrl, - item: items, + title: 'OpenAI Research', + link, + item: await fetchArticles(limit, 'Research'), }; } diff --git a/lib/routes/ornl/all-news.ts b/lib/routes/ornl/all-news.ts index cd0ea8b6822c3f..63d4bd566b7ec2 100644 --- a/lib/routes/ornl/all-news.ts +++ b/lib/routes/ornl/all-news.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.view-rows-main div.list-item-wrapper') + let items: DataItem[] = $('div.view-rows-main div.list-item-wrapper') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/oschina/column.ts b/lib/routes/oschina/column.ts index e3d78545fe2149..de8ce4b51977f2 100644 --- a/lib/routes/oschina/column.ts +++ b/lib/routes/oschina/column.ts @@ -24,9 +24,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language: string = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.news-item') + let items: DataItem[] = $('div.news-item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/oschina/event.ts b/lib/routes/oschina/event.ts index 13f8e9e852979f..65eab66a26e15a 100644 --- a/lib/routes/oschina/event.ts +++ b/lib/routes/oschina/event.ts @@ -31,9 +31,7 @@ export const handler = async (ctx: Context): Promise => { const $target: CheerioAPI = load(targetResponse); const language = $target('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('div.event-item') + let items: DataItem[] = $('div.event-item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/oshwhub/explore.ts b/lib/routes/oshwhub/explore.ts index d97c50b687cde9..19776ea9e1859b 100644 --- a/lib/routes/oshwhub/explore.ts +++ b/lib/routes/oshwhub/explore.ts @@ -70,9 +70,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = response.result.lists.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.result.lists.slice(0, limit).map((item): DataItem => { const title: string = item.name; const image: string | undefined = item.thumb?.startsWith('https:') ? item.thumb : `https:${item.thumb}`; const description: string | undefined = renderDescription({ diff --git a/lib/routes/ouc/hqsz.ts b/lib/routes/ouc/hqsz.ts index 233b44b81edb71..b22f58289729ba 100644 --- a/lib/routes/ouc/hqsz.ts +++ b/lib/routes/ouc/hqsz.ts @@ -1,4 +1,4 @@ -import CryptoJS from 'crypto-js/crypto-js'; +import CryptoJS from 'crypto-js'; import type { Route } from '@/types'; import cache from '@/utils/cache'; diff --git a/lib/routes/panewslab/profundity.ts b/lib/routes/panewslab/profundity.ts index 7035955ab47ff3..02d6ba9a7975b4 100644 --- a/lib/routes/panewslab/profundity.ts +++ b/lib/routes/panewslab/profundity.ts @@ -27,7 +27,10 @@ export const route: Route = { parameters: { category: '分类，见下表，默认为精选' }, radar: [ { - source: ['panewslab.com/', 'www.panewslab.com/zh/profundity/index.html'], + source: ['panewslab.com/'], + }, + { + source: ['www.panewslab.com/zh/profundity/index.html'], }, ], name: '深度', diff --git a/lib/routes/picnob.info/user.ts b/lib/routes/picnob.info/user.ts index d9fc359b991a64..f8ac27872201c0 100644 --- a/lib/routes/picnob.info/user.ts +++ b/lib/routes/picnob.info/user.ts @@ -6,6 +6,7 @@ import { ViewType } from '@/types'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +import wait from '@/utils/wait'; import type { Post, Profile, Pull, Status, Story } from './types'; @@ -42,7 +43,7 @@ export const route: Route = { const renderVideo = (video, poster) => ``; const renderImage = (src) => ``; const renderDescription = (type: Post['postType'], item: Post | Story) => { - let media = ''; + let media: string; switch (type) { case 'carousel': media = item.albumItems @@ -105,7 +106,7 @@ async function handler(ctx) { } if (attempt < 9) { // eslint-disable-next-line no-await-in-loop - await new Promise((resolve) => setTimeout(resolve, 3000)); + await wait(3000); } } diff --git a/lib/routes/picnob/user.ts b/lib/routes/picnob/user.ts index e76aa0ab1b7640..14a388a42f7484 100644 --- a/lib/routes/picnob/user.ts +++ b/lib/routes/picnob/user.ts @@ -1,51 +1,51 @@ -import { load } from 'cheerio'; -import type { ConnectResult, Options } from 'puppeteer-real-browser'; -import { connect } from 'puppeteer-real-browser'; +// import { load } from 'cheerio'; +// import type { ConnectResult, Options } from 'puppeteer-real-browser'; +// import { connect } from 'puppeteer-real-browser'; -import { config } from '@/config'; +// import { config } from '@/config'; import type { Route } from '@/types'; import { ViewType } from '@/types'; -import cache from '@/utils/cache'; -import { parseRelativeDate } from '@/utils/parse-date'; - -const realBrowserOption: Options = { - args: ['--start-maximized'], - turnstile: true, - headless: false, - // disableXvfb: true, - // ignoreAllFlags:true, - customConfig: { - chromePath: config.chromiumExecutablePath, - }, - connectOption: { - defaultViewport: null, - }, - plugins: [], -}; - -async function getPageWithRealBrowser(url: string, selector: string, conn: ConnectResult | null) { - try { - if (conn) { - const page = conn.page; - await page.goto(url, { timeout: 30000 }); - let verify: boolean | null = null; - const startDate = Date.now(); - while (!verify && Date.now() - startDate < 30000) { - // eslint-disable-next-line no-await-in-loop, no-restricted-syntax - verify = await page.evaluate((sel) => (document.querySelector(sel) ? true : null), selector).catch(() => null); - // eslint-disable-next-line no-await-in-loop - await new Promise((r) => setTimeout(r, 1000)); - } - return await page.content(); - } else { - const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(url)}&selector=${encodeURIComponent(selector)}`); - const json = await res.json(); - return (json.data?.at(0) || '') as string; - } - } catch { - return ''; - } -} +// import cache from '@/utils/cache'; +// import { parseRelativeDate } from '@/utils/parse-date'; + +// const realBrowserOption: Options = { +// args: ['--start-maximized'], +// turnstile: true, +// headless: false, +// // disableXvfb: true, +// // ignoreAllFlags:true, +// customConfig: { +// chromePath: config.chromiumExecutablePath, +// }, +// connectOption: { +// defaultViewport: null, +// }, +// plugins: [], +// }; + +// async function getPageWithRealBrowser(url: string, selector: string, conn: ConnectResult | null) { +// try { +// if (conn) { +// const page = conn.page; +// await page.goto(url, { timeout: 30000 }); +// let verify: boolean | null = null; +// const startDate = Date.now(); +// while (!verify && Date.now() - startDate < 30000) { +// // eslint-disable-next-line no-await-in-loop, no-restricted-syntax +// verify = await page.evaluate((sel) => (document.querySelector(sel) ? true : null), selector).catch(() => null); +// // eslint-disable-next-line no-await-in-loop +// await new Promise((r) => setTimeout(r, 1000)); +// } +// return await page.content(); +// } else { +// const res = await fetch(`${config.puppeteerRealBrowserService}?url=${encodeURIComponent(url)}&selector=${encodeURIComponent(selector)}`); +// const json = await res.json(); +// return (json.data?.at(0) || '') as string; +// } +// } catch { +// return ''; +// } +// } export const route: Route = { path: '/user/:id/:type?', @@ -57,8 +57,8 @@ export const route: Route = { }, features: { requireConfig: false, - requirePuppeteer: true, - antiCrawler: true, + requirePuppeteer: false, + antiCrawler: false, supportBT: false, supportPodcast: false, supportScihub: false, @@ -79,104 +79,108 @@ export const route: Route = { view: ViewType.Pictures, }; -async function handler(ctx) { - if (!config.puppeteerRealBrowserService && !config.chromiumExecutablePath) { - throw new Error('PUPPETEER_REAL_BROWSER_SERVICE or CHROMIUM_EXECUTABLE_PATH is required to use this route.'); - } - - // NOTE: 'picnob' is still available, but all requests to 'picnob' will be redirected to 'pixnoy' eventually - const baseUrl = 'https://www.pixnoy.com'; +function handler(ctx) { const id = ctx.req.param('id'); - const type = ctx.req.param('type') ?? 'profile'; - const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`; - - let conn: ConnectResult | null = null; - - if (!config.puppeteerRealBrowserService) { - conn = await connect(realBrowserOption); - - setTimeout(async () => { - if (conn) { - await conn.browser.close(); - } - }, 60000); - } - - const html = await getPageWithRealBrowser(profileUrl, '.post_box', conn); - if (!html) { - if (conn) { - await conn.browser.close(); - conn = null; - } - throw new Error('Failed to fetch user profile page. User may not exist or there are no posts available.'); - } - - const $ = load(html); - - const list = $('.post_box') - .toArray() - .map((item) => { - const $item = $(item); - const coverLink = $item.find('.cover_link').attr('href'); - const shortcode = coverLink?.split('/')?.[2]; - const image = $item.find('.cover .cover_link img'); - const title = image.attr('alt') || ''; - - return { - title, - description: `
    ${title}`, - link: `${baseUrl}${coverLink}`, - guid: shortcode, - pubDate: parseRelativeDate($item.find('.time .txt').text()), - }; - }); - - const jobs = list.map((item) => cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => await getPageWithRealBrowser(item.link, '.view', conn))); - - let htmlList: string[] = []; - if (conn) { - try { - for (const job of jobs) { - // eslint-disable-next-line no-await-in-loop - const html = await job; - htmlList.push(html); - } - } finally { - await conn.browser.close(); - conn = null; - } - } else { - htmlList = await Promise.all(jobs); - } - - const newDescription = htmlList.map((html) => { - if (!html) { - return ''; - } - const $ = load(html); - if ($('.video_img').length > 0) { - return `
    ${$('.sum_full').text()}`; - } else { - let description = ''; - for (const pic of $('.pic img').toArray()) { - const dataSrc = $(pic).attr('data-src'); - if (dataSrc) { - description += `
    `; - } - } - description += $('.sum_full').text(); - return description; - } - }); - - return { - title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`, - description: $('.info .sum').text(), - link: profileUrl, - image: $('.ava .pic img').attr('src'), - item: list.map((item, index) => ({ - ...item, - description: newDescription[index] || item.description, - })), - }; + return ctx.set('redirect', `/picnob.info/user/${id}`); + + // // Original puppeteer-real-browser implementation (deprecated) + // if (!config.puppeteerRealBrowserService && !config.chromiumExecutablePath) { + // throw new Error('PUPPETEER_REAL_BROWSER_SERVICE or CHROMIUM_EXECUTABLE_PATH is required to use this route.'); + // } + + // // NOTE: 'picnob' is still available, but all requests to 'picnob' will be redirected to 'picnob.info' eventually + // const baseUrl = 'https://www.pixnoy.com'; + // const id = ctx.req.param('id'); + // const type = ctx.req.param('type') ?? 'profile'; + // const profileUrl = `${baseUrl}/profile/${id}/${type === 'tagged' ? 'tagged/' : ''}`; + + // let conn: ConnectResult | null = null; + + // if (!config.puppeteerRealBrowserService) { + // conn = await connect(realBrowserOption); + + // setTimeout(async () => { + // if (conn) { + // await conn.browser.close(); + // } + // }, 60000); + // } + + // const html = await getPageWithRealBrowser(profileUrl, '.post_box', conn); + // if (!html) { + // if (conn) { + // await conn.browser.close(); + // conn = null; + // } + // throw new Error('Failed to fetch user profile page. User may not exist or there are no posts available.'); + // } + + // const $ = load(html); + + // const list = $('.post_box') + // .toArray() + // .map((item) => { + // const $item = $(item); + // const coverLink = $item.find('.cover_link').attr('href'); + // const shortcode = coverLink?.split('/')?.[2]; + // const image = $item.find('.cover .cover_link img'); + // const title = image.attr('alt') || ''; + + // return { + // title, + // description: `
    ${title}`, + // link: `${baseUrl}${coverLink}`, + // guid: shortcode, + // pubDate: parseRelativeDate($item.find('.time .txt').text()), + // }; + // }); + + // const jobs = list.map((item) => cache.tryGet(`picnob:user:${id}:${item.guid}:html`, async () => await getPageWithRealBrowser(item.link, '.view', conn))); + + // let htmlList: string[] = []; + // if (conn) { + // try { + // for (const job of jobs) { + // // eslint-disable-next-line no-await-in-loop + // const html = await job; + // htmlList.push(html); + // } + // } finally { + // await conn.browser.close(); + // conn = null; + // } + // } else { + // htmlList = await Promise.all(jobs); + // } + + // const newDescription = htmlList.map((html) => { + // if (!html) { + // return ''; + // } + // const $ = load(html); + // if ($('.video_img').length > 0) { + // return `
    ${$('.sum_full').text()}`; + // } else { + // let description = ''; + // for (const pic of $('.pic img').toArray()) { + // const dataSrc = $(pic).attr('data-src'); + // if (dataSrc) { + // description += `
    `; + // } + // } + // description += $('.sum_full').text(); + // return description; + // } + // }); + + // return { + // title: `${$('h1.fullname').text()} (@${id}) ${type === 'tagged' ? 'tagged' : 'public'} posts - Picnob`, + // description: $('.info .sum').text(), + // link: profileUrl, + // image: $('.ava .pic img').attr('src'), + // item: list.map((item, index) => ({ + // ...item, + // description: newDescription[index] || item.description, + // })), + // }; } diff --git a/lib/routes/pikabu/utils.ts b/lib/routes/pikabu/utils.ts index db1cc27f5b9e79..c279a2ef76cc75 100644 --- a/lib/routes/pikabu/utils.ts +++ b/lib/routes/pikabu/utils.ts @@ -19,7 +19,7 @@ const fixVideo = (element) => { .attr('style') .match(/url\((.+)\);/)[1]; const dataType = element.attr('data-type'); - let videoHtml = ''; + let videoHtml: string; if (dataType === 'video') { const videoId = element.attr('data-source').match(/\/embed\/(.+)$/)[1]; diff --git a/lib/routes/pingwest/user.ts b/lib/routes/pingwest/user.ts index 0cdd51c9cc565b..3fe00a1a8e6ae9 100644 --- a/lib/routes/pingwest/user.ts +++ b/lib/routes/pingwest/user.ts @@ -71,7 +71,7 @@ async function handler(ctx) { }); const $ = load(response.data.data.list); - let item = []; + let item: DataItem[]; const needFullText = option === 'fulltext'; switch (type) { case 'article': diff --git a/lib/routes/pixelstech/index.ts b/lib/routes/pixelstech/index.ts index 8fa99f48145d34..254566ad4fb36d 100644 --- a/lib/routes/pixelstech/index.ts +++ b/lib/routes/pixelstech/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.feed-item') + let items: DataItem[] = $('div.feed-item') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/pixiv/novel-api/content/utils.ts b/lib/routes/pixiv/novel-api/content/utils.ts index b440798dd3ad1d..ef5ec5629f1f52 100644 --- a/lib/routes/pixiv/novel-api/content/utils.ts +++ b/lib/routes/pixiv/novel-api/content/utils.ts @@ -131,6 +131,6 @@ export async function parseNovelContent(content: string, images: Record parseItems($(e))); + .map((e) => parseItems($(e), showImages)); return { title: $('title').first().text(), link, - language: $('html').attr('lang'), + language: $('html').attr('lang') as any, item: items, }; } diff --git a/lib/routes/pornhub/category.ts b/lib/routes/pornhub/category.ts index 46d1103e7b41e0..bb671e3a4e99aa 100644 --- a/lib/routes/pornhub/category.ts +++ b/lib/routes/pornhub/category.ts @@ -8,11 +8,11 @@ import { parseDate } from '@/utils/parse-date'; import { defaultDomain, renderDescription } from './utils'; export const route: Route = { - path: '/category/:caty', + path: '/category/:caty/:img?', categories: ['multimedia'], view: ViewType.Videos, example: '/pornhub/category/popular-with-women', - parameters: { caty: 'category, see [categories](https://www.pornhub.com/webmasters/categories)' }, + parameters: { caty: 'category, see [categories](https://www.pornhub.com/webmasters/categories)', img: 'show images, set to `img=1` to enable' }, features: { requireConfig: false, requirePuppeteer: false, @@ -28,7 +28,7 @@ export const route: Route = { }; async function handler(ctx) { - const category = ctx.req.param('caty'); + const { caty: category, img } = ctx.req.param(); const categories = await cache.tryGet('pornhub:categories', async () => { const { data } = await got(`${defaultDomain}/webmasters/categories`); @@ -52,10 +52,12 @@ async function handler(ctx) { throw new Error(response.message); } + const showImages = img === 'img=1'; + const list = response.videos.map((item) => ({ title: item.title, link: item.url, - description: renderDescription({ thumbs: item.thumbs }), + description: renderDescription({ thumbs: item.thumbs }, showImages), pubDate: parseDate(item.publish_date), category: [...new Set([...item.tags.map((t) => t.tag_name), ...item.categories.map((c) => c.category)])], })); diff --git a/lib/routes/pornhub/model.ts b/lib/routes/pornhub/model.ts index cb53a52c4cbb11..a55ea8ff9910d1 100644 --- a/lib/routes/pornhub/model.ts +++ b/lib/routes/pornhub/model.ts @@ -9,11 +9,16 @@ import { isValidHost } from '@/utils/valid-host'; import { getRadarDomin, headers, parseItems } from './utils'; export const route: Route = { - path: '/model/:username/:language?/:sort?', + path: '/model/:username/:language?/:sort?/:img?', categories: ['multimedia'], view: ViewType.Videos, example: '/pornhub/model/stacy-starando', - parameters: { language: 'language, see below', username: 'username, part of the url e.g. `pornhub.com/model/stacy-starando`', sort: 'sorting method, see below' }, + parameters: { + language: 'language, see below. defaults to www', + username: 'username, part of the url e.g. `pornhub.com/model/stacy-starando`', + sort: 'sorting method, see below. Defaults to mr (most recent)', + img: 'show images, set to `img=1` to enable', + }, features: { requireConfig: false, requirePuppeteer: false, @@ -30,7 +35,7 @@ export const route: Route = { }; async function handler(ctx): Promise { - const { language = 'www', username, sort = '' } = ctx.req.param(); + const { language = 'www', username, sort = '', img } = ctx.req.param(); const link = `https://${language}.pornhub.com/model/${username}/videos${sort ? `?o=${sort}` : ''}`; if (!isValidHost(language)) { throw new InvalidParameterError('Invalid language'); @@ -38,16 +43,17 @@ async function handler(ctx): Promise { const { data: response } = await got(link, { headers }); const $ = load(response); + const showImages = img === 'img=1'; const items = $('#mostRecentVideosSection .videoBox') .toArray() - .map((e) => parseItems($(e))); + .map((e) => parseItems($(e), showImages)); return { title: $('h1').first().text(), description: $('section.aboutMeSection').text().trim(), link, image: $('#getAvatar').attr('src'), - language: $('html').attr('lang'), + language: $('html').attr('lang') as any, item: items, }; } diff --git a/lib/routes/pornhub/pornstar.ts b/lib/routes/pornhub/pornstar.ts index 06489745e856a6..6c83d84861ce88 100644 --- a/lib/routes/pornhub/pornstar.ts +++ b/lib/routes/pornhub/pornstar.ts @@ -9,7 +9,7 @@ import { isValidHost } from '@/utils/valid-host'; import { getRadarDomin, headers, parseItems } from './utils'; export const route: Route = { - path: '/pornstar/:username/:language?/:sort?', + path: '/pornstar/:username/:language?/:sort?/:img?', categories: ['multimedia'], view: ViewType.Videos, example: '/pornhub/pornstar/june-liu/www/mr', @@ -18,7 +18,7 @@ export const route: Route = { description: 'username, part of the url e.g. `pornhub.com/pornstar/june-liu`', }, language: { - description: 'language', + description: 'language, defaults to `www` (English)', options: [ { value: 'www', label: 'English' }, { value: 'de', label: 'Deutsch' }, @@ -36,7 +36,7 @@ export const route: Route = { default: 'www', }, sort: { - description: 'sorting method, leave empty for `Best`', + description: 'sorting method, defaults to `mr` (Most Recent)', options: [ { label: 'Most Recent', @@ -56,6 +56,7 @@ export const route: Route = { }, ], }, + img: 'show images, set to `img=1` to enable', }, features: { requireConfig: false, @@ -73,7 +74,7 @@ export const route: Route = { }; async function handler(ctx): Promise { - const { language = 'www', username, sort = 'mr' } = ctx.req.param(); + const { language = 'www', username, sort = 'mr', img } = ctx.req.param(); let link = `https://${language}.pornhub.com/pornstar/${username}?o=${sort}`; if (!isValidHost(language)) { throw new InvalidParameterError('Invalid language'); @@ -83,17 +84,19 @@ async function handler(ctx): Promise { let $ = load(response); let items; + const showImages = img === 'img=1'; + if ($('.withBio').length === 0) { link = `https://${language}.pornhub.com/pornstar/${username}/videos?o=${sort}`; const { data: response } = await got(link, { headers }); $ = load(response); items = $('#mostRecentVideosSection .videoBox') .toArray() - .map((e) => parseItems($(e))); + .map((e) => parseItems($(e), showImages)); } else { items = $('#pornstarsVideoSection .videoBox') .toArray() - .map((e) => parseItems($(e))); + .map((e) => parseItems($(e), showImages)); } return { @@ -101,7 +104,7 @@ async function handler(ctx): Promise { description: $('section.aboutMeSection').text().trim(), link, image: $('#getAvatar').attr('src'), - language: $('html').attr('lang'), + language: $('html').attr('lang') as any, item: items, }; } diff --git a/lib/routes/pornhub/search.ts b/lib/routes/pornhub/search.ts index 62f2ebcf0107e6..79e22f46855ba6 100644 --- a/lib/routes/pornhub/search.ts +++ b/lib/routes/pornhub/search.ts @@ -6,11 +6,11 @@ import { parseDate } from '@/utils/parse-date'; import { defaultDomain, renderDescription } from './utils'; export const route: Route = { - path: '/search/:keyword', + path: '/search/:keyword/:img?', categories: ['multimedia'], view: ViewType.Videos, example: '/pornhub/search/stepsister', - parameters: { keyword: 'keyword' }, + parameters: { keyword: 'keyword', img: 'show images, set to `img=1` to enable' }, features: { requireConfig: false, requirePuppeteer: false, @@ -26,14 +26,16 @@ export const route: Route = { }; async function handler(ctx) { - const keyword = ctx.req.param('keyword'); + const { keyword, img } = ctx.req.param(); const currentUrl = `${defaultDomain}/webmasters/search?search=${keyword}`; const response = await got(currentUrl); + const showImages = img === 'img=1'; + const list = response.data.videos.map((item) => ({ title: item.title, link: item.url, - description: renderDescription({ thumbs: item.thumbs }), + description: renderDescription({ thumbs: item.thumbs }, showImages), pubDate: parseDate(item.publish_date), category: [...new Set([...item.tags.map((t) => t.tag_name), ...item.categories.map((c) => c.category)])], })); diff --git a/lib/routes/pornhub/users.ts b/lib/routes/pornhub/users.ts index b14991ac5250ec..03426ac34ca05a 100644 --- a/lib/routes/pornhub/users.ts +++ b/lib/routes/pornhub/users.ts @@ -8,10 +8,10 @@ import { isValidHost } from '@/utils/valid-host'; import { getRadarDomin, headers, parseItems } from './utils'; export const route: Route = { - path: '/users/:username/:language?', + path: '/users/:username/:language?/:img?', categories: ['multimedia'], example: '/pornhub/users/pornhubmodels', - parameters: { language: 'language, see below', username: 'username, part of the url e.g. `pornhub.com/users/pornhubmodels`' }, + parameters: { language: 'language, see below. defaults to `www` (English)', username: 'username, part of the url e.g. `pornhub.com/users/pornhubmodels`', img: 'show images, set to `img=1` to enable' }, features: { requireConfig: false, requirePuppeteer: false, @@ -28,7 +28,7 @@ export const route: Route = { }; async function handler(ctx): Promise { - const { language = 'www', username } = ctx.req.param(); + const { language = 'www', username, img } = ctx.req.param(); const link = `https://${language}.pornhub.com/users/${username}/videos`; if (!isValidHost(language)) { throw new InvalidParameterError('Invalid language'); @@ -36,16 +36,17 @@ async function handler(ctx): Promise { const { data: response } = await got(link, { headers }); const $ = load(response); + const showImages = img === 'img=1'; const items = $('.videoUList .videoBox') .toArray() - .map((e) => parseItems($(e))); + .map((e) => parseItems($(e), showImages)); return { title: $('.profileUserName a').text(), description: $('.aboutMeText').text().trim(), link, image: $('#getAvatar').attr('src'), - language: $('html').attr('lang'), + language: $('html').attr('lang') as any, allowEmpty: true, item: items, }; diff --git a/lib/routes/pornhub/utils.tsx b/lib/routes/pornhub/utils.tsx index 51607f38054cb5..3dd28016ec3edc 100644 --- a/lib/routes/pornhub/utils.tsx +++ b/lib/routes/pornhub/utils.tsx @@ -10,7 +10,7 @@ const headers = { hasVisited: 1, }; -const renderDescription = (data): string => +const renderDescription = (data, showImages = false): string => renderToString( <> {data.previewVideo ? ( @@ -18,9 +18,7 @@ const renderDescription = (data): string => ) : null} - {data.thumbs?.map((thumb, index) => ( - - ))} + {showImages && (data.thumbs ? data.thumbs.map((thumb, index) => ) : )} ); const extractDateFromImageUrl = (imageUrl) => { @@ -28,13 +26,16 @@ const extractDateFromImageUrl = (imageUrl) => { return matchResult ? matchResult.slice(1, 3).join('') : null; }; -const parseItems = (e) => ({ +const parseItems = (e, showImages = false) => ({ title: e.find('span.title a').text().trim(), link: defaultDomain + e.find('span.title a').attr('href'), - description: renderDescription({ - poster: e.find('img').data('mediumthumb'), - previewVideo: e.find('img').data('mediabook'), - }), + description: renderDescription( + { + poster: e.find('img').data('mediumthumb'), + previewVideo: e.find('img').data('mediabook'), + }, + showImages + ), author: e.find('.usernameWrap a').text(), pubDate: dayjs(extractDateFromImageUrl(e.find('img').data('mediumthumb'))).toDate() || parseRelativeDate(e.find('.added').text()), }); diff --git a/lib/routes/producereport/index.ts b/lib/routes/producereport/index.ts index ead272f1d52bd3..0d7b2890ce0ebc 100644 --- a/lib/routes/producereport/index.ts +++ b/lib/routes/producereport/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('table.views-table tbody tr') + let items: DataItem[] = $('table.views-table tbody tr') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/psyche/utils.ts b/lib/routes/psyche/utils.ts index 08f35f6b045ef5..c8dc72f17b104a 100644 --- a/lib/routes/psyche/utils.ts +++ b/lib/routes/psyche/utils.ts @@ -26,8 +26,8 @@ function format(article) { const type = article.type.toLowerCase(); let block = ''; - let banner = ''; - let authorsBio = ''; + let banner: string; + let authorsBio: string; switch (type) { case 'film': @@ -89,8 +89,7 @@ const getData = async (list) => { }) ); - let authors = ''; - authors = article.type === 'film' ? article.creditsShort : article.authors.map((author) => author.name).join(', '); + const authors: string = article.type === 'film' ? article.creditsShort : article.authors.map((author) => author.name).join(', '); item.description = capture.html(); item.author = authors; diff --git a/lib/routes/python/release.ts b/lib/routes/python/release.ts index d34555abbccd21..e55b1045337fea 100644 --- a/lib/routes/python/release.ts +++ b/lib/routes/python/release.ts @@ -19,9 +19,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.active-release-list-widget ol.list-row-container li') + let items: DataItem[] = $('div.active-release-list-widget ol.list-row-container li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/qdu/houqin.ts b/lib/routes/qdu/houqin.ts index 2303edb7d2a5e9..d8b3adf746645b 100644 --- a/lib/routes/qdu/houqin.ts +++ b/lib/routes/qdu/houqin.ts @@ -48,7 +48,6 @@ async function handler() { const path = item.find('a').attr('href'); const itemUrl = base + path; return cache.tryGet(itemUrl, async () => { - let description = ''; const result = await got(itemUrl); const $ = load(result.data); if ( @@ -69,7 +68,7 @@ async function handler() { 8 ); } - description = $('.v_news_content').html().trim(); + const description = $('.v_news_content').html()?.trim(); return { title: itemTitle, diff --git a/lib/routes/qdu/jwc.ts b/lib/routes/qdu/jwc.ts index b79f98abdfad83..9c9714efd0d0a9 100644 --- a/lib/routes/qdu/jwc.ts +++ b/lib/routes/qdu/jwc.ts @@ -49,7 +49,7 @@ async function handler() { let itemUrl = ''; itemUrl = path.startsWith('http') ? path : base + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; if (path.startsWith('http')) { description = itemTitle; } else { diff --git a/lib/routes/qlu/notice.ts b/lib/routes/qlu/notice.ts index 7609557929ae50..476efdc6ea9820 100644 --- a/lib/routes/qlu/notice.ts +++ b/lib/routes/qlu/notice.ts @@ -49,7 +49,7 @@ async function handler() { const path = item.find('.news_title').children().attr('href'); const itemUrl = path.startsWith('https') ? path : host + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; if (path.startsWith('https')) { description = itemTitle; } else { diff --git a/lib/routes/qq/kg/reply.ts b/lib/routes/qq/kg/reply.ts index 70dd6ea39c73b0..91fdcedd88b5f3 100644 --- a/lib/routes/qq/kg/reply.ts +++ b/lib/routes/qq/kg/reply.ts @@ -16,7 +16,7 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, - name: '用户作品评论动态', + name: '全民K歌 - 用户作品评论动态', maintainers: ['zhangxiang012'], handler, }; diff --git a/lib/routes/qq/kg/user.ts b/lib/routes/qq/kg/user.ts index e42812c2103d1e..44c7ec22a9f40f 100644 --- a/lib/routes/qq/kg/user.ts +++ b/lib/routes/qq/kg/user.ts @@ -19,7 +19,7 @@ export const route: Route = { supportPodcast: true, supportScihub: false, }, - name: '用户作品列表', + name: '全民K歌 - 用户作品列表', maintainers: ['zhangxiang012'], handler, }; diff --git a/lib/routes/qq/lol/news.ts b/lib/routes/qq/lol/news.ts index 80dcc36b5a4fdf..7bec8ca1b09650 100644 --- a/lib/routes/qq/lol/news.ts +++ b/lib/routes/qq/lol/news.ts @@ -34,9 +34,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(iconv.decode(Buffer.from(targetResponse), 'gbk')); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = response.data.result.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.result.slice(0, limit).map((item): DataItem => { const title: string = item.sTitle; const pubDate: number | string = item.sCreated; const linkUrl: string | undefined = item.iDocID ? `${item.iVideoId ? 'v/v2' : 'news'}/detail.shtml?docid=${item.iDocID}` : undefined; diff --git a/lib/routes/qq/news/user.ts b/lib/routes/qq/news/user.ts new file mode 100644 index 00000000000000..191542514e2a41 --- /dev/null +++ b/lib/routes/qq/news/user.ts @@ -0,0 +1,89 @@ +import { load } from 'cheerio'; + +import type { DataItem, Route } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +interface NewsItem { + id: string; + uinnick: string; + articletype: string; + longtitle: string; + url: string; + timestamp: number; + abstract: string; + miniProShareImage: string; +} + +export const route: Route = { + path: '/news/:uid/:detail?', + categories: ['social-media'], + example: '/qq/news/8QMZ2X5a5YUeujw=', + parameters: { + uid: '用户 ID, 用户主页 URL 中的最后一段部分', + detail: '是否抓取全文，该值只要不为空就抓取全文返回，否则只返回摘要', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: true, + supportScihub: false, + }, + radar: [ + { + source: ['news.qq.com/omn/author/:uid'], + target: '/qq/news/:uid', + }, + ], + name: '用户主页列表', + maintainers: ['hualiong'], + handler, +}; + +async function handler(ctx) { + const { uid, detail } = ctx.req.param(); + const url = `https://i.news.qq.com/getSubNewsMixedList?guestSuid=${uid}&tabId=om_index`; + const response = await ofetch<{ ret: number; newslist: NewsItem[] }>(url); + + let news = response.newslist.map( + (item) => + ({ + title: item.longtitle, + description: `

    ${item.abstract}

    `, + guid: item.id, + link: item.url, + author: item.uinnick, + pubDate: parseDate(item.timestamp * 1000), + }) satisfies DataItem + ); + + if (detail) { + news = await Promise.all( + response.newslist.map((item) => + cache.tryGet(item.id, async () => { + const description = + item.articletype === '0' + ? load(await ofetch(`https://news.qq.com/rain/a/${item.id}`))('.rich_media_content').html()! + : `

    ${item.abstract}

    文章包含非文本内容，请在浏览器中打开查看

    `; + return { + title: item.longtitle, + description, + guid: item.id, + link: item.url, + author: item.uinnick, + pubDate: parseDate(item.timestamp * 1000), + } satisfies DataItem; + }) + ) + ); + } + + return { + title: `${response.newslist[0].uinnick}的主页 - 腾讯网`, + link: `https://news.qq.com/omn/author/${uid}`, + item: news, + }; +} diff --git a/lib/routes/qq/pd/guild.ts b/lib/routes/qq/pd/guild.ts index 9bbfddcf74fecf..a56733d7bb89e6 100644 --- a/lib/routes/qq/pd/guild.ts +++ b/lib/routes/qq/pd/guild.ts @@ -56,7 +56,7 @@ async function handler(ctx: Context): Promise { } const sortType = sortMap[sort]; - let url = ''; + let url: string; let body = {}; let headers = {}; diff --git a/lib/routes/qq/pd/utils.ts b/lib/routes/qq/pd/utils.ts index 988b1b5c0dc29a..9ebdf4abc1fcc1 100644 --- a/lib/routes/qq/pd/utils.ts +++ b/lib/routes/qq/pd/utils.ts @@ -37,7 +37,7 @@ function parseText(text: string, props: FeedFontProps | undefined): string { } function parseDataItem(item: FeedPatternData, texts: string[], images: { [id: string]: FeedImage }): string { - let imageId = ''; + let imageId: string; switch (patternTypeMap[item.type] || undefined) { case 'text': return parseText(texts.shift() ?? '', item.props); diff --git a/lib/routes/radio/album.ts b/lib/routes/radio/album.ts index 4d8113da95f8fa..bd0b6523ef8510 100644 --- a/lib/routes/radio/album.ts +++ b/lib/routes/radio/album.ts @@ -73,7 +73,7 @@ async function handler(ctx) { const items = response.con.map((item) => { let enclosure_url = item.playUrlHigh ?? item.playUrlMedium ?? item.playUrlLow ?? item.playUrl; - enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url; + enclosure_url = enclosure_url.endsWith('.m3u8') ? item.downloadUrl : enclosure_url; const fileExt = new URL(enclosure_url).pathname.split('.').pop(); const enclosure_type = fileExt ? `audio/${audio_types[fileExt]}` : ''; diff --git a/lib/routes/radio/zhibo.ts b/lib/routes/radio/zhibo.ts index 9f56c7a14aabea..eca55fe4b6c976 100644 --- a/lib/routes/radio/zhibo.ts +++ b/lib/routes/radio/zhibo.ts @@ -72,7 +72,7 @@ async function handler(ctx) { const items = data.map((item) => { let enclosure_url = item.playUrlHigh ?? item.playUrlLow; - enclosure_url = /\.m3u8$/.test(enclosure_url) ? item.downloadUrl : enclosure_url; + enclosure_url = enclosure_url.endsWith('.m3u8') ? item.downloadUrl : enclosure_url; const file_ext = new URL(enclosure_url).pathname.split('.').pop(); const enclosure_type = file_ext ? `audio/${audio_types[file_ext]}` : ''; diff --git a/lib/routes/raspberrypi/magazine.ts b/lib/routes/raspberrypi/magazine.ts index 666abd77791f86..9490a0d8454e21 100644 --- a/lib/routes/raspberrypi/magazine.ts +++ b/lib/routes/raspberrypi/magazine.ts @@ -21,11 +21,9 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - const author: DataItem['author'] = $('meta[property="og:site_name"]').attr('content'); - items = $('div.o-grid--equal div.o-grid__col') + let items: DataItem[] = $('div.o-grid--equal div.o-grid__col') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/reuters/common.tsx b/lib/routes/reuters/common.tsx index d114c08555682f..02c7062d7392f4 100644 --- a/lib/routes/reuters/common.tsx +++ b/lib/routes/reuters/common.tsx @@ -1,6 +1,7 @@ import { load } from 'cheerio'; import { raw } from 'hono/html'; import { renderToString } from 'hono/jsx/dom/server'; +import type { JSX } from 'hono/jsx/jsx-runtime'; import type { Route } from '@/types'; import { ViewType } from '@/types'; @@ -256,7 +257,7 @@ async function handler(ctx) { guid: e.id, pubDate: parseDate(e.published_time), updated: parseDate(e.updated_time), - author: e.authors.map((e) => e.name).join(', '), + author: e.authors?.map((e) => e.name).join(', '), category: e.kicker.names, description: e.description, })); @@ -328,7 +329,10 @@ async function handler(ctx) { link: `https://www.reuters.com${section_id}`, item: items, }; - } catch { + } catch (error: any) { + if (error?.name !== 'FetchError') { + throw error; + } // Fallback to arc outboundfeeds if API fails const arcUrl = topic ? `https://www.reuters.com/arc/outboundfeeds/v4/mobile/section${section_id}?outputType=json` : `https://www.reuters.com/arc/outboundfeeds/v4/mobile/section/${category}/?outputType=json`; diff --git a/lib/routes/rockthejvm/articles.ts b/lib/routes/rockthejvm/articles.ts index a75dc0df633699..68a2d6df817a4e 100644 --- a/lib/routes/rockthejvm/articles.ts +++ b/lib/routes/rockthejvm/articles.ts @@ -23,9 +23,7 @@ export const handler = async (ctx: Context): Promise => { $('footer').remove(); - let items: DataItem[] = []; - - items = $('h2') + let items: DataItem[] = $('h2') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/samd/news.ts b/lib/routes/samd/news.ts index 3efe7aaa68808e..b98bb944d7612c 100644 --- a/lib/routes/samd/news.ts +++ b/lib/routes/samd/news.ts @@ -51,7 +51,8 @@ export const route: Route = { const items = await Promise.all( list.map((item) => cache.tryGet(item.link!, async () => { - const $ = load(await ofetch(item.link!)); + const html = await ofetch(item.link!); + const $ = load(html); const content = $('.content'); item.author = content.find('.author span').text(); diff --git a/lib/routes/samrdprc/index.ts b/lib/routes/samrdprc/index.ts index 38da5cd8ab5443..eca84611e45174 100644 --- a/lib/routes/samrdprc/index.ts +++ b/lib/routes/samrdprc/index.ts @@ -20,9 +20,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; - - items = $('div.boxl_ul ul li') + let items: DataItem[] = $('div.boxl_ul ul li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/sass/gs/index.ts b/lib/routes/sass/gs/index.ts index c3ea0e9ebf6d3b..4bda148e535df1 100644 --- a/lib/routes/sass/gs/index.ts +++ b/lib/routes/sass/gs/index.ts @@ -48,7 +48,7 @@ async function handler(ctx) { const itemUrl = path.startsWith('http') ? path : host + path; return cache.tryGet(itemUrl, async () => { - let description = ''; + let description: string; if (itemUrl) { const result = await got(itemUrl); const $ = load(result.data); diff --git a/lib/routes/sciencedirect/call-for-paper.tsx b/lib/routes/sciencedirect/call-for-paper.tsx index d43200ef0deda1..f1903b0b46bb10 100644 --- a/lib/routes/sciencedirect/call-for-paper.tsx +++ b/lib/routes/sciencedirect/call-for-paper.tsx @@ -42,7 +42,7 @@ async function handler(ctx) { try { data = JSON.parse(JSON.parse(scriptJSON)); } catch (error: any) { - throw new Error(`Failed to parse embedded script JSON: ${error.message}`); + throw new Error(`Failed to parse embedded script JSON: ${error.message}`, { cause: error }); } const cfpList = data?.callsForPapers?.list || []; diff --git a/lib/routes/sciencedirect/cf-email.ts b/lib/routes/sciencedirect/cf-email.ts index 21af4821b565a6..b193b6a55ead88 100644 --- a/lib/routes/sciencedirect/cf-email.ts +++ b/lib/routes/sciencedirect/cf-email.ts @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-code-point */ const decodeCFEmail = (encoded) => { const parseHex = (string, position) => Number.parseInt(string.slice(position, position + 2), 16); let decoded = ''; diff --git a/lib/routes/scientificamerican/podcast.ts b/lib/routes/scientificamerican/podcast.ts index 6a65d41537b42d..313138f8d61110 100644 --- a/lib/routes/scientificamerican/podcast.ts +++ b/lib/routes/scientificamerican/podcast.ts @@ -22,11 +22,9 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language: string = $('html').attr('lang') ?? 'en'; const data: string | undefined = response.match(/window\.__DATA__=JSON\.parse\(`(.*?)`\)/)?.[1]; - const parsedData = data ? JSON.parse(data.replaceAll('\\\\', '\\')) : undefined; + const parsedData = data ? JSON.parse(data.replaceAll(String.raw`\\`, '\\')) : undefined; - let items: DataItem[] = []; - - items = parsedData + let items: DataItem[] = parsedData ? parsedData.initialData.props.results.slice(0, limit).map((item): DataItem => { const title: string = item.title; const image: string | undefined = item.image_url; @@ -103,7 +101,7 @@ export const handler = async (ctx: Context): Promise => { const detailResponse = await ofetch(item.link); const detailData: string | undefined = detailResponse.match(/window\.__DATA__=JSON\.parse\(`(.*?)`\)/)?.[1]; - const parsedDetailData = detailData ? JSON.parse(detailData.replaceAll('\\\\', '\\')) : undefined; + const parsedDetailData = detailData ? JSON.parse(detailData.replaceAll(String.raw`\\`, '\\')) : undefined; if (!parsedDetailData) { return item; diff --git a/lib/routes/scientificamerican/templates/description.tsx b/lib/routes/scientificamerican/templates/description.tsx index bc1599bf307a04..1078746275584d 100644 --- a/lib/routes/scientificamerican/templates/description.tsx +++ b/lib/routes/scientificamerican/templates/description.tsx @@ -1,5 +1,6 @@ import { raw } from 'hono/html'; import { renderToString } from 'hono/jsx/dom/server'; +import type { JSX } from 'hono/jsx/jsx-runtime'; type DescriptionImage = { src?: string; diff --git a/lib/routes/scoop/apps.tsx b/lib/routes/scoop/apps.tsx index e5e9dc32de10a8..6d7cefa875606c 100644 --- a/lib/routes/scoop/apps.tsx +++ b/lib/routes/scoop/apps.tsx @@ -92,9 +92,7 @@ export const handler = async (ctx: Context): Promise => { }, }); - let items: DataItem[] = []; - - items = response.value.slice(0, limit).map((item): DataItem => { + const items: DataItem[] = response.value.slice(0, limit).map((item): DataItem => { const repositorySplits: string[] = item.Metadata.Repository.split(/\//); const repositoryName: string = repositorySplits.slice(-2).join('/'); const title = `${item.Name} ${item.Version} in ${repositoryName}`; diff --git a/lib/routes/scpta/news.ts b/lib/routes/scpta/news.ts index 90003dab1f40ce..1a8696e91222f9 100644 --- a/lib/routes/scpta/news.ts +++ b/lib/routes/scpta/news.ts @@ -68,7 +68,7 @@ async function handler(ctx) { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - let description = ''; + let description: string; try { const contentResponse = await got(item.link); const content = load(contentResponse.data); diff --git a/lib/routes/sdo/ff14risingstones/utils.tsx b/lib/routes/sdo/ff14risingstones/utils.tsx index 0d937660a27ce3..8839437e198110 100644 --- a/lib/routes/sdo/ff14risingstones/utils.tsx +++ b/lib/routes/sdo/ff14risingstones/utils.tsx @@ -159,7 +159,7 @@ export async function generateDynamicFeeds(dynamics: UserDynamic[]) { let title = `${dynamic.character_name}@${dynamic.group_name} ${dynamic.mask_content}`; let link: string | undefined; let description: string | undefined; - let detail: PostDetail | DutiesPartyDetail | FreeCompanyPartyDetail | NoviceNetworkParty | null = null; + let detail: PostDetail | DutiesPartyDetail | FreeCompanyPartyDetail | NoviceNetworkParty | null; switch (dynamic.from) { case DynamicSource.Post: diff --git a/lib/routes/sdu/yz.ts b/lib/routes/sdu/yz.ts new file mode 100644 index 00000000000000..64e18e7e577fc3 --- /dev/null +++ b/lib/routes/sdu/yz.ts @@ -0,0 +1,84 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +const host = 'https://www.yz.sdu.edu.cn'; + +export const route: Route = { + path: '/yz/:type?', + categories: ['university'], + example: '/sdu/yz/tzgg', + parameters: { type: '默认为`tzgg`' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: '研究生招生信息网', + maintainers: ['niuyi1017'], + handler, + description: `| 通知公告 | 招生拓展 | 政策文件 | +| -------- | -------- |-------- | +| tzgg | zstz | zcwj | `, +}; + +async function handler(ctx) { + const type = ctx.req.param('type') ?? 'tzgg'; + const pageUrl = `${host}/index/${type}.htm`; + + const response = await got(pageUrl); + const $ = load(response.data); + const typeName = $('.nyrtit .tit').text() || '研究生招生信息网'; + let list = $('.txtList li') + .toArray() + .map((element) => { + const $element = $(element); + let itemDate = $element.find('.times').text(); + itemDate = itemDate.slice(2) + '.' + itemDate.slice(0, 2); + const aTag = $element.find('a'); + const title = aTag.attr('title') || aTag.text().trim(); + const itemPath = aTag.attr('href') ?? ''; + const link = itemPath.startsWith('http') ? itemPath : new URL('/' + itemPath, pageUrl).href; + return { + title, + link, + pubDate: parseDate(itemDate), + }; + }); + + list = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const response = await got(item.link); + const $ = load(response.data); + + const content = $('.v_news_content'); + let description = ''; + if (content.length > 0) { + description = content.html()?.trim() ?? ''; + } + const attachments = $('ul[style="list-style-type:none;"]'); + if (attachments.length > 0) { + description += attachments.html()?.trim() ?? ''; + } + return { + ...item, + description, + }; + }) + ) + ); + + return { + title: `山东大学研究生招生信息网 - ${typeName}`, + description: $('title').text(), + link: pageUrl, + item: list, + }; +} diff --git a/lib/routes/sega/pjsekai.ts b/lib/routes/sega/pjsekai.ts index 3e164b46c8565b..3bcf376602d995 100644 --- a/lib/routes/sega/pjsekai.ts +++ b/lib/routes/sega/pjsekai.ts @@ -35,8 +35,8 @@ async function handler() { const posts = response.data || []; const list = await Promise.all( posts.map(async (post) => { - let link = ''; - let description = ''; + let link: string; + let description: string; const guid = post.displayOrder.toString() + post.id.toString(); // 双ID if (post.path.startsWith('information/')) { // information 公告 diff --git a/lib/routes/semiconductors/index.ts b/lib/routes/semiconductors/index.ts index 3790a62bdba873..3fc271130d6738 100644 --- a/lib/routes/semiconductors/index.ts +++ b/lib/routes/semiconductors/index.ts @@ -22,9 +22,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.col-sm-8') + let items: DataItem[] = $('div.col-sm-8') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/setn/index.ts b/lib/routes/setn/index.ts index 952012c7d2e033..e89573d4bacd8e 100644 --- a/lib/routes/setn/index.ts +++ b/lib/routes/setn/index.ts @@ -120,7 +120,7 @@ async function handler(ctx) { const content = load(detailResponse.data); - let head = {}; + let head: Record; try { head = JSON.parse(content('script[type="application/ld+json"]').first().text()); } catch { diff --git a/lib/routes/showstart/artist.ts b/lib/routes/showstart/artist.ts index 1a18e443a6d9a2..a8f7bc854f2c6a 100644 --- a/lib/routes/showstart/artist.ts +++ b/lib/routes/showstart/artist.ts @@ -41,5 +41,6 @@ async function handler(ctx: Context): Promise { description: artist.content, link: `${HOST}/artist/${artist.id}`, item: artist.activityList, + allowEmpty: true, }; } diff --git a/lib/routes/sicau/jiaowu.ts b/lib/routes/sicau/jiaowu.ts index 6f9e7c6e312ce4..c6c5bf64d5cc1e 100644 --- a/lib/routes/sicau/jiaowu.ts +++ b/lib/routes/sicau/jiaowu.ts @@ -68,7 +68,8 @@ export const route: Route = { items = await Promise.all( items.map((item) => cache.tryGet(item.link!, async () => { - const $ = load(await $get(item.link!)); + const html = await $get(item.link!); + const $ = load(html); item.description = $trim($('.text1[width="95%"] b').html()!); return item; }) diff --git a/lib/routes/sina/finance/rollnews.ts b/lib/routes/sina/finance/rollnews.ts new file mode 100644 index 00000000000000..347b0c3b13da81 --- /dev/null +++ b/lib/routes/sina/finance/rollnews.ts @@ -0,0 +1,59 @@ +import type { Route } from '@/types'; +import cache from '@/utils/cache'; + +import { getRollNewsList, parseArticle, parseRollNewsList } from '../utils'; + +export const route: Route = { + path: '/finance/rollnews/:lid?', + categories: ['new-media'], + example: '/sina/finance/rollnews', + parameters: { lid: '分区 id，见下表，默认为 `2519`' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['finance.sina.com.cn/roll', 'finance.sina.com.cn/'], + target: '/finance/rollnews', + }, + ], + name: '财经－滚动新闻', + maintainers: ['betterandbetterii'], + handler, + url: 'finance.sina.com.cn/roll', + description: `| 财经 | 股市 | 美股 | 中国概念股 | 港股 | 研究报告 | 全球市场 | 外汇 | +| ---- | ---- | ---- | ---------- | ---- | -------- | -------- | ---- | +| 2519 | 2671 | 2672 | 2673 | 2674 | 2675 | 2676 | 2487 |`, +}; + +async function handler(ctx) { + const map = { + 2519: '财经', + 2671: '股市', + 2672: '美股', + 2673: '中国概念股', + 2674: '港股', + 2675: '研究报告', + 2676: '全球市场', + 2487: '外汇', + }; + + const pageid = '384'; + const { lid = '2519' } = ctx.req.param(); + const { limit = '50' } = ctx.req.query(); + const response = await getRollNewsList(pageid, lid, limit); + const list = parseRollNewsList(response.data.result.data); + + const out = await Promise.all(list.map((item) => parseArticle(item, cache.tryGet))); + + return { + title: `新浪财经－${map[lid] ?? lid}滚动新闻`, + link: `https://finance.sina.com.cn/roll/#pageid=${pageid}&lid=${lid}&k=&num=${limit}&page=1`, + item: out, + }; +} diff --git a/lib/routes/sjtu/seiee/index.ts b/lib/routes/sjtu/seiee/index.ts index 89fec8c7aec25d..f07c8140d3fd4d 100644 --- a/lib/routes/sjtu/seiee/index.ts +++ b/lib/routes/sjtu/seiee/index.ts @@ -90,8 +90,9 @@ async function handler(ctx) { ) ); + const fallbackHtml = await ofetch(currentUrl); return { - title: $('title').text() || load(await ofetch(currentUrl))('title').text(), + title: $('title').text() || load(fallbackHtml)('title').text(), link: currentUrl, item: items, }; diff --git a/lib/routes/smartlink/index.ts b/lib/routes/smartlink/index.ts index 61cf7c03984e57..e0d04bebbb2337 100644 --- a/lib/routes/smartlink/index.ts +++ b/lib/routes/smartlink/index.ts @@ -12,7 +12,10 @@ function parseTitle(smartlinkUrl: string): string { const dateIndex = pathSegments.findIndex((segment) => dateRegex.test(segment)); // Use the segment after the date if found, otherwise use the last path segment - const titleSlug = dateIndex !== -1 && dateIndex < pathSegments.length - 1 ? pathSegments[dateIndex + 1] : pathSegments.at(-1) || ''; + let titleSlug = dateIndex !== -1 && dateIndex < pathSegments.length - 1 ? pathSegments[dateIndex + 1] : pathSegments.at(-1) || ''; + + // Remove .html/.htm extension if present + titleSlug = titleSlug.replace(/\.(html?|htm)$/i, ''); // Convert hyphens to spaces and capitalize each word return toTitleCase(titleSlug.replaceAll('-', ' ')); diff --git a/lib/routes/sspai/series-update.ts b/lib/routes/sspai/series-update.ts index 62f8638a28ad53..c57a9ee5e55079 100644 --- a/lib/routes/sspai/series-update.ts +++ b/lib/routes/sspai/series-update.ts @@ -34,7 +34,7 @@ async function handler(ctx) { const items = await Promise.all( response.data.data.map(async (item) => { - let description = ''; + let description: string; if (item.probation) { const res = await got(`https://sspai.com/api/v1/article/info/get?id=${item.id}&view=second&support_webp=true`); description = res.data.data.body; diff --git a/lib/routes/stcn/index.ts b/lib/routes/stcn/index.ts index 9fd2a955e82cad..9f9b2b20d40a62 100644 --- a/lib/routes/stcn/index.ts +++ b/lib/routes/stcn/index.ts @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = $('ul.infinite-list li') + let items: DataItem[] = $('ul.infinite-list li') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/stcn/kx.ts b/lib/routes/stcn/kx.ts index e3766fc82cc862..d580f65bd3e845 100644 --- a/lib/routes/stcn/kx.ts +++ b/lib/routes/stcn/kx.ts @@ -30,9 +30,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = response.data.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.slice(0, limit).map((item): DataItem => { const title: string = item.title; const description: string = item.content; const pubDate: number | string = item.time; diff --git a/lib/routes/stcn/rank.ts b/lib/routes/stcn/rank.ts index 6671db552ea0c0..9ad5abc95d0b81 100644 --- a/lib/routes/stcn/rank.ts +++ b/lib/routes/stcn/rank.ts @@ -30,9 +30,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(targetResponse); const language = $('html').attr('lang') ?? 'zh-CN'; - let items: DataItem[] = []; - - items = response.data.slice(0, limit).map((item): DataItem => { + let items: DataItem[] = response.data.slice(0, limit).map((item): DataItem => { const title: string = item.title; const linkUrl: string | undefined = item.url; diff --git a/lib/routes/surfshark/blog.ts b/lib/routes/surfshark/blog.ts index bf95e8c50ba06a..bc3de6db6d70db 100644 --- a/lib/routes/surfshark/blog.ts +++ b/lib/routes/surfshark/blog.ts @@ -31,9 +31,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.dg-article-single-card') + let items: DataItem[] = $('div.dg-article-single-card') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/taobao/mysql.ts b/lib/routes/taobao/mysql.ts index e516ddaa0090db..b545430bd25a56 100644 --- a/lib/routes/taobao/mysql.ts +++ b/lib/routes/taobao/mysql.ts @@ -19,10 +19,9 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'zh'; - let items: DataItem[] = []; let count = 0; - items = await Promise.all( + let items: DataItem[] = await Promise.all( $('h3 a.main') .toArray() .map(async (monthlyEl): Promise => { diff --git a/lib/routes/taptap/topic.ts b/lib/routes/taptap/topic.ts index 2954d437791876..270cefc5f02111 100644 --- a/lib/routes/taptap/topic.ts +++ b/lib/routes/taptap/topic.ts @@ -90,15 +90,13 @@ async function handler(ctx) { if (moment.reposted_moment.topic.footer_images) { description += imagePost(moment.reposted_moment.topic.footer_images); } - } else { - if (moment.topic.pin_video) { - description += videoPost(moment.topic.pin_video); - if (moment.topic.footer_images?.images) { - description += imagePost(moment.topic.footer_images.images); - } - } else { - description = await topicPost(appId, topicId, lang); + } else if (moment.topic.pin_video) { + description += videoPost(moment.topic.pin_video); + if (moment.topic.footer_images?.images) { + description += imagePost(moment.topic.footer_images.images); } + } else { + description = await topicPost(appId, topicId, lang); } return { title, diff --git a/lib/routes/telegram/channel-media.ts b/lib/routes/telegram/channel-media.ts index a00edbe581c8f2..79867dbfe07828 100644 --- a/lib/routes/telegram/channel-media.ts +++ b/lib/routes/telegram/channel-media.ts @@ -68,7 +68,7 @@ function sortThumb(thumb: Api.TypePhotoSize) { } function chooseLargestThumb(thumbs: Api.TypePhotoSize[]) { - thumbs = [...thumbs].sort((a, b) => sortThumb(a) - sortThumb(b)); + thumbs = [...thumbs].toSorted((a, b) => sortThumb(a) - sortThumb(b)); return thumbs.pop(); } diff --git a/lib/routes/tencent/news/coronavirus/data.tsx b/lib/routes/tencent/news/coronavirus/data.tsx index cfec69ede6a25e..392a04c89d189f 100644 --- a/lib/routes/tencent/news/coronavirus/data.tsx +++ b/lib/routes/tencent/news/coronavirus/data.tsx @@ -25,12 +25,8 @@ async function handler(ctx) { const nationalData = areaTree?.[0]; const provinceList = nationalData?.children; - let todayConfirm = 0; - let totalNowConfirm = 0; - let totalConfirm = 0; - let totalDead = 0; - let coronavirusData = {}; - let placeName = ''; + let coronavirusData: Record; + let placeName: string; if (!province || province === '中国' || province === '全国') { // 没有传参则取全国 @@ -51,10 +47,10 @@ async function handler(ctx) { if (!coronavirusData) { throw new InvalidParameterError(`未找到 ${placeName} 的疫情数据，请检查输入的省市名称是否正确`); } - todayConfirm = coronavirusData.today?.confirm; - totalNowConfirm = coronavirusData.total?.nowConfirm; - totalConfirm = coronavirusData.total?.confirm; - totalDead = coronavirusData.total?.dead; + const todayConfirm = coronavirusData.today?.confirm; + const totalNowConfirm = coronavirusData.total?.nowConfirm; + const totalConfirm = coronavirusData.total?.confirm; + const totalDead = coronavirusData.total?.dead; const pubDate = parseDate(coronavirusData.total?.mtime || lastUpdateTime); const title = `${placeName} - 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪`; diff --git a/lib/routes/tencent/qq/sdk/changelog.ts b/lib/routes/tencent/qq/sdk/changelog.ts index 6ff1a0ca20b5e6..bd4fc261567f39 100644 --- a/lib/routes/tencent/qq/sdk/changelog.ts +++ b/lib/routes/tencent/qq/sdk/changelog.ts @@ -25,8 +25,8 @@ export const route: Route = { async function handler(ctx) { const platform = ctx.req.param('platform'); - let title = ''; - let link = ''; + let title: string; + let link: string; if (platform === 'iOS') { title = 'iOS SDK 历史变更'; link = 'https://wiki.connect.qq.com/ios_sdk历史变更'; diff --git a/lib/routes/thebrain/blog.tsx b/lib/routes/thebrain/blog.tsx index 1d8588207d5306..85a882ddc10f28 100644 --- a/lib/routes/thebrain/blog.tsx +++ b/lib/routes/thebrain/blog.tsx @@ -21,9 +21,7 @@ export const handler = async (ctx: Context): Promise => { const $: CheerioAPI = load(response); const language = $('html').attr('lang') ?? 'en'; - let items: DataItem[] = []; - - items = $('div.blog-row') + let items: DataItem[] = $('div.blog-row') .slice(0, limit) .toArray() .map((el): Element => { diff --git a/lib/routes/theinitium/author.ts b/lib/routes/theinitium/author.ts index 253848047f0b52..d4e43d8a2162ce 100644 --- a/lib/routes/theinitium/author.ts +++ b/lib/routes/theinitium/author.ts @@ -7,10 +7,25 @@ const handler = (ctx) => processFeed('author', ctx); export const route: Route = { path: '/author/:type/:language?', name: '作者', + url: 'theinitium.com', maintainers: ['AgFlore'], parameters: { - type: '作者 ID，可从作者主页 URL 中获取，如 `https://theinitium.com/author/ninghuilulu`', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '作者 slug，可从作者主页 URL 中获取，如 `https://theinitium.com/author/initium-newsroom/`', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { @@ -19,6 +34,6 @@ export const route: Route = { }, ], handler, - example: '/theinitium/author/ninghuilulu/zh-hans', + example: '/theinitium/author/initium-newsroom', categories: ['new-media'], }; diff --git a/lib/routes/theinitium/channel.ts b/lib/routes/theinitium/channel.ts index da63ae056a1409..df91102c153ee7 100644 --- a/lib/routes/theinitium/channel.ts +++ b/lib/routes/theinitium/channel.ts @@ -6,24 +6,47 @@ const handler = (ctx) => processFeed('channel', ctx); export const route: Route = { path: '/channel/:type?/:language?', - name: '专题・栏目', + name: '栏目', + url: 'theinitium.com', maintainers: ['prnake', 'mintyfrankie'], parameters: { - type: '栏目，缺省为最新', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '栏目，缺省为最新（latest）', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。获取方式：登录 theinitium.com 后，从浏览器开发者工具中复制 Cookie。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { - source: ['theinitium.com/channel/:type'], + source: ['theinitium.com/latest/'], + target: '/channel/latest', + }, + { + source: ['theinitium.com/tag/:type'], target: '/channel/:type', }, ], handler, - example: '/theinitium/channel/latest/zh-hans', + example: '/theinitium/channel/latest', categories: ['new-media'], - description: `Type 栏目： + description: `Type 栏目（对应 Ghost 标签）： + +| 最新 | 速递 | 评论 | 国际 | 大陆 | 香港 | 台湾 | 科技 | 专题 | 日报 | 周报 | +| ------ | -------- | ------- | ------------- | -------- | -------- | ------ | ---------- | ------ | ----------- | ------ | +| latest | whatsnew | opinion | international | mainland | hongkong | taiwan | technology | feature | daily-brief | weekly | -| 最新 | 深度 | What’s New | 广场 | 科技 | 风物 | 特约 | ... | -| ------ | ------- | ---------- | ----------------- | ---------- | ------- | -------- | --- | -| latest | feature | news-brief | notes-and-letters | technology | culture | pick_up | ... |`, +:::tip +设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。 +:::`, }; diff --git a/lib/routes/theinitium/follow.ts b/lib/routes/theinitium/follow.ts index 50f9ceb7c71614..ef56fef97c6145 100644 --- a/lib/routes/theinitium/follow.ts +++ b/lib/routes/theinitium/follow.ts @@ -1,44 +1,19 @@ import type { Route } from '@/types'; -import { processFeed } from './utils'; - -const handler = (ctx) => processFeed('follow', ctx); - export const route: Route = { path: '/follow/articles/:language?', - name: '个人订阅追踪动态', + name: '个人订阅追踪动态（已停用）', maintainers: ['AgFlore'], parameters: { - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + language: '语言', }, - radar: [ - { - title: '作者', - source: ['theinitium.com/author/:type'], - target: '/author/:type', - }, - ], - handler, - example: '/theinitium/author/ninghuilulu/zh-hans', - categories: ['new-media'], - description: '需填入 Web 版认证 token, 也可选择直接在环境设置中填写明文的用户名和密码', - features: { - requireConfig: [ - { - name: 'INITIUM_BEARER_TOKEN', - optional: true, - description: `端传媒 Web 版认证 token。获取方式：登陆后打开端传媒站内任意页面，打开浏览器开发者工具中 “网络”(Network) 选项卡，筛选 URL 找到任一个地址为 \`api.initium.com\` 开头的请求，点击检查其 “消息头”，在 “请求头” 中找到Authorization字段，将其值复制填入配置即可。你的配置应该形如 \`INITIUM_BEARER_TOKEN: 'Bearer eyJxxxx......xx_U8'\`。使用 token 部署的好处是避免占据登陆设备数的额度，但这个 token 一般有效期为两周，因此只可作临时测试使用。`, - }, - { - name: 'INITIUM_USERNAME', - optional: true, - description: `端传媒用户名 （邮箱）`, - }, - { - name: 'INITIUM_PASSWORD', - optional: true, - description: `端传媒密码`, - }, - ], + radar: [], + handler: () => { + throw new Error('此路由已停用。端传媒迁移到 Ghost CMS 后不再支持个人追踪功能。请改用 /theinitium/channel/latest 或 /theinitium/tags/:tag 订阅。'); }, + example: '/theinitium/follow/articles', + categories: ['new-media'], + description: `:::warning +此路由已停用。端传媒已迁移到 Ghost CMS，不再支持通过 API 获取个人追踪内容。请改用标签或栏目订阅。 +:::`, }; diff --git a/lib/routes/theinitium/namespace.ts b/lib/routes/theinitium/namespace.ts index 6abfdcba1d00e4..14056e91fe024f 100644 --- a/lib/routes/theinitium/namespace.ts +++ b/lib/routes/theinitium/namespace.ts @@ -1,12 +1,20 @@ import type { Namespace } from '@/types'; export const namespace: Namespace = { - name: '端传媒', + name: 'The Initium', url: 'theinitium.com', - description: `通过提取文章全文，以提供比官方源更佳的阅读体验。 + description: `:::tip +Set the environment variable \`INITIUM_MEMBER_COOKIE\` to get the full text of paid articles. After logging in to theinitium.com, copy the Cookie from the browser developer tools. -::: warning -付费内容全文可能需要登陆获取，详情见部署页面的配置模块。 +Old environment variables \`INITIUM_USERNAME\`, \`INITIUM_PASSWORD\`, and \`INITIUM_BEARER_TOKEN\` are no longer used since the site migrated to Ghost CMS. :::`, - lang: 'zh-HK', + + zh: { + name: '端傳媒', + description: `:::tip +设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。登录 theinitium.com 后，从浏览器开发者工具中复制 Cookie。 + +旧的环境变量 \`INITIUM_USERNAME\`、\`INITIUM_PASSWORD\` 和 \`INITIUM_BEARER_TOKEN\` 已不再使用（网站已迁移至 Ghost CMS）。 +:::`, + }, }; diff --git a/lib/routes/theinitium/tags.ts b/lib/routes/theinitium/tags.ts index fb209acda53a0f..37155f1983c7bd 100644 --- a/lib/routes/theinitium/tags.ts +++ b/lib/routes/theinitium/tags.ts @@ -7,18 +7,33 @@ const handler = (ctx) => processFeed('tags', ctx); export const route: Route = { path: '/tags/:type/:language?', name: '话题・标签', + url: 'theinitium.com', maintainers: ['AgFlore'], parameters: { - type: '话题 ID，可从话题页 URL 中获取，如 `https://theinitium.com/tags/2019_10/`', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '标签 slug，可从标签页 URL 中获取，如 `https://theinitium.com/tag/south-korea/` 则为 `south-korea`', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { - source: ['theinitium.com/tags/:type'], + source: ['theinitium.com/tag/:type'], target: '/tags/:type', }, ], handler, - example: '/theinitium/tags/2019_10/zh-hans', + example: '/theinitium/tags/south-korea', categories: ['new-media'], }; diff --git a/lib/routes/theinitium/utils.ts b/lib/routes/theinitium/utils.ts index 902ff5aedcd38a..e630d919709eaa 100644 --- a/lib/routes/theinitium/utils.ts +++ b/lib/routes/theinitium/utils.ts @@ -1,168 +1,309 @@ import { load } from 'cheerio'; import type { Context } from 'hono'; -import { FetchError } from 'ofetch'; import { config } from '@/config'; import InvalidParameterError from '@/errors/types/invalid-parameter'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import logger from '@/utils/logger'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY='; +// Strip '-zh-hans' suffix from display names for cleanliness +const stripLangSuffix = (name: string) => name.replace(/-zh-hans$/i, ''); -export const processFeed = async (model: string, ctx: Context) => { - // model是channel/tag/etc.，而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话，那就是model是follow，type是articles。 - const type = ctx.req.param('type') ?? 'latest'; - const language = ctx.req.param('language') ?? 'zh-hans'; - let listUrl; - let listLink; - switch (model) { - case 'author': - listUrl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/author/${type}/`; - break; - case 'follow': - listUrl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`; - listLink = `https://theinitium.com/follow/`; - break; - case 'channel': - listUrl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/channel/${type}/`; - break; - case 'tags': - listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/tags/${type}/`; - break; - default: - throw new InvalidParameterError('wrong model'); - } +const GHOST_API_BASE = 'https://production-initium-media.ghost.io/ghost/api/content'; +const GHOST_CONTENT_KEY = 'a44a0409c222328d39e2c75293'; - const key = { - email: config.initium.username, - password: config.initium.password, - }; - const body = JSON.stringify(key); - - let token; - const cacheIn = await cache.get('initium:token'); - if (cacheIn) { - token = cacheIn; - } else if (config.initium.bearertoken) { - token = config.initium.bearertoken; - cache.set('initium:token', config.initium.bearertoken); - } else if (key.email === undefined) { - token = TOKEN; - } else { - const login = await got.post(`https://api.theinitium.com/api/v2/auth/login/?language=${language}`, { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Connection: 'keep-alive', - Authorization: TOKEN, - }, - body, - }); +// Old channel slugs → Ghost tag slugs mapping +const CHANNEL_TAG_MAP: Record = { + latest: '', // no filter = latest + whatsnew: 'whatsnew', + 'news-brief': 'whatsnew', + opinion: 'opinion', + international: 'international', + mainland: 'mainland', + hongkong: 'hong-kong', + taiwan: 'taiwan', + technology: 'technology', + feature: 'report', + report: 'report', + 'daily-brief': 'daily-brief', + weekly: 'weekly', +}; - token = 'token ' + login.data.token; - cache.set('initium:token', token); +// Ghost uses a language-based tagging system: +// - zh-hant (Traditional Chinese): uses base tag slug, e.g. "whatsnew", with internal tag #zh-hant +// - zh-hans (Simplified Chinese): uses suffixed tag slug, e.g. "whatsnew-zh-hans", with internal tag #zh-hans +// When no language is specified, we return all posts (both zh-hans and zh-hant mixed). +function applyLanguageToTagSlug(tagSlug: string, language: string): string { + if (language === 'zh-hans') { + return `${tagSlug}-zh-hans`; } + // zh-hant uses the base slug + return tagSlug; +} + +interface GhostPost { + id: string; + uuid: string; + slug: string; + title: string; + html: string; + feature_image?: string; + feature_image_caption?: string; + custom_excerpt?: string; + published_at: string; + updated_at: string; + url: string; + excerpt?: string; + access: boolean; + visibility?: string; + authors?: Array<{ name: string; slug: string }>; + tags?: Array<{ name: string; slug: string; visibility: string }>; + primary_author?: { name: string; slug: string }; + primary_tag?: { name: string; slug: string }; +} - const headers = { - Accept: '*/*', - Connection: 'keep-alive', - Authorization: token, +interface GhostResponse { + posts: GhostPost[]; + meta: { + pagination: { + page: number; + limit: number; + pages: number; + total: number; + }; }; +} + +async function ghostFetch(endpoint: string, params: Record = {}): Promise { + const url = new URL(`${GHOST_API_BASE}/${endpoint}/`); + url.searchParams.set('key', GHOST_CONTENT_KEY); + for (const [k, v] of Object.entries(params)) { + url.searchParams.set(k, v); + } + return await ofetch(url.href); +} - let response; +async function scrapeFullArticle(url: string, cookie: string): Promise { try { - response = await got(listUrl, { - headers, + const response = await ofetch(url, { + headers: { + Cookie: cookie, + }, + parseResponse: (txt) => txt, }); - } catch (error) { - if (error instanceof FetchError && error.statusCode === 401) { - // 401 说明 token 过期了，将它删掉 - await cache.set('initium:token', ''); + const $ = load(response); + const article = $('article'); + if (article.length === 0) { + return null; } - throw error; + // If paywall CTA present, cookie didn't work — fall back to Ghost preview + if (article.find('.gh-post-upgrade-cta').length > 0) { + return null; + } + return article.html(); + } catch (error) { + logger.warn(`Failed to scrape Initium article: ${url}`, error); + return null; } +} + +/** + * Clean Ghost Koenig editor card HTML for RSS consumption. + * Strips kg-* wrapper divs, converts bookmark cards to simple links, + * removes callout markup, etc. + */ +function cleanGhostHtml(html: string): string { + const $ = load(html, null, false); + + // Convert kg-bookmark-card to a simple link + $('a.kg-bookmark-container, a.kg-bookmark-card').each((_, el) => { + const $el = $(el); + const href = $el.attr('href') || ''; + const title = $el.find('.kg-bookmark-title').text().trim(); + const desc = $el.find('.kg-bookmark-description').text().trim(); + const replacement = title ? `

    ${title}${desc ? ` — ${desc}` : ''}

    ` : `

    ${href}

    `; + $el.replaceWith(replacement); + }); + + // Convert kg-callout-card: keep text, strip wrapper + $('.kg-callout-card').each((_, el) => { + const $el = $(el); + const text = $el.find('.kg-callout-text').html() || $el.html() || ''; + $el.replaceWith(`
    ${text}
    `); + }); + + // Convert kg-toggle-card: heading + content + $('.kg-toggle-card').each((_, el) => { + const $el = $(el); + const heading = $el.find('.kg-toggle-heading-text').text().trim(); + const content = $el.find('.kg-toggle-content').html() || ''; + $el.replaceWith(`${heading ? `

    ${heading}

    ` : ''}${content}`); + }); + + // Unwrap remaining kg-card divs (keep inner content) + $('.kg-card').each((_, el) => { + const $el = $(el); + $el.replaceWith($el.html() || ''); + }); + + // Remove figure wrapping around bookmark cards that we already replaced + $('figure.kg-bookmark-card').each((_, el) => { + const $el = $(el); + $el.replaceWith($el.html() || ''); + }); + + // Strip members-only paywall markers + $('p:contains("")').remove(); + + return $.html(); +} + +async function postsToItems(posts: GhostPost[]) { + const memberCookie = config.initium?.memberCookie; + + const items = await Promise.all( + posts.map(async (post) => { + const authors = post.authors?.map((a) => stripLangSuffix(a.name)) ?? []; + const categories = post.tags?.filter((t) => t.visibility === 'public').map((t) => stripLangSuffix(t.name)) ?? []; + + let description = post.html ? cleanGhostHtml(post.html) : post.html; - const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪'; - // 从v1直升的channel和tags里面是digests，v2新增的author和follow出来都是results - const articles = response.data.results ?? response.data.digests; - // 如果model=author，那就是avatar；否则都是cover，要么就没封面 - const image = response.data[model] && (response.data[model].cover || response.data[model].avatar); - - const getFullText = (slug) => - cache.tryGet(`theinitium:${slug}:${language}`, async () => { - let content = ''; - const { data } = await got(`https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`, { - headers, - }); - - if (data.lead.length) { - content += '

    「' + data.lead + '」

    '; + // For paid articles with truncated content, scrape full text if cookie available + if (!post.access && memberCookie) { + const fullHtml = (await cache.tryGet(`theinitium:full:${post.slug}`, () => scrapeFullArticle(post.url, memberCookie), config.cache.contentExpire)) as string | null; + if (fullHtml) { + description = cleanGhostHtml(fullHtml); + } } - if (data.byline.length) { - content += '

    ' + data.byline + '

    '; + + return { + title: post.title, + author: authors.join(', ') || post.primary_author?.name || '', + category: categories, + description, + link: post.url, + pubDate: parseDate(post.published_at), + updated: parseDate(post.updated_at), + guid: post.uuid, + banner: post.feature_image ?? undefined, + }; + }) + ); + + return items; +} + +export const processFeed = async (model: string, ctx: Context) => { + const type = ctx.req.param('type') ?? 'latest'; + const language = ctx.req.param('language') ?? ''; + + let filter = ''; + let listLink = ''; + let feedName = ''; + + switch (model) { + case 'channel': { + const baseTag = CHANNEL_TAG_MAP[type] ?? type; + if (baseTag === '') { + // "latest" = no tag filter, but we can still filter by language via internal tag + if (language === 'zh-hans' || language === 'zh-hant') { + filter = `tag:hash-${language}`; + } + } else { + const tagSlug = language ? applyLanguageToTagSlug(baseTag, language) : baseTag; + filter = `tag:${tagSlug}`; + } + listLink = type === 'latest' ? 'https://theinitium.com/latest/' : `https://theinitium.com/tag/${baseTag}/`; + feedName = type; + break; + } + case 'tags': { + const tagSlug = language ? applyLanguageToTagSlug(type, language) : type; + filter = `tag:${tagSlug}`; + listLink = `https://theinitium.com/tag/${type}/`; + feedName = type; + break; + } + case 'author': { + // Author slugs also have -zh-hans suffixed versions for simplified Chinese + const authorSlug = language === 'zh-hans' ? `${type}-zh-hans` : type; + filter = `author:${authorSlug}`; + listLink = `https://theinitium.com/author/${type}/`; + feedName = type; + break; + } + default: + throw new InvalidParameterError(`Unsupported model: ${model}`); + } + + const cacheKey = `theinitium:ghost:${model}:${type}:${language}`; + // Use routeExpire (5 min default) and refresh=false so cache actually expires + const data = (await cache.tryGet( + cacheKey, + async () => { + const params: Record = { + include: 'tags,authors', + limit: '20', + }; + if (filter) { + params.filter = filter; } - if (data.content) { - content += data.content.replace('