From fec30cb508dc8bfd78c366026dd7005777db16c8 Mon Sep 17 00:00:00 2001 From: Avazbek Olimov Date: Sun, 19 Jul 2026 12:27:00 +0300 Subject: [PATCH] =?UTF-8?q?=E2=80=A2=20feat:=20refactor=20bot=20for=20secu?= =?UTF-8?q?re=20and=20reliable=20production=20use?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add English and Russian localization with English as default - add admin commands for per-group language and settings - split persistent data into atomic JSON stores with backups - add five-minute media cache and Telegram file_id reuse - deduplicate concurrent downloads of the same video - harden URL validation and block private network access - preserve quiet group behavior without status-message noise - add rotating logs with 60-day retention - improve YouTube reliability with Node.js and yt-dlp retries - add Docker deployment, automatic yt-dlp updates and rollback - add unit and integration tests - rewrite README and add open-source community documentation --- .dockerignore | 11 + .env-example | 15 +- .gitattributes | 6 + .github/ISSUE_TEMPLATE/bug_report.yml | 21 + .github/dependabot.yml | 15 + .github/pull_request_template.md | 11 + .github/workflows/ci.yml | 37 + .gitignore | 4 + CODE_OF_CONDUCT.md | 5 + CONTRIBUTING.md | 24 + Dockerfile | 55 +- README.md | 370 +++---- SECURITY.md | 11 + app/download_backend.py | 486 ++++----- app/env_config.py | 9 +- app/i18n.py | 94 ++ app/jobs.py | 84 ++ app/logging_setup.py | 40 + app/media_cache.py | 81 ++ app/settings.py | 120 +++ app/storage.py | 285 ++++++ app/url_security.py | 86 ++ app/url_utils.py | 2 +- config.py | 40 +- docker-compose.yml | 15 +- example.config.py | 18 - install.sh | 419 ++------ main.py | 1366 +++++++++++-------------- pyproject.toml | 11 + requirements-dev.txt | 3 + requirements.txt | 3 +- scripts/docker-entrypoint.sh | 7 + scripts/update-ytdlp.sh | 126 +-- tests/test_application_integration.py | 123 +++ tests/test_download_backend.py | 149 +++ tests/test_jobs_and_cache.py | 53 + tests/test_logging_and_i18n.py | 23 + tests/test_storage.py | 80 ++ tests/test_url_security.py | 52 + 39 files changed, 2580 insertions(+), 1780 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/ci.yml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 app/i18n.py create mode 100644 app/jobs.py create mode 100644 app/logging_setup.py create mode 100644 app/media_cache.py create mode 100644 app/settings.py create mode 100644 app/storage.py create mode 100644 app/url_security.py delete mode 100644 example.config.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 scripts/docker-entrypoint.sh create mode 100644 tests/test_application_integration.py create mode 100644 tests/test_download_backend.py create mode 100644 tests/test_jobs_and_cache.py create mode 100644 tests/test_logging_and_i18n.py create mode 100644 tests/test_storage.py create mode 100644 tests/test_url_security.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0aeb601 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.idea +.env +.venv +__pycache__ +*.py[cod] +data +logs +tests +*.log diff --git a/.env-example b/.env-example index 8116654..9f2cab2 100644 --- a/.env-example +++ b/.env-example @@ -6,8 +6,19 @@ BOT_TOKEN= # Optional settings: # LOGS_CHAT_ID= # MAX_FILESIZE=52428800 -# OUTPUT_FOLDER=/tmp/yt-dlp-telegram -# COOKIES_FILE= +# WORKERS=2 +# MAX_QUEUE=200 +# UPLOAD_WORKERS=2 +# JOB_TIMEOUT_SECONDS=900 +# DEFAULT_LANGUAGE=en +# DELETE_ORIGINAL=true +# MEDIA_CACHE_ENABLED=true +# DISK_CACHE_MAX_FILES=5 +# DISK_CACHE_TTL_SECONDS=300 +# FILE_ID_CACHE_MAX_ITEMS=500 +# FILE_ID_CACHE_TTL_DAYS=30 +# COOKIES_FILE=/app/data/cookies.txt +# LOG_LEVEL=INFO YTDLP_JS_RUNTIMES=node YTDLP_REMOTE_COMPONENTS=ejs:github YTDLP_INSTAGRAM_IMPERSONATE=chrome diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1d91e01 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto +*.sh text eol=lf +Dockerfile text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.py text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..a5241fd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,21 @@ +name: Bug report +description: Report a reproducible problem +body: + - type: textarea + attributes: + label: Description + description: What happened and what did you expect? + validations: + required: true + - type: textarea + attributes: + label: Reproduction + description: Provide safe steps without tokens, cookies, private URLs, or chat data. + validations: + required: true + - type: input + attributes: + label: Version + - type: textarea + attributes: + label: Relevant redacted logs diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..31b58e1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f4ffd65 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,11 @@ +## Summary + +## User-visible behavior + +## Compatibility and configuration + +## Verification + +- [ ] Tests added or updated +- [ ] `ruff check`, `ruff format --check`, and `pytest` pass +- [ ] No secrets, cookies, media, logs, or private chat data are included diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e3c3d8a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install -r requirements-dev.txt + - run: python -m ruff check . + - run: python -m ruff format --check . + - run: python -m pytest + + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + with: + context: . + push: false + tags: linkdownloaderbotforgroups:test diff --git a/.gitignore b/.gitignore index a72755a..cb893c4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ venv/ # Runtime / cache data/ +logs/ +.pytest_cache/ +.ruff_cache/ +.idea/ diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..20a1f1b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# Code of Conduct + +Be respectful, constructive, and patient. Harassment, discrimination, personal attacks, and publication of another person's private information are not acceptable. Maintainers may edit or remove abusive contributions and restrict participation when necessary. + +Report conduct concerns privately to the repository owner. Reports will be reviewed fairly and confidentially where possible. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c6326df --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,24 @@ +# Contributing + +Thank you for helping improve Link Downloader Bot for Telegram Groups. + +## Development workflow + +1. Fork the repository and create a focused branch. +2. Install `requirements-dev.txt` in Python 3.11 or newer. +3. Preserve the quiet automatic-download behavior unless the change is explicitly discussed. +4. Add or update tests for behavior changes. +5. Run: + +```bash +python -m ruff check . +python -m ruff format --check . +python -m pytest +docker build -t linkdownloaderbotforgroups:test . +``` + +Do not include bot tokens, cookies, downloaded media, logs, or real private chat data in issues or commits. + +## Pull requests + +Explain the user-visible effect, compatibility impact, tests performed, and any configuration changes. Keep unrelated refactoring separate. Security reports must follow `SECURITY.md` rather than a public issue. diff --git a/Dockerfile b/Dockerfile index 280f7b0..d572e07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,57 @@ -FROM python:3.11-slim +FROM python:3.12-slim + +ARG NODE_VERSION=24.18.0 ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_NO_CACHE_DIR=1 + PIP_NO_CACHE_DIR=1 \ + DATA_DIR=/app/data \ + LOGS_DIR=/app/logs \ + OUTPUT_FOLDER=/app/data/cache WORKDIR /app -RUN apt-get update -y && apt-get install -y --no-install-recommends \ - ffmpeg ca-certificates nodejs \ - && rm -rf /var/lib/apt/lists/* +RUN set -eux; \ + apt-get update -y; \ + apt-get install -y --no-install-recommends ffmpeg ca-certificates gosu curl xz-utils; \ + case "$(dpkg --print-architecture)" in \ + amd64) node_arch='x64' ;; \ + arm64) node_arch='arm64' ;; \ + *) echo 'Unsupported architecture' >&2; exit 1 ;; \ + esac; \ + node_archive="node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ + curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${node_archive}"; \ + curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt"; \ + grep " ${node_archive}$" SHASUMS256.txt | sha256sum -c -; \ + tar -xJf "$node_archive" -C /usr/local/bin --strip-components=2 "node-v${NODE_VERSION}-linux-${node_arch}/bin/node"; \ + node --version; \ + rm -f "$node_archive" SHASUMS256.txt; \ + apt-get purge -y --auto-remove curl xz-utils; \ + rm -rf /var/lib/apt/lists/* COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt +RUN sed '/^yt-dlp/d' /app/requirements.txt > /tmp/requirements-base.txt \ + && pip install --no-cache-dir -r /tmp/requirements-base.txt + +# The updater changes this argument to refresh yt-dlp without invalidating the +# slower OS and stable Python dependency layers. +ARG YTDLP_CACHEBUST=initial +RUN echo "$YTDLP_CACHEBUST" >/tmp/ytdlp-cachebust \ + && pip install --no-cache-dir --upgrade 'yt-dlp[default,curl-cffi]' + +COPY app /app/app +COPY main.py config.py /app/ +COPY scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +RUN groupadd --gid 10001 bot \ + && useradd --uid 10001 --gid bot --no-create-home --home-dir /app bot \ + && mkdir -p /app/data/cache /app/logs \ + && chown -R bot:bot /app/data /app/logs + +RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh + +ENV XDG_CACHE_HOME=/tmp/.cache -# main.py is mounted from host by docker-compose (for easy editing) -# config.py is mounted from host by docker-compose +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["python", "-u", "/app/main.py"] diff --git a/README.md b/README.md index 401c1c8..1b05553 100644 --- a/README.md +++ b/README.md @@ -1,291 +1,177 @@ -# LinkDownloaderBotForGroups — Telegram-бот для скачивания видео по ссылкам в группах +# Link Downloader Bot for Telegram Groups -**LinkDownloaderBotForGroups** — это Telegram-бот для групп/супергрупп: Вы кидаете ссылку на видео (YouTube/Instagram/TikTok/VK/X/Facebook/Telegram и др.), бот скачивает ролик через **yt-dlp**, публикует видео в чат и (опционально) удаляет исходное сообщение со ссылкой. +A quiet, self-hosted Telegram bot that replaces video links with the actual video. It is designed for small private groups: paste a supported link, and the bot downloads the video, posts it silently in the same topic, then removes the original message only after success. -Ключевые слова (для поиска): **telegram bot downloader**, **yt-dlp telegram bot**, **скачать видео по ссылке в телеграм**, **бот для групп скачивает youtube instagram tiktok vk**. +The bot intentionally sends no “downloading” or failure messages for automatic requests. If a source is unavailable, the original link stays untouched and the technical reason is written to the rotating log. ---- +## Highlights -## Что умеет +- Supports YouTube, Instagram, TikTok, VK, X, Facebook and many other sites through [yt-dlp](https://github.com/yt-dlp/yt-dlp). +- Silent messages and no link previews. +- Telegram forum topic support. +- English and Russian interface; English is the default. +- Per-user automatic-download opt-out. +- Safe public-URL validation against local and private network addresses. +- Bounded worker queue and concurrent downloads. +- Single-flight deduplication: simultaneous copies of the same video are downloaded once. +- Reuses Telegram `file_id`, avoiding repeated downloads and uploads. +- Five-minute disk cache with automatic cleanup. +- Atomic JSON storage with backups and migration from the legacy `prefs.json`. +- Daily rotating logs retained for 60 days. +- Reproducible Docker deployment and an optional nightly yt-dlp updater with rollback. -* ✅ Скачивает видео по ссылкам и отправляет в чат как **video** (с поддержкой streaming). -* ✅ Работает в **группах и супергруппах**, включая **темы** (topics) — отвечает в нужном треде. -* ✅ Поддерживает множество источников (зависит от yt-dlp и доступности контента). -* ✅ **Без лишних уведомлений** (silent) и без превью ссылок. -* ✅ **Очередь и воркеры**: несколько скачиваний параллельно, чтобы не блокировать чат. -* ✅ **Персональный opt-out**: участник может отключить авто-скачивание для себя. -* ✅ Настраиваемые ограничения: размер файла, папка, cookies, чат логов. -* ✅ Установка одной командой через `install.sh` (Docker-рекомендуемый режим). +## How it works ---- +1. A member posts a video link. +2. The bot validates the URL and quietly downloads the video. +3. The bot publishes the video silently in the same chat topic. +4. After a successful upload, it removes the original link if it has permission. -## Как это выглядит в группе +When the same video is posted again, the bot normally sends the existing Telegram media by `file_id`. Captions and sender attribution are still generated independently for every group. -1. Участник отправляет ссылку на видео. -2. Бот скачивает ролик. -3. Бот публикует видео в группу. -4. Бот удаляет исходное сообщение со ссылкой (если у бота есть право **Delete messages**). +## Telegram setup -Под подписью к видео бот добавляет: +Create a bot with [@BotFather](https://t.me/BotFather), then: -* кликабельную ссылку на оригинал -* кто отправил ссылку +1. Disable **Group Privacy** under **Bot Settings → Group Privacy**. Otherwise the bot cannot see ordinary group messages. +2. Add the bot to a group. +3. Grant **Delete messages** if original links should be removed. ---- +No other administrator rights are required. -## Важно про права и приватность +## Quick start with Docker -Чтобы бот работал «по умолчанию» (видел **все** сообщения со ссылками и мог удалять сообщения): - -### 1) Отключите Group Privacy (иначе бот не видит обычные сообщения) - -В BotFather: - -* **Bot Settings → Group Privacy → Off** - -Если приватность включена — бот будет видеть только команды и упоминания, и «авто-скачивание без упоминания» работать не будет. - -### 2) Дайте боту права администратора в группе - -Минимально нужные права: - -* ✅ **Delete messages** (чтобы удалять исходную ссылку) - -Остальные права не требуются, но можно включать по желанию. - ---- - -## Быстрый старт (Docker, рекомендовано) - -### 1) Получите токен - -* В Telegram откройте **@BotFather** -* Создайте бота (`/newbot`) -* Скопируйте **BOT_TOKEN** - -> Совет по безопасности: токен храните только в `.env`, не коммитьте его в Git. - -### 2) Установите через install.sh +```bash +git clone https://github.com/Avazbek22/LinkDownloaderBotForGroups.git +cd LinkDownloaderBotForGroups +cp .env-example .env +nano .env +docker compose up -d --build +docker compose logs -f --tail=200 +``` -На сервере (Ubuntu/Debian): +Or inspect and run the installer on Debian/Ubuntu: ```bash curl -fsSL https://raw.githubusercontent.com/Avazbek22/LinkDownloaderBotForGroups/main/install.sh -o install.sh +less install.sh chmod +x install.sh ./install.sh ``` -Скрипт: - -* установит зависимости -* клонирует репозиторий -* попросит токен и запишет его в `.env` -* поднимет контейнер через Docker Compose - ---- - -## Конфигурация - -Настройки читаются из `.env` (или переменных окружения). Файл `config.py` **не содержит секретов** — он просто читает env. - -### .env (пример) - -```env -BOT_TOKEN=123456789:AA...your_token_here - -# Optional: -# LOGS_CHAT_ID=123456789 -# MAX_FILESIZE=52428800 -# OUTPUT_FOLDER=/tmp/yt-dlp-telegram -# COOKIES_FILE=/app/cookies.txt -YTDLP_JS_RUNTIMES=node -YTDLP_REMOTE_COMPONENTS=ejs:github -YTDLP_INSTAGRAM_IMPERSONATE=chrome -YTDLP_INSTAGRAM_RETRIES=8 -YTDLP_INSTAGRAM_FRAGMENT_RETRIES=8 -YTDLP_INSTAGRAM_SOCKET_TIMEOUT=30 +The installer preserves an existing `.env` and refuses to update a repository with tracked local modifications. + +## Configuration + +Only `BOT_TOKEN` is required. + +| Variable | Default | Description | +|---|---:|---| +| `BOT_TOKEN` | — | Telegram bot token | +| `LOGS_CHAT_ID` | empty | Optional chat for future critical operational notifications | +| `MAX_FILESIZE` | `52428800` | Maximum upload size in bytes | +| `WORKERS` | `2` | Concurrent download workers | +| `MAX_QUEUE` | `200` | In-memory queue capacity | +| `UPLOAD_WORKERS` | `2` | Concurrent local-file uploads to Telegram | +| `JOB_TIMEOUT_SECONDS` | `900` | Download deadline | +| `DEFAULT_LANGUAGE` | `en` | Default UI language: `en` or `ru` | +| `DELETE_ORIGINAL` | `true` | Remove a link after successful delivery | +| `MEDIA_CACHE_ENABLED` | `true` | Enable disk and Telegram `file_id` caches | +| `DISK_CACHE_MAX_FILES` | `5` | Maximum recent media files on disk | +| `DISK_CACHE_TTL_SECONDS` | `300` | Disk-cache lifetime after last use | +| `FILE_ID_CACHE_MAX_ITEMS` | `500` | Maximum persistent Telegram media entries | +| `FILE_ID_CACHE_TTL_DAYS` | `30` | Telegram media-cache lifetime | +| `COOKIES_FILE` | empty | Optional cookies file; `/app/data/cookies.txt` is convenient in Docker | +| `LOG_LEVEL` | `INFO` | Python logging level | +| `YTDLP_CONCURRENT_FRAGMENTS` | `4` | Concurrent fragments per download | + +See [.env-example](.env-example) for yt-dlp site-specific options. + +## Commands + +- `/start` and `/help` — show instructions. +- `/language` — show the current language. +- `/language en` or `/language ru` — change the group language; group administrators only. +- `/settings` — show the current group settings; group administrators only. +- `/delete_original on` or `/delete_original off` — configure successful-link deletion; group administrators only. +- `@BotName me` or `@BotName я` — toggle automatic downloads for yourself. +- When opted out, use `@BotName ` for a manual download. + +The bot remains silent for ordinary automatic failures by design. + +## Persistent data + +Runtime state is stored in `data/`: + +```text +data/ +├── settings.json # per-chat language and settings +├── users.json # per-user opt-out choices +├── state.json # welcome and migration state +├── media_cache.json # Telegram file_id cache +└── cache/ # short-lived downloaded media ``` -### Переменные окружения - -* **BOT_TOKEN** *(обязательно)* — токен бота. -* **LOGS_CHAT_ID** *(опционально)* — чат/канал/диалог для логов запросов (число). -* **MAX_FILESIZE** *(опционально)* — максимальный размер файла в байтах (по умолчанию 50 MB). -* **OUTPUT_FOLDER** *(опционально)* — временная папка для загрузок (по умолчанию `/tmp/yt-dlp-telegram`). -* **COOKIES_FILE** *(опционально)* — путь к cookies-файлу (если нужно для сложных сайтов/авторизации). -* **YTDLP_JS_RUNTIMES** *(по умолчанию `node`)* — JS runtime для YouTube extractor. -* **YTDLP_REMOTE_COMPONENTS** *(по умолчанию `ejs:github`)* — удалённые EJS-компоненты для устойчивости YouTube. -* **YTDLP_INSTAGRAM_IMPERSONATE** *(по умолчанию `chrome`)* — профиль impersonation для Instagram. -* **YTDLP_INSTAGRAM_RETRIES** *(по умолчанию `8`)* — retries для Instagram. -* **YTDLP_INSTAGRAM_FRAGMENT_RETRIES** *(по умолчанию `8`)* — fragment retries для Instagram. -* **YTDLP_INSTAGRAM_SOCKET_TIMEOUT** *(по умолчанию `30`)* — socket timeout для Instagram. - ---- - -## Управление ботом в группе - -### Авто-скачивание по умолчанию - -* Любая ссылка на видео → скачивание → отправка → удаление исходной ссылки. - -### Отключить авто-скачивание для себя - -В группе напишите: - -* `@ИмяБота @ВашНик` -* или `@ИмяБота me` -* или `@ИмяБота я` - -Повторите — включится обратно. - -### Ручной режим (когда Вы отключились) - -Если у Вас отключено авто — скачивание только с упоминанием: - -* `@ИмяБота <ссылка>` - ---- - -## Структура данных +Writes use a temporary file, `fsync`, and atomic replacement. A last-known-good `.bak` file is retained. Invalid JSON is quarantined rather than silently overwritten. Existing `data/prefs.json` is imported once and left untouched as a fallback. -Для сохранения настроек opt-out используется файл: +## Logs -* `data/prefs.json` - -Он монтируется в контейнер (Docker), поэтому переживает рестарты. - ---- - -## Обновление на сервере (Docker) +Application logs are written to stdout and `logs/bot.log`. They rotate daily at UTC midnight, with 60 daily files retained. Query strings are omitted from logged URLs to reduce accidental exposure of tokens. ```bash -cd /root/LinkDownloaderBotForGroups - -docker compose version >/dev/null 2>&1 && COMPOSE="docker compose" || COMPOSE="docker-compose" - -$COMPOSE -p linkdownloaderbotforgroups down - -git fetch --all --prune -# Важно: если Вы ничего локально не правили, будет чисто -# Если правили — сохраните свои изменения отдельно - -git pull --ff-only - -$COMPOSE -p linkdownloaderbotforgroups up -d --build -$COMPOSE -p linkdownloaderbotforgroups logs -f --tail=200 +docker compose logs -f --tail=200 +ls -la logs/ ``` -### Как сменить токен +## Automatic yt-dlp updates -1. Обновите `.env`: +Video extractors change frequently. `install.sh` enables a nightly systemd timer when systemd is available. The updater: -```bash -cd /root/LinkDownloaderBotForGroups -nano .env -``` +1. preserves the current Docker image as a rollback image; +2. refreshes only the yt-dlp image layer; +3. runs an import/version smoke test; +4. recreates the service; +5. restores the previous image if the new container does not stay running. -2. Пересоберите и перезапустите: +Updater output is stored in daily `logs/updater-YYYY-MM-DD.log` files and retained for 60 days. ```bash -docker compose version >/dev/null 2>&1 && COMPOSE="docker compose" || COMPOSE="docker-compose" -$COMPOSE -p linkdownloaderbotforgroups up -d --build -$COMPOSE -p linkdownloaderbotforgroups logs -f --tail=200 -``` - ---- - -## Диагностика и логи - -### Посмотреть статус - -```bash -cd /root/LinkDownloaderBotForGroups - -docker compose version >/dev/null 2>&1 && COMPOSE="docker compose" || COMPOSE="docker-compose" - -$COMPOSE -p linkdownloaderbotforgroups ps -$COMPOSE -p linkdownloaderbotforgroups logs --tail=200 +systemctl status linkdownloaderbotforgroups-yt-dlp-update.timer +sudo systemctl start linkdownloaderbotforgroups-yt-dlp-update.service ``` -### Посмотреть ресурсы +Other dependencies are deliberately updated through reviewed pull requests instead of unattended nightly upgrades. -```bash -docker stats --no-stream linkdownloaderbot -``` +## Development -### Проверить, что токен не пустой (без вывода токена) +Python 3.11 or newer is supported. ```bash -cd /root/LinkDownloaderBotForGroups - -# Проверка .env (покажет длину и head/tail) -token="$(grep -m1 '^BOT_TOKEN=' .env | cut -d= -f2- | tr -d '\r\n')" -echo "BOT_TOKEN length: ${#token}" -echo "BOT_TOKEN head/tail: ${token:0:5}...${token: -5}" - -# Проверка внутри контейнера (без печати токена) -docker exec -i linkdownloaderbot sh -lc 'python - < Optional[int]: - dur = meta.get("duration") - if isinstance(dur, (int, float)) and dur > 0: - return int(dur) - return None - +@dataclass(frozen=True) +class MediaMetadata: + url: str + info: dict[str, Any] + media_key: str + source_name: str -def format_size_bytes(fmt: Dict[str, Any], dur: Optional[int]) -> Tuple[Optional[int], bool]: - fs = fmt.get("filesize") - if isinstance(fs, int) and fs > 0: - return fs, True - fsa = fmt.get("filesize_approx") - if isinstance(fsa, int) and fsa > 0: - return fsa, True +def _duration(info: dict[str, Any]) -> int | None: + value = info.get("duration") + return int(value) if isinstance(value, (int, float)) and value > 0 else None - tbr = fmt.get("tbr") - if dur and isinstance(tbr, (int, float)) and tbr > 0: - est = int(dur * (float(tbr) * 1000.0 / 8.0)) - if est > 0: - return est, False - return None, False +def _size(fmt: dict[str, Any], duration: int | None) -> int | None: + for key in ("filesize", "filesize_approx"): + value = fmt.get(key) + if isinstance(value, (int, float)) and value > 0: + return int(value) + bitrate = fmt.get("tbr") + if duration and isinstance(bitrate, (int, float)) and bitrate > 0: + return int(duration * bitrate * 1000 / 8) + return None -def vcodec_pref_rank(vcodec: Any) -> int: - s = str(vcodec or "") - return 2 if s.startswith("avc1") else 1 +def _video_score(fmt: dict[str, Any]) -> tuple[int, int, int, float]: + height = min(int(fmt.get("height") or 0), 2160) + compatible = int(str(fmt.get("vcodec") or "").startswith("avc1")) + fps = min(int(fmt.get("fps") or 0), 120) + bitrate = float(fmt.get("tbr") or 0) + return height, compatible, fps, bitrate -def acodec_pref_rank(acodec: Any) -> int: - s = str(acodec or "") - return 2 if s.startswith("mp4a") else 1 +def _audio_score(fmt: dict[str, Any]) -> tuple[int, float]: + compatible = int(str(fmt.get("acodec") or "").startswith("mp4a")) + return compatible, float(fmt.get("abr") or fmt.get("tbr") or 0) -def best_progressive_mp4(meta: Dict[str, Any]) -> Optional[Dict[str, Any]]: - best = None - best_key = None +def select_format(info: dict[str, Any], max_bytes: int) -> tuple[str, str | None]: + """Select the best MP4 plan that is likely to fit the Telegram limit.""" + formats = [item for item in info.get("formats", []) if isinstance(item, dict)] + duration = _duration(info) + budget = int(max_bytes * 0.96) - for f in meta.get("formats", []) or []: - if f.get("ext") != "mp4": + progressive: list[dict[str, Any]] = [] + video_only: list[dict[str, Any]] = [] + audio_only: list[dict[str, Any]] = [] + for fmt in formats: + if not fmt.get("format_id"): continue - if f.get("vcodec") == "none" or f.get("acodec") == "none": - continue - - fid = f.get("format_id") - if not fid: - continue - - height = f.get("height") or 0 - fps = f.get("fps") or 0 - tbr = f.get("tbr") or 0 - - v_rank = vcodec_pref_rank(f.get("vcodec")) - a_rank = acodec_pref_rank(f.get("acodec")) - - key = (int(v_rank), int(a_rank), int(height), int(fps), float(tbr)) - - if best is None or key > best_key: - best = { - "kind": "progressive", - "format_spec": str(fid), - "merge_output_format": None, - } - best_key = key - - return best - - -def best_separate_mp4_m4a(meta: Dict[str, Any]) -> Optional[Dict[str, Any]]: - dur = duration_sec(meta) - - best_v = None - best_v_key = None - best_a = None - best_a_key = None - - for f in meta.get("formats", []) or []: - vcodec = f.get("vcodec") - acodec = f.get("acodec") - ext = f.get("ext") - - if ext == "mp4" and vcodec != "none" and acodec == "none": - fid = f.get("format_id") - if not fid: - continue - - height = f.get("height") or 0 - fps = f.get("fps") or 0 - tbr = f.get("tbr") or 0 - - v_rank = vcodec_pref_rank(vcodec) - key = (int(v_rank), int(height), int(fps), float(tbr)) - - if best_v is None or key > best_v_key: - size, conf = format_size_bytes(f, dur) - best_v = {"f": f, "size": size, "conf": conf} - best_v_key = key - - if vcodec == "none" and acodec != "none" and ext in ("m4a", "mp4"): - fid = f.get("format_id") - if not fid: + video = fmt.get("vcodec") not in {None, "none"} + audio = fmt.get("acodec") not in {None, "none"} + ext = fmt.get("ext") + if ext == "mp4" and video and audio: + if (_size(fmt, duration) or budget) <= budget: + progressive.append(fmt) + elif ext == "mp4" and video and not audio: + video_only.append(fmt) + elif ext in {"m4a", "mp4"} and audio and not video: + audio_only.append(fmt) + + pairs: list[tuple[dict[str, Any], dict[str, Any]]] = [] + for video in video_only: + video_size = _size(video, duration) + for audio in audio_only: + audio_size = _size(audio, duration) + if video_size is not None and audio_size is not None and video_size + audio_size > budget: continue - - abr = f.get("abr") or f.get("tbr") or 0 - a_rank = acodec_pref_rank(acodec) - key = (int(a_rank), float(abr)) - - if best_a is None or key > best_a_key: - size, conf = format_size_bytes(f, dur) - best_a = {"f": f, "size": size, "conf": conf} - best_a_key = key - - if not best_v or not best_a: - return None - - vf = best_v["f"] - af = best_a["f"] - return { - "kind": "separate", - "format_spec": f"{vf.get('format_id')}+{af.get('format_id')}", - "merge_output_format": "mp4", - } - - -def build_video_plan_like_main1(meta: Dict[str, Any]) -> Optional[Dict[str, Any]]: - p = best_progressive_mp4(meta) - if p: - return p - p = best_separate_mp4_m4a(meta) - if p: - return p - return None - - -def _parse_csv_list(raw: str) -> List[str]: - return [x.strip() for x in (raw or "").split(",") if x.strip()] - - -def _build_http_headers() -> Dict[str, str]: + pairs.append((video, audio)) + known_progressive = [fmt for fmt in progressive if _size(fmt, duration) is not None] + if known_progressive: + progressive = known_progressive + known_pairs = [ + pair for pair in pairs if _size(pair[0], duration) is not None and _size(pair[1], duration) is not None + ] + if known_pairs: + pairs = known_pairs + + best_progressive = max(progressive, key=_video_score) if progressive else None + best_pair = max(pairs, key=lambda pair: (_video_score(pair[0]), _audio_score(pair[1]))) if pairs else None + if best_progressive is not None and ( + best_pair is None or _video_score(best_progressive) >= _video_score(best_pair[0]) + ): + return str(best_progressive["format_id"]), None + if best_pair is not None: + video, audio = best_pair + return f"{video['format_id']}+{audio['format_id']}", "mp4" + + # yt-dlp applies the actual max_filesize guard. This fallback is needed when + # extractors do not expose enough size metadata to make a local decision. + return "b[ext=mp4]/bv*[ext=mp4]+ba[ext=m4a]/bv*+ba/b", "mp4" + + +def _csv(raw: str) -> list[str]: + return [item.strip() for item in (raw or "").split(",") if item.strip()] + + +def _headers() -> dict[str, str]: return { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" } -def _parse_impersonate_target(raw: str) -> Optional[Any]: - name = (raw or "").strip() - if not name or ImpersonateTarget is None: +def _impersonate(raw: str) -> Any | None: + if not raw or ImpersonateTarget is None: return None try: - return ImpersonateTarget.from_str(name) - except Exception: + return ImpersonateTarget.from_str(raw.strip()) + except (ValueError, TypeError): return None -def _youtube_api_opts() -> Dict[str, Any]: - runtimes = _parse_csv_list(env_config.YTDLP_JS_RUNTIMES) - remote_components = _parse_csv_list(env_config.YTDLP_REMOTE_COMPONENTS) - opts: Dict[str, Any] = {} - if runtimes: - opts["js_runtimes"] = {name: {} for name in runtimes} - if remote_components: - opts["remote_components"] = remote_components - return opts - - -def _instagram_api_opts(force_impersonate: bool) -> Dict[str, Any]: - opts: Dict[str, Any] = {} - if force_impersonate: - target = _parse_impersonate_target(env_config.YTDLP_INSTAGRAM_IMPERSONATE) +def _site_options(url: str, *, instagram_impersonate: bool = True) -> dict[str, Any]: + options: dict[str, Any] = {} + if is_youtube_url(url): + runtimes = _csv(env_config.YTDLP_JS_RUNTIMES) + components = _csv(env_config.YTDLP_REMOTE_COMPONENTS) + if runtimes: + options["js_runtimes"] = {runtime: {} for runtime in runtimes} + if components: + options["remote_components"] = components + if is_instagram_url(url): + options.update( + retries=env_config.YTDLP_INSTAGRAM_RETRIES, + fragment_retries=env_config.YTDLP_INSTAGRAM_FRAGMENT_RETRIES, + socket_timeout=env_config.YTDLP_INSTAGRAM_SOCKET_TIMEOUT, + ) + target = _impersonate(env_config.YTDLP_INSTAGRAM_IMPERSONATE) if instagram_impersonate else None if target is not None: - opts["impersonate"] = target - return opts + options["impersonate"] = target + return options -def _base_meta_opts(cookiefile_value: Optional[str], http_headers: Dict[str, str]) -> Dict[str, Any]: - opts: Dict[str, Any] = { +def _base_options(cookie_file: Path | None) -> dict[str, Any]: + options: dict[str, Any] = { "quiet": True, "no_warnings": True, + "noprogress": True, "noplaylist": True, "socket_timeout": 20, "retries": 5, - "http_headers": http_headers, + "http_headers": _headers(), } - if cookiefile_value: - opts["cookiefile"] = cookiefile_value - return opts - - -def _base_download_opts( - outtmpl: str, - max_send_bytes: int, - concurrent_fragments: int, - http_headers: Dict[str, str], -) -> Dict[str, Any]: - return { - "outtmpl": outtmpl, - "noplaylist": True, - "quiet": True, - "no_warnings": True, - "concurrent_fragment_downloads": concurrent_fragments, - "retries": 5, - "fragment_retries": 5, - "socket_timeout": 20, - "max_filesize": max_send_bytes, - "http_headers": http_headers, - } - + if cookie_file and cookie_file.is_file(): + options["cookiefile"] = os.fspath(cookie_file) + return options -def _extract_meta(url: str, opts: Dict[str, Any]) -> Dict[str, Any]: - with yt_dlp.YoutubeDL(opts) as ydl: - return ydl.extract_info(url, download=False) - -def _download(url: str, opts: Dict[str, Any]) -> Dict[str, Any]: - with yt_dlp.YoutubeDL(opts) as ydl: - return ydl.extract_info(url, download=True) - - -def download_with_ytdlp( - url: str, +def extract_metadata(url: str, cookie_file: Path | None = None) -> MediaMetadata: + options = _base_options(cookie_file) + options.update(_site_options(url)) + try: + with yt_dlp.YoutubeDL(options) as ydl: + info = ydl.extract_info(url, download=False) + except Exception: + if not is_instagram_url(url): + raise + fallback = _base_options(cookie_file) + with yt_dlp.YoutubeDL(fallback) as ydl: + info = ydl.extract_info(url, download=False) + if not isinstance(info, dict): + raise RuntimeError("extractor returned no metadata") + extractor = str(info.get("extractor_key") or info.get("extractor") or "generic").lower() + media_id = str(info.get("id") or info.get("display_id") or "").strip() + if not media_id: + raise RuntimeError("extractor returned no media id") + source = str(info.get("extractor_key") or info.get("extractor") or "Video") + return MediaMetadata(url=url, info=info, media_key=f"{extractor}:{media_id}", source_name=source) + + +def download_metadata( + metadata: MediaMetadata, out_prefix: str, - output_folder: str, + output_folder: Path, *, max_send_bytes: int, concurrent_fragments: int, -) -> Dict[str, Any]: - os.makedirs(output_folder, exist_ok=True) - outtmpl = os.path.join(output_folder, f"{out_prefix}.%(ext)s") - - cookies_file = getattr(config, "cookies_file", None) - cookiefile_value = None - if isinstance(cookies_file, str) and cookies_file.strip() and os.path.exists(cookies_file.strip()): - cookiefile_value = cookies_file.strip() - - http_headers = _build_http_headers() - - is_yt = is_youtube_url(url) - is_ig = is_instagram_url(url) - - meta_opts = _base_meta_opts(cookiefile_value, http_headers) - if is_yt: - meta_opts.update(_youtube_api_opts()) - if is_ig: - meta_opts.update( - { - "retries": env_config.YTDLP_INSTAGRAM_RETRIES, - "fragment_retries": env_config.YTDLP_INSTAGRAM_FRAGMENT_RETRIES, - "socket_timeout": env_config.YTDLP_INSTAGRAM_SOCKET_TIMEOUT, - } - ) - meta_opts.update(_instagram_api_opts(force_impersonate=True)) - + cookie_file: Path | None = None, + deadline: float | None = None, +) -> dict[str, Any]: + output_folder.mkdir(parents=True, exist_ok=True) + outtmpl = os.fspath(output_folder / f"{out_prefix}.%(ext)s") + format_spec, merge_format = select_format(metadata.info, max_send_bytes) + + def progress_hook(_status: dict[str, Any]) -> None: + if deadline is not None and time.monotonic() > deadline: + raise yt_dlp.utils.DownloadError("download deadline exceeded") + + options = _base_options(cookie_file) + options.update( + outtmpl=outtmpl, + concurrent_fragment_downloads=concurrent_fragments, + fragment_retries=5, + max_filesize=max_send_bytes, + format=format_spec, + progress_hooks=[progress_hook], + ) + if merge_format: + options["merge_output_format"] = merge_format + options.update(_site_options(metadata.url)) try: - meta = _extract_meta(url, meta_opts) + with yt_dlp.YoutubeDL(options) as ydl: + return ydl.process_ie_result(dict(metadata.info), download=True) except Exception: - if not is_ig: + if not (is_youtube_url(metadata.url) or is_instagram_url(metadata.url)): raise - # Fail-soft: Instagram fallback without forced impersonation and with base retries/timeouts. - fallback_meta_opts = _base_meta_opts(cookiefile_value, http_headers) - meta = _extract_meta(url, fallback_meta_opts) - - plan = build_video_plan_like_main1(meta) - if plan: - format_value = str(plan.get("format_spec")) - merge_value = plan.get("merge_output_format") - else: - format_value = "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/bv*+ba/b" - merge_value = "mp4" - - primary_opts = _base_download_opts( - outtmpl=outtmpl, - max_send_bytes=max_send_bytes, - concurrent_fragments=concurrent_fragments, - http_headers=http_headers, - ) - primary_opts["format"] = format_value - if merge_value: - primary_opts["merge_output_format"] = str(merge_value) - if cookiefile_value: - primary_opts["cookiefile"] = cookiefile_value - if is_yt: - primary_opts.update(_youtube_api_opts()) - if is_ig: - primary_opts.update( - { - "retries": env_config.YTDLP_INSTAGRAM_RETRIES, - "fragment_retries": env_config.YTDLP_INSTAGRAM_FRAGMENT_RETRIES, - "socket_timeout": env_config.YTDLP_INSTAGRAM_SOCKET_TIMEOUT, - } + fallback = _base_options(cookie_file) + fallback.update( + outtmpl=outtmpl, + concurrent_fragment_downloads=1, + fragment_retries=5, + max_filesize=max_send_bytes, + format=format_spec, + progress_hooks=[progress_hook], ) - primary_opts.update(_instagram_api_opts(force_impersonate=True)) - - try: - return _download(url, primary_opts) - except Exception as primary_err: - if is_ig: - # Fail-soft: retry Instagram without forced impersonation and with default retries/timeouts. - ig_fallback_opts = _base_download_opts( - outtmpl=outtmpl, - max_send_bytes=max_send_bytes, - concurrent_fragments=concurrent_fragments, - http_headers=http_headers, - ) - ig_fallback_opts["format"] = format_value - if merge_value: - ig_fallback_opts["merge_output_format"] = str(merge_value) - if cookiefile_value: - ig_fallback_opts["cookiefile"] = cookiefile_value - return _download(url, ig_fallback_opts) - - if is_yt: - # Extractor churn fallback for YouTube. - yt_fallback_opts = _base_download_opts( - outtmpl=outtmpl, - max_send_bytes=max_send_bytes, - concurrent_fragments=1, - http_headers=http_headers, - ) - yt_fallback_opts.update(_youtube_api_opts()) - yt_fallback_opts["format"] = "18/best[ext=mp4]/best" - return _download(url, yt_fallback_opts) - - raise primary_err + if merge_format: + fallback["merge_output_format"] = merge_format + fallback.update(_site_options(metadata.url, instagram_impersonate=False)) + with yt_dlp.YoutubeDL(fallback) as ydl: + return ydl.extract_info(metadata.url, download=True) + + +def find_downloaded_file(info: dict[str, Any], prefix: str, output_folder: Path) -> Path | None: + exact = output_folder / f"{prefix}.mp4" + if exact.is_file(): + return exact + candidates: list[Path] = [] + for path in output_folder.glob(f"{prefix}.*"): + lower = path.name.lower() + if not path.is_file() or lower.endswith((".part", ".ytdl", ".tmp", ".temp")): + continue + name_after_prefix = lower[len(prefix) :] + if name_after_prefix.startswith(".f") and name_after_prefix[2:].split(".", 1)[0].isdigit(): + continue + candidates.append(path) + if not candidates: + return None + candidates.sort(key=lambda path: (path.suffix.lower() == ".mp4", path.stat().st_mtime), reverse=True) + return candidates[0] diff --git a/app/env_config.py b/app/env_config.py index 2ca2d75..f957071 100644 --- a/app/env_config.py +++ b/app/env_config.py @@ -8,9 +8,12 @@ def _env_int(name: str, default: int) -> int: if not raw: return default try: - return int(raw) - except Exception: - return default + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + if value < 1: + raise RuntimeError(f"{name} must be positive") + return value def _env_str(name: str, default: str) -> str: diff --git a/app/i18n.py b/app/i18n.py new file mode 100644 index 0000000..24855b4 --- /dev/null +++ b/app/i18n.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Any + +TEXTS: dict[str, dict[str, str]] = { + "en": { + "private_help": ( + "Video downloader for Telegram groups\n\n" + "1. Add me to a group.\n" + "2. Disable Group Privacy in BotFather.\n" + "3. Grant permission to delete messages.\n\n" + "Post a video link and I will quietly replace it with the video.\n\n" + "Use /language en or /language ru to change the language.\n" + 'Source: GitHub' + ), + "group_help": ( + "How to use\n" + "Post a video link. I will download the video, publish it silently, and delete the original message after success.\n\n" + "Personal opt-out\n" + "Send {bot_mention} me to toggle automatic downloads for yourself.\n" + "When disabled, use {bot_mention} <link>.\n\n" + "Administrators can change the language with /language en or /language ru." + ), + "admin_hint": ( + "⚠️ Administrator permission recommended\n" + "Grant permission to delete messages so I can remove original links after a successful upload.\n\n" + ), + "already_welcomed": "The instructions were already sent. Use /help to show them again.", + "private_hint": "Use /help to see the instructions.", + "opted_out": ( + "{who}, automatic downloads are now disabled for you.\n" + "Mention {bot_mention} together with a link to download it." + ), + "opted_in": "{who}, automatic downloads are enabled again. You can simply post links.", + "language_current": "Current language: {language}. Available: en, ru.", + "language_changed": "Language changed to English.", + "language_admin_only": "Only group administrators can change the language.", + "language_invalid": "Supported languages: en, ru.", + "admin_only": "Only group administrators can change this setting.", + "settings_summary": "Group settings\nLanguage: {language}\nDelete original link: {delete_original}", + "delete_usage": "Use /delete_original on or /delete_original off.", + "delete_changed": "Deleting original links is now {state}.", + "state_on": "enabled", + "state_off": "disabled", + "caption": 'Original video · {source}\nFrom {sender}', + }, + "ru": { + "private_help": ( + "Бот для скачивания видео в группах Telegram\n\n" + "1. Добавьте меня в группу.\n" + "2. Отключите Group Privacy в BotFather.\n" + "3. Разрешите удалять сообщения.\n\n" + "Отправьте ссылку на видео — я тихо заменю её готовым видео.\n\n" + "Язык: /language en или /language ru.\n" + 'Исходный код: GitHub' + ), + "group_help": ( + "Как пользоваться\n" + "Отправьте ссылку на видео. Я скачаю и тихо опубликую видео, а после успеха удалю исходное сообщение.\n\n" + "Персональное отключение\n" + "Отправьте {bot_mention} я, чтобы отключить или включить автоматическое скачивание для себя.\n" + "Когда оно отключено, используйте {bot_mention} <ссылка>.\n\n" + "Администраторы могут изменить язык: /language en или /language ru." + ), + "admin_hint": ( + "⚠️ Рекомендуются права администратора\n" + "Разрешите удалять сообщения, чтобы я удалял исходные ссылки после успешной отправки.\n\n" + ), + "already_welcomed": "Инструкция уже отправлялась. Используйте /help, чтобы показать её снова.", + "private_hint": "Используйте /help, чтобы увидеть инструкцию.", + "opted_out": ( + "{who}, автоматическое скачивание для Вас отключено.\n" + "Для загрузки упомяните {bot_mention} вместе со ссылкой." + ), + "opted_in": "{who}, автоматическое скачивание снова включено. Можно просто отправлять ссылки.", + "language_current": "Текущий язык: {language}. Доступны: en, ru.", + "language_changed": "Язык изменён на русский.", + "language_admin_only": "Изменять язык могут только администраторы группы.", + "language_invalid": "Поддерживаемые языки: en, ru.", + "admin_only": "Изменять эту настройку могут только администраторы группы.", + "settings_summary": "Настройки группы\nЯзык: {language}\nУдаление исходной ссылки: {delete_original}", + "delete_usage": "Используйте /delete_original on или /delete_original off.", + "delete_changed": "Удаление исходных ссылок теперь {state}.", + "state_on": "включено", + "state_off": "выключено", + "caption": 'Ссылка на видео · {source}\nОт {sender}', + }, +} + + +def tr(locale: str, key: str, **values: Any) -> str: + catalog = TEXTS.get(locale, TEXTS["en"]) + template = catalog.get(key, TEXTS["en"].get(key, key)) + return template.format(**values) diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 0000000..f24ba05 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class Job: + job_id: str + chat_id: int + message_thread_id: int | None + original_message_id: int + user_id: int + url: str + url_key: str + sender_name: str + delete_original: bool + + +@dataclass +class Flight: + url_keys: set[str] + jobs: list[Job] = field(default_factory=list) + cursor: int = 0 + media_key: str | None = None + + +class FlightCoordinator: + """Coalesce equal URLs first and equal extractor media IDs after metadata.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._by_url: dict[str, Flight] = {} + self._by_media: dict[str, Flight] = {} + + def submit(self, job: Job) -> Flight | None: + with self._lock: + existing = self._by_url.get(job.url_key) + if existing is not None: + existing.jobs.append(job) + return None + flight = Flight(url_keys={job.url_key}, jobs=[job]) + self._by_url[job.url_key] = flight + return flight + + def promote(self, flight: Flight, media_key: str) -> bool: + with self._lock: + existing = self._by_media.get(media_key) + if existing is flight: + return True + if existing is not None: + existing.jobs.extend(flight.jobs[flight.cursor :]) + existing.url_keys.update(flight.url_keys) + for url_key in flight.url_keys: + self._by_url[url_key] = existing + flight.cursor = len(flight.jobs) + return False + flight.media_key = media_key + self._by_media[media_key] = flight + return True + + def pending(self, flight: Flight) -> list[Job]: + with self._lock: + jobs = list(flight.jobs[flight.cursor :]) + flight.cursor = len(flight.jobs) + return jobs + + def finish_if_idle(self, flight: Flight) -> bool: + with self._lock: + if flight.cursor < len(flight.jobs): + return False + for url_key in flight.url_keys: + if self._by_url.get(url_key) is flight: + self._by_url.pop(url_key, None) + if flight.media_key and self._by_media.get(flight.media_key) is flight: + self._by_media.pop(flight.media_key, None) + return True + + def abort(self, flight: Flight) -> list[Job]: + with self._lock: + jobs = list(flight.jobs[flight.cursor :]) + flight.cursor = len(flight.jobs) + self.finish_if_idle(flight) + return jobs diff --git a/app/logging_setup.py b/app/logging_setup.py new file mode 100644 index 0000000..ef595f7 --- /dev/null +++ b/app/logging_setup.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import logging +import re +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path + + +class RedactingFormatter(logging.Formatter): + _url_query = re.compile(r"(https?://[^\s?]+)\?[^\s]+", re.IGNORECASE) + + def format(self, record: logging.LogRecord) -> str: + return self._url_query.sub(r"\1?", super().format(record)) + + +def configure_logging(logs_dir: Path, level: str = "INFO") -> logging.Logger: + logs_dir.mkdir(parents=True, exist_ok=True) + root = logging.getLogger() + root.setLevel(getattr(logging, level, logging.INFO)) + root.handlers.clear() + + formatter = RedactingFormatter( + "%(asctime)sZ %(levelname)s %(name)s %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S", + ) + stream = logging.StreamHandler() + stream.setFormatter(formatter) + + file_handler = TimedRotatingFileHandler( + logs_dir / "bot.log", + when="midnight", + interval=1, + backupCount=60, + encoding="utf-8", + utc=True, + ) + file_handler.setFormatter(formatter) + root.addHandler(stream) + root.addHandler(file_handler) + return logging.getLogger("link_downloader_bot") diff --git a/app/media_cache.py b/app/media_cache.py new file mode 100644 index 0000000..0afaf8b --- /dev/null +++ b/app/media_cache.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import hashlib +import logging +import os +import threading +import time +from pathlib import Path + +LOG = logging.getLogger(__name__) + + +class DiskMediaCache: + def __init__(self, directory: Path, max_files: int = 5, ttl_seconds: int = 300) -> None: + self.directory = directory + self.max_files = max_files + self.ttl_seconds = ttl_seconds + self._lock = threading.RLock() + directory.mkdir(parents=True, exist_ok=True) + + @staticmethod + def prefix(media_key: str) -> str: + return "media-" + hashlib.sha256(media_key.encode("utf-8")).hexdigest()[:24] + + def get(self, media_key: str) -> Path | None: + with self._lock: + now = time.time() + candidates = [path for path in self.directory.glob(f"{self.prefix(media_key)}.*") if self._usable(path)] + if not candidates: + return None + path = max(candidates, key=lambda item: item.stat().st_mtime) + if now - path.stat().st_mtime > self.ttl_seconds: + self._unlink(path) + return None + os.utime(path, (now, now)) + return path + + def maintain(self) -> None: + with self._lock: + now = time.time() + files = [path for path in self.directory.iterdir() if self._usable(path)] + for path in files: + try: + if now - path.stat().st_mtime > self.ttl_seconds: + self._unlink(path) + except OSError: + LOG.exception("cannot inspect cache file path=%s", path) + files = [path for path in self.directory.iterdir() if self._usable(path)] + files.sort(key=lambda item: item.stat().st_mtime, reverse=True) + for path in files[self.max_files :]: + self._unlink(path) + self._remove_stale_intermediates(now) + + def remove_prefix_except(self, prefix: str, keep: Path | None = None) -> None: + with self._lock: + for path in self.directory.glob(f"{prefix}.*"): + if keep is not None and path == keep: + continue + self._unlink(path) + + def _remove_stale_intermediates(self, now: float) -> None: + for path in self.directory.iterdir(): + if not path.is_file() or self._usable(path): + continue + try: + if now - path.stat().st_mtime > self.ttl_seconds: + self._unlink(path) + except OSError: + LOG.exception("cannot inspect temporary file path=%s", path) + + @staticmethod + def _usable(path: Path) -> bool: + lower = path.name.lower() + return path.is_file() and not lower.endswith((".part", ".ytdl", ".tmp", ".temp", ".meta")) and ".f" not in lower + + @staticmethod + def _unlink(path: Path) -> None: + try: + path.unlink(missing_ok=True) + except OSError: + LOG.exception("cannot remove cache file path=%s", path) diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 0000000..630ff9d --- /dev/null +++ b/app/settings.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _load_dotenv(path: Path) -> None: + """Load a small, shell-free subset of .env for non-Docker launches.""" + if not path.is_file(): + return + for raw_line in path.read_text(encoding="utf-8-sig").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if value[:1] == value[-1:] and value[:1] in {'"', "'"}: + value = value[1:-1] + if key and key not in os.environ: + os.environ[key] = value + + +def _integer(name: str, default: int, minimum: int, maximum: int) -> int: + raw = (os.getenv(name) or "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + if not minimum <= value <= maximum: + raise RuntimeError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _boolean(name: str, default: bool) -> bool: + raw = (os.getenv(name) or "").strip().lower() + if not raw: + return default + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + raise RuntimeError(f"{name} must be true or false") + + +@dataclass(frozen=True) +class Settings: + token: str + logs_chat_id: int | None + data_dir: Path + output_dir: Path + logs_dir: Path + cookies_file: Path | None + max_filesize: int + workers: int + max_queue: int + upload_workers: int + concurrent_fragments: int + job_timeout: int + disk_cache_max_files: int + disk_cache_ttl: int + file_id_cache_max_items: int + file_id_cache_ttl_days: int + media_cache_enabled: bool + delete_original: bool + default_language: str + log_level: str + + +def load_settings(base_dir: Path | None = None) -> Settings: + base = (base_dir or Path(__file__).resolve().parents[1]).resolve() + _load_dotenv(base / ".env") + + token = (os.getenv("BOT_TOKEN") or "").strip() + if not token: + raise RuntimeError("BOT_TOKEN is not set") + + logs_raw = (os.getenv("LOGS_CHAT_ID") or "").strip() + try: + logs_chat_id = int(logs_raw) if logs_raw else None + except ValueError as exc: + raise RuntimeError("LOGS_CHAT_ID must be an integer") from exc + + default_language = (os.getenv("DEFAULT_LANGUAGE") or "en").strip().lower() + if default_language not in {"en", "ru"}: + raise RuntimeError("DEFAULT_LANGUAGE must be en or ru") + + data_dir = Path(os.getenv("DATA_DIR") or base / "data").expanduser().resolve() + output_dir = Path(os.getenv("OUTPUT_FOLDER") or data_dir / "cache").expanduser().resolve() + logs_dir = Path(os.getenv("LOGS_DIR") or base / "logs").expanduser().resolve() + cookies_raw = (os.getenv("COOKIES_FILE") or "").strip() + log_level = (os.getenv("LOG_LEVEL") or "INFO").strip().upper() + if log_level not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}: + raise RuntimeError("LOG_LEVEL must be DEBUG, INFO, WARNING, ERROR, or CRITICAL") + + return Settings( + token=token, + logs_chat_id=logs_chat_id, + data_dir=data_dir, + output_dir=output_dir, + logs_dir=logs_dir, + cookies_file=Path(cookies_raw).expanduser().resolve() if cookies_raw else None, + max_filesize=_integer("MAX_FILESIZE", 50 * 1024 * 1024, 1024 * 1024, 2 * 1024**3), + workers=_integer("WORKERS", 2, 1, 16), + max_queue=_integer("MAX_QUEUE", 200, 1, 10_000), + upload_workers=_integer("UPLOAD_WORKERS", 2, 1, 16), + concurrent_fragments=_integer("YTDLP_CONCURRENT_FRAGMENTS", 4, 1, 16), + job_timeout=_integer("JOB_TIMEOUT_SECONDS", 900, 30, 86_400), + disk_cache_max_files=_integer("DISK_CACHE_MAX_FILES", 5, 1, 100), + disk_cache_ttl=_integer("DISK_CACHE_TTL_SECONDS", 300, 30, 86_400), + file_id_cache_max_items=_integer("FILE_ID_CACHE_MAX_ITEMS", 500, 1, 100_000), + file_id_cache_ttl_days=_integer("FILE_ID_CACHE_TTL_DAYS", 30, 1, 3650), + media_cache_enabled=_boolean("MEDIA_CACHE_ENABLED", True), + delete_original=_boolean("DELETE_ORIGINAL", True), + default_language=default_language, + log_level=log_level, + ) diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..3b1d44c --- /dev/null +++ b/app/storage.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import logging +import os +import shutil +import threading +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +LOG = logging.getLogger(__name__) + + +class JsonFile: + def __init__(self, path: Path, default_factory: Callable[[], dict[str, Any]]) -> None: + self.path = path + self.backup_path = path.with_suffix(path.suffix + ".bak") + self.default_factory = default_factory + self.lock = threading.RLock() + self._data = self._load() + + def _read(self, path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("JSON root must be an object") + return value + + def _load(self) -> dict[str, Any]: + self.path.parent.mkdir(parents=True, exist_ok=True) + if not self.path.exists(): + data = self.default_factory() + self._write(data, make_backup=False) + return data + try: + return self._read(self.path) + except (OSError, ValueError, json.JSONDecodeError): + LOG.exception("invalid JSON store path=%s", self.path) + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + corrupt = self.path.with_name(f"{self.path.name}.corrupt-{stamp}") + try: + os.replace(self.path, corrupt) + except OSError: + LOG.exception("cannot quarantine corrupt store path=%s", self.path) + if self.backup_path.exists(): + try: + data = self._read(self.backup_path) + self._write(data, make_backup=False) + return data + except (OSError, ValueError, json.JSONDecodeError): + LOG.exception("invalid JSON backup path=%s", self.backup_path) + data = self.default_factory() + self._write(data, make_backup=False) + return data + + def _write(self, data: dict[str, Any], *, make_backup: bool = True) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_suffix(self.path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8", newline="\n") as handle: + json.dump(data, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + if make_backup and self.path.exists(): + shutil.copy2(self.path, self.backup_path) + os.replace(tmp, self.path) + + def snapshot(self) -> dict[str, Any]: + with self.lock: + return copy.deepcopy(self._data) + + def update(self, mutator: Callable[[dict[str, Any]], Any]) -> Any: + with self.lock: + updated = copy.deepcopy(self._data) + result = mutator(updated) + if updated != self._data: + self._write(updated) + self._data = updated + return result + + +class Storage: + def __init__(self, data_dir: Path, default_language: str = "en", delete_original: bool = True) -> None: + self.data_dir = data_dir + self.default_language = default_language + self.default_delete_original = delete_original + self.settings = JsonFile(data_dir / "settings.json", lambda: {"version": 1, "chats": {}}) + self.users = JsonFile(data_dir / "users.json", lambda: {"version": 1, "opt_out": {}}) + self.state = JsonFile( + data_dir / "state.json", + lambda: {"version": 1, "welcomed_groups": {}, "welcomed_private": {}, "migrations": {}}, + ) + self.media = JsonFile(data_dir / "media_cache.json", lambda: {"version": 1, "items": {}, "aliases": {}}) + self._migrate_legacy() + + def _migrate_legacy(self) -> None: + legacy = self.data_dir / "prefs.json" + state = self.state.snapshot() + if state.get("migrations", {}).get("prefs_v2") or not legacy.exists(): + return + try: + old = json.loads(legacy.read_text(encoding="utf-8")) + if not isinstance(old, dict): + raise ValueError("legacy JSON root must be an object") + old_opt_out = old.get("opt_out", {}) if isinstance(old.get("opt_out"), dict) else {} + old_groups = old.get("welcomed_groups", {}) if isinstance(old.get("welcomed_groups"), dict) else {} + old_private = old.get("welcomed_private", {}) if isinstance(old.get("welcomed_private"), dict) else {} + + def migrate_users(data: dict[str, Any]) -> None: + current = data.setdefault("opt_out", {}) + for chat_id, legacy_users in old_opt_out.items(): + if isinstance(legacy_users, dict): + current.setdefault(str(chat_id), {}).update(copy.deepcopy(legacy_users)) + + self.users.update(migrate_users) + + def migrate_state(data: dict[str, Any]) -> None: + data.setdefault("welcomed_groups", {}).update(copy.deepcopy(old_groups)) + data.setdefault("welcomed_private", {}).update(copy.deepcopy(old_private)) + data.setdefault("migrations", {})["prefs_v2"] = datetime.now(UTC).isoformat() + + self.state.update(migrate_state) + LOG.info("migrated legacy preferences path=%s", legacy) + except (OSError, ValueError, json.JSONDecodeError): + LOG.exception("cannot migrate legacy preferences path=%s", legacy) + + def chat_language(self, chat_id: int) -> str: + chat = self.settings.snapshot().get("chats", {}).get(str(chat_id), {}) + language = chat.get("language") if isinstance(chat, dict) else None + return language if language in {"en", "ru"} else self.default_language + + def set_chat_language(self, chat_id: int, language: str) -> None: + if language not in {"en", "ru"}: + raise ValueError("unsupported language") + + def mutate(data: dict[str, Any]) -> None: + data.setdefault("chats", {}).setdefault(str(chat_id), {})["language"] = language + + self.settings.update(mutate) + + def delete_original(self, chat_id: int) -> bool: + chat = self.settings.snapshot().get("chats", {}).get(str(chat_id), {}) + if isinstance(chat, dict) and isinstance(chat.get("delete_original"), bool): + return chat["delete_original"] + return self.default_delete_original + + def set_delete_original(self, chat_id: int, enabled: bool) -> None: + def mutate(data: dict[str, Any]) -> None: + data.setdefault("chats", {}).setdefault(str(chat_id), {})["delete_original"] = enabled + + self.settings.update(mutate) + + def is_opted_out(self, chat_id: int, user_id: int) -> bool: + users = self.users.snapshot().get("opt_out", {}).get(str(chat_id), {}) + return bool(users.get(str(user_id), False)) if isinstance(users, dict) else False + + def toggle_opt_out(self, chat_id: int, user_id: int) -> bool: + result = False + + def mutate(data: dict[str, Any]) -> None: + nonlocal result + root = data.setdefault("opt_out", {}) + users = root.setdefault(str(chat_id), {}) + result = not bool(users.get(str(user_id), False)) + if result: + users[str(user_id)] = True + else: + users.pop(str(user_id), None) + if not users: + root.pop(str(chat_id), None) + + self.users.update(mutate) + return result + + def was_welcomed(self, kind: str, identity: int) -> bool: + key = "welcomed_groups" if kind == "group" else "welcomed_private" + return bool(self.state.snapshot().get(key, {}).get(str(identity), False)) + + def mark_welcomed(self, kind: str, identity: int) -> None: + key = "welcomed_groups" if kind == "group" else "welcomed_private" + self.state.update(lambda data: data.setdefault(key, {}).__setitem__(str(identity), True)) + + def get_file_id(self, media_key: str, ttl_days: int) -> str | None: + item = self.media.snapshot().get("items", {}).get(media_key) + if not isinstance(item, dict) or not isinstance(item.get("file_id"), str): + return None + try: + created = datetime.fromisoformat(str(item["created_at"])) + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + if datetime.now(UTC) - created > timedelta(days=ttl_days): + self.remove_file_id(media_key) + return None + except (KeyError, ValueError, TypeError): + self.remove_file_id(media_key) + return None + + def touch(data: dict[str, Any]) -> None: + current = data.setdefault("items", {}).get(media_key) + if isinstance(current, dict): + current["last_used_at"] = datetime.now(UTC).isoformat() + + self.media.update(touch) + return item["file_id"] + + @staticmethod + def _alias_key(url_key: str) -> str: + return hashlib.sha256(url_key.encode("utf-8")).hexdigest() + + def get_cached_by_url(self, url_key: str, ttl_days: int) -> tuple[str, str, str] | None: + snapshot = self.media.snapshot() + media_key = snapshot.get("aliases", {}).get(self._alias_key(url_key)) + if not isinstance(media_key, str): + return None + file_id = self.get_file_id(media_key, ttl_days) + if file_id is None: + return None + item = self.media.snapshot().get("items", {}).get(media_key, {}) + source_name = str(item.get("source_name") or "Video") if isinstance(item, dict) else "Video" + return media_key, file_id, source_name + + def put_file_id( + self, + media_key: str, + file_id: str, + max_items: int, + *, + source_name: str = "Video", + url_keys: set[str] | None = None, + ) -> None: + now = datetime.now(UTC).isoformat() + + def mutate(data: dict[str, Any]) -> None: + items = data.setdefault("items", {}) + aliases = data.setdefault("aliases", {}) + items[media_key] = { + "file_id": file_id, + "source_name": source_name, + "created_at": now, + "last_used_at": now, + } + for url_key in url_keys or set(): + aliases[self._alias_key(url_key)] = media_key + while len(items) > max_items: + oldest = min(items, key=lambda key: str(items[key].get("last_used_at", ""))) + items.pop(oldest, None) + aliases = {alias: target for alias, target in aliases.items() if target != oldest} + data["aliases"] = aliases + + self.media.update(mutate) + + def remove_file_id(self, media_key: str) -> None: + def mutate(data: dict[str, Any]) -> None: + data.setdefault("items", {}).pop(media_key, None) + data["aliases"] = { + alias: target for alias, target in data.setdefault("aliases", {}).items() if target != media_key + } + + self.media.update(mutate) + + def prune_media_cache(self, ttl_days: int, max_items: int) -> None: + cutoff = datetime.now(UTC) - timedelta(days=ttl_days) + + def mutate(data: dict[str, Any]) -> None: + items = data.setdefault("items", {}) + for media_key, item in list(items.items()): + try: + created = datetime.fromisoformat(str(item["created_at"])) + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + if created < cutoff: + items.pop(media_key, None) + except (KeyError, TypeError, ValueError): + items.pop(media_key, None) + while len(items) > max_items: + oldest = min(items, key=lambda key: str(items[key].get("last_used_at", ""))) + items.pop(oldest, None) + data["aliases"] = { + alias: target for alias, target in data.setdefault("aliases", {}).items() if target in items + } + + self.media.update(mutate) diff --git a/app/url_security.py b/app/url_security.py new file mode 100644 index 0000000..0f2c858 --- /dev/null +++ b/app/url_security.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import ipaddress +import re +import socket +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + + +class UnsafeUrlError(ValueError): + pass + + +_TRACKING_KEYS = {"fbclid", "gclid", "igshid", "si"} + + +def _is_public_ip(value: str) -> bool: + try: + address = ipaddress.ip_address(value.split("%", 1)[0]) + except ValueError: + return False + return bool(address.is_global) + + +def validate_public_url(url: str) -> str: + if not isinstance(url, str) or len(url) > 4096: + raise UnsafeUrlError("invalid URL length") + try: + parts = urlsplit(url) + port = parts.port + except ValueError as exc: + raise UnsafeUrlError("invalid URL") from exc + if parts.scheme.lower() not in {"http", "https"}: + raise UnsafeUrlError("only HTTP(S) URLs are allowed") + if parts.username is not None or parts.password is not None: + raise UnsafeUrlError("credentials in URL are not allowed") + hostname = (parts.hostname or "").rstrip(".").lower() + if not hostname or hostname == "localhost" or hostname.endswith(".localhost"): + raise UnsafeUrlError("local host is not allowed") + if port is not None and not 1 <= port <= 65535: + raise UnsafeUrlError("invalid port") + + try: + literal = ipaddress.ip_address(hostname.split("%", 1)[0]) + except ValueError: + literal = None + if literal is not None: + if not literal.is_global: + raise UnsafeUrlError("non-public IP is not allowed") + return url + + try: + ascii_host = hostname.encode("idna").decode("ascii") + records = socket.getaddrinfo( + ascii_host, port or (443 if parts.scheme == "https" else 80), type=socket.SOCK_STREAM + ) + except (UnicodeError, socket.gaierror, OSError) as exc: + raise UnsafeUrlError("host cannot be resolved") from exc + addresses = {record[4][0] for record in records if record and record[4]} + if not addresses or not all(_is_public_ip(address) for address in addresses): + raise UnsafeUrlError("host resolves to a non-public IP") + return url + + +def normalized_url_key(url: str) -> str: + parts = urlsplit(url) + scheme = parts.scheme.lower() + host = (parts.hostname or "").rstrip(".").lower().encode("idna").decode("ascii") + port = parts.port + if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)): + host = f"{host}:{port}" + path = re.sub(r"/{2,}", "/", parts.path or "/") + query = [ + (key, value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + if key.lower() not in _TRACKING_KEYS and not key.lower().startswith("utm_") + ] + return urlunsplit((scheme, host, path, urlencode(sorted(query)), "")) + + +def safe_url_for_log(url: str) -> str: + try: + parts = urlsplit(url) + host = parts.hostname or "invalid" + return urlunsplit((parts.scheme, host, parts.path, "", ""))[:512] + except Exception: + return "" diff --git a/app/url_utils.py b/app/url_utils.py index ac9f293..e850bc4 100644 --- a/app/url_utils.py +++ b/app/url_utils.py @@ -7,7 +7,7 @@ def _host(url: str) -> str: try: host = (urlparse(url).netloc or "").lower() return host.replace("www.", "").strip() - except Exception: + except (TypeError, ValueError): return "" diff --git a/config.py b/config.py index 65dbda8..31f99be 100644 --- a/config.py +++ b/config.py @@ -1,32 +1,14 @@ -import os +"""Backward-compatible configuration facade. -# Required: Telegram bot token (from .env or environment variables) -token = (os.getenv("BOT_TOKEN") or "").strip() -if not token: - raise RuntimeError("BOT_TOKEN is not set. Put it into .env or environment variables.") +New code should import app.settings. Existing forks importing config continue to +receive the original attribute names. +""" -# Optional logs chat id (disabled by default) -logs_raw = (os.getenv("LOGS_CHAT_ID") or "").strip() -if logs_raw: - try: - logs = int(logs_raw) - except ValueError as ex: - raise RuntimeError("LOGS_CHAT_ID must be an integer.") from ex -else: - logs = None +from app.settings import load_settings -# Max file size (bytes). Default: 50 MB. -max_filesize_raw = (os.getenv("MAX_FILESIZE") or "").strip() -if max_filesize_raw: - try: - max_filesize = int(max_filesize_raw) - except ValueError as ex: - raise RuntimeError("MAX_FILESIZE must be an integer (bytes).") from ex -else: - max_filesize = 50 * 1024 * 1024 - -# Temp folder for downloads (can be overridden) -output_folder = (os.getenv("OUTPUT_FOLDER") or "/tmp/yt-dlp-telegram").strip() or "/tmp/yt-dlp-telegram" - -# Optional: cookies file path (disabled by default) -cookies_file = (os.getenv("COOKIES_FILE") or "").strip() or None +_settings = load_settings() +token = _settings.token +logs = _settings.logs_chat_id +max_filesize = _settings.max_filesize +output_folder = str(_settings.output_dir) +cookies_file = str(_settings.cookies_file) if _settings.cookies_file else None diff --git a/docker-compose.yml b/docker-compose.yml index e2a2c3c..6f06ef7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,18 @@ services: linkdownloaderbot: - build: . - container_name: linkdownloaderbot + image: linkdownloaderbotforgroups:local + build: + context: . restart: unless-stopped + init: true env_file: - .env volumes: - - ./main.py:/app/main.py:ro - - ./config.py:/app/config.py:ro - ./data:/app/data + - ./logs:/app/logs + read_only: true + tmpfs: + - /tmp:size=128m,mode=1777 + security_opt: + - no-new-privileges:true + stop_grace_period: 30s diff --git a/example.config.py b/example.config.py deleted file mode 100644 index a941fb9..0000000 --- a/example.config.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -# Example config (no secrets stored here). -# Put your real BOT_TOKEN into .env (or set it as an environment variable). - -token = (os.getenv("BOT_TOKEN") or "").strip() -if not token: - raise RuntimeError("BOT_TOKEN is not set. Put it into .env or environment variables.") - -logs_raw = (os.getenv("LOGS_CHAT_ID") or "").strip() -logs = int(logs_raw) if logs_raw else None - -max_filesize_raw = (os.getenv("MAX_FILESIZE") or "").strip() -max_filesize = int(max_filesize_raw) if max_filesize_raw else 50 * 1024 * 1024 - -output_folder = (os.getenv("OUTPUT_FOLDER") or "/tmp/yt-dlp-telegram").strip() or "/tmp/yt-dlp-telegram" - -cookies_file = (os.getenv("COOKIES_FILE") or "").strip() or None diff --git a/install.sh b/install.sh index fbf5f01..ef7505b 100644 --- a/install.sh +++ b/install.sh @@ -1,361 +1,112 @@ #!/usr/bin/env bash set -euo pipefail -REPO="https://github.com/Avazbek22/LinkDownloaderBotForGroups" -BRANCH="main" - -install_dir="${INSTALL_DIR:-$PWD/LinkDownloaderBotForGroups}" +REPOSITORY="${REPOSITORY:-https://github.com/Avazbek22/LinkDownloaderBotForGroups}" +BRANCH="${BRANCH:-main}" +INSTALL_DIR="${INSTALL_DIR:-$PWD/LinkDownloaderBotForGroups}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-linkdownloaderbotforgroups}" -SERVICE_KEY="${SERVICE_KEY:-linkdownloaderbot}" # имя сервиса в docker-compose.yml -UPDATER_SERVICE_NAME="${UPDATER_SERVICE_NAME:-linkdownloaderbotforgroups-yt-dlp-update.service}" -UPDATER_TIMER_NAME="${UPDATER_TIMER_NAME:-linkdownloaderbotforgroups-yt-dlp-update.timer}" - -PYTHON_BIN="python3" -VENV_DIR=".venv" - -# ---------- UI helpers ---------- -say() { echo -e "\n\033[1m\033[36m$*\033[0m"; } -ok() { echo -e "\033[32m✔\033[0m $*"; } -warn() { echo -e "\033[33m⚠\033[0m $*" >&2; } -die() { echo -e "\033[31m✖\033[0m $*" >&2; exit 1; } +SERVICE_KEY="linkdownloaderbot" -need_cmd() { command -v "$1" >/dev/null 2>&1; } +info() { printf '\n\033[1;36m%s\033[0m\n' "$*"; } +ok() { printf '\033[32m✓\033[0m %s\n' "$*"; } +die() { printf '\033[31m✗\033[0m %s\n' "$*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1; } as_root() { - if [[ "${EUID:-$(id -u)}" -eq 0 ]]; then - "$@" - else - need_cmd sudo || die "sudo not found. Run as root or install sudo." - sudo "$@" - fi -} - -apt_install() { - as_root apt-get update -y - as_root apt-get install -y --no-install-recommends "$@" -} - -read_yes_no() { - local prompt="$1" - local default="${2:-Y}" - local ans="" - read -r -p "$prompt " ans || true - ans="${ans:-$default}" - case "$ans" in - Y|y|yes|YES) return 0 ;; - *) return 1 ;; - esac -} - -detect_compose() { - if need_cmd docker && docker compose version >/dev/null 2>&1; then - echo "docker compose" - return 0 - fi - if need_cmd docker-compose; then - echo "docker-compose" - return 0 - fi - return 1 + if [[ "$(id -u)" == "0" ]]; then "$@"; else need sudo || die "sudo is required"; sudo "$@"; fi } -# ---------- deps ---------- -ensure_deps_basic() { - say "Installing dependencies" - apt_install git curl ca-certificates ffmpeg nodejs "$PYTHON_BIN" "$PYTHON_BIN-venv" "$PYTHON_BIN-pip" || true - ok "Dependencies are installed (or already present)." +compose() { + if docker compose version >/dev/null 2>&1; then docker compose "$@"; else docker-compose "$@"; fi } -ensure_env_key() { - local env_file="$1" - local key="$2" - local value="$3" - - if [[ ! -f "$env_file" ]]; then - return 0 +install_prerequisites() { + info "Checking prerequisites" + if ! need git || ! need docker; then + need apt-get || die "Install Git and Docker manually on this operating system" + as_root apt-get update -y + need git || as_root apt-get install -y git ca-certificates + need docker || as_root apt-get install -y docker.io fi - - if grep -q "^${key}=" "$env_file"; then - return 0 + need docker || die "Docker is unavailable" + if ! docker compose version >/dev/null 2>&1 && ! need docker-compose; then + as_root apt-get install -y docker-compose fi - - echo "${key}=${value}" >> "$env_file" + (docker compose version >/dev/null 2>&1 || need docker-compose) || die "Docker Compose is unavailable" + ok "Prerequisites are ready" } -# ---------- repo ---------- -clone_or_update_repo() { - say "Repository" - if [[ -d "$install_dir/.git" ]]; then - ok "Repo already exists: $install_dir" - git -C "$install_dir" fetch --all --prune - git -C "$install_dir" checkout "$BRANCH" - git -C "$install_dir" pull --ff-only - else - git clone --branch "$BRANCH" "$REPO" "$install_dir" - fi - ok "Repo is ready." -} - -# ---------- token / env ---------- -mask_token() { - local t="$1" - local len="${#t}" - if (( len <= 12 )); then - echo "$len chars" - return 0 - fi - echo "${t:0:5}...${t: -5} ($len chars)" -} - -read_bot_token() { - local token="${BOT_TOKEN:-}" - - if [[ -z "$token" ]]; then - # ВАЖНО: всё, что не токен — только в stderr, чтобы $(read_bot_token) не захватывал мусор - warn "" - warn "Enter your Telegram bot token." - warn "Input is hidden by default (security). Paste the token and press Enter." - warn "If you really want visible input, run: SHOW_TOKEN_INPUT=1 ./install.sh" - - if [[ "${SHOW_TOKEN_INPUT:-0}" == "1" ]]; then - >&2 printf "BOT_TOKEN: " - read -r token || true - else - >&2 printf "BOT_TOKEN: " - read -r -s token || true - >&2 echo +prepare_repository() { + info "Preparing repository" + if [[ -d "$INSTALL_DIR/.git" ]]; then + if [[ -n "$(git -C "$INSTALL_DIR" status --porcelain --untracked-files=no)" ]]; then + die "The existing repository has local changes: $INSTALL_DIR" fi - fi - - # cleanup (handles Windows CRLF and accidental spaces) - token="$(printf '%s' "$token" | tr -d '\r\n' | xargs)" - - if [[ -z "$token" ]]; then - die "Empty BOT_TOKEN." - fi - - # basic sanity check (warn only) - if [[ ! "$token" =~ ^[0-9]+:[A-Za-z0-9_-]{20,}$ ]]; then - warn "BOT_TOKEN format looks unusual. If the bot doesn't start, re-check the token in BotFather." - fi - - # stdout MUST contain only token - printf '%s' "$token" -} - -write_env_and_config() { - local env_path="$install_dir/.env" - local cfg_path="$install_dir/config.py" - - say "Config" - - local existing_token="" - if [[ -f "$env_path" ]]; then - existing_token="$(grep -m1 '^BOT_TOKEN=' "$env_path" | cut -d= -f2- | tr -d '\r\n' | xargs || true)" - fi - - local token="" - if [[ -n "${BOT_TOKEN:-}" ]]; then - token="$(printf '%s' "$BOT_TOKEN" | tr -d '\r\n' | xargs)" - ok "Using BOT_TOKEN from current environment: $(mask_token "$token")" - elif [[ -n "$existing_token" && "${FORCE_ENV:-0}" != "1" ]]; then - token="$existing_token" - ok "Using existing .env BOT_TOKEN: $(mask_token "$token")" + git -C "$INSTALL_DIR" fetch origin "$BRANCH" + git -C "$INSTALL_DIR" checkout "$BRANCH" + git -C "$INSTALL_DIR" pull --ff-only origin "$BRANCH" else - token="$(read_bot_token)" - { - echo "BOT_TOKEN=$token" - echo "# Optional:" - echo "# LOGS_CHAT_ID=123456789" - echo "# MAX_FILESIZE=52428800" - echo "# OUTPUT_FOLDER=/tmp/yt-dlp-telegram" - echo "# COOKIES_FILE=/app/cookies.txt" - echo "YTDLP_JS_RUNTIMES=node" - echo "YTDLP_REMOTE_COMPONENTS=ejs:github" - echo "YTDLP_INSTAGRAM_IMPERSONATE=chrome" - echo "YTDLP_INSTAGRAM_RETRIES=8" - echo "YTDLP_INSTAGRAM_FRAGMENT_RETRIES=8" - echo "YTDLP_INSTAGRAM_SOCKET_TIMEOUT=30" - } > "$env_path" - chmod 600 "$env_path" 2>/dev/null || true - ok ".env written: $(mask_token "$token")" - fi - - ensure_env_key "$env_path" "YTDLP_JS_RUNTIMES" "node" - ensure_env_key "$env_path" "YTDLP_REMOTE_COMPONENTS" "ejs:github" - ensure_env_key "$env_path" "YTDLP_INSTAGRAM_IMPERSONATE" "chrome" - ensure_env_key "$env_path" "YTDLP_INSTAGRAM_RETRIES" "8" - ensure_env_key "$env_path" "YTDLP_INSTAGRAM_FRAGMENT_RETRIES" "8" - ensure_env_key "$env_path" "YTDLP_INSTAGRAM_SOCKET_TIMEOUT" "30" - - # config.py без секретов (как у Вас сейчас рабочий вариант) - if [[ -f "$cfg_path" && "${FORCE_CONFIG:-0}" != "1" ]]; then - ok "config.py already exists — keeping it." - return 0 - fi - - cat > "$cfg_path" <<'PY' -import os - -# Required: Telegram bot token (from .env or environment variables) -token = (os.getenv("BOT_TOKEN") or "").strip() -if not token: - raise RuntimeError("BOT_TOKEN is not set. Put it into .env or environment variables.") - -# Optional logs chat id (disabled by default) -logs_raw = (os.getenv("LOGS_CHAT_ID") or "").strip() -if logs_raw: - try: - logs = int(logs_raw) - except ValueError as ex: - raise RuntimeError("LOGS_CHAT_ID must be an integer.") from ex -else: - logs = None - -# Max file size (bytes). Default: 50 MB. -max_filesize_raw = (os.getenv("MAX_FILESIZE") or "").strip() -if max_filesize_raw: - try: - max_filesize = int(max_filesize_raw) - except ValueError as ex: - raise RuntimeError("MAX_FILESIZE must be an integer (bytes).") from ex -else: - max_filesize = 50 * 1024 * 1024 - -# Temp folder for downloads (can be overridden) -output_folder = (os.getenv("OUTPUT_FOLDER") or "/tmp/yt-dlp-telegram").strip() or "/tmp/yt-dlp-telegram" - -# Optional: cookies file path (disabled by default) -cookies_file = (os.getenv("COOKIES_FILE") or "").strip() or None -PY - - ok "config.py written (no secrets)." -} - -# ---------- docker ---------- -ensure_docker() { - say "Docker" - if ! need_cmd docker; then - warn "Docker not found. Installing docker.io..." - apt_install docker.io - fi - - local compose_cmd="" - compose_cmd="$(detect_compose)" || true - if [[ -z "${compose_cmd:-}" ]]; then - warn "Docker Compose not found. Installing docker-compose..." - apt_install docker-compose + git clone --branch "$BRANCH" "$REPOSITORY" "$INSTALL_DIR" fi - - ok "Docker is ready." -} - -ensure_compose_override() { - # НЕ ТРОГАЕМ docker-compose.yml (чтобы git pull не конфликтовал). - # Делаем docker-compose.override.yml, который compose подхватит автоматически. - local override_path="$install_dir/docker-compose.override.yml" - - say "Docker Compose override" - cat > "$override_path" </dev/null || true - ok "Done." -} - -run_with_compose() { - say "Running with Docker Compose" - local compose_cmd - compose_cmd="$(detect_compose)" || die "docker compose / docker-compose not found" - - cd "$install_dir" - - # data folder for prefs.json etc. - mkdir -p data - - $compose_cmd -p "$COMPOSE_PROJECT" up -d --build - ok "Started. Logs:" - echo " cd \"$install_dir\"" - echo " $compose_cmd -p \"$COMPOSE_PROJECT\" logs -f --tail=200" -} - -# ---------- system mode ---------- -install_system_mode() { - say "System mode (venv)" - cd "$install_dir" - - if [[ ! -d "$VENV_DIR" ]]; then - "$PYTHON_BIN" -m venv "$VENV_DIR" - ok "venv created: $VENV_DIR" - fi - - "$VENV_DIR/bin/pip" install -r requirements.txt - ok "Dependencies installed." - - warn "Systemd generation is not included here; Docker mode is recommended on servers." - echo "Run manually:" - echo " cd \"$install_dir\"" - echo " $VENV_DIR/bin/python main.py" -} - -install_nightly_updater() { - say "Nightly yt-dlp updater" - - if ! need_cmd systemctl || [[ ! -d /run/systemd/system ]]; then - warn "systemd is unavailable. Skipping nightly updater setup (graceful fallback)." - return 0 - fi - - local service_tpl="$install_dir/scripts/systemd/linkdownloaderbotforgroups-yt-dlp-update.service" - local timer_tpl="$install_dir/scripts/systemd/linkdownloaderbotforgroups-yt-dlp-update.timer" - local service_out="/etc/systemd/system/$UPDATER_SERVICE_NAME" - local timer_out="/etc/systemd/system/$UPDATER_TIMER_NAME" - - [[ -f "$service_tpl" ]] || { warn "Missing $service_tpl"; return 0; } - [[ -f "$timer_tpl" ]] || { warn "Missing $timer_tpl"; return 0; } - - sed \ - -e "s|__INSTALL_DIR__|$install_dir|g" \ - -e "s|__COMPOSE_PROJECT__|$COMPOSE_PROJECT|g" \ - -e "s|__SERVICE_KEY__|$SERVICE_KEY|g" \ - "$service_tpl" | as_root tee "$service_out" >/dev/null - - as_root cp "$timer_tpl" "$timer_out" - +prepare_environment() { + info "Preparing configuration" + local env_file="$INSTALL_DIR/.env" + if [[ ! -f "$env_file" ]]; then + cp "$INSTALL_DIR/.env-example" "$env_file" + fi + if ! grep -Eq '^BOT_TOKEN=.+$' "$env_file"; then + local token="${BOT_TOKEN:-}" + if [[ -z "$token" ]]; then + printf 'Telegram BOT_TOKEN: ' >&2 + read -r -s token + printf '\n' >&2 + fi + [[ "$token" =~ ^[0-9]+:[A-Za-z0-9_-]{20,}$ ]] || die "BOT_TOKEN has an invalid format" + local temporary="$env_file.tmp" + awk -v token="$token" 'BEGIN{done=0} /^BOT_TOKEN=/{print "BOT_TOKEN=" token; done=1; next} {print} END{if(!done) print "BOT_TOKEN=" token}' \ + "$env_file" >"$temporary" + mv "$temporary" "$env_file" + chmod 600 "$env_file" + fi + mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/logs" + ok "Configuration is ready" +} + +start_bot() { + info "Building and starting the bot" + cd "$INSTALL_DIR" + compose -p "$COMPOSE_PROJECT" up -d --build + compose -p "$COMPOSE_PROJECT" ps + ok "Bot started" +} + +install_updater() { + [[ "${INSTALL_UPDATER:-1}" == "1" ]] || return 0 + need systemctl || return 0 + [[ -d /run/systemd/system ]] || return 0 + info "Installing nightly yt-dlp updater" + local service="/etc/systemd/system/linkdownloaderbotforgroups-yt-dlp-update.service" + local timer="/etc/systemd/system/linkdownloaderbotforgroups-yt-dlp-update.timer" + sed -e "s|__INSTALL_DIR__|$INSTALL_DIR|g" \ + -e "s|__COMPOSE_PROJECT__|$COMPOSE_PROJECT|g" \ + -e "s|__SERVICE_KEY__|$SERVICE_KEY|g" \ + "$INSTALL_DIR/scripts/systemd/linkdownloaderbotforgroups-yt-dlp-update.service" | as_root tee "$service" >/dev/null + as_root cp "$INSTALL_DIR/scripts/systemd/linkdownloaderbotforgroups-yt-dlp-update.timer" "$timer" as_root systemctl daemon-reload - as_root systemctl enable --now "$UPDATER_TIMER_NAME" - ok "Nightly updater enabled: $UPDATER_TIMER_NAME" + as_root systemctl enable --now linkdownloaderbotforgroups-yt-dlp-update.timer + ok "Nightly updater installed" } main() { - say "LinkDownloaderBotForGroups installer" - - ensure_deps_basic - clone_or_update_repo - write_env_and_config - - if read_yes_no "Install & run using Docker? [Y/n]" "Y"; then - ensure_docker - ensure_compose_override - stop_conflicting_containers - run_with_compose - else - install_system_mode - fi - - install_nightly_updater + install_prerequisites + prepare_repository + prepare_environment + start_bot + install_updater + printf '\nLogs: cd %q && docker compose -p %q logs -f --tail=200\n' "$INSTALL_DIR" "$COMPOSE_PROJECT" } main "$@" diff --git a/main.py b/main.py index e326feb..2f6920a 100644 --- a/main.py +++ b/main.py @@ -1,833 +1,621 @@ from __future__ import annotations import html -import json -import os -import re -import uuid +import logging import queue +import re +import signal import threading -from dataclasses import dataclass -from typing import Optional, Dict, Any -from urllib.parse import urlparse +import time +import uuid +from pathlib import Path +from typing import Any import telebot -import config -from app.download_backend import download_with_ytdlp - +from app.download_backend import MediaMetadata, download_metadata, extract_metadata, find_downloaded_file +from app.i18n import tr +from app.jobs import Flight, FlightCoordinator, Job +from app.logging_setup import configure_logging +from app.media_cache import DiskMediaCache +from app.settings import Settings, load_settings +from app.storage import Storage +from app.url_security import UnsafeUrlError, normalized_url_key, safe_url_for_log, validate_public_url -# ========================= -# Settings (simple and stable) -# ========================= - -WORKERS = 2 # how many downloads in parallel -MAX_QUEUE = 200 # queue limit to avoid RAM issues on busy groups -MAX_SEND_BYTES = int(getattr(config, "max_filesize", 50_000_000)) -YTDLP_CONCURRENT_FRAGMENTS = 4 - -# Persistent JSON "DB" (mounted via docker-compose to survive restarts) -DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -PREFS_PATH = os.path.join(DATA_DIR, "prefs.json") - -# About (for help messages) -AUTHOR_NAME = "Avazbek Olimov" REPO_URL = "https://github.com/Avazbek22/LinkDownloaderBotForGroups" +URL_RE = re.compile(r"https?://[^\s<>]+", re.IGNORECASE) -# ========================= -# Telegram init -# ========================= - -bot = telebot.TeleBot(config.token, threaded=True) -bot_lock = threading.RLock() - -jobs_q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=MAX_QUEUE) - - -def _bot_call(fn, *args, **kwargs): - with bot_lock: - return fn(*args, **kwargs) - - -# ========================= -# Helpers -# ========================= - -def _extract_first_url(text: str) -> Optional[str]: - if not text: +def extract_first_url(text: str) -> str | None: + match = URL_RE.search(text or "") + if not match: return None - m = re.search(r"(https?://\S+)", text.strip()) - if not m: - return None - url = m.group(1).strip() - url = url.rstrip(").,]}>\"'") - return url - - -def _try_send_message(chat_id: int, text: str, message_thread_id: Optional[int] = None) -> bool: - # Requirement: no notifications, no preview - try: - kwargs: Dict[str, Any] = { - "disable_web_page_preview": True, - "disable_notification": True, - } - if isinstance(message_thread_id, int): - kwargs["message_thread_id"] = message_thread_id - - _bot_call(bot.send_message, chat_id, text, **kwargs) - return True - except Exception: - return False - - -def _try_send_message_html(chat_id: int, html_text: str, message_thread_id: Optional[int] = None) -> bool: - # Requirement: no notifications, no preview + HTML allowed - try: - kwargs: Dict[str, Any] = { - "disable_web_page_preview": True, - "disable_notification": True, - "parse_mode": "HTML", - } - if isinstance(message_thread_id, int): - kwargs["message_thread_id"] = message_thread_id - - _bot_call(bot.send_message, chat_id, html_text, **kwargs) - return True - except Exception: - return False - - -def _safe_send_message(chat_id: int, text: str, message_thread_id: Optional[int] = None) -> None: - _try_send_message(chat_id, text, message_thread_id=message_thread_id) - - -def _safe_send_message_html(chat_id: int, html_text: str, message_thread_id: Optional[int] = None) -> None: - _try_send_message_html(chat_id, html_text, message_thread_id=message_thread_id) - - -def _safe_delete_message(chat_id: int, message_id: int) -> bool: - try: - _bot_call(bot.delete_message, chat_id, message_id) - return True - except Exception: - return False - - -def _log_request(message, url: str) -> None: - logs_chat = getattr(config, "logs", None) - if not logs_chat: - return - - try: - user = message.from_user - username = f"@{user.username}" if getattr(user, "username", None) else "(no username)" - chat_title = getattr(message.chat, "title", "") or "Group" - text = ( - f"Download request from {username} ({user.id})\n" - f"Chat: {chat_title} ({message.chat.id})\n" - f"URL: {url}" + return match.group(0).rstrip(").,;:!?]}>\"'") + + +class BotApplication: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.log = logging.getLogger("link_downloader_bot.app") + self.storage = Storage(settings.data_dir, settings.default_language, settings.delete_original) + self.storage.prune_media_cache(settings.file_id_cache_ttl_days, settings.file_id_cache_max_items) + self.disk_cache = DiskMediaCache( + settings.output_dir, + max_files=settings.disk_cache_max_files, + ttl_seconds=settings.disk_cache_ttl, ) - _bot_call(bot.send_message, logs_chat, text, disable_web_page_preview=True) - except Exception: - pass - - -def _is_intermediate_ytdlp_file(prefix: str, filename: str) -> bool: - """ - yt-dlp промежуточные файлы при separate выглядят как: - PREFIX.f137.mp4, PREFIX.f140.m4a, PREFIX.f248.webm и т.п. - """ - if not filename.startswith(prefix + "."): - return False - # .part / .ytdl / temp сразу считаем мусором/промежуточным - low = filename.lower() - if low.endswith(".part") or low.endswith(".ytdl") or low.endswith(".tmp") or low.endswith(".temp"): - return True - # PREFIX.f.ext - return re.match(rf"^{re.escape(prefix)}\.f\d+\.", filename) is not None - - -def _find_downloaded_file(info: Dict[str, Any], prefix: str, output_folder: str) -> Optional[str]: - """ - КРИТИЧНО: сначала ищем финальный файл PREFIX.mp4. - Именно он является результатом merge и содержит аудио. - """ - # 0) Best: final merged/progressive file by outtmpl base - final_mp4 = os.path.join(output_folder, f"{prefix}.mp4") - if os.path.exists(final_mp4): - return final_mp4 - - # 1) Sometimes yt-dlp gives final path in these keys - try: - for key in ("filepath", "_filename"): - fp = info.get(key) - if fp and os.path.exists(fp): - return fp - except Exception: - pass - - # 2) If info has requested_downloads, НЕ берём первый попавшийся mp4 (это часто video-only). - # Берём лучше "не промежуточный" и предпочитаем тот, что без ".f123." - try: - reqs = info.get("requested_downloads") or [] - candidates = [] - for r in reqs: - fp = r.get("filepath") - if not fp or not os.path.exists(fp): - continue - fn = os.path.basename(fp) - candidates.append(fp) - - # Prefer non-intermediate - non_intermediate = [fp for fp in candidates if not _is_intermediate_ytdlp_file(prefix, os.path.basename(fp))] - if non_intermediate: - # Prefer mp4 among them - mp4 = [fp for fp in non_intermediate if fp.lower().endswith(".mp4")] - return mp4[0] if mp4 else non_intermediate[0] - except Exception: - pass - - # 3) Fallback: scan folder, prefer exact PREFIX. (without .f123), then mp4, newest - try: - files = [] - for fn in os.listdir(output_folder): - if not fn.startswith(prefix): - continue - low = fn.lower() - if low.endswith(".part") or low.endswith(".ytdl") or low.endswith(".tmp") or low.endswith(".temp"): - continue - fp = os.path.join(output_folder, fn) - if not os.path.exists(fp): - continue - files.append(fp) - - if not files: - return None - - # Prefer "base" file: PREFIX. (no extra dots except ext) - base_like = [] - for fp in files: - fn = os.path.basename(fp) - if fn.startswith(prefix + ".") and fn.count(".") == 1: - base_like.append(fp) - - if base_like: - mp4 = [fp for fp in base_like if fp.lower().endswith(".mp4")] - if mp4: - return mp4[0] - return base_like[0] - - # Else prefer mp4 that is not intermediate - non_intermediate_mp4 = [ - fp for fp in files - if fp.lower().endswith(".mp4") and not _is_intermediate_ytdlp_file(prefix, os.path.basename(fp)) - ] - if non_intermediate_mp4: - # newest - non_intermediate_mp4.sort(key=lambda p: os.path.getmtime(p), reverse=True) - return non_intermediate_mp4[0] - - # last resort: newest file - files.sort(key=lambda p: os.path.getmtime(p), reverse=True) - return files[0] - except Exception: - return None - - -def _cleanup_files(prefix: str, output_folder: str) -> None: - try: - for fn in os.listdir(output_folder): - if fn.startswith(prefix): - fp = os.path.join(output_folder, fn) - try: - if os.path.exists(fp): - os.remove(fp) - except Exception: - pass - except Exception: - pass - - -def _download_with_ytdlp(url: str, out_prefix: str, output_folder: str) -> Dict[str, Any]: - return download_with_ytdlp( - url=url, - out_prefix=out_prefix, - output_folder=output_folder, - max_send_bytes=MAX_SEND_BYTES, - concurrent_fragments=YTDLP_CONCURRENT_FRAGMENTS, - ) - - -def _send_video_no_reply( - chat_id: int, - message_thread_id: Optional[int], - file_path: str, - caption_html: str, -) -> None: - size = os.path.getsize(file_path) - if size > MAX_SEND_BYTES: - raise RuntimeError("File too large for Telegram bot upload limit") - - kwargs: Dict[str, Any] = { - "supports_streaming": True, - "disable_notification": True, - "caption": caption_html, - "parse_mode": "HTML", - } - if isinstance(message_thread_id, int): - kwargs["message_thread_id"] = message_thread_id - - with open(file_path, "rb") as f: - _bot_call(bot.send_video, chat_id, f, **kwargs) - - -# ========================= -# Source detection -# ========================= - -def _detect_source(url: str) -> str: - try: - host = (urlparse(url).netloc or "").lower() - host = host.replace("www.", "").strip() - - mapping = [ - (("youtube.com", "youtu.be", "m.youtube.com"), "YouTube"), - (("instagram.com", "instagr.am"), "Instagram"), - (("tiktok.com",), "TikTok"), - (("vk.com", "vkvideo.ru"), "VK"), - (("twitter.com", "x.com"), "X"), - (("facebook.com", "fb.watch"), "Facebook"), - (("t.me",), "Telegram"), - ] - - for domains, name in mapping: - if any(host == d or host.endswith("." + d) for d in domains): - return name - - if host: - return host - return "Unknown" - except Exception: - return "Unknown" - - -def _format_sender_name(message) -> str: - user = message.from_user - first = (getattr(user, "first_name", "") or "").strip() - last = (getattr(user, "last_name", "") or "").strip() - - full = f"{first} {last}".strip() - if full: - return full - - username = getattr(user, "username", None) - if username: - return f"@{username}" - - return str(getattr(user, "id", "")) - - -def _html_escape_text(s: str) -> str: - return html.escape(s or "", quote=False) - - -def _html_escape_attr(s: str) -> str: - return html.escape(s or "", quote=True) - - -# ========================= -# Persistent prefs (JSON) -# ========================= - -_prefs_lock = threading.RLock() -_prefs_cache: Dict[str, Any] = {} - - -def _ensure_prefs_loaded() -> None: - global _prefs_cache - - os.makedirs(DATA_DIR, exist_ok=True) - - with _prefs_lock: - if _prefs_cache: - return + self.coordinator = FlightCoordinator() + self.queue: queue.Queue[Flight | None] = queue.Queue(maxsize=settings.max_queue) + self.stop_event = threading.Event() + self.upload_slots = threading.BoundedSemaphore(settings.upload_workers) + self.bot = telebot.TeleBot(settings.token, threaded=True) + self.bot_id = 0 + self.bot_username = "" + self.workers: list[threading.Thread] = [] + self.maintenance_thread: threading.Thread | None = None + self._register_handlers() + + @property + def mention(self) -> str: + return f"@{self.bot_username}" if self.bot_username else "@bot" + + def initialize_identity(self) -> None: + last_error: Exception | None = None + for attempt in range(1, 6): + try: + me = self.bot.get_me() + self.bot_id = int(me.id) + self.bot_username = str(me.username or "").lower() + return + except Exception as exc: # Telegram errors vary by transport/version + last_error = exc + self.log.warning("get_me failed attempt=%s", attempt, exc_info=True) + if attempt < 5: + time.sleep(min(attempt * 2, 8)) + raise RuntimeError("cannot initialize Telegram bot identity") from last_error + + def start(self) -> None: + self.disk_cache.maintain() + if self.settings.cookies_file and not self.settings.cookies_file.is_file(): + self.log.warning("cookies file does not exist path=%s", self.settings.cookies_file) + self.initialize_identity() + self._set_commands() + for index in range(self.settings.workers): + thread = threading.Thread(target=self._worker, name=f"download-worker-{index + 1}", daemon=True) + thread.start() + self.workers.append(thread) + self.maintenance_thread = threading.Thread( + target=self._maintenance_loop, + name="cache-maintenance", + daemon=True, + ) + self.maintenance_thread.start() + self.log.info( + "bot started username=%s workers=%s uploads=%s", + self.bot_username, + self.settings.workers, + self.settings.upload_workers, + ) + self.bot.infinity_polling(timeout=30, long_polling_timeout=30, allowed_updates=None) - if not os.path.exists(PREFS_PATH): - _prefs_cache = { - "version": 2, - "opt_out": {}, - "welcomed_groups": {}, - "welcomed_private": {}, - } - _save_prefs_locked() + def stop(self) -> None: + if self.stop_event.is_set(): return + self.stop_event.set() + self.bot.stop_polling() + for _ in self.workers: + try: + self.queue.put_nowait(None) + except queue.Full: + break + for thread in self.workers: + thread.join(timeout=10) + if self.maintenance_thread is not None: + self.maintenance_thread.join(timeout=2) + self.disk_cache.maintain() + self.log.info("bot stopped") + + def _worker(self) -> None: + while not self.stop_event.is_set(): + try: + flight = self.queue.get(timeout=1) + except queue.Empty: + continue + try: + if flight is None: + return + self._process_flight(flight) + except Exception: + self.log.exception("uncaught worker error") + self._operator_alert("A download worker failed unexpectedly. Check bot.log.") + if flight is not None: + self.coordinator.abort(flight) + finally: + self.queue.task_done() + + def _maintenance_loop(self) -> None: + interval = min(60, self.settings.disk_cache_ttl) + while not self.stop_event.wait(interval): + try: + self.disk_cache.maintain() + self.storage.prune_media_cache( + self.settings.file_id_cache_ttl_days, + self.settings.file_id_cache_max_items, + ) + except Exception: + self.log.exception("cache maintenance failed") + + def _process_flight(self, flight: Flight) -> None: + first = flight.jobs[0] + started = time.monotonic() + self.log.info("job metadata job_id=%s url=%s", first.job_id, safe_url_for_log(first.url)) + cache_profile = f"mp4:{self.settings.max_filesize}" + cached = ( + self.storage.get_cached_by_url(f"{first.url_key}|{cache_profile}", self.settings.file_id_cache_ttl_days) + if self.settings.media_cache_enabled + else None + ) + retry_jobs: list[Job] = [] + if cached is not None: + media_key, file_id, source_name = cached + if not self.coordinator.promote(flight, media_key): + return + metadata = MediaMetadata(first.url, {"id": media_key}, media_key, source_name) + while True: + batch = self.coordinator.pending(flight) + if batch: + retry_jobs = self._send_by_file_id(batch, file_id, metadata, media_key) + if retry_jobs: + break + if self.coordinator.finish_if_idle(flight): + self.log.info("Telegram cache hit job_id=%s media_key=%s", first.job_id, media_key) + return try: - with open(PREFS_PATH, "r", encoding="utf-8") as f: - data = json.load(f) - if not isinstance(data, dict): - raise ValueError("prefs not dict") - - if "opt_out" not in data or not isinstance(data.get("opt_out"), dict): - data["opt_out"] = {} - - if "welcomed_groups" not in data or not isinstance(data.get("welcomed_groups"), dict): - data["welcomed_groups"] = {} - - if "welcomed_private" not in data or not isinstance(data.get("welcomed_private"), dict): - data["welcomed_private"] = {} - - if "version" not in data: - data["version"] = 2 - - _prefs_cache = data + validate_public_url(first.url) + metadata = extract_metadata(first.url, self.settings.cookies_file) + final_url = metadata.info.get("webpage_url") + if isinstance(final_url, str): + validate_public_url(final_url) except Exception: - _prefs_cache = { - "version": 2, - "opt_out": {}, - "welcomed_groups": {}, - "welcomed_private": {}, - } - _save_prefs_locked() - - -def _save_prefs_locked() -> None: - tmp = PREFS_PATH + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - json.dump(_prefs_cache, f, ensure_ascii=False, indent=2) - os.replace(tmp, PREFS_PATH) - - -def _is_opted_out(chat_id: int, user_id: int) -> bool: - _ensure_prefs_loaded() - with _prefs_lock: - opt_out = _prefs_cache.get("opt_out", {}) - chat_key = str(chat_id) - users = opt_out.get(chat_key, {}) - return bool(users.get(str(user_id), False)) - - -def _toggle_opt_out(chat_id: int, user_id: int) -> bool: - _ensure_prefs_loaded() - with _prefs_lock: - opt_out = _prefs_cache.setdefault("opt_out", {}) - chat_key = str(chat_id) - users = opt_out.setdefault(chat_key, {}) - - user_key = str(user_id) - new_value = not bool(users.get(user_key, False)) - users[user_key] = new_value - - if not new_value: - users.pop(user_key, None) - if isinstance(users, dict) and not users: - opt_out.pop(chat_key, None) - - _save_prefs_locked() - return new_value - - -def _was_group_welcomed(chat_id: int) -> bool: - _ensure_prefs_loaded() - with _prefs_lock: - return bool(_prefs_cache.get("welcomed_groups", {}).get(str(chat_id), False)) - - -def _mark_group_welcomed(chat_id: int) -> None: - _ensure_prefs_loaded() - with _prefs_lock: - _prefs_cache.setdefault("welcomed_groups", {})[str(chat_id)] = True - _save_prefs_locked() - - -def _was_private_welcomed(user_id: int) -> bool: - _ensure_prefs_loaded() - with _prefs_lock: - return bool(_prefs_cache.get("welcomed_private", {}).get(str(user_id), False)) - - -def _mark_private_welcomed(user_id: int) -> None: - _ensure_prefs_loaded() - with _prefs_lock: - _prefs_cache.setdefault("welcomed_private", {})[str(user_id)] = True - _save_prefs_locked() - - -# ========================= -# Mention detection -# ========================= - -def _get_bot_username_lower() -> str: - try: - me = _bot_call(bot.get_me) - username = (getattr(me, "username", "") or "").strip() - return username.lower() - except Exception: - return "" - - -def _get_bot_id() -> int: - try: - me = _bot_call(bot.get_me) - return int(getattr(me, "id", 0) or 0) - except Exception: - return 0 - - -BOT_USERNAME_LOWER = _get_bot_username_lower() -BOT_ID = _get_bot_id() - - -def _contains_bot_mention(text: str) -> bool: - if not isinstance(text, str) or not text.strip(): - return False - if not BOT_USERNAME_LOWER: - return False - return f"@{BOT_USERNAME_LOWER}" in text.lower() - - -def _contains_sender_self_mention_or_me(text: str, sender_username: Optional[str]) -> bool: - if not isinstance(text, str) or not text.strip(): - return False - - t = text.lower() - - if re.search(r"(^|\s)me(\s|$)", t): - return True - - if re.search(r"(^|\s)я(\s|$)", t): - return True - - if sender_username: - return re.search(rf"@{re.escape(sender_username.lower())}\b", t) is not None - - return False - + self.log.exception("metadata failed job_id=%s url=%s", first.job_id, safe_url_for_log(first.url)) + self.coordinator.abort(flight) + return -# ========================= -# Help / About text -# ========================= + media_key = f"{metadata.media_key}:mp4:{self.settings.max_filesize}" + if cached is None and not self.coordinator.promote(flight, media_key): + self.log.info("job joined media flight job_id=%s media_key=%s", first.job_id, media_key) + return -def _help_text_html(is_group: bool) -> str: - bot_mention = f"@{BOT_USERNAME_LOWER}" if BOT_USERNAME_LOWER else "@" + file_id = None + if self.settings.media_cache_enabled: + file_id = self.storage.get_file_id(media_key, self.settings.file_id_cache_ttl_days) + file_path = self.disk_cache.get(media_key) if self.settings.media_cache_enabled else None + + while True: + batch = retry_jobs or self.coordinator.pending(flight) + retry_jobs = [] + if batch: + if file_id: + retry = self._send_by_file_id(batch, file_id, metadata, media_key) + if retry: + file_id = None + if file_path is None: + file_path = self._obtain_file(metadata, media_key) + file_id = self._send_from_file( + retry, + file_path, + metadata, + media_key, + {f"{key}|{cache_profile}" for key in flight.url_keys}, + ) + else: + if file_path is None: + file_path = self._obtain_file(metadata, media_key) + file_id = self._send_from_file( + batch, + file_path, + metadata, + media_key, + {f"{key}|{cache_profile}" for key in flight.url_keys}, + ) + if self.coordinator.finish_if_idle(flight): + break - if is_group: - usage = ( - "Как пользоваться\n" - "• Просто отправляйте ссылку на видео в группу — бот скачает видео, отправит его и затем удалит исходную ссылку.\n" - "• Подпись под видео: кликабельная «Ссылка на видео …» + «От Имя Фамилия».\n\n" - "Как отключить авто-скачивание для себя\n" - f"• Напишите в группе: {bot_mention} @ВашНик\n" - " (можно также написать «me» или «я» вместо ника)\n" - "• Повторите команду — включится обратно.\n\n" - "Режим вручную (когда Вы отключились)\n" - f"• Чтобы скачать: {bot_mention} <ссылка>\n" + self.disk_cache.maintain() + self.log.info( + "job complete job_id=%s media_key=%s elapsed=%.2f", + first.job_id, + media_key, + time.monotonic() - started, ) - else: - usage = ( - "Я бот для групп\n" - "Я скачиваю видео по ссылкам (YouTube/Instagram/TikTok/VK/X/Facebook/Telegram и др.) и публикую видео в группе.\n\n" - "Что нужно сделать\n" - "1) Добавьте меня в группу.\n" - "2) Дайте права администратора и разрешение удалять сообщения.\n\n" - "Как работает по умолчанию\n" - "• Любая ссылка на видео в группе → скачивание → отправка видео → удаление исходной ссылки.\n\n" - "Как отключить авто-скачивание для себя\n" - f"• В группе напишите: {bot_mention} @ВашНик\n" - " (или «me» / «я»)\n" - "• Повторите — включится обратно.\n\n" - "Когда авто отключено\n" - f"• Скачивание только так: {bot_mention} <ссылка>\n" - ) - - author = _html_escape_text(AUTHOR_NAME) - repo_attr = _html_escape_attr(REPO_URL) - - return ( - f"{usage}\n" - f"Автор: {author}\n" - f'Ссылка на репозиторий' - ) - -def _bot_admin_hint_html(chat_id: int) -> str: - try: - if not BOT_ID: - return "" - cm = _bot_call(bot.get_chat_member, chat_id, BOT_ID) - status = (getattr(cm, "status", "") or "").lower() - is_admin = status in ("administrator", "creator") - if is_admin: - return "" - return ( - "⚠️ Важно\n" - "Чтобы бот мог удалять исходные ссылки, назначьте его администратором и включите право «Удалять сообщения».\n\n" + def _obtain_file(self, metadata: MediaMetadata, media_key: str) -> Path: + cached = self.disk_cache.get(media_key) + if cached is not None: + self.log.info("disk cache hit media_key=%s", media_key) + return cached + prefix = self.disk_cache.prefix(media_key) + info = download_metadata( + metadata, + prefix, + self.settings.output_dir, + max_send_bytes=self.settings.max_filesize, + concurrent_fragments=self.settings.concurrent_fragments, + cookie_file=self.settings.cookies_file, + deadline=time.monotonic() + self.settings.job_timeout, ) - except Exception: - return "" - - -def _try_set_commands() -> None: - try: - commands = [ - telebot.types.BotCommand("start", "Инструкция"), - telebot.types.BotCommand("help", "Инструкция"), - ] - _bot_call(bot.set_my_commands, commands) - except Exception: - pass - - -_try_set_commands() - - -# ========================= -# Jobs -# ========================= - -@dataclass(frozen=True) -class Job: - chat_id: int - message_thread_id: Optional[int] - original_message_id: int - url: str - prefix: str - source_name: str - sender_full_name: str - notify_on_fail: bool - delete_original_on_success: bool - - -def _process_job(job_dict: Dict[str, Any]) -> None: - job = Job(**job_dict) - - output_folder = getattr(config, "output_folder", "/tmp/yt-dlp-telegram") or "/tmp/yt-dlp-telegram" - - try: - info = _download_with_ytdlp(job.url, job.prefix, output_folder) - file_path = _find_downloaded_file(info, job.prefix, output_folder) - - if not file_path or not os.path.exists(file_path): - raise RuntimeError("Downloaded file not found") - - url_attr = _html_escape_attr(job.url) - source_text = _html_escape_text(job.source_name) - sender_text = _html_escape_text(job.sender_full_name) - - caption_html = f'Ссылка на видео {source_text}\nОт {sender_text}' - - _send_video_no_reply( - chat_id=job.chat_id, - message_thread_id=job.message_thread_id, - file_path=file_path, - caption_html=caption_html, + path = find_downloaded_file(info, prefix, self.settings.output_dir) + if path is None or not path.is_file(): + self.disk_cache.remove_prefix_except(prefix) + raise RuntimeError("downloaded file not found") + if path.stat().st_size > self.settings.max_filesize: + self.disk_cache.remove_prefix_except(prefix) + raise RuntimeError("downloaded file exceeds MAX_FILESIZE") + self.disk_cache.remove_prefix_except(prefix, keep=path) + self.disk_cache.maintain() + self.log.info("download complete media_key=%s bytes=%s", media_key, path.stat().st_size) + return path + + def _send_by_file_id( + self, + jobs: list[Job], + file_id: str, + metadata: MediaMetadata, + media_key: str, + ) -> list[Job]: + for index, job in enumerate(jobs): + try: + self._send_video(job, file_id, metadata, upload=False) + self._after_success(job) + except Exception as exc: + if self._is_invalid_file_id(exc): + self.storage.remove_file_id(media_key) + self.log.warning("invalid Telegram file_id media_key=%s", media_key) + return jobs[index:] + self.log.exception("cached send failed job_id=%s chat_id=%s", job.job_id, job.chat_id) + return [] + + def _send_from_file( + self, + jobs: list[Job], + path: Path, + metadata: MediaMetadata, + media_key: str, + url_keys: set[str], + ) -> str | None: + file_id: str | None = None + for job in jobs: + try: + if file_id: + self._send_video(job, file_id, metadata, upload=False) + else: + response = self._send_video(job, path, metadata, upload=True) + video = getattr(response, "video", None) + candidate = getattr(video, "file_id", None) + if isinstance(candidate, str) and candidate: + file_id = candidate + if self.settings.media_cache_enabled: + self.storage.put_file_id( + media_key, + file_id, + self.settings.file_id_cache_max_items, + source_name=metadata.source_name, + url_keys=url_keys, + ) + self._after_success(job) + except Exception: + self.log.exception("video send failed job_id=%s chat_id=%s", job.job_id, job.chat_id) + return file_id + + def _send_video(self, job: Job, video: Path | str, metadata: MediaMetadata, *, upload: bool) -> Any: + language = self.storage.chat_language(job.chat_id) + caption = tr( + language, + "caption", + url=html.escape(job.url, quote=True), + source=html.escape(metadata.source_name, quote=False), + sender=html.escape(job.sender_name, quote=False), ) - - if job.delete_original_on_success: - _safe_delete_message(job.chat_id, job.original_message_id) - - except Exception: - if job.notify_on_fail: - _safe_send_message(job.chat_id, "Не удалось скачать", message_thread_id=job.message_thread_id) - finally: - _cleanup_files(job.prefix, output_folder) - - -def _worker_loop() -> None: - while True: - job = jobs_q.get() - try: - _process_job(job) - finally: - jobs_q.task_done() - - -for _ in range(WORKERS): - t = threading.Thread(target=_worker_loop, daemon=True) - t.start() - - -# ========================= -# Service: bot added to group (one-time per group) -# ========================= - -@bot.message_handler(content_types=["new_chat_members"]) -def handle_new_chat_members(message): - try: - if message.chat.type not in ("group", "supergroup"): - return - - new_members = getattr(message, "new_chat_members", None) or [] - if not new_members: - return - - is_me = False - for u in new_members: + kwargs: dict[str, Any] = { + "chat_id": job.chat_id, + "caption": caption, + "parse_mode": "HTML", + "supports_streaming": True, + "disable_notification": True, + } + if job.message_thread_id is not None: + kwargs["message_thread_id"] = job.message_thread_id + if upload: + with self.upload_slots, Path(video).open("rb") as handle: + return self.bot.send_video(video=handle, **kwargs) + return self.bot.send_video(video=video, **kwargs) + + def _after_success(self, job: Job) -> None: + if job.delete_original: try: - uid = int(getattr(u, "id", 0) or 0) + self.bot.delete_message(job.chat_id, job.original_message_id) except Exception: - uid = 0 - uname = (getattr(u, "username", "") or "").lower() - if (BOT_ID and uid == BOT_ID) or (BOT_USERNAME_LOWER and uname == BOT_USERNAME_LOWER): - is_me = True - break + self.log.exception("original delete failed job_id=%s chat_id=%s", job.job_id, job.chat_id) - if not is_me: - return + @staticmethod + def _is_invalid_file_id(exc: Exception) -> bool: + text = str(exc).lower() + return "file_id" in text or "file identifier" in text or "file reference" in text - chat_id = int(message.chat.id) - message_thread_id = getattr(message, "message_thread_id", None) + def _safe_message(self, chat_id: int, text: str, thread_id: int | None = None, *, html_mode: bool = False) -> bool: + try: + kwargs: dict[str, Any] = { + "disable_notification": True, + "disable_web_page_preview": True, + } + if html_mode: + kwargs["parse_mode"] = "HTML" + if thread_id is not None: + kwargs["message_thread_id"] = thread_id + self.bot.send_message(chat_id, text, **kwargs) + return True + except Exception: + self.log.exception("message send failed chat_id=%s", chat_id) + return False - if _was_group_welcomed(chat_id): + def _operator_alert(self, text: str) -> None: + if self.settings.logs_chat_id is None: return + try: + self.bot.send_message( + self.settings.logs_chat_id, + text, + disable_notification=True, + disable_web_page_preview=True, + ) + except Exception: + self.log.exception("operator alert failed") - html_text = _bot_admin_hint_html(chat_id) + _help_text_html(is_group=True) - - if _try_send_message_html(chat_id, html_text, message_thread_id=message_thread_id): - _mark_group_welcomed(chat_id) - - except Exception: - pass + def _is_admin(self, chat_id: int, user_id: int) -> bool: + try: + member = self.bot.get_chat_member(chat_id, user_id) + return str(getattr(member, "status", "")).lower() in {"administrator", "creator"} + except Exception: + self.log.exception("admin check failed chat_id=%s user_id=%s", chat_id, user_id) + return False + def _admin_hint(self, chat_id: int, language: str) -> str: + try: + member = self.bot.get_chat_member(chat_id, self.bot_id) + if str(getattr(member, "status", "")).lower() in {"administrator", "creator"} and bool( + getattr(member, "can_delete_messages", True) + ): + return "" + except Exception: + self.log.exception("bot permission check failed chat_id=%s", chat_id) + return "" + return tr(language, "admin_hint") -# ========================= -# Private: /start, /help, any text (one-time welcome per user) -# ========================= + def _help(self, chat_id: int, private: bool) -> str: + language = self.storage.chat_language(chat_id) + if private: + return tr(language, "private_help", repo_url=html.escape(REPO_URL, quote=True)) + return tr(language, "group_help", bot_mention=html.escape(self.mention, quote=False)) -@bot.message_handler(commands=["start", "help"]) -def handle_start_help(message): - try: - chat_type = getattr(message.chat, "type", "") - chat_id = int(getattr(message.chat, "id", 0) or 0) - message_thread_id = getattr(message, "message_thread_id", None) + def _set_commands(self) -> None: + try: + self.bot.set_my_commands( + [ + telebot.types.BotCommand("start", "Show instructions"), + telebot.types.BotCommand("help", "Show instructions"), + telebot.types.BotCommand("language", "Change language (admins)"), + telebot.types.BotCommand("settings", "Show group settings"), + telebot.types.BotCommand("delete_original", "Configure link deletion (admins)"), + ] + ) + except Exception: + self.log.exception("cannot set Telegram commands") - if chat_type == "private": - user_id = int(getattr(message.from_user, "id", 0) or 0) + def _register_handlers(self) -> None: + bot = self.bot - if (message.text or "").strip().lower().startswith("/help"): - _safe_send_message_html(chat_id, _help_text_html(is_group=False), message_thread_id=message_thread_id) - _mark_private_welcomed(user_id) + @bot.message_handler(content_types=["new_chat_members"]) + def new_members(message: Any) -> None: + if getattr(message.chat, "type", "") not in {"group", "supergroup"}: return + members = getattr(message, "new_chat_members", None) or [] + if not any(int(getattr(member, "id", 0) or 0) == self.bot_id for member in members): + return + chat_id = int(message.chat.id) + if self.storage.was_welcomed("group", chat_id): + return + language = self.storage.chat_language(chat_id) + text = self._admin_hint(chat_id, language) + self._help(chat_id, private=False) + if self._safe_message(chat_id, text, getattr(message, "message_thread_id", None), html_mode=True): + self.storage.mark_welcomed("group", chat_id) + + @bot.message_handler(commands=["start", "help"]) + def start_help(message: Any) -> None: + chat_type = getattr(message.chat, "type", "") + chat_id = int(message.chat.id) + private = chat_type == "private" + if not private and chat_type not in {"group", "supergroup"}: + return + self._safe_message( + chat_id, + self._help(chat_id, private=private), + getattr(message, "message_thread_id", None), + html_mode=True, + ) + if private: + self.storage.mark_welcomed("private", int(message.from_user.id)) + + @bot.message_handler(commands=["language"]) + def language(message: Any) -> None: + chat_id = int(message.chat.id) + current = self.storage.chat_language(chat_id) + parts = (message.text or "").split() + if len(parts) == 1: + self._safe_message(chat_id, tr(current, "language_current", language=current)) + return + requested = parts[1].lower() + if requested not in {"en", "ru"}: + self._safe_message(chat_id, tr(current, "language_invalid")) + return + if getattr(message.chat, "type", "") != "private" and not self._is_admin( + chat_id, int(message.from_user.id) + ): + self._safe_message(chat_id, tr(current, "language_admin_only")) + return + self.storage.set_chat_language(chat_id, requested) + self._safe_message(chat_id, tr(requested, "language_changed")) + + @bot.message_handler(commands=["settings"]) + def group_settings(message: Any) -> None: + chat_id = int(message.chat.id) + language_code = self.storage.chat_language(chat_id) + if getattr(message.chat, "type", "") not in {"group", "supergroup"}: + return + if not self._is_admin(chat_id, int(message.from_user.id)): + self._safe_message(chat_id, tr(language_code, "admin_only")) + return + enabled = self.storage.delete_original(chat_id) + state = tr(language_code, "state_on" if enabled else "state_off") + self._safe_message( + chat_id, + tr( + language_code, + "settings_summary", + language=language_code, + delete_original=state, + ), + getattr(message, "message_thread_id", None), + html_mode=True, + ) + + @bot.message_handler(commands=["delete_original"]) + def delete_original(message: Any) -> None: + chat_id = int(message.chat.id) + language_code = self.storage.chat_language(chat_id) + if getattr(message.chat, "type", "") not in {"group", "supergroup"}: + return + if not self._is_admin(chat_id, int(message.from_user.id)): + self._safe_message(chat_id, tr(language_code, "admin_only")) + return + parts = (message.text or "").split() + if len(parts) != 2 or parts[1].lower() not in {"on", "off"}: + self._safe_message(chat_id, tr(language_code, "delete_usage")) + return + enabled = parts[1].lower() == "on" + self.storage.set_delete_original(chat_id, enabled) + state = tr(language_code, "state_on" if enabled else "state_off") + self._safe_message(chat_id, tr(language_code, "delete_changed", state=state)) + + @bot.message_handler(func=lambda item: getattr(item.chat, "type", "") == "private", content_types=["text"]) + def private_text(message: Any) -> None: + chat_id = int(message.chat.id) + user_id = int(message.from_user.id) + if not self.storage.was_welcomed("private", user_id): + if self._safe_message(chat_id, self._help(chat_id, private=True), html_mode=True): + self.storage.mark_welcomed("private", user_id) + return + self._safe_message(chat_id, tr(self.storage.chat_language(chat_id), "private_hint")) - if not _was_private_welcomed(user_id): - _safe_send_message_html(chat_id, _help_text_html(is_group=False), message_thread_id=message_thread_id) - _mark_private_welcomed(user_id) - else: - _safe_send_message(chat_id, "Инструкция уже отправлялась. Нажмите /help чтобы показать её снова.", - message_thread_id=message_thread_id) - return - - if chat_type in ("group", "supergroup"): - _safe_send_message_html(chat_id, _help_text_html(is_group=True), message_thread_id=message_thread_id) - return - except Exception: - pass - - -@bot.message_handler(func=lambda m: getattr(m.chat, "type", "") == "private", content_types=["text"]) -def handle_private_any_text(message): - try: - chat_id = int(message.chat.id) - user_id = int(getattr(message.from_user, "id", 0) or 0) - message_thread_id = getattr(message, "message_thread_id", None) - - if not _was_private_welcomed(user_id): - _safe_send_message_html(chat_id, _help_text_html(is_group=False), message_thread_id=message_thread_id) - _mark_private_welcomed(user_id) - return - - _safe_send_message(chat_id, "Нажмите /help чтобы увидеть инструкцию.", message_thread_id=message_thread_id) - except Exception: - pass - - -# ========================= -# Main handler -# ========================= - -@bot.message_handler(func=lambda m: True, content_types=["text", "photo", "video", "document", "audio", "voice"]) -def handle_group_messages(message): - try: - if message.chat.type not in ("group", "supergroup"): - return - - if getattr(message.from_user, "is_bot", False): - return - - text = message.text if message.text else (message.caption if message.caption else "") - if not isinstance(text, str) or not text.strip(): - return - - if text.strip().startswith("/"): - return - - chat_id = int(message.chat.id) - user_id = int(message.from_user.id) - message_thread_id = getattr(message, "message_thread_id", None) - bot_mentioned = _contains_bot_mention(text) - - sender_username = getattr(message.from_user, "username", None) - if bot_mentioned and _contains_sender_self_mention_or_me(text, sender_username): - new_opt_out = _toggle_opt_out(chat_id, user_id) - - if sender_username: - who = f"@{sender_username}" - else: - who = _format_sender_name(message) - - if new_opt_out: - msg = ( - f"{who}, теперь для Вас авто-скачивание отключено.\n" - f"Чтобы скачать видео, упоминайте бота и вставляйте ссылку: @" - f"{BOT_USERNAME_LOWER} <ссылка>" - ) - else: - msg = ( - f"{who}, теперь для Вас включена автоотправка видео без упоминания бота.\n" - f"Можно просто отправлять ссылки." - ) - - _safe_send_message(chat_id, msg, message_thread_id=message_thread_id) - return - - url = _extract_first_url(text) - if not url: - return - - url_info = urlparse(url) - if not url_info.scheme or not url_info.netloc: - return - - _log_request(message, url) - - opted_out = _is_opted_out(chat_id, user_id) - - if opted_out and not bot_mentioned: - return - - notify_on_fail = bool(opted_out and bot_mentioned) - - job = { - "chat_id": chat_id, - "message_thread_id": message_thread_id if isinstance(message_thread_id, int) else None, - "original_message_id": int(message.message_id), - "url": url, - "prefix": uuid.uuid4().hex[:18], - "source_name": _detect_source(url), - "sender_full_name": _format_sender_name(message), - "notify_on_fail": notify_on_fail, - "delete_original_on_success": True, - } + @bot.message_handler( + func=lambda _message: True, + content_types=["text", "photo", "video", "document", "audio", "voice"], + ) + def group_message(message: Any) -> None: + self._handle_group_message(message) + def _handle_group_message(self, message: Any) -> None: try: - jobs_q.put_nowait(job) - except queue.Full: - if notify_on_fail: - _safe_send_message(chat_id, "Не удалось скачать", message_thread_id=message_thread_id) - - except Exception: - pass + if getattr(message.chat, "type", "") not in {"group", "supergroup"}: + return + if getattr(message.from_user, "is_bot", False): + return + text = message.text or message.caption or "" + if not text.strip() or text.lstrip().startswith("/"): + return + chat_id = int(message.chat.id) + user_id = int(message.from_user.id) + mentioned = bool(self.bot_username and re.search(rf"(^|\s)@{re.escape(self.bot_username)}\b", text.lower())) + username = getattr(message.from_user, "username", None) + if mentioned and self._self_mention(text, username): + opted_out = self.storage.toggle_opt_out(chat_id, user_id) + who = f"@{username}" if username else self._sender_name(message) + key = "opted_out" if opted_out else "opted_in" + self._safe_message( + chat_id, + tr(self.storage.chat_language(chat_id), key, who=who, bot_mention=self.mention), + getattr(message, "message_thread_id", None), + ) + return + url = extract_first_url(text) + if url is None: + return + if self.storage.is_opted_out(chat_id, user_id) and not mentioned: + return + validate_public_url(url) + job = Job( + job_id=uuid.uuid4().hex[:16], + chat_id=chat_id, + message_thread_id=( + int(message.message_thread_id) + if isinstance(getattr(message, "message_thread_id", None), int) + else None + ), + original_message_id=int(message.message_id), + user_id=user_id, + url=url, + url_key=normalized_url_key(url), + sender_name=self._sender_name(message), + delete_original=self.storage.delete_original(chat_id), + ) + flight = self.coordinator.submit(job) + if flight is None: + self.log.info("job joined URL flight job_id=%s url=%s", job.job_id, safe_url_for_log(url)) + return + try: + self.queue.put_nowait(flight) + self.log.info("job queued job_id=%s chat_id=%s url=%s", job.job_id, chat_id, safe_url_for_log(url)) + except queue.Full: + self.coordinator.abort(flight) + self.log.warning("queue full job_id=%s chat_id=%s", job.job_id, chat_id) + self._operator_alert("The download queue is full. Check bot.log.") + except UnsafeUrlError: + self.log.warning("unsafe URL rejected chat_id=%s", getattr(message.chat, "id", None), exc_info=True) + except Exception: + self.log.exception("group handler failed chat_id=%s", getattr(message.chat, "id", None)) + + @staticmethod + def _self_mention(text: str, username: str | None) -> bool: + lowered = text.lower() + if re.search(r"(^|\s)(me|я)(\s|$)", lowered): + return True + return bool(username and re.search(rf"(^|\s)@{re.escape(username.lower())}\b", lowered)) + + @staticmethod + def _sender_name(message: Any) -> str: + first = str(getattr(message.from_user, "first_name", "") or "").strip() + last = str(getattr(message.from_user, "last_name", "") or "").strip() + full = f"{first} {last}".strip() + username = getattr(message.from_user, "username", None) + return full or (f"@{username}" if username else str(message.from_user.id)) + + +def main() -> None: + settings = load_settings() + configure_logging(settings.logs_dir, settings.log_level) + application = BotApplication(settings) + + def shutdown(_signum: int, _frame: Any) -> None: + application.stop() + + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + try: + application.start() + finally: + application.stop() -bot.infinity_polling(timeout=30, long_polling_timeout=30) +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..152d1ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] +ignore = ["E501"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7d8f108 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==9.1.1 +ruff==0.15.22 diff --git a/requirements.txt b/requirements.txt index ce2662a..80de845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ +pyTelegramBotAPI==4.36.0 yt-dlp[default,curl-cffi] -pyTelegramBotAPI -requests-toolbelt>=1.0.0 diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh new file mode 100644 index 0000000..8e27001 --- /dev/null +++ b/scripts/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +# Bind-mounted directories retain host ownership. Fix only the two explicitly +# writable application paths, then permanently drop privileges. +chown -R bot:bot /app/data /app/logs +exec gosu bot "$@" diff --git a/scripts/update-ytdlp.sh b/scripts/update-ytdlp.sh index ba93c65..2feabca 100644 --- a/scripts/update-ytdlp.sh +++ b/scripts/update-ytdlp.sh @@ -4,96 +4,96 @@ set -euo pipefail ROOT_DIR="${ROOT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SERVICE_KEY="${SERVICE_KEY:-linkdownloaderbot}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-linkdownloaderbotforgroups}" +IMAGE_NAME="${IMAGE_NAME:-linkdownloaderbotforgroups:local}" +ROLLBACK_IMAGE="${IMAGE_NAME%:*}:rollback" -log() { - printf '[%s] %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" -} - +log() { printf '[%s] %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } need_cmd() { command -v "$1" >/dev/null 2>&1; } detect_compose() { - if need_cmd docker && docker compose version >/dev/null 2>&1; then + if docker compose version >/dev/null 2>&1; then echo "docker compose" - return 0 - fi - if need_cmd docker-compose; then + elif need_cmd docker-compose; then echo "docker-compose" - return 0 + else + return 1 fi - return 1 } -log_runtime() { - local py="$1" - local node_line="node:missing" - if need_cmd node; then - node_line="node:$(node --version 2>/dev/null || echo unknown)" - fi - local targets_line - targets_line="$("$py" -m yt_dlp --list-impersonate-targets 2>/dev/null | tr '\n' ' ' || true)" - if [[ -n "$targets_line" ]]; then - log "$node_line | impersonate-targets: $targets_line" - else - log "$node_line | impersonate-targets: unavailable" +update_venv() { + local python="$ROOT_DIR/.venv/bin/python" + local pip="$ROOT_DIR/.venv/bin/pip" + [[ -x "$python" && -x "$pip" ]] || return 1 + local before after + before="$($python -m yt_dlp --version 2>/dev/null || true)" + "$pip" install --upgrade 'yt-dlp[default,curl-cffi]' + after="$($python -m yt_dlp --version)" + log "venv yt-dlp: ${before:-unknown} -> $after" + if [[ "$before" != "$after" ]] && systemctl list-unit-files 2>/dev/null | grep -q '^linkdownloaderbotforgroups.service'; then + systemctl restart linkdownloaderbotforgroups.service fi } -update_venv_mode() { - local venv_python="$ROOT_DIR/.venv/bin/python" - local venv_pip="$ROOT_DIR/.venv/bin/pip" - [[ -x "$venv_python" && -x "$venv_pip" ]] || return 1 +update_docker() { + local compose before_id before_version after_version + compose="$(detect_compose)" + cd "$ROOT_DIR" + before_id="$(docker image inspect "$IMAGE_NAME" --format '{{.Id}}' 2>/dev/null || true)" + before_version="$($compose -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" python -m yt_dlp --version 2>/dev/null || true)" + if [[ -n "$before_id" ]]; then + docker image tag "$IMAGE_NAME" "$ROLLBACK_IMAGE" + fi - local before after - before="$("$venv_python" -m yt_dlp --version 2>/dev/null || true)" - "$venv_pip" install --upgrade 'yt-dlp[default,curl-cffi]' >/dev/null - after="$("$venv_python" -m yt_dlp --version 2>/dev/null || true)" - log_runtime "$venv_python" + $compose -p "$COMPOSE_PROJECT" build --pull \ + --build-arg "YTDLP_CACHEBUST=$(date -u '+%Y%m%dT%H%M%SZ')" "$SERVICE_KEY" + $compose -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" \ + python -c 'import main, telebot, yt_dlp; from app.settings import load_settings; s=load_settings(); telebot.TeleBot(s.token).get_me(); print(yt_dlp.version.__version__)' >/dev/null + after_version="$($compose -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" python -m yt_dlp --version)" - if [[ -n "$before" && -n "$after" && "$before" != "$after" ]]; then - log "yt-dlp changed in venv: $before -> $after" - if need_cmd systemctl && systemctl list-unit-files | grep -q '^linkdownloaderbotforgroups.service'; then - systemctl restart linkdownloaderbotforgroups.service || true - log "restarted linkdownloaderbotforgroups.service" - fi - else - log "yt-dlp unchanged in venv: ${after:-unknown}" + if ! $compose -p "$COMPOSE_PROJECT" up -d --no-deps "$SERVICE_KEY"; then + rollback_docker "$compose" + return 1 + fi + sleep 10 + local container_id running + container_id="$($compose -p "$COMPOSE_PROJECT" ps -q "$SERVICE_KEY")" + running="$(docker inspect --format '{{.State.Running}}' "$container_id" 2>/dev/null || true)" + if [[ -z "$container_id" || "$running" != "true" ]]; then + log "new container failed; rolling back" + rollback_docker "$compose" + return 1 fi - return 0 + log "docker yt-dlp: ${before_version:-unknown} -> $after_version" } -update_docker_mode() { - local compose_cmd - compose_cmd="$(detect_compose)" || return 1 - - local before after - before="$($compose_cmd -p "$COMPOSE_PROJECT" exec -T "$SERVICE_KEY" python -m yt_dlp --version 2>/dev/null || true)" - $compose_cmd -p "$COMPOSE_PROJECT" build "$SERVICE_KEY" >/dev/null - after="$($compose_cmd -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" python -m yt_dlp --version 2>/dev/null || true)" - - if [[ -n "$before" && -n "$after" && "$before" != "$after" ]]; then - $compose_cmd -p "$COMPOSE_PROJECT" up -d --no-deps "$SERVICE_KEY" >/dev/null - log "yt-dlp changed in docker: $before -> $after (service restarted)" - else - log "yt-dlp unchanged in docker: ${after:-unknown}" +rollback_docker() { + local compose="$1" + if docker image inspect "$ROLLBACK_IMAGE" >/dev/null 2>&1; then + docker image tag "$ROLLBACK_IMAGE" "$IMAGE_NAME" + $compose -p "$COMPOSE_PROJECT" up -d --no-deps --force-recreate "$SERVICE_KEY" fi +} - local node_line="node:unknown" - node_line="$($compose_cmd -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" sh -lc 'node --version 2>/dev/null || echo missing' 2>/dev/null || true)" - local targets_line - targets_line="$($compose_cmd -p "$COMPOSE_PROJECT" run --rm --no-deps "$SERVICE_KEY" python -m yt_dlp --list-impersonate-targets 2>/dev/null | tr '\n' ' ' || true)" - log "container-node:${node_line:-missing} | impersonate-targets: ${targets_line:-unavailable}" - return 0 +docker_runtime_exists() { + local compose + compose="$(detect_compose)" || return 1 + cd "$ROOT_DIR" + $compose -p "$COMPOSE_PROJECT" ps -a --services 2>/dev/null | grep -q "^${SERVICE_KEY}$" } main() { + mkdir -p "$ROOT_DIR/logs" + find "$ROOT_DIR/logs" -maxdepth 1 -type f -name 'updater-*.log' -mtime +60 -delete + exec >>"$ROOT_DIR/logs/updater-$(date -u '+%Y-%m-%d').log" 2>&1 cd "$ROOT_DIR" - if update_venv_mode; then + if docker_runtime_exists && update_docker; then exit 0 fi - if update_docker_mode; then + if update_venv; then exit 0 fi - log "No supported runtime found (.venv or docker compose). Nothing to update." + log "no supported runtime found" + exit 1 } main "$@" diff --git a/tests/test_application_integration.py b/tests/test_application_integration.py new file mode 100644 index 0000000..9cd844e --- /dev/null +++ b/tests/test_application_integration.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import main +from app.download_backend import MediaMetadata +from app.jobs import Job +from app.settings import Settings + + +class FakeBot: + def __init__(self) -> None: + self.sends = [] + self.deletes = [] + + def send_video(self, *, video, **kwargs): + self.sends.append((video, kwargs)) + return SimpleNamespace(video=SimpleNamespace(file_id="telegram-file-id")) + + def delete_message(self, chat_id, message_id): + self.deletes.append((chat_id, message_id)) + + +def _settings(tmp_path: Path) -> Settings: + return Settings( + token="123456:abcdefghijklmnopqrstuvwxyz", + logs_chat_id=None, + data_dir=tmp_path / "data", + output_dir=tmp_path / "cache", + logs_dir=tmp_path / "logs", + cookies_file=None, + max_filesize=50_000_000, + workers=2, + max_queue=200, + upload_workers=2, + concurrent_fragments=4, + job_timeout=60, + disk_cache_max_files=5, + disk_cache_ttl=300, + file_id_cache_max_items=500, + file_id_cache_ttl_days=30, + media_cache_enabled=True, + delete_original=True, + default_language="en", + log_level="INFO", + ) + + +def test_three_requests_download_and_upload_once(tmp_path, monkeypatch) -> None: + app = main.BotApplication(_settings(tmp_path)) + fake = FakeBot() + app.bot = fake + flight = None + for index in range(3): + job = Job( + str(index), + -(index + 1), + None, + index + 10, + index + 20, + "https://example.com/video", + "https://example.com/video", + f"User {index}", + True, + ) + submitted = app.coordinator.submit(job) + flight = submitted or flight + assert flight is not None + + metadata = MediaMetadata( + url="https://example.com/video", + info={"id": "video", "extractor": "Test"}, + media_key="test:video", + source_name="Test", + ) + video_path = tmp_path / "video.mp4" + video_path.write_bytes(b"video") + calls = {"downloads": 0} + + monkeypatch.setattr(main, "validate_public_url", lambda url: url) + monkeypatch.setattr(main, "extract_metadata", lambda *_args: metadata) + + def obtain(*_args): + calls["downloads"] += 1 + return video_path + + monkeypatch.setattr(app, "_obtain_file", obtain) + app._process_flight(flight) + + assert calls["downloads"] == 1 + assert len(fake.sends) == 3 + assert not isinstance(fake.sends[0][0], str) + assert fake.sends[1][0] == "telegram-file-id" + assert fake.sends[2][0] == "telegram-file-id" + assert len(fake.deletes) == 3 + + delayed = Job( + "delayed", + -4, + None, + 20, + 30, + "https://example.com/video", + "https://example.com/video", + "Later User", + True, + ) + delayed_flight = app.coordinator.submit(delayed) + assert delayed_flight is not None + monkeypatch.setattr(main, "extract_metadata", lambda *_args: (_ for _ in ()).throw(AssertionError("metadata call"))) + app._process_flight(delayed_flight) + assert calls["downloads"] == 1 + assert fake.sends[3][0] == "telegram-file-id" + + +def test_extract_first_url_trims_punctuation() -> None: + assert main.extract_first_url("look (https://example.com/video).") == "https://example.com/video" + + +def test_self_mention_requires_a_message_token() -> None: + assert main.BotApplication._self_mention("hello @alice", "alice") + assert not main.BotApplication._self_mention("https://example.com/@alice/video", "alice") diff --git a/tests/test_download_backend.py b/tests/test_download_backend.py new file mode 100644 index 0000000..7918481 --- /dev/null +++ b/tests/test_download_backend.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from pathlib import Path + +from app import download_backend +from app.download_backend import MediaMetadata, download_metadata, select_format + + +def test_selects_best_format_that_fits() -> None: + info = { + "duration": 60, + "formats": [ + { + "format_id": "small", + "ext": "mp4", + "vcodec": "avc1.4d401f", + "acodec": "mp4a.40.2", + "height": 720, + "filesize": 20_000_000, + }, + { + "format_id": "large", + "ext": "mp4", + "vcodec": "avc1.640028", + "acodec": "mp4a.40.2", + "height": 1080, + "filesize": 80_000_000, + }, + ], + } + assert select_format(info, 50_000_000) == ("small", None) + + +def test_combines_video_and_audio_sizes() -> None: + info = { + "formats": [ + { + "format_id": "v720", + "ext": "mp4", + "vcodec": "avc1", + "acodec": "none", + "height": 720, + "filesize": 30_000_000, + }, + { + "format_id": "v1080", + "ext": "mp4", + "vcodec": "avc1", + "acodec": "none", + "height": 1080, + "filesize": 49_000_000, + }, + { + "format_id": "audio", + "ext": "m4a", + "vcodec": "none", + "acodec": "mp4a", + "filesize": 5_000_000, + }, + ] + } + assert select_format(info, 50_000_000) == ("v720+audio", "mp4") + + +def test_known_size_wins_over_higher_unknown_format() -> None: + info = { + "formats": [ + { + "format_id": "known", + "ext": "mp4", + "vcodec": "avc1", + "acodec": "mp4a", + "height": 720, + "filesize": 20_000_000, + }, + { + "format_id": "unknown", + "ext": "mp4", + "vcodec": "avc1", + "acodec": "mp4a", + "height": 2160, + }, + ] + } + assert select_format(info, 50_000_000) == ("known", None) + + +def test_youtube_retry_reextracts_with_runtime_and_selected_format(monkeypatch, tmp_path: Path) -> None: + options_seen: list[dict] = [] + + class FakeYDL: + def __init__(self, options: dict) -> None: + self.options = options + options_seen.append(options) + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def process_ie_result(self, *_args, **_kwargs): + raise RuntimeError("expired media URL") + + def extract_info(self, *_args, **_kwargs): + return {"id": "video", "extractor": "youtube"} + + monkeypatch.setattr(download_backend.yt_dlp, "YoutubeDL", FakeYDL) + monkeypatch.setattr( + download_backend, + "_site_options", + lambda *_args, **_kwargs: {"js_runtimes": {"node": {}}}, + ) + metadata = MediaMetadata( + url="https://www.youtube.com/watch?v=video", + info={ + "formats": [ + { + "format_id": "video", + "ext": "mp4", + "vcodec": "avc1", + "acodec": "none", + "filesize": 4_000_000, + }, + { + "format_id": "audio", + "ext": "m4a", + "vcodec": "none", + "acodec": "mp4a", + "filesize": 1_000_000, + }, + ] + }, + media_key="youtube:video", + source_name="YouTube", + ) + + download_metadata( + metadata, + "retry", + tmp_path, + max_send_bytes=10_000_000, + concurrent_fragments=2, + ) + + assert len(options_seen) == 2 + assert options_seen[1]["format"] == "video+audio" + assert options_seen[1]["merge_output_format"] == "mp4" + assert options_seen[1]["js_runtimes"] == {"node": {}} diff --git a/tests/test_jobs_and_cache.py b/tests/test_jobs_and_cache.py new file mode 100644 index 0000000..ebfd044 --- /dev/null +++ b/tests/test_jobs_and_cache.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import os +import time + +from app.jobs import FlightCoordinator, Job +from app.media_cache import DiskMediaCache + + +def _job(job_id: str, url_key: str = "url") -> Job: + return Job(job_id, -1, None, 1, 2, "https://example.com/v", url_key, "User", True) + + +def test_coalesces_same_url_and_media() -> None: + coordinator = FlightCoordinator() + first = coordinator.submit(_job("a", "url-a")) + assert first is not None + assert coordinator.submit(_job("b", "url-a")) is None + second = coordinator.submit(_job("c", "url-c")) + assert second is not None + assert coordinator.promote(first, "youtube:id") + assert not coordinator.promote(second, "youtube:id") + assert [job.job_id for job in coordinator.pending(first)] == ["a", "b", "c"] + assert coordinator.finish_if_idle(first) + + +def test_new_job_prevents_flight_from_finishing() -> None: + coordinator = FlightCoordinator() + flight = coordinator.submit(_job("a")) + assert flight is not None + assert [job.job_id for job in coordinator.pending(flight)] == ["a"] + assert coordinator.submit(_job("b")) is None + assert not coordinator.finish_if_idle(flight) + assert [job.job_id for job in coordinator.pending(flight)] == ["b"] + assert coordinator.finish_if_idle(flight) + + +def test_disk_cache_ttl_and_lru(tmp_path) -> None: + cache = DiskMediaCache(tmp_path, max_files=2, ttl_seconds=30) + paths = [] + for index, key in enumerate(("a", "b", "c")): + path = tmp_path / f"{cache.prefix(key)}.mp4" + path.write_bytes(b"video") + os.utime(path, (time.time() + index, time.time() + index)) + paths.append(path) + cache.maintain() + assert not paths[0].exists() + assert paths[1].exists() and paths[2].exists() + + old = paths[1] + os.utime(old, (time.time() - 60, time.time() - 60)) + assert cache.get("b") is None + assert not old.exists() diff --git a/tests/test_logging_and_i18n.py b/tests/test_logging_and_i18n.py new file mode 100644 index 0000000..a1ef3ce --- /dev/null +++ b/tests/test_logging_and_i18n.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import logging + +from app.i18n import tr +from app.logging_setup import configure_logging + + +def test_log_redacts_url_queries(tmp_path) -> None: + logger = configure_logging(tmp_path, "INFO") + logger.info("failed https://example.com/video?token=very-secret") + for handler in logging.getLogger().handlers: + handler.flush() + content = (tmp_path / "bot.log").read_text(encoding="utf-8") + assert "very-secret" not in content + assert "?" in content + for handler in list(logging.getLogger().handlers): + handler.close() + logging.getLogger().removeHandler(handler) + + +def test_translation_falls_back_to_english() -> None: + assert "Current language" in tr("unknown", "language_current", language="en") diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..4c897e0 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +from app.storage import JsonFile, Storage + + +def test_migrates_legacy_preferences(tmp_path) -> None: + (tmp_path / "prefs.json").write_text( + json.dumps( + { + "opt_out": {"-1": {"42": True}}, + "welcomed_groups": {"-1": True}, + "welcomed_private": {"42": True}, + } + ), + encoding="utf-8", + ) + storage = Storage(tmp_path) + assert storage.is_opted_out(-1, 42) + assert storage.was_welcomed("group", -1) + assert storage.was_welcomed("private", 42) + assert (tmp_path / "prefs.json").exists() + + +def test_preferences_and_language_are_separate(tmp_path) -> None: + storage = Storage(tmp_path) + assert storage.chat_language(-10) == "en" + storage.set_chat_language(-10, "ru") + assert storage.chat_language(-10) == "ru" + storage.set_delete_original(-10, False) + assert storage.delete_original(-10) is False + assert storage.toggle_opt_out(-10, 7) is True + assert storage.toggle_opt_out(-10, 7) is False + assert "language" in (tmp_path / "settings.json").read_text(encoding="utf-8") + assert "opt_out" in (tmp_path / "users.json").read_text(encoding="utf-8") + + +def test_corrupt_store_recovers_from_backup(tmp_path) -> None: + path = tmp_path / "value.json" + store = JsonFile(path, lambda: {"version": 1, "value": 0}) + store.update(lambda data: data.__setitem__("value", 1)) + store.update(lambda data: data.__setitem__("value", 2)) + path.write_text("{broken", encoding="utf-8") + recovered = JsonFile(path, lambda: {"version": 1, "value": 0}) + assert recovered.snapshot()["value"] == 1 + assert list(tmp_path.glob("value.json.corrupt-*")) + + +def test_file_id_cache_expiry_and_limit(tmp_path) -> None: + storage = Storage(tmp_path) + storage.put_file_id("a", "id-a", max_items=2) + storage.put_file_id("b", "id-b", max_items=2) + storage.put_file_id("c", "id-c", max_items=2) + assert storage.get_file_id("a", 30) is None + assert storage.get_file_id("c", 30) == "id-c" + + expired = (datetime.now(UTC) - timedelta(days=2)).isoformat() + storage.media.update(lambda data: data["items"]["c"].__setitem__("created_at", expired)) + assert storage.get_file_id("c", 1) is None + + +def test_media_alias_is_hashed(tmp_path) -> None: + storage = Storage(tmp_path) + secret_url = "https://example.com/video?token=secret" + storage.put_file_id("media", "file-id", 10, source_name="Example", url_keys={secret_url}) + assert storage.get_cached_by_url(secret_url, 30) == ("media", "file-id", "Example") + assert secret_url not in (tmp_path / "media_cache.json").read_text(encoding="utf-8") + + +def test_prunes_expired_media_and_aliases(tmp_path) -> None: + storage = Storage(tmp_path) + storage.put_file_id("old", "file-id", 10, url_keys={"url"}) + expired = (datetime.now(UTC) - timedelta(days=10)).isoformat() + storage.media.update(lambda data: data["items"]["old"].__setitem__("created_at", expired)) + storage.prune_media_cache(ttl_days=1, max_items=10) + snapshot = storage.media.snapshot() + assert snapshot["items"] == {} + assert snapshot["aliases"] == {} diff --git a/tests/test_url_security.py b/tests/test_url_security.py new file mode 100644 index 0000000..8cd661d --- /dev/null +++ b/tests/test_url_security.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import socket + +import pytest + +from app.url_security import UnsafeUrlError, normalized_url_key, safe_url_for_log, validate_public_url + + +def _dns(address: str): + return [(socket.AF_INET6 if ":" in address else socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 443))] + + +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1/video", + "http://10.0.0.1/video", + "http://172.16.0.1/video", + "http://192.168.1.1/video", + "http://169.254.169.254/latest/meta-data", + "http://[::1]/video", + "http://[fc00::1]/video", + "file:///etc/passwd", + "http://user:pass@example.com/video", + "http://localhost/video", + ], +) +def test_rejects_unsafe_urls(url: str) -> None: + with pytest.raises(UnsafeUrlError): + validate_public_url(url) + + +def test_rejects_hostname_with_private_dns(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(socket, "getaddrinfo", lambda *_args, **_kwargs: _dns("192.168.1.10")) + with pytest.raises(UnsafeUrlError): + validate_public_url("https://example.com/video") + + +def test_accepts_public_dns(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(socket, "getaddrinfo", lambda *_args, **_kwargs: _dns("93.184.216.34")) + assert validate_public_url("https://example.com/video") == "https://example.com/video" + + +def test_normalization_removes_tracking_and_fragment() -> None: + left = normalized_url_key("HTTPS://Example.COM:443/video?utm_source=x&id=2&si=abc#part") + right = normalized_url_key("https://example.com/video?id=2") + assert left == right + + +def test_log_url_hides_query() -> None: + assert safe_url_for_log("https://example.com/video?token=secret") == "https://example.com/video"