From ed2cf298855fad1d143241453b70c22077fcbd59 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Thu, 15 Aug 2024 00:21:55 +0200 Subject: [PATCH 1/3] add steps about sig_helper and potoken + depreciate gluetun guide --- docs/index.md | 1 - docs/installation.md | 219 ++++++++++++++++++++++++++++++------------- mkdocs.yml | 1 - 3 files changed, 156 insertions(+), 65 deletions(-) diff --git a/docs/index.md b/docs/index.md index 098e4364..176730d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,7 +22,6 @@ - [NGINX reverse proxy setup](./nginx.md) - [Caddy reverse proxy setup](./caddy.md) - [Apache2 reverse proxy setup](./apache2.md) -- [Make Invidious requests data from YouTube through a VPN using Gluetun (in case your IP is blocked)](./gluetun.md) - [Database maintenance](./db-maintenance.md) - [CAPTCHA bug on Debian and Ubuntu](./captcha-bug.md) - [Registering users manually](./register-user.md) diff --git a/docs/installation.md b/docs/installation.md index 9101ac0f..45c0e98f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -12,6 +12,8 @@ Running Invidious requires at least 20GB disk space, 512MB of free RAM (so ~2G i Compiling Invidious requires at least 2.5GB of free RAM (We recommend to have at least 4GB installed). If you have less (e.g on a cheap VPS) you can setup a SWAP file or partition, so the combined amount is >= 4GB. +You need at least 1GB of RAM for the machine that will run the tool `youtube-trusted-session-generator` in the 1st step. Doesn't need to be the same machine as the one running Invidious, just a machine running on the same public IP address. + ## Docker **The Invidious docker image is only [available on Quay](https://quay.io/repository/invidious/invidious) because, unlike Docker Hub, [Quay is Free and Open Source Software](https://github.com/quay/quay/blob/master/LICENSE). This is reflected in the `docker-compose.yml` file used in this walk-through.** @@ -24,71 +26,105 @@ Ensure [Docker Engine](https://docs.docker.com/engine/install) and [Docker Compo Note: Currently the repository has to be cloned, this is because the `init-invidious-db.sh` file and the `config/sql` directory have to be mounted to the postgres container (See the volumes section in the docker-compose file below). This "problem" will be solved in the future. -```bash -git clone https://github.com/iv-org/invidious.git -cd invidious -``` +??? warning "About po_token and visitor_data identities" + + po_token known as Proof of Origin Token. This is an attestation token generated by a complex anti robot verification system created by Google named BotGuard/DroidGuard. It is used to confirm that the request is coming from a genuine device. + + These identity tokens (po_token and visitor_data) generated in this tutorial will make your entire Invidious session more easily traceable by YouTube because it is tied to a unique identifier. + + There is currently no official automatic tool to periodically change these tokens. This is working in progress but, for the time being, this is the solution the Invidious team is offering. + + If you want to be less traceable, you can always script the process by changing the identities every X hour. + + +1. Generate po_token and visitor_data identities for passing all verification checks on YouTube side: + ``` + docker run quay.io/invidious/youtube-trusted-session-generator + ``` + You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. + You will need to copy these two parameters in the third step. + +2. Execute these commands: + ```bash + git clone https://github.com/iv-org/invidious.git + cd invidious + ``` + +3. Edit the docker-compose.yml with this content: + + ```docker + version: "3" + services: + + invidious: + image: quay.io/invidious/invidious:latest + # image: quay.io/invidious/invidious:latest-arm64 # ARM64/AArch64 devices + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + # Please read the following file for a comprehensive list of all available + # configuration options and their associated syntax: + # https://github.com/iv-org/invidious/blob/master/config/config.example.yml + INVIDIOUS_CONFIG: | + db: + dbname: invidious + user: kemal + password: kemal + host: invidious-db + port: 5432 + check_tables: true + signature_server: inv_sig_helper:12999 + po_token: CHANGE_ME + visitor_data: CHANGE_ME + # external_port: + # domain: + # https_only: false + # statistics_enabled: false + hmac_key: "CHANGE_ME!!" + healthcheck: + test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 + interval: 30s + timeout: 5s + retries: 2 + logging: + options: + max-size: "1G" + max-file: "4" + depends_on: + - invidious-db + + inv_sig_helper: + image: quay.io/invidious/inv-sig-helper:latest + command: ["--tcp", "0.0.0.0:12999"] + environment: + - RUST_LOG=info + restart: unless-stopped + cap_drop: + - ALL + read_only: true + security_opt: + - no-new-privileges:true + + invidious-db: + image: docker.io/library/postgres:14 + restart: unless-stopped + volumes: + - postgresdata:/var/lib/postgresql/data + - ./config/sql:/config/sql + - ./docker/init-invidious-db.sh:/docker-entrypoint-initdb.d/init-invidious-db.sh + environment: + POSTGRES_DB: invidious + POSTGRES_USER: kemal + POSTGRES_PASSWORD: kemal + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] -Edit the docker-compose.yml with this content: - -```docker -version: "3" -services: - - invidious: - image: quay.io/invidious/invidious:latest - # image: quay.io/invidious/invidious:latest-arm64 # ARM64/AArch64 devices - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - # Please read the following file for a comprehensive list of all available - # configuration options and their associated syntax: - # https://github.com/iv-org/invidious/blob/master/config/config.example.yml - INVIDIOUS_CONFIG: | - db: - dbname: invidious - user: kemal - password: kemal - host: invidious-db - port: 5432 - check_tables: true - # external_port: - # domain: - # https_only: false - # statistics_enabled: false - hmac_key: "CHANGE_ME!!" - healthcheck: - test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 - interval: 30s - timeout: 5s - retries: 2 - logging: - options: - max-size: "1G" - max-file: "4" - depends_on: - - invidious-db - - invidious-db: - image: docker.io/library/postgres:14 - restart: unless-stopped volumes: - - postgresdata:/var/lib/postgresql/data - - ./config/sql:/config/sql - - ./docker/init-invidious-db.sh:/docker-entrypoint-initdb.d/init-invidious-db.sh - environment: - POSTGRES_DB: invidious - POSTGRES_USER: kemal - POSTGRES_PASSWORD: kemal - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - -volumes: - postgresdata: -``` + postgresdata: + ``` -Note: This compose is made for a true "production" setup, where Invidious is behind a reverse proxy. If you prefer to directly access Invidious, replace `127.0.0.1:3000:3000` with `3000:3000` under the `ports:` section. + Note: This compose is made for a true "production" setup, where Invidious is behind a reverse proxy. If you prefer to directly access Invidious, replace `127.0.0.1:3000:3000` with `3000:3000` under the `ports:` section. ### Docker-compose method (development) @@ -106,6 +142,31 @@ docker-compose up ### Linux +#### Generate po_token and visitor_data identities + +[Follow these instructions here on the official tool `youtube-trusted-session-generator`](https://github.com/iv-org/youtube-trusted-session-generator?tab=readme-ov-file#tutorial-without-docker) + +These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. + +??? warning "About po_token and visitor_data identities" + + po_token known as Proof of Origin Token. This is an attestation token generated by a complex anti robot verification system created by Google named BotGuard/DroidGuard. It is used to confirm that the request is coming from a genuine device. + + These identity tokens (po_token and visitor_data) generated in this tutorial will make your entire Invidious session more easily traceable by YouTube because it is tied to a unique identifier. + + There is currently no official automatic tool to periodically change these tokens. This is working in progress but, for the time being, this is the solution the Invidious team is offering. + + If you want to be less traceable, you can always script the process by changing the identities every X hour. + + +#### Run inv_sig_helper in background + +[Follow these instructions here on the official tool `inv_sig_helper`](https://github.com/iv-org/inv_sig_helper?tab=readme-ov-file#building-and-running-without-docker) and run it in the background with systemd for example. + +inv_sig_helper handle the "deciphering" of the video stream fetched from YouTube servers. As it is running untrusted code from Google themselves, make sure to isolate it by for example running it inside a LXC or locked down through systemd. + +Call for action: A systemd service example is welcome, [if you want to contribute to one](https://github.com/iv-org/documentation/edit/master/docs/installation.md#linux). + #### Install Crystal Follow the instructions for your distribution here: https://crystal-lang.org/install/ @@ -158,6 +219,10 @@ make # Configure config/config.yml as you like cp config/config.example.yml config/config.yml +# edit config.yaml to include po_token and visitor_data previously generated + +edit config/config.yaml + # Deploy the database ./invidious --migrate @@ -173,6 +238,30 @@ systemctl enable --now invidious.service ### MacOS +#### Generate po_token and visitor_data identities + +[Follow these instructions here on the official tool `youtube-trusted-session-generator`](https://github.com/iv-org/youtube-trusted-session-generator?tab=readme-ov-file#tutorial-without-docker) + +These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. + +??? warning "About po_token and visitor_data identities" + + po_token known as Proof of Origin Token. This is an attestation token generated by a complex anti robot verification system created by Google named BotGuard/DroidGuard. It is used to confirm that the request is coming from a genuine device. + + These identity tokens (po_token and visitor_data) generated in this tutorial will make your entire Invidious session more easily traceable by YouTube because it is tied to a unique identifier. + + There is currently no official automatic tool to periodically change these tokens. This is working in progress but, for the time being, this is the solution the Invidious team is offering. + + If you want to be less traceable, you can always script the process by changing the identities every X hour. + +#### Run inv_sig_helper in background + +[Follow these instructions here on the official tool `inv_sig_helper`](https://github.com/iv-org/inv_sig_helper?tab=readme-ov-file#building-and-running-without-docker) + +inv_sig_helper handle the "deciphering" of the video stream fetched from YouTube servers. As it is running untrusted code from Google themselves, make sure to isolate it by for example running it inside Docker or a VM. + +Call for action: An example here is welcome, [if you want to contribute to one](https://github.com/iv-org/documentation/edit/master/docs/installation.md#macos). + #### Install the dependencies ```bash @@ -211,7 +300,11 @@ psql invidious kemal < config/sql/playlist_videos.sql make # Configure config/config.yml as you like -cp config/config.example.yml config/config.yml +cp config/config.example.yml config/config.yml + +# edit config.yaml to include po_token and visitor_data previously generated + +edit config/config.yaml ``` ### Windows diff --git a/mkdocs.yml b/mkdocs.yml index 51657a8a..855b1775 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,7 +27,6 @@ nav: - 'nginx.md' - 'caddy.md' - 'apache2.md' - - 'gluetun.md' - 'db-maintenance.md' - 'captcha-bug.md' - 'register-user.md' From bb11c5e1fce74202c3275d1d64e1fba3a1871fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:39:53 +0200 Subject: [PATCH 2/3] reorder token position Co-authored-by: TheFrenchGhosty <47571719+TheFrenchGhosty@users.noreply.github.com> --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 45c0e98f..94997f9c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -75,8 +75,8 @@ Note: Currently the repository has to be cloned, this is because the `init-invid port: 5432 check_tables: true signature_server: inv_sig_helper:12999 - po_token: CHANGE_ME visitor_data: CHANGE_ME + po_token: CHANGE_ME # external_port: # domain: # https_only: false From 564fba4783803ed9e08b1ad7216511c65068547e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Sun, 18 Aug 2024 14:06:00 +0200 Subject: [PATCH 3/3] add note about token validity for same ip range --- docs/installation.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 94997f9c..53dd8959 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -41,16 +41,17 @@ Note: Currently the repository has to be cloned, this is because the `init-invid ``` docker run quay.io/invidious/youtube-trusted-session-generator ``` - You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. - You will need to copy these two parameters in the third step. + You have to run this command on the same public IP address as the one blocked by YouTube. Not necessarily the same machine, just the same public IP address. + You will need to copy these two parameters in the third step. + Subsequent usage of this same token will work on the same IP range or even the same ASN. The point is to generate this token on a blocked IP as "unblocked" IP addresses seems to not generate a token valid for passing the checks on a blocked IP. -2. Execute these commands: +3. Execute these commands: ```bash git clone https://github.com/iv-org/invidious.git cd invidious ``` -3. Edit the docker-compose.yml with this content: +4. Edit the docker-compose.yml with this content: ```docker version: "3" @@ -146,7 +147,11 @@ docker-compose up [Follow these instructions here on the official tool `youtube-trusted-session-generator`](https://github.com/iv-org/youtube-trusted-session-generator?tab=readme-ov-file#tutorial-without-docker) -These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. +These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. + +You have to run this command on the same public IP address as the one blocked by YouTube. Not necessarily the same machine, just the same public IP address. +You will need to copy these two parameters in the `config.yaml` file. +Subsequent usage of this same token will work on the same IP range or even the same ASN. The point is to generate this token on a blocked IP as "unblocked" IP addresses seems to not generate a token valid for passing the checks on a blocked IP. ??? warning "About po_token and visitor_data identities" @@ -242,7 +247,11 @@ systemctl enable --now invidious.service [Follow these instructions here on the official tool `youtube-trusted-session-generator`](https://github.com/iv-org/youtube-trusted-session-generator?tab=readme-ov-file#tutorial-without-docker) -These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. You have to run this command on the same public IP address as the Invidious server. Not necessarily the same machine, just the same public IP address. +These two parameters will be required for passing all verification checks on YouTube side and you will have to configure them in Invidious. + +You have to run this command on the same public IP address as the one blocked by YouTube. Not necessarily the same machine, just the same public IP address. +You will need to copy these two parameters in the `config.yaml` file. +Subsequent usage of this same token will work on the same IP range or even the same ASN. The point is to generate this token on a blocked IP as "unblocked" IP addresses seems to not generate a token valid for passing the checks on a blocked IP. ??? warning "About po_token and visitor_data identities"