translator-docker

.github/workflows/build_docker.yml

@@ -0,0 +1,42 @@
+name: Build and Push Docker Image
+
+on:
+    push:
+      branches:
+        - master
+      paths-ignore:
+        - 'scripts/**'
+        - '.gitignore'
+        - '.github/**'
+        - 'book/**'
+    workflow_dispatch:
+
+concurrency: build_docker
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Check out the repository to get the Dockerfile
+      - name: Dowload Dockerfile
+        run: wget https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/Dockerfile
+
+      # 2. Log into GitHub Container Registry
+      - name: Log in to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # 3. Build and push
+      - name: Build and push Docker image
+        run: |
+          # Define image name
+          IMAGE_NAME=ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image
+
+          # Build Docker image
+          docker build -t $IMAGE_NAME:latest .
+
+          # Push Docker image to GHCR
+          docker push $IMAGE_NAME:latest

.github/workflows/translate_af.yml

@@ -8,104 +8,65 @@ on:
       - 'scripts/**'
       - '.gitignore'
       - '.github/**'
+      - Dockerfile
   workflow_dispatch:
 
 concurrency: af
 
 permissions:
+  packages: write
   id-token: write
   contents: write
 
 jobs:
   run-translation:
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
     environment: prod
     env:
       LANGUAGE: Afrikaans
       BRANCH: af
-
+    
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
         with:
-          fetch-depth: 0 #Needed to download everything to be able to access the master & language branches
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.12
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip3 install openai tqdm tiktoken
-      
-      # Install Rust and Cargo
-      - name: Install Rust and Cargo
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+          fetch-depth: 0
 
-      # Install mdBook and Plugins
-      - name: Install mdBook and Plugins
+      - name: Download language branch
         run: |
-          cargo install mdbook
-          cargo install mdbook-alerts
-          cargo install mdbook-reading-time
-          cargo install mdbook-pagetoc
-          cargo install mdbook-tabs
-          cargo install mdbook-codename
+          git config --global --add safe.directory /__w/hacktricks/hacktricks
+          git config --global user.name 'Translator'
+          git config --global user.email '[email protected]'
+          git config pull.rebase false
+          git checkout $BRANCH
+          git pull
+          git checkout master
 
-      
-      - name: Update & install wget & translator.py
+      - name: Update & install translator.py (if needed)
         run: |
           sudo apt-get update
           sudo apt-get install wget -y
           mkdir scripts
           cd scripts
           wget https://raw.githubusercontent.com/carlospolop/hacktricks-cloud/master/scripts/translator.py
           cd ..
-
-      - name: Download language branch #Make sure we have last version
-        run: |
-          git config --global user.name 'Translator'
-          git config --global user.email '[email protected]'
-          git checkout "$BRANCH"
-          git pull
-          git checkout master
+      
 
       - name: Run translation script on changed files
         run: |
-          echo "Starting translations"
-          echo "Commit: $GITHUB_SHA"
-
-          # Export the OpenAI API key as an environment variable
           export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
-
-          # Run the translation script on each changed file
           git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do
             if echo "$file" | grep -qE '\.md$'; then
               echo -n "$file , " >> /tmp/file_paths.txt
-            else
-              echo "Skipping $file"
             fi
           done
-
-          echo "Translating $(cat /tmp/file_paths.txt)"
           python scripts/translator.py --language "$LANGUAGE" --branch "$BRANCH" --api-key "$OPENAI_API_KEY" -f "$(cat /tmp/file_paths.txt)" -t 3
 
-      # Push changes to the repository
-      - name: Commit and push changes
-        run: |
-          git checkout "$BRANCH"
-          git add -A
-          git commit -m "Translated $BRANCH files" || true
-          git push --set-upstream origin "$BRANCH"
-
-      # Build the mdBook
       - name: Build mdBook
-        run: MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build
+        run: |
+          MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
       
       # Login in AWs
       - name: Configure AWS credentials using OIDC
@@ -116,4 +77,4 @@ jobs:
 
       # Sync the build to S3
       - name: Sync to S3
-        run: aws s3 sync ./book s3://hacktricks-wiki/$BRANCH --delete
+        run: aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete

.github/workflows/translate_de.yml

@@ -8,104 +8,65 @@ on:
       - 'scripts/**'
       - '.gitignore'
       - '.github/**'
+      - Dockerfile
   workflow_dispatch:
 
 concurrency: de
 
 permissions:
+  packages: write
   id-token: write
   contents: write
 
 jobs:
   run-translation:
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
     environment: prod
     env:
       LANGUAGE: German
       BRANCH: de
-
+    
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
         with:
-          fetch-depth: 0 #Needed to download everything to be able to access the master & language branches
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.12
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip3 install openai tqdm tiktoken
-      
-      # Install Rust and Cargo
-      - name: Install Rust and Cargo
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+          fetch-depth: 0
 
-      # Install mdBook and Plugins
-      - name: Install mdBook and Plugins
+      - name: Download language branch
         run: |
-          cargo install mdbook
-          cargo install mdbook-alerts
-          cargo install mdbook-reading-time
-          cargo install mdbook-pagetoc
-          cargo install mdbook-tabs
-          cargo install mdbook-codename
+          git config --global --add safe.directory /__w/hacktricks/hacktricks
+          git config --global user.name 'Translator'
+          git config --global user.email '[email protected]'
+          git config pull.rebase false
+          git checkout $BRANCH
+          git pull
+          git checkout master
 
-      
-      - name: Update & install wget & translator.py
+      - name: Update & install translator.py (if needed)
         run: |
           sudo apt-get update
           sudo apt-get install wget -y
           mkdir scripts
           cd scripts
           wget https://raw.githubusercontent.com/carlospolop/hacktricks-cloud/master/scripts/translator.py
           cd ..
-
-      - name: Download language branch #Make sure we have last version
-        run: |
-          git config --global user.name 'Translator'
-          git config --global user.email '[email protected]'
-          git checkout "$BRANCH"
-          git pull
-          git checkout master
+      
 
       - name: Run translation script on changed files
         run: |
-          echo "Starting translations"
-          echo "Commit: $GITHUB_SHA"
-
-          # Export the OpenAI API key as an environment variable
           export OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
-
-          # Run the translation script on each changed file
           git diff --name-only HEAD~1 | grep -v "SUMMARY.md" | while read -r file; do
             if echo "$file" | grep -qE '\.md$'; then
               echo -n "$file , " >> /tmp/file_paths.txt
-            else
-              echo "Skipping $file"
             fi
           done
-
-          echo "Translating $(cat /tmp/file_paths.txt)"
           python scripts/translator.py --language "$LANGUAGE" --branch "$BRANCH" --api-key "$OPENAI_API_KEY" -f "$(cat /tmp/file_paths.txt)" -t 3
 
-      # Push changes to the repository
-      - name: Commit and push changes
-        run: |
-          git checkout "$BRANCH"
-          git add -A
-          git commit -m "Translated $BRANCH files" || true
-          git push --set-upstream origin "$BRANCH"
-
-      # Build the mdBook
       - name: Build mdBook
-        run: MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build
+        run: |
+          MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
       
       # Login in AWs
       - name: Configure AWS credentials using OIDC
@@ -116,4 +77,4 @@ jobs:
 
       # Sync the build to S3
       - name: Sync to S3
-        run: aws s3 sync ./book s3://hacktricks-wiki/$BRANCH --delete
+        run: aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete