github-azure-pipeline.yml

# Azure Pipeline: Sync Microsoft Learn Docs to Agent Skills
#
# This pipeline:
# 1. Checks out GitHub repo (contains products/, skills/)
# 2. Checks out ADO repo (contains src/, config/)
# 3. Runs docs2skills to crawl and classify docs
# 4. Pushes results back to GitHub repo

trigger: none

parameters:
  - name: fullSync
    displayName: 'Full Sync (use --full flag)'
    type: boolean
    default: false
  - name: targetBranch
    displayName: 'Target GitHub Branch'
    type: string
    default: 'main'
  - name: docs2SkillsADOBranch
    displayName: 'Docs2Skills ADO Branch'
    type: string
    default: 'main'

# schedules:
#   - cron: "0 2 * * *"  # Daily at 2:00 AM UTC
#     displayName: Run every day
#     branches:
#       include:
#         - main
#     always: true

variables:
  - group: Docs2Skills-Defaults
  - name: pythonVersion
    value: '3.12'

stages:
  - stage: __default
    jobs:
      - job: Job
        pool:
          vmImage: ubuntu-latest
        timeoutInMinutes: 240

        steps:
          # 1. Checkout GitHub repo (self - contains products/, skills/)
          - task: Checkout@1
            displayName: 'Checkout GitHub repo'
            inputs:
              repository: self
              persistCredentials: true

          # 2. Checkout ADO repo (source code)
          - task: Checkout@1
            displayName: 'Checkout ADO repo (code)'
            inputs:
              repository: git://Engineering/Learn.Docs2AgentSkills@refs/heads/${{ parameters.docs2SkillsADOBranch }}

          # 3. Setup Python
          - task: UsePythonVersion@0
            displayName: 'Setup Python $(pythonVersion)'
            inputs:
              versionSpec: '$(pythonVersion)'

          # 4. Install dependencies
          - task: Bash@3
            displayName: 'Install dependencies'
            inputs:
              targetType: inline
              script: |
                set -e
                cd $(Build.SourcesDirectory)/Learn.Docs2AgentSkills
                pip install --upgrade pip
                pip install -r requirements.txt

          # 5. Run docs2skills with Azure identity
          - task: AzureCLI@2
            displayName: 'Run docs2skills'
            inputs:
              azureSubscription: $(azureSubscription)
              scriptType: bash
              scriptLocation: inlineScript
              inlineScript: |
                set -e

                # Define paths
                ADO_REPO="$(Build.SourcesDirectory)/Learn.Docs2AgentSkills"
                GITHUB_REPO="$(Build.SourcesDirectory)/$(basename $(Build.Repository.Uri) .git)"
                PRODUCTS_DIR="$GITHUB_REPO/products"
                SKILLS_DIR="$GITHUB_REPO/skills"

                cd $ADO_REPO

                echo "========================================"
                echo "Starting docs sync"
                echo "========================================"
                echo "ADO repo (code): $ADO_REPO"
                echo "GitHub repo (data): $GITHUB_REPO"
                echo "Products dir: $PRODUCTS_DIR"
                echo "Skills dir: $SKILLS_DIR"
                echo "Azure OpenAI: $(azureOpenAIEndpoint)"
                echo "Deployment: $(azureOpenAIDeployment)"
                echo "Full Sync: ${{ parameters.fullSync }}"
                echo "Target Branch: ${{ parameters.targetBranch }}"
                echo "========================================"

                FULL_FLAG=""
                if [ "${{ parameters.fullSync }}" = "True" ]; then
                  FULL_FLAG="--full"
                  echo "Running with --full flag"
                    # check before delete
                    if [ -n "$PRODUCTS_DIR" ] && [ -d "$PRODUCTS_DIR" ]; then
                      rm -rf "$PRODUCTS_DIR"
                      mkdir -p "$PRODUCTS_DIR"
                    fi
                  
                    if [ -n "$SKILLS_DIR" ] && [ -d "$SKILLS_DIR" ]; then
                      rm -rf "$SKILLS_DIR"
                      mkdir -p "$SKILLS_DIR"
                    fi
                fi

                echo "========================================"
                echo "Processing Well-Architected Framework"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type waf \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Cloud Adoption Framework"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type caf \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Azure Architecture Center"
                echo "========================================"
                python -m src.docs2skills \
                  --config-type aac \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR

                echo "========================================"
                echo "Processing Azure Other Products"
                echo "========================================"
                python -m src.docs2skills \
                  --products-dir $PRODUCTS_DIR \
                  --skills-dir $SKILLS_DIR


                echo "========================================"
                echo "Sync completed"
                echo "========================================"
            env:
              AZURE_OPENAI_ENDPOINT: $(azureOpenAIEndpoint)
              AZURE_OPENAI_CHAT_DEPLOYMENT: $(azureOpenAIDeployment)

          # 6. Push to GitHub repo
          - task: Bash@3
            displayName: 'Push to GitHub'
            inputs:
              targetType: inline
              script: |
                set -e
                GITHUB_REPO=$(basename $(Build.Repository.Uri) .git)
                cd $(Build.SourcesDirectory)/$GITHUB_REPO

                git config user.email "docs2skills-pipeline@microsoft.com"
                git config user.name "Docs2Skills Pipeline"

                CHANGED_FILES=$(git status --porcelain products/ skills/ | wc -l)

                if [ "$CHANGED_FILES" -gt 0 ]; then
                  echo "Changed files: $CHANGED_FILES"
                  git status --short products/ skills/

                  git add products/ skills/
                  TIMESTAMP=$(date +"%Y-%m-%d %H:%M")
                  git commit -m "$TIMESTAMP - Generate skills from docs (code: ADO/${{ parameters.docs2SkillsADOBranch }}, target: GitHub/${{ parameters.targetBranch }})"
                  
                  TARGET_BRANCH="${{ parameters.targetBranch }}"
                  # Use refs/heads/ to ensure branch is created if it doesn't exist
                  git push origin HEAD:refs/heads/$TARGET_BRANCH

                  echo "Pushed changes to GitHub branch: $TARGET_BRANCH"
                else
                  echo "No changes detected"
                fi