Merge pull request #127 from Azure/copilot/enable-pipeline-token-substitution

feat: add {#[TOKEN_NAME]#} substitution step to publish pipelines (APIOps Toolkit parity)

Author: petehauge

.devcontainer/devcontainer.json

@@ -11,7 +11,7 @@
   "remoteEnv": {
     "PATH": "${containerEnv:PATH}:${containerWorkspaceFolder}/node_modules/.bin"
   },  
-  "postCreateCommand": "npm ci --ignore-scripts",
+  "postCreateCommand": "npm ci --ignore-scripts && WELCOME_SOURCE=\"source \\\"$PWD/.devcontainer/welcome.sh\\\"\" && (grep -qxF \"$WELCOME_SOURCE\" ~/.bashrc || echo \"$WELCOME_SOURCE\" >> ~/.bashrc) && mkdir -p ~/.config/vscode-dev-containers && touch ~/.config/vscode-dev-containers/first-run-notice-already-displayed",
   "customizations": {
     "vscode": {
       "settings": {
@@ -22,7 +22,9 @@
         "eslint.format.enable": true,
         "eslint.validate": ["typescript", "typescriptreact"],
         "js/ts.tsdk.path": "node_modules/typescript/lib",
-        "js/ts.tsdk.promptToUseWorkspaceVersion": true
+        "js/ts.tsdk.promptToUseWorkspaceVersion": true,
+        "extensions.ignoreRecommendations": true,
+        "extensions.showRecommendationsOnInstall": false
       },
       "extensions": [
         "dbaeumer.vscode-eslint",

.devcontainer/welcome.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Only print the banner in interactive terminals.
+if [[ $- != *i* ]]; then
+  return
+fi
+
+WELCOME_MARKER_FILE="$HOME/.config/apiops-cli/welcome-shown"
+
+# Print only once per container/user lifecycle.
+if [[ -f "$WELCOME_MARKER_FILE" ]]; then
+  return
+fi
+
+mkdir -p "$HOME/.config/apiops-cli"
+touch "$WELCOME_MARKER_FILE"
+
+printf "👋 Welcome to apiops-cli!\n\n"
+printf "📚 To get started, run:\n"
+printf "  npm ci && npm run build\n\n"

docs/README.md

@@ -42,6 +42,7 @@ Requires Node.js 22 or later.
 - **Incremental publish** — Deploy only changed resources via git diff
 - **Dry-run mode** — Preview changes before applying them
 - **CI/CD scaffolding** — `apiops init` generates GitHub Actions or Azure DevOps pipelines
+- **Token substitution** — Replace `{#[TOKEN_NAME]#}` placeholders in config files with pipeline secrets before publish
 - **Multiple auth methods** — Azure CLI, managed identity, workload identity (OIDC), service principal
 
 ## Documentation Structure
@@ -64,6 +65,7 @@ docs/
 │   ├── multi-environment.md           — Dev / staging / prod promotion
 │   ├── multi-team-workflows.md        — Selective extraction, CODEOWNERS
 │   ├── code-first-workflow.md         — IDE → git → CI/CD → APIM
+│   ├── token-substitution.md          — Pipeline token/placeholder substitution
 │   └── migration-from-v1.md           — Migrate from Azure/apiops toolkit
 ├── ci-cd/
 │   ├── github-actions.md              — GitHub Actions integration

docs/ci-cd/authentication-patterns.md

@@ -146,7 +146,7 @@ variables:
   - group: apim-config  # Contains APIM_RESOURCE_GROUP, APIM_SERVICE_NAME, etc.
 
 steps:
-  - task: NodeTool@0
+  - task: UseNode@1
     inputs:
       versionSpec: '22.x'
 

docs/ci-cd/azure-devops.md

@@ -65,7 +65,7 @@ flowchart TD
     J --> L[Log: create a pull request]
 ```
 
-1. **Node.js setup** — Installs Node.js 22.x via `NodeTool@0`
+1. **Node.js setup** — Installs Node.js 22.x via `UseNode@1`
 2. **Install dependencies** — Runs `npm ci`
 3. **Run extract** — Executes `apiops extract` via `AzureCLI@2` task, authenticating through the service connection
 4. **Publish artifacts** — Uploads the artifact directory as a pipeline artifact named `apim-artifacts`
@@ -127,24 +127,26 @@ The pipeline runs automatically when changes to artifact files or configuration
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `COMMIT_ID_CHOICE` | string | `publish-artifacts-in-last-commit` | Choose `publish-artifacts-in-last-commit` for incremental publish, or `publish-all-artifacts-in-repo` for a full publish |
-| `ENVIRONMENT` | string | `all` | Which environment to publish to: `all`, `dev`, or `prod` |
+| `ENVIRONMENT` | string | `dev` | Which environment to publish to (for example `dev` or `prod`) |
 
 ### Multi-Stage Deployment
 
-The pipeline generates one stage per environment. Stages run sequentially — each stage depends on the previous:
+The pipeline generates one stage per environment. The selected stage runs based on the `ENVIRONMENT` parameter.
 
 ```mermaid
 flowchart LR
-    A[Publish_dev] --> B[Publish_prod]
+  A[ENVIRONMENT=dev] --> B[Publish_dev]
+  C[ENVIRONMENT=prod] --> D[Publish_prod]
 ```
 
 Each stage:
 
-1. **Conditionally runs** — Only executes if the `ENVIRONMENT` parameter matches the stage name or is `all`
+1. **Conditionally runs** — Only executes when the `ENVIRONMENT` parameter matches the stage name
 2. **Uses a deployment job** — Wraps the publish step in a `deployment` job targeting an [Azure DevOps environment](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/environments) for approval gates
 3. **Loads per-environment variables** — Each stage uses its own variable group (`apim-dev`, `apim-prod`)
 4. **Authenticates per-environment** — Uses environment-specific service connections (`AZURE_SERVICE_CONNECTION_DEV`, `AZURE_SERVICE_CONNECTION_PROD`)
-5. **Applies overrides** — Passes `--override configuration.{env}.yaml` to apply [environment-specific overrides](../guides/environment-overrides.md)
+5. **Substitutes tokens** — Replaces `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with secret variable values before publishing
+6. **Applies overrides** — Passes `--overrides configuration.{env}.yaml` to apply [environment-specific overrides](../guides/environment-overrides.md)
 
 ### Publish Pipeline Walkthrough
 
@@ -155,15 +157,15 @@ For incremental publish (default), `--commit-id $(Build.SourceVersion)` is passe
   displayName: 'Publish to dev (incremental - last commit only)'
   condition: ne('${{ parameters.COMMIT_ID_CHOICE }}', 'publish-all-artifacts-in-repo')
   inputs:
-    azureSubscription: '$(AZURE_SERVICE_CONNECTION_DEV)'
+    azureSubscription: 'AZURE_SERVICE_CONNECTION_DEV'
     scriptType: 'bash'
     scriptLocation: 'inlineScript'
     inlineScript: |
       npx apiops publish \
         --resource-group $(APIM_RESOURCE_GROUP_DEV) \
         --service-name $(APIM_SERVICE_NAME_DEV) \
         --source ./apim-artifacts \
-        --override configuration.dev.yaml \
+        --overrides configuration.dev.yaml \
         --commit-id $(Build.SourceVersion) \
         --subscription-id $(AZURE_SUBSCRIPTION_ID)
 ```
@@ -293,6 +295,35 @@ In your `package.json`, pin to a specific version:
 }
 ```
 
+### Using Token Substitution
+
+To replace `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with secret variable values:
+
+1. **Install the [Replace Tokens extension](https://marketplace.visualstudio.com/items?itemName=qetza.replacetokens)** in your Azure DevOps organization (if not already installed).
+
+  You can do this via CLI:
+  ```bash
+  az devops extension install --publisher-id qetza --extension-id replacetokens
+  ```
+
+2. **Add secret variables** to the `apim-<env>` variable group. See the Azure DevOps documentation for [adding variables to a variable group](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups) and [marking variables as secret](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-secret-variables).
+
+   For example, to substitute `{#[BACKEND_URL]#}` in your configuration file:
+
+   `configuration.prod.yaml`:
+   ```yaml
+   backends:
+     - name: my-backend
+       properties:
+         url: "{#[BACKEND_URL]#}"
+   ```
+
+   Add `BACKEND_URL` as a secret variable in the `apim-prod` variable group with the actual backend URL as the value.
+
+3. The substitution step runs automatically before publish.
+
+See the [Token Substitution Guide](../guides/token-substitution.md) for full details, including migration from APIOps Toolkit.
+
 ---
 
 ## Troubleshooting
@@ -301,20 +332,22 @@ In your `package.json`, pin to a specific version:
 |---------|-------|-----|
 | `AzureCLI@2` fails with "service connection not found" | Variable group not linked or variable name mismatch | Verify the variable group is linked to the pipeline and `AZURE_SERVICE_CONNECTION` is defined |
 | Extract shows "No changes to commit" | APIM config hasn't changed since last extract | Expected behavior — no branch is created |
-| Publish stage is skipped | `ENVIRONMENT` parameter doesn't match the stage | Set `ENVIRONMENT` to `all` or the specific stage name |
+| Publish stage is skipped | `ENVIRONMENT` parameter doesn't match the stage | Set `ENVIRONMENT` to the specific stage name (for example `dev` or `prod`) |
 | `npm ci` fails | `package.json` or `package-lock.json` missing | Run `apiops init` to generate project files, then commit them |
 | "publish-all-artifacts-in-repo" deploys everything | Expected — this mode publishes all artifacts, ignoring git diff | Use `publish-artifacts-in-last-commit` (default) for incremental |
 | Approval gate blocks deployment | Environment checks configured | Approve in **Pipelines → Environments → {env}** |
+| Run is stuck with "This pipeline needs permission to access a resource" | Environment resource isn't authorized for pipeline use | Authorize the environment in Azure DevOps or run the prompt step that PATCHes `pipelinePermissions/environment/{id}` with `{"allPipelines":{"authorized":true}}` |
 | `--subscription-id` error | `AZURE_SUBSCRIPTION_ID` not set in variable group | Add it to the relevant variable group |
 
 ---
 
-## Related
+## Further Reading
 
 - [GitHub Actions Integration](github-actions.md) — alternative CI/CD platform
 - [apiops init](../commands/init.md) — generates pipeline files
 - [apiops extract](../commands/extract.md) — extract command reference
 - [apiops publish](../commands/publish.md) — publish command reference
 - [Authentication Guide](../guides/authentication.md) — auth methods and RBAC
 - [Environment Overrides](../guides/environment-overrides.md) — per-environment configuration
+- [Token Substitution](../guides/token-substitution.md) — pipeline placeholder substitution with `{#[TOKEN_NAME]#}`
 - [Filtering Resources](../guides/filtering-resources.md) — extract specific APIs

docs/ci-cd/github-actions.md

@@ -100,7 +100,8 @@ The workflow runs automatically when changes are pushed to `main` in these paths
 1. **Resolves the commit ID** — captures `GITHUB_SHA` for incremental publish.
 2. **Checks out the repository** with `fetch-depth: 2` (needed for git diff).
 3. **Authenticates with Azure** using OIDC federated credentials.
-4. **Runs `apiops publish`** in one of two modes:
+4. **Substitutes tokens** — replaces `{#[TOKEN_NAME]#}` placeholders in `configuration.<env>.yaml` with pipeline secret values.
+5. **Runs `apiops publish`** in one of two modes:
    - **Incremental** (default): uses `--commit-id` to publish only changed files.
    - **Full**: publishes all artifacts in the repository (useful for recovery or initial setup).
 
@@ -212,6 +213,24 @@ Use GitHub environment protection rules for production deployments:
 
 The publish workflow will pause and wait for approval before deploying to prod.
 
+### Using Token Substitution
+
+To replace `{#[TOKEN_NAME]#}` placeholders in your configuration YAML with pipeline secrets, add the secret mappings to the `env:` block of the generated substitution step:
+
+```yaml
+- name: Substitute tokens in configuration.prod.yaml
+  uses: cschleiden/replace-tokens@v1.3
+  with:
+    tokenPrefix: '{#['
+    tokenSuffix: ']#}'
+    files: '["configuration.prod.yaml"]'
+  env:
+    MY_SECRET: ${{ secrets.MY_SECRET }}
+    BACKEND_URL: ${{ secrets.BACKEND_URL }}
+```
+
+See the [Token Substitution Guide](../guides/token-substitution.md) for full details, including migration from APIOps Toolkit.
+
 ### Adding Environment Overrides
 
 To use [environment-specific overrides](../guides/environment-overrides.md), add the `--overrides` flag to the publish step in the workflow:
@@ -271,4 +290,5 @@ For authentication issues, see the [Authentication Guide](../guides/authenticati
 
 - [Authentication Guide](../guides/authentication.md) — all auth methods and RBAC roles
 - [Environment Overrides](../guides/environment-overrides.md) — per-environment configuration
+- [Token Substitution](../guides/token-substitution.md) — pipeline placeholder substitution with `{#[TOKEN_NAME]#}`
 - [Scenarios and Workflows](../guides/scenarios-and-workflows.md) — portal-first vs. code-first patterns