Merge pull request #165 from Azure/remove-spec-format-option

Remove unimplemented --spec-format and --otel options

Author: petehauge

README.md

@@ -47,7 +47,6 @@ Extract APIM service configuration to local artifact files.
 | `--output <dir>` | `./apim-artifacts` | Output directory |
 | `--filter <path>` | | Extract only matching resources |
 | `--no-transitive` | | Skip transitive dependencies |
-| `--spec-format <format>` | | Override API spec format |
 
 ```bash
 apiops extract --help
@@ -58,12 +57,11 @@ apiops extract \
   --service-name <name> \
   --output ./apim-artifacts
 
-# Extract a filtered subset of resources and override the API spec format
+# Extract a filtered subset of resources
 apiops extract \
   --resource-group <rg> \
   --service-name <name> \
-  --filter ./filter.yaml \
-  --spec-format openapi-v3-yaml
+  --filter ./filter.yaml
 
 # Authenticate explicitly with a service principal
 apiops extract \

docs/commands/extract.md

@@ -66,7 +66,6 @@ apiops extract \
 | `--output <dir>` | string | `./apim-artifacts` | No | Output directory path |
 | `--filter <path>` | string | — | No | Filter configuration YAML file |
 | `--no-transitive` | boolean | false (transitive ON) | No | Disable transitive dependency inclusion |
-| `--spec-format <format>` | string | — | No | API spec format: `openapi-v2-json`, `openapi-v3-json`, `openapi-v3-yaml` |
 
 ### Global flags
 

specs/contracts/cli-commands.md

@@ -15,7 +15,6 @@ These flags are available on **all** commands (`extract`, `publish`, `init`).
 | `--api-version <version>` | string | no | `2024-05-01` | APIM REST API version |
 | `--format <mode>` | string | no | `text` | Output format: `text` (human-readable) or `json` (machine-readable) |
 | `--verbose` | boolean | no | `false` | Enable debug-level output |
-| `--otel <path>` | string | no | — | OpenTelemetry configuration YAML file |
 | `--client-id <id>` | string | no | — | Service principal client ID (sets `AZURE_CLIENT_ID` for DefaultAzureCredential) |
 | `--client-secret <secret>` | string | no | — | Service principal client secret (sets `AZURE_CLIENT_SECRET`) |
 | `--tenant-id <id>` | string | no | — | Azure AD tenant ID (sets `AZURE_TENANT_ID`) |
@@ -35,7 +34,6 @@ Extract APIM configuration to local artifact files.
 | `--output <dir>` | string | no | `./apim-artifacts` | Output directory path |
 | `--filter <path>` | string | no | — | Filter configuration YAML file |
 | `--no-transitive` | boolean | no | `false` | Disable transitive dependency inclusion |
-| `--spec-format <format>` | string | no | — | API specification format (`openapi-v2-json`, `openapi-v3-json`, `openapi-v3-yaml`); native format used when omitted or conversion not possible |
 
 **stdout**: Per-resource status lines (one per extracted resource)  
 **stderr**: Structured log messages (JSON when `--verbose`)  
@@ -109,8 +107,6 @@ Initialize repository structure and CI/CD pipeline configuration.
 | `AZURE_SUBSCRIPTION_ID` | Azure subscription (fallback for `--subscription-id`) | extract, publish |
 | `AZURE_API_VERSION` | Override APIM REST API version (default: `2024-05-01`) | extract, publish |
 | `COMMIT_ID` | Git SHA for incremental publish | publish |
-| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTel exporter endpoint (used by OTel SDK) | extract, publish |
-| `APPLICATIONINSIGHTS_CONNECTION_STRING` | Azure Monitor connection string | extract, publish |
 
 ---
 

specs/data-model.md

@@ -150,9 +150,7 @@ interface ExtractConfig {
   outputDir: string;
   filter?: FilterConfig;
   includeTransitive: boolean;   // Default: true; false when --no-transitive
-  specFormat?: string;          // From --spec-format; undefined = native format
   verbose: boolean;
-  otelConfigPath?: string;
 }
 ```
 
@@ -204,7 +202,6 @@ interface PublishConfig {
   deleteUnmatched: boolean;    // Default: false; true when --delete-unmatched
   commitId?: string;           // From COMMIT_ID env var; triggers incremental publish
   verbose: boolean;
-  otelConfigPath?: string;
 }
 ```
 

specs/research.md

@@ -242,7 +242,6 @@ program
   .option('--api-version <version>', 'APIM REST API version', '2024-05-01')
   .option('--format <mode>', 'Output format (text|json)', 'text')
   .option('--verbose', 'Enable verbose output')
-  .option('--otel <path>', 'OpenTelemetry configuration file')
   .option('--client-id <id>', 'Service principal client ID')
   .option('--client-secret <secret>', 'Service principal client secret')
   .option('--tenant-id <id>', 'Azure AD tenant ID')
@@ -254,7 +253,6 @@ program.command('extract')
   .option('--output <dir>', 'Output directory', './apim-artifacts')
   .option('--filter <path>', 'Filter configuration YAML file')
   .option('--no-transitive', 'Disable transitive dependency inclusion')
-  .option('--spec-format <format>', 'API specification format')
   .action(extractAction)
 
 program.command('publish')
@@ -295,25 +293,6 @@ program.command('init')
 
 ---
 
-## R9: OpenTelemetry Integration
-
-**Decision**: `@opentelemetry/sdk-node` with auto-instrumentation for HTTP; `--otel <path>` accepts standard OTel config YAML file.
-
-**Rationale**:
-- spec requires structured logging to stderr + optional OTel export (clarification C5)
-- Standard OTel env vars (`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`) for server config
-- Config file can set resource attributes, exporter endpoint, and custom headers
-- `--otel <path>` flag consistent with `--filter <path>` and `--overrides <path>` pattern
-
-**Implementation**:
-- When `--otel` not specified: No OTel SDK initialized, no overhead
-- When `--otel <path>` specified: Read YAML config, initialize `NodeSDK` with OTLP exporter
-- Traces: One span per resource type extraction/publish, child spans per individual resource
-- Metrics: Resource count, duration, error count
-- Azure Monitor: Users set `APPLICATIONINSIGHTS_CONNECTION_STRING` env var + use Azure Monitor OTel exporter in config
-
----
-
 ## R10: Testing Strategy
 
 **Decision**: Vitest with three test tiers — unit, integration, contract.

specs/spec.md

@@ -15,7 +15,6 @@
 - Q: What progress feedback should the tool provide during long-running operations? → A: Per-resource status lines by default (e.g., `✓ apis/orders-api`). A `--log-level debug` option adds additional diagnostic output (HTTP requests, timing, retry details) useful for debugging.
 - Q: How should the tool handle API revisions during extract and publish? → A: Extract all revisions, stored as sub-folders under each API. Publish recreates all revisions in the correct order (root API first, then revisions with forced revision numbers).
 - Q: How should the tool parallelize extraction? → A: Cross-type parallel where safe (independent resource types extracted concurrently; resources within each type also concurrent). Publish remains dependency-ordered.
-- Q: Should the CLI emit structured telemetry/logging? → A: Structured logging to stderr by default. An `--otel <path>` flag accepts a standard OpenTelemetry configuration YAML file to enable trace and metric export. Credentials (API keys, connection strings) live in the OTel config file or environment variables, never as CLI flags.
 
 ### User Story 1 — Extract APIM Configuration (Priority: P1)
 
@@ -35,8 +34,6 @@ A platform engineer wants to capture the current state of an Azure API Managemen
 
 3. **Given** the user only wants a subset of resources, **When** the user provides a filter configuration file via `--filter filter.yaml`, **Then** only the resources matching the allowlist in the configuration file are extracted, along with any resources those filtered resources transitively depend on (e.g., backends, named values, loggers referenced in policies). The user can suppress transitive inclusion with `--no-transitive`.
 
-4. **Given** an APIM instance with APIs using different specification formats (OpenAPI v2/v3, GraphQL, SOAP/WSDL), **When** the user runs extract with a `--spec-format` option, **Then** API specifications are exported in the requested format where format conversion is possible, and in their native format otherwise.
-
 5. **Given** an APIM instance, **When** the user runs `apiops extract --format json`, **Then** machine-readable JSON progress output is written to stdout (resource counts, file paths written) suitable for CI/CD pipeline consumption.
 
 7. **Given** an APIM instance with APIs that have multiple revisions (e.g., `orders-api;rev=1`, `orders-api;rev=2`), **When** the user runs `apiops extract`, **Then** all revisions are extracted as sub-folders under the API directory, preserving revision numbers and metadata.
@@ -163,7 +160,7 @@ The development team wants the CLI to support adding new top-level commands (e.g
 - **FR-012**: System MUST support configuring the APIM REST API version per invocation via a CLI flag or configuration file.
 - **FR-024**: System MUST extract all API revisions as sub-folders under each API directory, preserving revision numbers. During publish, the tool MUST create the root API first, then apply revisions in order with forced revision numbers to prevent APIM from auto-assigning.
 - **FR-025**: During extraction, the system MUST parallelize across independent resource types (e.g., backends and loggers concurrently) and across resources within each type. Dependencies between types (e.g., APIs depend on version sets) MUST be respected. During publish, resource types MUST be processed in dependency order.
-- **FR-026**: All commands MUST emit structured log output to stderr by default (timestamps, log levels, resource context). An `--otel <path>` flag MUST accept a standard OpenTelemetry configuration YAML file to enable trace and metric export. The tool MUST pass this file to the OTel SDK (equivalent to setting `OTEL_CONFIG_FILE`). If `--otel` is not provided, no telemetry is exported. Credentials (API keys, connection strings) MUST NOT be accepted as separate CLI flags — they MUST reside in the OTel config file or environment variables.
+- **FR-026**: All commands MUST emit structured log output to stderr by default (timestamps, log levels, resource context).
 - **FR-013**: System MUST provide `--format json|text` on all commands for machine-readable output suitable for CI/CD pipeline consumption. The default format is `text` (human-readable). `--format json` writes structured JSON to stdout. This flag is independent of `--output` (extract directory) and `--source` (publish directory).
 - **FR-014**: System MUST handle paginated Azure REST API responses by following `nextLink` continuation tokens.
 - **FR-015**: System MUST implement retry logic with exponential backoff for transient HTTP failures and 429 rate-limit responses.
@@ -211,4 +208,5 @@ The development team wants the CLI to support adding new top-level commands (e.g
 - The tool is distributed as an npm package and/or standalone binary; distribution mechanism details are deferred to implementation planning.
 - APIM API secrets (named value values marked as secrets, subscription keys) are NOT extracted in plaintext — they must be handled via override configuration or key vault references.
 - GraphQL and SOAP/WSDL APIs are supported for extraction in their native formats; format conversion between these and OpenAPI is out of scope.
+- For REST APIs, specifications are extracted in OpenAPI v3 YAML only. The APIOps Toolkit's `API_SPECIFICATION_FORMAT` / `--spec-format` option (which allowed selecting `OpenAPIV2Json`, `OpenAPIV2Yaml`, `OpenAPIV3Json`, or `OpenAPIV3Yaml`) is intentionally not carried forward. This is a known breaking change from the APIOps Toolkit: users who relied on OpenAPI v2 output or JSON output will need to migrate to OpenAPI v3 YAML. SC-006 (toolkit artifact compatibility) covers directory layout, not spec format variants.
 - Instance-specific resources (authorization servers, OpenID Connect providers, certificates, caches, identity providers, portal configuration, content types/items, notifications, users, tenant settings) are intentionally excluded — they do not belong in environment-promotion pipelines.

specs/tasks.md

@@ -42,11 +42,11 @@
 - [x] T012 Implement resource descriptor ↔ ARM URI mapping in src/lib/resource-uri.ts (builds full ARM URL from ApimServiceContext + ResourceDescriptor, including workspace prefix)
 - [x] T013 Implement resource descriptor ↔ artifact file path mapping in src/lib/resource-path.ts (maps descriptor to directory/file paths per data-model.md artifact conventions)
 - [x] T014 [P] Implement structured logger in src/lib/logger.ts (stderr output, timestamps, log levels, --verbose support per FR-023/FR-026)
-- [x] T015 [P] Implement YAML config loader in src/lib/config-loader.ts (parse filter YAML, override YAML, OTel config with js-yaml; validate against FilterConfig/OverrideConfig schemas)
+- [x] T015 [P] Implement YAML config loader in src/lib/config-loader.ts (parse filter YAML, override YAML with js-yaml; validate against FilterConfig/OverrideConfig schemas)
 - [x] T016 Implement Azure REST HTTP client in src/clients/apim-client.ts (implements IApimClient: DefaultAzureCredential auth, nextLink pagination, Retry-After/429 handling, exponential backoff, provisioningState polling per research.md R1)
 - [x] T017 Implement filesystem artifact store in src/clients/artifact-store.ts (implements IArtifactStore: read/write resource JSON, policy XML, API specs, association files, wiki markdown; UTF-8 encoding; directory creation per contracts/iartifact-store.md)
 - [x] T018 [P] Implement parallel execution runner in src/lib/parallel-runner.ts (p-limit based concurrency control, Promise.allSettled, configurable concurrency per research.md R8)
-- [x] T019 Set up Commander program entry point in src/cli/index.ts (program name, version, global options --verbose/--otel/--format/--subscription-id/--cloud, subcommand registration pattern per FR-018)
+- [x] T019 Set up Commander program entry point in src/cli/index.ts (program name, version, global options --verbose/--format/--subscription-id/--cloud, subcommand registration pattern per FR-018)
 
 **Checkpoint**: Foundation ready — user story implementation can now begin
 
@@ -68,7 +68,7 @@
 - [ ] T025 [US1] Implement transitive dependency resolver in src/services/transitive-resolver.ts (scan policies for named value refs \{\{name\}\}, backend refs set-backend-service, policy fragment refs include-fragment; scan apiInformation.json for apiVersionSetId; fixed-point expansion; --no-transitive bypass per research.md R4)
 - [ ] T026 [US1] Implement secret redaction in src/services/secret-redactor.ts (detect properties.secret === true on named values, replace properties.value with redaction marker per research.md R5)
 - [ ] T027 [US1] Implement workspace-scoped extraction in src/services/workspace-extractor.ts (list workspaces, extract workspace-scoped resources under workspaces/{name}/ using same resource-extractor with workspace context prefix per FR-010)
-- [ ] T028 [US1] Register extract command in src/cli/extract-command.ts (Commander subcommand with --resource-group, --service-name, --output, --filter, --no-transitive, --spec-format flags; wire to extract-service per contracts/cli-commands.md)
+- [ ] T028 [US1] Register extract command in src/cli/extract-command.ts (Commander subcommand with --resource-group, --service-name, --output, --filter, --no-transitive flags; wire to extract-service per contracts/cli-commands.md)
 - [ ] T029 [US1] Implement JSON output mode for extract in src/cli/extract-command.ts (--format json: machine-readable JSON to stdout with resource counts and file paths per FR-013)
 
 **Checkpoint**: `apiops extract` works end-to-end for all 33 resource types with filtering, parallelism, and secret redaction
@@ -145,7 +145,7 @@
 ### Implementation for User Story 5
 
 - [ ] T052 [US5] Implement command auto-discovery in src/cli/index.ts (scan src/cli/*-command.ts files or use explicit registration array; new commands appear in --help automatically per FR-018/SC-007)
-- [ ] T053 [US5] Extract shared command infrastructure in src/cli/shared.ts (common option builders for --resource-group/--service-name/--subscription-id/--verbose/--otel; shared APIM client factory; shared artifact store factory)
+- [ ] T053 [US5] Extract shared command infrastructure in src/cli/shared.ts (common option builders for --resource-group/--service-name/--subscription-id/--verbose; shared APIM client factory; shared artifact store factory)
 - [ ] T054 [US5] Create command developer guide in src/cli/README.md (document how to add a new command: file naming, interface shape, option reuse, testing pattern)
 
 **Checkpoint**: Adding a new command requires only creating one file in src/cli/
@@ -156,9 +156,7 @@
 
 **Purpose**: Improvements that affect multiple user stories
 
-- [ ] T055 [P] Implement OTel integration in src/lib/otel-setup.ts (load --otel config YAML, initialize NodeSDK with OTLP exporter, create spans per resource type/resource, metrics for counts/duration per research.md R9)
 - [ ] T056 [P] Add --api-version global flag support in src/cli/index.ts and src/clients/apim-client.ts (override default 2024-05-01 per FR-012)
-- [ ] T057 [P] Implement --spec-format option for extract in src/services/api-extractor.ts (request specific format from APIM export API where conversion supported; fallback to native with warning per FR-011)
 - [ ] T058 Run quickstart.md validation (execute each quickstart command against a test APIM instance, verify expected outputs)
 - [ ] T059 Add bin entry to package.json and shebang to src/cli/index.ts for global npm install (`npx apiops` / `npm install -g`)
 
@@ -248,4 +246,4 @@ T029: JSON output mode         → src/cli/extract-command.ts
 4. Add US3 (CI/CD) → Test in GitHub Actions pipeline
 5. Add US4 (Init) → Test scaffold generation in empty repo
 6. Add US5 (Extensibility) → Verify new command pattern
-7. Polish → OTel, spec-format, quickstart validation
+7. Polish → quickstart validation

src/cli/index.ts

@@ -32,7 +32,6 @@ program
       .choices(['debug', 'info', 'warn', 'error'])
       .default('info'),
   )
-  .option('--otel <path>', 'Path to OpenTelemetry config YAML')
   .option('--format <type>', 'Output format: text or json', 'text')
   .option('--subscription-id <id>', 'Azure subscription ID')
   .option('--cloud <name>', 'Sovereign cloud: public, china, usgov, germany', 'public')

src/lib/config-loader.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 /**
  * YAML config loader with validation
- * Parse filter YAML, override YAML, and OTel config files
+ * Parse filter YAML and override YAML config files
  */
 
 import * as fs from 'node:fs/promises';
@@ -428,23 +428,3 @@ function isPlainObject(value: unknown): value is Record<string, unknown> {
     Object.prototype.toString.call(value) === '[object Object]'
   );
 }
-
-/**
- * Load and parse an OpenTelemetry configuration YAML file.
- * Returns undefined if file doesn't exist.
- */
-export async function loadOTelConfig(filePath: string): Promise<Record<string, unknown> | undefined> {
-  try {
-    const content = await fs.readFile(filePath, 'utf-8');
-    const parsed = (yaml.load(content) ?? {}) as Record<string, unknown>;
-    
-    logger.debug(`Loaded OTel config from ${filePath}`);
-    return parsed;
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      logger.debug(`OTel config file not found: ${filePath}`);
-      return undefined;
-    }
-    throw new Error(`Failed to load OTel config from ${filePath}: ${(error as Error).message}`, { cause: error });
-  }
-}

src/models/config.ts

@@ -14,7 +14,6 @@ export interface ExtractConfig {
   filter?: FilterConfig;
   includeTransitive: boolean;
   logLevel: LogLevel;
-  otelConfigPath?: string;
 }
 
 /**
@@ -81,7 +80,6 @@ export interface PublishConfig {
   deleteUnmatched: boolean;
   commitId?: string;
   logLevel: LogLevel;
-  otelConfigPath?: string;
 }
 
 /**