Merge branch 'v2-dev' into fix/DX-8736

Author: cs-raj

.talismanrc

@@ -1,6 +1,6 @@
 fileignoreconfig:
   - filename: pnpm-lock.yaml
-    checksum: f149ffa2864c1e5aa5b3f3363ea0d690e288168167d103775cb1f61be81151d5
+    checksum: 8eb82c59ad8b81af64587af55c6788d09a34d1b651d030d1ef01dce472716f2a
   - filename: packages/contentstack-branches/test/unit/helpers/stub-auth.ts
     checksum: 8cafd5994d3ec13ba9af74c80b330bfd14721ea4e0359b456598964a6c2913ce
   - filename: packages/contentstack-export/src/config/index.ts
@@ -18,7 +18,7 @@ fileignoreconfig:
   - filename: packages/contentstack-cli-cm-regex-validate/test/utils/connect-stack.test.ts
     checksum: 018980aa2b919967b9ef9ab1bdf635d4867fe21593fba5890afa443f440228ff
   - filename: packages/contentstack-bulk-operations/src/messages/index.ts
-    checksum: 07c2bf3f3130ad5e8b6a2971d76139e9b643c70a9ff5f7450adfb5c9bf3d7164
+    checksum: c8acefdd18f499a82d839e3cf99e34af9dd464f63f92282739411174ae7b7c37
   - filename: packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts
     checksum: 99a3c3eb422a17f73f4c8f15088004fa4c95df3545bdf510310c1f3a20e4c2c2
   - filename: packages/contentstack-bulk-operations/src/base-bulk-command.ts
@@ -39,4 +39,12 @@ fileignoreconfig:
     checksum: 88c7e1dee308fa4ae25e7815ead0842b1aac7ed669b02859cc8b25cba879595d
   - filename: packages/contentstack-bulk-operations/test/unit/services/taxonomy-service.test.ts
     checksum: abc5ac707341760cf59d5b8b1c4e13cf2c79955e2735c33e2db3ec6bc48eddb6
+  - filename: packages/contentstack-import/src/import/modules/assets.ts
+    checksum: cda61a9c90bb39f27c09951b8a2623851296aefd0d3220d066032287a712d899
+  - filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts
+    checksum: 63c6bff4d51842d8fa3cce88545259d0a2c3cfe71df95d303d993f692cee883b
+  - filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts
+    checksum: bc4a53f96be6a10786e00133245c7bdc43c965c8a98b753e3879e1110cf9c601
+  - filename: packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-cs-assets.test.ts
+    checksum: 1b6b5b0302d3efd9508e2e5d4fe3783da97a110735ca3106ce829cfe93874150
 version: '1.0'

packages/contentstack-asset-management/src/constants/index.ts

@@ -6,6 +6,8 @@ export const FALLBACK_AM_API_CONCURRENCY = 5;
 export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY;
 export const FALLBACK_AM_API_PAGE_SIZE = 100;
 export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5;
+/** Max assets/uids the CS Assets bulk delete/move endpoints accept per request. */
+export const CS_ASSETS_BULK_MUTATE_MAX_ITEMS = 100;
 
 /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */
 export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [

packages/contentstack-asset-management/src/export/assets.ts

@@ -2,12 +2,13 @@ import { resolve as pResolve } from 'node:path';
 import { Readable } from 'node:stream';
 import { mkdir, writeFile } from 'node:fs/promises';
 import chunk from 'lodash/chunk';
-import { configHandler, log, FsUtility } from '@contentstack/cli-utilities';
+import { log, FsUtility } from '@contentstack/cli-utilities';
 
 import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
 import type { ExportContext } from '../types/export-types';
 import { CSAssetsExportAdapter } from './base';
-import { writeStreamToFile, getArrayFromResponse } from '../utils/export-helpers';
+import { writeStreamToFile, getArrayFromResponse, getSecuredAssetAuth, SecuredAssetAuthError } from '../utils/export-helpers';
+import type { SecuredAssetAuth } from '../utils/export-helpers';
 import { forEachChunkedJsonStore } from '../utils/chunked-json-reader';
 import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry';
 import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter';
@@ -17,8 +18,11 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid'];
 
 type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string };
 
-/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */
-export type SpaceExportCounts = { assets: number; folders: number };
+/**
+ * Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities;
+ * failedAssets = metadata records missing after permanent page failures + binary download failures).
+ */
+export type SpaceExportCounts = { assets: number; folders: number; failedAssets: number };
 
 export default class ExportAssets extends CSAssetsExportAdapter {
   constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
@@ -53,11 +57,22 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     };
 
     log.debug(`Fetching folders and streaming assets for space ${workspace.space_uid}`, this.exportContext.context);
-    const [folders] = await Promise.all([
+    const [folders, streamResult] = await Promise.all([
       this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency),
       this.streamWorkspaceAssets(workspace.space_uid, workspace.uid, onPage, this.apiPageSize, this.apiFetchConcurrency),
     ]);
 
+    // Permanently failed metadata pages (or mid-export drift) — these records never reached
+    // assets.json, so the download loop can't see them. Count them as failures in the summary;
+    // they are recoverable via a re-export or a targeted query-export, so don't abort the run.
+    if (streamResult.missing > 0) {
+      log.error(
+        `${streamResult.missing} asset metadata record(s) could not be fetched for space ${workspace.space_uid} — ` +
+          'they are missing from this export. Re-export (or use query-export) to recover them.',
+        this.exportContext.context,
+      );
+    }
+
     if (fsWriter) fsWriter.completeFile(true);
     else await this.writeEmptyChunkedJson(assetsDir, 'assets.json');
     log.debug(`Wrote chunked assets metadata (${totalStreamed} item(s)) under ${assetsDir}`, this.exportContext.context);
@@ -78,22 +93,25 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null);
 
     log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
-    const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
+    const downloads = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
 
     const folderCount = getArrayFromResponse(folders, 'folders').length;
-    return { assets: assetsDownloaded, folders: folderCount };
+    return { assets: downloads.ok, folders: folderCount, failedAssets: streamResult.missing + downloads.fail };
   }
 
   /**
    * Download asset binaries by reading the just-written chunked `assets.json` back from disk
    * (one chunk at a time), so we never re-materialize the whole asset list in memory.
    */
-  private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<number> {
+  private async downloadWorkspaceAssets(
+    assetsDir: string,
+    spaceUid: string,
+    expectedDownloads: number,
+  ): Promise<{ ok: number; fail: number }> {
     const filesDir = pResolve(assetsDir, 'files');
     await mkdir(filesDir, { recursive: true });
 
     const securedAssets = this.exportContext.securedAssets ?? false;
-    const authtoken = securedAssets ? configHandler.get('authtoken') : null;
     log.debug(
       `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}, expected=${expectedDownloads}`,
       this.exportContext.context,
@@ -102,6 +120,9 @@ export default class ExportAssets extends CSAssetsExportAdapter {
 
     let downloadOk = 0;
     let downloadFail = 0;
+    // Set when a 401 persists after a forced token refresh — from then on, skip the network
+    // entirely and abort the phase, instead of individually failing every remaining asset.
+    let authFailure: SecuredAssetAuthError | null = null;
 
     await forEachChunkedJsonStore<AssetRecord>(
       assetsDir,
@@ -136,25 +157,60 @@ export default class ExportAssets extends CSAssetsExportAdapter {
       async (records) => {
         const valid = records.filter((asset) => this.isDownloadable(asset));
         if (valid.length === 0) return;
+        const chunkAuthFailure = authFailure;
+        if (chunkAuthFailure) {
+          // Auth already failed hard — count the remaining downloadables without hitting the network.
+          for (const rec of valid) {
+            downloadFail += 1;
+            this.tick(false, `asset: ${rec.filename ?? rec.file_name ?? rec.uid ?? 'asset'}`, chunkAuthFailure.message);
+          }
+          return;
+        }
+        // Resolve per chunk so long download phases pick up proactively refreshed OAuth tokens.
+        let auth: SecuredAssetAuth = securedAssets ? await getSecuredAssetAuth() : {};
         const apiBatches = chunk(valid, this.downloadAssetsBatchConcurrency);
         const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => {
           const asset = apiBatches[batchIndex][index] as AssetRecord;
           const uid = (asset.uid ?? asset._uid) as string;
           const url = asset.url as string;
           const filename = asset.filename ?? asset.file_name ?? 'asset';
           if (!url || !uid) return;
+          const knownAuthFailure = authFailure as SecuredAssetAuthError | null;
+          if (knownAuthFailure) {
+            downloadFail += 1;
+            this.tick(false, `asset: ${filename}`, knownAuthFailure.message);
+            return;
+          }
           try {
             const separator = url.includes('?') ? '&' : '?';
-            const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url;
+            const doFetch = () =>
+              fetch(
+                securedAssets && auth.authtoken ? `${url}${separator}authtoken=${auth.authtoken}` : url,
+                securedAssets && auth.headers ? { headers: auth.headers } : undefined,
+              );
             // Binary GET is idempotent — retry transient failures with backoff.
             const response = await withRetry(
               async () => {
                 let resp: Response;
                 try {
-                  resp = await fetch(downloadUrl);
+                  resp = await doFetch();
                 } catch (e) {
                   throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
                 }
+                if (securedAssets && resp.status === 401) {
+                  // Token expired or was revoked mid-run — force one refresh (deduped upstream)
+                  // and refetch. A second 401 means auth is unrecoverable: abort the phase.
+                  auth = await getSecuredAssetAuth(true);
+                  try {
+                    resp = await doFetch();
+                  } catch (e) {
+                    throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
+                  }
+                  if (resp.status === 401) {
+                    authFailure = new SecuredAssetAuthError(resp.status);
+                    throw authFailure;
+                  }
+                }
                 if (!resp.ok) {
                   if (isRetryableStatus(resp.status)) {
                     throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after')));
@@ -191,6 +247,16 @@ export default class ExportAssets extends CSAssetsExportAdapter {
       },
     );
 
+    const terminalAuthFailure = authFailure as SecuredAssetAuthError | null;
+    if (terminalAuthFailure) {
+      // Fail the space loudly — a "completed with errors" summary would bury the real cause.
+      log.error(
+        `Aborted asset downloads for space ${spaceUid}: ${terminalAuthFailure.message}`,
+        this.exportContext.context,
+      );
+      throw terminalAuthFailure;
+    }
+
     log.info(
       downloadFail === 0
         ? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
@@ -202,7 +268,7 @@ export default class ExportAssets extends CSAssetsExportAdapter {
       this.exportContext.context,
     );
 
-    return downloadOk;
+    return { ok: downloadOk, fail: downloadFail };
   }
 
 }

packages/contentstack-asset-management/src/export/spaces.ts

@@ -18,6 +18,8 @@ export type AssetExportCounts = {
   folders: number;
   assetTypes: number;
   fields: number;
+  /** Assets missing from the export: permanently failed metadata pages + failed binary downloads. */
+  failedAssets: number;
 };
 
 /**
@@ -53,7 +55,7 @@ export class ExportSpaces {
 
     if (!linkedWorkspaces.length) {
       log.debug('No linked workspaces to export', context);
-      return { assets: 0, folders: 0, assetTypes: 0, fields: 0 };
+      return { assets: 0, folders: 0, assetTypes: 0, fields: 0, failedAssets: 0 };
     }
 
     log.debug('Starting Contentstack Assets export process...', context);
@@ -105,6 +107,7 @@ export class ExportSpaces {
     // Real entity counts accumulated for the summary (Bug 3).
     let assetsTotal = 0;
     let foldersTotal = 0;
+    let failedAssetsTotal = 0;
     let assetTypesCount = 0;
     let fieldsCount = 0;
     try {
@@ -140,6 +143,7 @@ export class ExportSpaces {
           const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
           assetsTotal += spaceCounts.assets;
           foldersTotal += spaceCounts.folders;
+          failedAssetsTotal += spaceCounts.failedAssets;
           progress.completeProcess(spaceProcess, true);
           log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
         } catch (err) {
@@ -157,14 +161,20 @@ export class ExportSpaces {
       }
 
       log.info(
-        anySpaceFailed
+        anySpaceFailed || failedAssetsTotal > 0
           ? 'Contentstack Assets export completed with errors in one or more spaces'
           : 'Contentstack Assets export completed successfully',
         context,
       );
       log.debug('Contentstack Assets export completed', context);
 
-      return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount };
+      return {
+        assets: assetsTotal,
+        folders: foldersTotal,
+        assetTypes: assetTypesCount,
+        fields: fieldsCount,
+        failedAssets: failedAssetsTotal,
+      };
     } catch (err) {
       if (!bootstrapFailed) {
         // Mark any spaces that hadn't been processed as failed so the multibar

packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts

@@ -1,15 +1,16 @@
 import { resolve as pResolve } from 'node:path';
 import { mkdir, writeFile } from 'node:fs/promises';
 import { Readable } from 'node:stream';
-import { log, handleAndLogError, configHandler } from '@contentstack/cli-utilities';
+import { log, handleAndLogError } from '@contentstack/cli-utilities';
 
 import type { CsAssetsQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
 import type { ExportContext } from '../types/export-types';
 import ExportAssetTypes from '../export/asset-types';
 import ExportFields from '../export/fields';
 import { CSAssetsExportAdapter } from '../export/base';
 import chunk from 'lodash/chunk';
-import { getAssetItems, writeStreamToFile } from '../utils/export-helpers';
+import { getAssetItems, writeStreamToFile, getSecuredAssetAuth, SecuredAssetAuthError } from '../utils/export-helpers';
+import type { SecuredAssetAuth } from '../utils/export-helpers';
 import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry';
 import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter';
 
@@ -223,26 +224,64 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter {
     await mkdir(filesDir, { recursive: true });
 
     const securedAssets = this.exportContext.securedAssets ?? false;
-    const authtoken = securedAssets ? configHandler.get('authtoken') : null;
+    // Set when a 401 persists after a forced token refresh — from then on, skip the network
+    // entirely and abort the phase, instead of individually failing every remaining asset.
+    let authFailure: SecuredAssetAuthError | null = null;
+    // OAuth → Authorization: Bearer header; basic auth → ?authtoken= query param.
+    // Resolved lazily once per sequential batch (handlers within a batch share one resolve), so
+    // long download runs keep picking up proactively refreshed OAuth tokens.
+    let auth: SecuredAssetAuth = {};
+    let authBatchIndex = -1;
+    let authResolve: Promise<void> = Promise.resolve();
+    const ensureAuthForBatch = (batchIndex: number): Promise<void> => {
+      if (!securedAssets) return Promise.resolve();
+      if (batchIndex !== authBatchIndex) {
+        authBatchIndex = batchIndex;
+        authResolve = getSecuredAssetAuth().then((resolved) => {
+          auth = resolved;
+        });
+      }
+      return authResolve;
+    };
 
     const apiBatches = chunk(downloadable, this.downloadAssetsBatchConcurrency);
     const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => {
       const asset = apiBatches[batchIndex][index];
       const uid = String(asset.uid ?? asset._uid);
       const url = String(asset.url);
       const filename = String(asset.filename ?? asset.file_name ?? 'asset');
+      if (authFailure) return; // auth failed hard — don't hit the network for remaining assets
       try {
+        await ensureAuthForBatch(batchIndex);
         const separator = url.includes('?') ? '&' : '?';
-        const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url;
+        const doFetch = () =>
+          fetch(
+            securedAssets && auth.authtoken ? `${url}${separator}authtoken=${auth.authtoken}` : url,
+            securedAssets && auth.headers ? { headers: auth.headers } : undefined,
+          );
         // Binary GET is idempotent — retry transient failures with backoff.
         const response = await withRetry(
           async () => {
             let resp: Response;
             try {
-              resp = await fetch(downloadUrl);
+              resp = await doFetch();
             } catch (e) {
               throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
             }
+            if (securedAssets && resp.status === 401) {
+              // Token expired or was revoked mid-run — force one refresh (deduped upstream)
+              // and refetch. A second 401 means auth is unrecoverable: abort the phase.
+              auth = await getSecuredAssetAuth(true);
+              try {
+                resp = await doFetch();
+              } catch (e) {
+                throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
+              }
+              if (resp.status === 401) {
+                authFailure = new SecuredAssetAuthError(resp.status);
+                throw authFailure;
+              }
+            }
             if (!resp.ok) {
               if (isRetryableStatus(resp.status)) {
                 throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after')));
@@ -265,5 +304,15 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter {
     };
 
     await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler);
+
+    const terminalAuthFailure = authFailure as SecuredAssetAuthError | null;
+    if (terminalAuthFailure) {
+      // Fail the space loudly — silently skipping the rest would look like a successful export.
+      log.error(
+        `Aborted asset downloads for space ${spaceUid}: ${terminalAuthFailure.message}`,
+        this.exportContext.context,
+      );
+      throw terminalAuthFailure;
+    }
   }
 }