Merge pull request #295 from contentstack/feat/DX-9336

feat: integrate asset count API for exact pagination totals (DX-9336)

Author: naman-contentstack

.talismanrc

@@ -1,6 +1,6 @@
 fileignoreconfig:
   - filename: pnpm-lock.yaml
-    checksum: f149ffa2864c1e5aa5b3f3363ea0d690e288168167d103775cb1f61be81151d5
+    checksum: 8eb82c59ad8b81af64587af55c6788d09a34d1b651d030d1ef01dce472716f2a
   - filename: packages/contentstack-branches/test/unit/helpers/stub-auth.ts
     checksum: 8cafd5994d3ec13ba9af74c80b330bfd14721ea4e0359b456598964a6c2913ce
   - filename: packages/contentstack-export/src/config/index.ts
@@ -18,7 +18,7 @@ fileignoreconfig:
   - filename: packages/contentstack-cli-cm-regex-validate/test/utils/connect-stack.test.ts
     checksum: 018980aa2b919967b9ef9ab1bdf635d4867fe21593fba5890afa443f440228ff
   - filename: packages/contentstack-bulk-operations/src/messages/index.ts
-    checksum: 07c2bf3f3130ad5e8b6a2971d76139e9b643c70a9ff5f7450adfb5c9bf3d7164
+    checksum: ba38daefd561d9acd018b6ebcb65847eceab6d375b28988d7b5624dcea9ccfe0
   - filename: packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts
     checksum: 99a3c3eb422a17f73f4c8f15088004fa4c95df3545bdf510310c1f3a20e4c2c2
   - filename: packages/contentstack-bulk-operations/src/base-bulk-command.ts
@@ -39,4 +39,6 @@ fileignoreconfig:
     checksum: 88c7e1dee308fa4ae25e7815ead0842b1aac7ed669b02859cc8b25cba879595d
   - filename: packages/contentstack-bulk-operations/test/unit/services/taxonomy-service.test.ts
     checksum: abc5ac707341760cf59d5b8b1c4e13cf2c79955e2735c33e2db3ec6bc48eddb6
+  - filename: packages/contentstack-import/src/import/modules/assets.ts
+    checksum: cda61a9c90bb39f27c09951b8a2623851296aefd0d3220d066032287a712d899
 version: '1.0'

packages/contentstack-asset-management/src/export/assets.ts

@@ -17,8 +17,11 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid'];
 
 type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string };
 
-/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */
-export type SpaceExportCounts = { assets: number; folders: number };
+/**
+ * Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities;
+ * failedAssets = metadata records missing after permanent page failures + binary download failures).
+ */
+export type SpaceExportCounts = { assets: number; folders: number; failedAssets: number };
 
 export default class ExportAssets extends CSAssetsExportAdapter {
   constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
@@ -53,11 +56,22 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     };
 
     log.debug(`Fetching folders and streaming assets for space ${workspace.space_uid}`, this.exportContext.context);
-    const [folders] = await Promise.all([
+    const [folders, streamResult] = await Promise.all([
       this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency),
       this.streamWorkspaceAssets(workspace.space_uid, workspace.uid, onPage, this.apiPageSize, this.apiFetchConcurrency),
     ]);
 
+    // Permanently failed metadata pages (or mid-export drift) — these records never reached
+    // assets.json, so the download loop can't see them. Count them as failures in the summary;
+    // they are recoverable via a re-export or a targeted query-export, so don't abort the run.
+    if (streamResult.missing > 0) {
+      log.error(
+        `${streamResult.missing} asset metadata record(s) could not be fetched for space ${workspace.space_uid} — ` +
+          'they are missing from this export. Re-export (or use query-export) to recover them.',
+        this.exportContext.context,
+      );
+    }
+
     if (fsWriter) fsWriter.completeFile(true);
     else await this.writeEmptyChunkedJson(assetsDir, 'assets.json');
     log.debug(`Wrote chunked assets metadata (${totalStreamed} item(s)) under ${assetsDir}`, this.exportContext.context);
@@ -78,17 +92,21 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null);
 
     log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
-    const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
+    const downloads = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
 
     const folderCount = getArrayFromResponse(folders, 'folders').length;
-    return { assets: assetsDownloaded, folders: folderCount };
+    return { assets: downloads.ok, folders: folderCount, failedAssets: streamResult.missing + downloads.fail };
   }
 
   /**
    * Download asset binaries by reading the just-written chunked `assets.json` back from disk
    * (one chunk at a time), so we never re-materialize the whole asset list in memory.
    */
-  private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<number> {
+  private async downloadWorkspaceAssets(
+    assetsDir: string,
+    spaceUid: string,
+    expectedDownloads: number,
+  ): Promise<{ ok: number; fail: number }> {
     const filesDir = pResolve(assetsDir, 'files');
     await mkdir(filesDir, { recursive: true });
 
@@ -202,7 +220,7 @@ export default class ExportAssets extends CSAssetsExportAdapter {
       this.exportContext.context,
     );
 
-    return downloadOk;
+    return { ok: downloadOk, fail: downloadFail };
   }
 
 }

packages/contentstack-asset-management/src/export/spaces.ts

@@ -18,6 +18,8 @@ export type AssetExportCounts = {
   folders: number;
   assetTypes: number;
   fields: number;
+  /** Assets missing from the export: permanently failed metadata pages + failed binary downloads. */
+  failedAssets: number;
 };
 
 /**
@@ -53,7 +55,7 @@ export class ExportSpaces {
 
     if (!linkedWorkspaces.length) {
       log.debug('No linked workspaces to export', context);
-      return { assets: 0, folders: 0, assetTypes: 0, fields: 0 };
+      return { assets: 0, folders: 0, assetTypes: 0, fields: 0, failedAssets: 0 };
     }
 
     log.debug('Starting Contentstack Assets export process...', context);
@@ -105,6 +107,7 @@ export class ExportSpaces {
     // Real entity counts accumulated for the summary (Bug 3).
     let assetsTotal = 0;
     let foldersTotal = 0;
+    let failedAssetsTotal = 0;
     let assetTypesCount = 0;
     let fieldsCount = 0;
     try {
@@ -140,6 +143,7 @@ export class ExportSpaces {
           const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
           assetsTotal += spaceCounts.assets;
           foldersTotal += spaceCounts.folders;
+          failedAssetsTotal += spaceCounts.failedAssets;
           progress.completeProcess(spaceProcess, true);
           log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
         } catch (err) {
@@ -157,14 +161,20 @@ export class ExportSpaces {
       }
 
       log.info(
-        anySpaceFailed
+        anySpaceFailed || failedAssetsTotal > 0
           ? 'Contentstack Assets export completed with errors in one or more spaces'
           : 'Contentstack Assets export completed successfully',
         context,
       );
       log.debug('Contentstack Assets export completed', context);
 
-      return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount };
+      return {
+        assets: assetsTotal,
+        folders: foldersTotal,
+        assetTypes: assetTypesCount,
+        fields: fieldsCount,
+        failedAssets: failedAssetsTotal,
+      };
     } catch (err) {
       if (!bootstrapFailed) {
         // Mark any spaces that hadn't been processed as failed so the multibar

packages/contentstack-asset-management/src/types/cs-assets-api.ts

@@ -159,6 +159,21 @@ export type SearchAssetsParams = {
   limit?: number;
 };
 
+/** Response shape of GET /api/bff/spaces/{space_uid}/assets/count. */
+export type AssetCountResponse = {
+  count: number;
+};
+
+/**
+ * Result of streaming a workspace's assets. `missing` is the gap between the authoritative
+ * count-API total and the records actually streamed (permanently failed pages and/or items
+ * changed mid-export) — surfaced as failures in the export summary rather than aborting the run.
+ */
+export type StreamWorkspaceAssetsResult = {
+  streamed: number;
+  missing: number;
+};
+
 /** Response shape from POST /api/search for assets. */
 export type SearchAssetsResponse = {
   count?: number;
@@ -174,14 +189,15 @@ export interface ICSAssetsAdapter {
   listSpaces(pageSize?: number, fetchConcurrency?: number): Promise<SpacesListResponse>;
   getSpace(spaceUid: string): Promise<SpaceResponse>;
   getWorkspaceFields(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise<FieldsResponse>;
-  getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise<unknown>;
+  getAssetsCount(spaceUid: string, workspaceUid?: string): Promise<number>;
+  getFoldersCount(spaceUid: string, workspaceUid?: string): Promise<number>;
   streamWorkspaceAssets(
     spaceUid: string,
     workspaceUid: string | undefined,
     onPage: (items: unknown[]) => void | Promise<void>,
     pageSize?: number,
     fetchConcurrency?: number,
-  ): Promise<number>;
+  ): Promise<StreamWorkspaceAssetsResult>;
   getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise<unknown>;
   getWorkspaceAssetTypes(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise<AssetTypesResponse>;
   searchAssets(params: SearchAssetsParams): Promise<SearchAssetsResponse>;