Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ fileignoreconfig:
- filename: packages/contentstack-query-export/.env-example
checksum: 922c7aa9c788ab60b987de2b0a2aee6d90843c463a8bbc29201e4efe31081187
- filename: pnpm-lock.yaml
checksum: bb5303f2fe64f90ae95d2738363267fb0bfcfeb71f025c2110d4cec87ff84d95
checksum: 3d2eaabf1df366efee1759156465c6aefa68f30d372717de2cdc3e41946aa3d8
- filename: packages/contentstack-import/src/utils/build-import-spaces-options.ts
checksum: fe0cb6cb5903515982af1e3642f2a19233207d35f13dc205cebeda0aa399f8b5
- filename: packages/contentstack-export/src/export/modules/stack.ts
Expand Down
49 changes: 0 additions & 49 deletions packages/contentstack-asset-management/README.md

This file was deleted.

44 changes: 43 additions & 1 deletion packages/contentstack-asset-management/src/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,17 @@ export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB;
export const AM_MAIN_PROCESS_NAME = 'Asset Management 2.0';

/**
* Process names for Asset Management 2.0 export progress (for tick labels).
* Process names for Asset Management 2.0 export/import progress.
*
* In the new per-space layout each entry below corresponds to a single row in
* the multibar:
* - {@link AM_FIELDS} / {@link AM_ASSET_TYPES} are the shared bootstrap rows
* (one execution per org, ahead of per-space work).
* - {@link AM_IMPORT_FIELDS} / {@link AM_IMPORT_ASSET_TYPES} are the import
* equivalents.
* - One additional row per space is added dynamically via
* {@link getSpaceProcessName} and ticks include folders + metadata + asset
* transfer for that space.
*/
export const PROCESS_NAMES = {
AM_SPACE_METADATA: 'Space metadata',
Expand All @@ -51,6 +61,38 @@ export const PROCESS_NAMES = {
AM_IMPORT_ASSETS: 'Import assets',
} as const;

/**
* Maximum visual length of a per-space process row label. The CLIProgressManager
* truncates anything over 20 characters; reserve 6 chars for the `Space ` prefix
* so the trailing space uid keeps 14 chars before truncation.
*/
const SPACE_PROCESS_NAME_PREFIX = 'Space ';
const SPACE_PROCESS_NAME_MAX_UID_LEN = 14;

/**
* Returns the multibar row label for a single AM 2.0 space.
* The label is bounded so CLIProgressManager.formatProcessName doesn't truncate
* it mid-string; the full uid is still used for tick item labels and structured
* logs, only the row label itself is shortened for display.
*/
export function getSpaceProcessName(spaceUid: string): string {
const safeUid = spaceUid ?? '';
const trimmed =
safeUid.length > SPACE_PROCESS_NAME_MAX_UID_LEN
? safeUid.substring(0, SPACE_PROCESS_NAME_MAX_UID_LEN)
: safeUid;
return `${SPACE_PROCESS_NAME_PREFIX}${trimmed}`;
}

/**
* Detects whether a process name belongs to a per-space progress row, used by
* the export/import strategy registries to aggregate counts for the final
* summary across all spaces.
*/
export function isSpaceProcessName(processName: string): boolean {
return typeof processName === 'string' && processName.startsWith(SPACE_PROCESS_NAME_PREFIX);
}

/**
* Status messages for each process (exporting, fetching, importing, failed).
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportAssetTypes extends AssetManagementExportAdapter {
protected processName: string = PROCESS_NAMES.AM_ASSET_TYPES;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}
Expand All @@ -24,7 +26,13 @@ export default class ExportAssetTypes extends AssetManagementExportAdapter {
} else {
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items);
this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null);
await this.writeItemsToChunkedJson(
dir,
'asset-types.json',
'asset_types',
['uid', 'title', 'category', 'file_extension'],
items,
);
this.tick(true, `asset_types (${items.length})`, null);
}
}
23 changes: 14 additions & 9 deletions packages/contentstack-asset-management/src/export/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,17 @@ export default class ExportAssets extends AssetManagementExportAdapter {
this.getWorkspaceAssets(workspace.space_uid, workspace.uid),
]);

const assetItems = getAssetItems(assetsData);
const downloadableCount = assetItems.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))).length;
// Per-space total: 1 folder write + 1 metadata write + N per-asset downloads.
// The shared module-level total is just a placeholder before this point; update
// it now so the multibar row shows real progress as downloads tick in.
this.progressOrParent?.updateProcessTotal?.(this.processName, 2 + downloadableCount);

await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2));
this.tick(true, `folders: ${workspace.space_uid}`, null);
log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context);

const assetItems = getAssetItems(assetsData);
log.debug(
assetItems.length === 0
? `No assets for space ${workspace.space_uid}, wrote empty assets.json`
Expand All @@ -60,7 +66,7 @@ export default class ExportAssets extends AssetManagementExportAdapter {
: `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`,
this.exportContext.context,
);
this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null);
this.tick(true, `metadata: ${workspace.space_uid} (${assetItems.length})`, null);

log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid);
Expand All @@ -87,8 +93,6 @@ export default class ExportAssets extends AssetManagementExportAdapter {
`Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`,
this.exportContext.context,
);
let lastError: string | null = null;
let allSuccess = true;
let downloadOk = 0;
let downloadFail = 0;

Expand Down Expand Up @@ -118,24 +122,25 @@ export default class ExportAssets extends AssetManagementExportAdapter {
const filePath = pResolve(assetFolderPath, filename);
await writeStreamToFile(nodeStream, filePath);
downloadOk += 1;
// Per-asset tick so the per-space progress bar moves in real time.
this.tick(true, `asset: ${filename}`, null);
log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context);
} catch (e) {
allSuccess = false;
downloadFail += 1;
lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
this.tick(false, `asset: ${filename}`, err);
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
}
});

this.tick(allSuccess, `downloads: ${spaceUid}`, lastError);
log.info(
allSuccess
downloadFail === 0
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
this.exportContext.context,
);
log.debug(
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, allSuccess=${allSuccess}`,
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
this.exportContext.context,
);
}
Expand Down
11 changes: 10 additions & 1 deletion packages/contentstack-asset-management/src/export/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
protected readonly exportContext: ExportContext;
protected progressManager: CLIProgressManager | null = null;
protected parentProgressManager: CLIProgressManager | null = null;
protected readonly processName: string = AM_MAIN_PROCESS_NAME;
protected processName: string = AM_MAIN_PROCESS_NAME;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig);
Expand All @@ -30,6 +30,15 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
this.parentProgressManager = parent;
}

/**
* Override the default progress process name for {@link tick}/{@link updateStatus}
* calls. Used by the per-space orchestrator so each module's ticks land on the
* row for the space currently being exported.
*/
public setProcessName(name: string): void {
this.processName = name;
}

protected get progressOrParent(): CLIProgressManager | null {
return this.parentProgressManager ?? this.progressManager;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportFields extends AssetManagementExportAdapter {
protected processName: string = PROCESS_NAMES.AM_FIELDS;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}
Expand All @@ -25,6 +27,6 @@ export default class ExportFields extends AssetManagementExportAdapter {
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
this.tick(true, PROCESS_NAMES.AM_FIELDS, null);
this.tick(true, `fields (${items.length})`, null);
}
}
71 changes: 52 additions & 19 deletions packages/contentstack-asset-management/src/export/spaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import { log, CLIProgressManager, configHandler, handleAndLogError } from '@cont

import type { AssetManagementExportOptions, AssetManagementAPIConfig } from '../types/asset-management-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
import { AM_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index';
import ExportAssetTypes from './asset-types';
import ExportFields from './fields';
import ExportWorkspace from './workspaces';
Expand Down Expand Up @@ -55,12 +54,18 @@ export class ExportSpaces {
await mkdir(spacesRootPath, { recursive: true });
log.debug(`Spaces root path: ${spacesRootPath}`, context);

const totalSteps = 2 + linkedWorkspaces.length * 4;
const progress = this.createProgress();
progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps);
progress
.startProcess(AM_MAIN_PROCESS_NAME)
.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME);
// Multibar layout: two shared bootstrap rows + one row per space. Per-space
// totals start at 1 and are bumped to (2 + downloadableCount) inside
// ExportAssets.start once we know the asset count for that space.
progress.addProcess(PROCESS_NAMES.AM_FIELDS, 1);
progress.addProcess(PROCESS_NAMES.AM_ASSET_TYPES, 1);
const spaceProcessNames = new Map<string, string>();
for (const ws of linkedWorkspaces) {
const spaceProcess = getSpaceProcessName(ws.space_uid);
spaceProcessNames.set(ws.space_uid, spaceProcess);
progress.addProcess(spaceProcess, 1);
}

const apiConfig: AssetManagementAPIConfig = {
baseURL: assetManagementUrl,
Expand All @@ -82,39 +87,67 @@ export class ExportSpaces {
await mkdir(sharedAssetTypesDir, { recursive: true });

const firstSpaceUid = linkedWorkspaces[0].space_uid;
let bootstrapFailed = false;
let anySpaceFailed = false;
try {
progress.startProcess(PROCESS_NAMES.AM_FIELDS);
progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES);

const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext);
exportAssetTypes.setParentProgressManager(progress);
const exportFields = new ExportFields(apiConfig, exportContext);
exportFields.setParentProgressManager(progress);
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
try {
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true);
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true);
} catch (bootstrapErr) {
bootstrapFailed = true;
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, false);
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, false);
throw bootstrapErr;
}

for (const ws of linkedWorkspaces) {
progress.updateStatus(`Exporting space: ${ws.space_uid}...`, AM_MAIN_PROCESS_NAME);
const spaceProcess = spaceProcessNames.get(ws.space_uid)!;
progress.startProcess(spaceProcess);
log.debug(`Exporting space: ${ws.space_uid}`, context);
const spaceDir = pResolve(spacesRootPath, ws.space_uid);
try {
const exportWorkspace = new ExportWorkspace(apiConfig, exportContext);
exportWorkspace.setParentProgressManager(progress);
await exportWorkspace.start(ws, spaceDir, branchName || 'main');
await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
progress.completeProcess(spaceProcess, true);
log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
} catch (err) {
// Per-space failure: mark the row failed and continue with the next
// space so partial export results are preserved (matches import).
anySpaceFailed = true;
log.debug(`Failed to export workspace for space ${ws.space_uid}: ${err}`, context);
progress.tick(
false,
`space: ${ws.space_uid}`,
(err as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_SPACE_METADATA].FAILED,
AM_MAIN_PROCESS_NAME,
handleAndLogError(
err,
{ ...(context as Record<string, unknown>), spaceUid: ws.space_uid },
`Failed to export space ${ws.space_uid}`,
);
throw err;
progress.completeProcess(spaceProcess, false);
}
}

progress.completeProcess(AM_MAIN_PROCESS_NAME, true);
log.info('Asset Management export completed successfully', context);
log.info(
anySpaceFailed
? 'Asset Management export completed with errors in one or more spaces'
: 'Asset Management export completed successfully',
context,
);
log.debug('Asset Management 2.0 export completed', context);
} catch (err) {
progress.completeProcess(AM_MAIN_PROCESS_NAME, false);
if (!bootstrapFailed) {
// Mark any spaces that hadn't been processed as failed so the multibar
// doesn't leave dangling pending rows.
for (const [, spaceProcess] of spaceProcessNames) {
progress.completeProcess(spaceProcess, false);
}
}
handleAndLogError(err, { ...(context as Record<string, unknown>) }, 'Asset Management export failed');
throw err;
}
Expand Down
Loading
Loading