diff --git a/tools/scripts/pieces/update-pieces-metadata.ts b/tools/scripts/pieces/update-pieces-metadata.ts index 62f9c2e3cf..82f2647496 100644 --- a/tools/scripts/pieces/update-pieces-metadata.ts +++ b/tools/scripts/pieces/update-pieces-metadata.ts @@ -2,12 +2,11 @@ import assert from 'node:assert'; import { PieceMetadata } from '../../../packages/pieces/community/framework/src'; import { StatusCodes } from 'http-status-codes'; import { HttpHeader } from '../../../packages/pieces/community/common/src'; -import { findAllPieces } from '../utils/piece-script-utils'; +import { AP_CLOUD_API_BASE, findAllPieces, findNewPieces, pieceMetadataExists } from '../utils/piece-script-utils'; assert(process.env['AP_CLOUD_API_KEY'], 'API Key is not defined'); const { AP_CLOUD_API_KEY } = process.env; -const AP_CLOUD_API_BASE = 'https://cloud.activepieces.com/api/v1'; const insertPieceMetadata = async ( pieceMetadata: PieceMetadata @@ -30,28 +29,7 @@ const insertPieceMetadata = async ( } }; -const pieceMetadataExists = async ( - pieceName: string, - pieceVersion: string -): Promise => { - const cloudResponse = await fetch( - `${AP_CLOUD_API_BASE}/pieces/${pieceName}?version=${pieceVersion}` - ); - - const pieceExist: Record = { - [StatusCodes.OK]: true, - [StatusCodes.NOT_FOUND]: false - }; - - if ( - pieceExist[cloudResponse.status] === null || - pieceExist[cloudResponse.status] === undefined - ) { - throw new Error(await cloudResponse.text()); - } - return pieceExist[cloudResponse.status]; -}; const insertMetadataIfNotExist = async (pieceMetadata: PieceMetadata) => { console.info( @@ -72,16 +50,14 @@ const insertMetadataIfNotExist = async (pieceMetadata: PieceMetadata) => { }; const insertMetadata = async (piecesMetadata: PieceMetadata[]) => { - for (const pieceMetadata of piecesMetadata) { - await insertMetadataIfNotExist(pieceMetadata); - } + await Promise.all(piecesMetadata.map(insertMetadataIfNotExist)) }; const main = async () => { console.log('update pieces metadata: started') - const piecesMetadata = await findAllPieces() + const piecesMetadata = await findNewPieces() await insertMetadata(piecesMetadata) console.log('update pieces metadata: completed') diff --git a/tools/scripts/utils/piece-script-utils.ts b/tools/scripts/utils/piece-script-utils.ts index 261acb792a..b9a4abe3df 100644 --- a/tools/scripts/utils/piece-script-utils.ts +++ b/tools/scripts/utils/piece-script-utils.ts @@ -3,9 +3,10 @@ import { readdir, stat } from 'node:fs/promises' import { resolve, join } from 'node:path' import { cwd } from 'node:process' import { PieceMetadata } from '../../../packages/pieces/community/framework/src' -import { PieceCategory, extractPieceFromModule } from '../../../packages/shared/src' +import { extractPieceFromModule } from '../../../packages/shared/src' import * as semver from 'semver' import { readPackageJson } from './files' +import { StatusCodes } from 'http-status-codes' type Piece = { name: string; displayName: string; @@ -15,8 +16,10 @@ type Piece = { metadata(): Omit; }; +export const AP_CLOUD_API_BASE = 'https://cloud.activepieces.com/api/v1'; export const PIECES_FOLDER = 'packages/pieces' export const COMMUNITY_PIECE_FOLDER = 'packages/pieces/community' +export const NON_PIECES_PACKAGES = ['@activepieces/pieces-framework', '@activepieces/pieces-common'] const validateSupportedRelease = (minRelease: string | undefined, maxRelease: string | undefined) => { if (minRelease !== undefined && !semver.valid(minRelease)) { @@ -51,10 +54,6 @@ export function getCommunityPieceFolder(pieceName: string): string { return join(COMMUNITY_PIECE_FOLDER, pieceName) } -export async function findPiece(pieceName: string): Promise { - const pieces = await findAllPieces() - return pieces.find((p) => p.name === pieceName) ?? null -} export async function findAllPiecesDirectoryInSource(): Promise { const piecesPath = resolve(cwd(), 'packages', 'pieces') @@ -64,10 +63,47 @@ export async function findAllPiecesDirectoryInSource(): Promise { return [...paths, ...enterprisePiecesPaths] } -export async function findPieceDirectoryInSource(pieceName: string): Promise { - const piecesPath = await findAllPiecesDirectoryInSource(); - const piecePath = piecesPath.find((p) => p.includes(pieceName)) - return piecePath ?? null +export const pieceMetadataExists = async ( + pieceName: string, + pieceVersion: string +): Promise => { + const cloudResponse = await fetch( + `${AP_CLOUD_API_BASE}/pieces/${pieceName}?version=${pieceVersion}` + ); + + const pieceExist: Record = { + [StatusCodes.OK]: true, + [StatusCodes.NOT_FOUND]: false + }; + + if ( + pieceExist[cloudResponse.status] === null || + pieceExist[cloudResponse.status] === undefined + ) { + throw new Error(await cloudResponse.text()); + } + + return pieceExist[cloudResponse.status]; +}; + +export async function findNewPieces(): Promise { + const paths = await findAllDistPaths() + const changedPieces = (await Promise.all(paths.map(async (folderPath) => { + const packageJson = await readPackageJson(folderPath); + if (NON_PIECES_PACKAGES.includes(packageJson.name)) { + return null; + } + const exists = await pieceMetadataExists(packageJson.name, packageJson.version) + if (!exists) { + try { + return loadPieceFromFolder(folderPath); + } catch (ex) { + return null; + } + } + return null; + }))).filter((piece): piece is PieceMetadata => piece !== null) + return changedPieces; } export async function findAllPieces(): Promise {