From adfb4d5c199374141410b750f11c0cf94875ad20 Mon Sep 17 00:00:00 2001 From: maayarosama Date: Tue, 28 Jan 2025 11:29:05 +0200 Subject: [PATCH 1/8] Reworte handle method inside script file not rollback and to output each batch nodes that successfully and failed to deploy on --- .../grid_client/scripts/mass_deployments.ts | 194 +++++++++++++----- 1 file changed, 137 insertions(+), 57 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index d694bccf9a..1cfcc392af 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -1,21 +1,117 @@ +import { Contract } from "@threefold/tfchain_client"; +import { ValidationError } from "@threefold/types"; + import { + Deployment, + DeploymentResultContracts, + events, FarmFilterOptions, FilterOptions, generateString, GridClient, + KycStatus, MachineModel, MachinesModel, NetworkModel, NodeInfo, + Operations, TwinDeployment, } from "../src"; import { config, getClient } from "./client_loader"; import { log } from "./utils"; -async function pingNodes( - grid3: GridClient, - nodes: NodeInfo[], -): Promise> { +async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { + const kycStatus = await grid3.machines.twinDeploymentHandler.kyc.status(); + if (kycStatus !== KycStatus.verified) { + throw new ValidationError( + "Your account is not verified. Please sign into Threefold Dashboard or Connect mobile app to complete your KYC verification.", + ); + } + + events.emit("logs", "Validating workloads"); + await grid3.machines.twinDeploymentHandler.validate(twinDeployments); + await grid3.machines.twinDeploymentHandler.checkNodesCapacity(twinDeployments); + await grid3.machines.twinDeploymentHandler.checkFarmIps(twinDeployments); + + const contracts: DeploymentResultContracts = { + created: [], + updated: [], + deleted: [], + }; + + const resultContracts: DeploymentResultContracts = { + created: [], + updated: [], + deleted: [], + }; + + const successfulNodesSet: number[] = []; + const failedNodesSet: number[] = []; + const original_deployments: Deployment[] = []; + + const deploymentPromises: Promise[] = []; + + for (const twinDeployment of twinDeployments) { + deploymentPromises.push( + (async () => { + try { + const extrinsics = await grid3.machines.twinDeploymentHandler.PrepareExtrinsic(twinDeployment, contracts); + const extrinsicResults: Contract[] = + await grid3.machines.twinDeploymentHandler.tfclient.applyAllExtrinsics([ + ...extrinsics.nodeExtrinsics, + ...extrinsics.nameExtrinsics, + ]); + + for (const contract of extrinsicResults) { + const updatedContract = contracts.updated.filter(c => c["contractId"] === contract.contractId); + if (updatedContract.length === 0) contracts.created.push(contract); + } + + if (twinDeployment.operation === Operations.deploy) { + events.emit("logs", `Sending deployment to node_id: ${twinDeployment.nodeId}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.deployment.contract_id = contract.contractId; + resultContracts.created.push(contract); + break; + } + } + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + successfulNodesSet.push(twinDeployment.nodeId); + } else if (twinDeployment.operation === Operations.update) { + const old_contract_id = twinDeployment.deployment.contract_id; + if (old_contract_id) { + original_deployments.push( + await grid3.machines.twinDeploymentHandler.getDeploymentFromFactory(old_contract_id), + ); + } + events.emit("logs", `Updating deployment with contract_id: ${twinDeployment.deployment.contract_id}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.nodeId = contract.contractType.nodeContract.nodeId; + resultContracts.updated.push(contract); + break; + } + } + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + successfulNodesSet.push(twinDeployment.nodeId); + } + } catch (error) { + events.emit("logs", `Failed deployment on node_id: ${twinDeployment.nodeId} due to error: ${error}`); + successfulNodesSet.push(twinDeployment.nodeId); + } + })(), + ); + } + + const res = await Promise.allSettled(deploymentPromises); + console.log("Deployment Results in handle:", res); + log("Successful Nodes: " + successfulNodesSet); + log("Failed Nodes: " + failedNodesSet); + return { resultContracts, successfulNodesSet, failedNodesSet }; +} + +async function pingNodes(grid3: GridClient, nodes: NodeInfo[]) { const pingPromises = nodes.map(async node => { try { const res = await grid3.zos.pingNode({ nodeId: node.nodeId }); @@ -31,11 +127,9 @@ async function pingNodes( return result; } - async function main() { const grid3 = await getClient(); - // Timeout for deploying vm is 2 min grid3.clientOptions.deploymentTimeoutMinutes = 2; await grid3._connect(); @@ -43,9 +137,11 @@ async function main() { const offlineNodes: number[] = []; let failedCount = 0; let successCount = 0; - const batchSize = 50; - const totalVMs = 250; + const batchSize = 10; + const totalVMs = 50; const batches = totalVMs / batchSize; + const allSuccessfulNodes = new Set(); + const allFailedNodes = new Set(); // resources const cru = 1; @@ -86,30 +182,20 @@ async function main() { const results = await pingNodes(grid3, nodes); console.timeEnd("Ping Nodes"); - // Check nodes results results.forEach(({ node, res, error }) => { if (res) { console.log(`Node ${node.nodeId} is online`); } else { offlineNodes.push(node.nodeId); console.log(`Node ${node.nodeId} is offline`); - if (error) { - console.error("Error:", error); - } + if (error) console.error("Error:", error); } }); const onlineNodes = nodes.filter(node => !offlineNodes.includes(node.nodeId)); - - // Batch Deployment const batchVMs: MachinesModel[] = []; - for (let i = 0; i < batchSize; i++) { + for (let i = 0; i < batchSize && onlineNodes.length > 0; i++) { const vmName = "vm" + generateString(8); - - if (onlineNodes.length <= 0) { - errors.push("No online nodes available for deployment"); - continue; - } const selectedNode = onlineNodes.pop(); // create vm node Object @@ -141,66 +227,60 @@ async function main() { vms.network = n; vms.machines = [vm]; vms.metadata = ""; - vms.description = "Test deploying vm with name " + vm.name + " via ts grid3 client - Batch " + (batch + 1); + vms.description = `Test deploying vm with name ${vm.name} - Batch ${batch + 1}`; batchVMs.push(vms); } const allTwinDeployments: TwinDeployment[] = []; - const deploymentPromises = batchVMs.map(async (vms, index) => { + const deploymentPromises = batchVMs.map(async vms => { try { - const [twinDeployments, _, __] = await grid3.machines._createDeployment(vms); - return { twinDeployments, batchIndex: index }; + const [twinDeployments] = await grid3.machines._createDeployment(vms); + return twinDeployments; } catch (error) { log(`Error creating deployment for batch ${batch + 1}: ${error}`); - return { twinDeployments: null, batchIndex: index }; + return []; } }); console.time("Preparing Batch " + (batch + 1)); const deploymentResults = await Promise.allSettled(deploymentPromises).then(results => results.flatMap(r => (r.status === "fulfilled" ? r.value : [])), ); - console.timeEnd("Preparing Batch " + (batch + 1)); - - for (const { twinDeployments } of deploymentResults) { - if (twinDeployments) { - allTwinDeployments.push(...twinDeployments); - } - } - - try { - await grid3.machines.twinDeploymentHandler.handle(allTwinDeployments); - log(`Successfully handled and saved contracts for some twin deployments`); - } catch (error) { - errors.push(error); - failedCount += batchSize; - log(`Error handling contracts for twin deployments: ${error}`); - } - successCount = totalVMs - failedCount; + allTwinDeployments.push(...deploymentResults); - console.timeEnd("Batch " + (batch + 1)); - } + if (allTwinDeployments.length > 0) { + try { + const { resultContracts, successfulNodesSet, failedNodesSet } = await handle(grid3, allTwinDeployments); - console.timeEnd("Total Deployment Time"); + successfulNodesSet.forEach(node => allSuccessfulNodes.add(node)); + failedNodesSet.forEach(node => allFailedNodes.add(node)); - log("Successful Deployments: " + successCount); - log("Failed Deployments: " + failedCount); + log(`Successfully handled deployments for Batch ${batch + 1}`); + successCount = allSuccessfulNodes.size; + } catch (error) { + errors.push(`Error handling deployments for Batch ${batch + 1}: ${error}`); + failedCount = allFailedNodes.size; + } + } else { + log(`No deployments created for Batch ${batch + 1}`); + } - // List of failed deployments' errors - log("Failed deployments errors: "); - for (let i = 0; i < errors.length; i++) { - log(errors[i]); + log(`Batch ${batch + 1} Summary:`); + log(`- Successful Deployments: ${successCount}`); + log(`- Failed Deployments: ${failedCount}`); log("---------------------------------------------"); } - // List of offline nodes - log("Failed Nodes: "); - for (let i = 0; i < offlineNodes.length; i++) { - log(offlineNodes[i]); - log("---------------------------------------------"); - } + console.timeEnd("Total Deployment Time"); + + log("Final Summary:"); + log(`- Total Successful Deployments: ${successCount}`); + log(`- Total Failed Deployments: ${failedCount}`); + log(`- Offline Nodes: ${offlineNodes.join(", ")}`); + log(`- All Successful Deployments on Nodes: ${Array.from(allSuccessfulNodes).join(", ")}`); + log(`- All Failed Deployments on Nodes: ${Array.from(allFailedNodes).join(", ")}`); await grid3.disconnect(); } From 6986f14138d9a9868d5c904d12a0f0ec85ae5035 Mon Sep 17 00:00:00 2001 From: maayarosama Date: Tue, 28 Jan 2025 14:47:58 +0200 Subject: [PATCH 2/8] Removing unused console logs --- packages/grid_client/scripts/mass_deployments.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index 1cfcc392af..4316cd698d 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -105,7 +105,6 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { } const res = await Promise.allSettled(deploymentPromises); - console.log("Deployment Results in handle:", res); log("Successful Nodes: " + successfulNodesSet); log("Failed Nodes: " + failedNodesSet); return { resultContracts, successfulNodesSet, failedNodesSet }; From be75a2edca539b842e53746a99cfad7b61e91d1b Mon Sep 17 00:00:00 2001 From: maayarosama Date: Sun, 2 Feb 2025 18:07:45 +0200 Subject: [PATCH 3/8] Refactoring mass deployment script to count failed deployments --- .../grid_client/scripts/mass_deployments.ts | 220 ++++++++++-------- 1 file changed, 125 insertions(+), 95 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index 4316cd698d..6877521c6c 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -1,4 +1,4 @@ -import { Contract } from "@threefold/tfchain_client"; +import { Contract, ExtrinsicResult } from "@threefold/tfchain_client"; import { ValidationError } from "@threefold/types"; import { @@ -6,6 +6,7 @@ import { DeploymentResultContracts, events, FarmFilterOptions, + Features, FilterOptions, generateString, GridClient, @@ -16,17 +17,16 @@ import { NodeInfo, Operations, TwinDeployment, + WorkloadTypes, } from "../src"; import { config, getClient } from "./client_loader"; import { log } from "./utils"; - async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { const kycStatus = await grid3.machines.twinDeploymentHandler.kyc.status(); - if (kycStatus !== KycStatus.verified) { + if (kycStatus !== KycStatus.verified) throw new ValidationError( "Your account is not verified. Please sign into Threefold Dashboard or Connect mobile app to complete your KYC verification.", ); - } events.emit("logs", "Validating workloads"); await grid3.machines.twinDeploymentHandler.validate(twinDeployments); @@ -38,75 +38,107 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { updated: [], deleted: [], }; - const resultContracts: DeploymentResultContracts = { created: [], updated: [], deleted: [], }; + let nodeExtrinsics: ExtrinsicResult[] = []; + let nameExtrinsics: ExtrinsicResult[] = []; + let deletedExtrinsics: ExtrinsicResult[] = []; - const successfulNodesSet: number[] = []; - const failedNodesSet: number[] = []; - const original_deployments: Deployment[] = []; - - const deploymentPromises: Promise[] = []; + const successfulNodesSet = new Set(); + const failedNodesSet = new Set(); for (const twinDeployment of twinDeployments) { - deploymentPromises.push( - (async () => { - try { - const extrinsics = await grid3.machines.twinDeploymentHandler.PrepareExtrinsic(twinDeployment, contracts); - const extrinsicResults: Contract[] = - await grid3.machines.twinDeploymentHandler.tfclient.applyAllExtrinsics([ - ...extrinsics.nodeExtrinsics, - ...extrinsics.nameExtrinsics, - ]); + for (const workload of twinDeployment.deployment.workloads) { + if (!twinDeployment.network) { + break; + } + if (workload.type === WorkloadTypes.network || workload.type === WorkloadTypes.networklight) { + events.emit("logs", `Updating network workload with name: ${workload.name}`); + twinDeployment.network.updateWorkload(twinDeployment.nodeId, workload); + } + } + const extrinsics = await grid3.machines.twinDeploymentHandler.PrepareExtrinsic(twinDeployment, contracts); + nodeExtrinsics = nodeExtrinsics.concat(extrinsics.nodeExtrinsics); + nameExtrinsics = nameExtrinsics.concat(extrinsics.nameExtrinsics); + deletedExtrinsics = deletedExtrinsics.concat(extrinsics.deletedExtrinsics); + } - for (const contract of extrinsicResults) { - const updatedContract = contracts.updated.filter(c => c["contractId"] === contract.contractId); - if (updatedContract.length === 0) contracts.created.push(contract); - } + const extrinsicResults: Contract[] = await grid3.machines.twinDeploymentHandler.tfclient.applyAllExtrinsics( + [...nodeExtrinsics, ...nameExtrinsics], + ); + + for (const contract of extrinsicResults) { + const updatedContract = contracts.updated.filter(c => c["contractId"] === contract.contractId); + if (updatedContract.length === 0) contracts.created.push(contract); + } - if (twinDeployment.operation === Operations.deploy) { - events.emit("logs", `Sending deployment to node_id: ${twinDeployment.nodeId}`); - for (const contract of extrinsicResults) { - if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { - twinDeployment.deployment.contract_id = contract.contractId; + await Promise.allSettled( + twinDeployments.map(async twinDeployment => { + try { + if (twinDeployment.operation === Operations.deploy) { + events.emit("logs", `Sending deployment to node_id: ${twinDeployment.nodeId}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.deployment.contract_id = contract.contractId; + if ( + twinDeployment.returnNetworkContracts || + !( + twinDeployment.deployment.workloads.length === 1 && + twinDeployment.deployment.workloads[0].type === WorkloadTypes.network + ) + ) resultContracts.created.push(contract); - break; - } + break; } - await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); - successfulNodesSet.push(twinDeployment.nodeId); - } else if (twinDeployment.operation === Operations.update) { - const old_contract_id = twinDeployment.deployment.contract_id; - if (old_contract_id) { - original_deployments.push( - await grid3.machines.twinDeploymentHandler.getDeploymentFromFactory(old_contract_id), - ); - } - events.emit("logs", `Updating deployment with contract_id: ${twinDeployment.deployment.contract_id}`); - for (const contract of extrinsicResults) { - if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { - twinDeployment.nodeId = contract.contractType.nodeContract.nodeId; + } + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + successfulNodesSet.add(twinDeployment.nodeId); + events.emit( + "logs", + `A deployment has been created on node_id: ${twinDeployment.nodeId} with contract_id: ${twinDeployment.deployment.contract_id}`, + ); + } else if (twinDeployment.operation === Operations.update) { + events.emit("logs", `Updating deployment with contract_id: ${twinDeployment.deployment.contract_id}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.nodeId = contract.contractType.nodeContract.nodeId; + if ( + twinDeployment.returnNetworkContracts || + !( + twinDeployment.deployment.workloads.length === 1 && + twinDeployment.deployment.workloads[0].type === WorkloadTypes.network + ) + ) resultContracts.updated.push(contract); - break; - } + break; } - await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); - successfulNodesSet.push(twinDeployment.nodeId); } - } catch (error) { - events.emit("logs", `Failed deployment on node_id: ${twinDeployment.nodeId} due to error: ${error}`); - successfulNodesSet.push(twinDeployment.nodeId); + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + successfulNodesSet.add(twinDeployment.nodeId); + events.emit("logs", `Deployment has been updated with contract_id: ${twinDeployment.deployment.contract_id}`); } - })(), - ); + } catch (error) { + failedNodesSet.add(twinDeployment.nodeId); + events.emit("logs", `Failed to process node_id: ${twinDeployment.nodeId} due to error: ${error}`); + } + }), + ); + + const deletedResult = await grid3.machines.twinDeploymentHandler.tfclient.applyAllExtrinsics( + deletedExtrinsics, + ); + if (deletedExtrinsics.length > 0) { + for (const id of deletedResult) { + resultContracts.deleted.push({ contractId: id }); + events.emit("logs", `Deployment has been deleted with contract_id: ${id}`); + } } - const res = await Promise.allSettled(deploymentPromises); - log("Successful Nodes: " + successfulNodesSet); - log("Failed Nodes: " + failedNodesSet); + await grid3.machines.twinDeploymentHandler.waitForDeployments(twinDeployments); + await grid3.machines.twinDeploymentHandler.saveNetworks(twinDeployments, contracts); return { resultContracts, successfulNodesSet, failedNodesSet }; } @@ -136,11 +168,9 @@ async function main() { const offlineNodes: number[] = []; let failedCount = 0; let successCount = 0; - const batchSize = 10; - const totalVMs = 50; + const batchSize = 3; + const totalVMs = 6; const batches = totalVMs / batchSize; - const allSuccessfulNodes = new Set(); - const allFailedNodes = new Set(); // resources const cru = 1; @@ -175,6 +205,7 @@ async function main() { availableFor: await grid3.twins.get_my_twin_id(), farmIds: farmIds, randomize: true, + features: [Features.yggdrasil], } as FilterOptions); console.time("Ping Nodes"); @@ -187,17 +218,24 @@ async function main() { } else { offlineNodes.push(node.nodeId); console.log(`Node ${node.nodeId} is offline`); - if (error) console.error("Error:", error); + if (error) { + console.error("Error:", error); + } } }); const onlineNodes = nodes.filter(node => !offlineNodes.includes(node.nodeId)); const batchVMs: MachinesModel[] = []; - for (let i = 0; i < batchSize && onlineNodes.length > 0; i++) { + + for (let i = 0; i < batchSize; i++) { const vmName = "vm" + generateString(8); + + if (onlineNodes.length <= 0) { + errors.push("No online nodes available for deployment"); + continue; + } const selectedNode = onlineNodes.pop(); - // create vm node Object const vm = new MachineModel(); vm.name = vmName; vm.node_id = selectedNode.nodeId; @@ -214,72 +252,64 @@ async function main() { SSH_KEY: config.ssh_key, }; - // create network model for each vm const n = new NetworkModel(); n.name = "nw" + generateString(5); n.ip_range = "10.238.0.0/16"; n.addAccess = true; - // create VMs Object for each vm const vms = new MachinesModel(); vms.name = "batch" + (batch + 1); vms.network = n; vms.machines = [vm]; vms.metadata = ""; - vms.description = `Test deploying vm with name ${vm.name} - Batch ${batch + 1}`; + vms.description = "Test deploying vm with name " + vm.name + " via ts grid3 client - Batch " + (batch + 1); batchVMs.push(vms); } const allTwinDeployments: TwinDeployment[] = []; - const deploymentPromises = batchVMs.map(async vms => { + const deploymentPromises = batchVMs.map(async (vms, index) => { try { - const [twinDeployments] = await grid3.machines._createDeployment(vms); - return twinDeployments; + const [twinDeployments, _, __] = await grid3.machines._createDeployment(vms); + return { twinDeployments, batchIndex: index }; } catch (error) { log(`Error creating deployment for batch ${batch + 1}: ${error}`); - return []; + return { twinDeployments: null, batchIndex: index }; } }); console.time("Preparing Batch " + (batch + 1)); const deploymentResults = await Promise.allSettled(deploymentPromises).then(results => results.flatMap(r => (r.status === "fulfilled" ? r.value : [])), ); + console.timeEnd("Preparing Batch " + (batch + 1)); - allTwinDeployments.push(...deploymentResults); - - if (allTwinDeployments.length > 0) { - try { - const { resultContracts, successfulNodesSet, failedNodesSet } = await handle(grid3, allTwinDeployments); - - successfulNodesSet.forEach(node => allSuccessfulNodes.add(node)); - failedNodesSet.forEach(node => allFailedNodes.add(node)); - - log(`Successfully handled deployments for Batch ${batch + 1}`); - successCount = allSuccessfulNodes.size; - } catch (error) { - errors.push(`Error handling deployments for Batch ${batch + 1}: ${error}`); - failedCount = allFailedNodes.size; + for (const { twinDeployments } of deploymentResults) { + if (twinDeployments) { + allTwinDeployments.push(...twinDeployments); } - } else { - log(`No deployments created for Batch ${batch + 1}`); } - log(`Batch ${batch + 1} Summary:`); - log(`- Successful Deployments: ${successCount}`); - log(`- Failed Deployments: ${failedCount}`); - log("---------------------------------------------"); + try { + const { resultContracts, successfulNodesSet, failedNodesSet } = await handle(grid3, allTwinDeployments); + log(`Successfully handled and saved contracts for twin deployments`); + log(`Successful Nodes: ${Array.from(successfulNodesSet).join(", ")}`); + log(`Failed Nodes: ${Array.from(failedNodesSet).join(", ")}`); + } catch (error) { + errors.push(error); + failedCount += batchSize; + log(`Error handling contracts for twin deployments: ${error}`); + } + + successCount = totalVMs - failedCount; + console.timeEnd("Batch " + (batch + 1)); } console.timeEnd("Total Deployment Time"); - log("Final Summary:"); - log(`- Total Successful Deployments: ${successCount}`); - log(`- Total Failed Deployments: ${failedCount}`); - log(`- Offline Nodes: ${offlineNodes.join(", ")}`); - log(`- All Successful Deployments on Nodes: ${Array.from(allSuccessfulNodes).join(", ")}`); - log(`- All Failed Deployments on Nodes: ${Array.from(allFailedNodes).join(", ")}`); + log("Successful Deployments: " + successCount); + log("Failed Deployments: " + failedCount); + log("Failed Nodes: " + offlineNodes); await grid3.disconnect(); } From 438576d3396d918c92d1d2e122cbd63ad272bddb Mon Sep 17 00:00:00 2001 From: maayarosama Date: Sun, 9 Feb 2025 21:07:20 +0200 Subject: [PATCH 4/8] Setting addAccess to false and refactoring handle method and adding promise to calling of handle method --- .../grid_client/scripts/mass_deployments.ts | 189 +++++++++--------- 1 file changed, 94 insertions(+), 95 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index 6877521c6c..d282215030 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -6,7 +6,6 @@ import { DeploymentResultContracts, events, FarmFilterOptions, - Features, FilterOptions, generateString, GridClient, @@ -47,19 +46,15 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { let nameExtrinsics: ExtrinsicResult[] = []; let deletedExtrinsics: ExtrinsicResult[] = []; - const successfulNodesSet = new Set(); - const failedNodesSet = new Set(); - for (const twinDeployment of twinDeployments) { for (const workload of twinDeployment.deployment.workloads) { - if (!twinDeployment.network) { - break; - } + if (!twinDeployment.network) break; if (workload.type === WorkloadTypes.network || workload.type === WorkloadTypes.networklight) { events.emit("logs", `Updating network workload with name: ${workload.name}`); twinDeployment.network.updateWorkload(twinDeployment.nodeId, workload); } } + const extrinsics = await grid3.machines.twinDeploymentHandler.PrepareExtrinsic(twinDeployment, contracts); nodeExtrinsics = nodeExtrinsics.concat(extrinsics.nodeExtrinsics); nameExtrinsics = nameExtrinsics.concat(extrinsics.nameExtrinsics); @@ -71,61 +66,61 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { ); for (const contract of extrinsicResults) { - const updatedContract = contracts.updated.filter(c => c["contractId"] === contract.contractId); - if (updatedContract.length === 0) contracts.created.push(contract); + const updatedContract = contracts.updated.find(c => c["contractId"] === contract.contractId); + if (!updatedContract) contracts.created.push(contract); } - await Promise.allSettled( - twinDeployments.map(async twinDeployment => { - try { - if (twinDeployment.operation === Operations.deploy) { - events.emit("logs", `Sending deployment to node_id: ${twinDeployment.nodeId}`); - for (const contract of extrinsicResults) { - if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { - twinDeployment.deployment.contract_id = contract.contractId; - if ( - twinDeployment.returnNetworkContracts || - !( - twinDeployment.deployment.workloads.length === 1 && - twinDeployment.deployment.workloads[0].type === WorkloadTypes.network - ) + const successfulNodes = new Set(); + const failedNodes = new Set(); + + for (const twinDeployment of twinDeployments) { + try { + if (twinDeployment.operation === Operations.deploy) { + events.emit("logs", `Sending deployment to node_id: ${twinDeployment.nodeId}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.deployment.contract_id = contract.contractId; + if ( + twinDeployment.returnNetworkContracts || + !( + twinDeployment.deployment.workloads.length === 1 && + twinDeployment.deployment.workloads[0].type === WorkloadTypes.network ) - resultContracts.created.push(contract); - break; - } + ) + resultContracts.created.push(contract); + break; } - await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); - successfulNodesSet.add(twinDeployment.nodeId); - events.emit( - "logs", - `A deployment has been created on node_id: ${twinDeployment.nodeId} with contract_id: ${twinDeployment.deployment.contract_id}`, - ); - } else if (twinDeployment.operation === Operations.update) { - events.emit("logs", `Updating deployment with contract_id: ${twinDeployment.deployment.contract_id}`); - for (const contract of extrinsicResults) { - if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { - twinDeployment.nodeId = contract.contractType.nodeContract.nodeId; - if ( - twinDeployment.returnNetworkContracts || - !( - twinDeployment.deployment.workloads.length === 1 && - twinDeployment.deployment.workloads[0].type === WorkloadTypes.network - ) + } + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + events.emit( + "logs", + `A deployment has been created on node_id: ${twinDeployment.nodeId} with contract_id: ${twinDeployment.deployment.contract_id}`, + ); + } else if (twinDeployment.operation === Operations.update) { + events.emit("logs", `Updating deployment with contract_id: ${twinDeployment.deployment.contract_id}`); + for (const contract of extrinsicResults) { + if (twinDeployment.deployment.challenge_hash() === contract.contractType.nodeContract.deploymentHash) { + twinDeployment.nodeId = contract.contractType.nodeContract.nodeId; + if ( + twinDeployment.returnNetworkContracts || + !( + twinDeployment.deployment.workloads.length === 1 && + twinDeployment.deployment.workloads[0].type === WorkloadTypes.network ) - resultContracts.updated.push(contract); - break; - } + ) + resultContracts.updated.push(contract); + break; } - await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); - successfulNodesSet.add(twinDeployment.nodeId); - events.emit("logs", `Deployment has been updated with contract_id: ${twinDeployment.deployment.contract_id}`); } - } catch (error) { - failedNodesSet.add(twinDeployment.nodeId); - events.emit("logs", `Failed to process node_id: ${twinDeployment.nodeId} due to error: ${error}`); + await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); + events.emit("logs", `Deployment has been updated with contract_id: ${twinDeployment.deployment.contract_id}`); } - }), - ); + successfulNodes.add(twinDeployment.nodeId); + } catch (e) { + failedNodes.add(twinDeployment.nodeId); + events.emit("logs", `Deployment failed on node_id: ${twinDeployment.nodeId} with error: ${e}`); + } + } const deletedResult = await grid3.machines.twinDeploymentHandler.tfclient.applyAllExtrinsics( deletedExtrinsics, @@ -139,7 +134,7 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { await grid3.machines.twinDeploymentHandler.waitForDeployments(twinDeployments); await grid3.machines.twinDeploymentHandler.saveNetworks(twinDeployments, contracts); - return { resultContracts, successfulNodesSet, failedNodesSet }; + return { resultContracts, successfulNodes, failedNodes }; } async function pingNodes(grid3: GridClient, nodes: NodeInfo[]) { @@ -166,11 +161,11 @@ async function main() { const errors: any = []; const offlineNodes: number[] = []; - let failedCount = 0; - let successCount = 0; const batchSize = 3; const totalVMs = 6; const batches = totalVMs / batchSize; + const allSuccessfulNodes: number[] = []; + const allFailedNodes = new Set(); // resources const cru = 1; @@ -205,7 +200,6 @@ async function main() { availableFor: await grid3.twins.get_my_twin_id(), farmIds: farmIds, randomize: true, - features: [Features.yggdrasil], } as FilterOptions); console.time("Ping Nodes"); @@ -218,22 +212,14 @@ async function main() { } else { offlineNodes.push(node.nodeId); console.log(`Node ${node.nodeId} is offline`); - if (error) { - console.error("Error:", error); - } + if (error) console.error("Error:", error); } }); const onlineNodes = nodes.filter(node => !offlineNodes.includes(node.nodeId)); const batchVMs: MachinesModel[] = []; - - for (let i = 0; i < batchSize; i++) { + for (let i = 0; i < batchSize && onlineNodes.length > 0; i++) { const vmName = "vm" + generateString(8); - - if (onlineNodes.length <= 0) { - errors.push("No online nodes available for deployment"); - continue; - } const selectedNode = onlineNodes.pop(); const vm = new MachineModel(); @@ -255,61 +241,74 @@ async function main() { const n = new NetworkModel(); n.name = "nw" + generateString(5); n.ip_range = "10.238.0.0/16"; - n.addAccess = true; + n.addAccess = false; const vms = new MachinesModel(); vms.name = "batch" + (batch + 1); vms.network = n; vms.machines = [vm]; vms.metadata = ""; - vms.description = "Test deploying vm with name " + vm.name + " via ts grid3 client - Batch " + (batch + 1); + vms.description = `Test deploying vm with name ${vm.name} - Batch ${batch + 1}`; batchVMs.push(vms); } const allTwinDeployments: TwinDeployment[] = []; - const deploymentPromises = batchVMs.map(async (vms, index) => { + const deploymentPromises = batchVMs.map(async vms => { try { - const [twinDeployments, _, __] = await grid3.machines._createDeployment(vms); - return { twinDeployments, batchIndex: index }; + const [twinDeployments] = await grid3.machines._createDeployment(vms); + return twinDeployments; } catch (error) { log(`Error creating deployment for batch ${batch + 1}: ${error}`); - return { twinDeployments: null, batchIndex: index }; + return []; } }); console.time("Preparing Batch " + (batch + 1)); const deploymentResults = await Promise.allSettled(deploymentPromises).then(results => results.flatMap(r => (r.status === "fulfilled" ? r.value : [])), ); - console.timeEnd("Preparing Batch " + (batch + 1)); + allTwinDeployments.push(...deploymentResults); + let batchSuccessfulNodes: Set = new Set(); + let batchFailedNodes: Set = new Set(); - for (const { twinDeployments } of deploymentResults) { - if (twinDeployments) { - allTwinDeployments.push(...twinDeployments); - } - } + if (allTwinDeployments.length > 0) { + const results = await Promise.allSettled([handle(grid3, allTwinDeployments)]); - try { - const { resultContracts, successfulNodesSet, failedNodesSet } = await handle(grid3, allTwinDeployments); - log(`Successfully handled and saved contracts for twin deployments`); - log(`Successful Nodes: ${Array.from(successfulNodesSet).join(", ")}`); - log(`Failed Nodes: ${Array.from(failedNodesSet).join(", ")}`); - } catch (error) { - errors.push(error); - failedCount += batchSize; - log(`Error handling contracts for twin deployments: ${error}`); - } + results.forEach(result => { + if (result.status === "fulfilled") { + const { resultContracts, successfulNodes, failedNodes } = result.value; + batchSuccessfulNodes = successfulNodes; + batchFailedNodes = failedNodes; - successCount = totalVMs - failedCount; - console.timeEnd("Batch " + (batch + 1)); + successfulNodes.forEach(node => allSuccessfulNodes.push(node)); + failedNodes.forEach(node => allFailedNodes.add(node)); + + log(`Successfully handled deployments for Batch ${batch + 1}`); + } else { + errors.push(`Error handling deployments for Batch ${batch + 1}: ${result.reason}`); + } + }); + + log(`Batch ${batch + 1} Summary:`); + log(`- Successful Deployments on Nodes: ${Array.from(batchSuccessfulNodes).join(", ")}`); + log(`- Failed Deployments on Nodes: ${Array.from(batchFailedNodes).join(", ")}`); + log(`- Successful Deployments: ${batchSuccessfulNodes.size}`); + log(`- Failed Deployments: ${batchFailedNodes.size}`); + log("---------------------------------------------"); + } else { + log(`No deployments created for Batch ${batch + 1}`); + } } console.timeEnd("Total Deployment Time"); - log("Successful Deployments: " + successCount); - log("Failed Deployments: " + failedCount); - log("Failed Nodes: " + offlineNodes); + log("Final Summary:"); + log(`- Total Successful Deployments: ${allSuccessfulNodes.length}`); + log(`- Total Failed Deployments: ${totalVMs - allSuccessfulNodes.length}`); + log(`- Offline Nodes: ${offlineNodes.join(", ")}`); + log(`- All Successful Deployments on Nodes: ${Array.from(allSuccessfulNodes).join(", ")}`); + log(`- All Failed Deployments on Nodes: ${Array.from(allFailedNodes).join(", ")}`); await grid3.disconnect(); } From e58303666796f706f6213941d1b8e0e394b7ee9f Mon Sep 17 00:00:00 2001 From: maayarosama Date: Mon, 10 Feb 2025 15:58:26 +0200 Subject: [PATCH 5/8] Fixing random empty batch data --- .../grid_client/scripts/mass_deployments.ts | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index d282215030..a17b066dfa 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -20,6 +20,38 @@ import { } from "../src"; import { config, getClient } from "./client_loader"; import { log } from "./utils"; + +async function waitForDeployments(grid3: GridClient, twinDeployments: TwinDeployment[]) { + const timeout = grid3.machines.twinDeploymentHandler.config.deploymentTimeoutMinutes; + const results = await Promise.allSettled( + twinDeployments.map(t => { + if ([Operations.deploy, Operations.update].includes(t.operation)) { + events.emit("logs", `Waiting for deployment with contract_id: ${t.deployment.contract_id} to be ready`); + return grid3.machines.twinDeploymentHandler.waitForDeployment(t, timeout); + } + + return null; + }), + ); + + const passedDeployments: TwinDeployment[] = []; + const failedDeployments: { twinDeployment: TwinDeployment; reason: any }[] = []; + + results.forEach((result, index) => { + const twinDeployment = twinDeployments[index]; + if (result.status === "fulfilled") { + passedDeployments.push(twinDeployment); + } else { + failedDeployments.push({ + twinDeployment, + reason: result.reason, + }); + } + }); + + return { passedDeployments, failedDeployments }; +} + async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { const kycStatus = await grid3.machines.twinDeploymentHandler.kyc.status(); if (kycStatus !== KycStatus.verified) @@ -72,7 +104,6 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { const successfulNodes = new Set(); const failedNodes = new Set(); - for (const twinDeployment of twinDeployments) { try { if (twinDeployment.operation === Operations.deploy) { @@ -115,9 +146,7 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { await grid3.machines.twinDeploymentHandler.sendToNode(twinDeployment); events.emit("logs", `Deployment has been updated with contract_id: ${twinDeployment.deployment.contract_id}`); } - successfulNodes.add(twinDeployment.nodeId); } catch (e) { - failedNodes.add(twinDeployment.nodeId); events.emit("logs", `Deployment failed on node_id: ${twinDeployment.nodeId} with error: ${e}`); } } @@ -132,8 +161,15 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { } } - await grid3.machines.twinDeploymentHandler.waitForDeployments(twinDeployments); - await grid3.machines.twinDeploymentHandler.saveNetworks(twinDeployments, contracts); + try { + const { passedDeployments, failedDeployments } = await waitForDeployments(grid3, twinDeployments); + passedDeployments.forEach(deployment => successfulNodes.add(deployment.nodeId)); + failedDeployments.forEach(deployment => failedNodes.add(deployment.twinDeployment.nodeId)); + await grid3.machines.twinDeploymentHandler.saveNetworks(twinDeployments, contracts); + } catch (error) { + events.emit("logs", `Deployment Error: ${error}`); + } + return { resultContracts, successfulNodes, failedNodes }; } @@ -271,9 +307,10 @@ async function main() { allTwinDeployments.push(...deploymentResults); let batchSuccessfulNodes: Set = new Set(); let batchFailedNodes: Set = new Set(); - if (allTwinDeployments.length > 0) { - const results = await Promise.allSettled([handle(grid3, allTwinDeployments)]); + const handlePromises = [handle(grid3, allTwinDeployments)]; + + const results = await Promise.allSettled(handlePromises); results.forEach(result => { if (result.status === "fulfilled") { From 0e0199b7c5cee4bcb264207d8e7772e5a8ea4984 Mon Sep 17 00:00:00 2001 From: maayarosama Date: Mon, 10 Feb 2025 16:34:58 +0200 Subject: [PATCH 6/8] Adding try and catch around validation function calls --- packages/grid_client/scripts/mass_deployments.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index a17b066dfa..64a60290a5 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -58,11 +58,14 @@ async function handle(grid3: GridClient, twinDeployments: TwinDeployment[]) { throw new ValidationError( "Your account is not verified. Please sign into Threefold Dashboard or Connect mobile app to complete your KYC verification.", ); - - events.emit("logs", "Validating workloads"); - await grid3.machines.twinDeploymentHandler.validate(twinDeployments); - await grid3.machines.twinDeploymentHandler.checkNodesCapacity(twinDeployments); - await grid3.machines.twinDeploymentHandler.checkFarmIps(twinDeployments); + try { + events.emit("logs", "Validating workloads"); + await grid3.machines.twinDeploymentHandler.validate(twinDeployments); + await grid3.machines.twinDeploymentHandler.checkNodesCapacity(twinDeployments); + await grid3.machines.twinDeploymentHandler.checkFarmIps(twinDeployments); + } catch (error) { + events.emit("logs", "Error Validating workloads:" + error); + } const contracts: DeploymentResultContracts = { created: [], From e0b8a40b416009b4e991c114f286f67be05f28c5 Mon Sep 17 00:00:00 2001 From: maayarosama Date: Tue, 11 Feb 2025 12:09:44 +0200 Subject: [PATCH 7/8] Formating logs --- .../grid_client/scripts/mass_deployments.ts | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index 64a60290a5..97569d7084 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -331,10 +331,18 @@ async function main() { }); log(`Batch ${batch + 1} Summary:`); - log(`- Successful Deployments on Nodes: ${Array.from(batchSuccessfulNodes).join(", ")}`); - log(`- Failed Deployments on Nodes: ${Array.from(batchFailedNodes).join(", ")}`); - log(`- Successful Deployments: ${batchSuccessfulNodes.size}`); - log(`- Failed Deployments: ${batchFailedNodes.size}`); + log( + `- Successful Deployments on Nodes: ${ + Array.from(batchSuccessfulNodes).length ? Array.from(batchSuccessfulNodes).join(", ") : "-" + }`, + ); + log( + `- Failed Deployments on Nodes: ${ + Array.from(batchFailedNodes).length ? Array.from(batchFailedNodes).join(", ") : "-" + }`, + ); + log(`- Successful Deployments: ${batchSuccessfulNodes.size ?? 0}`); + log(`- Failed Deployments: ${batchFailedNodes.size ?? 0}`); log("---------------------------------------------"); } else { log(`No deployments created for Batch ${batch + 1}`); @@ -344,11 +352,19 @@ async function main() { console.timeEnd("Total Deployment Time"); log("Final Summary:"); - log(`- Total Successful Deployments: ${allSuccessfulNodes.length}`); + log(`- Total Successful Deployments: ${allSuccessfulNodes.length ?? 0}`); log(`- Total Failed Deployments: ${totalVMs - allSuccessfulNodes.length}`); - log(`- Offline Nodes: ${offlineNodes.join(", ")}`); - log(`- All Successful Deployments on Nodes: ${Array.from(allSuccessfulNodes).join(", ")}`); - log(`- All Failed Deployments on Nodes: ${Array.from(allFailedNodes).join(", ")}`); + log(`- Offline Nodes: ${offlineNodes.length ? offlineNodes.join(", ") : "-"}`); + log( + `- All Successful Deployments on Nodes: ${ + Array.from(allSuccessfulNodes).length ? Array.from(allSuccessfulNodes).join(", ") : "-" + }`, + ); + log( + `- All Failed Deployments on Nodes: ${ + Array.from(allFailedNodes).length ? Array.from(allFailedNodes).join(", ") : "-" + }`, + ); await grid3.disconnect(); } From 52c96cd764485487555b445ddb804ed9cfede67a Mon Sep 17 00:00:00 2001 From: maayarosama Date: Tue, 11 Feb 2025 13:19:38 +0200 Subject: [PATCH 8/8] Adding features to filterNodes --- packages/grid_client/scripts/mass_deployments.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/grid_client/scripts/mass_deployments.ts b/packages/grid_client/scripts/mass_deployments.ts index 97569d7084..d67fb093be 100644 --- a/packages/grid_client/scripts/mass_deployments.ts +++ b/packages/grid_client/scripts/mass_deployments.ts @@ -6,6 +6,7 @@ import { DeploymentResultContracts, events, FarmFilterOptions, + Features, FilterOptions, generateString, GridClient, @@ -239,6 +240,7 @@ async function main() { availableFor: await grid3.twins.get_my_twin_id(), farmIds: farmIds, randomize: true, + features: [Features.yggdrasil], } as FilterOptions); console.time("Ping Nodes");