diff --git a/src/business/runtime-state/config/remote/remote-config-runtime-state.ts b/src/business/runtime-state/config/remote/remote-config-runtime-state.ts index 1fd170797..88b4b0062 100644 --- a/src/business/runtime-state/config/remote/remote-config-runtime-state.ts +++ b/src/business/runtime-state/config/remote/remote-config-runtime-state.ts @@ -544,6 +544,7 @@ export class RemoteConfigRuntimeState implements RemoteConfigRuntimeStateApi { cluster.dnsConsensusNodePattern, ), node.blockNodeMap, + node.externalBlockNodeMap, ), ); } diff --git a/src/commands/block-node.ts b/src/commands/block-node.ts index e3f9e29ef..8235ab8dc 100644 --- a/src/commands/block-node.ts +++ b/src/commands/block-node.ts @@ -42,6 +42,7 @@ import {DeploymentStateSchema} from '../data/schema/model/remote/deployment-stat import {ConsensusNode} from '../core/model/consensus-node.js'; import {NetworkCommand} from './network.js'; import {type ClusterSchema} from '../data/schema/model/common/cluster-schema.js'; +import {ExternalBlockNodeStateSchema} from '../data/schema/model/remote/state/external-block-node-state-schema.js'; interface BlockNodeDeployConfigClass { chartVersion: string; @@ -109,6 +110,22 @@ interface BlockNodeUpgradeContext { config: BlockNodeUpgradeConfigClass; } +interface BlockNodeAddExternalConfigClass { + clusterRef: ClusterReferenceName; + deployment: DeploymentName; + devMode: boolean; + quiet: boolean; + context: string; + externalBlockNodeAddress: string; + newExternalBlockNodeComponent: ExternalBlockNodeStateSchema; + namespace: NamespaceName; + priorityMapping: Record; +} + +interface BlockNodeAddExternalContext { + config: BlockNodeAddExternalConfigClass; +} + @injectable() export class BlockNodeCommand extends BaseCommand { public constructor() { @@ -121,6 +138,8 @@ export class BlockNodeCommand extends BaseCommand { private static readonly UPGRADE_CONFIGS_NAME: string = 'upgradeConfigs'; + private static readonly ADD_EXTERNAL_CONFIGS_NAME: string = 'addConfigs'; + public static readonly ADD_FLAGS_LIST: CommandFlags = { required: [flags.deployment], optional: [ @@ -139,6 +158,11 @@ export class BlockNodeCommand extends BaseCommand { ], }; + public static readonly ADD_EXTERNAL_FLAGS_LIST: CommandFlags = { + required: [flags.deployment, flags.externalBlockNodeAddress], + optional: [flags.clusterRef, flags.devMode, flags.quiet, flags.priorityMapping], + }; + public static readonly DESTROY_FLAGS_LIST: CommandFlags = { required: [flags.deployment], optional: [flags.chartDirectory, flags.clusterRef, flags.devMode, flags.force, flags.quiet, flags.id], @@ -252,7 +276,7 @@ export class BlockNodeCommand extends BaseCommand { private updateConsensusNodesPostGenesis(): SoloListrTask { return { title: 'Copy block-nodes.json to consensus nodes', - task: async ({config: {priorityMapping, namespace}}): Promise => { + task: async ({config: {priorityMapping}}): Promise => { const nodeAliases: string[] = Object.keys(priorityMapping); const filteredConsensusNodes: ConsensusNode[] = this.remoteConfig @@ -260,13 +284,24 @@ export class BlockNodeCommand extends BaseCommand { .filter((node): boolean => nodeAliases.includes(node.name)); for (const node of filteredConsensusNodes) { - await NetworkCommand.createAndCopyBlockNodeJsonFileForConsensusNode( - node, - namespace, - this.logger, - this.k8Factory, - this.remoteConfig, - ); + await NetworkCommand.createAndCopyBlockNodeJsonFileForConsensusNode(node, this.logger, this.k8Factory); + } + }, + }; + } + + private updateConsensusNodesPostGenesisForExternal(): SoloListrTask { + return { + title: 'Copy block-nodes.json to consensus nodes', + task: async ({config: {priorityMapping}}): Promise => { + const nodeAliases: string[] = Object.keys(priorityMapping); + + const filteredConsensusNodes: ConsensusNode[] = this.remoteConfig + .getConsensusNodes() + .filter((node): boolean => nodeAliases.includes(node.name)); + + for (const node of filteredConsensusNodes) { + await NetworkCommand.createAndCopyBlockNodeJsonFileForConsensusNode(node, this.logger, this.k8Factory); } }, }; @@ -287,6 +322,45 @@ export class BlockNodeCommand extends BaseCommand { }; } + private updateConsensusNodesInRemoteConfigForExternalBlockNode(): SoloListrTask { + return { + title: 'Update consensus nodes in remote config', + task: async ({config: {newExternalBlockNodeComponent, priorityMapping}}): Promise => { + const state: DeploymentStateSchema = this.remoteConfig.configuration.state; + const nodeAliases: string[] = Object.keys(priorityMapping); + + for (const node of state.consensusNodes.filter((node): boolean => + nodeAliases.includes(Templates.renderNodeAliasFromNumber(node.metadata.id)), + )) { + const priority: number = priorityMapping[Templates.renderNodeAliasFromNumber(node.metadata.id)]; + + node.externalBlockNodeMap.push([newExternalBlockNodeComponent.id, priority]); + } + + this.remoteConfig.configuration.state.consensusNodes = state.consensusNodes; + + await this.remoteConfig.persist(); + }, + }; + } + + private handleConsensusNodeUpdatingForExternalBlockNode(): SoloListrTask { + return { + title: 'Update consensus nodes', + task: (_, task): SoloListr => { + const subTasks: SoloListrTask[] = [ + this.updateConsensusNodesInRemoteConfigForExternalBlockNode(), + ]; + + if (this.remoteConfig.configuration.state.ledgerPhase !== LedgerPhase.UNINITIALIZED) { + subTasks.push(this.updateConsensusNodesPostGenesisForExternal()); + } + + return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT); + }, + }; + } + public async add(argv: ArgvStruct): Promise { // eslint-disable-next-line @typescript-eslint/typedef,unicorn/no-this-assignment const self = this; @@ -716,6 +790,77 @@ export class BlockNodeCommand extends BaseCommand { return true; } + public async addExternal(argv: ArgvStruct): Promise { + let lease: Lock; + + const tasks: SoloListr = this.taskList.newTaskList( + [ + { + title: 'Initialize', + task: async (context_, task): Promise> => { + await this.localConfig.load(); + await this.remoteConfig.loadAndValidate(argv); + lease = await this.leaseManager.create(); + + this.configManager.update(argv); + + flags.disablePrompts(BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.optional); + + const allFlags: CommandFlag[] = [ + ...BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.required, + ...BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.optional, + ]; + + await this.configManager.executePrompt(task, allFlags); + + const config: BlockNodeAddExternalConfigClass = this.configManager.getConfig( + BlockNodeCommand.ADD_EXTERNAL_CONFIGS_NAME, + allFlags, + ) as BlockNodeAddExternalConfigClass; + + context_.config = config; + + config.clusterRef = this.getClusterReference(); + config.context = this.getClusterContext(config.clusterRef); + config.namespace = await this.getNamespace(task); + + config.priorityMapping = Templates.parseBlockNodePriorityMapping( + config.priorityMapping as any, + this.remoteConfig.getConsensusNodes(), + ); + + const id: ComponentId = this.remoteConfig.configuration.state.externalBlockNodes.length + 1; + const [address, port] = Templates.parseExternalBlockAddress(config.externalBlockNodeAddress); + config.newExternalBlockNodeComponent = new ExternalBlockNodeStateSchema(id, address, port); + + return ListrLock.newAcquireLockTask(lease, task); + }, + }, + this.addExternalBlockNodeComponent(), + this.handleConsensusNodeUpdatingForExternalBlockNode(), + ], + constants.LISTR_DEFAULT_OPTIONS.DEFAULT, + undefined, + 'block node add', + ); + + if (tasks.isRoot()) { + try { + await tasks.run(); + } catch (error) { + throw new SoloError(`Error deploying block node: ${error.message}`, error); + } finally { + await lease?.release(); + } + } else { + this.taskList.registerCloseFunction(async (): Promise => { + await lease?.release(); + }); + } + + return true; + } + /** * Gives the port used for liveness check based on the chart version and image tag (if set) */ @@ -777,6 +922,19 @@ export class BlockNodeCommand extends BaseCommand { }; } + /** Adds the block node component to remote config. */ + private addExternalBlockNodeComponent(): SoloListrTask { + return { + title: 'Add external block node component in remote config', + skip: (): boolean => !this.remoteConfig.isLoaded(), + task: async ({config: {newExternalBlockNodeComponent}}): Promise => { + this.remoteConfig.configuration.state.externalBlockNodes.push(newExternalBlockNodeComponent); + + await this.remoteConfig.persist(); + }, + }; + } + /** Adds the block node component to remote config. */ private removeBlockNodeComponent(): SoloListrTask { return { diff --git a/src/commands/command-definitions/block-command-definition.ts b/src/commands/command-definitions/block-command-definition.ts index d09bed6dd..c48f09809 100644 --- a/src/commands/command-definitions/block-command-definition.ts +++ b/src/commands/command-definitions/block-command-definition.ts @@ -34,6 +34,8 @@ export class BlockCommandDefinition extends BaseCommandDefinition { public static readonly NODE_DESTROY = 'destroy'; public static readonly NODE_UPGRADE = 'upgrade'; + public static readonly NODE_ADD_EXTERNAL = 'add-external'; + public static readonly ADD_COMMAND: string = `${BlockCommandDefinition.COMMAND_NAME} ${BlockCommandDefinition.NODE_SUBCOMMAND_NAME} ${BlockCommandDefinition.NODE_ADD}` as const; public static readonly DESTROY_COMMAND: string = @@ -58,7 +60,6 @@ export class BlockCommandDefinition extends BaseCommandDefinition { this.blockNodeCommand.add, BlockNodeCommand.ADD_FLAGS_LIST, [constants.HELM, constants.KUBECTL], - false, ), ) .addSubcommand( @@ -70,7 +71,6 @@ export class BlockCommandDefinition extends BaseCommandDefinition { this.blockNodeCommand.destroy, BlockNodeCommand.DESTROY_FLAGS_LIST, [constants.HELM, constants.KUBECTL], - false, ), ) .addSubcommand( @@ -82,7 +82,17 @@ export class BlockCommandDefinition extends BaseCommandDefinition { this.blockNodeCommand.upgrade, BlockNodeCommand.UPGRADE_FLAGS_LIST, [constants.HELM, constants.KUBECTL], - false, + ), + ) + .addSubcommand( + new Subcommand( + BlockCommandDefinition.NODE_ADD_EXTERNAL, + 'Add an external block node for the specified deployment. ' + + 'You can specify the priority and consensus nodes to which to connect or use the default settings.', + this.blockNodeCommand, + this.blockNodeCommand.addExternal, + BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST, + [constants.HELM, constants.KUBECTL], ), ), ) diff --git a/src/commands/flags.ts b/src/commands/flags.ts index 294b18119..18bcc72d0 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -1189,6 +1189,18 @@ export class Flags { prompt: undefined, }; + public static readonly externalBlockNodeAddress: CommandFlag = { + constName: 'externalBlockNodeAddress', + name: 'address', + definition: { + describe: + "Configure external block node addresses, port can be provided after ':'" + + `(default port: ${constants.BLOCK_NODE_PORT})`, + type: 'string', + }, + prompt: undefined, + }; + public static readonly applicationProperties: CommandFlag = { constName: 'applicationProperties', name: 'application-properties', @@ -2953,6 +2965,7 @@ export class Flags { Flags.blockNodeChartVersion, Flags.blockNodeCfg, Flags.priorityMapping, + Flags.externalBlockNodeAddress, Flags.realm, Flags.shard, Flags.username, diff --git a/src/commands/network.ts b/src/commands/network.ts index dd90cd345..530a9b939 100644 --- a/src/commands/network.ts +++ b/src/commands/network.ts @@ -67,7 +67,6 @@ import {Secret} from '../integration/kube/resources/secret/secret.js'; import * as versions from '../../version.js'; import {SoloLogger} from '../core/logging/solo-logger.js'; import {K8Factory} from '../integration/kube/k8-factory.js'; -import {RemoteConfigRuntimeStateApi} from '../business/runtime-state/api/remote-config-runtime-state-api.js'; import {K8Helper} from '../business/utils/k8-helper.js'; export interface NetworkDeployConfigClass { @@ -981,16 +980,19 @@ export class NetworkCommand extends BaseCommand { }, { title: 'Copy gRPC TLS Certificates', - task: (context_, parentTask): SoloListr => + task: ( + {config: {grpcTlsCertificatePath, grpcWebTlsCertificatePath, grpcTlsKeyPath, grpcWebTlsKeyPath}}, + parentTask, + ): SoloListr => this.certificateManager.buildCopyTlsCertificatesTasks( parentTask, - context_.config.grpcTlsCertificatePath, - context_.config.grpcWebTlsCertificatePath, - context_.config.grpcTlsKeyPath, - context_.config.grpcWebTlsKeyPath, + grpcTlsCertificatePath, + grpcWebTlsCertificatePath, + grpcTlsKeyPath, + grpcWebTlsKeyPath, ), - skip: (context_): boolean => - !context_.config.grpcTlsCertificatePath && !context_.config.grpcWebTlsCertificatePath, + skip: ({config: {grpcTlsCertificatePath, grpcWebTlsCertificatePath}}): boolean => + !grpcTlsCertificatePath && !grpcWebTlsCertificatePath, }, { title: 'Prepare staging directory', @@ -999,22 +1001,20 @@ export class NetworkCommand extends BaseCommand { [ { title: 'Copy Gossip keys to staging', - task: (context_): void => { - const config: NetworkDeployConfigClass = context_.config; - this.keyManager.copyGossipKeysToStaging(config.keysDir, config.stagingKeysDir, config.nodeAliases); + task: ({config: {keysDir, stagingKeysDir, nodeAliases}}): void => { + this.keyManager.copyGossipKeysToStaging(keysDir, stagingKeysDir, nodeAliases); }, }, { title: 'Copy gRPC TLS keys to staging', - task: (context_): void => { - const config: NetworkDeployConfigClass = context_.config; - for (const nodeAlias of config.nodeAliases) { + task: ({config: {nodeAliases, keysDir, stagingKeysDir}}): void => { + for (const nodeAlias of nodeAliases) { const tlsKeyFiles: PrivateKeyAndCertificateObject = this.keyManager.prepareTlsKeyFilePaths( nodeAlias, - config.keysDir, + keysDir, ); - this.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir); + this.keyManager.copyNodeKeysToStaging(tlsKeyFiles, stagingKeysDir); } }, }, @@ -1025,12 +1025,10 @@ export class NetworkCommand extends BaseCommand { }, { title: 'Copy node keys to secrets', - task: (context_, parentTask): SoloListr => { - const config: NetworkDeployConfigClass = context_.config; - + task: ({config: {stagingDir, consensusNodes, contexts}}, parentTask): SoloListr => { // set up the subtasks return parentTask.newListr( - this.platformInstaller.copyNodeKeys(config.stagingDir, config.consensusNodes, config.contexts), + this.platformInstaller.copyNodeKeys(stagingDir, consensusNodes, contexts), constants.LISTR_DEFAULT_OPTIONS.WITH_CONCURRENCY, ); }, @@ -1055,19 +1053,20 @@ export class NetworkCommand extends BaseCommand { }, { title: `Install chart '${constants.SOLO_DEPLOYMENT_CHART}'`, - task: async (context_): Promise => { - const config: NetworkDeployConfigClass = context_.config; - for (const [clusterReference] of config.clusterRefs) { + task: async ({config}): Promise => { + const {namespace, clusterRefs, valuesArgMap, chartDirectory} = config; + + for (const [clusterReference] of clusterRefs) { const isInstalled: boolean = await this.chartManager.isChartInstalled( - config.namespace, + namespace, constants.SOLO_DEPLOYMENT_CHART, - config.clusterRefs.get(clusterReference), + clusterRefs.get(clusterReference), ); if (isInstalled) { await this.chartManager.uninstall( - config.namespace, + namespace, constants.SOLO_DEPLOYMENT_CHART, - config.clusterRefs.get(clusterReference), + clusterRefs.get(clusterReference), ); config.isUpgrade = true; } @@ -1079,13 +1078,13 @@ export class NetworkCommand extends BaseCommand { ); await this.chartManager.upgrade( - config.namespace, + namespace, constants.SOLO_DEPLOYMENT_CHART, constants.SOLO_DEPLOYMENT_CHART, - context_.config.chartDirectory || constants.SOLO_TESTING_CHART_URL, + chartDirectory || constants.SOLO_TESTING_CHART_URL, config.soloChartVersion, - config.valuesArgMap[clusterReference], - config.clusterRefs.get(clusterReference), + valuesArgMap[clusterReference], + clusterRefs.get(clusterReference), ); showVersionBanner(this.logger, constants.SOLO_DEPLOYMENT_CHART, config.soloChartVersion); } @@ -1095,12 +1094,11 @@ export class NetworkCommand extends BaseCommand { { title: 'Check for load balancer', skip: ({config: {loadBalancerEnabled}}): boolean => loadBalancerEnabled === false, - task: (context_, task): SoloListr => { + task: ({config: {consensusNodes, namespace}}, task): SoloListr => { const subTasks: SoloListrTask[] = []; - const config: NetworkDeployConfigClass = context_.config; //Add check for network node service to be created and load balancer to be assigned (if load balancer is enabled) - for (const consensusNode of config.consensusNodes) { + for (const consensusNode of consensusNodes) { subTasks.push({ title: `Load balancer is assigned for: ${chalk.yellow(consensusNode.name)}, cluster: ${chalk.yellow(consensusNode.cluster)}`, task: async (): Promise => { @@ -1111,7 +1109,7 @@ export class NetworkCommand extends BaseCommand { svc = await this.k8Factory .getK8(consensusNode.context) .services() - .list(config.namespace, [ + .list(namespace, [ `solo.hedera.com/node-id=${consensusNode.nodeId},solo.hedera.com/type=network-node-svc`, ]); @@ -1288,15 +1286,13 @@ export class NetworkCommand extends BaseCommand { { title: `Copy ${constants.BLOCK_NODES_JSON_FILE}`, skip: ({config: {blockNodeComponents}}): boolean => blockNodeComponents.length === 0, - task: async ({config: {namespace, consensusNodes}}): Promise => { + task: async ({config: {consensusNodes}}): Promise => { try { for (const consensusNode of consensusNodes) { await NetworkCommand.createAndCopyBlockNodeJsonFileForConsensusNode( consensusNode, - namespace, this.logger, this.k8Factory, - this.remoteConfig, ); } } catch (error) { @@ -1331,27 +1327,26 @@ export class NetworkCommand extends BaseCommand { /** * @param consensusNode - the targeted consensus node - * @param namespace * @param logger * @param k8Factory - * @param remoteConfig */ public static async createAndCopyBlockNodeJsonFileForConsensusNode( consensusNode: ConsensusNode, - namespace: NamespaceName, logger: SoloLogger, k8Factory: K8Factory, - remoteConfig: RemoteConfigRuntimeStateApi, ): Promise { - const {nodeId, context, name: nodeAlias, blockNodeMap} = consensusNode; - - const blockNodeIds: Set = new Set(blockNodeMap.map(([id]): ComponentId => id)); + const { + nodeId, + context, + name: nodeAlias, + blockNodeMap, + externalBlockNodeMap, + namespace: namespaceNameAsString, + } = consensusNode; - const blockNodeComponents: BlockNodeStateSchema[] = remoteConfig.configuration.state.blockNodes.filter( - (blockNode): boolean => blockNodeIds.has(blockNode.metadata.id), - ); + const namespace: NamespaceName = NamespaceName.of(namespaceNameAsString); - const blockNodesJsonData: string = new BlockNodesJsonWrapper(blockNodeMap, blockNodeComponents).toJSON(); + const blockNodesJsonData: string = new BlockNodesJsonWrapper(blockNodeMap, externalBlockNodeMap).toJSON(); const blockNodesJsonFilename: string = `${constants.BLOCK_NODES_JSON_FILE.replace('.json', '')}-${nodeId}.json`; const blockNodesJsonPath: string = PathEx.join(constants.SOLO_CACHE_DIR, blockNodesJsonFilename); @@ -1531,16 +1526,14 @@ export class NetworkCommand extends BaseCommand { return { title: 'Add node and proxies to remote config', skip: (): boolean => !this.remoteConfig.isLoaded(), - task: async (context_): Promise => { - const {namespace} = context_.config; - - for (const consensusNode of context_.config.consensusNodes) { + task: async ({config: {consensusNodes, namespace, isUpgrade, releaseTag}}): Promise => { + for (const consensusNode of consensusNodes) { const componentId: ComponentId = Templates.renderComponentIdFromNodeAlias(consensusNode.name); const clusterReference: ClusterReferenceName = consensusNode.cluster; this.remoteConfig.configuration.components.changeNodePhase(componentId, DeploymentPhase.REQUESTED); - if (context_.config.isUpgrade) { + if (isUpgrade) { this.logger.info('Do not add envoy and haproxy components again during upgrade'); } else { // do not add new envoy or haproxy components if they already exist @@ -1555,13 +1548,11 @@ export class NetworkCommand extends BaseCommand { ); } } - if (context_.config.releaseTag) { + if (releaseTag) { // update the solo chart version to match the deployed version - this.remoteConfig.updateComponentVersion( - ComponentTypes.ConsensusNode, - new SemVer(context_.config.releaseTag), - ); + this.remoteConfig.updateComponentVersion(ComponentTypes.ConsensusNode, new SemVer(releaseTag)); } + await this.remoteConfig.persist(); }, }; diff --git a/src/commands/node/tasks.ts b/src/commands/node/tasks.ts index 237225334..79e0576b2 100644 --- a/src/commands/node/tasks.ts +++ b/src/commands/node/tasks.ts @@ -90,6 +90,8 @@ import { type Context, type DeploymentName, type Optional, + type Realm, + type Shard, type SoloListr, type SoloListrTask, type SoloListrTaskWrapper, @@ -180,8 +182,8 @@ export class NodeCommandTasks { } private getFileUpgradeId(deploymentName: DeploymentName): FileId { - const realm = this.localConfig.configuration.realmForDeployment(deploymentName); - const shard = this.localConfig.configuration.shardForDeployment(deploymentName); + const realm: Realm = this.localConfig.configuration.realmForDeployment(deploymentName); + const shard: Shard = this.localConfig.configuration.shardForDeployment(deploymentName); return FileId.fromString(entityId(shard, realm, constants.UPGRADE_FILE_ID_NUM)); } @@ -190,19 +192,21 @@ export class NodeCommandTasks { // also the platform zip file is ~80Mb in size requiring a lot of transactions since the max // transaction size is 6Kb and in practice we need to send the file as 4Kb chunks. // Note however that in DAB phase-2, we won't need to trigger this fake upgrade process - const zipper = new Zippy(this.logger); - const upgradeConfigDirectory = PathEx.join(stagingDirectory, 'mock-upgrade', 'data', 'config'); + const zipper: Zippy = new Zippy(this.logger); + const upgradeConfigDirectory: string = PathEx.join(stagingDirectory, 'mock-upgrade', 'data', 'config'); if (!fs.existsSync(upgradeConfigDirectory)) { fs.mkdirSync(upgradeConfigDirectory, {recursive: true}); } // bump field hedera.config.version or use the version passed in - const fileBytes = fs.readFileSync(PathEx.joinWithRealPath(stagingDirectory, 'templates', 'application.properties')); - const lines = fileBytes.toString().split('\n'); - const newLines = []; + const fileBytes: Buffer = fs.readFileSync( + PathEx.joinWithRealPath(stagingDirectory, 'templates', 'application.properties'), + ); + const lines: string[] = fileBytes.toString().split('\n'); + const newLines: string[] = []; for (let line of lines) { line = line.trim(); - const parts = line.split('='); + const parts: string[] = line.split('='); if (parts.length === 2) { if (parts[0] === 'hedera.config.version') { const version: string = upgradeVersion ?? String(Number.parseInt(parts[1]) + 1); @@ -1591,7 +1595,7 @@ export class NodeCommandTasks { fs.writeFileSync(genesisNetworkJson, genesisNetworkData.toJSON()); } - public prepareStagingDirectory(nodeAliasesProperty: string) { + public prepareStagingDirectory(nodeAliasesProperty: string): AnyListrContext { return { title: 'Prepare staging directory', task: (context_, task) => { @@ -2403,13 +2407,14 @@ export class NodeCommandTasks { new ConsensusNode( config.nodeAlias, Templates.nodeIdFromNodeAlias(config.nodeAlias), - config.namespace, + config.namespace.name, undefined, context, config.consensusNodes[0].dnsBaseDomain, config.consensusNodes[0].dnsConsensusNodePattern, Templates.renderFullyQualifiedNetworkSvcName(config.namespace, config.nodeAlias), [], + [], ), k8, +constants.HEDERA_NODE_EXTERNAL_GOSSIP_PORT, @@ -3316,6 +3321,7 @@ export class NodeCommandTasks { cluster.dnsConsensusNodePattern, ), [], + [], ), ); } diff --git a/src/core/block-nodes-json-wrapper.ts b/src/core/block-nodes-json-wrapper.ts index 6361a77ae..db369bb48 100644 --- a/src/core/block-nodes-json-wrapper.ts +++ b/src/core/block-nodes-json-wrapper.ts @@ -11,6 +11,7 @@ import {inject} from 'tsyringe-neo'; import {InjectTokens} from './dependency-injection/inject-tokens.js'; import {patchInject} from './dependency-injection/container-helper.js'; import {type RemoteConfigRuntimeStateApi} from '../business/runtime-state/api/remote-config-runtime-state-api.js'; +import {ExternalBlockNodeStateSchema} from '../data/schema/model/remote/state/external-block-node-state-schema.js'; type BlockNodeConnectionData = | { @@ -29,26 +30,40 @@ interface BlockNodesJsonStructure { blockItemBatchSize: number; } +/** + * Wrapper used to generate `block-nodes.json` file + * for the consensus node used to configure block node connections. + */ export class BlockNodesJsonWrapper implements ToJSON { private readonly remoteConfig: RemoteConfigRuntimeStateApi; + private readonly blockNodes: BlockNodeStateSchema[]; + private readonly externalBlockNodes: ExternalBlockNodeStateSchema[]; public constructor( private readonly blockNodeMap: PriorityMapping[], - private readonly blockNodeComponents: BlockNodeStateSchema[], + private readonly externalBlockNodeMap: PriorityMapping[], @inject(InjectTokens.RemoteConfigRuntimeState) remoteConfig?: RemoteConfigRuntimeStateApi, ) { this.remoteConfig = patchInject(remoteConfig, InjectTokens.RemoteConfigRuntimeState, this.constructor.name); + this.blockNodes = this.remoteConfig.configuration.state.blockNodes; + this.externalBlockNodes = this.remoteConfig.configuration.state.externalBlockNodes; } public toJSON(): string { return JSON.stringify(this.buildBlockNodesJsonStructure()); } - public buildBlockNodesJsonStructure(): BlockNodesJsonStructure { + private buildBlockNodesJsonStructure(): BlockNodesJsonStructure { + // Figure out field name for port + const useLegacyPortName: boolean = lt( + this.remoteConfig.configuration.versions.consensusNode, + versions.MINIMUM_HIERO_CONSENSUS_NODE_VERSION_FOR_LEGACY_PORT_NAME_FOR_BLOCK_NODES_JSON_FILE, + ); + const blockNodeConnectionData: BlockNodeConnectionData[] = []; for (const [id, priority] of this.blockNodeMap) { - const blockNodeComponent: BlockNodeStateSchema = this.blockNodeComponents.find( + const blockNodeComponent: BlockNodeStateSchema = this.blockNodes.find( (component): boolean => component.metadata.id === id, ); @@ -70,11 +85,18 @@ export class BlockNodesJsonWrapper implements ToJSON { const port: number = useLegacyPort ? constants.BLOCK_NODE_PORT_LEGACY : constants.BLOCK_NODE_PORT; - // Figure out field name for port - const useLegacyPortName: boolean = lt( - this.remoteConfig.configuration.versions.consensusNode, - versions.MINIMUM_HIERO_CONSENSUS_NODE_VERSION_FOR_LEGACY_PORT_NAME_FOR_BLOCK_NODES_JSON_FILE, + blockNodeConnectionData.push( + useLegacyPortName ? {address, port, priority} : {address, streamingPort: port, priority}, ); + } + + for (const [id, priority] of this.externalBlockNodeMap) { + const blockNodeComponent: ExternalBlockNodeStateSchema = this.externalBlockNodes.find( + (component): boolean => component.id === id, + ); + + const address: string = blockNodeComponent.address; + const port: number = blockNodeComponent.port; blockNodeConnectionData.push( useLegacyPortName ? {address, port, priority} : {address, streamingPort: port, priority}, diff --git a/src/core/config/remote/api/component-factory-api.ts b/src/core/config/remote/api/component-factory-api.ts index ecd4cfd91..464f1ce33 100644 --- a/src/core/config/remote/api/component-factory-api.ts +++ b/src/core/config/remote/api/component-factory-api.ts @@ -41,6 +41,7 @@ export interface ComponentFactoryApi { phase: DeploymentPhase.REQUESTED | DeploymentPhase.STARTED, portForwardConfigs?: PortForwardConfig[], blockNodeMap?: PriorityMapping[], + externalBlockNodeIds?: PriorityMapping[], ): ConsensusNodeStateSchema; createConsensusNodeComponentsFromNodeIds( diff --git a/src/core/config/remote/component-factory.ts b/src/core/config/remote/component-factory.ts index 7b87b86f2..08f83c210 100644 --- a/src/core/config/remote/component-factory.ts +++ b/src/core/config/remote/component-factory.ts @@ -84,6 +84,7 @@ export class ComponentFactory implements ComponentFactoryApi { phase: DeploymentPhase.REQUESTED | DeploymentPhase.STARTED, portForwardConfigs?: PortForwardConfig[], blockNodeMap: PriorityMapping[] = [], + externalBlockNodeMap: PriorityMapping[] = [], ): ConsensusNodeStateSchema { const metadata: ComponentStateMetadataSchema = new ComponentStateMetadataSchema( id, @@ -93,7 +94,7 @@ export class ComponentFactory implements ComponentFactoryApi { portForwardConfigs, ); - return new ConsensusNodeStateSchema(metadata, blockNodeMap); + return new ConsensusNodeStateSchema(metadata, blockNodeMap, externalBlockNodeMap); } public createConsensusNodeComponentsFromNodeIds( diff --git a/src/core/model/consensus-node.ts b/src/core/model/consensus-node.ts index 202ed2261..4cfb3d41a 100644 --- a/src/core/model/consensus-node.ts +++ b/src/core/model/consensus-node.ts @@ -1,19 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 -import {type NodeAlias} from '../../types/aliases.js'; -import {type ClusterReferenceName, type PriorityMapping} from '../../types/index.js'; +import {type NodeAlias, type NodeId} from '../../types/aliases.js'; +import { + type ClusterReferenceName, + type Context, + type NamespaceNameAsString, + type PriorityMapping, +} from '../../types/index.js'; export class ConsensusNode { public constructor( public readonly name: NodeAlias, - public readonly nodeId: number, - public readonly namespace: string, + public readonly nodeId: NodeId, + public readonly namespace: NamespaceNameAsString, public readonly cluster: ClusterReferenceName, - public readonly context: string, + public readonly context: Context, public readonly dnsBaseDomain: string, public readonly dnsConsensusNodePattern: string, public readonly fullyQualifiedDomainName: string, public readonly blockNodeMap: PriorityMapping[], + public readonly externalBlockNodeMap: PriorityMapping[], ) { if (!context) { throw new Error(`ConsensusNode context cannot be empty or null. Call stack: ${new Error().stack}`); diff --git a/src/core/profile-manager.ts b/src/core/profile-manager.ts index b9edbe817..29cb6f36a 100644 --- a/src/core/profile-manager.ts +++ b/src/core/profile-manager.ts @@ -257,7 +257,6 @@ export class ProfileManager { throw new SoloError(`Configuration file path is missing for: ${flag.name}`); } - // use the default flag value to rename the file provided by the user const fileName: string = path.basename(filePath); const destinationPath: string = PathEx.join(stagingDirectory, 'templates', fileName); this.logger.debug(`Copying configuration file to staging: ${filePath} -> ${destinationPath}`); @@ -297,18 +296,24 @@ export class ProfileManager { PathEx.joinWithRealPath(stagingDirectory, 'templates', 'application.env'), yamlRoot, ); + try { - if (this.remoteConfig.configuration.state.blockNodes.length === 0) { + if ( + this.remoteConfig.configuration.state.blockNodes.length === 0 && + this.remoteConfig.configuration.state.externalBlockNodes.length === 0 + ) { return; } } catch { + // quick fix for tests where field on remote config are unaccessible return; } - const blockNodes: BlockNodeStateSchema[] = this.remoteConfig.configuration.components.state.blockNodes; - for (const node of consensusNodes) { - const blockNodesJsonData: string = new BlockNodesJsonWrapper(node.blockNodeMap, blockNodes).toJSON(); + const blockNodesJsonData: string = new BlockNodesJsonWrapper( + node.blockNodeMap, + node.externalBlockNodeMap, + ).toJSON(); let nodeIndex: number = 0; diff --git a/src/core/templates.ts b/src/core/templates.ts index d3b9fb047..f1b1edd8c 100644 --- a/src/core/templates.ts +++ b/src/core/templates.ts @@ -407,6 +407,11 @@ export class Templates { return [`solo.hedera.com/node-name=${nodeAlias}`, 'solo.hedera.com/type=network-node']; } + public static parseExternalBlockAddress(raw: string): [string, number] { + const [address, port] = raw.includes(':') ? raw.split(':') : [raw, constants.BLOCK_NODE_PORT]; + return [address, +port]; + } + public static parseBlockNodePriorityMapping(rawString: string, nodes: ConsensusNode[]): Record { const mapping: Record = {}; @@ -420,8 +425,6 @@ export class Templates { // eslint-disable-next-line prefer-const let [nodeAlias, priority] = data.split('=') as [NodeAlias, number | undefined]; - priority = !priority && nodeAliasesToPriorityMapping.length === 1 ? 2 : 1; - mapping[nodeAlias] = +priority || 1; } diff --git a/src/data/schema/model/remote/deployment-state-schema.ts b/src/data/schema/model/remote/deployment-state-schema.ts index 58c051136..d67bdcf71 100644 --- a/src/data/schema/model/remote/deployment-state-schema.ts +++ b/src/data/schema/model/remote/deployment-state-schema.ts @@ -12,6 +12,7 @@ import {ExplorerStateSchema} from './state/explorer-state-schema.js'; import {BlockNodeStateSchema} from './state/block-node-state-schema.js'; import {ComponentIdsSchema} from './state/component-ids-schema.js'; import {DeploymentStateStructure} from './interfaces/deployment-state-structure.js'; +import {ExternalBlockNodeStateSchema} from './state/external-block-node-state-schema.js'; @Exclude() export class DeploymentStateSchema implements DeploymentStateStructure { @@ -51,6 +52,10 @@ export class DeploymentStateSchema implements DeploymentStateStructure { @Type((): typeof ExplorerStateSchema => ExplorerStateSchema) public explorers: ExplorerStateSchema[]; + @Expose() + @Type((): typeof ExternalBlockNodeStateSchema => ExternalBlockNodeStateSchema) + public externalBlockNodes: ExternalBlockNodeStateSchema[]; + public constructor( ledgerPhase?: LedgerPhase, componentIds?: ComponentIdsSchema, @@ -61,6 +66,7 @@ export class DeploymentStateSchema implements DeploymentStateStructure { haProxies?: HaProxyStateSchema[], envoyProxies?: EnvoyProxyStateSchema[], explorers?: ExplorerStateSchema[], + externalBlockNodes?: ExternalBlockNodeStateSchema[], ) { this.ledgerPhase = ledgerPhase; this.componentIds = componentIds || new ComponentIdsSchema(); @@ -71,5 +77,6 @@ export class DeploymentStateSchema implements DeploymentStateStructure { this.haProxies = haProxies || []; this.envoyProxies = envoyProxies || []; this.explorers = explorers || []; + this.externalBlockNodes = externalBlockNodes || []; } } diff --git a/src/data/schema/model/remote/state/consensus-node-state-schema.ts b/src/data/schema/model/remote/state/consensus-node-state-schema.ts index b9ac235c6..c879abb46 100644 --- a/src/data/schema/model/remote/state/consensus-node-state-schema.ts +++ b/src/data/schema/model/remote/state/consensus-node-state-schema.ts @@ -10,8 +10,16 @@ export class ConsensusNodeStateSchema extends BaseStateSchema { @Expose() public blockNodeMap: PriorityMapping[]; - public constructor(metadata?: ComponentStateMetadataSchema, blockNodeMap?: PriorityMapping[]) { + @Expose() + public externalBlockNodeMap: PriorityMapping[]; + + public constructor( + metadata?: ComponentStateMetadataSchema, + blockNodeMap?: PriorityMapping[], + externalBlockNodeMap?: PriorityMapping[], + ) { super(metadata); this.blockNodeMap = blockNodeMap || []; + this.externalBlockNodeMap = externalBlockNodeMap || []; } } diff --git a/src/data/schema/model/remote/state/external-block-node-state-schema.ts b/src/data/schema/model/remote/state/external-block-node-state-schema.ts new file mode 100644 index 000000000..623e9eefa --- /dev/null +++ b/src/data/schema/model/remote/state/external-block-node-state-schema.ts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +import {Exclude, Expose} from 'class-transformer'; +import {type ComponentId} from '../../../../../types/index.js'; + +@Exclude() +export class ExternalBlockNodeStateSchema { + @Expose() + public id: number; + + @Expose() + public address: string; + + @Expose() + public port: number; + + public constructor(id?: ComponentId, address?: string, port?: number) { + this.id = id; + this.address = address; + this.port = port; + } +} diff --git a/test/e2e/commands/block-node.test.ts b/test/e2e/commands/block-node.test.ts index cb01b2ba8..824aafbb0 100644 --- a/test/e2e/commands/block-node.test.ts +++ b/test/e2e/commands/block-node.test.ts @@ -81,15 +81,16 @@ new EndToEndTestSuiteBuilder() BlockNodeTest.add(options, ['node2']); - it('Wait for block node 2 to come online', async (): Promise => { - testLogger.showUser('Sleeping for 2 minutes to allow block node 2 to sync.'); - await sleep(Duration.ofMinutes(2)); - testLogger.showUser('Finished sleeping.'); - }).timeout(Duration.ofMinutes(3).toMillis()); - BlockNodeTest.verifyBlockNodesJson(options, 'node1', [1], [2]); BlockNodeTest.verifyBlockNodesJson(options, 'node2', [2]); + BlockNodeTest.addExternal(options, 'test-address-1', ['node1']); + BlockNodeTest.addExternal(options, 'test-address-2:3030', ['node2']); + + // External Block Nodes + BlockNodeTest.verifyBlockNodesJson(options, 'node1', [], [], 'test-address-1', constants.BLOCK_NODE_PORT); + BlockNodeTest.verifyBlockNodesJson(options, 'node2', [], [], 'test-address-2', 3030); + BlockNodeTest.destroy(options); describe('Should write log metrics', async (): Promise => { diff --git a/test/e2e/commands/tests/block-node-test.ts b/test/e2e/commands/tests/block-node-test.ts index 8790d944e..9b49256cf 100644 --- a/test/e2e/commands/tests/block-node-test.ts +++ b/test/e2e/commands/tests/block-node-test.ts @@ -57,6 +57,42 @@ export class BlockNodeTest extends BaseCommandTest { return argv; } + private static soloBlockNodeAddExternalArgv( + testName: string, + deployment: DeploymentName, + clusterReference: ClusterReferenceName, + address: string, + nodeAliases?: NodeAliases, + ): string[] { + const {newArgv, argvPushGlobalFlags, optionFromFlag} = BlockNodeTest; + + const argv: string[] = newArgv(); + argv.push( + BlockCommandDefinition.COMMAND_NAME, + BlockCommandDefinition.NODE_SUBCOMMAND_NAME, + BlockCommandDefinition.NODE_ADD_EXTERNAL, + optionFromFlag(Flags.deployment), + deployment, + optionFromFlag(Flags.clusterRef), + clusterReference, + optionFromFlag(Flags.externalBlockNodeAddress), + address, + ); + + if (nodeAliases !== undefined && nodeAliases.length > 0) { + const stringBuilder: string[] = []; + + for (const nodeAlias of nodeAliases) { + stringBuilder.push(`${nodeAlias}=1`); + } + + argv.push(optionFromFlag(Flags.priorityMapping), stringBuilder.join(',')); + } + + argvPushGlobalFlags(argv, testName, false, true); + return argv; + } + private static soloBlockNodeDestroyArgv( testName: string, deployment: DeploymentName, @@ -101,6 +137,18 @@ export class BlockNodeTest extends BaseCommandTest { }).timeout(Duration.ofMinutes(5).toMillis()); } + public static addExternal(options: BaseTestOptions, address: string, nodeAliases?: NodeAliases): void { + const {testName, deployment, clusterReferenceNameArray} = options; + + const {soloBlockNodeAddExternalArgv} = BlockNodeTest; + + it(`${testName}: block node add-external`, async (): Promise => { + await main( + soloBlockNodeAddExternalArgv(testName, deployment, clusterReferenceNameArray[0], address, nodeAliases), + ); + }).timeout(Duration.ofMinutes(5).toMillis()); + } + public static destroy(options: BaseTestOptions): void { const {testName, deployment, clusterReferenceNameArray} = options; const {soloBlockNodeDestroyArgv} = BlockNodeTest; @@ -142,6 +190,8 @@ export class BlockNodeTest extends BaseCommandTest { nodeAlias: NodeAlias, blockNodeIds: ComponentId[], excludedBlockNodeIds: ComponentId[] = [], + expectedExternalAddress?: string, + expectedExternalPort?: number, ): void { const {namespace, contexts, testName} = options; @@ -161,6 +211,14 @@ export class BlockNodeTest extends BaseCommandTest { for (const excludedBlockNodeId of excludedBlockNodeIds) { expect(output).to.not.include(`block-node-${excludedBlockNodeId}`); } + + if (expectedExternalAddress !== undefined) { + expect(output).to.include(expectedExternalAddress); + } + + if (expectedExternalPort !== undefined) { + expect(output).to.include(expectedExternalPort.toString()); + } }); } } diff --git a/test/unit/commands/base.test.ts b/test/unit/commands/base.test.ts index c0686b1c9..d131a01a7 100644 --- a/test/unit/commands/base.test.ts +++ b/test/unit/commands/base.test.ts @@ -171,6 +171,7 @@ describe('BaseCommand', () => { 'dnsConsensusNodePattern', 'fullyQualifiedDomainName', [], + [], ), new ConsensusNode( 'node2', @@ -182,6 +183,7 @@ describe('BaseCommand', () => { 'dnsConsensusNodePattern', 'fullyQualifiedDomainName', [], + [], ), ]; diff --git a/test/unit/commands/network.test.ts b/test/unit/commands/network.test.ts index 755b14a1a..ca959d31f 100644 --- a/test/unit/commands/network.test.ts +++ b/test/unit/commands/network.test.ts @@ -300,7 +300,9 @@ describe('NetworkCommand unit tests', (): void => { options.remoteConfig.getConsensusNodes = sinon .stub() - .returns([new ConsensusNode('node1', 0, 'solo-e2e', 'cluster', 'context-1', 'base', 'pattern', 'fqdn', [])]); + .returns([ + new ConsensusNode('node1', 0, 'solo-e2e', 'cluster', 'context-1', 'base', 'pattern', 'fqdn', [], []), + ]); options.remoteConfig.getContexts = sinon.stub().returns(['context-1']); const stubbedClusterReferences: ClusterReferences = new Map([['cluster', 'context1']]); diff --git a/test/unit/core/profile-manager.test.ts b/test/unit/core/profile-manager.test.ts index 6554cc788..266e6e3b8 100644 --- a/test/unit/core/profile-manager.test.ts +++ b/test/unit/core/profile-manager.test.ts @@ -46,6 +46,7 @@ describe('ProfileManager', () => { dnsConsensusNodePattern: 'network-{nodeAlias}-svc.{namespace}.svc', fullyQualifiedDomainName: 'network-node1-svc.test-namespace.svc.cluster.local', blockNodeMap: [], + externalBlockNodeMap: [], }, { name: 'node2', @@ -57,6 +58,7 @@ describe('ProfileManager', () => { dnsConsensusNodePattern: 'network-{nodeAlias}-svc.{namespace}.svc', fullyQualifiedDomainName: 'network-node2-svc.test-namespace.svc.cluster.local', blockNodeMap: [], + externalBlockNodeMap: [], }, { name: 'node3', @@ -68,6 +70,7 @@ describe('ProfileManager', () => { dnsConsensusNodePattern: 'network-{nodeAlias}-svc.{namespace}.svc', fullyQualifiedDomainName: 'network-node3-svc.test-namespace.svc.cluster.local', blockNodeMap: [], + externalBlockNodeMap: [], }, ]; diff --git a/test/unit/core/templates.test.ts b/test/unit/core/templates.test.ts index 207451421..5fcdca2e3 100644 --- a/test/unit/core/templates.test.ts +++ b/test/unit/core/templates.test.ts @@ -16,6 +16,7 @@ describe('core/templates', (): void => { dnsConsensusNodePattern: 'network-{nodeAlias}-svc.{namespace}.svc', fullyQualifiedDomainName: 'network-node1-svc.solo.svc.cluster.local', blockNodeMap: [], + externalBlockNodeMap: [], }, { name: 'node2', @@ -27,6 +28,7 @@ describe('core/templates', (): void => { dnsConsensusNodePattern: '{nodeId}.consensus.prod', fullyQualifiedDomainName: '2.consensus.prod.us-west-2.gcp.charlie.sphere', blockNodeMap: [], + externalBlockNodeMap: [], }, ];