From 0f63992d4dfe79737ff6c9b064d6878cf903bfcf Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 11:36:54 +0100 Subject: [PATCH 01/19] [refactor-runner-adxt-783] Started refactoring --- components/command/filemanager.go | 17 +++-- components/command/osCommand.go | 15 ++--- components/command/package.go | 11 ++- components/command/runner.go | 71 ++++++++++++++++++-- components/command/unixOSCommand.go | 8 +-- components/command/utils.go | 6 +- components/command/windowsOSCommand.go | 6 +- components/os/linux.go | 2 +- components/os/linux_packagemanagers.go | 8 +-- components/os/linux_servicemanagers.go | 8 +-- components/os/macos.go | 2 +- components/os/macos_packagemanagers.go | 2 +- components/os/macos_servicemanagers.go | 4 +- components/os/os.go | 8 +-- components/os/windows.go | 2 +- components/os/windows_servicemanagers.go | 4 +- components/remote/os.go | 2 +- scenarios/aws/microVMs/microvms/provision.go | 8 +-- scenarios/aws/microVMs/microvms/runner.go | 6 +- 19 files changed, 122 insertions(+), 68 deletions(-) diff --git a/components/command/filemanager.go b/components/command/filemanager.go index 580ad2b38..55f11d9d9 100644 --- a/components/command/filemanager.go +++ b/components/command/filemanager.go @@ -10,19 +10,18 @@ import ( "runtime" "strings" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) type FileManager struct { - runner *Runner + runner Runner command OSCommand } -func NewFileManager(runner *Runner) *FileManager { +func NewFileManager(runner Runner) *FileManager { return &FileManager{ runner: runner, - command: runner.osCommand, + command: runner.OsCommand(), } } @@ -31,13 +30,13 @@ func (fm *FileManager) IsPathAbsolute(path string) bool { } // CreateDirectoryFromPulumiString if it does not exist from directory name as a Pulumi String -func (fm *FileManager) CreateDirectoryFromPulumiString(name string, remotePath pulumi.String, useSudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (fm *FileManager) CreateDirectoryFromPulumiString(name string, remotePath pulumi.String, useSudo bool, opts ...pulumi.ResourceOption) (Command, error) { return fm.command.CreateDirectory(fm.runner, name, remotePath, useSudo, opts...) } // CreateDirectoryForFile if the directory does not exist // To avoid pulumi.URN collisions if multiple files use the same directory, use the full filePath as URN and path.Split out the folderPath for creation -func (fm *FileManager) CreateDirectoryForFile(remotePath string, useSudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (fm *FileManager) CreateDirectoryForFile(remotePath string, useSudo bool, opts ...pulumi.ResourceOption) (Command, error) { // if given just a directory path, path.Split returns "" as file // eg. path.Split("/a/b/c/") -> "/a/b/c/", "" folderPath, _ := path.Split(remotePath) @@ -45,12 +44,12 @@ func (fm *FileManager) CreateDirectoryForFile(remotePath string, useSudo bool, o } // CreateDirectory if it does not exist -func (fm *FileManager) CreateDirectory(remotePath string, useSudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (fm *FileManager) CreateDirectory(remotePath string, useSudo bool, opts ...pulumi.ResourceOption) (Command, error) { return fm.command.CreateDirectory(fm.runner, "create-directory-"+remotePath, pulumi.String(remotePath), useSudo, opts...) } // TempDirectory creates a temporary directory -func (fm *FileManager) TempDirectory(folderName string, opts ...pulumi.ResourceOption) (*remote.Command, string, error) { +func (fm *FileManager) TempDirectory(folderName string, opts ...pulumi.ResourceOption) (Command, string, error) { tempDir := path.Join(fm.command.GetTemporaryDirectory(), folderName) folderCmd, err := fm.CreateDirectory(tempDir, false, opts...) return folderCmd, tempDir, err @@ -59,7 +58,7 @@ func (fm *FileManager) TempDirectory(folderName string, opts ...pulumi.ResourceO // HomeDirectory creates a directory in home directory, if it does not exist // A home directory is a file system directory on a multi-user operating system containing files for a given user of the system. // It does not require sudo, using sudo in home directory allows to change default ownership and it is discouraged. -func (fm *FileManager) HomeDirectory(folderName string, opts ...pulumi.ResourceOption) (*remote.Command, string, error) { +func (fm *FileManager) HomeDirectory(folderName string, opts ...pulumi.ResourceOption) (Command, string, error) { homeDir := path.Join(fm.command.GetHomeDirectory(), folderName) folderCmd, err := fm.CreateDirectory(homeDir, false, opts...) return folderCmd, homeDir, err diff --git a/components/command/osCommand.go b/components/command/osCommand.go index b95136bfe..294433a7b 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -3,7 +3,6 @@ package command import ( "strings" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -13,11 +12,11 @@ type OSCommand interface { GetHomeDirectory() string CreateDirectory( - runner *Runner, + runner Runner, resourceName string, remotePath pulumi.StringInput, useSudo bool, - opts ...pulumi.ResourceOption) (*remote.Command, error) + opts ...pulumi.ResourceOption) (Command, error) BuildCommandString( command pulumi.StringInput, @@ -28,7 +27,7 @@ type OSCommand interface { IsPathAbsolute(path string) bool - NewCopyFile(runner *Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) } // ------------------------------ @@ -38,13 +37,13 @@ type OSCommand interface { const backupExtension = "pulumi.backup" func createDirectory( - runner *Runner, + runner Runner, name string, createCmd string, deleteCmd string, useSudo bool, opts ...pulumi.ResourceOption, -) (*remote.Command, error) { +) (Command, error) { // If the folder was previously created, make sure to delete it before creating it. opts = append(opts, pulumi.DeleteBeforeReplace(true)) return runner.Command(name, @@ -73,13 +72,13 @@ func buildCommandString( } func copyRemoteFile( - runner *Runner, + runner Runner, name string, createCommand pulumi.StringInput, deleteCommand pulumi.StringInput, useSudo bool, opts ...pulumi.ResourceOption, -) (*remote.Command, error) { +) (Command, error) { return runner.Command(name, &Args{ Create: createCommand, diff --git a/components/command/package.go b/components/command/package.go index 6051aa19a..e734934c5 100644 --- a/components/command/package.go +++ b/components/command/package.go @@ -5,14 +5,13 @@ import ( "github.com/DataDog/test-infra-definitions/common/namer" "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) type GenericPackageManager struct { namer namer.Namer - updateDBCommand *remote.Command - runner *Runner + updateDBCommand Command + runner Runner opts []pulumi.ResourceOption installCmd string updateCmd string @@ -20,7 +19,7 @@ type GenericPackageManager struct { } func NewGenericPackageManager( - runner *Runner, + runner Runner, name string, installCmd string, updateCmd string, @@ -37,7 +36,7 @@ func NewGenericPackageManager( return packageManager } -func (m *GenericPackageManager) Ensure(packageRef string, transform Transformer, checkBinary string, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (m *GenericPackageManager) Ensure(packageRef string, transform Transformer, checkBinary string, opts ...pulumi.ResourceOption) (Command, error) { opts = append(opts, m.opts...) if m.updateCmd != "" { updateDB, err := m.updateDB(opts) @@ -76,7 +75,7 @@ func (m *GenericPackageManager) Ensure(packageRef string, transform Transformer, return cmd, nil } -func (m *GenericPackageManager) updateDB(opts []pulumi.ResourceOption) (*remote.Command, error) { +func (m *GenericPackageManager) updateDB(opts []pulumi.ResourceOption) (Command, error) { if m.updateDBCommand != nil { return m.updateDBCommand, nil } diff --git a/components/command/runner.go b/components/command/runner.go index 9208a1c48..776253dfa 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -53,7 +53,27 @@ type runnerConfiguration struct { connection remote.ConnectionInput } -type Runner struct { +type Command interface { + pulumi.Resource +} + +var _ Command = &remote.Command{} +var _ Command = &local.Command{} + +type Runner interface { + Environment() config.Env + Namer() namer.Namer + Config() runnerConfiguration + OsCommand() OSCommand + + Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) + NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) +} + +var _ Runner = &RemoteRunner{} +var _ Runner = &LocalRunner{} + +type RemoteRunner struct { e config.Env namer namer.Namer waitCommand *remote.Command @@ -62,7 +82,7 @@ type Runner struct { options []pulumi.ResourceOption } -type RunnerArgs struct { +type RemoteRunnerArgs struct { ParentResource pulumi.Resource ConnectionName string Connection remote.ConnectionInput @@ -71,8 +91,8 @@ type RunnerArgs struct { OSCommand OSCommand } -func NewRunner(e config.Env, args RunnerArgs) (*Runner, error) { - runner := &Runner{ +func NewRemoteRunner(e config.Env, args RemoteRunnerArgs) (*RemoteRunner, error) { + runner := &RemoteRunner{ e: e, namer: namer.NewNamer(e.Ctx(), "remote").WithPrefix(args.ConnectionName), config: runnerConfiguration{ @@ -101,7 +121,23 @@ func NewRunner(e config.Env, args RunnerArgs) (*Runner, error) { return runner, nil } -func (r *Runner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (r *RemoteRunner) Environment() config.Env { + return r.e +} + +func (r *RemoteRunner) Namer() namer.Namer { + return r.namer +} + +func (r *RemoteRunner) Config() runnerConfiguration { + return r.config +} + +func (r *RemoteRunner) OsCommand() OSCommand { + return r.osCommand +} + +func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) { if args.Sudo && r.config.user != "" { r.e.Ctx().Log.Info(fmt.Sprintf("warning: running sudo command on a runner with user %s, discarding user", r.config.user), nil) } @@ -109,7 +145,7 @@ func (r *Runner) Command(name string, args *Args, opts ...pulumi.ResourceOption) return remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) } -func (r *Runner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) } @@ -138,7 +174,28 @@ func NewLocalRunner(e config.Env, args LocalRunnerArgs) *LocalRunner { return localRunner } -func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (*local.Command, error) { +func (r *LocalRunner) Environment() config.Env { + return r.e +} + +func (r *LocalRunner) Namer() namer.Namer { + return r.namer +} + +func (r *LocalRunner) Config() runnerConfiguration { + return r.config +} + +func (r *LocalRunner) OsCommand() OSCommand { + return r.osCommand +} + +func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) { opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) return local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) } + +func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + // TODO + return nil, nil +} diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 21a94e376..3e62191f1 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -27,12 +27,12 @@ func NewUnixOSCommand() OSCommand { // CreateDirectory if it does not exist func (unixOSCommand) CreateDirectory( - runner *Runner, + runner Runner, name string, remotePath pulumi.StringInput, useSudo bool, opts ...pulumi.ResourceOption, -) (*remote.Command, error) { +) (Command, error) { createCmd := fmt.Sprintf("mkdir -p %v", remotePath) deleteCmd := fmt.Sprintf(`bash -c 'if [ -z "$(ls -A %v)" ]; then rm -d %v; fi'`, remotePath, remotePath) // check if directory already exist @@ -72,7 +72,7 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { return strings.HasPrefix(path, "/") } -func (fs unixOSCommand) NewCopyFile(runner *Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (fs unixOSCommand) NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { tempRemotePath := localPath.ToStringOutput().ApplyT(func(path string) string { return filepath.Join(runner.osCommand.GetTemporaryDirectory(), filepath.Base(path)) }).(pulumi.StringOutput) @@ -117,7 +117,7 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, return formattedCommand } -func (fs unixOSCommand) moveRemoteFile(runner *Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (fs unixOSCommand) moveRemoteFile(runner *RemoteRunner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) copyCommand := pulumi.Sprintf(`cp '%v' '%v'`, source, destination) createCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; fi; %v'`, destination, destination, backupPath, copyCommand) diff --git a/components/command/utils.go b/components/command/utils.go index 2dd361c8d..96a4a2ede 100644 --- a/components/command/utils.go +++ b/components/command/utils.go @@ -5,9 +5,9 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -type ReadyFunc func(*Runner) (*remote.Command, error) +type ReadyFunc func(*RemoteRunner) (*remote.Command, error) -func WaitForCloudInit(runner *Runner) (*remote.Command, error) { +func WaitForCloudInit(runner *RemoteRunner) (*remote.Command, error) { return runner.Command( "wait-cloud-init", &Args{ @@ -17,7 +17,7 @@ func WaitForCloudInit(runner *Runner) (*remote.Command, error) { }) } -func WaitForSuccessfulConnection(runner *Runner) (*remote.Command, error) { +func WaitForSuccessfulConnection(runner *RemoteRunner) (*remote.Command, error) { return runner.Command( "wait-successful-connection", &Args{ diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index c5fb0e3c5..e494f378f 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -20,12 +20,12 @@ func NewWindowsOSCommand() OSCommand { // CreateDirectory if it does not exist func (fs windowsOSCommand) CreateDirectory( - runner *Runner, + runner Runner, name string, remotePath pulumi.StringInput, _ bool, opts ...pulumi.ResourceOption, -) (*remote.Command, error) { +) (Command, error) { useSudo := false return createDirectory( runner, @@ -78,7 +78,7 @@ func (fs windowsOSCommand) IsPathAbsolute(path string) bool { return false } -func (fs windowsOSCommand) NewCopyFile(runner *Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (fs windowsOSCommand) NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return remote.NewCopyFile(runner.e.Ctx(), runner.namer.ResourceName("copy", name), &remote.CopyFileArgs{ Connection: runner.config.connection, LocalPath: localPath, diff --git a/components/os/linux.go b/components/os/linux.go index b060193cb..69e09f3cb 100644 --- a/components/os/linux.go +++ b/components/os/linux.go @@ -7,7 +7,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newLinuxOS(e config.Env, desc Descriptor, runner *command.Runner) OS { +func newLinuxOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/linux_packagemanagers.go b/components/os/linux_packagemanagers.go index d263f9cbe..badeef933 100644 --- a/components/os/linux_packagemanagers.go +++ b/components/os/linux_packagemanagers.go @@ -5,19 +5,19 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func newAptManager(runner *command.Runner) PackageManager { +func newAptManager(runner *command.RemoteRunner) PackageManager { return command.NewGenericPackageManager(runner, "apt", "apt-get install -y", "apt-get update -y", pulumi.StringMap{"DEBIAN_FRONTEND": pulumi.String("noninteractive")}) } -func newYumManager(runner *command.Runner) PackageManager { +func newYumManager(runner *command.RemoteRunner) PackageManager { return command.NewGenericPackageManager(runner, "yum", "yum install -y", "", nil) } -func newDnfManager(runner *command.Runner) PackageManager { +func newDnfManager(runner *command.RemoteRunner) PackageManager { return command.NewGenericPackageManager(runner, "dnf", "dnf install -y", "", nil) } -func newZypperManager(runner *command.Runner) PackageManager { +func newZypperManager(runner *command.RemoteRunner) PackageManager { return command.NewGenericPackageManager(runner, "zypper", "zypper -n install", "zypper -n refresh", nil) } diff --git a/components/os/linux_servicemanagers.go b/components/os/linux_servicemanagers.go index ad4d9460b..58e4666ef 100644 --- a/components/os/linux_servicemanagers.go +++ b/components/os/linux_servicemanagers.go @@ -11,10 +11,10 @@ import ( type systemdServiceManager struct { e config.Env - runner *command.Runner + runner *command.RemoteRunner } -func newSystemdServiceManager(e config.Env, runner *command.Runner) ServiceManager { +func newSystemdServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { return &systemdServiceManager{e: e, runner: runner} } @@ -35,10 +35,10 @@ func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform co type sysvinitServiceManager struct { e config.Env - runner *command.Runner + runner *command.RemoteRunner } -func newSysvinitServiceManager(e config.Env, runner *command.Runner) ServiceManager { +func newSysvinitServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { return &sysvinitServiceManager{e: e, runner: runner} } diff --git a/components/os/macos.go b/components/os/macos.go index c66be16f8..904ef8d12 100644 --- a/components/os/macos.go +++ b/components/os/macos.go @@ -5,7 +5,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newMacOS(e config.Env, desc Descriptor, runner *command.Runner) OS { +func newMacOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/macos_packagemanagers.go b/components/os/macos_packagemanagers.go index 0cda0e3de..954a72b4d 100644 --- a/components/os/macos_packagemanagers.go +++ b/components/os/macos_packagemanagers.go @@ -5,7 +5,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func newBrewManager(runner *command.Runner) PackageManager { +func newBrewManager(runner *command.RemoteRunner) PackageManager { return command.NewGenericPackageManager(runner, "brew", "brew install -y", "brew update -y", pulumi.StringMap{"NONINTERACTIVE": pulumi.String("1")}) } diff --git a/components/os/macos_servicemanagers.go b/components/os/macos_servicemanagers.go index a29d00f83..0b0a67a69 100644 --- a/components/os/macos_servicemanagers.go +++ b/components/os/macos_servicemanagers.go @@ -11,10 +11,10 @@ import ( type macOSServiceManager struct { e config.Env - runner *command.Runner + runner *command.RemoteRunner } -func newMacOSServiceManager(e config.Env, runner *command.Runner) ServiceManager { +func newMacOSServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { return &macOSServiceManager{e: e, runner: runner} } diff --git a/components/os/os.go b/components/os/os.go index a9570a2a3..904b3276e 100644 --- a/components/os/os.go +++ b/components/os/os.go @@ -29,7 +29,7 @@ type ServiceManager interface { type OS interface { Descriptor() Descriptor - Runner() *command.Runner + Runner() *command.RemoteRunner FileManager() *command.FileManager PackageManager() PackageManager ServiceManger() ServiceManager @@ -40,7 +40,7 @@ var _ OS = &os{} // os is a generic implementation of OS interface type os struct { descriptor Descriptor - runner *command.Runner + runner *command.RemoteRunner fileManager *command.FileManager packageManager PackageManager serviceManager ServiceManager @@ -50,7 +50,7 @@ func (o os) Descriptor() Descriptor { return o.descriptor } -func (o os) Runner() *command.Runner { +func (o os) Runner() *command.RemoteRunner { return o.runner } @@ -69,7 +69,7 @@ func (o os) ServiceManger() ServiceManager { func NewOS( e config.Env, descriptor Descriptor, - runner *command.Runner, + runner *command.RemoteRunner, ) OS { switch descriptor.Family() { case LinuxFamily: diff --git a/components/os/windows.go b/components/os/windows.go index a6e9dec78..f09e5aebe 100644 --- a/components/os/windows.go +++ b/components/os/windows.go @@ -5,7 +5,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newWindowsOS(e config.Env, desc Descriptor, runner *command.Runner) OS { +func newWindowsOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/windows_servicemanagers.go b/components/os/windows_servicemanagers.go index b75015b23..98a645c06 100644 --- a/components/os/windows_servicemanagers.go +++ b/components/os/windows_servicemanagers.go @@ -9,10 +9,10 @@ import ( type windowsServiceManager struct { e config.Env - runner *command.Runner + runner *command.RemoteRunner } -func newWindowsServiceManager(e config.Env, runner *command.Runner) ServiceManager { +func newWindowsServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { return &windowsServiceManager{e: e, runner: runner} } diff --git a/components/remote/os.go b/components/remote/os.go index 984e0d7c6..988db2746 100644 --- a/components/remote/os.go +++ b/components/remote/os.go @@ -20,7 +20,7 @@ func InitHost(e config.Env, conn remote.ConnectionOutput, osDesc os.Descriptor, } // Now we can create the runner - runner, err := command.NewRunner(e, command.RunnerArgs{ + runner, err := command.NewRemoteRunner(e, command.RemoteRunnerArgs{ ParentResource: host, ConnectionName: host.Name(), Connection: conn, diff --git a/scenarios/aws/microVMs/microvms/provision.go b/scenarios/aws/microVMs/microvms/provision.go index 5fcd726e4..b9042cdc1 100644 --- a/scenarios/aws/microVMs/microvms/provision.go +++ b/scenarios/aws/microVMs/microvms/provision.go @@ -237,9 +237,9 @@ func provisionRemoteMicroVMs(vmCollections []*VMCollection, instanceEnv *Instanc } pc := createProxyConnection(domain.ip, "root", microVMSSHKey, conn.ToConnectionOutput()) - remoteRunner, err := command.NewRunner( + remoteRunner, err := command.NewRemoteRunner( collection.instance.e, - command.RunnerArgs{ + command.RemoteRunnerArgs{ ParentResource: domain.lvDomain, Connection: pc, ConnectionName: collection.instance.instanceNamer.ResourceName("conn", domain.domainID), @@ -299,9 +299,9 @@ func provisionLocalMicroVMs(vmCollections []*VMCollection) ([]pulumi.Resource, e return nil, err } - remoteRunner, err := command.NewRunner( + remoteRunner, err := command.NewRemoteRunner( *collection.instance.e, - command.RunnerArgs{ + command.RemoteRunnerArgs{ ParentResource: domain.lvDomain, Connection: conn, ConnectionName: domain.domainNamer.ResourceName("provision-conn"), diff --git a/scenarios/aws/microVMs/microvms/runner.go b/scenarios/aws/microVMs/microvms/runner.go index 7a71b6932..3f6ffe07a 100644 --- a/scenarios/aws/microVMs/microvms/runner.go +++ b/scenarios/aws/microVMs/microvms/runner.go @@ -10,7 +10,7 @@ import ( ) type Runner struct { - remoteRunner *command.Runner + remoteRunner *command.RemoteRunner localRunner *command.LocalRunner } @@ -24,7 +24,7 @@ func NewRunner(options ...func(*Runner)) *Runner { return &runner } -func WithRemoteRunner(runner *command.Runner) func(*Runner) { +func WithRemoteRunner(runner *command.RemoteRunner) func(*Runner) { return func(a *Runner) { a.remoteRunner = runner } @@ -61,7 +61,7 @@ func (a *Runner) LocalCommand(name string, args *command.Args, opts ...pulumi.Re return a.localRunner.Command(name, args, opts...) } -func (a *Runner) GetRemoteRunner() (*command.Runner, error) { +func (a *Runner) GetRemoteRunner() (*command.RemoteRunner, error) { if a.remoteRunner == nil { return nil, errors.New("remote runner not initialized") } From 505767274aa8693c8ac4dca46241b2ff43f8886e Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 13:34:14 +0100 Subject: [PATCH 02/19] [refactor-runner-adxt-783] wip: components/os --- components/command/osCommand.go | 2 +- components/command/package.go | 2 +- components/command/runner.go | 42 +++++++++- components/command/unixOSCommand.go | 12 +-- components/command/utils.go | 7 +- components/command/windowsOSCommand.go | 8 +- components/os/linux.go | 2 +- components/os/linux_packagemanagers.go | 8 +- components/os/linux_servicemanagers.go | 13 ++-- components/os/macos.go | 2 +- components/os/macos_packagemanagers.go | 2 +- components/os/macos_servicemanagers.go | 7 +- components/os/os.go | 13 ++-- components/os/windows.go | 2 +- components/os/windows_servicemanagers.go | 7 +- scenarios/aws/ec2/vm.go | 3 +- scenarios/aws/microVMs/microvms/libvirt.go | 2 +- scenarios/aws/microVMs/microvms/libvirt_fs.go | 10 +-- scenarios/aws/microVMs/microvms/network.go | 2 +- scenarios/aws/microVMs/microvms/pool.go | 8 +- scenarios/aws/microVMs/microvms/provision.go | 28 ++++--- scenarios/aws/microVMs/microvms/run.go | 6 +- scenarios/aws/microVMs/microvms/runner.go | 77 ------------------- scenarios/aws/microVMs/microvms/volume.go | 6 +- 24 files changed, 111 insertions(+), 160 deletions(-) delete mode 100644 scenarios/aws/microVMs/microvms/runner.go diff --git a/components/command/osCommand.go b/components/command/osCommand.go index 294433a7b..e3d27b118 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -27,7 +27,7 @@ type OSCommand interface { IsPathAbsolute(path string) bool - NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) } // ------------------------------ diff --git a/components/command/package.go b/components/command/package.go index e734934c5..f9c48613c 100644 --- a/components/command/package.go +++ b/components/command/package.go @@ -26,7 +26,7 @@ func NewGenericPackageManager( env pulumi.StringMap, ) *GenericPackageManager { packageManager := &GenericPackageManager{ - namer: namer.NewNamer(runner.e.Ctx(), name), + namer: namer.NewNamer(runner.Environment().Ctx(), name), runner: runner, installCmd: installCmd, updateCmd: updateCmd, diff --git a/components/command/runner.go b/components/command/runner.go index 776253dfa..a384ceaa0 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -55,10 +55,37 @@ type runnerConfiguration struct { type Command interface { pulumi.Resource + + Stdout() pulumi.StringOutput + Stderr() pulumi.StringOutput +} + +type LocalCommand struct { + local.Command +} + +type RemoteCommand struct { + remote.Command } -var _ Command = &remote.Command{} -var _ Command = &local.Command{} +var _ Command = &RemoteCommand{} +var _ Command = &LocalCommand{} + +func (c *LocalCommand) Stdout() pulumi.StringOutput { + return c.Stdout() +} + +func (c *LocalCommand) Stderr() pulumi.StringOutput { + return c.Stderr() +} + +func (c *RemoteCommand) Stdout() pulumi.StringOutput { + return c.Stdout() +} + +func (c *RemoteCommand) Stderr() pulumi.StringOutput { + return c.Stderr() +} type Runner interface { Environment() config.Env @@ -68,6 +95,7 @@ type Runner interface { Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + PulumiOptions() []pulumi.ResourceOption } var _ Runner = &RemoteRunner{} @@ -76,7 +104,7 @@ var _ Runner = &LocalRunner{} type RemoteRunner struct { e config.Env namer namer.Namer - waitCommand *remote.Command + waitCommand Command config runnerConfiguration osCommand OSCommand options []pulumi.ResourceOption @@ -149,6 +177,10 @@ func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.Str return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) } +func (r *RemoteRunner) PulumiOptions() []pulumi.ResourceOption { + return r.options +} + type LocalRunner struct { e config.Env namer namer.Namer @@ -199,3 +231,7 @@ func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.Stri // TODO return nil, nil } + +func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { + return []pulumi.ResourceOption{} +} diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 3e62191f1..a0a634260 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -72,17 +72,17 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { return strings.HasPrefix(path, "/") } -func (fs unixOSCommand) NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (fs unixOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { tempRemotePath := localPath.ToStringOutput().ApplyT(func(path string) string { - return filepath.Join(runner.osCommand.GetTemporaryDirectory(), filepath.Base(path)) + return filepath.Join(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) }).(pulumi.StringOutput) - tempCopyFile, err := remote.NewCopyFile(runner.e.Ctx(), runner.namer.ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: runner.config.connection, + tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: runner.Config().connection, LocalPath: localPath, RemotePath: tempRemotePath, Triggers: pulumi.Array{localPath, tempRemotePath}, - }, utils.MergeOptions(runner.options, opts...)...) + }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) if err != nil { return nil, err @@ -117,7 +117,7 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, return formattedCommand } -func (fs unixOSCommand) moveRemoteFile(runner *RemoteRunner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (fs unixOSCommand) moveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) copyCommand := pulumi.Sprintf(`cp '%v' '%v'`, source, destination) createCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; fi; %v'`, destination, destination, backupPath, copyCommand) diff --git a/components/command/utils.go b/components/command/utils.go index 96a4a2ede..03fa532ed 100644 --- a/components/command/utils.go +++ b/components/command/utils.go @@ -1,13 +1,12 @@ package command import ( - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -type ReadyFunc func(*RemoteRunner) (*remote.Command, error) +type ReadyFunc func(Runner) (Command, error) -func WaitForCloudInit(runner *RemoteRunner) (*remote.Command, error) { +func WaitForCloudInit(runner Runner) (Command, error) { return runner.Command( "wait-cloud-init", &Args{ @@ -17,7 +16,7 @@ func WaitForCloudInit(runner *RemoteRunner) (*remote.Command, error) { }) } -func WaitForSuccessfulConnection(runner *RemoteRunner) (*remote.Command, error) { +func WaitForSuccessfulConnection(runner Runner) (Command, error) { return runner.Command( "wait-successful-connection", &Args{ diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index e494f378f..9d6d8b358 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -78,11 +78,11 @@ func (fs windowsOSCommand) IsPathAbsolute(path string) bool { return false } -func (fs windowsOSCommand) NewCopyFile(runner *RemoteRunner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return remote.NewCopyFile(runner.e.Ctx(), runner.namer.ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: runner.config.connection, +func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + return remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: runner.Config().connection, LocalPath: localPath, RemotePath: remotePath, Triggers: pulumi.Array{localPath, remotePath}, - }, utils.MergeOptions(runner.options, opts...)...) + }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) } diff --git a/components/os/linux.go b/components/os/linux.go index 69e09f3cb..f70a85d1b 100644 --- a/components/os/linux.go +++ b/components/os/linux.go @@ -7,7 +7,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newLinuxOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { +func newLinuxOS(e config.Env, desc Descriptor, runner command.Runner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/linux_packagemanagers.go b/components/os/linux_packagemanagers.go index badeef933..b41344a92 100644 --- a/components/os/linux_packagemanagers.go +++ b/components/os/linux_packagemanagers.go @@ -5,19 +5,19 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func newAptManager(runner *command.RemoteRunner) PackageManager { +func newAptManager(runner command.Runner) PackageManager { return command.NewGenericPackageManager(runner, "apt", "apt-get install -y", "apt-get update -y", pulumi.StringMap{"DEBIAN_FRONTEND": pulumi.String("noninteractive")}) } -func newYumManager(runner *command.RemoteRunner) PackageManager { +func newYumManager(runner command.Runner) PackageManager { return command.NewGenericPackageManager(runner, "yum", "yum install -y", "", nil) } -func newDnfManager(runner *command.RemoteRunner) PackageManager { +func newDnfManager(runner command.Runner) PackageManager { return command.NewGenericPackageManager(runner, "dnf", "dnf install -y", "", nil) } -func newZypperManager(runner *command.RemoteRunner) PackageManager { +func newZypperManager(runner command.Runner) PackageManager { return command.NewGenericPackageManager(runner, "zypper", "zypper -n install", "zypper -n refresh", nil) } diff --git a/components/os/linux_servicemanagers.go b/components/os/linux_servicemanagers.go index 58e4666ef..b78da21c7 100644 --- a/components/os/linux_servicemanagers.go +++ b/components/os/linux_servicemanagers.go @@ -5,20 +5,19 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/command" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) type systemdServiceManager struct { e config.Env - runner *command.RemoteRunner + runner command.Runner } -func newSystemdServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { +func newSystemdServiceManager(e config.Env, runner command.Runner) ServiceManager { return &systemdServiceManager{e: e, runner: runner} } -func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { cmdName := s.e.CommonNamer().ResourceName("running", serviceName) cmdArgs := command.Args{ Sudo: true, @@ -35,14 +34,14 @@ func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform co type sysvinitServiceManager struct { e config.Env - runner *command.RemoteRunner + runner command.Runner } -func newSysvinitServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { +func newSysvinitServiceManager(e config.Env, runner command.Runner) ServiceManager { return &sysvinitServiceManager{e: e, runner: runner} } -func (s *sysvinitServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (s *sysvinitServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { cmdName := s.e.CommonNamer().ResourceName("running", serviceName) // To the difference of systemctl the restart doesn't work if the service isn't already running // so instead we run a stop command that we allow to fail and then a start command diff --git a/components/os/macos.go b/components/os/macos.go index 904ef8d12..b9f637b6f 100644 --- a/components/os/macos.go +++ b/components/os/macos.go @@ -5,7 +5,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newMacOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { +func newMacOS(e config.Env, desc Descriptor, runner command.Runner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/macos_packagemanagers.go b/components/os/macos_packagemanagers.go index 954a72b4d..511a675f0 100644 --- a/components/os/macos_packagemanagers.go +++ b/components/os/macos_packagemanagers.go @@ -5,7 +5,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func newBrewManager(runner *command.RemoteRunner) PackageManager { +func newBrewManager(runner command.Runner) PackageManager { return command.NewGenericPackageManager(runner, "brew", "brew install -y", "brew update -y", pulumi.StringMap{"NONINTERACTIVE": pulumi.String("1")}) } diff --git a/components/os/macos_servicemanagers.go b/components/os/macos_servicemanagers.go index 0b0a67a69..8ebd8a8d4 100644 --- a/components/os/macos_servicemanagers.go +++ b/components/os/macos_servicemanagers.go @@ -5,20 +5,19 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/command" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) type macOSServiceManager struct { e config.Env - runner *command.RemoteRunner + runner command.Runner } -func newMacOSServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { +func newMacOSServiceManager(e config.Env, runner command.Runner) ServiceManager { return &macOSServiceManager{e: e, runner: runner} } -func (s *macOSServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (s *macOSServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { cmdName := s.e.CommonNamer().ResourceName("running", serviceName) cmdArgs := command.Args{ Sudo: true, diff --git a/components/os/os.go b/components/os/os.go index 904b3276e..0a3a8d130 100644 --- a/components/os/os.go +++ b/components/os/os.go @@ -6,7 +6,6 @@ import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/command" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -15,12 +14,12 @@ type PackageManager interface { // Ensure ensures that a package is installed // checkBinary is a binary that should be checked before running the install command, if it is not empty it will first run the `command -v checkBinary` command and if it fails it will run the installCmd, // if it succeeds we consider the package is already installed - Ensure(packageRef string, transform command.Transformer, checkBinary string, opts ...pulumi.ResourceOption) (*remote.Command, error) + Ensure(packageRef string, transform command.Transformer, checkBinary string, opts ...pulumi.ResourceOption) (command.Command, error) } type ServiceManager interface { // EnsureStarted starts or restarts (may be stop+start depending on implementation) the service if already running - EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) + EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) } // FileManager needs to be added here as well instead of the command package @@ -29,7 +28,7 @@ type ServiceManager interface { type OS interface { Descriptor() Descriptor - Runner() *command.RemoteRunner + Runner() command.Runner FileManager() *command.FileManager PackageManager() PackageManager ServiceManger() ServiceManager @@ -40,7 +39,7 @@ var _ OS = &os{} // os is a generic implementation of OS interface type os struct { descriptor Descriptor - runner *command.RemoteRunner + runner command.Runner fileManager *command.FileManager packageManager PackageManager serviceManager ServiceManager @@ -50,7 +49,7 @@ func (o os) Descriptor() Descriptor { return o.descriptor } -func (o os) Runner() *command.RemoteRunner { +func (o os) Runner() command.Runner { return o.runner } @@ -69,7 +68,7 @@ func (o os) ServiceManger() ServiceManager { func NewOS( e config.Env, descriptor Descriptor, - runner *command.RemoteRunner, + runner command.Runner, ) OS { switch descriptor.Family() { case LinuxFamily: diff --git a/components/os/windows.go b/components/os/windows.go index f09e5aebe..304f1d0fa 100644 --- a/components/os/windows.go +++ b/components/os/windows.go @@ -5,7 +5,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" ) -func newWindowsOS(e config.Env, desc Descriptor, runner *command.RemoteRunner) OS { +func newWindowsOS(e config.Env, desc Descriptor, runner command.Runner) OS { os := &os{ descriptor: desc, runner: runner, diff --git a/components/os/windows_servicemanagers.go b/components/os/windows_servicemanagers.go index 98a645c06..fb2afdf9e 100644 --- a/components/os/windows_servicemanagers.go +++ b/components/os/windows_servicemanagers.go @@ -3,20 +3,19 @@ package os import ( "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/command" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) type windowsServiceManager struct { e config.Env - runner *command.RemoteRunner + runner command.Runner } -func newWindowsServiceManager(e config.Env, runner *command.RemoteRunner) ServiceManager { +func newWindowsServiceManager(e config.Env, runner command.Runner) ServiceManager { return &windowsServiceManager{e: e, runner: runner} } -func (s *windowsServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (s *windowsServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { cmdName := s.e.CommonNamer().ResourceName("running", serviceName) cmdArgs := command.Args{ Create: pulumi.String("Restart-Service -Name " + serviceName), diff --git a/scenarios/aws/ec2/vm.go b/scenarios/aws/ec2/vm.go index e3568fffd..85fed7a3e 100644 --- a/scenarios/aws/ec2/vm.go +++ b/scenarios/aws/ec2/vm.go @@ -12,7 +12,6 @@ import ( "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/resources/aws/ec2" - goremote "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi-random/sdk/v4/go/random" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -110,7 +109,7 @@ func NewVM(e aws.Environment, name string, params ...VMOption) (*remote.Host, er }) } -func InstallECRCredentialsHelper(e aws.Environment, vm *remote.Host) (*goremote.Command, error) { +func InstallECRCredentialsHelper(e aws.Environment, vm *remote.Host) (command.Command, error) { ecrCredsHelperInstall, err := vm.OS.PackageManager().Ensure("amazon-ecr-credential-helper", nil, "") if err != nil { return nil, err diff --git a/scenarios/aws/microVMs/microvms/libvirt.go b/scenarios/aws/microVMs/microvms/libvirt.go index 3a0f22496..68105dc10 100644 --- a/scenarios/aws/microVMs/microvms/libvirt.go +++ b/scenarios/aws/microVMs/microvms/libvirt.go @@ -47,7 +47,7 @@ func newLibvirtFS(ctx *pulumi.Context, vmset *vmconfig.VMSet, pools map[vmconfig } } -func createDomainConsoleLog(runner *Runner, domainName string, resourceName string, depends []pulumi.Resource) (pulumi.Resource, error) { +func createDomainConsoleLog(runner command.Runner, domainName string, resourceName string, depends []pulumi.Resource) (pulumi.Resource, error) { args := command.Args{ Create: pulumi.Sprintf("truncate --size=0 %s", resources.GetConsolePath(domainName)), } diff --git a/scenarios/aws/microVMs/microvms/libvirt_fs.go b/scenarios/aws/microVMs/microvms/libvirt_fs.go index 2ac4ac685..e69725c59 100644 --- a/scenarios/aws/microVMs/microvms/libvirt_fs.go +++ b/scenarios/aws/microVMs/microvms/libvirt_fs.go @@ -211,7 +211,7 @@ func NewLibvirtFSCustomRecipe(ctx *pulumi.Context, vmset *vmconfig.VMSet, pools } } -func refreshFromBackingStore(volume LibvirtVolume, runner *Runner, urlPath string, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func refreshFromBackingStore(volume LibvirtVolume, runner command.Runner, urlPath string, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { var downloadCmd string var refreshCmd string @@ -246,7 +246,7 @@ func refreshFromBackingStore(volume LibvirtVolume, runner *Runner, urlPath strin return []pulumi.Resource{res}, err } -func downloadAndExtractRootfs(fs *LibvirtFilesystem, runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func downloadAndExtractRootfs(fs *LibvirtFilesystem, runner command.Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { var waitFor []pulumi.Resource var downloadSpecs []filesystemImageDownload var retrieveImage []pulumi.Resource @@ -334,7 +334,7 @@ func downloadAndExtractRootfs(fs *LibvirtFilesystem, runner *Runner, depends []p return waitFor, nil } -func extractImage(fsImage *filesystemImage, runner *Runner, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func extractImage(fsImage *filesystemImage, runner command.Runner, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { // Extract archive from the download path assuming it is xz compressed extractTopLevelArchive := command.Args{ Create: pulumi.Sprintf("rm %s || true; xz -d %s", fsImage.imagePath, fsImage.downloadPath()), @@ -347,7 +347,7 @@ func extractImage(fsImage *filesystemImage, runner *Runner, namer namer.Namer, d return []pulumi.Resource{res}, nil } -func (fs *LibvirtFilesystem) SetupLibvirtFilesystem(providerFn LibvirtProviderFn, runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func (fs *LibvirtFilesystem) SetupLibvirtFilesystem(providerFn LibvirtProviderFn, runner command.Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { // Downloading the base images for the volumes is the slowest part of the entire setup. // We want this step to start as soon as our remote VMs are ready. Therefore, we do not // make it depend on any other step. @@ -363,7 +363,7 @@ func (fs *LibvirtFilesystem) SetupLibvirtFilesystem(providerFn LibvirtProviderFn return setupLibvirtFilesystem(fs, runner, providerFn, depends) } -func setupLibvirtFilesystem(fs *LibvirtFilesystem, runner *Runner, providerFn LibvirtProviderFn, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func setupLibvirtFilesystem(fs *LibvirtFilesystem, runner command.Runner, providerFn LibvirtProviderFn, depends []pulumi.Resource) ([]pulumi.Resource, error) { var waitFor []pulumi.Resource for _, vol := range fs.volumes { setupLibvirtVMVolumeDone, err := vol.SetupLibvirtVMVolume(fs.ctx, runner, providerFn, fs.isLocal, depends) diff --git a/scenarios/aws/microVMs/microvms/network.go b/scenarios/aws/microVMs/microvms/network.go index f134f89b6..b1a682cc6 100644 --- a/scenarios/aws/microVMs/microvms/network.go +++ b/scenarios/aws/microVMs/microvms/network.go @@ -93,7 +93,7 @@ func getMicroVMGroupSubnet(taken []string) (string, error) { return "", fmt.Errorf("getMicroVMGroupSubnet: could not find subnet") } -func allowNFSPortsForBridge(ctx *pulumi.Context, isLocal bool, bridge pulumi.StringOutput, runner *Runner, resourceNamer namer.Namer, microVMGroupSubnet string) ([]pulumi.Resource, error) { +func allowNFSPortsForBridge(ctx *pulumi.Context, isLocal bool, bridge pulumi.StringOutput, runner command.Runner, resourceNamer namer.Namer, microVMGroupSubnet string) ([]pulumi.Resource, error) { sudoPassword := GetSudoPassword(ctx, isLocal) iptablesAllowTCPArgs := command.Args{ Create: pulumi.Sprintf(iptablesTCPRule, iptablesAddRuleFlag, bridge, microVMGroupSubnet, tcpRPCInfoPorts), diff --git a/scenarios/aws/microVMs/microvms/pool.go b/scenarios/aws/microVMs/microvms/pool.go index 6187b8f84..a8d7adefa 100644 --- a/scenarios/aws/microVMs/microvms/pool.go +++ b/scenarios/aws/microVMs/microvms/pool.go @@ -14,7 +14,7 @@ import ( ) type LibvirtPool interface { - SetupLibvirtPool(ctx *pulumi.Context, runner *Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) + SetupLibvirtPool(ctx *pulumi.Context, runner command.Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) Name() string Type() vmconfig.PoolType Path() string @@ -67,7 +67,7 @@ Setup for remote pool and local pool is different for a number of reasons: the target environment and the pulumi host are the same machine. It is simpler to use this API locally than have a complicated permissions setup. */ -func remoteGlobalPool(p *globalLibvirtPool, runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func remoteGlobalPool(p *globalLibvirtPool, runner command.Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { poolXMLWrittenArgs := command.Args{ Create: pulumi.Sprintf("echo \"%s\" > %s", p.poolXML, p.poolXMLPath), Delete: pulumi.Sprintf("rm -f %s", p.poolXMLPath), @@ -139,7 +139,7 @@ func localGlobalPool(ctx *pulumi.Context, p *globalLibvirtPool, providerFn Libvi return []pulumi.Resource{poolReady}, nil } -func (p *globalLibvirtPool) SetupLibvirtPool(ctx *pulumi.Context, runner *Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func (p *globalLibvirtPool) SetupLibvirtPool(ctx *pulumi.Context, runner command.Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { if isLocal { return localGlobalPool(ctx, p, providerFn, depends) } @@ -200,7 +200,7 @@ func NewRAMBackedLibvirtPool(ctx *pulumi.Context, disk *vmconfig.Disk) (LibvirtP }, nil } -func (p *rambackedLibvirtPool) SetupLibvirtPool(ctx *pulumi.Context, runner *Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func (p *rambackedLibvirtPool) SetupLibvirtPool(ctx *pulumi.Context, runner command.Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) ([]pulumi.Resource, error) { buildSharedDiskInRamfsArgs := command.Args{ Create: pulumi.Sprintf(sharedDiskCmd, p.poolPath, p.poolSize, p.baseImagePath), Delete: pulumi.Sprintf("umount %[1]s && rm -r %[1]s", p.poolPath), diff --git a/scenarios/aws/microVMs/microvms/provision.go b/scenarios/aws/microVMs/microvms/provision.go index b9042cdc1..f9c2d450c 100644 --- a/scenarios/aws/microVMs/microvms/provision.go +++ b/scenarios/aws/microVMs/microvms/provision.go @@ -64,7 +64,7 @@ func readMicroVMSSHKey(instance *Instance, depends []pulumi.Resource) (pulumi.St args := command.Args{ Create: pulumi.Sprintf("cat %s", filepath.Join(GetWorkingDirectory(instance.Arch), "ddvm_rsa")), } - done, err := instance.runner.RemoteCommand(instance.instanceNamer.ResourceName("read-microvm-ssh-key"), &args, pulumi.DependsOn(depends)) + done, err := instance.runner.Command(instance.instanceNamer.ResourceName("read-microvm-ssh-key"), &args, pulumi.DependsOn(depends)) if err != nil { return pulumi.StringOutput{}, nil, err } @@ -72,7 +72,7 @@ func readMicroVMSSHKey(instance *Instance, depends []pulumi.Resource) (pulumi.St return s, []pulumi.Resource{done}, err } -func setupSSHAllowEnv(runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func setupSSHAllowEnv(runner command.Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { args := command.Args{ Create: pulumi.Sprintf("echo -e 'AcceptEnv DD_API_KEY\n' | sudo tee -a /etc/ssh/sshd_config"), } @@ -83,7 +83,7 @@ func setupSSHAllowEnv(runner *Runner, depends []pulumi.Resource) ([]pulumi.Resou return []pulumi.Resource{done}, nil } -func reloadSSHD(runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func reloadSSHD(runner command.Runner, depends []pulumi.Resource) ([]pulumi.Resource, error) { args := command.Args{ Create: pulumi.Sprintf("sudo systemctl reload sshd.service"), } @@ -94,7 +94,7 @@ func reloadSSHD(runner *Runner, depends []pulumi.Resource) ([]pulumi.Resource, e return []pulumi.Resource{done}, nil } -func mountMicroVMDisks(runner *Runner, disks []resources.DomainDisk, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func mountMicroVMDisks(runner command.Runner, disks []resources.DomainDisk, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { var waitFor []pulumi.Resource for _, d := range disks { @@ -117,7 +117,7 @@ func mountMicroVMDisks(runner *Runner, disks []resources.DomainDisk, namer namer return waitFor, nil } -func setDockerDataRoot(runner *Runner, disks []resources.DomainDisk, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func setDockerDataRoot(runner command.Runner, disks []resources.DomainDisk, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { var waitFor []pulumi.Resource for _, d := range disks { @@ -142,7 +142,7 @@ func setDockerDataRoot(runner *Runner, disks []resources.DomainDisk, namer namer return waitFor, nil } -func enableNFSConnKiller(runner *Runner, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func enableNFSConnKiller(runner command.Runner, namer namer.Namer, depends []pulumi.Resource) ([]pulumi.Resource, error) { args := command.Args{ Create: pulumi.Sprintf("systemctl enable --now kill-dead-nfs-connections.timer"), Sudo: true, @@ -173,7 +173,7 @@ func provisionMetalInstance(instance *Instance) ([]pulumi.Resource, error) { return reloadSSHDDone, nil } -func prepareLibvirtSSHKeys(runner *Runner, localRunner *command.LocalRunner, resourceNamer namer.Namer, pair sshKeyPair, depends []pulumi.Resource) ([]pulumi.Resource, error) { +func prepareLibvirtSSHKeys(runner command.Runner, localRunner *command.LocalRunner, resourceNamer namer.Namer, pair sshKeyPair, depends []pulumi.Resource) ([]pulumi.Resource, error) { sshGenArgs := command.Args{ Create: pulumi.Sprintf("rm -f %s && rm -f %s && ssh-keygen -t rsa -b 4096 -f %s -q -N \"\" && cat %s", pair.privateKey, pair.publicKey, pair.privateKey, pair.publicKey), Delete: pulumi.Sprintf("rm %s && rm %s", pair.privateKey, pair.publicKey), @@ -249,21 +249,20 @@ func provisionRemoteMicroVMs(vmCollections []*VMCollection, instanceEnv *Instanc if err != nil { return nil, err } - microRunner := NewRunner(WithRemoteRunner(remoteRunner)) - mountDisksDone, err := mountMicroVMDisks(microRunner, domain.Disks, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) + mountDisksDone, err := mountMicroVMDisks(remoteRunner, domain.Disks, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) if err != nil { return nil, err } - setDockerDataRootDone, err := setDockerDataRoot(microRunner, domain.Disks, domain.domainNamer, mountDisksDone) + setDockerDataRootDone, err := setDockerDataRoot(remoteRunner, domain.Disks, domain.domainNamer, mountDisksDone) if err != nil { return nil, err } deps := append(readKeyDone, setDockerDataRootDone...) deps = append(deps, domain.lvDomain) - reloadSSHDDone, err := reloadSSHD(microRunner, deps) + reloadSSHDDone, err := reloadSSHD(remoteRunner, deps) if err != nil { return nil, err } @@ -311,22 +310,21 @@ func provisionLocalMicroVMs(vmCollections []*VMCollection) ([]pulumi.Resource, e if err != nil { return nil, err } - microVMRunner := NewRunner(WithRemoteRunner(remoteRunner)) if collection.instance.IsMacOSHost() { - nfsConnKillerDone, err := enableNFSConnKiller(microVMRunner, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) + nfsConnKillerDone, err := enableNFSConnKiller(remoteRunner, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) if err != nil { return nil, err } waitFor = append(waitFor, nfsConnKillerDone...) } - mountDisksDone, err := mountMicroVMDisks(microVMRunner, domain.Disks, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) + mountDisksDone, err := mountMicroVMDisks(remoteRunner, domain.Disks, domain.domainNamer, []pulumi.Resource{domain.lvDomain}) if err != nil { return nil, err } - setDockerDataRootDone, err := setDockerDataRoot(microVMRunner, domain.Disks, domain.domainNamer, mountDisksDone) + setDockerDataRootDone, err := setDockerDataRoot(remoteRunner, domain.Disks, domain.domainNamer, mountDisksDone) if err != nil { return nil, err } diff --git a/scenarios/aws/microVMs/microvms/run.go b/scenarios/aws/microVMs/microvms/run.go index 0a5ce4d40..517669839 100644 --- a/scenarios/aws/microVMs/microvms/run.go +++ b/scenarios/aws/microVMs/microvms/run.go @@ -34,7 +34,7 @@ type Instance struct { instance *remoteComp.Host Arch string instanceNamer namer.Namer - runner *Runner + runner command.Runner libvirtURI pulumi.StringOutput } @@ -192,9 +192,9 @@ func configureInstance(instance *Instance, m *config.DDMicroVMConfig) ([]pulumi. OSCommand: osCommand, }) if instance.Arch != LocalVMSet { - instance.runner = NewRunner(WithRemoteRunner(instance.instance.OS.Runner())) + instance.runner = instance.instance.OS.Runner() } else { - instance.runner = NewRunner(WithLocalRunner(localRunner)) + instance.runner = localRunner } shouldProvision := m.GetBoolWithDefault(m.MicroVMConfig, config.DDMicroVMProvisionEC2Instance, true) diff --git a/scenarios/aws/microVMs/microvms/runner.go b/scenarios/aws/microVMs/microvms/runner.go deleted file mode 100644 index 3f6ffe07a..000000000 --- a/scenarios/aws/microVMs/microvms/runner.go +++ /dev/null @@ -1,77 +0,0 @@ -package microvms - -import ( - "errors" - - "github.com/DataDog/test-infra-definitions/components/command" - "github.com/pulumi/pulumi-command/sdk/go/command/local" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" - "github.com/pulumi/pulumi/sdk/v3/go/pulumi" -) - -type Runner struct { - remoteRunner *command.RemoteRunner - localRunner *command.LocalRunner -} - -func NewRunner(options ...func(*Runner)) *Runner { - runner := Runner{} - - for _, opts := range options { - opts(&runner) - } - - return &runner -} - -func WithRemoteRunner(runner *command.RemoteRunner) func(*Runner) { - return func(a *Runner) { - a.remoteRunner = runner - } -} - -func WithLocalRunner(runner *command.LocalRunner) func(*Runner) { - return func(a *Runner) { - a.localRunner = runner - } -} - -func (a *Runner) Command(name string, args *command.Args, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - if a.remoteRunner != nil { - return a.remoteRunner.Command(name, args, opts...) - } - if a.localRunner != nil { - return a.localRunner.Command(name, args, opts...) - } - - panic("no runner initialized") -} - -func (a *Runner) RemoteCommand(name string, args *command.Args, opts ...pulumi.ResourceOption) (*remote.Command, error) { - if a.remoteRunner == nil { - return nil, errors.New("remote runner not initialized for Runner instance") - } - return a.remoteRunner.Command(name, args, opts...) -} - -func (a *Runner) LocalCommand(name string, args *command.Args, opts ...pulumi.ResourceOption) (*local.Command, error) { - if a.localRunner == nil { - return nil, errors.New("local runner not initialized for Runner instance") - } - return a.localRunner.Command(name, args, opts...) -} - -func (a *Runner) GetRemoteRunner() (*command.RemoteRunner, error) { - if a.remoteRunner == nil { - return nil, errors.New("remote runner not initialized") - } - - return a.remoteRunner, nil -} - -func (a *Runner) GetLocalRunner() (*command.LocalRunner, error) { - if a.localRunner == nil { - return nil, errors.New("local runner not initialized") - } - return a.localRunner, nil -} diff --git a/scenarios/aws/microVMs/microvms/volume.go b/scenarios/aws/microVMs/microvms/volume.go index cc2b32797..2e9df3ef4 100644 --- a/scenarios/aws/microVMs/microvms/volume.go +++ b/scenarios/aws/microVMs/microvms/volume.go @@ -12,7 +12,7 @@ import ( ) type LibvirtVolume interface { - SetupLibvirtVMVolume(ctx *pulumi.Context, runner *Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) (pulumi.Resource, error) + SetupLibvirtVMVolume(ctx *pulumi.Context, runner command.Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) (pulumi.Resource, error) UnderlyingImage() *filesystemImage FullResourceName(...string) string Key() string @@ -99,7 +99,7 @@ func NewLibvirtVolume( } } -func remoteLibvirtVolume(v *volume, runner *Runner, depends []pulumi.Resource) (pulumi.Resource, error) { +func remoteLibvirtVolume(v *volume, runner command.Runner, depends []pulumi.Resource) (pulumi.Resource, error) { var baseVolumeReady pulumi.Resource volumeXMLPath := fmt.Sprintf("/tmp/volume-%s.xml", v.filesystemImage.imageName) @@ -156,7 +156,7 @@ func localLibvirtVolume(v *volume, ctx *pulumi.Context, providerFn LibvirtProvid return stgvolReady, nil } -func (v *volume) SetupLibvirtVMVolume(ctx *pulumi.Context, runner *Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) (pulumi.Resource, error) { +func (v *volume) SetupLibvirtVMVolume(ctx *pulumi.Context, runner command.Runner, providerFn LibvirtProviderFn, isLocal bool, depends []pulumi.Resource) (pulumi.Resource, error) { if isLocal { return localLibvirtVolume(v, ctx, providerFn, depends) } From f645c6d5d4e974a013eb32f2f9c14eb4ec8522c0 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 13:58:48 +0100 Subject: [PATCH 03/19] [refactor-runner-adxt-783] Can build --- components/command/runner.go | 28 +++++++++++--------- components/datadog/agent/host_linuxos.go | 6 ++--- components/datadog/agent/host_os.go | 4 +-- components/datadog/agent/host_windowsos.go | 7 +++-- components/docker/component.go | 11 ++++---- components/kubernetes/kind.go | 4 +-- scenarios/aws/microVMs/microvms/provision.go | 4 +-- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index a384ceaa0..a262547bf 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -56,8 +56,8 @@ type runnerConfiguration struct { type Command interface { pulumi.Resource - Stdout() pulumi.StringOutput - Stderr() pulumi.StringOutput + StdoutOutput() pulumi.StringOutput + StderrOutput() pulumi.StringOutput } type LocalCommand struct { @@ -71,20 +71,20 @@ type RemoteCommand struct { var _ Command = &RemoteCommand{} var _ Command = &LocalCommand{} -func (c *LocalCommand) Stdout() pulumi.StringOutput { - return c.Stdout() +func (c *LocalCommand) StdoutOutput() pulumi.StringOutput { + return c.Command.Stdout } -func (c *LocalCommand) Stderr() pulumi.StringOutput { - return c.Stderr() +func (c *LocalCommand) StderrOutput() pulumi.StringOutput { + return c.Command.Stderr } -func (c *RemoteCommand) Stdout() pulumi.StringOutput { - return c.Stdout() +func (c *RemoteCommand) StdoutOutput() pulumi.StringOutput { + return c.Command.Stdout } -func (c *RemoteCommand) Stderr() pulumi.StringOutput { - return c.Stderr() +func (c *RemoteCommand) StderrOutput() pulumi.StringOutput { + return c.Command.Stderr } type Runner interface { @@ -170,7 +170,9 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO r.e.Ctx().Log.Info(fmt.Sprintf("warning: running sudo command on a runner with user %s, discarding user", r.config.user), nil) } - return remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) + cmd, err := remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) + + return &RemoteCommand{*cmd}, err } func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { @@ -224,7 +226,9 @@ func (r *LocalRunner) OsCommand() OSCommand { func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) { opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) - return local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) + cmd, err := local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) + + return &LocalCommand{*cmd}, err } func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { diff --git a/components/datadog/agent/host_linuxos.go b/components/datadog/agent/host_linuxos.go index f72cdc84a..b163cd2e4 100644 --- a/components/datadog/agent/host_linuxos.go +++ b/components/datadog/agent/host_linuxos.go @@ -2,13 +2,13 @@ package agent import ( "fmt" + "strings" + "github.com/DataDog/test-infra-definitions/components/command" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" remoteComp "github.com/DataDog/test-infra-definitions/components/remote" - "strings" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -62,6 +62,6 @@ func (am *agentLinuxManager) getAgentConfigFolder() string { return "/etc/datadog-agent" } -func (am *agentLinuxManager) restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (am *agentLinuxManager) restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { return am.targetOS.ServiceManger().EnsureRestarted("datadog-agent", transform, opts...) } diff --git a/components/datadog/agent/host_os.go b/components/datadog/agent/host_os.go index 05b498604..8d8de1d6d 100644 --- a/components/datadog/agent/host_os.go +++ b/components/datadog/agent/host_os.go @@ -2,13 +2,13 @@ package agent import ( "fmt" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/command" "github.com/DataDog/test-infra-definitions/components/os" remoteComp "github.com/DataDog/test-infra-definitions/components/remote" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -16,7 +16,7 @@ import ( type agentOSManager interface { getInstallCommand(version agentparams.PackageVersion, additionalInstallParameters []string) (string, error) getAgentConfigFolder() string - restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) + restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) } func getOSManager(host *remoteComp.Host) agentOSManager { diff --git a/components/datadog/agent/host_windowsos.go b/components/datadog/agent/host_windowsos.go index a4817dc16..63283b04c 100644 --- a/components/datadog/agent/host_windowsos.go +++ b/components/datadog/agent/host_windowsos.go @@ -20,7 +20,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -68,7 +67,7 @@ for ($i=0; $i -lt 3; $i++) { }; $exitCode = (Start-Process -Wait msiexec -PassThru -ArgumentList '/qn /i %s APIKEY=%%s %s').ExitCode Get-Content %s -Exit $exitCode +Exit $exitCode `, url, localFilename, localFilename, strings.Join(additionalInstallParameters, " "), logFilePath) return cmd, nil } @@ -77,7 +76,7 @@ func (am *agentWindowsManager) getAgentConfigFolder() string { return `C:\ProgramData\Datadog` } -func (am *agentWindowsManager) restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (am *agentWindowsManager) restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) { // TODO: When we introduce Namer in components, we should use it here. cmdName := am.host.Name() + "-" + "restart-agent" // Retry restart several time, workaround to https://datadoghq.atlassian.net/browse/WINA-747 @@ -92,7 +91,7 @@ while ($tries -lt 5) { } Start-Sleep -Seconds $sleepTime $sleepTime = $sleepTime * 2 - $tries++ + $tries++ } Get-Content stderr.txt Exit $exitCode diff --git a/components/docker/component.go b/components/docker/component.go index 6db241712..72d9a1451 100644 --- a/components/docker/component.go +++ b/components/docker/component.go @@ -13,7 +13,6 @@ import ( "github.com/DataDog/test-infra-definitions/components/command" remoteComp "github.com/DataDog/test-infra-definitions/components/remote" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -64,7 +63,7 @@ func (d *Manager) Export(ctx *pulumi.Context, out *ManagerOutput) error { return components.Export(ctx, d, out) } -func (d *Manager) ComposeFileUp(composeFilePath string, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (d *Manager) ComposeFileUp(composeFilePath string, opts ...pulumi.ResourceOption) (command.Command, error) { opts = utils.MergeOptions(d.opts, opts...) composeHash, err := utils.FileHash(composeFilePath) @@ -93,7 +92,7 @@ func (d *Manager) ComposeFileUp(composeFilePath string, opts ...pulumi.ResourceO ) } -func (d *Manager) ComposeStrUp(name string, composeManifests []ComposeInlineManifest, envVars pulumi.StringMap, opts ...pulumi.ResourceOption) (*remote.Command, error) { +func (d *Manager) ComposeStrUp(name string, composeManifests []ComposeInlineManifest, envVars pulumi.StringMap, opts ...pulumi.ResourceOption) (command.Command, error) { opts = utils.MergeOptions(d.opts, opts...) homeCmd, composePath, err := d.Host.OS.FileManager().HomeDirectory(name+"-compose-tmp", opts...) @@ -140,7 +139,7 @@ func (d *Manager) ComposeStrUp(name string, composeManifests []ComposeInlineMani ) } -func (d *Manager) install() (*remote.Command, error) { +func (d *Manager) install() (command.Command, error) { opts := []pulumi.ResourceOption{pulumi.Parent(d)} opts = utils.MergeOptions(d.opts, opts...) dockerInstall, err := d.Host.OS.PackageManager().Ensure("docker.io", nil, "docker", opts...) @@ -178,7 +177,7 @@ func (d *Manager) install() (*remote.Command, error) { groupCmd, err := d.Host.OS.Runner().Command( d.namer.ResourceName("group"), &command.Args{ - Create: pulumi.Sprintf("usermod -a -G docker %s", whoami.Stdout), + Create: pulumi.Sprintf("usermod -a -G docker %s", whoami.StdoutOutput()), Sudo: true, }, utils.MergeOptions(opts, utils.PulumiDependsOn(whoami))..., @@ -190,7 +189,7 @@ func (d *Manager) install() (*remote.Command, error) { return groupCmd, err } -func (d *Manager) installCompose() (*remote.Command, error) { +func (d *Manager) installCompose() (command.Command, error) { opts := append(d.opts, pulumi.Parent(d)) installCompose := pulumi.Sprintf("bash -c '(docker-compose version | grep %s) || (curl --retry 10 -fsSLo /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/%s/docker-compose-linux-$(uname -p) && sudo chmod 755 /usr/local/bin/docker-compose)'", composeVersion, composeVersion) return d.Host.OS.Runner().Command( diff --git a/components/kubernetes/kind.go b/components/kubernetes/kind.go index b2b43757f..37b9a2706 100644 --- a/components/kubernetes/kind.go +++ b/components/kubernetes/kind.go @@ -100,7 +100,7 @@ func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion st // Patch Kubeconfig based on private IP output // Also add skip tls - clusterComp.KubeConfig = pulumi.All(kubeConfigCmd.Stdout, vm.Address).ApplyT(func(args []interface{}) string { + clusterComp.KubeConfig = pulumi.All(kubeConfigCmd.StdoutOutput(), vm.Address).ApplyT(func(args []interface{}) string { allowInsecure := regexp.MustCompile("certificate-authority-data:.+").ReplaceAllString(args[0].(string), "insecure-skip-tls-verify: true") return strings.ReplaceAll(allowInsecure, "0.0.0.0", args[1].(string)) }).(pulumi.StringOutput) @@ -161,7 +161,7 @@ func NewLocalKindCluster(env config.Env, name string, kubeVersion string, opts . return err } - clusterComp.KubeConfig = kubeConfigCmd.Stdout + clusterComp.KubeConfig = kubeConfigCmd.StdoutOutput() clusterComp.ClusterName = kindClusterName.ToStringOutput() return nil diff --git a/scenarios/aws/microVMs/microvms/provision.go b/scenarios/aws/microVMs/microvms/provision.go index f9c2d450c..4fd42087a 100644 --- a/scenarios/aws/microVMs/microvms/provision.go +++ b/scenarios/aws/microVMs/microvms/provision.go @@ -68,7 +68,7 @@ func readMicroVMSSHKey(instance *Instance, depends []pulumi.Resource) (pulumi.St if err != nil { return pulumi.StringOutput{}, nil, err } - s := pulumi.ToSecret(done.Stdout).(pulumi.StringOutput) + s := pulumi.ToSecret(done.StdoutOutput()).(pulumi.StringOutput) return s, []pulumi.Resource{done}, err } @@ -189,7 +189,7 @@ func prepareLibvirtSSHKeys(runner command.Runner, localRunner *command.LocalRunn // We override the runner-level user here with root, and construct the path to the default users .ssh directory, // in order to write the public ssh key in the correct file. sshWriteArgs := command.Args{ - Create: pulumi.Sprintf("echo '%s' >> $(getent passwd 1000 | cut -d: -f6)/.ssh/authorized_keys", sshgenDone.Stdout), + Create: pulumi.Sprintf("echo '%s' >> $(getent passwd 1000 | cut -d: -f6)/.ssh/authorized_keys", sshgenDone.StderrOutput()), Sudo: true, } From 254bf8ff52cc18b3bd19264c7aa5f75d69665a43 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 14:18:26 +0100 Subject: [PATCH 04/19] [refactor-runner-adxt-783] Copy file --- components/command/runner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index a262547bf..d970dfe13 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -232,8 +232,7 @@ func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOp } func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - // TODO - return nil, nil + return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) } func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { From e9255ef09d59dfb7366a4bb8d49925326bcce19b Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 14:40:43 +0100 Subject: [PATCH 05/19] [refactor-runner-adxt-783] ad: Fixed --- components/activedirectory/params.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/activedirectory/params.go b/components/activedirectory/params.go index dad9b6cb6..a185b59df 100644 --- a/components/activedirectory/params.go +++ b/components/activedirectory/params.go @@ -3,7 +3,6 @@ package activedirectory import ( "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/command" - pulumiRemote "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/pulumiverse/pulumi-time/sdk/go/time" ) @@ -49,7 +48,7 @@ func WithDomain(domainFqdn, domainAdmin, domainAdminPassword string) Option { } func (adCtx *activeDirectoryContext) joinActiveDirectoryDomain(params *JoinDomainConfiguration) error { - var joinCmd *pulumiRemote.Command + var joinCmd command.Command joinCmd, err := adCtx.comp.host.OS.Runner().Command(adCtx.comp.namer.ResourceName("join-domain"), &command.Args{ Create: pulumi.Sprintf(` Add-Computer -DomainName %s -Credential (New-Object System.Management.Automation.PSCredential -ArgumentList %s, %s) @@ -90,7 +89,7 @@ func WithDomainController(domainFqdn, adminPassword string) func(*Configuration) } func (adCtx *activeDirectoryContext) installDomainController(params *DomainControllerConfiguration) error { - var installCmd *pulumiRemote.Command + var installCmd command.Command installCmd, err := adCtx.comp.host.OS.Runner().Command(adCtx.comp.namer.ResourceName("install-forest"), &command.Args{ Create: pulumi.Sprintf(` Add-WindowsFeature -name ad-domain-services -IncludeManagementTools; From 2b64d469feee9b87a786188feeab3651a44fe8eb Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 11 Dec 2024 15:34:51 +0100 Subject: [PATCH 06/19] [refactor-runner-adxt-783] runner: Fixed lock warning --- components/command/runner.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index d970dfe13..1f5d48d8e 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -61,11 +61,11 @@ type Command interface { } type LocalCommand struct { - local.Command + *local.Command } type RemoteCommand struct { - remote.Command + *remote.Command } var _ Command = &RemoteCommand{} @@ -172,7 +172,11 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO cmd, err := remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) - return &RemoteCommand{*cmd}, err + if err != nil { + return &RemoteCommand{cmd}, nil + } + + return nil, err } func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { @@ -228,7 +232,11 @@ func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOp opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) cmd, err := local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) - return &LocalCommand{*cmd}, err + if err != nil { + return &LocalCommand{cmd}, nil + } + + return nil, err } func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { From ab0b21a9e5798912a41c000291462601c31d914d Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Thu, 12 Dec 2024 11:40:42 +0100 Subject: [PATCH 07/19] [refactor-runner-adxt-783] wip --- components/command/osCommand.go | 1 + components/command/runner.go | 64 ++++++++++++++++++++++++++ components/command/unixOSCommand.go | 47 +++++++++---------- components/command/windowsOSCommand.go | 8 ++++ main.go | 11 ++++- resources/local/podman/vm.go | 19 ++++++++ 6 files changed, 125 insertions(+), 25 deletions(-) diff --git a/components/command/osCommand.go b/components/command/osCommand.go index e3d27b118..155adcda8 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -17,6 +17,7 @@ type OSCommand interface { remotePath pulumi.StringInput, useSudo bool, opts ...pulumi.ResourceOption) (Command, error) + MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) BuildCommandString( command pulumi.StringInput, diff --git a/components/command/runner.go b/components/command/runner.go index 1f5d48d8e..f5ac7a597 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "path/filepath" "github.com/pulumi/pulumi-command/sdk/go/command/local" "github.com/pulumi/pulumi-command/sdk/go/command/remote" @@ -95,6 +96,8 @@ type Runner interface { Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) PulumiOptions() []pulumi.ResourceOption } @@ -246,3 +249,64 @@ func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.Stri func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { return []pulumi.ResourceOption{} } + +func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + create := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) + delete := pulumi.Sprintf("Remove-Item -Path '%v'", dst) + useSudo := false // TODO A + + return r.Command(name, + &Args{ + Create: create, + Delete: delete, + Sudo: useSudo, + Triggers: pulumi.Array{create, delete, pulumi.BoolPtr(useSudo)}, + }, opts...) +} + +func (r *LocalRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + create := pulumi.Sprintf("cp '%v' '%v'", src, dst) + delete := pulumi.Sprintf("rm '%v'", dst) + useSudo := false // TODO A + + return r.Command(name, + &Args{ + Create: create, + Delete: delete, + Sudo: useSudo, + Triggers: pulumi.Array{create, delete, pulumi.BoolPtr(useSudo)}, + }, opts...) +} + +func (r *RemoteRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + return remote.NewCopyFile(r.Environment().Ctx(), r.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: r.Config().connection, + LocalPath: src, + RemotePath: dst, + Triggers: pulumi.Array{src, dst}, + }, utils.MergeOptions(r.PulumiOptions(), opts...)...) +} + +func (r *RemoteRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + tempRemotePath := src.ToStringOutput().ApplyT(func(path string) string { + return filepath.Join(r.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) + }).(pulumi.StringOutput) + + tempCopyFile, err := remote.NewCopyFile(r.Environment().Ctx(), r.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: r.Config().connection, + LocalPath: src, + RemotePath: tempRemotePath, + Triggers: pulumi.Array{src, tempRemotePath}, + }, utils.MergeOptions(r.PulumiOptions(), opts...)...) + + if err != nil { + return nil, err + } + + moveCommand, err := r.OsCommand().MoveRemoteFile(r, name, tempRemotePath, dst, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) + if err != nil { + return nil, err + } + + return moveCommand, err +} diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index a0a634260..59067c25c 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -2,13 +2,11 @@ package command import ( "fmt" - "path/filepath" "strings" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/alessio/shellescape" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -73,27 +71,28 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { } func (fs unixOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - tempRemotePath := localPath.ToStringOutput().ApplyT(func(path string) string { - return filepath.Join(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) - }).(pulumi.StringOutput) - - tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: runner.Config().connection, - LocalPath: localPath, - RemotePath: tempRemotePath, - Triggers: pulumi.Array{localPath, tempRemotePath}, - }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) - - if err != nil { - return nil, err - } - - moveCommand, err := fs.moveRemoteFile(runner, name, tempRemotePath, remotePath, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) - if err != nil { - return nil, err - } - - return moveCommand, err + return runner.CopyUnixFile(name, localPath, remotePath, opts...) + // tempRemotePath := localPath.ToStringOutput().ApplyT(func(path string) string { + // return filepath.Join(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) + // }).(pulumi.StringOutput) + + // tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + // Connection: runner.Config().connection, + // LocalPath: localPath, + // RemotePath: tempRemotePath, + // Triggers: pulumi.Array{localPath, tempRemotePath}, + // }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) + + // if err != nil { + // return nil, err + // } + + // moveCommand, err := fs.MoveRemoteFile(runner, name, tempRemotePath, remotePath, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) + // if err != nil { + // return nil, err + // } + + // return moveCommand, err } func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, user string) pulumi.StringInput { @@ -117,7 +116,7 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, return formattedCommand } -func (fs unixOSCommand) moveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { +func (fs unixOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) copyCommand := pulumi.Sprintf(`cp '%v' '%v'`, source, destination) createCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; fi; %v'`, destination, destination, backupPath, copyCommand) diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index 9d6d8b358..9f5692d13 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -86,3 +86,11 @@ func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, re Triggers: pulumi.Array{localPath, remotePath}, }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) } + +func (fs windowsOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { + backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) + copyCommand := pulumi.Sprintf(`Copy-Item -Path %v -Destination %v`, source, destination) + createCommand := pulumi.Sprintf(`if (Test-Path %v) { Move-Item -Force -Path %v -Destination %v }; %v`, destination, destination, backupPath, copyCommand) + deleteCommand := pulumi.Sprintf(`if (Test-Path %v) { Move-Item -Force -Path %v -Destination %v } else { Remove-Item -Force -Path %v }`, backupPath, backupPath, destination, destination) + return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) +} diff --git a/main.go b/main.go index 4bfc97ce0..eb8c968f2 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/DataDog/test-infra-definitions/registry" - "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" ) @@ -20,6 +19,16 @@ const ( func main() { pulumi.Run(func(ctx *pulumi.Context) error { + // TODO A + // e, _ := local.NewEnvironment(ctx) + // runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewUnixOSCommand()}) + // // _, err := runner.NewCopyFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) + // fm := command.NewFileManager(runner) + // fm.CreateDirectory("/tmp/hmm", false) + // _, err := fm.CopyFile("copy-file", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) + + // return err + scenarioName := os.Getenv(scenarioEnvVarName) rootConfig := config.New(ctx, "") if s := rootConfig.Get(scenarioParamName); s != "" { diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index 6ec2bf8e2..5116be11a 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -23,6 +23,21 @@ var dockerfileContent string var customDockerConfig = "{}" func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.ResourceOption) (address pulumi.StringOutput, user string, port int, err error) { + // TODO A: Unix command / windows + // runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewUnixOSCommand()}) + // fileManager := command.NewFileManager(runner) + + // // runner.CopyUnixFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) + + // runner.Command("hey-ho", &command.Args{ + // Create: pulumi.String("cp /tmp/hey /tmp/ho"), + // Delete: pulumi.String("rm /tmp/ho"), + // }) + + // println("END") + + // return pulumi.StringOutput{}, "", -1, fmt.Errorf("not implemented") + interpreter := []string{"/bin/bash", "-c"} if runtime.GOOS == "windows" { interpreter = []string{"powershell", "-Command"} @@ -41,14 +56,18 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou // TODO clean up the folder on stack destroy // Requires a Runner refactor to reuse crossplatform commands err = os.MkdirAll(dataPath, 0700) + // _, err = fileManager.CreateDirectory(dataPath, false) if err != nil { return pulumi.StringOutput{}, "", -1, err } + println("DIR") dockerfilePath := path.Join(dataPath, "Dockerfile") + // _, err = fileManager.CopyInlineFile(pulumi.String(dockerfileContent), dockerfilePath) err = os.WriteFile(dockerfilePath, []byte(dockerfileContent), 0600) if err != nil { return pulumi.StringOutput{}, "", -1, err } + println("COPY") // Use a config to avoid docker hooks that can call vault or other services (credHelpers) err = os.WriteFile(path.Join(dataPath, "config.json"), []byte(customDockerConfig), 0600) From c0c3ac407fd3911e3938b474be1fc42241cae56e Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Fri, 13 Dec 2024 17:12:08 +0100 Subject: [PATCH 08/19] [refactor-runner-adxt-783] Fixed little error --- scenarios/aws/microVMs/microvms/provision.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/aws/microVMs/microvms/provision.go b/scenarios/aws/microVMs/microvms/provision.go index 4fd42087a..68a0fa102 100644 --- a/scenarios/aws/microVMs/microvms/provision.go +++ b/scenarios/aws/microVMs/microvms/provision.go @@ -189,7 +189,7 @@ func prepareLibvirtSSHKeys(runner command.Runner, localRunner *command.LocalRunn // We override the runner-level user here with root, and construct the path to the default users .ssh directory, // in order to write the public ssh key in the correct file. sshWriteArgs := command.Args{ - Create: pulumi.Sprintf("echo '%s' >> $(getent passwd 1000 | cut -d: -f6)/.ssh/authorized_keys", sshgenDone.StderrOutput()), + Create: pulumi.Sprintf("echo '%s' >> $(getent passwd 1000 | cut -d: -f6)/.ssh/authorized_keys", sshgenDone.StdoutOutput()), Sudo: true, } From c49c5fe5bd00a483c6f9b9a528f366dfc9fd0117 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 16 Dec 2024 17:28:52 +0100 Subject: [PATCH 09/19] Fixed bug --- components/command/runner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index f5ac7a597..01d5737d6 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -176,10 +176,10 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO cmd, err := remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) if err != nil { - return &RemoteCommand{cmd}, nil + return nil, err } - return nil, err + return &RemoteCommand{cmd}, nil } func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { @@ -236,10 +236,10 @@ func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOp cmd, err := local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) if err != nil { - return &LocalCommand{cmd}, nil + return nil, err } - return nil, err + return &LocalCommand{cmd}, nil } func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { From d5ab53e9eec1240f80cdad8974cd809f2a2342b6 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Tue, 17 Dec 2024 09:57:06 +0100 Subject: [PATCH 10/19] [refactor-runner-adxt-783] Cleaned code --- components/command/runner.go | 40 +++++++++++++------------- components/command/unixOSCommand.go | 21 -------------- components/command/windowsOSCommand.go | 8 +----- main.go | 10 ------- resources/local/podman/vm.go | 7 ++--- 5 files changed, 23 insertions(+), 63 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index 01d5737d6..c95e88f7c 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -24,7 +24,7 @@ type Args struct { Sudo bool } -func (args *Args) toLocalCommandArgs(config runnerConfiguration, osCommand OSCommand) *local.CommandArgs { +func (args *Args) toLocalCommandArgs(config RunnerConfiguration, osCommand OSCommand) *local.CommandArgs { return &local.CommandArgs{ Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), @@ -34,7 +34,7 @@ func (args *Args) toLocalCommandArgs(config runnerConfiguration, osCommand OSCom } } -func (args *Args) toRemoteCommandArgs(config runnerConfiguration, osCommand OSCommand) *remote.CommandArgs { +func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCommand) *remote.CommandArgs { return &remote.CommandArgs{ Connection: config.connection, Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), @@ -49,7 +49,7 @@ func (args *Args) toRemoteCommandArgs(config runnerConfiguration, osCommand OSCo // Examples: swapping `args.Delete` with `args.Create`, or adding `args.Triggers`, or editing the name type Transformer func(name string, args Args) (string, Args) -type runnerConfiguration struct { +type RunnerConfiguration struct { user string connection remote.ConnectionInput } @@ -91,7 +91,7 @@ func (c *RemoteCommand) StderrOutput() pulumi.StringOutput { type Runner interface { Environment() config.Env Namer() namer.Namer - Config() runnerConfiguration + Config() RunnerConfiguration OsCommand() OSCommand Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) @@ -108,7 +108,7 @@ type RemoteRunner struct { e config.Env namer namer.Namer waitCommand Command - config runnerConfiguration + config RunnerConfiguration osCommand OSCommand options []pulumi.ResourceOption } @@ -126,7 +126,7 @@ func NewRemoteRunner(e config.Env, args RemoteRunnerArgs) (*RemoteRunner, error) runner := &RemoteRunner{ e: e, namer: namer.NewNamer(e.Ctx(), "remote").WithPrefix(args.ConnectionName), - config: runnerConfiguration{ + config: RunnerConfiguration{ connection: args.Connection, user: args.User, }, @@ -160,7 +160,7 @@ func (r *RemoteRunner) Namer() namer.Namer { return r.namer } -func (r *RemoteRunner) Config() runnerConfiguration { +func (r *RemoteRunner) Config() RunnerConfiguration { return r.config } @@ -193,7 +193,7 @@ func (r *RemoteRunner) PulumiOptions() []pulumi.ResourceOption { type LocalRunner struct { e config.Env namer namer.Namer - config runnerConfiguration + config RunnerConfiguration osCommand OSCommand } @@ -207,7 +207,7 @@ func NewLocalRunner(e config.Env, args LocalRunnerArgs) *LocalRunner { e: e, namer: namer.NewNamer(e.Ctx(), "local"), osCommand: args.OSCommand, - config: runnerConfiguration{ + config: RunnerConfiguration{ user: args.User, }, } @@ -223,7 +223,7 @@ func (r *LocalRunner) Namer() namer.Namer { return r.namer } -func (r *LocalRunner) Config() runnerConfiguration { +func (r *LocalRunner) Config() RunnerConfiguration { return r.config } @@ -251,30 +251,30 @@ func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { } func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - create := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) - delete := pulumi.Sprintf("Remove-Item -Path '%v'", dst) + createCmd := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) + deleteCmd := pulumi.Sprintf("Remove-Item -Path '%v'", dst) useSudo := false // TODO A return r.Command(name, &Args{ - Create: create, - Delete: delete, + Create: createCmd, + Delete: deleteCmd, Sudo: useSudo, - Triggers: pulumi.Array{create, delete, pulumi.BoolPtr(useSudo)}, + Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, }, opts...) } func (r *LocalRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - create := pulumi.Sprintf("cp '%v' '%v'", src, dst) - delete := pulumi.Sprintf("rm '%v'", dst) + createCmd := pulumi.Sprintf("cp '%v' '%v'", src, dst) + deleteCmd := pulumi.Sprintf("rm '%v'", dst) useSudo := false // TODO A return r.Command(name, &Args{ - Create: create, - Delete: delete, + Create: createCmd, + Delete: deleteCmd, Sudo: useSudo, - Triggers: pulumi.Array{create, delete, pulumi.BoolPtr(useSudo)}, + Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, }, opts...) } diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 59067c25c..58a313a35 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -72,27 +72,6 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { func (fs unixOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return runner.CopyUnixFile(name, localPath, remotePath, opts...) - // tempRemotePath := localPath.ToStringOutput().ApplyT(func(path string) string { - // return filepath.Join(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) - // }).(pulumi.StringOutput) - - // tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ - // Connection: runner.Config().connection, - // LocalPath: localPath, - // RemotePath: tempRemotePath, - // Triggers: pulumi.Array{localPath, tempRemotePath}, - // }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) - - // if err != nil { - // return nil, err - // } - - // moveCommand, err := fs.MoveRemoteFile(runner, name, tempRemotePath, remotePath, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) - // if err != nil { - // return nil, err - // } - - // return moveCommand, err } func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, user string) pulumi.StringInput { diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index 9f5692d13..8c20a4362 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -6,7 +6,6 @@ import ( "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -79,12 +78,7 @@ func (fs windowsOSCommand) IsPathAbsolute(path string) bool { } func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: runner.Config().connection, - LocalPath: localPath, - RemotePath: remotePath, - Triggers: pulumi.Array{localPath, remotePath}, - }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) + return runner.CopyWindowsFile(name, localPath, remotePath, opts...) } func (fs windowsOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { diff --git a/main.go b/main.go index eb8c968f2..0c3df24bf 100644 --- a/main.go +++ b/main.go @@ -19,16 +19,6 @@ const ( func main() { pulumi.Run(func(ctx *pulumi.Context) error { - // TODO A - // e, _ := local.NewEnvironment(ctx) - // runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewUnixOSCommand()}) - // // _, err := runner.NewCopyFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) - // fm := command.NewFileManager(runner) - // fm.CreateDirectory("/tmp/hmm", false) - // _, err := fm.CopyFile("copy-file", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) - - // return err - scenarioName := os.Getenv(scenarioEnvVarName) rootConfig := config.New(ctx, "") if s := rootConfig.Get(scenarioParamName); s != "" { diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index 5116be11a..aa29124a9 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -27,16 +27,13 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou // runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewUnixOSCommand()}) // fileManager := command.NewFileManager(runner) - // // runner.CopyUnixFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) + // runner.CopyUnixFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) // runner.Command("hey-ho", &command.Args{ // Create: pulumi.String("cp /tmp/hey /tmp/ho"), // Delete: pulumi.String("rm /tmp/ho"), // }) - - // println("END") - - // return pulumi.StringOutput{}, "", -1, fmt.Errorf("not implemented") + // TODO END interpreter := []string{"/bin/bash", "-c"} if runtime.GOOS == "windows" { From 2448f9d546cfc55b2b49e3c2e75cc90b3c64e24c Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Tue, 17 Dec 2024 15:22:41 +0100 Subject: [PATCH 11/19] [refactor-runner-adxt-783] command: Fixed env + refactored podman vm.go --- components/command/runner.go | 59 ++++++++++++++++++++--------- resources/local/podman/vm.go | 72 +++++++++++++++--------------------- 2 files changed, 71 insertions(+), 60 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index c95e88f7c..599ec7548 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -22,27 +22,42 @@ type Args struct { Environment pulumi.StringMap RequirePasswordFromStdin bool Sudo bool + // Only used for local commands + LocalAssetPaths pulumi.StringArrayInput + LocalDir pulumi.StringInput } -func (args *Args) toLocalCommandArgs(config RunnerConfiguration, osCommand OSCommand) *local.CommandArgs { +func (args *Args) toLocalCommandArgs(config RunnerConfiguration, osCommand OSCommand) (*local.CommandArgs, error) { return &local.CommandArgs{ - Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Triggers: args.Triggers, - Stdin: args.Stdin, + Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Environment: args.Environment, + Triggers: args.Triggers, + Stdin: args.Stdin, + AssetPaths: args.LocalAssetPaths, + Dir: args.LocalDir, + }, nil +} + +func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCommand) (*remote.CommandArgs, error) { + // Ensure no local arguments are passed to remote commands + if args.LocalAssetPaths != nil { + return nil, fmt.Errorf("local asset paths are not supported in remote commands") + } + if args.LocalDir != nil { + return nil, fmt.Errorf("local dir is not supported in remote commands") } -} -func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCommand) *remote.CommandArgs { return &remote.CommandArgs{ - Connection: config.connection, - Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Triggers: args.Triggers, - Stdin: args.Stdin, - } + Connection: config.connection, + Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Environment: args.Environment, + Triggers: args.Triggers, + Stdin: args.Stdin, + }, nil } // Transformer is a function that can be used to modify the command name and args. @@ -173,7 +188,12 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO r.e.Ctx().Log.Info(fmt.Sprintf("warning: running sudo command on a runner with user %s, discarding user", r.config.user), nil) } - cmd, err := remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toRemoteCommandArgs(r.config, r.osCommand), utils.MergeOptions(r.options, opts...)...) + remoteArgs, err := args.toRemoteCommandArgs(r.config, r.osCommand) + if err != nil { + return nil, err + } + + cmd, err := remote.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), remoteArgs, utils.MergeOptions(r.options, opts...)...) if err != nil { return nil, err @@ -233,7 +253,12 @@ func (r *LocalRunner) OsCommand() OSCommand { func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) { opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand)) - cmd, err := local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), args.toLocalCommandArgs(r.config, r.osCommand), opts...) + localArgs, err := args.toLocalCommandArgs(r.config, r.osCommand) + if err != nil { + return nil, err + } + + cmd, err := local.NewCommand(r.e.Ctx(), r.namer.ResourceName("cmd", name), localArgs, opts...) if err != nil { return nil, err diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index aa29124a9..706fb758a 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -6,11 +6,10 @@ import ( "path" "runtime" - "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/utils" + "github.com/DataDog/test-infra-definitions/components/command" resourceslocal "github.com/DataDog/test-infra-definitions/resources/local" - "github.com/pulumi/pulumi-command/sdk/go/command/local" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -23,22 +22,14 @@ var dockerfileContent string var customDockerConfig = "{}" func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.ResourceOption) (address pulumi.StringOutput, user string, port int, err error) { - // TODO A: Unix command / windows - // runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewUnixOSCommand()}) - // fileManager := command.NewFileManager(runner) - - // runner.CopyUnixFile("copy-hey-ho", pulumi.String("/tmp/hey"), pulumi.String("/tmp/ho")) - - // runner.Command("hey-ho", &command.Args{ - // Create: pulumi.String("cp /tmp/hey /tmp/ho"), - // Delete: pulumi.String("rm /tmp/ho"), - // }) - // TODO END - - interpreter := []string{"/bin/bash", "-c"} + var osCommand command.OSCommand if runtime.GOOS == "windows" { - interpreter = []string{"powershell", "-Command"} + osCommand = command.NewWindowsOSCommand() + } else { + osCommand = command.NewUnixOSCommand() } + runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: osCommand}) + fileManager := command.NewFileManager(runner) publicKey, err := os.ReadFile(e.DefaultPublicKeyPath()) if err != nil { @@ -52,53 +43,48 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou dataPath := path.Join(homeDir, ".localpodman") // TODO clean up the folder on stack destroy // Requires a Runner refactor to reuse crossplatform commands - err = os.MkdirAll(dataPath, 0700) - // _, err = fileManager.CreateDirectory(dataPath, false) + dataDir, err := fileManager.CreateDirectory(dataPath, false) if err != nil { return pulumi.StringOutput{}, "", -1, err } - println("DIR") + dockerfilePath := path.Join(dataPath, "Dockerfile") - // _, err = fileManager.CopyInlineFile(pulumi.String(dockerfileContent), dockerfilePath) - err = os.WriteFile(dockerfilePath, []byte(dockerfileContent), 0600) + dockerFile, err := fileManager.CopyInlineFile(pulumi.String(dockerfileContent), dockerfilePath, pulumi.DependsOn([]pulumi.Resource{dataDir})) if err != nil { return pulumi.StringOutput{}, "", -1, err } - println("COPY") // Use a config to avoid docker hooks that can call vault or other services (credHelpers) - err = os.WriteFile(path.Join(dataPath, "config.json"), []byte(customDockerConfig), 0600) + dockerConfig, err := fileManager.CopyInlineFile(pulumi.String(customDockerConfig), path.Join(dataPath, "config.json"), pulumi.DependsOn([]pulumi.Resource{dataDir})) if err != nil { return pulumi.StringOutput{}, "", -1, err } podmanCommand := "podman --config " + dataPath - opts = utils.MergeOptions(opts, e.WithProviders(config.ProviderCommand)) - // TODO use NewLocalRunner - // requires a refactor to pass interpreter - buildPodman, err := local.NewCommand(e.Ctx(), e.CommonNamer().ResourceName("podman-build", args.Name), &local.CommandArgs{ - Interpreter: pulumi.ToStringArray(interpreter), - Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, - Create: pulumi.Sprintf("%s build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", podmanCommand, args.Name), - Delete: pulumi.Sprintf("%s rmi %s", podmanCommand, args.Name), - Triggers: pulumi.Array{}, - AssetPaths: pulumi.StringArray{}, - Dir: pulumi.String(dataPath), + opts = utils.MergeOptions(opts, pulumi.DependsOn([]pulumi.Resource{dockerFile, dockerConfig})) + buildPodman, err := runner.Command("podman-build"+args.Name, &command.Args{ + Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, + Create: pulumi.Sprintf("%s build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", podmanCommand, args.Name), + Delete: pulumi.Sprintf("%s rmi %s", podmanCommand, args.Name), + Triggers: pulumi.Array{}, + LocalAssetPaths: pulumi.StringArray{}, + LocalDir: pulumi.String(dataPath), }, opts...) + if err != nil { return pulumi.StringOutput{}, "", -1, err } opts = utils.MergeOptions(opts, pulumi.DependsOn([]pulumi.Resource{buildPodman})) - runPodman, err := local.NewCommand(e.Ctx(), e.CommonNamer().ResourceName("podman-run", args.Name), &local.CommandArgs{ - Interpreter: pulumi.ToStringArray(interpreter), - Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, - Create: pulumi.Sprintf("%s run -d --name=%[2]s_run -p 50022:22 %[2]s", podmanCommand, args.Name), - Delete: pulumi.Sprintf("%s stop %[2]s_run && podman rm %[2]s_run", podmanCommand, args.Name), - Triggers: pulumi.Array{}, - AssetPaths: pulumi.StringArray{}, - Dir: pulumi.String(dataPath), + runPodman, err := runner.Command("podman-run"+args.Name, &command.Args{ + Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, + Create: pulumi.Sprintf("%s run -d --name=%[2]s_run -p 50022:22 %[2]s", podmanCommand, args.Name), + Delete: pulumi.Sprintf("%s stop %[2]s_run && podman rm %[2]s_run", podmanCommand, args.Name), + Triggers: pulumi.Array{}, + LocalAssetPaths: pulumi.StringArray{}, + LocalDir: pulumi.String(dataPath), }, opts...) + if err != nil { return pulumi.StringOutput{}, "", -1, err } @@ -106,7 +92,7 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou e.Ctx().Log.Info("Running with container of type ubuntu", nil) // hack to wait for the container to be up - ipAddress := runPodman.Stdout.ApplyT(func(_ string) string { + ipAddress := runPodman.StdoutOutput().ApplyT(func(_ string) string { return "localhost" }).(pulumi.StringOutput) From 74d4b0292b49f412fd765450440a05225354ccac Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 18 Dec 2024 14:27:16 +0100 Subject: [PATCH 12/19] [refactor-runner-adxt-783] Tested and cleaned code --- components/command/osCommand.go | 9 +++++++++ components/command/runner.go | 4 ++-- components/command/unixOSCommand.go | 6 +++--- components/command/windowsOSCommand.go | 6 +++--- resources/local/podman/vm.go | 9 +-------- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/components/command/osCommand.go b/components/command/osCommand.go index 155adcda8..bfad1350c 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -1,6 +1,7 @@ package command import ( + "runtime" "strings" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -88,3 +89,11 @@ func copyRemoteFile( Triggers: pulumi.Array{createCommand, deleteCommand, pulumi.BoolPtr(useSudo)}, }, opts...) } + +func NewLocalOSCommand() OSCommand { + if runtime.GOOS == "windows" { + return NewWindowsOSCommand() + } else { + return NewUnixOSCommand() + } +} diff --git a/components/command/runner.go b/components/command/runner.go index 599ec7548..0505e031e 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -278,7 +278,7 @@ func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { createCmd := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) deleteCmd := pulumi.Sprintf("Remove-Item -Path '%v'", dst) - useSudo := false // TODO A + useSudo := false return r.Command(name, &Args{ @@ -292,7 +292,7 @@ func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, func (r *LocalRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { createCmd := pulumi.Sprintf("cp '%v' '%v'", src, dst) deleteCmd := pulumi.Sprintf("rm '%v'", dst) - useSudo := false // TODO A + useSudo := false return r.Command(name, &Args{ diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 58a313a35..6013501e4 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -97,8 +97,8 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, func (fs unixOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) - copyCommand := pulumi.Sprintf(`cp '%v' '%v'`, source, destination) - createCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; fi; %v'`, destination, destination, backupPath, copyCommand) - deleteCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; else rm -f '%v'; fi'`, backupPath, backupPath, destination, destination) + copyCommand := pulumi.Sprintf(`cp "%v" "%v"`, source, destination) + createCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; fi; %v'`, destination, destination, backupPath, copyCommand) + deleteCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; else rm -f "%v"; fi'`, backupPath, backupPath, destination, destination) return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) } diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index 8c20a4362..bdbaaf09e 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -83,8 +83,8 @@ func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, re func (fs windowsOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) - copyCommand := pulumi.Sprintf(`Copy-Item -Path %v -Destination %v`, source, destination) - createCommand := pulumi.Sprintf(`if (Test-Path %v) { Move-Item -Force -Path %v -Destination %v }; %v`, destination, destination, backupPath, copyCommand) - deleteCommand := pulumi.Sprintf(`if (Test-Path %v) { Move-Item -Force -Path %v -Destination %v } else { Remove-Item -Force -Path %v }`, backupPath, backupPath, destination, destination) + copyCommand := pulumi.Sprintf(`Copy-Item -Path '%v' -Destination '%v'`, source, destination) + createCommand := pulumi.Sprintf(`if (Test-Path '%v') { Move-Item -Force -Path '%v' -Destination '%v' }; %v`, destination, destination, backupPath, copyCommand) + deleteCommand := pulumi.Sprintf(`if (Test-Path '%v') { Move-Item -Force -Path '%v' -Destination '%v' } else { Remove-Item -Force -Path %v }`, backupPath, backupPath, destination, destination) return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) } diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index 706fb758a..8f823ef3f 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -4,7 +4,6 @@ import ( _ "embed" "os" "path" - "runtime" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/command" @@ -22,13 +21,7 @@ var dockerfileContent string var customDockerConfig = "{}" func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.ResourceOption) (address pulumi.StringOutput, user string, port int, err error) { - var osCommand command.OSCommand - if runtime.GOOS == "windows" { - osCommand = command.NewWindowsOSCommand() - } else { - osCommand = command.NewUnixOSCommand() - } - runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: osCommand}) + runner := command.NewLocalRunner(&e, command.LocalRunnerArgs{OSCommand: command.NewLocalOSCommand()}) fileManager := command.NewFileManager(runner) publicKey, err := os.ReadFile(e.DefaultPublicKeyPath()) From 18bea625cdddb18a59497adf56429b5d1e1fd49d Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 18 Dec 2024 14:30:46 +0100 Subject: [PATCH 13/19] [refactor-runner-adxt-783] Cleaned code --- components/command/osCommand.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/command/osCommand.go b/components/command/osCommand.go index bfad1350c..21e933abd 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -93,7 +93,7 @@ func copyRemoteFile( func NewLocalOSCommand() OSCommand { if runtime.GOOS == "windows" { return NewWindowsOSCommand() - } else { - return NewUnixOSCommand() } + + return NewUnixOSCommand() } From 3656bdaeefa95533c132b19da6f29de2fb9172af Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 18 Dec 2024 14:39:41 +0100 Subject: [PATCH 14/19] [refactor-runner-adxt-783] Cleaned code --- resources/local/podman/vm.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index 8f823ef3f..242271106 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -64,7 +64,6 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou LocalAssetPaths: pulumi.StringArray{}, LocalDir: pulumi.String(dataPath), }, opts...) - if err != nil { return pulumi.StringOutput{}, "", -1, err } @@ -77,7 +76,6 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou LocalAssetPaths: pulumi.StringArray{}, LocalDir: pulumi.String(dataPath), }, opts...) - if err != nil { return pulumi.StringOutput{}, "", -1, err } From 36bcf5a9bcbce388ebbb9d2c8751e6d164924686 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Wed, 18 Dec 2024 17:45:44 +0100 Subject: [PATCH 15/19] [refactor-runner-adxt-783] env: Vars are exported and not passed to the shell anymore --- components/command/runner.go | 28 +++++++++++++--------------- components/command/unixOSCommand.go | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/components/command/runner.go b/components/command/runner.go index 0505e031e..ea1393a77 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -29,14 +29,13 @@ type Args struct { func (args *Args) toLocalCommandArgs(config RunnerConfiguration, osCommand OSCommand) (*local.CommandArgs, error) { return &local.CommandArgs{ - Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Environment: args.Environment, - Triggers: args.Triggers, - Stdin: args.Stdin, - AssetPaths: args.LocalAssetPaths, - Dir: args.LocalDir, + Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Triggers: args.Triggers, + Stdin: args.Stdin, + AssetPaths: args.LocalAssetPaths, + Dir: args.LocalDir, }, nil } @@ -50,13 +49,12 @@ func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCo } return &remote.CommandArgs{ - Connection: config.connection, - Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), - Environment: args.Environment, - Triggers: args.Triggers, - Stdin: args.Stdin, + Connection: config.connection, + Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user), + Triggers: args.Triggers, + Stdin: args.Stdin, }, nil } diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 6013501e4..fbc40d8e2 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -58,7 +58,7 @@ func (fs unixOSCommand) BuildCommandString(command pulumi.StringInput, env pulum var envVars pulumi.StringArray for varName, varValue := range env { - envVars = append(envVars, pulumi.Sprintf(`%v="%v"`, varName, varValue)) + envVars = append(envVars, pulumi.Sprintf(`export %v="%v";`, varName, varValue)) } return buildCommandString(formattedCommand, envVars, func(envVarsStr pulumi.StringOutput) pulumi.StringInput { From d1ef99fb49cef5d9dae1716eb208d28f2cefd557 Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 23 Dec 2024 09:36:20 +0100 Subject: [PATCH 16/19] [refactor-runner-adxt-783] renamed: MoveRemoteFile -> MoveFile --- components/command/osCommand.go | 2 +- components/command/runner.go | 2 +- components/command/unixOSCommand.go | 2 +- components/command/windowsOSCommand.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/command/osCommand.go b/components/command/osCommand.go index 21e933abd..2165eec5e 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -18,7 +18,7 @@ type OSCommand interface { remotePath pulumi.StringInput, useSudo bool, opts ...pulumi.ResourceOption) (Command, error) - MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) + MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) BuildCommandString( command pulumi.StringInput, diff --git a/components/command/runner.go b/components/command/runner.go index ea1393a77..0ecc53a33 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -326,7 +326,7 @@ func (r *RemoteRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, op return nil, err } - moveCommand, err := r.OsCommand().MoveRemoteFile(r, name, tempRemotePath, dst, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) + moveCommand, err := r.OsCommand().MoveFile(r, name, tempRemotePath, dst, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) if err != nil { return nil, err } diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index fbc40d8e2..7f80824f9 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -95,7 +95,7 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, return formattedCommand } -func (fs unixOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { +func (fs unixOSCommand) MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) copyCommand := pulumi.Sprintf(`cp "%v" "%v"`, source, destination) createCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; fi; %v'`, destination, destination, backupPath, copyCommand) diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index bdbaaf09e..e4b227575 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -81,7 +81,7 @@ func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, re return runner.CopyWindowsFile(name, localPath, remotePath, opts...) } -func (fs windowsOSCommand) MoveRemoteFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { +func (fs windowsOSCommand) MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) copyCommand := pulumi.Sprintf(`Copy-Item -Path '%v' -Destination '%v'`, source, destination) createCommand := pulumi.Sprintf(`if (Test-Path '%v') { Move-Item -Force -Path '%v' -Destination '%v' }; %v`, destination, destination, backupPath, copyCommand) From a6e0c8ed342bd5e63135af11cdc09acf762656ee Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 23 Dec 2024 09:43:41 +0100 Subject: [PATCH 17/19] [refactor-runner-adxt-783] copy*file are not exported anymore --- components/command/filemanager.go | 2 +- components/command/runner.go | 21 +++++++++++---------- components/command/unixOSCommand.go | 2 +- components/command/windowsOSCommand.go | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/components/command/filemanager.go b/components/command/filemanager.go index 55f11d9d9..bee2c626c 100644 --- a/components/command/filemanager.go +++ b/components/command/filemanager.go @@ -65,7 +65,7 @@ func (fm *FileManager) HomeDirectory(folderName string, opts ...pulumi.ResourceO } func (fm *FileManager) CopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return fm.runner.NewCopyFile(name, localPath, remotePath, opts...) + return fm.runner.newCopyFile(name, localPath, remotePath, opts...) } func (fm *FileManager) CopyInlineFile(fileContent pulumi.StringInput, remotePath string, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { diff --git a/components/command/runner.go b/components/command/runner.go index 0ecc53a33..63529e9bb 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -106,12 +106,13 @@ type Runner interface { Namer() namer.Namer Config() RunnerConfiguration OsCommand() OSCommand + PulumiOptions() []pulumi.ResourceOption Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) - NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) - CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) - CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) - PulumiOptions() []pulumi.ResourceOption + + newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) } var _ Runner = &RemoteRunner{} @@ -200,7 +201,7 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO return &RemoteCommand{cmd}, nil } -func (r *RemoteRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *RemoteRunner) newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) } @@ -265,7 +266,7 @@ func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOp return &LocalCommand{cmd}, nil } -func (r *LocalRunner) NewCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *LocalRunner) newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) } @@ -273,7 +274,7 @@ func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { return []pulumi.ResourceOption{} } -func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *LocalRunner) copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { createCmd := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) deleteCmd := pulumi.Sprintf("Remove-Item -Path '%v'", dst) useSudo := false @@ -287,7 +288,7 @@ func (r *LocalRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, }, opts...) } -func (r *LocalRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *LocalRunner) copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { createCmd := pulumi.Sprintf("cp '%v' '%v'", src, dst) deleteCmd := pulumi.Sprintf("rm '%v'", dst) useSudo := false @@ -301,7 +302,7 @@ func (r *LocalRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opt }, opts...) } -func (r *RemoteRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *RemoteRunner) copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { return remote.NewCopyFile(r.Environment().Ctx(), r.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ Connection: r.Config().connection, LocalPath: src, @@ -310,7 +311,7 @@ func (r *RemoteRunner) CopyWindowsFile(name string, src, dst pulumi.StringInput, }, utils.MergeOptions(r.PulumiOptions(), opts...)...) } -func (r *RemoteRunner) CopyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { +func (r *RemoteRunner) copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { tempRemotePath := src.ToStringOutput().ApplyT(func(path string) string { return filepath.Join(r.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) }).(pulumi.StringOutput) diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 7f80824f9..8698fb190 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -71,7 +71,7 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { } func (fs unixOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return runner.CopyUnixFile(name, localPath, remotePath, opts...) + return runner.copyUnixFile(name, localPath, remotePath, opts...) } func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, user string) pulumi.StringInput { diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index e4b227575..2cf9bf5f4 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -78,7 +78,7 @@ func (fs windowsOSCommand) IsPathAbsolute(path string) bool { } func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return runner.CopyWindowsFile(name, localPath, remotePath, opts...) + return runner.copyWindowsFile(name, localPath, remotePath, opts...) } func (fs windowsOSCommand) MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { From 8654e0593a94ed7062e2a58ea908441df2a78f5c Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 23 Dec 2024 10:58:59 +0100 Subject: [PATCH 18/19] [refactor-runner-adxt-783] copy*file: Moved implementation to commands --- components/command/osCommand.go | 2 + components/command/runner.go | 68 +------------------------- components/command/unixOSCommand.go | 42 +++++++++++++++- components/command/windowsOSCommand.go | 26 +++++++++- 4 files changed, 70 insertions(+), 68 deletions(-) diff --git a/components/command/osCommand.go b/components/command/osCommand.go index 2165eec5e..7b0983289 100644 --- a/components/command/osCommand.go +++ b/components/command/osCommand.go @@ -30,6 +30,8 @@ type OSCommand interface { IsPathAbsolute(path string) bool NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + copyLocalFile(runner *LocalRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) + copyRemoteFile(runner *RemoteRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) } // ------------------------------ diff --git a/components/command/runner.go b/components/command/runner.go index 63529e9bb..8ff4832d2 100644 --- a/components/command/runner.go +++ b/components/command/runner.go @@ -2,7 +2,6 @@ package command import ( "fmt" - "path/filepath" "github.com/pulumi/pulumi-command/sdk/go/command/local" "github.com/pulumi/pulumi-command/sdk/go/command/remote" @@ -111,8 +110,6 @@ type Runner interface { Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) - copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) - copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) } var _ Runner = &RemoteRunner{} @@ -202,7 +199,7 @@ func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceO } func (r *RemoteRunner) newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) + return r.osCommand.copyRemoteFile(r, name, localPath, remotePath, opts...) } func (r *RemoteRunner) PulumiOptions() []pulumi.ResourceOption { @@ -267,70 +264,9 @@ func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOp } func (r *LocalRunner) newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return r.osCommand.NewCopyFile(r, name, localPath, remotePath, opts...) + return r.osCommand.copyLocalFile(r, name, localPath, remotePath, opts...) } func (r *LocalRunner) PulumiOptions() []pulumi.ResourceOption { return []pulumi.ResourceOption{} } - -func (r *LocalRunner) copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - createCmd := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) - deleteCmd := pulumi.Sprintf("Remove-Item -Path '%v'", dst) - useSudo := false - - return r.Command(name, - &Args{ - Create: createCmd, - Delete: deleteCmd, - Sudo: useSudo, - Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, - }, opts...) -} - -func (r *LocalRunner) copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - createCmd := pulumi.Sprintf("cp '%v' '%v'", src, dst) - deleteCmd := pulumi.Sprintf("rm '%v'", dst) - useSudo := false - - return r.Command(name, - &Args{ - Create: createCmd, - Delete: deleteCmd, - Sudo: useSudo, - Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, - }, opts...) -} - -func (r *RemoteRunner) copyWindowsFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return remote.NewCopyFile(r.Environment().Ctx(), r.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: r.Config().connection, - LocalPath: src, - RemotePath: dst, - Triggers: pulumi.Array{src, dst}, - }, utils.MergeOptions(r.PulumiOptions(), opts...)...) -} - -func (r *RemoteRunner) copyUnixFile(name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - tempRemotePath := src.ToStringOutput().ApplyT(func(path string) string { - return filepath.Join(r.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) - }).(pulumi.StringOutput) - - tempCopyFile, err := remote.NewCopyFile(r.Environment().Ctx(), r.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ - Connection: r.Config().connection, - LocalPath: src, - RemotePath: tempRemotePath, - Triggers: pulumi.Array{src, tempRemotePath}, - }, utils.MergeOptions(r.PulumiOptions(), opts...)...) - - if err != nil { - return nil, err - } - - moveCommand, err := r.OsCommand().MoveFile(r, name, tempRemotePath, dst, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) - if err != nil { - return nil, err - } - - return moveCommand, err -} diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 8698fb190..9514aeb41 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -2,11 +2,13 @@ package command import ( "fmt" + "path/filepath" "strings" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/alessio/shellescape" + "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -71,7 +73,7 @@ func (fs unixOSCommand) IsPathAbsolute(path string) bool { } func (fs unixOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return runner.copyUnixFile(name, localPath, remotePath, opts...) + return runner.newCopyFile(name, localPath, remotePath, opts...) } func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, user string) pulumi.StringInput { @@ -102,3 +104,41 @@ func (fs unixOSCommand) MoveFile(runner Runner, name string, source, destination deleteCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; else rm -f "%v"; fi'`, backupPath, backupPath, destination, destination) return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) } + +func (fs unixOSCommand) copyLocalFile(runner *LocalRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + createCmd := pulumi.Sprintf("cp '%v' '%v'", src, dst) + deleteCmd := pulumi.Sprintf("rm '%v'", dst) + useSudo := false + + return runner.Command(name, + &Args{ + Create: createCmd, + Delete: deleteCmd, + Sudo: useSudo, + Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, + }, opts...) +} + +func (fs unixOSCommand) copyRemoteFile(runner *RemoteRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + tempRemotePath := src.ToStringOutput().ApplyT(func(path string) string { + return filepath.Join(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path)) + }).(pulumi.StringOutput) + + tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: runner.Config().connection, + LocalPath: src, + RemotePath: tempRemotePath, + Triggers: pulumi.Array{src, tempRemotePath}, + }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) + + if err != nil { + return nil, err + } + + moveCommand, err := runner.OsCommand().MoveFile(runner, name, tempRemotePath, dst, true, utils.MergeOptions(opts, utils.PulumiDependsOn(tempCopyFile))...) + if err != nil { + return nil, err + } + + return moveCommand, err +} diff --git a/components/command/windowsOSCommand.go b/components/command/windowsOSCommand.go index 2cf9bf5f4..440cdbaf5 100644 --- a/components/command/windowsOSCommand.go +++ b/components/command/windowsOSCommand.go @@ -6,6 +6,7 @@ import ( "github.com/DataDog/test-infra-definitions/common/utils" + "github.com/pulumi/pulumi-command/sdk/go/command/remote" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -78,7 +79,7 @@ func (fs windowsOSCommand) IsPathAbsolute(path string) bool { } func (fs windowsOSCommand) NewCopyFile(runner Runner, name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { - return runner.copyWindowsFile(name, localPath, remotePath, opts...) + return runner.newCopyFile(name, localPath, remotePath, opts...) } func (fs windowsOSCommand) MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { @@ -88,3 +89,26 @@ func (fs windowsOSCommand) MoveFile(runner Runner, name string, source, destinat deleteCommand := pulumi.Sprintf(`if (Test-Path '%v') { Move-Item -Force -Path '%v' -Destination '%v' } else { Remove-Item -Force -Path %v }`, backupPath, backupPath, destination, destination) return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) } + +func (fs windowsOSCommand) copyLocalFile(runner *LocalRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + createCmd := pulumi.Sprintf("Copy-Item -Path '%v' -Destination '%v'", src, dst) + deleteCmd := pulumi.Sprintf("Remove-Item -Path '%v'", dst) + useSudo := false + + return runner.Command(name, + &Args{ + Create: createCmd, + Delete: deleteCmd, + Sudo: useSudo, + Triggers: pulumi.Array{createCmd, deleteCmd, pulumi.BoolPtr(useSudo)}, + }, opts...) +} + +func (fs windowsOSCommand) copyRemoteFile(runner *RemoteRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + return remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{ + Connection: runner.Config().connection, + LocalPath: src, + RemotePath: dst, + Triggers: pulumi.Array{src, dst}, + }, utils.MergeOptions(runner.PulumiOptions(), opts...)...) +} From ea4c5e1bf450e4f20135fd4ab21a524b32d845cb Mon Sep 17 00:00:00 2001 From: Celian Raimbault Date: Mon, 23 Dec 2024 13:59:18 +0100 Subject: [PATCH 19/19] [refactor-runner-adxt-783] Undone path sanitize modifications for copy linux file --- components/command/unixOSCommand.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/command/unixOSCommand.go b/components/command/unixOSCommand.go index 9514aeb41..35241eed6 100644 --- a/components/command/unixOSCommand.go +++ b/components/command/unixOSCommand.go @@ -99,9 +99,9 @@ func formatCommandIfNeeded(command pulumi.StringInput, sudo bool, password bool, func (fs unixOSCommand) MoveFile(runner Runner, name string, source, destination pulumi.StringInput, sudo bool, opts ...pulumi.ResourceOption) (Command, error) { backupPath := pulumi.Sprintf("%v.%s", destination, backupExtension) - copyCommand := pulumi.Sprintf(`cp "%v" "%v"`, source, destination) - createCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; fi; %v'`, destination, destination, backupPath, copyCommand) - deleteCommand := pulumi.Sprintf(`bash -c 'if [ -f "%v" ]; then mv -f "%v" "%v"; else rm -f "%v"; fi'`, backupPath, backupPath, destination, destination) + copyCommand := pulumi.Sprintf(`cp '%v' '%v'`, source, destination) + createCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; fi; %v'`, destination, destination, backupPath, copyCommand) + deleteCommand := pulumi.Sprintf(`bash -c 'if [ -f '%v' ]; then mv -f '%v' '%v'; else rm -f '%v'; fi'`, backupPath, backupPath, destination, destination) return copyRemoteFile(runner, fmt.Sprintf("move-file-%s", name), createCommand, deleteCommand, sudo, utils.MergeOptions(opts, pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))...) }