Skip to content

[ADXT-783] Refactor runner command arguments #1325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 27 commits into from
Jan 3, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
0f63992
[refactor-runner-adxt-783] Started refactoring
CelianR Dec 11, 2024
5057672
[refactor-runner-adxt-783] wip: components/os
CelianR Dec 11, 2024
f645c6d
[refactor-runner-adxt-783] Can build
CelianR Dec 11, 2024
254bf8f
[refactor-runner-adxt-783] Copy file
CelianR Dec 11, 2024
e9255ef
[refactor-runner-adxt-783] ad: Fixed
CelianR Dec 11, 2024
2b64d46
[refactor-runner-adxt-783] runner: Fixed lock warning
CelianR Dec 11, 2024
ab0b21a
[refactor-runner-adxt-783] wip
CelianR Dec 12, 2024
c0c3ac4
[refactor-runner-adxt-783] Fixed little error
CelianR Dec 13, 2024
c49c5fe
Fixed bug
CelianR Dec 16, 2024
d5ab53e
[refactor-runner-adxt-783] Cleaned code
CelianR Dec 17, 2024
2448f9d
[refactor-runner-adxt-783] command: Fixed env + refactored podman vm.go
CelianR Dec 17, 2024
74d4b02
[refactor-runner-adxt-783] Tested and cleaned code
CelianR Dec 18, 2024
846ecc0
Merge branch 'main' into celian/refactor-runner-adxt-783
CelianR Dec 18, 2024
18bea62
[refactor-runner-adxt-783] Cleaned code
CelianR Dec 18, 2024
3656bda
[refactor-runner-adxt-783] Cleaned code
CelianR Dec 18, 2024
36bcf5a
[refactor-runner-adxt-783] env: Vars are exported and not passed to t…
CelianR Dec 18, 2024
d1ef99f
[refactor-runner-adxt-783] renamed: MoveRemoteFile -> MoveFile
CelianR Dec 23, 2024
a6e0c8e
[refactor-runner-adxt-783] copy*file are not exported anymore
CelianR Dec 23, 2024
8654e05
[refactor-runner-adxt-783] copy*file: Moved implementation to commands
CelianR Dec 23, 2024
924ee08
Merge branch 'main' into celian/refactor-runner-adxt-783
CelianR Dec 23, 2024
ea4c5e1
[refactor-runner-adxt-783] Undone path sanitize modifications for cop…
CelianR Dec 23, 2024
40ffa48
Refactored command args
CelianR Dec 24, 2024
901b29a
[refactor-runner-continued-adxt-783] wip merge
CelianR Jan 3, 2025
884e85f
[fix-option] Fixed option package commit
CelianR Jan 3, 2025
b0c5a47
Merge branch 'celian/fix-option' into celian/refactor-runner-continue…
CelianR Jan 3, 2025
9948468
Merge branch 'main' into celian/refactor-runner-continued-adxt-783
CelianR Jan 3, 2025
b3e0308
[refactor-runner-continued-adxt-783] lint: Fixed linter
CelianR Jan 3, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions components/command/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func (m *GenericPackageManager) Ensure(packageRef string, transform Transformer,
}

cmdName := m.namer.ResourceName("install-"+packageRef, utils.StrHash(cmdStr))
cmdArgs := Args{
var cmdArgs RunnerCommandArgs = &Args{
Create: pulumi.String(cmdStr),
Environment: m.env,
Sudo: true,
Expand All @@ -65,7 +65,7 @@ func (m *GenericPackageManager) Ensure(packageRef string, transform Transformer,
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

cmd, err := m.runner.Command(cmdName, &cmdArgs, opts...)
cmd, err := m.runner.Command(cmdName, cmdArgs, opts...)
if err != nil {
return nil, err
}
Expand Down
63 changes: 47 additions & 16 deletions components/command/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ import (
"github.com/DataDog/test-infra-definitions/common/utils"
)

type RunnerCommandArgs interface {
Arguments() *Args
}

type Args struct {
Create pulumi.StringInput
Update pulumi.StringInput
Expand All @@ -26,27 +30,54 @@ type Args struct {
LocalDir pulumi.StringInput
}

func (args *Args) toLocalCommandArgs(config RunnerConfiguration, osCommand OSCommand) (*local.CommandArgs, error) {
type LocalArgs struct {
Args
// Only used for local commands
LocalAssetPaths pulumi.StringArrayInput
LocalDir pulumi.StringInput
}

var _ RunnerCommandArgs = &Args{}
var _ RunnerCommandArgs = &LocalArgs{}

func (args *Args) Arguments() *Args {
return args
}

func (args *LocalArgs) Arguments() *Args {
return &args.Args
}

func toLocalCommandArgs(cmdArgs RunnerCommandArgs, config RunnerConfiguration, osCommand OSCommand) (*local.CommandArgs, error) {
// Retrieve local specific arguments if provided
var assetsPath pulumi.StringArrayInput
var dir pulumi.StringInput
if localArgs, ok := cmdArgs.(*LocalArgs); ok {
assetsPath = localArgs.LocalAssetPaths
dir = localArgs.LocalDir
}

args := cmdArgs.Arguments()

return &local.CommandArgs{
Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user),
Update: osCommand.BuildCommandString(args.Update, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user),
Delete: osCommand.BuildCommandString(args.Delete, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user),
Triggers: args.Triggers,
Stdin: args.Stdin,
AssetPaths: args.LocalAssetPaths,
Dir: args.LocalDir,
AssetPaths: assetsPath,
Dir: dir,
}, nil
}

func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCommand) (*remote.CommandArgs, error) {
func toRemoteCommandArgs(cmdArgs RunnerCommandArgs, config RunnerConfiguration, osCommand OSCommand) (*remote.CommandArgs, error) {
// Ensure no local arguments are passed to remote commands
if args.LocalAssetPaths != nil {
return nil, fmt.Errorf("local asset paths are not supported in remote commands")
}
if args.LocalDir != nil {
return nil, fmt.Errorf("local dir is not supported in remote commands")
if _, ok := cmdArgs.(*LocalArgs); ok {
return nil, fmt.Errorf("local arguments are not allowed for remote commands")
}

args := cmdArgs.Arguments()

return &remote.CommandArgs{
Connection: config.connection,
Create: osCommand.BuildCommandString(args.Create, args.Environment, args.Sudo, args.RequirePasswordFromStdin, config.user),
Expand All @@ -59,7 +90,7 @@ func (args *Args) toRemoteCommandArgs(config RunnerConfiguration, osCommand OSCo

// Transformer is a function that can be used to modify the command name and args.
// Examples: swapping `args.Delete` with `args.Create`, or adding `args.Triggers`, or editing the name
type Transformer func(name string, args Args) (string, Args)
type Transformer func(name string, args RunnerCommandArgs) (string, RunnerCommandArgs)

type RunnerConfiguration struct {
user string
Expand Down Expand Up @@ -107,7 +138,7 @@ type Runner interface {
OsCommand() OSCommand
PulumiOptions() []pulumi.ResourceOption

Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error)
Command(name string, args RunnerCommandArgs, opts ...pulumi.ResourceOption) (Command, error)

newCopyFile(name string, localPath, remotePath pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error)
}
Expand Down Expand Up @@ -179,12 +210,12 @@ func (r *RemoteRunner) OsCommand() OSCommand {
return r.osCommand
}

func (r *RemoteRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) {
if args.Sudo && r.config.user != "" {
func (r *RemoteRunner) Command(name string, args RunnerCommandArgs, opts ...pulumi.ResourceOption) (Command, error) {
if args.Arguments().Sudo && r.config.user != "" {
r.e.Ctx().Log.Info(fmt.Sprintf("warning: running sudo command on a runner with user %s, discarding user", r.config.user), nil)
}

remoteArgs, err := args.toRemoteCommandArgs(r.config, r.osCommand)
remoteArgs, err := toRemoteCommandArgs(args, r.config, r.osCommand)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -247,9 +278,9 @@ func (r *LocalRunner) OsCommand() OSCommand {
return r.osCommand
}

func (r *LocalRunner) Command(name string, args *Args, opts ...pulumi.ResourceOption) (Command, error) {
func (r *LocalRunner) Command(name string, args RunnerCommandArgs, opts ...pulumi.ResourceOption) (Command, error) {
opts = utils.MergeOptions[pulumi.ResourceOption](opts, r.e.WithProviders(config.ProviderCommand))
localArgs, err := args.toLocalCommandArgs(r.config, r.osCommand)
localArgs, err := toLocalCommandArgs(args, r.config, r.osCommand)
if err != nil {
return nil, err
}
Expand Down
10 changes: 6 additions & 4 deletions components/datadog/agent/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,10 @@ func (h *HostAgent) installAgent(env config.Env, params *agentparams.Params, bas
// For this reason we have another `restartAgentServices` in `installIntegrationConfigsAndFiles` that is triggered when an integration is deleted.
_, err = h.manager.restartAgentServices(
// Transformer used to add triggers to the restart command
func(name string, args command.Args) (string, command.Args) {
func(name string, cmdArgs command.RunnerCommandArgs) (string, command.RunnerCommandArgs) {
args := *cmdArgs.Arguments()
args.Triggers = pulumi.Array{configFiles["datadog.yaml"], configFiles["system-probe.yaml"], configFiles["security-agent.yaml"], pulumi.String(intgHash)}
return name, args
return name, &args
},
utils.PulumiDependsOn(h),
)
Expand Down Expand Up @@ -214,11 +215,12 @@ func (h *HostAgent) installIntegrationConfigsAndFiles(
restartCmd, err := h.manager.restartAgentServices(
// Use a transformer to inject triggers on intg hash and move `restart` command from `Create` to `Delete`
// so that it's run after the `Delete` commands of the integrations.
func(name string, args command.Args) (string, command.Args) {
func(name string, cmdArgs command.RunnerCommandArgs) (string, command.RunnerCommandArgs) {
args := *cmdArgs.Arguments()
args.Triggers = pulumi.Array{pulumi.String(hash)}
args.Delete = args.Create
args.Create = nil
return name + "-on-intg-removal", args
return name + "-on-intg-removal", &args
})
if err != nil {
return nil, "", err
Expand Down
4 changes: 2 additions & 2 deletions components/datadog/agent/host_windowsos.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ while ($tries -lt 5) {
Exit $exitCode
`

cmdArgs := command.Args{
var cmdArgs command.RunnerCommandArgs = &command.Args{
Create: pulumi.String(cmd),
}

Expand All @@ -106,7 +106,7 @@ while ($tries -lt 5) {
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

return am.host.OS.Runner().Command(cmdName, &cmdArgs, opts...)
return am.host.OS.Runner().Command(cmdName, cmdArgs, opts...)
}

func getAgentURL(version agentparams.PackageVersion) (string, error) {
Expand Down
8 changes: 4 additions & 4 deletions components/os/linux_servicemanagers.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func newSystemdServiceManager(e config.Env, runner command.Runner) ServiceManage

func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) {
cmdName := s.e.CommonNamer().ResourceName("running", serviceName)
cmdArgs := command.Args{
var cmdArgs command.RunnerCommandArgs = &command.Args{
Sudo: true,
Create: pulumi.String("systemctl restart " + serviceName),
}
Expand All @@ -29,7 +29,7 @@ func (s *systemdServiceManager) EnsureRestarted(serviceName string, transform co
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

return s.runner.Command(cmdName, &cmdArgs, opts...)
return s.runner.Command(cmdName, cmdArgs, opts...)
}

type sysvinitServiceManager struct {
Expand All @@ -45,7 +45,7 @@ func (s *sysvinitServiceManager) EnsureRestarted(serviceName string, transform c
cmdName := s.e.CommonNamer().ResourceName("running", serviceName)
// To the difference of systemctl the restart doesn't work if the service isn't already running
// so instead we run a stop command that we allow to fail and then a start command
cmdArgs := command.Args{
var cmdArgs command.RunnerCommandArgs = &command.Args{
Sudo: false,
Create: pulumi.String(fmt.Sprintf("sudo stop %[1]s; sudo start %[1]s", serviceName)),
}
Expand All @@ -55,5 +55,5 @@ func (s *sysvinitServiceManager) EnsureRestarted(serviceName string, transform c
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

return s.runner.Command(cmdName, &cmdArgs, opts...)
return s.runner.Command(cmdName, cmdArgs, opts...)
}
4 changes: 2 additions & 2 deletions components/os/macos_servicemanagers.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func newMacOSServiceManager(e config.Env, runner command.Runner) ServiceManager

func (s *macOSServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) {
cmdName := s.e.CommonNamer().ResourceName("running", serviceName)
cmdArgs := command.Args{
var cmdArgs command.RunnerCommandArgs = &command.Args{
Sudo: true,
Create: pulumi.String(fmt.Sprintf("launchctl stop %s && launchctl start %s", serviceName, serviceName)),
}
Expand All @@ -29,5 +29,5 @@ func (s *macOSServiceManager) EnsureRestarted(serviceName string, transform comm
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

return s.runner.Command(cmdName, &cmdArgs, opts...)
return s.runner.Command(cmdName, cmdArgs, opts...)
}
4 changes: 2 additions & 2 deletions components/os/windows_servicemanagers.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ func newWindowsServiceManager(e config.Env, runner command.Runner) ServiceManage

func (s *windowsServiceManager) EnsureRestarted(serviceName string, transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error) {
cmdName := s.e.CommonNamer().ResourceName("running", serviceName)
cmdArgs := command.Args{
var cmdArgs command.RunnerCommandArgs = &command.Args{
Create: pulumi.String("Restart-Service -Name " + serviceName),
}

Expand All @@ -26,5 +26,5 @@ func (s *windowsServiceManager) EnsureRestarted(serviceName string, transform co
cmdName, cmdArgs = transform(cmdName, cmdArgs)
}

return s.runner.Command(cmdName, &cmdArgs, opts...)
return s.runner.Command(cmdName, cmdArgs, opts...)
}
24 changes: 14 additions & 10 deletions resources/local/podman/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,23 +56,27 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou
podmanCommand := "podman --config " + dataPath

opts = utils.MergeOptions(opts, utils.PulumiDependsOn(dockerFile, dockerConfig))
buildPodman, err := runner.Command("podman-build"+args.Name, &command.Args{
Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))},
Create: pulumi.Sprintf("%s build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", podmanCommand, args.Name),
Delete: pulumi.Sprintf("%s rmi %s", podmanCommand, args.Name),
Triggers: pulumi.Array{},
buildPodman, err := runner.Command("podman-build"+args.Name, &command.LocalArgs{
Args: command.Args{
Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))},
Create: pulumi.Sprintf("%s build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", podmanCommand, args.Name),
Delete: pulumi.Sprintf("%s rmi %s", podmanCommand, args.Name),
Triggers: pulumi.Array{},
},
LocalAssetPaths: pulumi.StringArray{},
LocalDir: pulumi.String(dataPath),
}, opts...)
if err != nil {
return pulumi.StringOutput{}, "", -1, err
}
opts = utils.MergeOptions(opts, utils.PulumiDependsOn(buildPodman))
runPodman, err := runner.Command("podman-run"+args.Name, &command.Args{
Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))},
Create: pulumi.Sprintf("%s run -d --name=%[2]s_run -p 50022:22 %[2]s", podmanCommand, args.Name),
Delete: pulumi.Sprintf("%s stop %[2]s_run && podman rm %[2]s_run", podmanCommand, args.Name),
Triggers: pulumi.Array{},
runPodman, err := runner.Command("podman-run"+args.Name, &command.LocalArgs{
Args: command.Args{
Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))},
Create: pulumi.Sprintf("%s run -d --name=%[2]s_run -p 50022:22 %[2]s", podmanCommand, args.Name),
Delete: pulumi.Sprintf("%s stop %[2]s_run && podman rm %[2]s_run", podmanCommand, args.Name),
Triggers: pulumi.Array{},
},
LocalAssetPaths: pulumi.StringArray{},
LocalDir: pulumi.String(dataPath),
}, opts...)
Expand Down
Loading