diff --git a/generator/resources/ec2_linux_onprem_test_matrix.json b/generator/resources/ec2_linux_onprem_test_matrix.json index fa16915df..fa86860dc 100644 --- a/generator/resources/ec2_linux_onprem_test_matrix.json +++ b/generator/resources/ec2_linux_onprem_test_matrix.json @@ -1,14 +1,26 @@ [ { - "os": "al2", - "username": "ec2-user", + "os": "ubuntu-25", + "username": "ubuntu", "instanceType":"t3a.medium", - "installAgentCommand": "go run ./install/install_agent.go rpm", - "agentStartCommand": "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m onPremise -s -c ", - "ami": "cloudwatch-agent-integration-test-al2*", - "caCertPath": "/etc/ssl/certs/ca-bundle.crt", + "installAgentCommand": "go run ./install/install_agent.go deb", + "ami": "cloudwatch-agent-integration-test-ubuntu-25*", + "caCertPath": "/etc/ssl/certs/ca-certificates.crt", "arc": "amd64", - "binaryName": "amazon-cloudwatch-agent.rpm", - "family": "linux" + "binaryName": "amazon-cloudwatch-agent.deb", + "family": "linux", + "agentStartCommand": "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m onPremise -s -c " + }, + { + "os": "debian-12", + "username": "admin", + "instanceType": "c6g.large", + "installAgentCommand": "go run ./install/install_agent.go deb", + "ami": "cloudwatch-agent-integration-test-debian-12-arm64*", + "caCertPath": "/etc/ssl/certs/ca-certificates.crt", + "arc": "arm64", + "binaryName": "amazon-cloudwatch-agent.deb", + "family": "linux", + "agentStartCommand": "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m onPremise -s -c " } ] \ No newline at end of file diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 79cb55ade..2cdbef074 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -77,6 +77,9 @@ var testTypeToTestConfig = map[string][]testConfig{ "ec2_linux_wd_nvidia": { {testDir: "./test/workload_discovery"}, }, + "ec2_linux_onprem": { + {testDir: "./test/cloudwatchlogs"}, + }, testTypeKeyEc2Linux: { {testDir: "./test/ca_bundle"}, {testDir: "./test/cloudwatchlogs"}, @@ -429,26 +432,12 @@ var partitionTests = map[string]partition{ }, } -func copyAllEC2LinuxTestForOnpremTesting() { - /* Some tests need to be fixed in order to run in both environment, so for now for PoC, run one that works. - testTypeToTestConfig["ec2_linux_onprem"] = testTypeToTestConfig[testTypeKeyEc2Linux] - */ - testTypeToTestConfig["ec2_linux_onprem"] = []testConfig{ - { - testDir: "./test/lvm", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - } -} - func main() { useE2E := flag.Bool("e2e", false, "Use e2e test matrix generation") flag.Parse() configMap := testTypeToTestConfig - if !*useE2E { - copyAllEC2LinuxTestForOnpremTesting() - } else { + if *useE2E { configMap = testTypeToTestConfigE2E } diff --git a/terraform/ec2/linux/main.tf b/terraform/ec2/linux/main.tf index 747c500fb..6d88a5b96 100644 --- a/terraform/ec2/linux/main.tf +++ b/terraform/ec2/linux/main.tf @@ -37,6 +37,12 @@ locals { binary_uri = var.is_canary ? "${var.s3_bucket}/release/amazon_linux/${var.arc}/latest/${var.binary_name}" : "${var.s3_bucket}/integration-test/binary/${var.cwa_github_sha}/linux/${var.arc}/${var.binary_name}" // list of test that require instance reboot reboot_required_tests = tolist(["./test/restart"]) + + // On-premises specific configuration + is_onprem = var.is_onprem + + // Pre-test setup command + pre_test_setup_cmd = local.is_onprem ? "echo 'Pre-test setup: Replacing {instance_id} and $${aws:InstanceId} placeholders in test resource configs'; find . -path '*/resources/*.json' -exec sed -i 's/{instance_id}/${module.linux_common.cwagent_id}/g' {} \\; -exec sed -i 's/$${aws:InstanceId}/${module.linux_common.cwagent_id}/g' {} \\; && echo 'Updated all config files in resources directories'" : var.pre_test_setup } ##################################################################### @@ -53,24 +59,54 @@ resource "null_resource" "integration_test_setup" { # Prepare Integration Test provisioner "remote-exec" { - inline = [ - "echo sha ${var.cwa_github_sha}", - "sudo cloud-init status --wait", - "echo clone ${var.github_test_repo} branch ${var.github_test_repo_branch} and install agent", - # check for vendor directory specifically instead of overall test repo to avoid issues with SELinux - "if [ ! -d amazon-cloudwatch-agent-test/vendor ]; then", - "echo 'Vendor directory (test repo dependencies) not found, cloning...'", - "sudo rm -r amazon-cloudwatch-agent-test", - "git clone --branch ${var.github_test_repo_branch} ${var.github_test_repo} -q", - "else", - "echo 'Test repo already exists, skipping clone'", - "fi", - "cd amazon-cloudwatch-agent-test", - "git rev-parse --short HEAD", - "aws s3 cp --no-progress s3://${local.binary_uri} .", - "export PATH=$PATH:/snap/bin:/usr/local/go/bin", - var.install_agent, - ] + inline = concat( + [ + "echo sha ${var.cwa_github_sha}", + "sudo cloud-init status --wait", + "echo clone ${var.github_test_repo} branch ${var.github_test_repo_branch} and install agent", + # check for vendor directory specifically instead of overall test repo to avoid issues with SELinux + "if [ ! -d amazon-cloudwatch-agent-test/vendor ]; then", + "echo 'Vendor directory (test repo dependencies) not found, cloning...'", + "sudo rm -r amazon-cloudwatch-agent-test", + "git clone --branch ${var.github_test_repo_branch} ${var.github_test_repo} -q", + "else", + "echo 'Test repo already exists, skipping clone'", + "fi", + "cd amazon-cloudwatch-agent-test", + "git rev-parse --short HEAD", + "aws s3 cp --no-progress s3://${local.binary_uri} .", + "export PATH=$PATH:/snap/bin:/usr/local/go/bin", + ], + + # On-premises specific setup + local.is_onprem ? [ + "sudo mkdir -p ~/.aws", + "echo creating credentials file that the agent uses by default for onprem", + "printf '[default]\\nregion = us-west-2\\n' | sudo tee ~/.aws/config", + "echo attempting to assume role for on-premises credentials", + "ASSUME_ROLE_OUTPUT=$(aws sts assume-role --role-arn ${module.linux_common.cwa_onprem_assumed_iam_role_arm} --role-session-name onpremtest --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' --output text)", + "if [ $? -ne 0 ]; then echo 'Failed to assume role'; exit 1; fi", + "echo 'Creating default credentials'", + "printf '[default]\\naws_access_key_id=%s\\naws_secret_access_key=%s\\naws_session_token=%s\\n' $ASSUME_ROLE_OUTPUT | sudo tee ~/.aws/credentials>/dev/null", + "echo verifying credentials are working", + "aws sts get-caller-identity || echo 'Credentials test failed'", + "echo turning off imds access in order to make agent start with onprem mode", + "aws ec2 modify-instance-metadata-options --instance-id ${module.linux_common.cwagent_id} --http-endpoint disabled", + "echo waiting for IMDS to be fully disabled", + "sleep 10", + "sudo mkdir -p /opt/aws/amazon-cloudwatch-agent/etc", + "printf '[credentials]\\n shared_credential_profile = \"default\"\\n shared_credential_file = \"/home/${var.user}/.aws/credentials\"\\n' | sudo tee /opt/aws/amazon-cloudwatch-agent/etc/common-config.toml>/dev/null", + "echo setting environment variables for agent", + "echo 'RUN_IN_AWS=false' | sudo tee -a /opt/aws/amazon-cloudwatch-agent/etc/env-config", + "echo 'INSTANCE_ID=${module.linux_common.cwagent_id}' | sudo tee -a /opt/aws/amazon-cloudwatch-agent/etc/env-config", + "echo 'export RUN_IN_AWS=false' | sudo tee -a /etc/environment", + "echo 'export INSTANCE_ID=${module.linux_common.cwagent_id}' | sudo tee -a /etc/environment", + ] : [], + + [ + var.install_agent, + ] + ) } depends_on = [ @@ -128,11 +164,12 @@ resource "null_resource" "integration_test_run" { inline = concat( [ "echo Preparing environment...", - "sudo yum install amazon-cloudwatch-agent -y", + "nohup bash -c 'while true; do sudo shutdown -c; sleep 30; done' >/dev/null 2>&1 &", ], # SELinux test setup (if enabled) var.is_selinux_test ? [ + "sudo yum install amazon-cloudwatch-agent -y", "echo Running SELinux test setup...", "sudo yum install selinux-policy selinux-policy-targeted policycoreutils-python-utils selinux-policy-devel -y", "sudo setenforce 1", @@ -153,14 +190,40 @@ resource "null_resource" "integration_test_run" { "export LOCAL_STACK_HOST_NAME=${var.local_stack_host_name}", "export AWS_REGION=${var.region}", "export PATH=$PATH:/snap/bin:/usr/local/go/bin", + ], + + [ "echo Running integration test...", "cd ~/amazon-cloudwatch-agent-test", - "nohup bash -c 'while true; do sudo shutdown -c; sleep 30; done' >/dev/null 2>&1 &", + ], + + # On-premises specific environment variables + local.is_onprem ? [ + "export RUN_IN_AWS=false", + "export AWS_EC2_METADATA_DISABLED=true", + "export AWS_PROFILE=default", + "export AWS_SHARED_CREDENTIALS_FILE=~/.aws/credentials", + "export AWS_CONFIG_FILE=~/.aws/config", + "echo 'Environment variables for on-premises test:'", + "echo 'AWS_REGION='$AWS_REGION", + "echo 'RUN_IN_AWS='$RUN_IN_AWS", + "echo 'AWS_EC2_METADATA_DISABLED='$AWS_EC2_METADATA_DISABLED", + "echo 'AWS_PROFILE='$AWS_PROFILE", + "echo 'Instance ID parameter: ${module.linux_common.cwagent_id}'", + "echo 'Testing AWS credentials:'", + "aws sts get-caller-identity || echo 'AWS credentials test failed'", + "echo 'Testing agent credentials:'", + "sudo aws sts get-caller-identity || echo 'Agent credentials test failed'", + "echo 'Pre-test setup: Replacing {instance_id} and $${aws:InstanceId} placeholders in test resource configs'; find . -path '${var.test_dir}/resources/*.json' -exec sed -i 's/{instance_id}/${module.linux_common.cwagent_id}/g' {} \\; -exec sed -i 's/$${aws:InstanceId}/${module.linux_common.cwagent_id}/g' {} \\; && echo 'Updated all config files in resources directories'" + ] : [ "echo Running sanity test...", "go test ./test/sanity -p 1 -v", + ], + + [ var.pre_test_setup, - # Integration test execution - "go test ${var.test_dir} -p 1 -timeout 1h -computeType=EC2 -bucket=${var.s3_bucket} -plugins='${var.plugin_tests}' -excludedTests='${var.excluded_tests}' -cwaCommitSha=${var.cwa_github_sha} -caCertPath=${var.ca_cert_path} -proxyUrl=${module.linux_common.proxy_instance_proxy_ip} -instanceId=${module.linux_common.cwagent_id} ${length(regexall("/amp", var.test_dir)) > 0 ? "-ampWorkspaceId=${module.amp[0].workspace_id} " : ""}-v" + # Integration test execution with conditional agent start command + "go test ${var.test_dir} -p 1 -timeout 1h -computeType=EC2 -bucket=${var.s3_bucket} -plugins='${var.plugin_tests}' -excludedTests='${var.excluded_tests}' -cwaCommitSha=${var.cwa_github_sha} -caCertPath=${var.ca_cert_path} -proxyUrl=${module.linux_common.proxy_instance_proxy_ip} -instanceId=${module.linux_common.cwagent_id} ${local.is_onprem ? "-agentStartCommand='${var.agent_start}'" : ""} ${length(regexall("/amp", var.test_dir)) > 0 ? "-ampWorkspaceId=${module.amp[0].workspace_id} " : ""}-v" ], ) } diff --git a/terraform/ec2/linux/variables.tf b/terraform/ec2/linux/variables.tf index 6038b2ed8..3eb84350c 100644 --- a/terraform/ec2/linux/variables.tf +++ b/terraform/ec2/linux/variables.tf @@ -127,4 +127,10 @@ variable "agent_start" { description = "default command should be for ec2 with linux" type = string default = "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c " +} + +variable "is_onprem" { + description = "Whether to run in on-premises mode instead of EC2 mode" + type = bool + default = false } \ No newline at end of file diff --git a/terraform/ec2/linux_onprem/main.tf b/terraform/ec2/linux_onprem/main.tf deleted file mode 100644 index 5e19ec2c2..000000000 --- a/terraform/ec2/linux_onprem/main.tf +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -module "linux_common" { - source = "../common/linux" - - region = var.region - ec2_instance_type = var.ec2_instance_type - ssh_key_name = var.ssh_key_name - ami = var.ami - ssh_key_value = var.ssh_key_value - user = var.user - arc = var.arc - test_name = var.test_name - test_dir = var.test_dir - is_canary = var.is_canary -} - -module "reboot_common" { - source = "../common/linux_reboot" - - test_dir = var.test_dir - reboot_required_tests = local.reboot_required_tests - private_key_content = module.linux_common.private_key_content - cwagent_public_ip = module.linux_common.cwagent_public_ip - user = var.user - - depends_on = [ - null_resource.integration_test_setup, - ] -} - -locals { - // list of test that require instance reboot - reboot_required_tests = tolist(["./test/restart"]) - // Canary downloads latest binary. Integration test downloads binary connect to git hash. - binary_uri = var.is_canary ? "${var.s3_bucket}/release/amazon_linux/${var.arc}/latest/${var.binary_name}" : "${var.s3_bucket}/integration-test/binary/${var.cwa_github_sha}/linux/${var.arc}/${var.binary_name}" -} - -##################################################################### -# Execute test -##################################################################### - -resource "null_resource" "integration_test_setup" { - connection { - type = "ssh" - user = var.user - private_key = module.linux_common.private_key_content - host = module.linux_common.cwagent_public_ip - } - - # Prepare Integration Test. - ## Disabling imds endpoint here in order to keep the ability to ssh. If launching an instance with it disabled, ssh doesn't work. - ## If imds is not accessible, and RUN_IN_AWS env variable isn't set to true, then the agent considers it being in an onprem host. - provisioner "remote-exec" { - inline = [ - "echo sha ${var.cwa_github_sha}", - "sudo cloud-init status --wait", - "echo clone and install agent", - "git clone --branch ${var.github_test_repo_branch} ${var.github_test_repo}", - "cd amazon-cloudwatch-agent-test", - "aws s3 cp s3://${local.binary_uri} .", - "export PATH=$PATH:/snap/bin:/usr/local/go/bin", - "echo installing agent", - var.install_agent, - "sudo mkdir -p ~/.aws", - "sudo mkdir -p /.aws", - "echo creating credentials file that the agent uses by default for onprem", - "printf '\n[profile AmazonCloudWatchAgent]\nregion = us-west-2' | sudo tee -a /.aws/config", - "printf '\n[AmazonCloudWatchAgent]\naws_access_key_id=%s\naws_secret_access_key=%s\naws_session_token=%s' $(aws sts assume-role --role-arn ${module.linux_common.cwa_onprem_assumed_iam_role_arm} --role-session-name onpremtest --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' --output text) | sudo tee -a /.aws/credentials>/dev/null", - "printf '[credentials]\n shared_credential_profile = \"AmazonCloudWatchAgent\"\n shared_credential_file = \"/.aws/credentials\"' | sudo tee /opt/aws/amazon-cloudwatch-agent/etc/common-config.toml>/dev/null", - "echo write the same credentials as default profile as well. AWS SDK clients used for testing looks for default. Without this, would have needed to specify profile name in the test code", - "printf '\n[default]\nregion = us-west-2' | sudo tee -a ~/.aws/config", - "printf '\n[default]\naws_access_key_id=%s\naws_secret_access_key=%s\naws_session_token=%s' $(aws sts assume-role --role-arn ${module.linux_common.cwa_onprem_assumed_iam_role_arm} --role-session-name test --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' --output text) | sudo tee -a ~/.aws/credentials>/dev/null", - "echo turning off imds access in order to make agent start with onprem mode", - "aws ec2 modify-instance-metadata-options --instance-id ${module.linux_common.cwagent_id} --http-endpoint disabled", - ] - } - - depends_on = [ - module.linux_common, - ] -} - -resource "null_resource" "integration_test_run" { - connection { - type = "ssh" - user = var.user - private_key = module.linux_common.private_key_content - host = module.linux_common.cwagent_public_ip - } - - #Run sanity check and integration test - provisioner "remote-exec" { - inline = [ - "echo prepare environment", - "export LOCAL_STACK_HOST_NAME=${var.local_stack_host_name}", - "export AWS_REGION=${var.region}", - "export PATH=$PATH:/snap/bin:/usr/local/go/bin", - "echo run integration test", - "cd ~/amazon-cloudwatch-agent-test", - "nohup bash -c 'while true; do sudo shutdown -c; sleep 30; done' >/dev/null 2>&1 &", - "echo run sanity test && go test ./test/sanity -p 1 -v", - "go test ${var.test_dir} -p 1 -timeout 1h -computeType=EC2 -bucket=${var.s3_bucket} -plugins='${var.plugin_tests}' -cwaCommitSha=${var.cwa_github_sha} -caCertPath=${var.ca_cert_path} -proxyUrl=${module.linux_common.proxy_instance_proxy_ip} -instanceId=${module.linux_common.cwagent_id} -agentStartCommand='${var.agent_start}' -v", - ] - } - - depends_on = [ - null_resource.integration_test_setup, - module.reboot_common, - ] -} \ No newline at end of file diff --git a/terraform/ec2/linux_onprem/providers.tf b/terraform/ec2/linux_onprem/providers.tf deleted file mode 100644 index d8a1f722b..000000000 --- a/terraform/ec2/linux_onprem/providers.tf +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -provider "aws" { - region = var.region -} \ No newline at end of file diff --git a/terraform/ec2/linux_onprem/variables.tf b/terraform/ec2/linux_onprem/variables.tf deleted file mode 100644 index 6a6017683..000000000 --- a/terraform/ec2/linux_onprem/variables.tf +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -variable "region" { - type = string - default = "us-west-2" -} - -variable "ec2_instance_type" { - type = string - default = "t3a.medium" -} - -variable "ssh_key_name" { - type = string - default = "" -} - -variable "ami" { - type = string - default = "cloudwatch-agent-integration-test-ubuntu*" -} - -variable "ssh_key_value" { - type = string - default = "" -} - -variable "user" { - type = string - default = "" -} - -variable "install_agent" { - description = "go run ./install/install_agent.go deb or go run ./install/install_agent.go rpm" - type = string - default = "go run ./install/install_agent.go rpm" -} - -variable "ca_cert_path" { - type = string - default = "" -} - -variable "arc" { - type = string - default = "amd64" - - validation { - condition = contains(["amd64", "arm64"], var.arc) - error_message = "Valid values for arc are (amd64, arm64)." - } -} - -variable "binary_name" { - type = string - default = "" -} - -variable "local_stack_host_name" { - type = string - default = "localhost.localstack.cloud" -} - -variable "s3_bucket" { - type = string - default = "" -} - -variable "test_name" { - type = string - default = "" -} - -variable "test_dir" { - type = string - default = "" -} - -variable "cwa_github_sha" { - type = string - default = "" -} - -variable "github_test_repo" { - type = string - default = "https://github.com/aws/amazon-cloudwatch-agent-test.git" -} - -variable "github_test_repo_branch" { - type = string - default = "main" -} - -variable "is_canary" { - type = bool - default = false -} - -variable "plugin_tests" { - type = string - default = "" -} - -variable "agent_start" { - description = "default command is for onprem with linux" - type = string - default = "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m onPremise -s -c " -} \ No newline at end of file diff --git a/test/cloudwatchlogs/publish_logs_test.go b/test/cloudwatchlogs/publish_logs_test.go index faa166b18..f02bd3d05 100644 --- a/test/cloudwatchlogs/publish_logs_test.go +++ b/test/cloudwatchlogs/publish_logs_test.go @@ -95,7 +95,12 @@ func init() { func TestWriteLogsToCloudWatch(t *testing.T) { // this uses the {instance_id} placeholder in the agent configuration, // so we need to determine the host's instance ID for validation - instanceId := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() + instanceId := env.InstanceId + if instanceId == "" { + // Fallback to IMDS if not provided via command line (for backward compatibility) + instanceId = awsservice.GetInstanceId() + } log.Printf("Found instance id %s", instanceId) defer awsservice.DeleteLogGroupAndStream(instanceId, instanceId) @@ -181,7 +186,12 @@ func TestAutoRemovalFileRotation(t *testing.T) { // 3. The file should be rotated again, and a new log line of size GREATER THAN N should be written // 4. All three log lines, in full, should be visible in CloudWatch Logs func TestRotatingLogsDoesNotSkipLines(t *testing.T) { - instanceId := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() + instanceId := env.InstanceId + if instanceId == "" { + // Fallback to IMDS if not provided via command line (for backward compatibility) + instanceId = awsservice.GetInstanceId() + } cfgFilePath := "resources/config_log_rotated.json" log.Printf("Found instance id %s", instanceId) @@ -238,7 +248,12 @@ func TestRotatingLogsDoesNotSkipLines(t *testing.T) { } func TestLogGroupClass(t *testing.T) { - instanceId := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() + instanceId := env.InstanceId + if instanceId == "" { + // Fallback to IMDS if not provided via command line (for backward compatibility) + instanceId = awsservice.GetInstanceId() + } logFile, err := os.Create(logFilePath) agentRuntime := 20 * time.Second // default flush interval is 5 seconds if err != nil { @@ -310,7 +325,12 @@ func writeSleepRestart(t *testing.T, f *os.File, configPath string, linesPerLoop } func autoRemovalTestCleanup() { - instanceId := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() + instanceId := env.InstanceId + if instanceId == "" { + // Fallback to IMDS if not provided via command line (for backward compatibility) + instanceId = awsservice.GetInstanceId() + } awsservice.DeleteLogGroupAndStream(instanceId, instanceId) paths, _ := filepath.Glob(logFilePath + "*") for _, p := range paths { @@ -320,7 +340,12 @@ func autoRemovalTestCleanup() { // checkData queries CWL and verifies the number of log lines. func checkData(t *testing.T, start time.Time, lineCount int) { - instanceId := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() + instanceId := env.InstanceId + if instanceId == "" { + // Fallback to IMDS if not provided via command line (for backward compatibility) + instanceId = awsservice.GetInstanceId() + } end := time.Now() // Sleep to ensure backend stores logs. time.Sleep(time.Second * 60)