From e3f30858fa72f70d2354179f903abc9f04ce4bcc Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 18 Aug 2025 18:37:02 +0000 Subject: [PATCH 1/4] created test files --- .../logging_processor-saphana/config.yaml | 1 + .../logging_processor-saphana/input.log | 9 ++ .../output_fluentbit.yaml | 130 ++++++++++++++++++ .../output_otel.yaml | 1 + 4 files changed, 141 insertions(+) create mode 100644 transformation_test/testdata/logging_processor-saphana/config.yaml create mode 100644 transformation_test/testdata/logging_processor-saphana/input.log create mode 100644 transformation_test/testdata/logging_processor-saphana/output_fluentbit.yaml create mode 100644 transformation_test/testdata/logging_processor-saphana/output_otel.yaml diff --git a/transformation_test/testdata/logging_processor-saphana/config.yaml b/transformation_test/testdata/logging_processor-saphana/config.yaml new file mode 100644 index 0000000000..b198c5fd0e --- /dev/null +++ b/transformation_test/testdata/logging_processor-saphana/config.yaml @@ -0,0 +1 @@ +- type: saphana diff --git a/transformation_test/testdata/logging_processor-saphana/input.log b/transformation_test/testdata/logging_processor-saphana/input.log new file mode 100644 index 0000000000..5b7ac3d4cc --- /dev/null +++ b/transformation_test/testdata/logging_processor-saphana/input.log @@ -0,0 +1,9 @@ +[12034]{-1}[-1/-1] 2025-08-18 09:12:45.123456 i indexserver Main.cpp(00123) : Started indexserver successfully. +[12035]{200001}[15/40637290] 2025-08-18 09:13:12.654321 w backup BackupManager.cpp(00245) : Backup duration exceeded expected threshold. +[12036]{-1}[-1/-1] 2025-08-18 09:14:33.987654 e commlib commlibImpl.cpp(00456) : ERROR: comm::connect to Host: 127.0.0.1, port: 30015, Error: exception 1: no.2110021 (Basis/IO/Stream/impl/NetworkChannel.cpp:2991) +System error: SO_ERROR has pending error for socket. rc=111: Connection refused. channel={={={this=139745749931800, fd=22, refCnt=1, local=127.0.0.1/58700_tcp, remote=127.0.0.1/30015_tcp, state=ConnectWait, pending=[----]}}} +exception throw location: + 1: 0x00007f1937d9095c in .LTHUNK27.lto_priv.2256+0x558 (libhdbbasis.so) + 2: 0x0000563f44888831 in _GLOBAL__sub_I_setServiceStarting.cpp.lto_priv.239+0x520 (hdbnsutil) +[12037]{-1}[-1/-1] 2025-08-18 09:15:10.555432 d statistics StatCollector.cpp(00333) : Collected performance metrics. +[12038]{200002}[16/40637291] 2025-08-18 09:16:20.111111 i indexserver Main.cpp(00130) : New client connection established. thread_id=12038 diff --git a/transformation_test/testdata/logging_processor-saphana/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-saphana/output_fluentbit.yaml new file mode 100644 index 0000000000..ccc24ca2e8 --- /dev/null +++ b/transformation_test/testdata/logging_processor-saphana/output_fluentbit.yaml @@ -0,0 +1,130 @@ +- entries: + - jsonPayload: + component: indexserver + message: Started indexserver successfully. + severity_flag: i + thread_id: "12034" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + severity: 200.0 + sourceLocation: + file: Main.cpp + function: "" + line: 123.0 + timestamp: 2025-08-18T09:12:45.123456000Z + - jsonPayload: + component: backup + connection_id: "200001" + message: Backup duration exceeded expected threshold. + severity_flag: w + thread_id: "12035" + transaction_id: "15" + update_transaction_id: "40637290" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + severity: 400.0 + sourceLocation: + file: BackupManager.cpp + function: "" + line: 245.0 + timestamp: 2025-08-18T09:13:12.654321000Z + - jsonPayload: + component: commlib + message: "ERROR: comm::connect to Host: 127.0.0.1, port: 30015, Error: exception 1: no.2110021 (Basis/IO/Stream/impl/NetworkChannel.cpp:2991)" + severity_flag: e + thread_id: "12036" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + severity: 500.0 + sourceLocation: + file: commlibImpl.cpp + function: "" + line: 456.0 + timestamp: 2025-08-18T09:14:33.987654000Z + - jsonPayload: + message: "System error: SO_ERROR has pending error for socket. rc=111: Connection refused. channel={={={this=139745749931800, fd=22, refCnt=1, local=127.0.0.1/58700_tcp, remote=127.0.0.1/30015_tcp, state=ConnectWait, pending=[----]}}}" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + sourceLocation: + file: "" + function: "" + line: 0.0 + timestamp: now + - jsonPayload: + message: "exception throw location:" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + sourceLocation: + file: "" + function: "" + line: 0.0 + timestamp: now + - jsonPayload: + message: " 1: 0x00007f1937d9095c in .LTHUNK27.lto_priv.2256+0x558 (libhdbbasis.so)" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + sourceLocation: + file: "" + function: "" + line: 0.0 + timestamp: now + - jsonPayload: + message: " 2: 0x0000563f44888831 in _GLOBAL__sub_I_setServiceStarting.cpp.lto_priv.239+0x520 (hdbnsutil)" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + sourceLocation: + file: "" + function: "" + line: 0.0 + timestamp: now + - jsonPayload: + component: statistics + message: Collected performance metrics. + severity_flag: d + thread_id: "12037" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + severity: 100.0 + sourceLocation: + file: StatCollector.cpp + function: "" + line: 333.0 + timestamp: 2025-08-18T09:15:10.555432000Z + - jsonPayload: + component: indexserver + connection_id: "200002" + message: New client connection established. thread_id=12038 + severity_flag: i + thread_id: "12038" + transaction_id: "16" + update_transaction_id: "40637291" + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/saphana + logName: projects/my-project/logs/transformation_test + severity: 200.0 + sourceLocation: + file: Main.cpp + function: "" + line: 130.0 + timestamp: 2025-08-18T09:16:20.111111000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-saphana/output_otel.yaml b/transformation_test/testdata/logging_processor-saphana/output_otel.yaml new file mode 100644 index 0000000000..4a8920aad8 --- /dev/null +++ b/transformation_test/testdata/logging_processor-saphana/output_otel.yaml @@ -0,0 +1 @@ +- config_error: processor "processor0" not supported in pipeline "transformation_test" From 9e2c3f870b142f2e4bb8bea2e854cea29dea6e2e Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 18 Aug 2025 18:45:26 +0000 Subject: [PATCH 2/4] sap hana refactor --- apps/saphana.go | 111 ++++++++++++++++++++---------------------------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/apps/saphana.go b/apps/saphana.go index 26f6a60c0e..db0ffa8d26 100644 --- a/apps/saphana.go +++ b/apps/saphana.go @@ -18,45 +18,55 @@ import ( "context" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) -type LoggingProcessorSapHanaTrace struct { - confgenerator.ConfigComponent `yaml:",inline"` -} +type LoggingProcessorMacroSapHanaTrace struct{} -func (LoggingProcessorSapHanaTrace) Type() string { +func (LoggingProcessorMacroSapHanaTrace) Type() string { return "saphana" } -func (p LoggingProcessorSapHanaTrace) Components(ctx context.Context, tag string, uid string) []fluentbit.Component { - c := confgenerator.LoggingProcessorParseRegex{ - // Undocumented Format: [thread_id]{connection_id}[transaction_id/update_transaction_id] timestamp severity_flag component source_file : message - // Sample line: [7893]{200068}[20/40637286] 2021-11-04 13:13:25.025767 w FileIO FileSystem.cpp(00085) : Unsupported file system "ext4" for "/usr/sap/MMM/SYS/global/hdb/data/mnt00001" - // Sample line: [18048]{-1}[-1/-1] 2020-11-10 12:24:23.424024 i Crypto RootKeyStoreAccessor.cpp(00818) : Created new root key /usr/sap/MMM/SYS/global/hdb/security/ssfs:HDB_SERVER/3/PERSISTENCE - // Sample line: [18048]{-1}[-1/-1] 2020-11-10 12:24:20.988943 e commlib commlibImpl.cpp(00986) : ERROR: comm::connect to Host: 127.0.0.1, port: 30001, Error: exception 1: no.2110017 (Basis/IO/Stream/impl/NetworkChannel.cpp:2989) - // System error: SO_ERROR has pending error for socket. rc=111: Connection refused. channel={={={this=139745749931736, fd=21, refCnt=1, local=127.0.0.1/58654_tcp, remote=127.0.0.1/30001_tcp, state=ConnectWait, pending=[----]}}} - // exception throw location: - // 1: 0x00007f1937d9095c in .LTHUNK27.lto_priv.2256+0x558 (libhdbbasis.so) - // ... - // 25: 0x0000563f44888831 in _GLOBAL__sub_I_setServiceStarting.cpp.lto_priv.239+0x520 (hdbnsutil) - Regex: `^\[(?\d+)\]\{(?-?\d+)\}\[(?-?\d+)\/(?-?\d+)\]\s+(?