out_cloudwatch_logs: increase MAX_EVENT_LEN to 1MB with tests

Increase MAX_EVENT_LEN from 262,118 bytes (256 KiB) to 1,000,000 bytes
(1 MB) to better align with AWS CloudWatch's documented maximum event
size of 1,048,576 bytes (1 MiB). The 1 MB limit provides a ~4.6% safety
margin to account for JSON encoding overhead.

Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586
bytes) succeed, though we use a conservative limit for production safety.

Add runtime tests to validate the new limit:
- event_size_at_limit: Validates events at exactly MAX_EVENT_LEN (1MB) are accepted
- event_size_over_limit: Validates events exceeding MAX_EVENT_LEN are truncated
- event_truncation_with_backslash: Validates backslash handling at truncation boundary

Signed-off-by: Shelby Hagman <[email protected]>

Author: ShelbyZ

plugins/out_cloudwatch_logs/cloudwatch_api.h

@@ -21,9 +21,10 @@
 #define FLB_OUT_CLOUDWATCH_API
 
 /*
- * The CloudWatch API documents that the maximum payload is 1,048,576 bytes
- * For reasons that are under investigation, using that number in this plugin
- * leads to API errors. No issues have been seen setting it to 1,000,000 bytes.
+ * The CloudWatch API documents that the maximum payload is 1,048,576 bytes.
+ * This is the total size limit for the entire PutLogEvents request payload buffer.
+ * Individual events are capped at MAX_EVENT_LEN (1,000,000 bytes) as a conservative
+ * safety margin to account for JSON encoding overhead and per-event metadata.
  */
 #define PUT_LOG_EVENTS_PAYLOAD_SIZE    1048576
 #define MAX_EVENTS_PER_PUT             10000
@@ -43,8 +44,17 @@
 /* Maximum number of character limits including both the Attributes key and its value */
 #define ATTRIBUTES_MAX_LEN 300
 
-/* 256KiB minus 26 bytes for the event */
-#define MAX_EVENT_LEN      262118
+/*
+ * https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
+ * AWS CloudWatch's documented maximum event size is 1,048,576 bytes (1 MiB),
+ * including JSON encoding overhead (structure, escaping, etc.).
+ * 
+ * Setting MAX_EVENT_LEN to 1,000,000 bytes (1 MB) provides a ~4.6% safety margin
+ * to account for JSON encoding overhead and ensure reliable operation.
+ * Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586 bytes)
+ * succeed, though we use a conservative limit for production safety.
+ */
+#define MAX_EVENT_LEN      1000000
 
 /* Prefix used for entity fields only */
 #define AWS_ENTITY_PREFIX "aws_entity"

tests/runtime/out_cloudwatch.c

@@ -5,10 +5,17 @@
 /* Test data */
 #include "data/td/json_td.h" /* JSON_TD */
 
+/* CloudWatch API constants */
+#include "../../plugins/out_cloudwatch_logs/cloudwatch_api.h"
+
 #define ERROR_ALREADY_EXISTS "{\"__type\":\"ResourceAlreadyExistsException\"}"
 /* not a real error code, but tests that the code can respond to any error */
 #define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}"
 
+/* JSON structure constants for test message generation */
+static const char *TEST_JSON_PREFIX = "{\"message\":\"";
+static const char *TEST_JSON_SUFFIX = "\"}";
+
 /* It writes a big JSON message (copied from TD test) */
 void flb_test_cloudwatch_success(void)
 {
@@ -393,6 +400,145 @@ void flb_test_cloudwatch_error_put_retention_policy(void)
     flb_destroy(ctx);
 }
 
+/* Helper function to create a large JSON message of specified size */
+static char* create_large_json_message(size_t target_size)
+{
+    size_t prefix_len = strlen(TEST_JSON_PREFIX);
+    size_t suffix_len = strlen(TEST_JSON_SUFFIX);
+    size_t overhead = prefix_len + suffix_len;
+    size_t data_size;
+    char *json;
+
+    /* Reject target_size too small for valid JSON structure */
+    if (target_size < overhead + 1) {
+        return NULL;
+    }
+
+    json = flb_malloc(target_size + 1);
+    if (!json) {
+        return NULL;
+    }
+
+    /* Build JSON: prefix + data + suffix */
+    memcpy(json, TEST_JSON_PREFIX, prefix_len);
+    data_size = target_size - overhead;
+
+    /* Fill with 'A' characters */
+    memset(json + prefix_len, 'A', data_size);
+
+    /* Close JSON object */
+    memcpy(json + prefix_len + data_size, TEST_JSON_SUFFIX, suffix_len);
+    json[target_size] = '\0';
+
+    /* Caller must free */
+    return json;
+}
+
+/* Helper to setup and run a CloudWatch test with custom JSON data */
+static void run_cloudwatch_test_with_data(char *data, size_t data_len)
+{
+    int ret;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+
+    setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1);
+
+    ctx = flb_create();
+    TEST_CHECK(ctx != NULL);
+
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    TEST_CHECK(in_ffd >= 0);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL);
+    TEST_CHECK(out_ffd >= 0);
+    flb_output_set(ctx, out_ffd, "match", "test", NULL);
+    flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL);
+    flb_output_set(ctx, out_ffd, "log_group_name", "fluent", NULL);
+    flb_output_set(ctx, out_ffd, "log_stream_prefix", "from-fluent-", NULL);
+    flb_output_set(ctx, out_ffd, "auto_create_group", "On", NULL);
+    flb_output_set(ctx, out_ffd, "net.keepalive", "Off", NULL);
+    flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL);
+
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    if (data) {
+        flb_lib_push(ctx, in_ffd, data, data_len);
+    }
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
+/* Test event size at maximum allowed limit (should succeed without truncation) */
+void flb_test_cloudwatch_event_size_at_limit(void)
+{
+    char *large_json;
+
+    /* Create message at MAX_EVENT_LEN */
+    large_json = create_large_json_message(MAX_EVENT_LEN);
+    TEST_CHECK(large_json != NULL);
+
+    if (large_json) {
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
+/* Test event size exceeding limit (should be truncated to MAX_EVENT_LEN) */
+void flb_test_cloudwatch_event_size_over_limit(void)
+{
+    char *large_json;
+
+    /* Create message exceeding MAX_EVENT_LEN by 1 byte to test truncation */
+    large_json = create_large_json_message(MAX_EVENT_LEN + 1);
+    TEST_CHECK(large_json != NULL);
+
+    if (large_json) {
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
+/* Test event with trailing backslash at truncation boundary */
+void flb_test_cloudwatch_event_truncation_with_backslash(void)
+{
+    char *large_json;
+    size_t prefix_len = strlen(TEST_JSON_PREFIX);
+    size_t suffix_len = strlen(TEST_JSON_SUFFIX);
+    size_t total_len;
+    size_t data_len;
+    size_t i;
+
+    /* Create base message exceeding MAX_EVENT_LEN */
+    large_json = create_large_json_message(MAX_EVENT_LEN + 100);
+    TEST_CHECK(large_json != NULL);
+
+    if (large_json) {
+        total_len = strlen(large_json);
+        data_len = total_len - prefix_len - suffix_len;
+
+        /* Replace pairs of characters with valid escape sequence "\\" */
+        for (i = 98; i < data_len - 1; i += 100) {
+            large_json[prefix_len + i] = '\\';
+            large_json[prefix_len + i + 1] = '\\';
+        }
+
+        size_t boundary = MAX_EVENT_LEN - 1; /* index in full JSON string */
+        /* Ensure a backslash is at the exact truncation boundary */
+        if (boundary + 1 < total_len - suffix_len) {
+            large_json[boundary] = '\\';
+            large_json[boundary + 1] = '\\';
+        }
+
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
 /* Test list */
 TEST_LIST = {
     {"success", flb_test_cloudwatch_success },
@@ -405,5 +551,8 @@ TEST_LIST = {
     {"put_retention_policy_success", flb_test_cloudwatch_put_retention_policy_success },
     {"already_exists_create_group_put_retention_policy", flb_test_cloudwatch_already_exists_create_group_put_retention_policy },
     {"error_put_retention_policy", flb_test_cloudwatch_error_put_retention_policy },
+    {"event_size_at_limit", flb_test_cloudwatch_event_size_at_limit },
+    {"event_size_over_limit", flb_test_cloudwatch_event_size_over_limit },
+    {"event_truncation_with_backslash", flb_test_cloudwatch_event_truncation_with_backslash },
     {NULL, NULL}
 };