out_cloudwatch_logs: increase MAX_EVENT_LEN to 1MB with tests

Increase MAX_EVENT_LEN from 262,118 bytes (256 KiB) to 1,000,000 bytes
(1 MB) to better align with AWS CloudWatch's documented maximum event
size of 1,048,576 bytes (1 MiB). The 1 MB limit provides a ~4.6% safety
margin to account for JSON encoding overhead.

Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586
bytes) succeed, though we use a conservative limit for production safety.

Add runtime tests to validate the new limit:
- event_size_near_limit: Validates events just under 1MB are accepted
- event_size_at_aws_max: Validates events near AWS max are truncated
- event_truncation_with_backslash: Validates backslash handling at
  truncation boundary

Signed-off-by: Shelby Hagman <[email protected]>

Author: ShelbyZ

plugins/out_cloudwatch_logs/cloudwatch_api.h

@@ -43,8 +43,17 @@
 /* Maximum number of character limits including both the Attributes key and its value */
 #define ATTRIBUTES_MAX_LEN 300
 
-/* 256KiB minus 26 bytes for the event */
-#define MAX_EVENT_LEN      262118
+/*
+ * https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
+ * AWS CloudWatch's documented maximum event size is 1,048,576 bytes (1 MiB),
+ * including JSON encoding overhead (structure, escaping, etc.).
+ * 
+ * Setting MAX_EVENT_LEN to 1,000,000 bytes (1 MB) provides a ~4.6% safety margin
+ * to account for JSON encoding overhead and ensure reliable operation.
+ * Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586 bytes)
+ * succeed, though we use a conservative limit for production safety.
+ */
+#define MAX_EVENT_LEN      1000000
 
 /* Prefix used for entity fields only */
 #define AWS_ENTITY_PREFIX "aws_entity"

tests/runtime/out_cloudwatch.c

@@ -9,6 +9,9 @@
 /* not a real error code, but tests that the code can respond to any error */
 #define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}"
 
+/* MAX_EVENT_LEN from cloudwatch_api.h */
+#define MAX_EVENT_LEN 1000000
+
 /* It writes a big JSON message (copied from TD test) */
 void flb_test_cloudwatch_success(void)
 {
@@ -393,6 +396,134 @@ void flb_test_cloudwatch_error_put_retention_policy(void)
     flb_destroy(ctx);
 }
 
+/* Helper function to create a large JSON message of specified size */
+static char* create_large_json_message(size_t target_size)
+{
+    char *json;
+    size_t i;
+    size_t data_size;
+    
+    /* Account for JSON structure: {"message":"..."} = 15 chars + data */
+    if (target_size < 20) {
+        target_size = 20;
+    }
+    
+    json = flb_malloc(target_size + 1);
+    if (!json) {
+        return NULL;
+    }
+    
+    /* Start JSON object */
+    strcpy(json, "{\"message\":\"");
+    data_size = target_size - 15; /* Subtract JSON structure overhead */
+    
+    /* Fill with 'A' characters */
+    for (i = 0; i < data_size; i++) {
+        json[12 + i] = 'A';
+    }
+    
+    /* Close JSON object */
+    strcpy(json + 12 + data_size, "\"}");
+    json[target_size] = '\0';
+    
+    return json;
+}
+
+/* Helper to setup and run a CloudWatch test with custom JSON data */
+static void run_cloudwatch_test_with_data(char *data, size_t data_len)
+{
+    int ret;
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+
+    setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1);
+
+    ctx = flb_create();
+
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    TEST_CHECK(in_ffd >= 0);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL);
+    TEST_CHECK(out_ffd >= 0);
+    flb_output_set(ctx, out_ffd, "match", "test", NULL);
+    flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL);
+    flb_output_set(ctx, out_ffd, "log_group_name", "fluent", NULL);
+    flb_output_set(ctx, out_ffd, "log_stream_prefix", "from-fluent-", NULL);
+    flb_output_set(ctx, out_ffd, "auto_create_group", "On", NULL);
+    flb_output_set(ctx, out_ffd, "net.keepalive", "Off", NULL);
+    flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL);
+
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    if (data) {
+        flb_lib_push(ctx, in_ffd, data, data_len);
+    }
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
+/* Test event size just under the limit (should succeed) */
+void flb_test_cloudwatch_event_size_near_limit(void)
+{
+    char *large_json;
+
+    /* Create message just under MAX_EVENT_LEN (999,000 bytes) */
+    large_json = create_large_json_message(999000);
+    TEST_CHECK(large_json != NULL);
+    
+    if (large_json) {
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
+/* Test event size at AWS maximum (should be truncated to MAX_EVENT_LEN) */
+void flb_test_cloudwatch_event_size_at_aws_max(void)
+{
+    char *large_json;
+
+    /* Create message near AWS max (1,048,576 bytes) */
+    large_json = create_large_json_message(1048500);
+    TEST_CHECK(large_json != NULL);
+    
+    if (large_json) {
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
+/* Test event with trailing backslash at truncation boundary */
+void flb_test_cloudwatch_event_truncation_with_backslash(void)
+{
+    char *large_json;
+    size_t i;
+
+    /* Create message with backslashes near the truncation point */
+    large_json = flb_malloc(1000100);
+    TEST_CHECK(large_json != NULL);
+    
+    if (large_json) {
+        strcpy(large_json, "{\"message\":\"");
+        /* Fill with pattern that includes backslashes */
+        for (i = 0; i < 999980; i++) {
+            if (i % 100 == 99) {
+                large_json[12 + i] = '\\';
+            } else {
+                large_json[12 + i] = 'A';
+            }
+        }
+        strcpy(large_json + 12 + 999980, "\"}");
+        
+        run_cloudwatch_test_with_data(large_json, strlen(large_json));
+        flb_free(large_json);
+    }
+}
+
 /* Test list */
 TEST_LIST = {
     {"success", flb_test_cloudwatch_success },
@@ -405,5 +536,8 @@ TEST_LIST = {
     {"put_retention_policy_success", flb_test_cloudwatch_put_retention_policy_success },
     {"already_exists_create_group_put_retention_policy", flb_test_cloudwatch_already_exists_create_group_put_retention_policy },
     {"error_put_retention_policy", flb_test_cloudwatch_error_put_retention_policy },
+    {"event_size_near_limit", flb_test_cloudwatch_event_size_near_limit },
+    {"event_size_at_aws_max", flb_test_cloudwatch_event_size_at_aws_max },
+    {"event_truncation_with_backslash", flb_test_cloudwatch_event_truncation_with_backslash },
     {NULL, NULL}
 };