Merge pull request #3150 from rockwotj/dft-mmms

franz: use IEC units instead of SI

Author: rockwotj

CHANGELOG.md

@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
 - Field `avro_raw_json` was deprecated in favor of `avro.raw_unions` for processor `schema_registry_decode`. (@rockwotj)
 - The `snowpipe_streaming` output now has better error handling for authentication failures when uploading to cloud storage. (@rockwotj)
 - Field `schema_evolution.new_column_type_mapping` for `snowpipe_streaming` is deprecated and can be replaced with `schema_evolution.processors`. (@rockwotj)
+- Increased the default values for `max_message_bytes` and `broker_write_max_bytes` by using IEC units instead of SI units. This better matches defaults in Redpanda and Kafka. (@rockwotj)
 
 ## 4.45.1 - 2025-01-17
 

docs/modules/components/pages/outputs/kafka_franz.adoc

@@ -93,8 +93,8 @@ output:
     idempotent_write: true
     compression: "" # No default (optional)
     timeout: 10s
-    max_message_bytes: 1MB
-    broker_write_max_bytes: 100MB
+    max_message_bytes: 1MiB
+    broker_write_max_bytes: 100MiB
 ```
 
 --
@@ -767,7 +767,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -784,7 +784,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

docs/modules/components/pages/outputs/ockam_kafka.adoc

@@ -91,8 +91,8 @@ output:
       idempotent_write: true
       compression: "" # No default (optional)
       timeout: 10s
-      max_message_bytes: 1MB
-      broker_write_max_bytes: 100MB
+      max_message_bytes: 1MiB
+      broker_write_max_bytes: 100MiB
       topic: "" # No default (required)
       key: "" # No default (optional)
       partition: ${! meta("partition") } # No default (optional)
@@ -479,7 +479,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -496,7 +496,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

docs/modules/components/pages/outputs/redpanda.adoc

@@ -80,8 +80,8 @@ output:
     idempotent_write: true
     compression: "" # No default (optional)
     timeout: 10s
-    max_message_bytes: 1MB
-    broker_write_max_bytes: 100MB
+    max_message_bytes: 1MiB
+    broker_write_max_bytes: 100MiB
 ```
 
 --
@@ -650,7 +650,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -667,7 +667,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

docs/modules/components/pages/outputs/redpanda_migrator.adoc

@@ -87,8 +87,8 @@ output:
     idempotent_write: true
     compression: "" # No default (optional)
     timeout: 10s
-    max_message_bytes: 1MB
-    broker_write_max_bytes: 100MB
+    max_message_bytes: 1MiB
+    broker_write_max_bytes: 100MiB
 ```
 
 --
@@ -741,7 +741,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -758,7 +758,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

docs/modules/components/pages/outputs/redpanda_migrator_offsets.adoc

@@ -74,8 +74,8 @@ output:
     offset_commit_timestamp: ${! @kafka_offset_commit_timestamp }
     offset_metadata: ${! @kafka_offset_metadata }
     timeout: 10s
-    max_message_bytes: 1MB
-    broker_write_max_bytes: 100MB
+    max_message_bytes: 1MiB
+    broker_write_max_bytes: 100MiB
     max_retries: 0
     backoff:
       initial_interval: 1s
@@ -542,7 +542,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -559,7 +559,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

docs/modules/components/pages/redpanda/about.adoc

@@ -60,8 +60,8 @@ redpanda:
   idempotent_write: true
   compression: "" # No default (optional)
   timeout: 10s
-  max_message_bytes: 1MB
-  broker_write_max_bytes: 100MB
+  max_message_bytes: 1MiB
+  broker_write_max_bytes: 100MiB
 ```
 --
 ======
@@ -573,7 +573,7 @@ The maximum space in bytes than an individual message may take, messages larger
 
 *Type*: `string`
 
-*Default*: `"1MB"`
+*Default*: `"1MiB"`
 
 ```yml
 # Examples
@@ -590,7 +590,7 @@ The upper bound for the number of bytes written to a broker connection in a sing
 
 *Type*: `string`
 
-*Default*: `"100MB"`
+*Default*: `"100MiB"`
 
 ```yml
 # Examples

internal/impl/kafka/franz_writer.go

@@ -53,13 +53,13 @@ func FranzProducerLimitsFields() []*service.ConfigField {
 		service.NewStringField(kfwFieldMaxMessageBytes).
 			Description("The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`.").
 			Advanced().
-			Default("1MB").
+			Default("1MiB").
 			Example("100MB").
 			Example("50mib"),
 		service.NewStringField(kfwFieldBrokerWriteMaxBytes).
 			Description("The upper bound for the number of bytes written to a broker connection in a single write. This field corresponds to Kafka's `socket.request.max.bytes`.").
 			Advanced().
-			Default("100MB").
+			Default("100MiB").
 			Example("128MB").
 			Example("50mib"),
 	}