Prevent OOM from malformed snappy payloads by validating decoded length

A specially crafted remote-write request can declare an extremely large
decoded length while providing only a small encoded payload. Prometheus
allocates memory based on the declared decoded size, so a single request
can trigger an allocation of ~2.5 GB. A few such requests are enough to
crash the process with OOM.

Here's the script that can be used to reproduce the issue:

echo
"97eab4890a170a085f5f6e616d655f5f120b746573745f6d6574726963121009000000000000f03f10d48fc9b2a333"
\
  | xxd -r -p \
  | curl -X POST \
      "http://127.0.0.1:9090/api/v1/write" \
      -H "Content-Type: application/x-protobuf" \
      -H "Content-Encoding: snappy" \
      -H "X-Prometheus-Remote-Write-Version: 0.1.0" \
      --data-binary @-

This change adds a hard limit: the requested decoded length must be less
than 32 MB. Requests exceeding the limit are rejected with HTTP 400
before any allocation occurs.

Signed-off-by: Max Kotliar <[email protected]>

Author: makasim

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Unreleased
 
+:warning: This release introduces a 32MB hard limit on decoded snappy payloads to prevent potential OOM attacks. Requests with decoded payloads exceeding this limit will be rejected with HTTP 400 status code. See #1917 for details. :warning:
+
+* [BUGFIX] exp/api: Reject malformed snappy payloads declaring huge decoded sizes. Enforce a 32MB decoded-size limit to prevent OOM from oversized remote-write requests. #1917.
+
 ## 1.23.2 / 2025-09-05
 
 This release is made to upgrade to prometheus/common v0.66.1, which drops the dependencies github.com/grafana/regexp and go.uber.org/atomic and replaces gopkg.in/yaml.v2 with go.yaml.in/yaml/v2 (a drop-in replacement).

exp/api/remote/remote_api.go

@@ -448,6 +448,14 @@ func WithWriteHandlerMiddlewares(middlewares ...func(http.Handler) http.Handler)
 	}
 }
 
+// maxDecodedSize limits the maximum allowed size of decompressed snappy payloads.
+// This protects against maliciously crafted payloads that could cause excessive memory
+// allocation and potentially lead to out-of-memory (OOM) conditions.
+// All usual payloads should be much smaller than this limit and pass without any problems.
+//
+// See more in https://github.com/prometheus/client_golang/pull/1917
+const maxDecodedSize = 32 * 1024 * 1024
+
 // SnappyDecodeMiddleware returns a middleware that checks if the request body is snappy-encoded and decompresses it.
 // If the request body is not snappy-encoded, it returns an error.
 // Used by default in NewHandler.
@@ -479,6 +487,18 @@ func SnappyDecodeMiddleware(logger *slog.Logger) func(http.Handler) http.Handler
 				return
 			}
 
+			decodedSize, err := snappy.DecodedLen(bodyBytes)
+			if err != nil {
+				logger.Error("Error snappy decoding remote write request", "err", err)
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+			if decodedSize > maxDecodedSize {
+				logger.Error("Error snappy decoding remote write request: decoded size exceeds maximum allowed", "decodedSize", decodedSize, "maxDecodedSize", maxDecodedSize)
+				http.Error(w, "decoded size exceeds maximum allowed", http.StatusBadRequest)
+				return
+			}
+
 			decompressed, err := snappy.Decode(nil, bodyBytes)
 			if err != nil {
 				// TODO(bwplotka): Add more context to responded error?

exp/api/remote/remote_api_test.go

@@ -14,18 +14,22 @@
 package remote
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
 	"errors"
 	"io"
 	"log/slog"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/klauspost/compress/snappy"
 	"github.com/prometheus/common/model"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/testing/protocmp"
@@ -295,3 +299,136 @@ func TestRemoteAPI_Write_WithHandler(t *testing.T) {
 		}
 	})
 }
+
+func TestSnappyDecodeMiddleware(t *testing.T) {
+	tLogger := slog.Default()
+
+	var actReq *writev2.Request
+	successH := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("failed to read body: %v", err)
+		}
+		actReq = &writev2.Request{}
+		if err := proto.Unmarshal(b, actReq); err != nil {
+			t.Fatalf("failed to unmarshal request: %v", err)
+		}
+
+		w.WriteHeader(http.StatusOK)
+	})
+
+	mw := SnappyDecodeMiddleware(tLogger)(successH)
+
+	t.Run("success", func(t *testing.T) {
+		// populated by successH handler
+		actReq = nil
+		expReq := testV2()
+
+		serializedExpReq, err := proto.Marshal(expReq)
+		if err != nil {
+			t.Fatal(err)
+		}
+		compressedExpReq := snappy.Encode(nil, serializedExpReq)
+
+		// Create HTTP request
+		r := httptest.NewRequest("POST", "/api/v1/write", bytes.NewReader(compressedExpReq))
+		r.Header.Set("Content-Encoding", "snappy")
+		rw := httptest.NewRecorder()
+
+		mw.ServeHTTP(rw, r)
+
+		if rw.Code != http.StatusOK {
+			t.Fatalf("expected status 200, got %d: %s", rw.Code, rw.Body.String())
+		}
+		if diff := cmp.Diff(expReq, actReq, protocmp.Transform()); diff != "" {
+			t.Fatalf("unexpected request after decoding: %s", diff)
+		}
+	})
+
+	t.Run("too_big", func(t *testing.T) {
+		// populated by successH handler
+		actReq = nil
+
+		s := writev2.NewSymbolTable()
+		expReq := &writev2.Request{}
+		for i := 0; i < 1024*1024; i++ {
+			expReq.Timeseries = append(expReq.Timeseries, &writev2.TimeSeries{
+				Metadata: &writev2.Metadata{
+					Type:    writev2.Metadata_METRIC_TYPE_COUNTER,
+					HelpRef: s.Symbolize("My lovely counter" + strconv.Itoa(i)),
+				},
+				LabelsRefs: s.SymbolizeLabels([]string{"__name__", "metric" + strconv.Itoa(i), "foo", "bar" + strconv.Itoa(i)}, nil),
+				Samples: []*writev2.Sample{
+					{Value: 1.1, Timestamp: 1214141},
+					{Value: 1.5, Timestamp: 1214180},
+				},
+			})
+		}
+
+		serializedExpReq, err := proto.Marshal(expReq)
+		if err != nil {
+			t.Fatal(err)
+		}
+		compressedExpReq := snappy.Encode(nil, serializedExpReq)
+
+		// Create HTTP request
+		r := httptest.NewRequest("POST", "/api/v1/write", bytes.NewReader(compressedExpReq))
+		r.Header.Set("Content-Encoding", "snappy")
+		rw := httptest.NewRecorder()
+
+		mw.ServeHTTP(rw, r)
+
+		if rw.Code != http.StatusBadRequest {
+			t.Fatalf("expected status 400, got %d", rw.Code)
+		}
+
+		body := rw.Body.String()
+		if !strings.Contains(body, "decoded size exceeds maximum allowed") {
+			t.Fatalf("expected error message about size limit, got: %s", body)
+		}
+	})
+
+	t.Run("crafted_decode_len", func(t *testing.T) {
+		// Snappy format: varint(decoded_len) + compressed_data
+		// For a claimed size of 33MB (exceeds 32MB limit), we need varint encoding
+		dst := make([]byte, binary.MaxVarintLen64)
+		binary.PutUvarint(dst, uint64(33*1024*1024))
+		// Add some dummy compressed data. Doesn't need to be valid.
+		dst = append(dst, []byte{0x00, 0x01, 0x02}...)
+
+		r := httptest.NewRequest("POST", "/api/v1/write", bytes.NewReader(dst))
+		r.Header.Set("Content-Encoding", "snappy")
+		rw := httptest.NewRecorder()
+
+		mw.ServeHTTP(rw, r)
+
+		if rw.Code != http.StatusBadRequest {
+			t.Fatalf("expected status 400, got %d", rw.Code)
+		}
+
+		body := rw.Body.String()
+		if !strings.Contains(body, "decoded size exceeds maximum allowed") {
+			t.Fatalf("expected error message about size limit, got: %s", body)
+		}
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		// Completely invalid snappy data
+		invalidData := []byte{0xff, 0xff, 0xff, 0xff}
+
+		r := httptest.NewRequest("POST", "/api/v1/write", bytes.NewReader(invalidData))
+		r.Header.Set("Content-Encoding", "snappy")
+		rw := httptest.NewRecorder()
+
+		mw.ServeHTTP(rw, r)
+
+		if rw.Code != http.StatusBadRequest {
+			t.Fatalf("expected status 400, got %d", rw.Code)
+		}
+
+		body := rw.Body.String()
+		if !strings.Contains(body, "corrupt input") {
+			t.Fatalf("expected error message about corrupt input, got: %s", body)
+		}
+	})
+}