transport: add parallel transport and improve resumable transport

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/google/go-containerregistry v0.20.6
 	github.com/gpustack/gguf-parser-go v0.22.1
 	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.9.1
 )
 
 require (
@@ -16,6 +17,7 @@ require (
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.9.3 // indirect
 	github.com/henvic/httpretty v0.1.4 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -25,6 +27,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/vbatts/tar-split v0.12.1 // indirect
 	golang.org/x/crypto v0.35.0 // indirect

go.sum

@@ -1,5 +1,6 @@
 github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8=
 github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -18,6 +19,8 @@ github.com/gpustack/gguf-parser-go v0.22.1 h1:FRnEDWqT0Rcplr/R9ctCRSN2+3DhVsf6dn
 github.com/gpustack/gguf-parser-go v0.22.1/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
 github.com/henvic/httpretty v0.1.4 h1:Jo7uwIRWVFxkqOnErcoYfH90o3ddQyVrSANeS4cxYmU=
 github.com/henvic/httpretty v0.1.4/go.mod h1:Dn60sQTZfbt2dYsdUSNsCljyF4AfdqnuJFDLJA1I4AM=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -37,10 +40,15 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d h1:3VwvTjiRPA7cqtgOWddEL+JrcijMlXUmj99c/6YyZoY=
 github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d/go.mod h1:tAG61zBM1DYRaGIPloumExGvScf08oHuo0kFoOqdbT0=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

tools/benchmarks/parallelget/README.md

@@ -0,0 +1,110 @@
+# ParallelGet Benchmark Tool
+
+A command-line benchmarking tool that compares the performance of standard HTTP GET requests against parallelized requests using the `transport/parallel` package.
+
+## Features
+
+- **Performance Comparison**: Downloads the same URL twice (standard vs parallel) and compares timing
+- **Response Validation**: Ensures both downloads produce identical results (byte-for-byte comparison)
+- **Configurable Parameters**: Adjustable chunk size and concurrency settings
+- **Detailed Metrics**: Reports download speeds, timing differences, and performance improvements
+- **Dynamic Progress Display**: Shows real-time progress bars during downloads with percentage and byte counts
+- **Clean Output**: User-friendly performance summary with emojis and clear formatting
+
+## Usage
+
+```bash
+go run ./tools/benchmarks/parallelget <url> [flags]
+```
+
+or
+
+```bash
+go build ./tools/benchmarks/parallelget
+./parallelget <url> [flags]
+```
+
+### Arguments
+
+- `<url>`: The HTTP URL to benchmark (required)
+
+### Flags
+
+- `--chunk-size int`: Minimum chunk size in bytes for parallelization (default: 1MB)
+- `--max-concurrent uint`: Maximum concurrent requests for parallel transport (default: 4)
+- `-h, --help`: Show help information
+
+### Examples
+
+```bash
+# Basic usage
+./parallelget https://example.com/large-file.zip
+
+# Custom chunk size (512KB) and higher concurrency
+./parallelget https://example.com/large-file.zip --chunk-size 524288 --max-concurrent 8
+
+# Small chunk size for testing with smaller files
+./parallelget https://httpbin.org/bytes/10485760 --chunk-size 262144 --max-concurrent 6
+```
+
+## Output
+
+The tool provides detailed output including:
+
+1. **Configuration**: Shows the chunk size and concurrency settings
+2. **Progress**: Real-time updates for each benchmark phase
+3. **Individual Results**: Download speed and timing for each approach
+4. **Validation**: Confirms that both downloads produced identical content
+5. **Performance Summary**: 
+   - Speedup factor (e.g., "3.2x faster")
+   - Time saved/penalty
+   - Detailed timing breakdown
+
+### Sample Output
+
+```
+Benchmarking HTTP GET performance for: https://example.com/large-file.zip
+Configuration: chunk-size=1048576 bytes, max-concurrent=4
+
+Running non-parallel benchmark...
+  Progress: [██████████████████████████████] 100.0% (10485760/10485760 bytes)
+✓ Non-parallel: 10485760 bytes in 2.1s (4.76 MB/s)
+Running parallel benchmark...
+  Progress: [██████████████████████████████] 100.0% (10485760/10485760 bytes)
+✓ Parallel: 10485760 bytes in 650ms (15.38 MB/s)
+Validating response consistency...
+✓ Responses match perfectly
+
+============================================================
+PERFORMANCE COMPARISON
+============================================================
+🚀 Parallel was 3.23x faster than non-parallel
+⏱️  Time saved: 1.45s (69.0%)
+
+Detailed timing:
+  Non-parallel: 2.1s
+  Parallel:     650ms
+  Difference:   -1.45s
+```
+
+## How It Works
+
+1. **Non-Parallel Benchmark**: Uses `net/http.DefaultClient` with `net/http.DefaultTransport`
+2. **Parallel Benchmark**: Uses `net/http.DefaultClient` with `transport/parallel.ParallelTransport` wrapping `net/http.DefaultTransport`
+3. **Response Storage**: Both responses are written to temporary files for validation
+4. **Validation**: Performs byte-by-byte comparison to ensure identical content
+5. **Cleanup**: Automatically removes temporary files after completion
+
+## Notes
+
+- The tool requires the server to support HTTP range requests (`Accept-Ranges: bytes`) for parallel downloads to work
+- If the server doesn't support range requests or the file is too small, the parallel transport will automatically fall back to a single request
+- Temporary files are automatically cleaned up, even if the tool exits unexpectedly
+- The tool validates that both downloads produce identical results before reporting performance metrics
+
+## Use Cases
+
+- **Performance Testing**: Evaluate the effectiveness of parallel downloads for different URLs
+- **Configuration Tuning**: Find optimal chunk size and concurrency settings for specific servers or file types  
+- **Server Compatibility**: Test whether servers properly support range requests
+- **Network Optimization**: Understand the impact of parallel downloads on different network conditions