perf(storage): add write_object upload strategies benchmarking suite

Cargo.toml

@@ -356,6 +356,7 @@ members = [
   "src/storage/benchmarks/appendable_object",
   "src/storage/benchmarks/random",
   "src/storage/benchmarks/w1r3",
+  "src/storage/benchmarks/write_object",
   "src/storage/examples",
   "src/storage/tests/scenarios",
   "src/wkt",
@@ -445,6 +446,7 @@ humantime             = { default-features = false, version = "2" }
 hyper                 = { default-features = false, version = "1.10" }
 jiff                  = { default-features = false, version = "0.2.32" }
 jsonwebtoken          = { default-features = false, version = "11" }
+libc                  = { default-features = false, version = "0.2.183" }
 markdown              = { default-features = false, version = "1.0" }
 opentelemetry         = { default-features = false, version = "0.32" }
 opentelemetry-proto   = { default-features = false, version = "0.32" }

src/storage/benchmarks/write_object/Cargo.toml

@@ -0,0 +1,41 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name                 = "storage-benchmark-write-object"
+version              = "0.0.0"
+publish              = false
+edition.workspace    = true
+authors.workspace    = true
+license.workspace    = true
+repository.workspace = true
+keywords.workspace   = true
+categories.workspace = true
+
+[dependencies]
+anyhow.workspace            = true
+bytes.workspace             = true
+clap                        = { workspace = true, features = ["derive", "env", "help", "std", "usage"] }
+crc32c.workspace            = true
+google-cloud-auth.workspace = true
+google-cloud-storage        = { workspace = true, features = ["default-rustls-provider", "unstable-stream"] }
+libc.workspace              = true
+rand.workspace              = true
+tempfile.workspace          = true
+tokio                       = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] }
+tracing-subscriber          = { workspace = true, features = ["env-filter", "fmt", "std"] }
+uuid.workspace              = true
+
+[lints]
+workspace = true

src/storage/benchmarks/write_object/README.md

@@ -0,0 +1,114 @@
+# Cloud Storage WriteObject Upload Benchmark
+
+Benchmarks and compares upload strategies and checksum modes for
+[`write_object`](crate::client::Storage::write_object) in the Google Cloud
+Storage (GCS) Rust client library.
+
+## Motivation & Scenarios
+
+When uploading seekable data (such as local disk files), SDKs face a protocol
+trade-off regarding data integrity validation and throughput:
+
+- **Option A (`Option_A_Unbuffered_Baseline`)**:
+  - Code: `.send_unbuffered()`
+  - Single continuous stream, 0 application RAM buffer.
+  - Checksum calculated on the fly; verified client-side upon completion.
+- **Option B (`Option_B_Unbuffered_2Pass`)**:
+  - Code: `.precompute_checksums().await?.send_unbuffered()`
+  - **Pass 1:** Scans the local file to compute SIMD CRC32C checksum (reads from
+    physical disk if cold, populates page cache).
+  - **Pass 2:** Streams data via a single continuous PUT request with
+    server-side validation header (`x-goog-hash`). Reads from OS page cache.
+  - 0 application RAM buffer.
+- **Option C (`Option_C_Buffered_Chunked`)**:
+  - Code: `.send_buffered()`
+  - 1-pass chunked upload using sequential 8 MiB HTTP PUT requests.
+  - Server-side validation enforced by attaching checksum to the final chunk.
+  - Requires 8 MiB in-memory buffer per upload.
+
+## Benchmark Matrix (5 Size Tiers)
+
+1. **12 MiB (Single-Shot Multipart)**: Exercises single-shot multipart upload
+   path (below default 16 MiB threshold).
+1. **64 MiB (Small Resumable)**: 8 chunks in Option C vs. 1 continuous stream in
+   Option B.
+1. **512 MiB (Medium Resumable)**: 64 chunks in Option C vs. 1 continuous stream
+   in Option B.
+1. **2 GiB (Large Resumable)**: 256 chunks in Option C vs. 1 continuous stream
+   in Option B.
+1. **8 GiB (Stress Resumable)**: 1,024 chunks in Option C vs. 1 continuous
+   stream in Option B.
+
+## Pre-flight Check & 512 KiB Global Warmup
+
+Before creating large test files or running measured iterations, the benchmark
+performs a single **512 KiB pre-flight warmup check**:
+
+- Validates Google Cloud authentication and bucket write/delete permissions. If
+  authentication or permissions fail, the benchmark aborts immediately with
+  remediation guidance.
+- Primes DNS resolution and the TLS 1.3 keep-alive connection pool.
+- Eliminates the need for redundant multi-gigabyte warmup uploads during actual
+  scenario testing.
+
+## Page Cache, Storage Medium & Disk Cleanup
+
+- **Physical SSD Storage:** By default, test files are created under
+  `/usr/local/google/tmp/rust-write-object-benchmarking-data` on physical SSD
+  storage. This can be overridden via `--temp-dir=/path/to/dir`.
+- **Cold Cache Eviction:** By default, `--cold-cache=true` is enabled. Before
+  every measured iteration, the benchmark calls `posix_fadvise(DONTNEED)` once
+  to evict the test file from the OS page cache (RAM), ensuring a cold physical
+  disk read.
+- **Disk Cleanup:** Upon benchmark completion, the temporary file is
+  automatically removed from physical disk.
+
+## Pre-requisites
+
+- Authenticate with GCP credentials:
+  ```bash
+  gcloud auth application-default login
+  ```
+- Set required benchmark environment variables:
+  ```bash
+  # Target GCS bucket for uploads (required)
+  export GOOGLE_CLOUD_RUST_BENCHMARKS_BUCKET="rust-write-object-benchmark-bucket"
+
+  # Directory on physical SSD for temporary test file generation (required)
+  export GOOGLE_CLOUD_RUST_BENCHMARKS_DATA_PATH="/usr/local/google/tmp/rust-write-object-benchmarking-data"
+
+  # Directory for saving CSV latencies and JSON summary metrics (optional)
+  # If omitted, metrics are printed to the terminal and file writing is skipped.
+  export GOOGLE_CLOUD_RUST_BENCHMARKS_STATS_OUTPUT_PATH="/usr/local/google/tmp/rust-write-object-benchmarking-data/results"
+  ```
+
+## Running the Benchmark
+
+### Run All 5 Tiers (12 MiB, 64 MiB, 512 MiB, 2 GiB, 8 GiB)
+
+```bash
+chmod +x run_all.sh
+./run_all.sh
+```
+
+### Run a Single Configuration
+
+```bash
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 12582912 \
+  --scenario all \
+  --measured-iterations 5
+```
+
+### CLI Options Reference
+
+| Flag                    | Env Variable                                     | Default             | Description                                          |
+| :---------------------- | :----------------------------------------------- | :------------------ | :--------------------------------------------------- |
+| `--bucket-name`         | `GOOGLE_CLOUD_RUST_BENCHMARKS_BUCKET`            | *(none, required)*  | Target bucket name                                   |
+| `--temp-dir`            | `GOOGLE_CLOUD_RUST_BENCHMARKS_DATA_PATH`         | *(none, required)*  | Physical SSD directory for scratch file              |
+| `--output-dir`          | `GOOGLE_CLOUD_RUST_BENCHMARKS_STATS_OUTPUT_PATH` | *(none, optional)*  | Output folder for CSV and JSON reports               |
+| `--object-size`         |                                                  | `67108864` (64 MiB) | Object size in bytes                                 |
+| `--scenario`            |                                                  | `all`               | Scenario (`option-a`, `option-b`, `option-c`, `all`) |
+| `--cold-cache`          |                                                  | `true`              | Evict page cache between iterations                  |
+| `--cleanup`             |                                                  | `true`              | Delete uploaded object after iteration               |
+| `--measured-iterations` |                                                  | `5`                 | Measured iterations per scenario                     |

src/storage/benchmarks/write_object/run_all.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# Parse extra args passed directly to script
+EXTRA_ARGS=( "$@" )
+
+echo "Building benchmark..."
+cargo build --release -p storage-benchmark-write-object
+
+echo "================================================================="
+echo "Running Tier 1: 12 MiB (Single-Shot Multipart Upload)"
+echo "================================================================="
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 12582912 \
+  "${EXTRA_ARGS[@]}"
+
+echo "================================================================="
+echo "Running Tier 2: 64 MiB (Small Resumable Upload - 8 chunks)"
+echo "================================================================="
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 67108864 \
+  "${EXTRA_ARGS[@]}"
+
+echo "================================================================="
+echo "Running Tier 3: 512 MiB (Medium Resumable Upload - 64 chunks)"
+echo "================================================================="
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 536870912 \
+  "${EXTRA_ARGS[@]}"
+
+echo "================================================================="
+echo "Running Tier 4: 2 GiB (Large Resumable Upload - 256 chunks)"
+echo "================================================================="
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 2147483648 \
+  "${EXTRA_ARGS[@]}"
+
+echo "================================================================="
+echo "Running Tier 5: 8 GiB (Stress Resumable Upload - 1,024 chunks)"
+echo "================================================================="
+cargo run --release -p storage-benchmark-write-object -- \
+  --object-size 8589934592 \
+  "${EXTRA_ARGS[@]}"
+
+echo "================================================================="
+echo "All benchmarks complete!"
+echo "================================================================="

src/storage/benchmarks/write_object/src/args.rs

@@ -0,0 +1,100 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::{Parser, ValueEnum};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+pub enum UploadScenario {
+    /// Option A: Baseline 1-pass unbuffered stream (no precomputed hash, client-side validation only)
+    OptionA,
+    /// Option B: 2-pass unbuffered stream (Pass 1: precomputed hash, Pass 2: continuous stream, server-side validation)
+    OptionB,
+    /// Option C: 1-pass buffered chunked upload (8 MiB chunks in RAM, server-side validation on final chunk)
+    OptionC,
+    /// Run all three scenarios (Option A, Option B, and Option C) for side-by-side comparison
+    All,
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about = "Benchmark write_object upload strategies in GCS", long_about = None)]
+pub struct Args {
+    /// The name of the bucket to use for the benchmark.
+    #[arg(long, env = "GOOGLE_CLOUD_RUST_BENCHMARKS_BUCKET")]
+    pub bucket_name: Option<String>,
+
+    /// Number of measured iterations per scenario.
+    #[arg(long, default_value_t = 5)]
+    pub measured_iterations: usize,
+
+    /// The size of the object to upload in bytes.
+    #[arg(long, default_value_t = 67_108_864)] // 64 MiB default
+    pub object_size: u64,
+
+    /// Upload scenario / strategy to benchmark.
+    #[arg(long, value_enum, default_value_t = UploadScenario::All)]
+    pub scenario: UploadScenario,
+
+    /// Whether to evict the test file from the OS page cache (RAM) before each iteration to
+    /// simulate a cold physical disk read.
+    #[arg(long, action = clap::ArgAction::Set, default_value_t = true)]
+    pub cold_cache: bool,
+
+    /// Directory on physical SSD storage for creating the temporary test file.
+    #[arg(long, env = "GOOGLE_CLOUD_RUST_BENCHMARKS_DATA_PATH")]
+    pub temp_dir: Option<String>,
+
+    /// Directory for saving output artifacts (raw CSV latencies and summary JSON).
+    /// If omitted or empty, file reporting is skipped and results are printed to the terminal.
+    #[arg(long, env = "GOOGLE_CLOUD_RUST_BENCHMARKS_STATS_OUTPUT_PATH")]
+    pub output_dir: Option<String>,
+
+    /// Whether to delete test objects from GCS after each iteration.
+    #[arg(long, action = clap::ArgAction::Set, default_value_t = true)]
+    pub cleanup: bool,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_args_parsing_with_boolean_values() {
+        let args = Args::try_parse_from([
+            "benchmark",
+            "--bucket-name",
+            "test-bucket",
+            "--temp-dir",
+            "/tmp/test-data",
+            "--object-size",
+            "12582912",
+            "--scenario",
+            "option-a",
+            "--cleanup",
+            "false",
+            "--cold-cache",
+            "false",
+            "--measured-iterations",
+            "1",
+        ])
+        .unwrap();
+
+        assert_eq!(args.bucket_name.as_deref(), Some("test-bucket"));
+        assert_eq!(args.temp_dir.as_deref(), Some("/tmp/test-data"));
+        assert_eq!(args.object_size, 12_582_912);
+        assert_eq!(args.scenario, UploadScenario::OptionA);
+        assert!(!args.cleanup);
+        assert!(!args.cold_cache);
+        assert_eq!(args.measured_iterations, 1);
+    }
+}