perf: add NBI benchmark python script; update docs

Signed-off-by: Tony Chen <a122774007@gmail.com>

Author: Nahemah1022

docs/assets/nbi/nbi_scale.png

@@ -380,3 +380,7 @@ $ ais ls ais://.sys-inventory --props chunked,size
 NAME                                              CHUNKED         SIZE
 aws/@#/training-data-v5/inv-Tp4nR7kWx             yes             1.28GiB
 ```
+
+## Performance
+
+A latency-vs-scale benchmark comparing NBI listing, AIS regular listing, and direct S3 access (boto3) from 1K to 80K objects is available at [`python/tests/perf/nbi/`](../python/tests/perf/nbi/README.md).

docs/nbi.md

@@ -10,6 +10,7 @@ braceexpand>=0.1.7
 msgspec>=0.18.5
 xxhash>=3.5.0
 boto3>=1.37.10
+tqdm>=4.66.3
 pylint>=3.2.7
 pytest>=8.3.5,<9.0.0  # TODO: The newly released pytest 9.0.0 is incompatible with pytest-xdist on Python 3.10.x. Revisit and remove this upper limit.
 pytest-xdist>=3.6.1

python/aistore/common_requirements

@@ -13,7 +13,7 @@
 
 
 @dataclass
-class BenchmarkResult:
+class BenchmarkResult:  # pylint: disable=too-many-instance-attributes
     """Structured benchmark result data"""
 
     bucket_name: str
@@ -71,7 +71,7 @@ def compare_to_baseline(self, baseline: BenchmarkResult):
             )
 
 
-def benchmark_sequential_get(
+def benchmark_sequential_get(  # pylint: disable=too-many-locals
     client: Client, bucket_name: str, num_objects: int
 ) -> BenchmarkResult:
     """Benchmark getting objects one by one sequentially."""
@@ -119,7 +119,7 @@ def benchmark_sequential_get(
     return result
 
 
-def benchmark_get_batch(
+def benchmark_get_batch(  # pylint: disable=too-many-locals
     client: Client,
     bucket_name: str,
     num_objects: int,
@@ -287,7 +287,8 @@ def print_comprehensive_results(results: List[BenchmarkComparison]):
 
     print(f"\n{'='*120}")
     print(
-        f"{'Test Configuration':<35} {'Duration':<10} {'vs Baseline':<20} {'Speedup':<10} {'Throughput':<15} {'vs Baseline':<12}"
+        f"{'Test Configuration':<35} {'Duration':<10} {'vs Baseline':<20}"
+        f" {'Speedup':<10} {'Throughput':<15} {'vs Baseline':<12}"
     )
     print(f"{'='*120}")
 
@@ -315,7 +316,8 @@ def print_comprehensive_results(results: List[BenchmarkComparison]):
             improvement_str = "N/A"
 
         print(
-            f"{comp.test_config:<35} {result.duration:<10.2f} {time_vs_baseline:<20} {speedup_str:<10} {throughput_str:<15} {improvement_str:<12}"
+            f"{comp.test_config:<35} {result.duration:<10.2f} {time_vs_baseline:<20}"
+            f" {speedup_str:<10} {throughput_str:<15} {improvement_str:<12}"
         )
 
     print(f"{'='*120}")
@@ -334,7 +336,9 @@ def print_comprehensive_results(results: List[BenchmarkComparison]):
         print(f"   Total Objects:    {result.total_objects:,}")
         print(f"   Duration:         {result.duration:.3f} seconds")
         print(
-            f"   Success Rate:     {result.successful_reads}/{result.total_objects} ({100*result.successful_reads/result.total_objects:.1f}%)"
+            f"   Success Rate:     "
+            f"{result.successful_reads}/{result.total_objects}"
+            f" ({100*result.successful_reads/result.total_objects:.1f}%)"
         )
         print(f"   Throughput:       {result.throughput_objects_per_sec:.1f} objects/s")
         print(f"   Data Throughput:  {result.throughput_mib_per_sec:.2f} MiB/s")
@@ -386,32 +390,32 @@ def save_results_to_json(results: List[BenchmarkComparison], filename: str = Non
 def main():
     """Main comprehensive benchmarking function."""
     # Configuration
-    AISTORE_URL = "http://<ais-endpt>:51080"  # Replace <ais-endpt> with the actual AIStore endpoint
-    BUCKET_NAME = "benchmark-10KiB"  # Updated to match populate_bucket.py
-    TOTAL_OBJECTS = 10000  # 100,000 objects as requested
+    aistore_url = "http://<ais-endpt>:51080"  # Replace with actual endpoint
+    bucket_name = "benchmark-10KiB"
+    total_objects = 10000
 
     print("Starting comprehensive batch benchmark...")
-    print(f"AIStore URL: {AISTORE_URL}")
-    print(f"Bucket: {BUCKET_NAME}")
-    print(f"Total objects per test: {TOTAL_OBJECTS:,}")
+    print(f"AIStore URL: {aistore_url}")
+    print(f"Bucket: {bucket_name}")
+    print(f"Total objects per test: {total_objects:,}")
 
     # Create client with optimized pool size
-    client = Client(AISTORE_URL, max_pool_size=1000)
+    client = Client(aistore_url, max_pool_size=1000)
 
     # Verify bucket exists
     try:
-        bucket = client.bucket(BUCKET_NAME)
+        bucket = client.bucket(bucket_name)
         bucket.head()
-        print(f"Bucket '{BUCKET_NAME}' found - proceeding with benchmark")
+        print(f"Bucket '{bucket_name}' found - proceeding with benchmark")
     except Exception as e:
-        print(f"Error accessing bucket '{BUCKET_NAME}': {e}")
+        print(f"Error accessing bucket '{bucket_name}': {e}")
         print("Make sure to run populate_bucket.py first to create test objects")
         print("The bucket should contain at least 100,000 objects for this benchmark")
         return
 
     # Run comprehensive benchmark
     try:
-        results = run_comprehensive_benchmark(client, BUCKET_NAME, TOTAL_OBJECTS)
+        results = run_comprehensive_benchmark(client, bucket_name, total_objects)
 
         if not results:
             print("No benchmark results obtained. Check for errors above.")

python/tests/perf/__init__.py

@@ -55,9 +55,8 @@ def upload_object(self, bucket, obj_name: str, content: bytes, retries: int = 3)
                 if attempt < retries - 1:
                     time.sleep(0.1 * (2**attempt))  # Exponential backoff
                     continue
-                else:
-                    with self.lock:
-                        self.stats["failed"] += 1
+                with self.lock:
+                    self.stats["failed"] += 1
                     self.logger.error(
                         "Failed to upload %s after %d attempts: %s",
                         obj_name,

python/tests/perf/get_batch/get_batch_bench.py

@@ -0,0 +1,93 @@
+# NBI Latency-vs-Scale Benchmark
+
+Measures listing latency of [Native Bucket Inventory (NBI)](../../../../docs/nbi.md) against two baselines — AIS regular listing and direct S3 access via boto3 — across bucket sizes from 1K to 80K objects.
+
+## Background
+
+| Path | How it works |
+|---|---|
+| **AIS Regular** | Client sends one request to the AIS proxy. One designated target walks `ListObjectsV2` sequentially (1000 keys/page) and returns the aggregated result. |
+| **NBI creation** | A one-time job that walks S3 (same as Regular) and writes the full bucket namespace as binary inventory chunks on each target's local disk. |
+| **NBI listing** | Subsequent listings read the pre-built inventory chunks from local disk — no S3 calls. |
+| **S3 Direct** | Client calls boto3 `ListObjectsV2` in a pagination loop directly against S3 (MaxKeys=1000, same page size as AIS). |
+
+## Setup
+
+### 1. Connect to the cluster
+
+```bash
+export AIS_ENDPOINT=http://<proxy>:51080
+ais show cluster
+```
+
+### 2. Generate source objects (one-time)
+
+Objects are 1-byte files uploaded via `aisloader` to an S3-backed bucket. The subdirectory prefix isolates each scale point so they can coexist in the same bucket.
+
+```bash
+for subdir in 1k 2k 5k 10k 20k 40k 50k 80k; do
+    count=$(echo $subdir | sed 's/k/*1000/' | bc)
+    aisloader -bucket=my-bucket -provider=aws \
+        -subdir=bench/${subdir} \
+        -pctput=100 -maxputs=${count} \
+        -minsize=1 -maxsize=1 \
+        -numworkers=16 -cleanup=false -skiplist \
+        -duration=10m
+done
+```
+
+Or run each prefix individually:
+
+```bash
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/1k  -pctput=100 -maxputs=1000  -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/2k  -pctput=100 -maxputs=2000  -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/5k  -pctput=100 -maxputs=5000  -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/10k -pctput=100 -maxputs=10000 -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/20k -pctput=100 -maxputs=20000 -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/40k -pctput=100 -maxputs=40000 -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/50k -pctput=100 -maxputs=50000 -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+aisloader -bucket=my-bucket -provider=aws -subdir=bench/80k -pctput=100 -maxputs=80000 -minsize=1 -maxsize=1 -numworkers=16 -cleanup=false -skiplist -duration=10m
+```
+
+### 3. Install dependencies
+
+```bash
+pip install aistore boto3
+```
+
+### 4. Run the benchmark
+
+```bash
+python3 bench.py
+```
+
+Flags:
+
+| Flag | Default | Description |
+|---|---|---|
+| `--endpoint` | `$AIS_ENDPOINT` | AIS proxy URL |
+| `--bucket` | `my-bucket` | S3 bucket name |
+| `--provider` | `aws` | backend provider |
+| `--runs` | `20` | listing repetitions per scale point |
+
+## Results
+
+![NBI benchmark result](../../../../docs/assets/nbi/nbi_scale.png)
+
+```
+==========================================================================================================================
+NBI latency-vs-scale  (20 runs each)
+==========================================================================================================================
+Objects     Creation           --- Regular (ms) ---                 --- NBI (ms) ---                   --- S3 Direct (ms) ---          Speedup
+                           p50      p95      p99       sd       p50      p95      p99       sd       p50      p95      p99       sd    (R/N)
+--------------------------------------------------------------------------------------------------------------------------
+ 1K             472ms       179      429      429       90        25      110      110       20       259      302      302       21     7.2x
+ 2K             559ms       321      603      603       97        32      106      106       21       548      604      604       38     9.9x
+ 5K            1094ms       826     1190     1190      103        60      135      135       29      1343     1640     1640       88    13.8x
+10K            1934ms      1766     2201     2201      160       164      199      199       39      3100     4842     4842      608    10.7x
+20K            3455ms      3541     3978     3978      194       271      394      394       38      5475     6140     6140      277    13.1x
+40K            6578ms      7249     8075     8075      460       577      682      682       61     11246    14341    14341     1089    12.6x
+50K            7978ms      9110    10072    10072      592       674      841      841       69     13873    21246    21246     1770    13.5x
+80K           13375ms     14724    15973    15973      633      1244     1367     1367       61     22180    28541    28541     1731    11.8x
+==========================================================================================================================
+```