|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +## Prerequisites: ################################################################################# |
| 4 | +# - aistore cluster with at least 3 active targets |
| 5 | +# (cleanup mode requires CountActiveTs >= 2; we take one target down, so need 3) |
| 6 | +# - ais (CLI) |
| 7 | +# |
| 8 | +## What this tests: |
| 9 | +# - 'ais start rebalance --cleanup' on a cluster where rebalance has left |
| 10 | +# misplaced copies behind (via maintenance + rejoin). |
| 11 | +# - Asserts (after every rebalance phase): logical object count is preserved |
| 12 | +# end-to-end (no data loss); physical count grows after rejoin-rebalance |
| 13 | +# (misplaced copies exist) and returns to baseline after cleanup (copies |
| 14 | +# reclaimed); 'ais storage summary' agrees with 'ais ls --summary'. |
| 15 | +# |
| 16 | +## Usage: |
| 17 | +## reb-cleanup-mode.sh [--bucket BUCKET] [--num-shards N] |
| 18 | +# |
| 19 | +## Example: |
| 20 | +## ./ais/test/scripts/reb-cleanup-mode.sh |
| 21 | +## ./ais/test/scripts/reb-cleanup-mode.sh --num-shards 9999 |
| 22 | + |
| 23 | +if ! [ -x "$(command -v ais)" ]; then |
| 24 | + echo "Error: ais (CLI) not installed" >&2 |
| 25 | + exit 1 |
| 26 | +fi |
| 27 | + |
| 28 | +## defaults |
| 29 | +bucket="ais://reb-cleanup-$RANDOM" |
| 30 | +num_shards=9999 |
| 31 | + |
| 32 | +while (( "$#" )); do |
| 33 | + case "${1}" in |
| 34 | + --bucket) bucket=$2; shift; shift;; |
| 35 | + --num-shards) num_shards=$2; shift; shift;; |
| 36 | + *) echo "fatal: unknown argument '${1}'"; exit 1;; |
| 37 | + esac |
| 38 | +done |
| 39 | + |
| 40 | +## uncomment for verbose output |
| 41 | +## set -x |
| 42 | + |
| 43 | +## state to be restored on exit |
| 44 | +node="" |
| 45 | +bucket_created=false |
| 46 | + |
| 47 | +cleanup() { |
| 48 | + rc=$? |
| 49 | + ## bring node out of maintenance if we left it there mid-test |
| 50 | + [[ -z "$node" ]] || ais cluster add-remove-nodes stop-maintenance "$node" --yes 1>/dev/null 2>&1 |
| 51 | + [[ "$bucket_created" == "false" ]] || ais rmb "$bucket" --yes 1>/dev/null 2>&1 |
| 52 | + exit $rc |
| 53 | +} |
| 54 | +trap cleanup EXIT INT TERM |
| 55 | + |
| 56 | +## --- measurement helpers (all use stable user-facing CLI surface) --- |
| 57 | + |
| 58 | +## logical count: number of unique objects in BMD (data-integrity invariant) |
| 59 | +get_logical() { ais ls "$bucket" --count-only | awk '{print $2}' | tr -d ','; } |
| 60 | + |
| 61 | +## physical count: LOMs across all mountpaths, including misplaced copies |
| 62 | +get_physical() { ais ls "$bucket" --summary -H | awk '{print $3}'; } |
| 63 | + |
| 64 | +## storage summary: cached object count (cross-validates get_physical) |
| 65 | +get_usage() { ais storage summary "$bucket" -H | awk '{print $2}'; } |
| 66 | + |
| 67 | +assert_eq() { |
| 68 | + local label="$1" actual="$2" expected="$3" |
| 69 | + [[ "$actual" == "$expected" ]] || \ |
| 70 | + { echo "FAIL: $label: got $actual, expected $expected"; exit 1; } |
| 71 | +} |
| 72 | + |
| 73 | +assert_gt() { |
| 74 | + local label="$1" actual="$2" floor="$3" |
| 75 | + [[ $actual -gt $floor ]] || \ |
| 76 | + { echo "FAIL: $label: got $actual, expected > $floor"; exit 1; } |
| 77 | +} |
| 78 | + |
| 79 | +## 'ais wait rebalance' has a race window where it returns before a just-launched |
| 80 | +## reb is registered as running. A second wait, after a brief pause, picks up |
| 81 | +## the actual running xaction. |
| 82 | +## TODO: fix 'ais wait rebalance' to wait for next state change (separate CLI work). |
| 83 | +wait_reb() { |
| 84 | + ais wait rebalance |
| 85 | + sleep 1 |
| 86 | + ais wait rebalance |
| 87 | +} |
| 88 | + |
| 89 | +## --- precheck: need >= 3 active targets --- |
| 90 | +nat=$(ais show cluster target | awk '/^t\[/ && /online/' | wc -l) |
| 91 | +[[ $nat -ge 3 ]] || { echo "FAIL: need >= 3 active targets, got $nat"; exit 1; } |
| 92 | + |
| 93 | +## --- step 1: create bucket and fill with shards --- |
| 94 | +ais create $bucket 1>/dev/null || exit $? |
| 95 | +bucket_created=true |
| 96 | +ais archive gen-shards "${bucket}/shard-{00001..$(printf %05d $num_shards)}.tar" \ |
| 97 | + --fcount 1 --num-workers 10 1>/dev/null || exit $? |
| 98 | + |
| 99 | +## --- step 2: baseline (logical and physical must agree initially) --- |
| 100 | +logical_initial=$(get_logical) |
| 101 | +physical_initial=$(get_physical) |
| 102 | +usage_initial=$(get_usage) |
| 103 | +assert_eq "initial logical" "$logical_initial" "$num_shards" |
| 104 | +assert_eq "initial physical" "$physical_initial" "$num_shards" |
| 105 | +assert_eq "initial storage-summary" "$usage_initial" "$num_shards" |
| 106 | + |
| 107 | +## --- step 3: pick a random target and put it in maintenance --- |
| 108 | +node=$(ais advanced random-node) |
| 109 | +ais cluster add-remove-nodes start-maintenance "$node" --yes 1>/dev/null || exit $? |
| 110 | +wait_reb |
| 111 | + |
| 112 | +## --- step 4: post-maintenance invariants --- |
| 113 | +## logical preserved (rebalance redistributed; nothing lost) |
| 114 | +## physical preserved (redistribution, not duplication) |
| 115 | +assert_eq "post-maint logical" "$(get_logical)" "$logical_initial" |
| 116 | +assert_eq "post-maint physical" "$(get_physical)" "$physical_initial" |
| 117 | +assert_eq "post-maint usage" "$(get_usage)" "$usage_initial" |
| 118 | + |
| 119 | +## --- step 5: bring node back; triggers rejoin-rebalance --- |
| 120 | +ais cluster add-remove-nodes stop-maintenance "$node" --yes 1>/dev/null || exit $? |
| 121 | +node="" ## node is back; clear so trap doesn't try to stop-maintenance again |
| 122 | +wait_reb |
| 123 | + |
| 124 | +## --- step 6: post-rejoin invariants --- |
| 125 | +## logical still preserved (full maintenance cycle without data loss) |
| 126 | +## physical MUST exceed initial (rejoin-reb left misplaced copies behind — |
| 127 | +## that's exactly what cleanup is for; if this assertion fails, the test is |
| 128 | +## not exercising cleanup work and any subsequent "no-op cleanup" PASS would |
| 129 | +## be meaningless) |
| 130 | +assert_eq "post-rejoin logical" "$(get_logical)" "$logical_initial" |
| 131 | +physical_rejoin=$(get_physical) |
| 132 | +usage_rejoin=$(get_usage) |
| 133 | +assert_gt "post-rejoin physical" "$physical_rejoin" "$physical_initial" |
| 134 | +assert_gt "post-rejoin usage" "$usage_rejoin" "$usage_initial" |
| 135 | + |
| 136 | +## --- step 7: run cleanup --- |
| 137 | +ais start rebalance --cleanup 1>/dev/null || exit $? |
| 138 | +wait_reb |
| 139 | + |
| 140 | +## --- step 8: post-cleanup invariants --- |
| 141 | +## logical preserved (cleanup must never remove the last copy of an object) |
| 142 | +## physical back to initial (all misplaced copies reclaimed) |
| 143 | +## usage agrees with physical |
| 144 | +assert_eq "post-cleanup logical" "$(get_logical)" "$logical_initial" |
| 145 | +assert_eq "post-cleanup physical" "$(get_physical)" "$physical_initial" |
| 146 | +assert_eq "post-cleanup usage" "$(get_usage)" "$usage_initial" |
| 147 | + |
| 148 | +echo "PASS: cleanup reclaimed $((physical_rejoin - physical_initial)) misplaced copies" |
0 commit comments