sys: cpu and memory (cgroup v2 in a constrained container)

* test cgroup v2 manually
  - `docker run --rm --cpus=1.5 --memory=512m ...`
* add README section
* add TestMain to `sys.Init()` only once
------
* part four, prev. commit: 3fb3aeb

Signed-off-by: Alex Aizman <alex.aizman@gmail.com>

Author: alex-aizman

sys/README.md

@@ -22,6 +22,7 @@ The `sys` package provides lightweight runtime visibility into host and containe
   - [Container memory (cgroup v1)](#container-memory-cgroup-v1)
 - [Container detection](#container-detection)
 - [Fallback](#fallback)
+- [Example: testing `sys` package inside a constrained container](#example-testing-sys-package-inside-a-constrained-container)
 - [Current limitations and future plans](#current-limitations-and-future-plans)
   - [cgroup v1 deprecation](#cgroup-v1-deprecation)
   - [Hardcoded thresholds](#hardcoded-thresholds)
@@ -171,7 +172,86 @@ The package follows these rules:
 - for CPU: preserve a usable percentage whenever possible
 - for memory: prefer host stats over failing when container-specific files cannot be read
 
-This makes the package resilient across bare-metal, VMs, containers, and mixed environments, at the cost of occasionally returning an approximation instead of a strict container-scoped answer.
+## Example: testing `sys` package inside a constrained container
+
+A simple way to validate container-aware CPU and memory reporting is to compare the same test run on the host and inside a Docker container with explicit CPU and memory limits.
+
+### Host run
+
+```bash
+go test -v -tags=debug
+```
+
+Example output:
+
+```text
+=== RUN   TestNumCPU
+--- PASS: TestNumCPU (0.00s)
+=== RUN   TestLoadAvg
+    sys_test.go:41: Load average: 0.63, 0.49, 0.49
+--- PASS: TestLoadAvg (0.00s)
+=== RUN   TestMaxProcs
+--- PASS: TestMaxProcs (0.00s)
+=== RUN   TestMemoryStats
+    sys_test.go:76: Memory stats: {used 29GiB, free 2GiB, buffcache 20GiB, actfree 23GiB}
+    sys_test.go:79: Either swap is off or failed to read its stats
+--- PASS: TestMemoryStats (0.00s)
+=== RUN   TestProcAndMaxLoad
+    sys_test.go:110: First call: load=0, extreme=false
+    ...
+    sys_test.go:133: Second call: load=3, extreme=false
+    sys_test.go:145: Process CPU usage:   1.85%
+--- PASS: TestProcAndMaxLoad (5.69s)
+PASS
+ok      github.com/NVIDIA/aistore/sys   5.696s
+```
+
+### Container run
+
+```bash
+docker run --rm \
+  --cpus=1.5 \
+  --memory=512m \
+  -v "$PWD":/src -w /src \
+  -v "$HOME/go/pkg/mod":/go/pkg/mod \
+  -v "$HOME/.cache/go-build":/root/.cache/go-build \
+  golang:1.25 \
+  go test ./sys -run . -v -count=1 2>&1
+```
+
+Example output:
+
+```text
+=== RUN   TestNumCPU
+--- PASS: TestNumCPU (0.00s)
+=== RUN   TestLoadAvg
+    sys_test.go:41: Load average: 0.36, 0.41, 0.47
+--- PASS: TestLoadAvg (0.00s)
+=== RUN   TestMaxProcs
+--- PASS: TestMaxProcs (0.00s)
+=== RUN   TestMemoryStats
+    sys_test.go:76: Memory stats: {used 29MiB, free 483MiB, buffcache 152KiB, actfree 483MiB}
+    sys_test.go:79: Either swap is off or failed to read its stats
+--- PASS: TestMemoryStats (0.00s)
+=== RUN   TestProcAndMaxLoad
+    sys_test.go:110: First call: load=0, extreme=false
+    ...
+    sys_test.go:133: Second call: load=15, extreme=false
+    sys_test.go:145: Process CPU usage:  14.82%
+--- PASS: TestProcAndMaxLoad (5.70s)
+```
+
+The comparison illustrates several points:
+
+* On the host, `TestMemoryStats` reports host-scale memory totals.
+* Inside the container, the same test reports memory bounded by the cgroup limit (`--memory=512m`) rather than the host's physical RAM.
+* `TestNumCPU` and `TestMaxProcs` exercise init-time CPU detection and container-aware CPU count.
+* `TestProcAndMaxLoad` burns CPU in-process and verifies that `MaxLoad2()` reports non-zero utilization on a subsequent sample.
+* Swap may report as zero or be unavailable inside short-lived containers; this is not unusual.
+
+To further confirm container-scoped memory accounting, rerun the container example with a different limit (for example, `--memory=4G`). `TestMemoryStats` should then report a total close to 4 GiB instead of 512 MiB.
+
+> The container example above assumes cgroup v2 - the default on modern Linux distributions and container runtimes.
 
 ## Current limitations and future plans
 

sys/init.go

@@ -28,6 +28,8 @@ func init() {
 
 func isContainerized() bool { return contDetected || contForced }
 
+// func CgroupVer() int { return cgroupVer } // uncomment for unit tests
+
 // container-aware CPU count; GOMAXPROCS; okv2 if { container && can read cgroup v2 }
 // - AIS node (`aisnode`) calls Init() once upon startup
 // - external modules that skip it still get a sane NumCPU() - see above

sys/sys_test.go

@@ -17,14 +17,23 @@ import (
 	"github.com/NVIDIA/aistore/tools/tassert"
 )
 
+// example running it in a constrained container:
+// docker run --rm --cpus=1.5 --memory=512m   -v "$PWD":/src -w /src \
+//            -v "$HOME/go/pkg/mod":/go/pkg/mod -v "$HOME/.cache/go-build":/root/.cache/go-build \
+//            golang:1.25   go test ./sys -run . -v -count=1 2>&
+
+func TestMain(m *testing.M) {
+	sys.Init(false /*forceCont*/)
+	os.Exit(m.Run())
+}
+
 func checkSkipOS(t *testing.T, oss ...string) {
 	if slices.Contains(oss, runtime.GOOS) {
 		t.Skipf("skipping test for %s platform", runtime.GOOS)
 	}
 }
 
 func TestNumCPU(t *testing.T) {
-	sys.Init(false)
 	checkSkipOS(t, "darwin")
 	if sys.NumCPU() < 1 || sys.NumCPU() > runtime.NumCPU() {
 		t.Errorf("Wrong number of CPUs %d (%d)", sys.NumCPU(), runtime.NumCPU())
@@ -64,30 +73,14 @@ func TestMemoryStats(t *testing.T) {
 	tassert.Errorf(t, mem.ActualUsed+mem.ActualFree == mem.Total, "ActualUsed + ActualFree must equal Total: %+v", mem)
 	tassert.Errorf(t, mem.Used >= mem.ActualUsed, "Used must be >= ActualUsed (BuffCache=%d): %+v", mem.BuffCache, mem)
 
+	// t.Logf("cgroup version: %d", sys.CgroupVer()) // uncomment for visibility
+
 	var sb cos.SB
 	sb.Grow(80)
 	mem.Str(&sb)
 	t.Logf("Memory stats: %s", sb)
 
-	checkSkipOS(t, "darwin")
-
-	var memHost sys.MemStat
-	err = memHost.Get()
-	tassert.CheckFatal(t, err)
-
-	// NOTE: force "containerized" even though it'll then fail to read cgroups
-	sys.Init(true /*forceCont*/)
-
-	var memCont sys.MemStat
-	err = memCont.Get()
-	tassert.CheckFatal(t, err)
-
-	// this check in particular would make more sense if running in a container
-	tassert.Errorf(t, memHost.Total >= memCont.Total,
-		"Container's memory total is greater than the host one.\nOS: %+v\nContainer: %+v", memHost, memCont)
-
-	if memHost.SwapTotal == 0 && memHost.SwapFree == 0 {
-		// Not an error(e.g, Jenkins VM has swap off) - just a warning
+	if mem.SwapTotal == 0 && mem.SwapFree == 0 {
 		t.Log("Either swap is off or failed to read its stats")
 	}
 }
@@ -96,7 +89,6 @@ func TestProcAndMaxLoad(t *testing.T) {
 	if testing.Short() {
 		t.Skipf("skipping %s in short mode", t.Name())
 	}
-	sys.Init(false)
 	checkSkipOS(t, "darwin")
 
 	pid := os.Getpid()