Skip to content

Commit 416fc04

Browse files
committed
feat: native bucket inventory (cli)
* new top-level command `ais nbi {create | rm | show}` ---- * part nineteen, prev. commit: ed7ac5d Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent ed7ac5d commit 416fc04

9 files changed

Lines changed: 288 additions & 97 deletions

File tree

cmd/cli/cli/app.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
22
/*
3-
* Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.
44
*/
55
package cli
66

@@ -243,6 +243,7 @@ func (a *acli) setupCommands(emptyCmdline bool) {
243243
showCmdPerformance,
244244
remClusterCmd,
245245
mlCmd,
246+
nbiCmd,
246247
a.getAliasCmd(),
247248
}
248249

cmd/cli/cli/bucket_hdlr.go

Lines changed: 0 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -286,15 +286,6 @@ var (
286286
waitFlag,
287287
waitJobXactFinishedFlag,
288288
},
289-
cmdCreateNBI: {
290-
nbiNameFlag,
291-
invPrefixFlag,
292-
nameOnlyFlag,
293-
allPropsFlag,
294-
nbiPagesPerChunkFlag,
295-
nbiMaxEntriesPerChunkFlag,
296-
nbiForceFlag,
297-
},
298289
}
299290
)
300291

@@ -351,21 +342,6 @@ var (
351342
bcmplop{additionalCompletions: []cli.BashCompleteFunc{bpropCompletions}},
352343
),
353344
}
354-
bucketCmdCreateNBI = cli.Command{
355-
Name: cmdCreateNBI,
356-
Usage: "Create bucket inventory for subsequent distributed listing,\n" +
357-
indent1 + "e.g.:\n" +
358-
indent1 + "\t* create-inventory s3://abc\t- create inventory with default (name, size) properties;\n" +
359-
indent1 + "\t* create-inventory s3://abc --inv-name my-first-inventory\t- same, with a custom inventory name;\n" +
360-
indent1 + "\t* create-inventory s3://abc --prefix images/\t- inventory only objects under 'images/';\n" +
361-
indent1 + "\t* create-inventory s3://abc --all\t- inventory with all object properties;\n" +
362-
indent1 + "\t* create-inventory s3://abc --name-only\t- lightweight: object names only;\n" +
363-
indent1 + "\t* create-inventory ais://@remais/xyz --inv-pages 2\t- remote AIS, with 2 pages per chunk.",
364-
ArgsUsage: bucketArgument,
365-
Flags: sortFlags(bucketCmdsFlags[cmdCreateNBI]),
366-
Action: createInventoryHandler,
367-
BashComplete: bucketCompletions(bcmplop{}),
368-
}
369345

370346
bucketCmd = cli.Command{
371347
Name: commandBucket,
@@ -433,7 +409,6 @@ var (
433409
makeAlias(&showCmdBucket, &mkaliasOpts{newName: commandShow}),
434410
},
435411
},
436-
bucketCmdCreateNBI,
437412
},
438413
}
439414
)
@@ -1029,69 +1004,6 @@ proceed:
10291004
}
10301005
}
10311006

1032-
//
1033-
// createInventoryHandler
1034-
//
1035-
1036-
func createInventoryHandler(c *cli.Context) error {
1037-
if c.NArg() == 0 {
1038-
return missingArgumentsError(c, c.Command.ArgsUsage)
1039-
}
1040-
if c.NArg() > 1 {
1041-
return incorrectUsageMsg(c, "", c.Args()[1:])
1042-
}
1043-
bck, err := parseBckURI(c, c.Args().Get(0), false)
1044-
if err != nil {
1045-
return err
1046-
}
1047-
1048-
// msg
1049-
msg := &apc.CreateNBIMsg{}
1050-
msg.SetFlag(apc.LsNoDirs)
1051-
1052-
if flagIsSet(c, invPrefixFlag) {
1053-
msg.Prefix = parseStrFlag(c, invPrefixFlag)
1054-
}
1055-
1056-
switch {
1057-
case flagIsSet(c, nameOnlyFlag):
1058-
msg.SetFlag(apc.LsNameOnly)
1059-
msg.Props = apc.GetPropsName // abs. minimum
1060-
case flagIsSet(c, allPropsFlag):
1061-
msg.AddProps(apc.GetPropsAll...) // all supported props
1062-
default:
1063-
msg.AddProps(apc.GetPropsName, apc.GetPropsSize, apc.GetPropsCached) // NOTE default
1064-
}
1065-
1066-
// inv name
1067-
if flagIsSet(c, nbiNameFlag) {
1068-
msg.Name = parseStrFlag(c, nbiNameFlag)
1069-
if err := cos.CheckAlphaPlus(msg.Name, "inventory name"); err != nil {
1070-
return err
1071-
}
1072-
}
1073-
1074-
// advanced
1075-
if flagIsSet(c, nbiPagesPerChunkFlag) {
1076-
a := parseIntFlag(c, nbiPagesPerChunkFlag)
1077-
msg.PagesPerChunk = int64(a)
1078-
}
1079-
if flagIsSet(c, nbiMaxEntriesPerChunkFlag) {
1080-
a := parseIntFlag(c, nbiMaxEntriesPerChunkFlag)
1081-
msg.MaxEntriesPerChunk = int64(a)
1082-
}
1083-
msg.Force = flagIsSet(c, nbiForceFlag)
1084-
1085-
// do
1086-
xid, err := api.CreateBucketInventory(apiBP, bck, msg)
1087-
if err != nil {
1088-
return V(err)
1089-
}
1090-
1091-
actionDone(c, fmt.Sprintf("Creating inventory %s. %s", bck.Cname(""), toMonitorMsg(c, xid, "")))
1092-
return nil
1093-
}
1094-
10951007
////////////
10961008
// lsbCtx //
10971009
////////////

cmd/cli/cli/const.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const (
3333
commandPerf = "performance"
3434
commandStorage = "storage"
3535
commandTLS = "tls"
36+
commandNBI = "nbi"
3637

3738
commandSearch = "search"
3839
)
@@ -1401,8 +1402,6 @@ var (
14011402
// bucket inventory
14021403
//
14031404

1404-
const cmdCreateNBI = apc.ActCreateNBI
1405-
14061405
// Deprecated: Feb 2026 - planned removal by April–May 2026.
14071406
// S3-specific bucket inventory support will be replaced by the native API and CLI
14081407
// supporting all remote buckets and all backends

cmd/cli/cli/nbi_hdlr.go

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2+
// This file handles commands that interact with the cluster.
3+
/*
4+
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
5+
*/
6+
package cli
7+
8+
import (
9+
"fmt"
10+
"sort"
11+
12+
"github.com/NVIDIA/aistore/api"
13+
"github.com/NVIDIA/aistore/api/apc"
14+
"github.com/NVIDIA/aistore/cmd/cli/teb"
15+
"github.com/NVIDIA/aistore/cmn/cos"
16+
17+
"github.com/urfave/cli"
18+
)
19+
20+
const (
21+
createNBIUsage = "Create bucket inventory (snapshot) for subsequent fast listing,\n" +
22+
indent1 + "e.g.:\n" +
23+
indent1 + "\t* ais nbi create s3://abc\t- create inventory with default (name, size) properties;\n" +
24+
indent1 + "\t* ais nbi create s3://abc --inv-name my-first-inventory\t- same, with a custom inventory name;\n" +
25+
indent1 + "\t* ais nbi create s3://abc --prefix images/\t- inventory only objects under 'images/';\n" +
26+
indent1 + "\t* ais nbi create s3://abc --all\t- inventory with all object properties;\n" +
27+
indent1 + "\t* ais nbi create s3://abc --name-only\t- lightweight: object names only;\n" +
28+
indent1 + "\t* ais nbi create ais://@remais/xyz --inv-pages 2\t- remote AIS, with 2 pages per chunk."
29+
30+
removeNBIUsage = "Remove bucket inventory,\n" +
31+
indent1 + "e.g.:\n" +
32+
indent1 + "\t* ais nbi rm s3://abc\t- remove inventory for the bucket;\n" +
33+
indent1 + "\t* ais nbi rm s3://abc --inv-name my-first-inventory\t- remove specific named inventory."
34+
35+
showNBIUsage = "Show bucket inventory,\n" +
36+
indent1 + "e.g.:\n" +
37+
indent1 + "\t* ais nbi show s3://abc\t- show inventory details for the bucket;\n" +
38+
indent1 + "\t* ais nbi show s3://abc --inv-name my-first-inventory\t- show specific named inventory."
39+
)
40+
41+
// flags
42+
var (
43+
nbiCmdFlags = map[string][]cli.Flag{
44+
commandCreate: {
45+
nbiNameFlag,
46+
invPrefixFlag,
47+
nameOnlyFlag,
48+
allPropsFlag,
49+
nbiPagesPerChunkFlag,
50+
nbiMaxEntriesPerChunkFlag,
51+
nbiForceFlag,
52+
},
53+
commandRemove: {
54+
nbiNameFlag,
55+
},
56+
commandShow: {
57+
nbiNameFlag,
58+
verboseFlag,
59+
},
60+
}
61+
)
62+
63+
// commands
64+
var (
65+
cmdCreateNBI = cli.Command{
66+
Name: commandCreate,
67+
Usage: createNBIUsage,
68+
ArgsUsage: bucketArgument,
69+
Flags: sortFlags(nbiCmdFlags[commandCreate]),
70+
Action: createNBIHandler,
71+
BashComplete: bucketCompletions(bcmplop{}),
72+
}
73+
cmdRemoveNBI = cli.Command{
74+
Name: commandRemove,
75+
Usage: removeNBIUsage,
76+
ArgsUsage: bucketArgument,
77+
Flags: sortFlags(nbiCmdFlags[commandRemove]),
78+
Action: removeNBIHandler,
79+
BashComplete: bucketCompletions(bcmplop{}),
80+
}
81+
cmdShowNBI = cli.Command{
82+
Name: commandShow,
83+
Usage: showNBIUsage,
84+
ArgsUsage: bucketArgument,
85+
Flags: sortFlags(nbiCmdFlags[commandShow]),
86+
Action: showNBIHandler,
87+
BashComplete: bucketCompletions(bcmplop{}),
88+
}
89+
90+
// top-level
91+
nbiCmd = cli.Command{
92+
Name: commandNBI,
93+
Usage: "Manage native bucket inventory (NBI) - create (to facilitate faster list-objects), show, or remove bucket inventory snapshots",
94+
Subcommands: []cli.Command{
95+
cmdCreateNBI,
96+
cmdRemoveNBI,
97+
cmdShowNBI, // TODO -- FIXME: add `ais show nbi` alias
98+
},
99+
}
100+
)
101+
102+
//
103+
// createNBIHandler
104+
//
105+
106+
func createNBIHandler(c *cli.Context) error {
107+
if c.NArg() == 0 {
108+
return missingArgumentsError(c, c.Command.ArgsUsage)
109+
}
110+
if c.NArg() > 1 {
111+
return incorrectUsageMsg(c, "", c.Args()[1:])
112+
}
113+
bck, err := parseBckURI(c, c.Args().Get(0), false)
114+
if err != nil {
115+
return err
116+
}
117+
118+
// msg
119+
msg := &apc.CreateNBIMsg{}
120+
msg.SetFlag(apc.LsNoDirs)
121+
122+
if flagIsSet(c, invPrefixFlag) {
123+
msg.Prefix = parseStrFlag(c, invPrefixFlag)
124+
}
125+
126+
switch {
127+
case flagIsSet(c, nameOnlyFlag):
128+
msg.SetFlag(apc.LsNameOnly)
129+
msg.Props = apc.GetPropsName // abs. minimum
130+
case flagIsSet(c, allPropsFlag):
131+
msg.AddProps(apc.GetPropsAll...) // all supported props
132+
default:
133+
msg.AddProps(apc.GetPropsName, apc.GetPropsSize, apc.GetPropsCached) // NOTE default
134+
}
135+
136+
// inv name
137+
if flagIsSet(c, nbiNameFlag) {
138+
msg.Name = parseStrFlag(c, nbiNameFlag)
139+
if err := cos.CheckAlphaPlus(msg.Name, "inventory name"); err != nil {
140+
return err
141+
}
142+
}
143+
144+
// advanced
145+
if flagIsSet(c, nbiPagesPerChunkFlag) {
146+
a := parseIntFlag(c, nbiPagesPerChunkFlag)
147+
msg.PagesPerChunk = int64(a)
148+
}
149+
if flagIsSet(c, nbiMaxEntriesPerChunkFlag) {
150+
a := parseIntFlag(c, nbiMaxEntriesPerChunkFlag)
151+
msg.MaxEntriesPerChunk = int64(a)
152+
}
153+
msg.Force = flagIsSet(c, nbiForceFlag)
154+
155+
// do
156+
xid, err := api.CreateNBI(apiBP, bck, msg)
157+
if err != nil {
158+
return V(err)
159+
}
160+
161+
actionDone(c, fmt.Sprintf("Creating inventory %s. %s", bck.Cname(""), toMonitorMsg(c, xid, "")))
162+
return nil
163+
}
164+
165+
//
166+
// removeNBIHandler
167+
//
168+
169+
func removeNBIHandler(c *cli.Context) error {
170+
if c.NArg() == 0 {
171+
return missingArgumentsError(c, c.Command.ArgsUsage)
172+
}
173+
if c.NArg() > 1 {
174+
return incorrectUsageMsg(c, "", c.Args()[1:])
175+
}
176+
177+
// bucket
178+
bck, err := parseBckURI(c, c.Args().Get(0), false)
179+
if err != nil {
180+
return err
181+
}
182+
183+
// inv name
184+
var (
185+
s string
186+
invName = parseStrFlag(c, nbiNameFlag)
187+
)
188+
if invName != "" {
189+
if err := cos.CheckAlphaPlus(invName, "inventory name"); err != nil {
190+
return err
191+
}
192+
s = " '" + invName + "'"
193+
}
194+
if err := api.DestroyNBI(apiBP, bck, invName); err != nil {
195+
return V(err)
196+
}
197+
198+
actionDone(c, fmt.Sprintf("Removed inventory%s for bucket %s", s, bck.String()))
199+
return nil
200+
}
201+
202+
//
203+
// showNBIHandler
204+
//
205+
206+
func showNBIHandler(c *cli.Context) error {
207+
if c.NArg() == 0 {
208+
return missingArgumentsError(c, c.Command.ArgsUsage)
209+
}
210+
if c.NArg() > 1 {
211+
return incorrectUsageMsg(c, "", c.Args()[1:])
212+
}
213+
214+
bck, err := parseBckURI(c, c.Args().Get(0), false)
215+
if err != nil {
216+
return err
217+
}
218+
219+
invName := parseStrFlag(c, nbiNameFlag)
220+
if invName != "" {
221+
if err := cos.CheckAlphaPlus(invName, "inventory name"); err != nil {
222+
return err
223+
}
224+
}
225+
226+
infos, err := api.GetNBI(apiBP, bck, invName)
227+
if err != nil {
228+
return V(err)
229+
}
230+
231+
type extInfo struct {
232+
apc.NBIInfo
233+
Bucket string
234+
}
235+
236+
lst := make([]*extInfo, 0, len(infos))
237+
for _, v := range infos {
238+
var ext extInfo
239+
ext.NBIInfo = *v
240+
ext.Bucket = bck.Cname("")
241+
lst = append(lst, &ext)
242+
}
243+
sort.Slice(lst, func(i, j int) bool { return lst[i].Name < lst[j].Name })
244+
245+
if flagIsSet(c, verboseFlag) {
246+
return teb.Print(lst, teb.NBITmplVerbose)
247+
}
248+
return teb.Print(lst, teb.NBITmpl)
249+
}

0 commit comments

Comments
 (0)