Skip to content

Commit 9eaaba8

Browse files
alex-aizmanalex-aizman
committed
core: object chunks; chunk manifest
* add StoreCompleted() and StorePartial() - the latter for checkpointing * add lom.CompleteUfest() to persist (manifest, LOM) atomically * add chunk #1 fixup logic; add rollback * micro-optimize GetChunk() for sequential access ------ * chunk manifest to support io.ReaderAt (with this, all LomReader interfaces) ------ * unit: fix CSM resolver registration typo * unit: add testing parameter to StoreCompleted() for non-existing chunks ------ * cleanup and refactor; unify error handling * part eight, prev. commit: 7c8a770 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 82db204 commit 9eaaba8

4 files changed

Lines changed: 232 additions & 143 deletions

File tree

‎ais/tgts3mpt.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -475,8 +475,8 @@ func (t *target) completeMpt(w http.ResponseWriter, r *http.Request, items []str
475475
ecode, errF := poi.finalize()
476476
freePOI(poi)
477477

478-
// .6 write manifest
479-
if err := manifest.Store(lom); err != nil {
478+
// .6 write manifest // TODO -- FIXME: niy
479+
if err := manifest.StoreCompleted(lom, true); err != nil {
480480
s3.WriteMptErr(w, r, errF, ecode, lom, uploadID)
481481
return
482482
}

‎core/lchunk_test.go‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
4242
bucketLocal = "CHUNK_TEST_Local"
4343
)
4444

45+
const unitTestingWithNoRealChunks = true
46+
4547
localBck := cmn.Bck{Name: bucketLocal, Provider: apc.AIS, Ns: cmn.NsGlobal}
4648

4749
fs.CSM.Reg(fs.ObjectType, &fs.ObjectContentResolver{}, true)
@@ -142,7 +144,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
142144
manifest := createChunkManifest(testFileSize, 3, chunkSizes, "test-session-001", lom)
143145

144146
// Store manifest
145-
err := manifest.Store(lom)
147+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
146148
Expect(err).NotTo(HaveOccurred())
147149

148150
// After store, manifest should be marked as completed
@@ -202,7 +204,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
202204
chunkSizes := []int64{testFileSize}
203205
manifest := createChunkManifest(testFileSize, 1, chunkSizes, "single-chunk-session", lom)
204206

205-
err := manifest.Store(lom)
207+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
206208
Expect(err).NotTo(HaveOccurred())
207209

208210
loadedManifest := &core.Ufest{}
@@ -231,7 +233,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
231233

232234
manifest := createChunkManifest(testFileSize, numChunks, chunkSizes, "many-chunks-session", lom)
233235

234-
err := manifest.Store(lom)
236+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
235237
Expect(err).NotTo(HaveOccurred())
236238

237239
loadedManifest := &core.Ufest{}
@@ -267,7 +269,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
267269
testTime := time.Date(2025, 8, 2, 15, 30, 45, 0, time.UTC)
268270
manifest.Created = testTime
269271

270-
err := manifest.Store(lom)
272+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
271273
Expect(err).NotTo(HaveOccurred())
272274

273275
loadedManifest := &core.Ufest{}
@@ -293,7 +295,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
293295
{Siz: 524000, Num: 2, Path: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "def456")},
294296
}
295297

296-
err := manifest.Store(lom)
298+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
297299
Expect(err).To(HaveOccurred())
298300
Expect(err.Error()).To(ContainSubstring("invalid chunk-manifest"))
299301
})
@@ -307,7 +309,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
307309
manifest.Num = 0
308310
manifest.Chunks = []core.Uchunk{}
309311

310-
err := manifest.Store(lom)
312+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
311313
Expect(err).To(HaveOccurred())
312314
Expect(err.Error()).To(ContainSubstring("invalid chunk-manifest"))
313315
})
@@ -335,7 +337,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
335337
manifest.Chunks[i].Cksum = cos.NewCksum(cos.ChecksumCesXxh, longVal)
336338
}
337339

338-
err := manifest.Store(lom)
340+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
339341
Expect(err).To(HaveOccurred())
340342
Expect(err.Error()).To(ContainSubstring("too large"))
341343
})
@@ -359,7 +361,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
359361
// Store valid manifest first
360362
chunkSizes := []int64{testFileSize}
361363
manifest := createChunkManifest(testFileSize, 1, chunkSizes, "version-test", lom)
362-
err := manifest.Store(lom)
364+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
363365
Expect(err).NotTo(HaveOccurred())
364366

365367
// Corrupt the version byte
@@ -382,7 +384,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
382384
// Store valid manifest
383385
chunkSizes := []int64{testFileSize}
384386
manifest := createChunkManifest(testFileSize, 1, chunkSizes, "checksum-test", lom)
385-
err := manifest.Store(lom)
387+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
386388
Expect(err).NotTo(HaveOccurred())
387389

388390
b, err := fs.GetXattr(localFQN, xattrChunk)
@@ -408,7 +410,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
408410
// Store valid manifest first
409411
chunkSizes := []int64{testFileSize}
410412
manifest := createChunkManifest(testFileSize, 1, chunkSizes, "truncated-test", lom)
411-
err := manifest.Store(lom)
413+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
412414
Expect(err).NotTo(HaveOccurred())
413415

414416
// Truncate the xattr data
@@ -437,7 +439,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
437439
{Siz: 548576, Num: 2, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "validchecksum"), MD5: "md5hash2"}, // total = 1048576
438440
}
439441

440-
err := manifest.Store(lom)
442+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
441443
Expect(err).NotTo(HaveOccurred())
442444

443445
loadedManifest := &core.Ufest{}
@@ -462,7 +464,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
462464
{Siz: 548576, Num: 2, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "validchecksum"), MD5: "md5hash2"}, // total = 1048576
463465
}
464466

465-
err := manifest.Store(lom)
467+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
466468
Expect(err).NotTo(HaveOccurred())
467469

468470
loadedManifest := &core.Ufest{}
@@ -489,7 +491,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
489491
{Siz: 0, Num: 3, Cksum: cos.NewCksum(cos.ChecksumCesXxh, "empty2"), MD5: "md5_3"}, // another zero - total = 1048576
490492
}
491493

492-
err := manifest.Store(lom)
494+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
493495
Expect(err).NotTo(HaveOccurred())
494496

495497
loadedManifest := &core.Ufest{}
@@ -513,7 +515,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
513515
{Siz: testFileSize, Num: 1, Path: "test", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "checksum"), MD5: "md5hash"},
514516
}
515517

516-
err := manifest.Store(lom)
518+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
517519
Expect(err).NotTo(HaveOccurred())
518520

519521
loadedManifest := &core.Ufest{}
@@ -536,7 +538,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
536538
{Siz: testFileSize, Num: 1, Path: "test", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "checksum"), MD5: "md5hash"},
537539
}
538540

539-
err := manifest.Store(lom)
541+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
540542
Expect(err).NotTo(HaveOccurred())
541543

542544
loadedManifest := &core.Ufest{}
@@ -569,7 +571,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
569571
{Siz: testFileSize, Num: 1, Path: "chunk1", Cksum: cos.NewCksum(cos.ChecksumCesXxh, "abc123"), MD5: "md5hash"},
570572
}
571573

572-
err := manifest.Store(lom)
574+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
573575
Expect(err).NotTo(HaveOccurred())
574576

575577
loadedManifest := &core.Ufest{}
@@ -605,7 +607,7 @@ var _ = Describe("Chunk Manifest Xattrs", func() {
605607
// Initially not completed
606608
Expect(manifest.Completed()).To(BeFalse())
607609

608-
err := manifest.Store(lom)
610+
err := manifest.StoreCompleted(lom, unitTestingWithNoRealChunks)
609611
Expect(err).NotTo(HaveOccurred())
610612

611613
// After store, should be completed

‎core/lom_test.go‎

Lines changed: 10 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"github.com/NVIDIA/aistore/core/mock"
2121
"github.com/NVIDIA/aistore/fs"
2222
"github.com/NVIDIA/aistore/tools/readers"
23-
"github.com/NVIDIA/aistore/tools/trand"
2423

2524
onexxh "github.com/OneOfOne/xxhash"
2625
. "github.com/onsi/ginkgo/v2"
@@ -73,7 +72,7 @@ var _ = Describe("LOM", func() {
7372

7473
fs.CSM.Reg(fs.ObjectType, &fs.ObjectContentResolver{}, true)
7574
fs.CSM.Reg(fs.WorkfileType, &fs.WorkfileContentResolver{}, true)
76-
fs.CSM.Reg(fs.ObjChunkType, &fs.ObjectContentResolver{}, true)
75+
fs.CSM.Reg(fs.ObjChunkType, &fs.ObjChunkContentResolver{}, true)
7776

7877
bmd := mock.NewBaseBownerMock(
7978
meta.NewBck(
@@ -1070,68 +1069,6 @@ var _ = Describe("LOM", func() {
10701069
})
10711070
})
10721071

1073-
Describe("Chunked Objects – round-trip store/load", func() {
1074-
const (
1075-
objName = "chunked/roundtrip.bin"
1076-
// total size 1 MiB split into three chunks
1077-
sz1 = int64(400_000)
1078-
sz2 = int64(400_000)
1079-
sz3 = int64(cos.MiB) - sz1 - sz2
1080-
totalSize = int64(cos.MiB)
1081-
)
1082-
1083-
var (
1084-
fqn = mis[0].MakePathFQN(&localBckB, fs.ObjectType, objName)
1085-
// helper to build a manifest with paths + fake checksums
1086-
newManifest = func() *core.Ufest {
1087-
return &core.Ufest{
1088-
Size: totalSize,
1089-
Num: 3,
1090-
Chunks: []core.Uchunk{
1091-
{Siz: sz1, Num: 3, Path: "ch/" + objName + ".00001", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
1092-
{Siz: sz2, Num: 2, Path: "ch/" + objName + ".00002", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
1093-
{Siz: sz3, Num: 1, Path: "ch/" + objName + ".00003", Cksum: cos.NewCksum(cos.ChecksumCesXxh, trand.String(16))},
1094-
},
1095-
}
1096-
}
1097-
)
1098-
1099-
It("should persist and reload chunk manifest & flag", func() {
1100-
// ── 1. create Chunk-#1 and initial LOM ────────────────────────────────
1101-
createTestFile(fqn, int(sz1))
1102-
lom := newBasicLom(fqn, totalSize)
1103-
manifest := newManifest()
1104-
1105-
// store manifest, then persist normal LOM metadata
1106-
Expect(manifest.Store(lom)).To(Succeed())
1107-
Expect(persist(lom)).To(Succeed())
1108-
lom.UncacheUnless()
1109-
1110-
// ── 2. reload the object (fresh LOM) ──────────────────────────────────
1111-
re := newBasicLom(fqn)
1112-
Expect(re.Load(false, false)).To(Succeed())
1113-
1114-
// NOTE: IsChunked() is still false at this point because the
1115-
// manifest hasn’t been loaded yet – load it now, which also
1116-
// re-sets the lmflChunk flag in memory.
1117-
var out core.Ufest
1118-
Expect(out.Load(re)).To(Succeed())
1119-
1120-
// flag should now be restored
1121-
// TODO -- FIXME Expect(re.IsChunked()).To(BeTrue())
1122-
Expect(re.Lsize()).To(Equal(totalSize))
1123-
1124-
// compare manifest contents
1125-
Expect(out.Num).To(Equal(manifest.Num))
1126-
Expect(out.Size).To(Equal(manifest.Size))
1127-
for i := range 3 {
1128-
Expect(out.Chunks[i].Siz).To(Equal(manifest.Chunks[i].Siz))
1129-
Expect(out.Chunks[i].Path).To(Equal(manifest.Chunks[i].Path))
1130-
Expect(out.Chunks[i].Cksum.Equal(manifest.Chunks[i].Cksum)).To(BeTrue())
1131-
}
1132-
})
1133-
})
1134-
11351072
Describe("UfestReader", func() {
11361073
const (
11371074
fileSize = 1 * cos.GiB
@@ -1142,7 +1079,7 @@ var _ = Describe("LOM", func() {
11421079
testObject := "chunked/large-file.bin"
11431080
localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
11441081

1145-
createDummyFile(localFQN)
1082+
createTestFile(localFQN, 0)
11461083
lom := newBasicLom(localFQN, fileSize)
11471084

11481085
Expect(cos.CreateDir(filepath.Dir(localFQN))).NotTo(HaveOccurred())
@@ -1180,7 +1117,7 @@ var _ = Describe("LOM", func() {
11801117
Expect(ufest.Size).To(Equal(int64(fileSize)))
11811118

11821119
// Store manifest (this will mark it as completed)
1183-
err := ufest.Store(lom)
1120+
err := ufest.StoreCompleted(lom)
11841121
Expect(err).NotTo(HaveOccurred())
11851122
Expect(ufest.Completed()).To(BeTrue())
11861123

@@ -1225,7 +1162,7 @@ var _ = Describe("LOM", func() {
12251162
testObject := "chunked/empty-file.bin"
12261163
localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
12271164

1228-
createDummyFile(localFQN)
1165+
createTestFile(localFQN, 0)
12291166
lom := newBasicLom(localFQN, 0)
12301167
ufest := core.NewUfest("empty-test-"+cos.GenTie(), lom)
12311168

@@ -1239,7 +1176,7 @@ var _ = Describe("LOM", func() {
12391176
err = ufest.Add(chunk, 0, 1)
12401177
Expect(err).NotTo(HaveOccurred())
12411178

1242-
err = ufest.Store(lom)
1179+
err = ufest.StoreCompleted(lom)
12431180
Expect(err).NotTo(HaveOccurred())
12441181

12451182
reader, err := ufest.NewReader()
@@ -1255,7 +1192,7 @@ var _ = Describe("LOM", func() {
12551192

12561193
By("testing incomplete manifest")
12571194
fqn2 := mis[0].MakePathFQN(&localBckB, fs.ObjectType, "incomplete.bin")
1258-
createDummyFile(fqn2)
1195+
createTestFile(fqn2, 0)
12591196
lom2 := newBasicLom(fqn2)
12601197
ufest2 := core.NewUfest("incomplete-test-"+cos.GenTie(), lom2)
12611198

@@ -1269,7 +1206,7 @@ var _ = Describe("LOM", func() {
12691206
testObject := "chunked/size-test.bin"
12701207
localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
12711208

1272-
createDummyFile(localFQN)
1209+
createTestFile(localFQN, 0)
12731210
lom := newBasicLom(localFQN)
12741211
ufest := core.NewUfest("size-test-"+cos.GenTie(), lom)
12751212

@@ -1286,7 +1223,7 @@ var _ = Describe("LOM", func() {
12861223
Expect(err).NotTo(HaveOccurred())
12871224

12881225
lom.SetSize(200)
1289-
err = ufest.Store(lom)
1226+
err = ufest.StoreCompleted(lom)
12901227
Expect(err).NotTo(HaveOccurred())
12911228

12921229
reader, err := ufest.NewReader()
@@ -1311,7 +1248,7 @@ var _ = Describe("LOM", func() {
13111248
testObject := "chunked/many-small-chunks.bin"
13121249
localFQN := mis[0].MakePathFQN(&localBckB, fs.ObjectType, testObject)
13131250

1314-
createDummyFile(localFQN)
1251+
createTestFile(localFQN, 0)
13151252
lom := newBasicLom(localFQN, totalFileSize)
13161253
ufest := core.NewUfest("multi-chunk-test-"+cos.GenTie(), lom)
13171254

@@ -1330,7 +1267,7 @@ var _ = Describe("LOM", func() {
13301267
Expect(err).NotTo(HaveOccurred())
13311268
}
13321269

1333-
err := ufest.Store(lom)
1270+
err := ufest.StoreCompleted(lom)
13341271
Expect(err).NotTo(HaveOccurred())
13351272

13361273
reader, err := ufest.NewReader()

0 commit comments

Comments
 (0)