Skip to content

Commit 73bc5a4

Browse files
angeliniangelini
committed
Remove dir fuzz error
1 parent bddb60b commit 73bc5a4

File tree

3 files changed

+173
-47
lines changed

3 files changed

+173
-47
lines changed

‎cmd/fuzz-test/main.go

+99-47
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package main
22

33
import (
44
"context"
5-
"errors"
65
"fmt"
76
"io/fs"
87
stdlog "log"
@@ -180,7 +179,7 @@ func (o AddFileOp) Apply() error {
180179
}
181180

182181
func (o AddFileOp) String() string {
183-
return fmt.Sprintf("AddFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
182+
return fmt.Sprintf("AddFile(%s, %d)", Join(o.dir, o.name), len(o.content))
184183
}
185184

186185
type UpdateFileOp struct {
@@ -198,7 +197,7 @@ func newUpdateFileOp(base string, project int64) Operation {
198197
return SkipOp{}
199198
}
200199

201-
return AddFileOp{
200+
return UpdateFileOp{
202201
base: base,
203202
dir: dir,
204203
name: files[rand.Intn(len(files))],
@@ -211,7 +210,7 @@ func (o UpdateFileOp) Apply() error {
211210
}
212211

213212
func (o UpdateFileOp) String() string {
214-
return fmt.Sprintf("UpdateFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
213+
return fmt.Sprintf("UpdateFile(%s, %d)", Join(o.dir, o.name), len(o.content))
215214
}
216215

217216
type AddDirOp struct {
@@ -277,7 +276,36 @@ func (o RemoveFileOp) Apply() error {
277276
}
278277

279278
func (o RemoveFileOp) String() string {
280-
return fmt.Sprintf("RemoveFileOp(%s)", Join(o.dir, o.name))
279+
return fmt.Sprintf("RemoveFile(%s)", Join(o.dir, o.name))
280+
}
281+
282+
type RemoveDirOp struct {
283+
base string
284+
dir string
285+
name string
286+
}
287+
288+
func newRemoveDirOp(base string, project int64) Operation {
289+
dir := fmt.Sprint(project)
290+
dirs := objectFilter(walkDir(Join(base, dir)), typeDirectory)
291+
292+
if len(dirs) == 0 {
293+
return SkipOp{}
294+
}
295+
296+
return RemoveDirOp{
297+
base: base,
298+
dir: dir,
299+
name: dirs[rand.Intn(len(dirs))],
300+
}
301+
}
302+
303+
func (o RemoveDirOp) Apply() error {
304+
return os.RemoveAll(Join(o.base, o.dir, o.name))
305+
}
306+
307+
func (o RemoveDirOp) String() string {
308+
return fmt.Sprintf("RemoveDir(%s)", Join(o.dir, o.name))
281309
}
282310

283311
type AddSymlinkOp struct {
@@ -320,12 +348,12 @@ func (o AddSymlinkOp) Apply() error {
320348
}
321349

322350
func (o AddSymlinkOp) String() string {
323-
return fmt.Sprintf("AddSymlinkOp(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
351+
return fmt.Sprintf("AddSymlink(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
324352
}
325353

326354
type OpConstructor func(dir string, project int64) Operation
327355

328-
var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newAddSymlinkOp}
356+
var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newRemoveDirOp, newAddSymlinkOp}
329357

330358
func randomOperation(baseDir string, project int64) Operation {
331359
var operation Operation = SkipOp{}
@@ -340,71 +368,97 @@ func randomOperation(baseDir string, project int64) Operation {
340368
return operation
341369
}
342370

343-
func createDirs(projects int) (string, string, string, string, error) {
371+
type Directories struct {
372+
base string
373+
reset string
374+
previous string
375+
step string
376+
}
377+
378+
func createDirectories(projects int) (*Directories, error) {
344379
var dirs []string
345380

346-
for _, name := range []string{"base", "continue", "reset", "step"} {
381+
for _, name := range []string{"base", "reset", "previous", "step"} {
347382
dir, err := os.MkdirTemp("", fmt.Sprintf("dl-ft-%s-", name))
348383
if err != nil {
349-
return "", "", "", "", fmt.Errorf("cannot create tmp dir: %w", err)
384+
return nil, fmt.Errorf("cannot create tmp dir: %w", err)
350385
}
351386

352387
for projectIdx := 1; projectIdx <= projects; projectIdx++ {
353388
err = os.MkdirAll(filepath.Join(dir, fmt.Sprint(projectIdx)), 0755)
354389
if err != nil {
355-
return "", "", "", "", fmt.Errorf("cannot create project dir: %w", err)
390+
return nil, fmt.Errorf("cannot create project dir: %w", err)
356391
}
357392
}
358393
dirs = append(dirs, dir)
359394
}
360395

361-
return dirs[0], dirs[1], dirs[2], dirs[3], nil
396+
return &Directories{
397+
base: dirs[0],
398+
reset: dirs[1],
399+
previous: dirs[2],
400+
step: dirs[3],
401+
}, nil
402+
}
403+
404+
func (d *Directories) Log(ctx context.Context) {
405+
logger.Info(ctx, "base", zap.String("path", d.base))
406+
logger.Info(ctx, "reset", zap.String("path", d.reset))
407+
logger.Info(ctx, "prev", zap.String("path", d.previous))
408+
logger.Info(ctx, "step", zap.String("path", d.step))
362409
}
363410

364-
func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, baseDir, continueDir, resetDir, stepDir string) error {
411+
func (d *Directories) RemoveAll() {
412+
os.RemoveAll(d.base)
413+
os.RemoveAll(d.reset)
414+
os.RemoveAll(d.previous)
415+
os.RemoveAll(d.step)
416+
}
417+
418+
func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, dirs *Directories) (int64, error) {
365419
err := operation.Apply()
366420
if err != nil {
367-
return fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
421+
return -1, fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
368422
}
369423

370-
version, _, err := client.Update(ctx, project, projectDir(baseDir, project))
424+
version, _, err := client.Update(ctx, project, projectDir(dirs.base, project))
371425
if err != nil {
372-
return fmt.Errorf("failed to update project %d: %w", project, err)
426+
return -1, fmt.Errorf("failed to update project %d: %w", project, err)
373427
}
374428

375-
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(continueDir, project), "")
429+
os.RemoveAll(projectDir(dirs.reset, project))
430+
err = os.MkdirAll(projectDir(dirs.reset, project), 0755)
376431
if err != nil {
377-
return fmt.Errorf("failed to rebuild continue project %d: %w", project, err)
432+
return -1, fmt.Errorf("failed to create reset dir %s: %w", projectDir(dirs.reset, project), err)
378433
}
379434

380-
os.RemoveAll(projectDir(resetDir, project))
381-
err = os.MkdirAll(projectDir(resetDir, project), 0755)
435+
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.reset, project), "")
382436
if err != nil {
383-
return fmt.Errorf("failed to create reset dir %s: %w", projectDir(resetDir, project), err)
437+
return -1, fmt.Errorf("failed to rebuild from reset, project %d: %w", project, err)
384438
}
385439

386-
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(resetDir, project), "")
440+
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.previous, project), "")
387441
if err != nil {
388-
return fmt.Errorf("failed to rebuild reset project %d: %w", project, err)
442+
return -1, fmt.Errorf("failed to rebuild from previous version, project %d: %w", project, err)
389443
}
390444

391-
os.RemoveAll(projectDir(stepDir, project))
392-
err = os.MkdirAll(projectDir(stepDir, project), 0755)
445+
os.RemoveAll(projectDir(dirs.step, project))
446+
err = os.MkdirAll(projectDir(dirs.step, project), 0755)
393447
if err != nil {
394-
return fmt.Errorf("failed to create step dir %s: %w", projectDir(stepDir, project), err)
448+
return -1, fmt.Errorf("failed to create step dir %s: %w", projectDir(dirs.step, project), err)
395449
}
396450

397451
stepVersion := int64(rand.Intn(int(version)))
398-
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(stepDir, project), "")
452+
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(dirs.step, project), "")
399453
if err != nil {
400-
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
454+
return -1, fmt.Errorf("failed to rebuild step to version %v, project %d: %w", stepVersion, project, err)
401455
}
402-
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(stepDir, project), "")
456+
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(dirs.step, project), "")
403457
if err != nil {
404-
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
458+
return -1, fmt.Errorf("failed to rebuild step from version %v, project %d: %w", stepVersion, project, err)
405459
}
406460

407-
return nil
461+
return stepVersion, nil
408462
}
409463

410464
type MatchError struct {
@@ -507,35 +561,35 @@ func logOpLog(ctx context.Context, opLog []Operation) {
507561
}
508562
}
509563

510-
func verifyDirs(ctx context.Context, projects int, baseDir, continueDir, resetDir, stepDir string) error {
564+
func verifyDirs(ctx context.Context, projects int, dirs *Directories, stepVersion int64) error {
511565
for projectIdx := 1; projectIdx <= projects; projectIdx++ {
512566
project := int64(projectIdx)
513567

514-
matchErrors, err := compareDirs(project, projectDir(baseDir, project), projectDir(resetDir, project))
568+
matchErrors, err := compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.reset, project))
515569
if err != nil {
516570
return fmt.Errorf("failed to compare base & reset dirs: %w", err)
517571
}
518572
if len(matchErrors) > 0 {
519573
logMatchErrors(ctx, matchErrors)
520-
return errors.New("reset directory match error")
574+
return fmt.Errorf("reset directory match error, project %d", project)
521575
}
522576

523-
matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(continueDir, project))
577+
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.previous, project))
524578
if err != nil {
525-
return fmt.Errorf("failed to compare base & continue dirs: %w", err)
579+
return fmt.Errorf("failed to compare base & previous dirs: %w", err)
526580
}
527581
if len(matchErrors) > 0 {
528582
logMatchErrors(ctx, matchErrors)
529-
return errors.New("continue directory match error")
583+
return fmt.Errorf("previous version directory match error, project %d", project)
530584
}
531585

532-
matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(stepDir, project))
586+
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.step, project))
533587
if err != nil {
534588
return fmt.Errorf("failed to compare base & step dirs: %w", err)
535589
}
536590
if len(matchErrors) > 0 {
537591
logMatchErrors(ctx, matchErrors)
538-
return errors.New("step directory match error")
592+
return fmt.Errorf("step from verion %d directory match error, project %d", stepVersion, project)
539593
}
540594
}
541595
return nil
@@ -552,29 +606,27 @@ func fuzzTest(ctx context.Context, client *dlc.Client, projects, iterations int)
552606
}
553607
}
554608

555-
baseDir, continueDir, resetDir, stepDir, err := createDirs(projects)
609+
dirs, err := createDirectories(projects)
556610
if err != nil {
557611
return err
558612
}
559-
defer os.RemoveAll(baseDir)
560-
defer os.RemoveAll(continueDir)
561-
defer os.RemoveAll(resetDir)
562-
defer os.RemoveAll(stepDir)
613+
// defer dirs.RemoveAll()
614+
dirs.Log(ctx)
563615

564616
var opLog []Operation
565617

566618
for iterIdx := 0; iterIdx < iterations; iterIdx++ {
567619
project := int64(rand.Intn(projects) + 1)
568620

569-
operation := randomOperation(baseDir, project)
621+
operation := randomOperation(dirs.base, project)
570622
opLog = append(opLog, operation)
571623

572-
err := runIteration(ctx, client, project, operation, baseDir, continueDir, resetDir, stepDir)
624+
stepVersion, err := runIteration(ctx, client, project, operation, dirs)
573625
if err != nil {
574626
return fmt.Errorf("failed to run iteration %d: %w", iterIdx, err)
575627
}
576628

577-
err = verifyDirs(ctx, projects, baseDir, continueDir, resetDir, stepDir)
629+
err = verifyDirs(ctx, projects, dirs, stepVersion)
578630
if err != nil {
579631
logOpLog(ctx, opLog)
580632
return err

‎test/client_combined_test.go

+64
Original file line numberDiff line numberDiff line change
@@ -621,3 +621,67 @@ func TestCombinedWithPrefixDirectoryBug(t *testing.T) {
621621
"abc": {content: "abc v2"},
622622
})
623623
}
624+
625+
func TestCombinedAddAndRemoveInDirBug(t *testing.T) {
626+
tc := util.NewTestCtx(t, auth.Project, 1)
627+
defer tc.Close()
628+
629+
writeProject(tc, 1, 1)
630+
writeEmptyDir(tc, 1, 1, nil, "a/")
631+
632+
c, fs, close := createTestClient(tc)
633+
defer close()
634+
635+
tmpDir := emptyTmpDir(t)
636+
defer os.RemoveAll(tmpDir)
637+
638+
rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
639+
version: 1,
640+
count: 1,
641+
})
642+
643+
verifyDir(t, tmpDir, 1, map[string]expectedFile{
644+
"a/": {content: "", fileType: typeDirectory},
645+
})
646+
647+
writeFile(t, tmpDir, "a/b", "a/b v1")
648+
649+
update(tc, c, 1, tmpDir, expectedResponse{
650+
version: 2,
651+
count: 1,
652+
})
653+
654+
removeFile(t, tmpDir, "a/b")
655+
656+
update(tc, c, 1, tmpDir, expectedResponse{
657+
version: 3,
658+
count: 2,
659+
})
660+
661+
updateStream := newMockUpdateServer(tc.Context(), 1, map[string]expectedObject{
662+
"b": {content: "b v4"},
663+
"a/": {deleted: true},
664+
})
665+
err := fs.Update(updateStream)
666+
require.NoError(t, err, "fs.Update")
667+
668+
os.RemoveAll(tmpDir)
669+
err = os.MkdirAll(tmpDir, 0755)
670+
require.NoError(t, err, "recreate tmp dir %v", tmpDir)
671+
672+
rebuild(tc, c, 1, i(2), tmpDir, nil, expectedResponse{
673+
version: 2,
674+
count: 1,
675+
})
676+
677+
debugObjects(tc)
678+
679+
rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
680+
version: 4,
681+
count: 2,
682+
})
683+
684+
verifyDir(t, tmpDir, 4, map[string]expectedFile{
685+
"b": {content: "b v4"},
686+
})
687+
}

‎test/shared_test.go

+10
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,16 @@ func writeFile(t *testing.T, dir string, path string, content string) {
243243
require.NoError(t, err, "write file %v", path)
244244
}
245245

246+
func removeFile(t *testing.T, dir string, path string) {
247+
err := os.Remove(filepath.Join(dir, path))
248+
require.NoError(t, err, "remove file %v", path)
249+
}
250+
251+
func removeDir(t *testing.T, dir string, path string) {
252+
err := os.RemoveAll(filepath.Join(dir, path))
253+
require.NoError(t, err, "remove dir %v", path)
254+
}
255+
246256
func emptyTmpDir(t *testing.T) string {
247257
dir, err := os.MkdirTemp("", "dateilager_tests_")
248258
require.NoError(t, err, "create temp dir")

0 commit comments

Comments
 (0)