wip

Makefile

@@ -90,10 +90,11 @@ release/assets.tar.gz: assets/*
 	tar -zcf $@ assets
 
 release: build
-release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64
-release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64
-release: release/webui_linux_amd64 release/webui_macos_amd64 release/webui_macos_arm64
-release: release/assets.tar.gz release/migrations.tar.gz
+release: release/client_linux_amd64
+# release: release/server_linux_amd64 release/server_macos_amd64 release/server_macos_arm64
+# release: release/client_linux_amd64 release/client_macos_amd64 release/client_macos_arm64
+# release: release/webui_linux_amd64 release/webui_macos_amd64 release/webui_macos_arm64
+# release: release/assets.tar.gz release/migrations.tar.gz
 
 test: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
 test: migrate

cmd/fuzz-test/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"io/fs"
 	stdlog "log"
 	"math/rand"
@@ -452,6 +453,15 @@ type MatchError struct {
 	actualContent   string
 }
 
+func isDirEmpty(path string) bool {
+	file, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	_, err = file.Readdirnames(1)
+	return err == io.EOF
+}
+
 func compareDirs(project int64, expected string, actual string) ([]MatchError, error) {
 	var errors []MatchError
 
@@ -461,6 +471,11 @@ func compareDirs(project int64, expected string, actual string) ([]MatchError, e
 	for path, expectedType := range expectedObjects {
 		actualType, found := actualObjects[path]
 		if !found {
+			// FIXME: This is to handle a special case where we leave empty directories behind
+			//        in the source directory.
+			if actualType == typeDirectory && isDirEmpty(filepath.Join(expected, path)) {
+				continue
+			}
 			errors = append(errors, MatchError{
 				project:      project,
 				path:         path,
@@ -524,9 +539,9 @@ func logMatchErrors(ctx context.Context, matchErrors []MatchError) {
 
 		switch {
 		case matchErr.expectedType == -1:
-			logger.Info(ctx, "missing file in target dir", props...)
+			logger.Info(ctx, "extra file in rebuild dir", props...)
 		case matchErr.actualType == -1:
-			logger.Info(ctx, "missing file in source dir", props...)
+			logger.Info(ctx, "missing file in rebuild dir", props...)
 		case matchErr.expectedType != matchErr.actualType:
 			props = append(props, zap.String("expected", typeStr(matchErr.expectedType)), zap.String("actual", typeStr(matchErr.actualType)))
 			logger.Info(ctx, "object type mismatch", props...)

internal/files/preallocate_linux.go

@@ -9,7 +9,7 @@ import (
 )
 
 func PreAllocate(file *os.File, size int64) error {
-	if size > 0 {
+	if size > 512*1024 {
 		return unix.Fallocate(int(file.Fd()), 0, 0, size)
 	}
 	return nil

internal/files/writer.go

@@ -46,6 +46,7 @@ func writeObject(rootDir string, cacheObjectsDir string, reader *db.TarReader, h
 		}
 		hashHex := hex.EncodeToString(content)
 		return makeSymlink(filepath.Join(cacheObjectsDir, hashHex, header.Name), path)
+
 	case tar.TypeReg:
 		dir := filepath.Dir(path)
 		_, err := retryFileErrors(dir, func() (interface{}, error) {
@@ -87,6 +88,7 @@ func writeObject(rootDir string, cacheObjectsDir string, reader *db.TarReader, h
 
 	case tar.TypeSymlink:
 		return makeSymlink(header.Linkname, path)
+
 	case 'D':
 		err := os.Remove(path)
 		if errors.Is(err, fs.ErrNotExist) {

pkg/cli/client.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"os"
+	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -108,9 +110,17 @@ func ClientExecute() {
 	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Second)
 	defer cancel()
 
+	os.RemoveAll("client.prof")
+	file, err := os.Create("client.prof")
+	if err != nil {
+		panic(fmt.Sprintf("cannot open profile path client.prod: %v", err))
+	}
+	_ = pprof.StartCPUProfile(file)
+	defer pprof.StopCPUProfile()
+
 	cmd := NewClientCommand()
 
-	err := cmd.ExecuteContext(ctx)
+	err = cmd.ExecuteContext(ctx)
 
 	client := client.FromContext(cmd.Context())
 	if client != nil {