Compute usage from the sizes in the manifest.

This is a more accurate calculation of the files that were supplied by the
user. In particular, it avoids counting the sizes of extra files that were
added by the application, e.g., non-gobbler-related dotfiles.

Author: LTLA

delete.go

@@ -109,7 +109,7 @@ func deleteAssetHandler(reqpath string, globals *globalConfiguration) error {
     if _, err := os.Stat(asset_dir); errors.Is(err, os.ErrNotExist) {
         return nil
     }
-    to_free, err := computeUsage(asset_dir)
+    to_free, err := computeAssetUsage(asset_dir)
     if err != nil {
         return fmt.Errorf("failed to compute usage for %s; %v", asset_dir, err)
     }
@@ -206,7 +206,7 @@ func deleteVersionHandler(reqpath string, globals *globalConfiguration) error {
     if _, err := os.Stat(version_dir); errors.Is(err, os.ErrNotExist) {
         return nil
     }
-    to_free, err := computeUsage(version_dir)
+    to_free, err := computeVersionUsage(version_dir)
     if err != nil {
         return fmt.Errorf("failed to compute usage for %s; %v", version_dir, err)
     }

delete_test.go

@@ -53,6 +53,11 @@ func mockRegistryForDeletion(project, asset string, versions []string) (string,
             return "", fmt.Errorf("failed to write a placeholder file; %w", err)
         }
         expected_size += len(message)
+
+        err = reindexDirectory(reg, project, asset, v)
+        if err != nil {
+            return "", fmt.Errorf("failed to reindex the directory; %w", err)
+        }
     }
 
     err = os.WriteFile(filepath.Join(project_dir, usageFileName), []byte(fmt.Sprintf(`{ "total": %d }`, expected_size)), 0644)
@@ -334,7 +339,7 @@ func TestDeleteVersion(t *testing.T) {
             if err != nil {
                 t.Fatalf("failed to read usage after deletion; %v", err)
             }
-            expected, err := computeUsage(filepath.Join(asset_dir, survivor))
+            expected, err := computeVersionUsage(filepath.Join(asset_dir, survivor))
             if err != nil {
                 t.Fatalf("failed to compute usage for the survivor; %v", err)
             }

probation.go

@@ -144,7 +144,7 @@ func baseProbationHandler(reqpath string, globals *globalConfiguration, approve
         }
 
     } else {
-        freed, err := computeUsage(version_dir)
+        freed, err := computeVersionUsage(version_dir)
         if err != nil {
             return fmt.Errorf("failed to compute usage for %q; %w", version_dir, err)
         }

probation_test.go

@@ -58,6 +58,11 @@ func mockProbationVersion(reg, project, asset, version string) error {
         return fmt.Errorf("failed to create some mock files; %w", err)
     }
 
+    err = reindexDirectory(reg, project, asset, version)
+    if err != nil {
+        return fmt.Errorf("failed to reindex the directory; %w", err)
+    }
+
     err = os.WriteFile(filepath.Join(project_dir, usageFileName), []byte(fmt.Sprintf(`{ "total": %d }`, len(contents))), 0644)
     if err != nil {
         return fmt.Errorf("failed to create mock usage; %w", err)

upload.go

@@ -198,7 +198,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error {
     }
 
     {
-        extra, err := computeUsage(version_dir)
+        extra, err := computeVersionUsage(version_dir)
         if err != nil {
             return fmt.Errorf("failed to compute usage for the new version at %q; %w", version_dir, err)
         }

upload_test.go

@@ -128,7 +128,7 @@ func TestUploadHandlerSimple(t *testing.T) {
     if err != nil {
         t.Fatalf("failed to read the usage; %v", err)
     }
-    expected_usage, err := computeUsage(project_dir)
+    expected_usage, err := computeProjectUsage(project_dir)
     if err != nil {
         t.Fatalf("failed to compute the expected usage; %v", err)
     }
@@ -382,7 +382,7 @@ func TestUploadHandlerUpdate(t *testing.T) {
         if err != nil {
             t.Fatalf("failed to read the usage; %v", err)
         }
-        expected_usage, err := computeUsage(project_dir)
+        expected_usage, err := computeProjectUsage(project_dir)
         if err != nil {
             t.Fatalf("failed to compute the expected usage; %v", err)
         }

usage.go

@@ -4,8 +4,6 @@ import (
     "os"
     "encoding/json"
     "fmt"
-    "io/fs"
-    "strings"
     "path/filepath"
     "time"
     "net/http"
@@ -34,37 +32,72 @@ func readUsage(path string) (*usageMetadata, error) {
     return &output, nil
 }
 
-func computeUsage(dir string) (int64, error) {
+func computeProjectUsage(path string) (int64, error) {
     var total int64
     total = 0
 
-    err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
-        if err != nil {
-            return fmt.Errorf("failed to walk into %q; %w", path, err)
+    assets, err := os.ReadDir(path)
+    if err != nil {
+        return total, fmt.Errorf("failed to list assets; %w", err)
+    }
+
+    for _, aentry := range assets {
+        if !aentry.IsDir() {
+            continue
         }
 
-        if info.IsDir() {
-            return nil
+        asset := aentry.Name()
+        asize, err := computeAssetUsage(filepath.Join(path, asset))
+        if err != nil {
+            return total, fmt.Errorf("failed to get usage for asset %q; %w", asset, err) 
         }
 
-        // Skipping internal files.
-        base := filepath.Base(path)
-        if strings.HasPrefix(base, "..") {
-            return nil
+        total += asize
+    }
+
+    return total, nil
+}
+
+func computeAssetUsage(path string) (int64, error) {
+    var total int64
+    total = 0
+
+    versions, err := os.ReadDir(path)
+    if err != nil {
+        return total, fmt.Errorf("failed to list versions; %w", err)
+    }
+
+    for _, ventry := range versions {
+        if !ventry.IsDir() {
+            continue
         }
 
-        // Skipping symlinks.
-        restat, err := info.Info()
+        version := ventry.Name()
+        vsize, err := computeVersionUsage(filepath.Join(path, version))
         if err != nil {
-            return fmt.Errorf("failed to stat %q; %w", path, err)
-        }
-        if restat.Mode() & os.ModeSymlink == os.ModeSymlink {
-            return nil
+            return total, fmt.Errorf("failed to get usage for version %q; %w", version, err) 
         }
 
-        total += restat.Size()
-        return nil
-    })
+        total += vsize
+    }
+
+    return total, nil
+}
+
+func computeVersionUsage(path string) (int64, error) {
+    var total int64
+    total = 0
+
+    man, err := readManifest(path)
+    if err != nil {
+        return total, fmt.Errorf("failed to open manifest; %w", err)
+    }
+
+    for _, mm := range man {
+        if mm.Link == nil {
+            total += mm.Size
+        }
+    }
 
     return total, err
 }
@@ -111,7 +144,7 @@ func refreshUsageHandler(reqpath string, globals *globalConfiguration) (*usageMe
     }
     defer globals.Locks.Unlock(project_dir)
 
-    new_usage, err := computeUsage(project_dir)
+    new_usage, err := computeProjectUsage(project_dir)
     if err != nil {
         return nil, fmt.Errorf("failed to compute usage for %q; %w", *(incoming.Project), err)
     }

usage_test.go

@@ -82,15 +82,29 @@ func TestComputeUsage(t *testing.T) {
         t.Fatalf("failed to create mock file; %v", err)
     }
 
-    // Actually running some tests.
-    total, err := computeUsage(src)
+    // Executing the transfer and computing the size.
+    reg, err := os.MkdirTemp("", "")
+    if err != nil {
+        t.Fatalf("failed to create the registry; %v", err)
+    }
+
+    project := "pokemon"
+    asset := "pikachu"
+    version := "yellow"
+    err = transferDirectory(src, reg, project, asset, version)
+    if err != nil {
+        t.Fatalf("failed to perform the transfer; %v", err)
+    }
+
+    total, err := computeProjectUsage(filepath.Join(reg, project))
     if err != nil {
         t.Fatalf("failed to create compute usage; %v", err)
     }
     if total != int64(expected_size) {
         t.Fatalf("sum of file sizes is different from expected (%d, got %d)", expected_size, total)
     }
 
+    // Symlinks are ignored.
     err = os.Symlink(
         filepath.Join(src, "moves", "grass", "razor_leaf"), 
         filepath.Join(src, "moves", "grass", "vine_whip"),
@@ -99,12 +113,19 @@ func TestComputeUsage(t *testing.T) {
         t.Fatalf("failed to create mock file; %v", err)
     }
 
-    total, err = computeUsage(src)
+    version = "green"
+    err = transferDirectory(src, reg, project, asset, version)
+    if err != nil {
+        t.Fatalf("failed to perform the transfer; %v", err)
+    }
+
+    combined_total, err := computeProjectUsage(filepath.Join(reg, project))
     if err != nil {
         t.Fatalf("failed to create compute usage; %v", err)
     }
-    if total != int64(expected_size) {
-        t.Fatalf("sum of file sizes is different from expected (%d, got %d) when ignoring soft links", expected_size, total)
+    total_added := combined_total - total
+    if total_added != int64(expected_size) {
+        t.Fatalf("sum of file sizes is different from expected (%d, got %d)", expected_size, total_added)
     }
 }
 
@@ -125,18 +146,22 @@ func TestRefreshUsageHandler(t *testing.T) {
 
     expected_size := int64(0)
     for _, asset := range []string{ "WHEE", "STUFF", "BLAH" } {
-        version_dir := filepath.Join(project_dir, asset)
-        err := os.Mkdir(version_dir, 0755)
+        src, err := os.MkdirTemp("", "test-")
         if err != nil {
-            t.Fatalf("failed to create asset directory; %v", err)
+            t.Fatalf("failed to create tempdir; %v", err)
         }
 
         message := "I am " + asset
-        err = os.WriteFile(filepath.Join(version_dir, "thingy"), []byte(message), 0644)
+        err = os.WriteFile(filepath.Join(src, "thingy"), []byte(message), 0644)
         if err != nil {
             t.Fatalf("failed to write a mock file; %v", err)
         }
 
+        err = transferDirectory(src, reg, project_name, asset, "v1")
+        if err != nil {
+            t.Fatalf("failed to perform the transfer; %v", err)
+        }
+
         expected_size += int64(len(message))
     }