tests: add tests for stats consistency, #8898 #9003

add tests ensuring:
- borg info and create report same "This archive" deduplicated size
- before/after borg recreate it reports same "This archive" deduplicated size
- this/all archive(s) stats are same if 1 archive is in repo
- all archives stats is same for borg create and borg info

note that some stats differences are expected.

Author: ThomasWaldmann

src/borg/testsuite/archiver.py

@@ -1617,6 +1617,68 @@ def test_info(self):
         info_archive = self.cmd('info', '--first', '1', self.repository_location)
         assert 'Archive name: test\n' in info_archive
 
+    def test_info_matches_create_deduplicated_size(self):
+        # Create two identical files to ensure intra-archive deduplication happens,
+        # so the deduplicated size is visibly different from compressed size.
+        data = b'X' * (1024 * 80)
+        self.create_regular_file('file1', contents=data)
+        self.create_regular_file('file2', contents=data)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        create_json = json.loads(self.cmd('create', '--json', self.repository_location + '::test', 'input'))
+        info_json = json.loads(self.cmd('info', '--json', self.repository_location + '::test'))
+
+        # Compare deduplicated size of "This archive" between create and info.
+        dedup_create = create_json['archive']['stats']['deduplicated_size']
+        assert len(info_json['archives']) == 1
+        dedup_info = info_json['archives'][0]['stats']['deduplicated_size']
+        assert dedup_create == dedup_info
+
+    def test_info_matches_create_all_archives_deduplicated_size(self):
+        # Verify that the "All archives" deduplicated size reported by `borg create --json`
+        # matches the one reported by `borg info --json <repo>`.
+        data = b'Y' * (1024 * 80)
+        self.create_regular_file('file1', contents=data)
+        self.create_regular_file('file2', contents=data)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        # Create two archives so that the "All archives" section is meaningful and stable.
+        self.cmd('create', '--json', self.repository_location + '::a1', 'input')
+        create_json2 = json.loads(self.cmd('create', '--json', self.repository_location + '::a2', 'input'))
+        info_repo_json = json.loads(self.cmd('info', '--json', self.repository_location))
+
+        # In JSON, the repository-wide deduplicated size corresponds to cache.stats.unique_size.
+        dedup_create_all = create_json2['cache']['stats']['unique_size']
+        dedup_info_all = info_repo_json['cache']['stats']['unique_size']
+        assert dedup_create_all == dedup_info_all
+
+    def test_single_archive_all_equals_this_info_and_create(self):
+        # With a single archive in the repository, the deduplicated size for
+        # "This archive" and "All archives" should be identical, both in
+        # `borg create --json` and in `borg info --json` (repo and archive views).
+        data = b'Z' * (1024 * 80)
+        self.create_regular_file('file1', contents=data)
+        self.create_regular_file('file2', contents=data)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        create_json = json.loads(self.cmd('create', '--json', self.repository_location + '::only', 'input'))
+
+        # From create --json
+        dedup_this_create = create_json['archive']['stats']['deduplicated_size']
+        dedup_all_create = create_json['cache']['stats']['unique_size']
+
+        # From info --json (archive and repository views)
+        info_archive_json = json.loads(self.cmd('info', '--json', self.repository_location + '::only'))
+        info_repo_json = json.loads(self.cmd('info', '--json', self.repository_location))
+        assert len(info_archive_json['archives']) == 1
+        dedup_this_info = info_archive_json['archives'][0]['stats']['deduplicated_size']
+        dedup_all_info = info_repo_json['cache']['stats']['unique_size']
+
+        # create and info shall give the same numbers
+        assert dedup_this_create == dedup_this_info
+        assert dedup_all_create == dedup_all_info
+        # accounting for "all archives" includes metadata chunks, for "this archive" it does not,
+        # thus a mismatch is expected.
+        assert dedup_this_create != dedup_all_create
+        assert dedup_this_info != dedup_all_info
+
     def test_info_json(self):
         self.create_regular_file('file1', size=1024 * 80)
         self.cmd('init', '--encryption=repokey', self.repository_location)
@@ -4021,6 +4083,25 @@ def test_exit_codes(self):
             self.cmd('create', self.repository_location + '::archive', 'input', fork=True,
                      exit_code=Repository.InvalidRepository.exit_mcode)
 
+    def test_original_size_stable_across_recreate(self):
+        # Test that changes in archive metadata (like number of chunks) do not influence the original size.
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+
+        def original_size(archive_name):
+            info = json.loads(self.cmd('info', '--json', f"{self.repository_location}::{archive_name}"))
+            return info['archives'][0]['stats']['original_size']
+
+        sizes = [12345, 67890]
+        self.create_regular_file('file1', size=sizes[0])
+        self.create_regular_file('file2', size=sizes[1])
+
+        self.cmd('create', '--compression=none', self.repository_location + '::archive', 'input')
+        assert original_size('archive') == sum(sizes)
+
+        # Recreate with different chunker params to try to reproduce #8898.
+        self.cmd('recreate', '--chunker-params=10,12,11,63', self.repository_location + '::archive')
+        assert original_size('archive') == sum(sizes)
+
 
 @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available')
 class ArchiverTestCaseBinary(ArchiverTestCase):