Automatically free up backup storage space

lib/argument_parser.py

@@ -367,7 +367,10 @@ def argument_parser() -> argparse.ArgumentParser:
 indicate a unit in bytes. The number will be interpreted as a number
 of bytes. Case does not matter, so all of the following specify
 15 megabytes: 15MB, 15Mb, 15mB, 15mb, 15M, and 15m. Old backups
-will be deleted until at least that much space is free."""))
+will be deleted until at least that much space is free.
+
+Alternatively, this argument can be "auto". This will cause Vintage Backup to delete old backups
+only when creating a new backup fails due to the backup media to running out of space."""))
 
     deletion_group.add_argument("--delete-after", metavar="TIME", help=format_help(
 """After a successful backup, delete backups if they are older than the time span in the argument.

lib/backup.py

@@ -413,6 +413,7 @@ def create_new_backup(
         logger.info("There is a staging folder leftover from previous incomplete backup.")
         logger.info("Deleting %s ...", staging_backup_path)
         fs.delete_directory_tree(staging_backup_path)
+        fs.log_free_space(backup_location)
 
     backup_info.confirm_user_location_is_unchanged(user_data_location, backup_location)
     backup_info.record_user_location(user_data_location, backup_location)
@@ -713,6 +714,8 @@ def log_backup_size(free_up_parameter: str | None, backup_space_taken: int) -> N
         free_up_parameter: The value given to the --free-up command line option
         backup_space_taken: The space taken by the most recent backup
     """
+    if free_up_parameter == "auto":
+        free_up_parameter = ""
     free_up = fs.parse_storage_space(free_up_parameter or "0")
     free_up_percent = math.ceil(100*backup_space_taken/free_up) if free_up else 0
     free_up_text = f" ({free_up_percent}% of --free-up)" if free_up else ""

lib/backup_deletion.py

@@ -41,7 +41,7 @@ def delete_oldest_backups_for_space(
     Raises:
         CommandLineError: If the --free-up parameter is larger than the entire backup storage media.
     """
-    if not space_requirement:
+    if not space_requirement or space_requirement == "auto":
         return
 
     total_storage = shutil.disk_usage(backup_location).total
@@ -131,7 +131,40 @@ def delete_single_backup(backup: Path, verify_checksum_result_folder: Path | Non
     except OSError:
         pass
 
-    logger.info("Free space: %s", fs.byte_units(shutil.disk_usage(backup.parent.parent).free))
+    fs.log_free_space(backup.parent.parent)
+
+
+def delete_oldest_backup(
+        backup_location: Path,
+        min_backups_remaining: int,
+        verify_checksum_result_folder: Path | None) -> None:
+    """
+    Delete the oldest backup at the specified location.
+
+    Arguments:
+        backup_location: The base directory holding all dated backups.
+        min_backups_remaining: The minimum number of backups that should remain after deletion
+            operations.
+        verify_checksum_result_folder: If the checksum of the backup is being verified prior to
+            deletion, put the verification result files in this folder.
+
+    Raises:
+        CommandLineError: If there are no backups to delete or one remaining backup.
+    """
+    backups = util.all_backups(backup_location)
+    if not backups:
+        raise CommandLineError("No backups to delete.")
+
+    if len(backups) == 1:
+        raise CommandLineError("Last remaining backup will not be deleted.")
+
+    if len(backups) <= min_backups_remaining:
+        raise CommandLineError("Reached maximum number of backup deletions this session.")
+
+    oldest_backup = backups[0]
+    logger.info("")
+    logger.info("Deleting oldest backup: %s", oldest_backup)
+    delete_single_backup(oldest_backup, verify_checksum_result_folder)
 
 
 def delete_backups(
@@ -270,12 +303,14 @@ def check_time_span_parameters(args: argparse.Namespace) -> None:
         last_time_span_str = time_span_str
 
 
-def delete_old_backups(args: argparse.Namespace) -> None:
+def delete_old_backups(args: argparse.Namespace, *, delete_oldest: bool = False) -> None:
     """
     Delete the oldest backups by various criteria in the command line options.
 
     Arguments:
         args: Parsed command line
+        delete_oldest: Whether to delete the oldest backup first. The argument --max-deletions is
+            still respected.
 
     Note: The argument `args.max_deletions` is reduced by the number of deletions to make sure that
         option is respected if multiple rounds of backup deletions are required.
@@ -296,10 +331,17 @@ def delete_old_backups(args: argparse.Namespace) -> None:
         verify_checksum_result_folder = fs.path_or_none(args.verify_checksum_before_deletion)
         max_deletions = int(args.max_deletions or backup_count)
         min_backups_remaining = max(backup_count - max_deletions, 1)
+
+        if delete_oldest:
+            delete_oldest_backup(
+                backup_folder, min_backups_remaining, verify_checksum_result_folder)
+
         delete_too_frequent_backups(
             backup_folder, args, min_backups_remaining, verify_checksum_result_folder)
+
         delete_oldest_backups_for_space(
             backup_folder, args.free_up, verify_checksum_result_folder, min_backups_remaining)
+
         delete_backups_older_than(
             backup_folder, args.delete_after, verify_checksum_result_folder, min_backups_remaining)
 

lib/filesystem.py

@@ -221,6 +221,11 @@ def parse_storage_space(space_requirement: str) -> float:
         raise CommandLineError(f"Invalid storage space value: {space_requirement}") from None
 
 
+def log_free_space(path: Path) -> None:
+    """Log the amount of free space at a location."""
+    logger.info("Free space: %s", byte_units(shutil.disk_usage(path).free))
+
+
 def write_directory(output: TextIO, directory: Path, file_names: list[str]) -> None:
     """
     Write the full path of a directory followed by a list of files it contains.
@@ -295,3 +300,23 @@ def classify_path(path: Path) -> str:
         else "Folder" if path.is_dir()
         else "File" if path.is_file()
         else "Unknown")
+
+
+def folder_size(directory: Path) -> int:
+    """
+    Calculate the total size of all files in a directory tree.
+
+    Arguments:
+        directory: A folder whose contents will be measured for size
+
+    Returns:
+        int: The total size of all files in the directory tree.
+    """
+    inode_sizes: dict[int, int] = {}  # inode --> size of file
+    for folder, _, file_names in directory.walk():
+        for file_name in file_names:
+            path = folder/file_name
+            stat = path.stat(follow_symlinks=False)
+            inode_sizes[stat.st_ino] = stat.st_size
+
+    return sum(inode_sizes.values())

lib/main.py

@@ -53,12 +53,23 @@ def backup_cycle(args: argparse.Namespace) -> None:
         CommandLineError: If the backup storage media runs out of space and --free-up cannot delete
             enough old backups to make room
     """
+    delete_oldest = False
+
     while True:
         try:
-            delete_old_backups(args)
+            delete_old_backups(args, delete_oldest=delete_oldest)
+            if delete_oldest:
+                logger.info("")
+                logger.info("Restarting backup")
+            delete_oldest = False
             start_backup(args)
             break
         except exc.OutOfSpaceError as error:
+            if args.free_up == "auto":
+                logger.info("Not enough space to complete backup.")
+                delete_oldest = True
+                continue
+
             if not args.free_up:
                 raise
 

testing/test.py

@@ -1966,13 +1966,7 @@ def used_space(self, path: Path | str) -> int:
         if not path.is_relative_to(self.base_path):
             raise ValueError(f"{path} is not a subdirectory of {self.base_path}")
 
-        total_used = 0
-        for directory, _, file_names in self.base_path.walk():
-            for file_name in file_names:
-                file_path = directory/file_name
-                total_used += file_path.stat(follow_symlinks=False).st_size
-
-        return total_used
+        return fs.folder_size(self.base_path)
 
     def total_size(self) -> int:
         """Returns the total size of this mock drive."""
@@ -2504,6 +2498,127 @@ def test_error_raised_when_no_free_up_and_and_no_space(self) -> None:
               self.assertRaises(OutOfSpaceError)):
             main.default_action(args)
 
+    def test_free_up_auto_does_nothing_when_enough_space_for_backup(self) -> None:
+        """Test that --free-up auto does nothing when there is sufficient space for a backup."""
+        create_user_data(self.user_path)
+        data_size = fs.folder_size(self.user_path)
+        backup_count = 4
+        mock_storage = DiskUsageMock(self.backup_path, (backup_count + 1)*data_size)
+        for _ in range(backup_count):
+            with (patch("lib.backup.shutil.disk_usage", mock_storage),
+                  patch("lib.backup_deletion.shutil.disk_usage", mock_storage),
+                  patch("lib.backup.datetime", Now_Mock())):
+                main_assert_no_error_log([
+                    "-u", str(self.user_path),
+                    "-b", str(self.backup_path),
+                    "--force-copy",
+                    "--free-up", "auto"],
+                    self)
+
+        all_backups = util.all_backups(self.backup_path)
+        self.assertEqual(len(all_backups), backup_count)
+
+    def test_free_up_auto_deletes_old_backups_when_not_enough_space_for_backup(self) -> None:
+        """Test that --free-up auto deletes old backups to make room for new ones."""
+        create_user_data(self.user_path)
+        data_size = fs.folder_size(self.user_path)
+        backup_count = 5
+        backup_storage_count = 3
+        storage_space = int((backup_storage_count + 0.5)*data_size)
+        mock_storage = DiskUsageMock(self.backup_path, storage_space)
+        for _ in range(backup_count):
+            with (patch("lib.backup.shutil.disk_usage", mock_storage),
+                  patch("lib.backup_deletion.shutil.disk_usage", mock_storage),
+                  patch("lib.backup.shutil.copy2", MockCopy2()),
+                  patch("lib.backup.datetime", Now_Mock())):
+                exit_code = main_no_log([
+                    "-u", str(self.user_path),
+                    "-b", str(self.backup_path),
+                    "--force-copy",
+                    "--free-up", "auto"])
+
+            self.assertEqual(exit_code, 0)
+
+        all_backups = util.all_backups(self.backup_path)
+        self.assertEqual(len(all_backups), backup_storage_count)
+
+    def test_free_up_auto_raises_error_if_backup_cannot_be_deleted(self) -> None:
+        """Test that an exception is raised if --free-up auto fails to delete a backup."""
+        create_user_data(self.user_path)
+        data_size = fs.folder_size(self.user_path)
+        mock_storage = DiskUsageMock(self.backup_path, int(1.5*data_size))
+        default_backup(self.user_path, self.backup_path)
+        with (patch("lib.backup.shutil.disk_usage", mock_storage),
+              patch("lib.backup.shutil.copy2", MockCopy2()),
+              patch("lib.backup_deletion.shutil.disk_usage", mock_storage),
+              self.assertLogs(level=logging.ERROR) as logs):
+            exit_code = main_no_log([
+                "-u", str(self.user_path),
+                "-b", str(self.backup_path),
+                "--force-copy",
+                "--free-up", "auto"])
+
+        self.assertEqual(exit_code, 1)
+        self.assertEqual(logs.output, ["ERROR:root:Last remaining backup will not be deleted."])
+
+    def test_delete_oldest_backup_deletes_oldest_backup(self) -> None:
+        """Test that delete_oldest_backup() deletes only the oldest backup."""
+        create_old_daily_backups(self.backup_path, 10)
+        backups = util.all_backups(self.backup_path)
+        expected_backups = backups[1:]
+        deletion.delete_oldest_backup(self.backup_path, 0, None)
+        remaining_backups = util.all_backups(self.backup_path)
+        self.assertEqual(expected_backups, remaining_backups)
+
+    def test_delete_oldest_backup_raises_exception_if_no_backups(self) -> None:
+        """Test delete_oldest_backup() raises an exception if there are no backups to delete."""
+        with self.assertRaises(CommandLineError) as error:
+            deletion.delete_oldest_backup(self.backup_path, 0, None)
+        self.assertEqual(error.exception.args, ("No backups to delete.",))
+
+    def test_delete_oldest_backup_raises_exception_if_only_one_backup(self) -> None:
+        """Test delete_oldest_backup() raises an exception if there is one backup left."""
+        create_old_monthly_backups(self.backup_path, 1)
+        with self.assertRaises(CommandLineError) as error:
+            deletion.delete_oldest_backup(self.backup_path, 0, None)
+        self.assertEqual(error.exception.args, ("Last remaining backup will not be deleted.",))
+
+    def test_delete_oldest_backup_raises_exception_if_max_deletions_reached(self) -> None:
+        """Test delete_oldest_backup() raises an exception if there is one backup left."""
+        backup_count = 10
+        create_old_monthly_backups(self.backup_path, backup_count)
+        with self.assertRaises(CommandLineError) as error:
+            deletion.delete_oldest_backup(self.backup_path, backup_count, None)
+        self.assertEqual(
+            error.exception.args,
+            ("Reached maximum number of backup deletions this session.",))
+
+    def test_delete_oldest_backup_verifies_checksum(self) -> None:
+        """Test that delete_oldest_backup() verifies a checksum file if one exists."""
+        create_user_data(self.user_path)
+        with patch("lib.backup.datetime", Now_Mock()):
+            exit_code = main_no_log([
+                "-u", str(self.user_path),
+                "-b", str(self.backup_path),
+                "--checksum"])
+        self.assertEqual(exit_code, 0)
+
+        oldest_backup, = util.all_backups(self.backup_path)
+        checksum_file = oldest_backup/verify.checksum_file_name
+        self.assertTrue(checksum_file.exists())
+
+        changed_file = oldest_backup/"root_file.txt"
+        self.assertTrue(changed_file.exists())
+        changed_file.write_text("corrupted", encoding="utf8")
+
+        default_backup(self.user_path, self.backup_path)
+        deletion.delete_oldest_backup(self.backup_path, 0, self.user_path)
+
+        checksum_verify_file = self.user_path/verify.verify_checksum_file_name
+        self.assertTrue(checksum_verify_file.exists())
+        _, line, _ = checksum_verify_file.read_text(encoding="utf8").split("\n")
+        self.assertTrue(line.startswith(str(changed_file.relative_to(oldest_backup))))
+
 
 class MoveBackupsTests(TestCaseWithTemporaryFilesAndFolders):
     """Test moving backup sets to a different location."""
@@ -6621,3 +6736,37 @@ def error_chmod(*_: Any, **__: Any) -> Never:  # ruff:ignore[any-type]
         self.assertTrue(folder_path.is_dir())
         self.assertEqual(error.exception.args, (error_message,))
         os.chmod(folder_path, stat.S_IWRITE, follow_symlinks=False)  # ruff:ignore[os-chmod]
+
+    def test_size_of_empty_folder_is_zero(self) -> None:
+        """Test that an empty folder has zero size."""
+        self.assertEqual(fs.folder_size(self.user_path), 0)
+
+    def test_size_of_folder_with_one_file_is_size_of_file(self) -> None:
+        """Test that the size of a folder with one file is the size of that file."""
+        file_size = 10_000_000
+        create_large_files(self.user_path, file_size)
+        self.assertEqual(fs.folder_size(self.user_path), file_size)
+
+    def test_size_of_folder_with_subfolders_and_files_is_size_of_all_files_in_tree(self) -> None:
+        """Test that all files in a directory tree get summed to find total size."""
+        for num in range(3):
+            (self.user_path/str(num)).mkdir()
+
+        file_size = 5_000_000
+        create_large_files(self.user_path, file_size)
+        base_file = self.user_path/"base_file.txt"
+        base_file.write_text(file_size*"B")
+        total_size = 4*file_size
+        self.assertEqual(fs.folder_size(self.user_path), total_size)
+
+    def test_size_of_folder_tree_with_hard_links_does_not_double_count(self) -> None:
+        """Test that the sizes of files that are hardlinked together are not double-counted."""
+        create_user_data(self.user_path)
+        default_backup(self.user_path, self.backup_path)
+        single_backup_size = fs.folder_size(self.backup_path)
+
+        # All new files at backup location are hardlinked to first backup.
+        default_backup(self.user_path, self.backup_path)
+        all_backups_size = fs.folder_size(self.backup_path)
+
+        self.assertEqual(single_backup_size, all_backups_size)

wiki/delete.md

@@ -13,7 +13,8 @@ Backup deletions after a backup ensure that most of the time the next backup can
 
 Specify how much disk space should be kept free at the backup location.
 If there is less space before or after a backup, old backups will be deleted until this amount of space is free.
-This parameter can be just a number or a number with a byte unit.
+This parameter can be just a number, a number with a byte unit, or the word `auto`.
+
 For example,
 
 `--free-up "10 GB"`
@@ -27,6 +28,14 @@ If there is space between the number and unit like `10 GB`, then the whole param
 This size of this parameter should be an overestimate of the space needed for each backup.
 This depends on how much new data is added between backups and how often files are copied instead of hard-linked (see the [`--hard-link-count`](backup.md#--hard-link-count) and [`--copy-probability`](backup.md#--copy-probability) parameters).
 
+If the parameter is
+
+`--free-up auto`
+
+then old backups are only deleted when the backup location cannot complete a backup due to running out of space.
+This option maximizes the use of space at the backup location.
+However, this can increase the amount of time a backup takes to complete since multiple deletions may be required to create enough space to make a new a backup.
+
 If the backup storage media runs out of space during a backup and this parameter is used, then
 1. The backup process will abort,
 2. Old backups will be deleted according to `--free-up`, and