create: rebuild the files cache from an archive of the same group

When the local files cache is missing, borg rebuilds it by reading the archive
this one continues from the repository. That archive was looked up by matching
the series name only:

    archives = self.manifest.archives.list(match=[self.archive_name], ...)

Archive series names are not unique across hosts, so in a repository shared by
multiple machines or users this could pick a foreign archive: if host2 backed up
its own "home" series after host1, host1 would rebuild its files cache from
host2's archive. Almost nothing matches there, so borg reads and chunks
everything again - the files cache silently stops working for everyone but the
host that happened to write last.

The lookup now matches the archive attributes given by the new --group-by
option, defaulting to name,host. Valid keys are name, host and user; tags are
not usable because a new archive is not known to belong to the tag group of an
existing one, and an empty value is rejected because an archive must not
continue an arbitrary unrelated archive.

The host and user an archive gets stamped with now come from
archive_hostname() / archive_username() in helpers, so the metadata written by
create and the lookup done by the cache can not drift apart.

Note that the local files cache file name is still derived from the series name
alone. It lives on the client, so it is per host already, and keeping the name
avoids invalidating everybody's files cache.

Author: ThomasWaldmann

src/borg/archive.py

@@ -10,7 +10,6 @@
 from contextlib import contextmanager
 from datetime import timedelta
 from functools import partial
-from getpass import getuser
 from io import BytesIO
 from itertools import groupby, zip_longest
 from collections.abc import Iterator
@@ -32,6 +31,7 @@
 from .helpers import BackupSymlinkParentError, BackupPathTraversalError
 from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError, BackupTimeoutError
 from .helpers import HardLinkManager
+from .helpers import archive_hostname, archive_username
 from .helpers import ChunkIteratorFileWrapper, open_item
 from .helpers import Error, IntegrityError, set_ec, sig_int
 from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
@@ -51,7 +51,6 @@
 from .manifest import Manifest
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem, ItemDiff
-from . import platform
 from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth
 from .repository import Repository, NoManifestError
 from .repoobj import RepoObj
@@ -770,8 +769,8 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m
             "item_ptrs": item_ptrs,  # see #1473
             "command_line": join_cmd(sys.argv),
             "cwd": self.cwd,
-            "hostname": os.environ.get("BORG_HOSTNAME") or platform.get_hostname(),
-            "username": os.environ.get("BORG_USERNAME") or getuser(),
+            "hostname": archive_hostname(),
+            "username": archive_username(),
             "time": nominal.isoformat(timespec="microseconds"),
             "start": start.isoformat(timespec="microseconds"),
             "end": end.isoformat(timespec="microseconds"),

src/borg/archiver/create_cmd.py

@@ -17,6 +17,8 @@
 from ..constants import *  # NOQA
 from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec
 from ..helpers import archivename_validator, DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime
+from ..helpers import FilesCacheGroupBySpec
+from ..helpers.parseformat import FILES_CACHE_GROUP_BY_KEYS
 from ..helpers import octal_int, nonnegative_seconds
 from ..helpers import read_input_map
 from ..helpers import eval_escapes
@@ -313,7 +315,12 @@ def create_inner(archive, cache, fso):
         logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed))
         if not dry_run:
             with Cache(
-                repository, manifest, progress=args.progress, cache_mode=args.files_cache_mode, archive_name=args.name
+                repository,
+                manifest,
+                progress=args.progress,
+                cache_mode=args.files_cache_mode,
+                archive_name=args.name,
+                archive_group_by=tuple(args.group_by.split(",")),
             ) as cache:
                 archive = Archive(
                     manifest,
@@ -791,6 +798,21 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         done by comparing multiple file metadata values with previous values kept in
         the files cache.
 
+        The files cache is kept locally, one per archive series. If it is missing (e.g. on a
+        fresh machine or after the local cache was removed), borg rebuilds it by reading the
+        archive this one continues from the repository. That archive is the newest one having
+        the same archive attributes as given by ``--group-by``, by default the same series
+        name and the same host. Matching the series name alone would be wrong in a repository
+        shared by multiple machines or users, because they may use the same series name for
+        their own, unrelated data - borg would then rebuild the files cache from a foreign
+        archive, where almost nothing matches, and read and chunk everything again.
+
+        Give ``--group-by name`` if the same series is written by different hosts on purpose
+        and they see the same files, or add ``user`` if one host backs up the same series as
+        different users. Beware of grouping by an attribute that is not stable over time: if
+        e.g. the hostname changes for every backup run (as it might for containers), borg will
+        never find an archive to rebuild the files cache from.
+
         This comparison can operate in different modes as given by ``--files-cache``:
 
         - ctime,size,inode (default on POSIX systems)
@@ -1221,6 +1243,18 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             help="operate files cache in MODE. default: %s (on Windows: %s, because ctime is "
             "file creation time there)." % (FILES_CACHE_MODE_UI_DEFAULT_POSIX, FILES_CACHE_MODE_UI_DEFAULT_WIN32),
         )
+        fs_group.add_argument(
+            "--group-by",
+            metavar="KEYS",
+            dest="group_by",
+            action=Highlander,
+            type=FilesCacheGroupBySpec,
+            default="name,host",
+            help="comma-separated list of archive attributes identifying the archives this archive "
+            "belongs to; the newest of them is the archive the files cache is rebuilt from, if the "
+            "local files cache is missing. valid keys are: {}; default is: "
+            "name,host".format(", ".join(FILES_CACHE_GROUP_BY_KEYS)),
+        )
         fs_group.add_argument(
             "--files-changed",
             metavar="MODE",

src/borg/cache.py

@@ -26,6 +26,7 @@
 from .constants import CHUNKINDEX_SMALL_FRAGMENT_CAP, CHUNKINDEX_MERGE_ATTEMPTS, CHUNKINDEX_INVALID_SENTINEL
 from .hashindex import ChunkIndex, ChunkIndexEntry, ChunkIndexEntryFormat
 from .helpers import get_cache_dir
+from .helpers import archive_hostname, archive_username
 from .helpers import chunkit
 from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list
 from .helpers import format_file_size, safe_encode
@@ -58,6 +59,27 @@ def files_cache_name(archive_name, files_cache_name="files"):
     return files_cache_name + "." + suffix
 
 
+def archive_group_patterns(archive_name, group_by):
+    """
+    Build the match patterns selecting the archives belonging to the same group as a new archive.
+
+    The new archive is named *archive_name* and gets stamped with this host and this user, so the
+    patterns describe the archives it continues, e.g. ["name:home", "host:myhost"] for the default
+    grouping. See "borg help match-archives" for the pattern syntax.
+    """
+    patterns = []
+    for group_by_key in group_by:
+        if group_by_key == "name":
+            patterns.append(f"name:{archive_name}")
+        elif group_by_key == "host":
+            patterns.append(f"host:{archive_hostname()}")
+        elif group_by_key == "user":
+            patterns.append(f"user:{archive_username()}")
+        else:
+            raise ValueError(f"invalid group-by key: {group_by_key}")
+    return patterns
+
+
 def discover_files_cache_names(path, files_cache_name="files"):
     """
     Return a list of all files cache file names in the given directory.
@@ -200,6 +222,7 @@ def __new__(
         progress=False,
         cache_mode=FILES_CACHE_MODE_DISABLED,
         archive_name=None,
+        archive_group_by=(),
         start_backup=None,
     ):
         return AdHocWithFilesCache(
@@ -209,6 +232,7 @@ def __new__(
             progress=progress,
             cache_mode=cache_mode,
             archive_name=archive_name,
+            archive_group_by=archive_group_by,
             start_backup=start_backup,
         )
 
@@ -225,8 +249,9 @@ class FilesCacheMixin:
 
     FILES_CACHE_NAME = "files"
 
-    def __init__(self, cache_mode, archive_name=None, start_backup=None):
+    def __init__(self, cache_mode, archive_name=None, archive_group_by=(), start_backup=None):
         self.archive_name = archive_name  # ideally a SERIES name
+        self.archive_group_by = archive_group_by  # archive attributes identifying the previous archive
         assert not ("c" in cache_mode and "m" in cache_mode)
         assert "d" in cache_mode or "c" in cache_mode or "m" in cache_mode
         self.cache_mode = cache_mode
@@ -294,9 +319,13 @@ def _build_files_cache(self):
 
         from .archive import Archive
 
-        # get the latest archive with the IDENTICAL name, supporting archive series:
+        # Get the latest archive of the same group, supporting archive series. Matching the name
+        # alone is not enough in a repository shared by multiple hosts or users, because they may
+        # use the same series name for their own, unrelated data - we would then build our files
+        # cache from a foreign archive, which just wastes time as almost nothing would match.
+        match = archive_group_patterns(self.archive_name, self.archive_group_by)
         try:
-            archives = self.manifest.archives.list(match=[self.archive_name], sort_by=["ts"], last=1)
+            archives = self.manifest.archives.list(match=match, sort_by=["ts"], last=1)
         except PermissionDenied:  # maybe repo is in write-only mode?
             archives = None
         if not archives:
@@ -1203,13 +1232,14 @@ def __init__(
         progress=False,
         cache_mode=FILES_CACHE_MODE_DISABLED,
         archive_name=None,
+        archive_group_by=(),
         start_backup=None,
     ):
         """
         :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
         :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison
         """
-        FilesCacheMixin.__init__(self, cache_mode, archive_name, start_backup)
+        FilesCacheMixin.__init__(self, cache_mode, archive_name, archive_group_by, start_backup)
         ChunksMixin.__init__(self)
         assert isinstance(manifest, Manifest)
         self.manifest = manifest

src/borg/helpers/__init__.py

@@ -28,7 +28,7 @@
 from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special
 from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime
 from .fs import HardLinkManager
-from .misc import sysinfo, log_multi, consume
+from .misc import sysinfo, log_multi, consume, archive_hostname, archive_username
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode
 from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd
@@ -39,6 +39,7 @@
     FilesystemDirSpec,
     SortBySpec,
     GroupBySpec,
+    FilesCacheGroupBySpec,
     CompressionSpec,
     ChunkerParams,
     DigestAlgos,

src/borg/helpers/misc.py

@@ -4,6 +4,7 @@
 import platform  # python stdlib import - if this fails, check that cwd != src/borg/
 import sys
 from collections import deque
+from getpass import getuser
 from itertools import islice
 
 from ..logger import create_logger
@@ -15,6 +16,18 @@
 from ..constants import ROBJ_FILE_STREAM
 
 
+def archive_hostname():
+    """Return the hostname a new archive is stamped with."""
+    from ..platform import get_hostname
+
+    return os.environ.get("BORG_HOSTNAME") or get_hostname()
+
+
+def archive_username():
+    """Return the username a new archive is stamped with."""
+    return os.environ.get("BORG_USERNAME") or getuser()
+
+
 def sysinfo():
     show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
     if show_sysinfo == "no":

src/borg/helpers/parseformat.py

@@ -582,24 +582,35 @@ def SortBySpec(text):
     return text.replace("timestamp", "ts").replace("archive", "name")
 
 
-def GroupBySpec(text):
+def GroupBySpec(text, valid_keys=None, allow_ungrouped=True):
     """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group."""
     from ..manifest import AI_GROUP_BY_KEYS
 
+    valid_keys = AI_GROUP_BY_KEYS if valid_keys is None else valid_keys
     if text in ("", "none"):
+        if not allow_ungrouped:
+            raise ArgumentTypeError("At least one group-by key is required (valid keys: %s)" % ", ".join(valid_keys))
         return ""  # idempotency: the normalized value must pass validation again
     seen = set()
     for group_key in text.split(","):
-        if group_key not in AI_GROUP_BY_KEYS:
-            raise ArgumentTypeError(
-                "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS))
-            )
+        if group_key not in valid_keys:
+            raise ArgumentTypeError("Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(valid_keys)))
         if group_key in seen:
             raise ArgumentTypeError("Duplicate group-by key: %s" % group_key)
         seen.add(group_key)
     return text
 
 
+# A new archive is not known to belong to the tag group of an existing archive, and it must not
+# continue an arbitrary unrelated archive, so this grouping is more restricted than prune's.
+FILES_CACHE_GROUP_BY_KEYS = ["name", "host", "user"]
+
+
+def FilesCacheGroupBySpec(text):
+    """Validate the group-by keys usable for finding the archive a new archive continues."""
+    return GroupBySpec(text, valid_keys=FILES_CACHE_GROUP_BY_KEYS, allow_ungrouped=False)
+
+
 SIZE_UNITS = ("si", "iec", "raw")
 
 _warned_units: set[str] = set()  # invalid BORG_UNITS values already complained about

src/borg/testsuite/archiver/create_cmd_test.py

@@ -7,6 +7,7 @@
 import socket
 import stat
 import subprocess
+from pathlib import Path
 
 import pytest
 from blake3 import blake3
@@ -2035,3 +2036,62 @@ def no_close_time_write(self, now, force=False, clear=False):
     with changedir("output"):
         cmd(archiver, "extract", "test")
     assert_dirs_equal("input", "output/input")
+
+
+def _remove_files_cache(archiver, archive_name):
+    """Remove the local files cache of an archive series, forcing a rebuild from the repository."""
+    from ...cache import files_cache_name
+    from ...helpers import get_cache_dir
+
+    repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"]
+    cache_file = Path(get_cache_dir(repo_id, create=False)) / files_cache_name(archive_name)
+    cache_file.unlink()
+
+
+def test_files_cache_rebuild_ignores_other_hosts(archivers, request, monkeypatch):
+    """The files cache must be rebuilt from an archive of the same host, not from a foreign one."""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    # host1 backs up its "home" series ...
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    cmd(archiver, "create", "home", "input")
+    host1_id = cmd(archiver, "repo-list", "--format={id}{NL}").strip()
+
+    # ... and afterwards host2 backs up its own, unrelated "home" series into the same repository,
+    # so the newest archive named "home" is not host1's any more.
+    monkeypatch.setenv("BORG_HOSTNAME", "host2")
+    cmd(archiver, "create", "home", "input")
+
+    # host1 lost its local files cache and has to rebuild it from the repository.
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    _remove_files_cache(archiver, "home")
+    output = cmd(archiver, "create", "--debug", "home", "input")
+    assert "Building files cache from" in output
+    assert host1_id in output  # host2's archive would be useless here
+
+
+def test_files_cache_rebuild_group_by_name_only(archivers, request, monkeypatch):
+    """--group-by name restores the previous behaviour of matching the series name only."""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    cmd(archiver, "create", "home", "input")
+    monkeypatch.setenv("BORG_HOSTNAME", "host2")
+    cmd(archiver, "create", "home", "input")
+    host2_id = cmd(archiver, "repo-list", "--format={id}{NL}", "--last", "1").strip()
+
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    _remove_files_cache(archiver, "home")
+    output = cmd(archiver, "create", "--debug", "--group-by", "name", "home", "input")
+    assert host2_id in output  # the newest archive of the series, whatever host made it
+
+
+def test_files_cache_rebuild_group_by_invalid(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--group-by", "", "home", "input", exit_code=2)
+    assert "At least one group-by key is required" in output

src/borg/testsuite/cache_test.py

@@ -678,3 +678,34 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch):
         finally:
             cache.close()
         repository.flush()
+
+
+def test_archive_group_patterns(monkeypatch):
+    from ..cache import archive_group_patterns
+
+    monkeypatch.setenv("BORG_HOSTNAME", "myhost")
+    monkeypatch.setenv("BORG_USERNAME", "myuser")
+    assert archive_group_patterns("home", ()) == []
+    assert archive_group_patterns("home", ("name",)) == ["name:home"]
+    assert archive_group_patterns("home", ("name", "host")) == ["name:home", "host:myhost"]
+    assert archive_group_patterns("home", ("name", "host", "user")) == ["name:home", "host:myhost", "user:myuser"]
+    with pytest.raises(ValueError, match="invalid group-by key: tags"):
+        archive_group_patterns("home", ("tags",))
+
+
+def test_files_cache_group_by_spec():
+    from argparse import ArgumentTypeError
+
+    from ..helpers import FilesCacheGroupBySpec
+
+    assert FilesCacheGroupBySpec("name,host") == "name,host"
+    assert FilesCacheGroupBySpec("name") == "name"
+    # the parsed value is fed through the spec again by the argument parser, so it must be stable:
+    assert FilesCacheGroupBySpec(FilesCacheGroupBySpec("name,host")) == FilesCacheGroupBySpec("name,host")
+    # tags are not usable here: a new archive is not known to belong to the tag group of an existing one.
+    with pytest.raises(ArgumentTypeError, match="Invalid group-by key: tags"):
+        FilesCacheGroupBySpec("name,tags")
+    # a new archive must not continue an arbitrary unrelated archive:
+    for text in ("", "none"):
+        with pytest.raises(ArgumentTypeError, match="At least one group-by key is required"):
+            FilesCacheGroupBySpec(text)