diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 0ff6642a0e..5ba81f1fe1 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -13,11 +13,12 @@ from ..helpers.nanorst import rst_to_terminal from ..manifest import Manifest, AI_HUMAN_SORT_KEYS from ..patterns import PatternMatcher -from ..legacyremote import LegacyRemoteRepository +from ..legacy.remote import LegacyRemoteRepository from ..remote import RemoteRepository -from ..legacyrepository import LegacyRepository +from ..legacy.repository import LegacyRepository from ..repository import Repository -from ..repoobj import RepoObj, RepoObj1 +from ..repoobj import RepoObj +from ..legacy.repoobj import RepoObj1 from ..patterns import ( ArgparsePatternAction, ArgparseExcludeFileAction, diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index 71088f8ded..e37ddf57b2 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -10,7 +10,7 @@ from ..helpers.argparsing import ArgumentParser, ArgumentTypeError from ..item import ChunkListEntry from ..manifest import Manifest -from ..legacyrepository import LegacyRepository +from ..legacy.repository import LegacyRepository from ..repository import Repository from ..logger import create_logger diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index e187257ed2..bd8dce0bfb 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -487,7 +487,7 @@ def safe_unlink(path): Use this when deleting potentially large files when recovering from a VFS error such as ENOSPC. It can help a full file system recover. Refer to the "File system interaction" section - in legacyrepository.py for further explanations. + in legacy/repository.py for further explanations. """ path_obj = Path(path) try: diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index bb38092d03..38762a485a 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -1335,9 +1335,9 @@ def ellipsis_truncate(msg, space): class BorgJsonEncoder(json.JSONEncoder): def default(self, o): - from ..legacyrepository import LegacyRepository + from ..legacy.repository import LegacyRepository from ..repository import Repository - from ..legacyremote import LegacyRemoteRepository + from ..legacy.remote import LegacyRemoteRepository from ..remote import RemoteRepository from ..archive import Archive from ..cache import AdHocWithFilesCache diff --git a/src/borg/legacy/__init__.py b/src/borg/legacy/__init__.py new file mode 100644 index 0000000000..ef99c3bc19 --- /dev/null +++ b/src/borg/legacy/__init__.py @@ -0,0 +1,8 @@ +""" +Borg Legacy Package — Borg 1.x compatibility layer. + +This package contains all code specific to reading/writing Borg 1.x repositories. +It is required for ``borg transfer --from-borg1`` and ``borg serve`` (serving v1 clients). + +This package can be removed entirely when Borg 1.x support is dropped. +""" diff --git a/src/borg/legacyremote.py b/src/borg/legacy/remote.py similarity index 98% rename from src/borg/legacyremote.py rename to src/borg/legacy/remote.py index 3fbc586084..d94215a648 100644 --- a/src/borg/legacyremote.py +++ b/src/borg/legacy/remote.py @@ -16,24 +16,24 @@ from xxhash import xxh64 -from . import __version__ -from .compress import Compressor -from .constants import * # NOQA -from .helpers import Error, ErrorWithTraceback, IntegrityError -from .helpers import bin_to_hex -from .helpers import get_limited_unpacker -from .helpers import replace_placeholders -from .helpers import format_file_size -from .helpers import safe_unlink -from .helpers import prepare_subprocess_env, ignore_sigint -from .helpers import get_socket_filename -from .fslocking import LockTimeout, NotLocked, NotMyLock, LockFailed -from .logger import create_logger -from .helpers import msgpack -from .legacyrepository import LegacyRepository -from .version import parse_version, format_version -from .helpers.datastruct import EfficientCollectionQueue -from .platform import is_win32 +from .. import __version__ +from ..compress import Compressor +from ..constants import * # NOQA +from ..helpers import Error, ErrorWithTraceback, IntegrityError +from ..helpers import bin_to_hex +from ..helpers import get_limited_unpacker +from ..helpers import replace_placeholders +from ..helpers import format_file_size +from ..helpers import safe_unlink +from ..helpers import prepare_subprocess_env, ignore_sigint +from ..helpers import get_socket_filename +from ..fslocking import LockTimeout, NotLocked, NotMyLock, LockFailed +from ..logger import create_logger +from ..helpers import msgpack +from .repository import LegacyRepository +from ..version import parse_version, format_version +from ..helpers.datastruct import EfficientCollectionQueue +from ..platform import is_win32 logger = create_logger(__name__) diff --git a/src/borg/legacy/repoobj.py b/src/borg/legacy/repoobj.py new file mode 100644 index 0000000000..99ac49f174 --- /dev/null +++ b/src/borg/legacy/repoobj.py @@ -0,0 +1,75 @@ +"""Legacy RepoObj1 — Borg 1.x repository object format. + +Moved from borg.repoobj as part of the legacy code separation. +""" + +from ..constants import * # NOQA +from ..helpers import workarounds +from ..compress import Compressor, get_compressor + +# Workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode +AUTHENTICATED_NO_KEY = "authenticated_no_key" in workarounds + + +class RepoObj1: # legacy + @classmethod + def extract_crypted_data(cls, data: bytes) -> bytes: + # used for crypto type detection + return data + + def __init__(self, key): + self.key = key + self.compressor = get_compressor("lz4", legacy_mode=True) + + def id_hash(self, data: bytes) -> bytes: + return self.key.id_hash(data) + + def format( + self, + id: bytes, + meta: dict, + data: bytes, + compress: bool = True, + size: int = None, + ctype: int = None, + clevel: int = None, + ro_type: str = None, + ) -> bytes: + assert isinstance(id, bytes) + assert meta == {} + assert isinstance(data, (bytes, memoryview)) + assert ro_type is not None + assert compress or size is not None and ctype is not None and clevel is not None + if compress: + assert size is None or size == len(data) + meta, data_compressed = self.compressor.compress(meta, data) + else: + assert isinstance(size, int) + data_compressed = data # is already compressed, must include type/level bytes + data_encrypted = self.key.encrypt(id, data_compressed) + return data_encrypted + + def parse_meta(self, id: bytes, cdata: bytes) -> dict: + raise NotImplementedError("parse_meta is not available for RepoObj1") + + def parse( + self, id: bytes, cdata: bytes, decompress: bool = True, want_compressed: bool = False, ro_type: str = None + ) -> tuple[dict, bytes]: + assert not (not decompress and not want_compressed), "invalid parameter combination!" + assert isinstance(id, bytes) + assert isinstance(cdata, bytes) + assert ro_type is not None + data_compressed = self.key.decrypt(id, cdata) + compressor_cls, compression_level = Compressor.detect(data_compressed[:2]) + compressor = compressor_cls(level=compression_level, legacy_mode=True) + meta_compressed = {} + meta_compressed["ctype"] = compressor.ID + meta_compressed["clevel"] = compressor.level + meta_compressed["csize"] = len(data_compressed) + if decompress: + meta, data = compressor.decompress(None, data_compressed) + if not AUTHENTICATED_NO_KEY: + self.key.assert_id(id, data) + else: + meta, data = None, None + return meta_compressed if want_compressed else meta, data_compressed if want_compressed else data diff --git a/src/borg/legacyrepository.py b/src/borg/legacy/repository.py similarity index 99% rename from src/borg/legacyrepository.py rename to src/borg/legacy/repository.py index 9ef97a1a1b..288c5a60d7 100644 --- a/src/borg/legacyrepository.py +++ b/src/borg/legacy/repository.py @@ -14,22 +14,22 @@ import xxhash -from .constants import * # NOQA -from .hashindex import NSIndex1Entry, NSIndex1 -from .helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size -from .helpers import Location -from .helpers import ProgressIndicatorPercent -from .helpers import bin_to_hex, hex_to_bin -from .helpers import secure_erase, safe_unlink -from .helpers import msgpack -from .helpers.lrucache import LRUCache -from .fslocking import Lock, LockError, LockErrorT -from .logger import create_logger -from .manifest import Manifest, NoManifestError -from .platform import SaveFile, SyncFile, sync_dir, safe_fadvise -from .repoobj import RepoObj -from .checksums import crc32 -from .crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError +from ..constants import * # NOQA +from ..hashindex import NSIndex1Entry, NSIndex1 +from ..helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size +from ..helpers import Location +from ..helpers import ProgressIndicatorPercent +from ..helpers import bin_to_hex, hex_to_bin +from ..helpers import secure_erase, safe_unlink +from ..helpers import msgpack +from ..helpers.lrucache import LRUCache +from ..fslocking import Lock, LockError, LockErrorT +from ..logger import create_logger +from ..manifest import Manifest, NoManifestError +from ..platform import SaveFile, SyncFile, sync_dir, safe_fadvise +from ..repoobj import RepoObj +from ..checksums import crc32 +from ..crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError logger = create_logger(__name__) diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 95589c28f5..3087c9d7a3 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -117,7 +117,7 @@ def ids(self, *, deleted=False): def _get_archive_meta(self, id: bytes) -> dict: # get all metadata directly from the ArchiveItem in the repo. - from .legacyrepository import LegacyRepository + from .legacy.repository import LegacyRepository from .repository import Repository try: diff --git a/src/borg/remote.py b/src/borg/remote.py index 5fdafb4223..b8ff6d3c3c 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -36,7 +36,7 @@ from .logger import create_logger, borg_serve_log_queue from .manifest import NoManifestError from .helpers import msgpack -from .legacyrepository import LegacyRepository +from .legacy.repository import LegacyRepository from .repository import Repository, StoreObjectNotFound from .version import parse_version, format_version from .helpers.datastruct import EfficientCollectionQueue diff --git a/src/borg/repoobj.py b/src/borg/repoobj.py index a5aecf3dae..4d2814b4ef 100644 --- a/src/borg/repoobj.py +++ b/src/borg/repoobj.py @@ -6,7 +6,7 @@ from .constants import * # NOQA from .helpers import msgpack, workarounds from .helpers.errors import IntegrityError -from .compress import Compressor, LZ4_COMPRESSOR, get_compressor +from .compress import Compressor, LZ4_COMPRESSOR # Workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode AUTHENTICATED_NO_KEY = "authenticated_no_key" in workarounds @@ -139,65 +139,5 @@ def parse( return meta_compressed if want_compressed else meta, data_compressed if want_compressed else data -class RepoObj1: # legacy - @classmethod - def extract_crypted_data(cls, data: bytes) -> bytes: - # used for crypto type detection - return data - - def __init__(self, key): - self.key = key - self.compressor = get_compressor("lz4", legacy_mode=True) - - def id_hash(self, data: bytes) -> bytes: - return self.key.id_hash(data) - - def format( - self, - id: bytes, - meta: dict, - data: bytes, - compress: bool = True, - size: int = None, - ctype: int = None, - clevel: int = None, - ro_type: str = None, - ) -> bytes: - assert isinstance(id, bytes) - assert meta == {} - assert isinstance(data, (bytes, memoryview)) - assert ro_type is not None - assert compress or size is not None and ctype is not None and clevel is not None - if compress: - assert size is None or size == len(data) - meta, data_compressed = self.compressor.compress(meta, data) - else: - assert isinstance(size, int) - data_compressed = data # is already compressed, must include type/level bytes - data_encrypted = self.key.encrypt(id, data_compressed) - return data_encrypted - - def parse_meta(self, id: bytes, cdata: bytes) -> dict: - raise NotImplementedError("parse_meta is not available for RepoObj1") - - def parse( - self, id: bytes, cdata: bytes, decompress: bool = True, want_compressed: bool = False, ro_type: str = None - ) -> tuple[dict, bytes]: - assert not (not decompress and not want_compressed), "invalid parameter combination!" - assert isinstance(id, bytes) - assert isinstance(cdata, bytes) - assert ro_type is not None - data_compressed = self.key.decrypt(id, cdata) - compressor_cls, compression_level = Compressor.detect(data_compressed[:2]) - compressor = compressor_cls(level=compression_level, legacy_mode=True) - meta_compressed = {} - meta_compressed["ctype"] = compressor.ID - meta_compressed["clevel"] = compressor.level - meta_compressed["csize"] = len(data_compressed) - if decompress: - meta, data = compressor.decompress(None, data_compressed) - if not AUTHENTICATED_NO_KEY: - self.key.assert_id(id, data) - else: - meta, data = None, None - return meta_compressed if want_compressed else meta, data_compressed if want_compressed else data +# Backward compatibility: RepoObj1 has moved to borg.legacy.repoobj +from .legacy.repoobj import RepoObj1 # noqa: F401 diff --git a/src/borg/testsuite/legacyrepository_test.py b/src/borg/testsuite/legacyrepository_test.py index a97a094c97..28e10adcfb 100644 --- a/src/borg/testsuite/legacyrepository_test.py +++ b/src/borg/testsuite/legacyrepository_test.py @@ -13,9 +13,9 @@ from ..helpers import msgpack from ..fslocking import Lock, LockFailed from ..platformflags import is_win32 -from ..legacyremote import LegacyRemoteRepository, InvalidRPCMethod, PathNotAllowed -from ..legacyrepository import LegacyRepository, LoggedIO -from ..legacyrepository import MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT2, TAG_PUT, TAG_COMMIT +from ..legacy.remote import LegacyRemoteRepository, InvalidRPCMethod, PathNotAllowed +from ..legacy.repository import LegacyRepository, LoggedIO +from ..legacy.repository import MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT2, TAG_PUT, TAG_COMMIT from ..repoobj import RepoObj from .hashindex_test import H @@ -664,7 +664,7 @@ def test_subtly_corrupted_hints_without_integrity(repository, caplog): repository.put(H(3), fchunk(b"1234")) # Do a compaction run. # The corrupted refcount is detected and logged as a warning, but compaction proceeds. - caplog.set_level(logging.WARNING, logger="borg.legacyrepository") + caplog.set_level(logging.WARNING, logger="borg.legacy.repository") repository.commit(compact=True) assert "Corrupted segment reference count" in caplog.text # We verify that the repository is still consistent. diff --git a/src/borg/testsuite/repoobj_test.py b/src/borg/testsuite/repoobj_test.py index e05cc989ff..748749ef21 100644 --- a/src/borg/testsuite/repoobj_test.py +++ b/src/borg/testsuite/repoobj_test.py @@ -4,7 +4,8 @@ from ..crypto.key import PlaintextKey from ..helpers.errors import IntegrityError from ..repository import Repository -from ..repoobj import RepoObj, RepoObj1 +from ..repoobj import RepoObj +from ..legacy.repoobj import RepoObj1 from ..compress import LZ4