diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 6ceca538cf..0000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: '{build}' - -environment: - matrix: - - PYTHON: C:\Python38-x64 - -# Disable automatic builds -build: off - -# Build artifacts: all wheel and exe files in the dist folder -artifacts: - - path: 'dist\*.whl' - - path: 'dist\*.exe' - -install: -- ps: scripts\win-download-openssl.ps1 -- ps: | - & $env:PYTHON\python.exe -m venv borg-env - borg-env\Scripts\activate.ps1 - python -m pip install -U pip - pip install -r requirements.d/development.txt - pip install wheel pyinstaller - -build_script: -- ps: | - borg-env\Scripts\activate.ps1 - scripts\win-build.ps1 diff --git a/.coafile b/.coafile deleted file mode 100644 index 9fb3b096fb..0000000000 --- a/.coafile +++ /dev/null @@ -1,41 +0,0 @@ -[all] -# note: put developer specific settings into ~/.coarc (e.g. editor = ...) -max_line_length = 255 -use_spaces = True -ignore = src/borg/(chunker.c|compress.c|hashindex.c|item.c), - src/borg/algorithms/(checksums.c|crc32*.c), - src/borg/algorithms/blake2/**, - src/borg/algorithms/lz4/**, - src/borg/algorithms/xxh64/**, - src/borg/algorithms/zstd/**, - src/borg/crypto/low_level.c, - src/borg/platform/*.c - -[all.general] -files = src/borg/**/*.(py|pyx|c) -bears = SpaceConsistencyBear, FilenameBear, InvalidLinkBear, LineLengthBear -file_naming_convention = snake - -[all.python] -files = src/borg/**/*.py -bears = PEP8Bear, PyDocStyleBear, PyLintBear -pep_ignore = E123,E125,E126,E127,E128,E226,E301,E309,E402,F401,F405,F811,W690 -pylint_disable = C0103, C0111, C0112, C0122, C0123, C0301, C0302, C0325, C0330, C0411, C0412, C0413, C1801, - I1101, - W0102, W0104, W0106, W0108, W0120, W0201, W0212, W0221, W0231, W0401, W0404, - W0511, W0603, W0611, W0612, W0613, W0614, W0621, W0622, W0640, W0702, W0703, - W1201, W1202, W1401, - R0101, R0201, R0204, R0901, R0902, R0903, R0904, R0911, R0912, R0913, R0914, R0915, - R0916, R1701, R1704, R1705, R1706, R1710, - E0102, E0202, E0401, E0601, E0611, E0702, E1101, E1102, E1120, E1129, E1130 -pydocstyle_ignore = D100, D101, D102, D103, D104, D105, D200, D201, D202, D203, D204, D205, D209, D210, - D212, D213, D300, D301, D400, D401, D402, D403, D404 - -[all.c] -files = src/borg/**/*.c -bears = CPPCheckBear - -[all.html] -files = src/borg/**/*.html -bears = HTMLLintBear -htmllint_ignore = * diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..181769026d --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Migrate code style to Black +7957af562d5ce8266b177039783be4dc8bdd7898 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index efee2039aa..253fa27b41 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,5 +1,6 @@ # These are supported funding model platforms -# github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -# liberapay: borgbackup +github: borgbackup +liberapay: borgbackup +open_collective: borgbackup custom: ['https://www.borgbackup.org/support/fund.html'] diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index e20150c801..455c8535bc 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -5,14 +5,14 @@ Thank you for reporting an issue. - Borgbackup documentation: http://borgbackup.readthedocs.io/en/stable/index.html - FAQ: https://borgbackup.readthedocs.io/en/stable/faq.html and - - open issues in Github tracker: https://github.com/borgbackup/borg/issues + - open issues in GitHub tracker: https://github.com/borgbackup/borg/issues If you cannot find a similar problem, then create a new issue. Please fill in as much of the template as possible. --> -## Have you checked borgbackup docs, FAQ, and open Github issues? +## Have you checked borgbackup docs, FAQ, and open GitHub issues? No @@ -46,7 +46,7 @@ results and any steps you took to troubleshoot. How to run benchmark: http://borgbackup.readthedocs.io/en/stable/usage/benchmark.html *IMPORTANT* - Please mark logs and text output from terminal commands -or else Github will not display them correctly. +or else GitHub will not display them correctly. An example is provided below. Example: diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000000..f382af79ad --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,15 @@ +# https://black.readthedocs.io/en/stable/integrations/github_actions.html#usage +# see also what we use locally, requirements.d/codestyle.txt - should be the same version here. + +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable + with: + version: "~= 23.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c7a84cd15..4f91c3ed31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ on: - '**.c' - '**.h' - '**.yml' + - '**.toml' - '**.cfg' - '**.ini' - 'requirements.d/*' @@ -23,6 +24,7 @@ on: - '**.c' - '**.h' - '**.yml' + - '**.toml' - '**.cfg' - '**.ini' - 'requirements.d/*' @@ -31,58 +33,56 @@ on: jobs: lint: - runs-on: ubuntu-latest - timeout-minutes: 10 + runs-on: ubuntu-22.04 + timeout-minutes: 5 steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Lint with flake8 - run: | - pip install flake8 - flake8 src scripts conftest.py + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 - pytest: + linux: needs: lint strategy: + fail-fast: true matrix: include: - - os: ubuntu-20.04 - python-version: '3.8' - toxenv: py38-fuse2 - - os: ubuntu-20.04 + - os: ubuntu-22.04 python-version: '3.9' - toxenv: py39-fuse3 - - os: ubuntu-20.04 + toxenv: mypy + - os: ubuntu-22.04 + python-version: '3.11' + toxenv: docs + - os: ubuntu-22.04 + python-version: '3.9' + toxenv: py39-fuse2 + - os: ubuntu-22.04 python-version: '3.10' toxenv: py310-fuse3 - - os: macos-10.15 # macos-latest is macos 11.6.2 and hanging at test_fuse, #6099 - python-version: '3.8' - toxenv: py38-fuse2 + - os: ubuntu-22.04 + python-version: '3.11' + toxenv: py311-fuse3 + - os: ubuntu-22.04 + python-version: '3.12' + toxenv: py312-fuse3 env: - # Configure pkg-config to use OpenSSL from Homebrew - PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig TOXENV: ${{ matrix.toxenv }} runs-on: ${{ matrix.os }} - timeout-minutes: 40 + timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: # just fetching 1 commit is not enough for setuptools-scm, so we fetch all fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} @@ -91,24 +91,13 @@ jobs: ${{ runner.os }}- - name: Install Linux packages - if: ${{ runner.os == 'Linux' }} run: | sudo apt-get update - sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential - sudo apt-get install -y libxxhash-dev || true - sudo apt-get install -y libb2-dev || true + sudo apt-get install -y pkg-config build-essential + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module - - name: Install macOS packages - if: ${{ runner.os == 'macOS' }} - run: | - brew install pkg-config || brew upgrade pkg-config - brew install zstd || brew upgrade zstd - brew install lz4 || brew upgrade lz4 - brew install openssl@1.1 || brew upgrade openssl@1.1 - brew install --cask macfuse || brew upgrade --cask macfuse # Required for Python llfuse module - - name: Install Python requirements run: | python -m pip install --upgrade pip setuptools wheel @@ -117,16 +106,129 @@ jobs: run: | # pip install -e . python setup.py -v develop - - name: run pytest via tox + - name: run tox env + env: + XDISTN: "4" run: | # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. #sudo -E bash -c "tox -e py" tox --skip-missing-interpreters - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 env: OS: ${{ runner.os }} python: ${{ matrix.python-version }} with: token: ${{ secrets.CODECOV_TOKEN }} env_vars: OS, python + + macOS: + + needs: linux + strategy: + fail-fast: true + matrix: + include: + - os: macos-14 + python-version: '3.11' + toxenv: py311-none # note: no fuse testing, due to #6099, see also #6196. + + env: + # Configure pkg-config to use OpenSSL from Homebrew + PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH" + TOXENV: ${{ matrix.toxenv }} + + runs-on: ${{ matrix.os }} + timeout-minutes: 180 + + steps: + - uses: actions/checkout@v4 + with: + # just fetching 1 commit is not enough for setuptools-scm, so we fetch all + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install macOS packages + run: brew bundle install + + - name: Install Python requirements + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r requirements.d/development.txt + - name: Install borgbackup + env: + # we already have that in the global env, but something is broken and overwrites that. + # so, set it here, again. + PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH" + run: | + pip install -ve . + - name: run tox env + env: + # we already have that in the global env, but something is broken and overwrites that. + # so, set it here, again. + PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH" + XDISTN: "6" + run: | + # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. + #sudo -E bash -c "tox -e py" + tox --skip-missing-interpreters + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + OS: ${{ runner.os }} + python: ${{ matrix.python-version }} + with: + token: ${{ secrets.CODECOV_TOKEN }} + env_vars: OS, python + + windows: + + if: false # build is broken, thus disabled, see #8264 + runs-on: windows-latest + timeout-minutes: 120 + needs: linux + + env: + SETUPTOOLS_USE_DISTUTILS: stdlib # Needed for pip to work - https://www.msys2.org/docs/python/#known-issues + PY_COLORS: 1 + + defaults: + run: + shell: msys2 {0} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + - name: Install dependencies + run: ./scripts/msys2-install-deps development + - name: Build + run: | + # build borg.exe + SETUPTOOLS_USE_DISTUTILS=stdlib pip install -e . + pyinstaller -y scripts/borg.exe.spec + # build sdist and wheel in dist/... + SETUPTOOLS_USE_DISTUTILS=stdlib python -m build + - uses: actions/upload-artifact@v4 + with: + name: borg-windows + path: dist/borg.exe + - name: Run tests + run: | + ./dist/borg.exe -V + pytest -n4 --benchmark-skip -vv -rs -k "not remote" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 864d71f4e9..c22089b9a4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,7 +14,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: actions: read contents: read @@ -29,38 +29,41 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # just fetching 1 commit is not enough for setuptools-scm, so we fetch all fetch-depth: 0 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + - name: Set up Python + uses: actions/setup-python@v5 with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - + python-version: 3.11 - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - - name: Install requirements, build and install Borg + - name: Install requirements run: | - sudo apt install libacl1-dev - pip3 install -r requirements.d/development.txt - pip3 install -e . - + sudo apt-get update + sudo apt-get install -y pkg-config build-essential + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + - name: Build and install Borg + run: | + python3 -m venv ../borg-env + source ../borg-env/bin/activate + pip3 install -r requirements.d/development.txt + pip3 install -ve . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.gitignore b/.gitignore index a4ff44cd12..c76d68d177 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ src/borg/crypto/low_level.c src/borg/hashindex.c src/borg/item.c src/borg/chunker.c -src/borg/algorithms/checksums.c +src/borg/checksums.c src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..5b6d0d390a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: +- repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 + hooks: + - id: ruff diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..784ad2981a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,33 @@ +# .readthedocs.yaml - Read the Docs configuration file. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + jobs: + post_checkout: + - git fetch --unshallow + apt_packages: + - build-essential + - pkg-config + - libacl1-dev + - libssl-dev + - liblz4-dev + - libzstd-dev + - libxxhash-dev + +python: + install: + - requirements: requirements.d/development.lock.txt + - requirements: requirements.d/docs.txt + - method: pip + path: . + +sphinx: + configuration: docs/conf.py + +formats: + - htmlzip diff --git a/AUTHORS b/AUTHORS index aebc8a1ddb..5378b3b6c0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -44,27 +44,3 @@ Attic Patches and Suggestions - Johann Klähn - Petros Moisiadis - Thomas Waldmann - -BLAKE2 ------- - -Borg includes BLAKE2: Copyright 2012, Samuel Neves , licensed under the terms -of the CC0, the OpenSSL Licence, or the Apache Public License 2.0. - -Slicing CRC32 -------------- - -Borg includes a fast slice-by-8 implementation of CRC32, Copyright 2011-2015 Stephan Brumme, -licensed under the terms of a zlib license. See http://create.stephan-brumme.com/crc32/ - -Folding CRC32 -------------- - -Borg includes an extremely fast folding implementation of CRC32, Copyright 2013 Intel Corporation, -licensed under the terms of the zlib license. - -xxHash ------- - -XXH64, a fast non-cryptographic hash algorithm. Copyright 2012-2016 Yann Collet, -licensed under a BSD 2-clause license. diff --git a/Brewfile b/Brewfile new file mode 100644 index 0000000000..cefb1d3cdc --- /dev/null +++ b/Brewfile @@ -0,0 +1,12 @@ +brew 'pkg-config' +brew 'zstd' +brew 'lz4' +brew 'xxhash' +brew 'openssl@3.0' + +# osxfuse (aka macFUSE) is only required for "borg mount", +# but won't work on github actions' workers. +# it requires installing a kernel extension, so some users +# may want it and some won't. + +#cask 'osxfuse' diff --git a/LICENSE b/LICENSE index e425c4775a..6551b9d530 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2015-2022 The Borg Collective (see AUTHORS file) +Copyright (C) 2015-2023 The Borg Collective (see AUTHORS file) Copyright (C) 2010-2014 Jonas Borgström All rights reserved. diff --git a/MANIFEST.in b/MANIFEST.in index dcbe185492..caf027ad90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ # stuff we need to include into the sdist is handled automatically by # setuptools_scm - it includes all git-committed files. # but we want to exclude some committed files/dirs not needed in the sdist: -exclude .coafile .editorconfig .gitattributes .gitignore .mailmap Vagrantfile +exclude .editorconfig .gitattributes .gitignore .mailmap Vagrantfile prune .github include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c include src/borg/platform/syncfilerange.c diff --git a/README.rst b/README.rst index 44551953b4..00157d847c 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,23 @@ -|screencast_basic| +This is borg2! +-------------- + +Please note that this is the README for borg2 / master branch. + +For the stable version's docs, please see there: + +https://borgbackup.readthedocs.io/en/stable/ + +Borg2 is currently in beta testing and might get major and/or +breaking changes between beta releases (and there is no beta to +next-beta upgrade code, so you will have to delete and re-create repos). + +Thus, **DO NOT USE BORG2 FOR YOUR PRODUCTION BACKUPS!** Please help with +testing it, but set it up *additionally* to your production backups. + +TODO: the screencasts need a remake using borg2, see there: + +https://github.com/borgbackup/borg/issues/6303 -More screencasts: `installation`_, `advanced usage`_ What is BorgBackup? ------------------- @@ -8,11 +25,11 @@ What is BorgBackup? BorgBackup (short: Borg) is a deduplicating backup program. Optionally, it supports compression and authenticated encryption. -The main goal of Borg is to provide an efficient and secure way to backup data. +The main goal of Borg is to provide an efficient and secure way to back up data. The data deduplication technique used makes Borg suitable for daily backups since only changes are stored. -The authenticated encryption technique makes it suitable for backups to not -fully trusted targets. +The authenticated encryption technique makes it suitable for backups to targets not +fully trusted. See the `installation manual`_ or, if you have already downloaded Borg, ``docs/installation.rst`` to get started with Borg. @@ -56,8 +73,9 @@ Main features * quick detection of unmodified files **Data encryption** - All data can be protected using 256-bit AES encryption, data integrity and - authenticity is verified using HMAC-SHA256. Data is encrypted clientside. + All data can be protected client-side using 256-bit authenticated encryption + (AES-OCB or chacha20-poly1305), ensuring data confidentiality, integrity and + authenticity. **Obfuscation** Optionally, borg can actively obfuscate e.g. the size of files / chunks to @@ -86,7 +104,7 @@ Main features you can just run them on these platforms: * Linux - * Mac OS X + * macOS * FreeBSD * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet) * Cygwin (experimental, no binaries yet) @@ -100,35 +118,31 @@ Main features Easy to use ~~~~~~~~~~~ -Initialize a new backup repository (see ``borg init --help`` for encryption options):: +For ease of use, set the BORG_REPO environment variable:: - $ borg init -e repokey /path/to/repo + $ export BORG_REPO=/path/to/repo -Create a backup archive:: +Create a new backup repository (see ``borg rcreate --help`` for encryption options):: - $ borg create /path/to/repo::Saturday1 ~/Documents + $ borg rcreate -e repokey-aes-ocb -Now doing another backup, just to show off the great deduplication:: +Create a new backup archive:: - $ borg create -v --stats /path/to/repo::Saturday2 ~/Documents - ----------------------------------------------------------------------------- - Archive name: Saturday2 - Archive fingerprint: 622b7c53c... - Time (start): Sat, 2016-02-27 14:48:13 - Time (end): Sat, 2016-02-27 14:48:14 - Duration: 0.88 seconds - Number of files: 163 - ----------------------------------------------------------------------------- - Original size Compressed size Deduplicated size - This archive: 6.85 MB 6.85 MB 30.79 kB <-- ! - All archives: 13.69 MB 13.71 MB 6.88 MB + $ borg create Monday1 ~/Documents - Unique chunks Total chunks - Chunk index: 167 330 - ----------------------------------------------------------------------------- +Now doing another backup, just to show off the great deduplication:: + $ borg create -v --stats Monday2 ~/Documents + Repository: /path/to/repo + Archive name: Monday2 + Archive fingerprint: 7714aef97c1a24539cc3dc73f79b060f14af04e2541da33d54c7ee8e81a00089 + Time (start): Mon, 2022-10-03 19:57:35 +0200 + Time (end): Mon, 2022-10-03 19:57:35 +0200 + Duration: 0.01 seconds + Number of files: 24 + Original size: 29.73 MB + Deduplicated size: 520 B -For a graphical frontend refer to our complementary project `BorgWeb `_. Helping, Donations and Bounties, becoming a Patron -------------------------------------------------- @@ -171,11 +185,7 @@ see ``docs/support.rst`` in the source distribution). .. start-badges -|doc| |build| |coverage| |bestpractices| |bounties| - -.. |bounties| image:: https://api.bountysource.com/badge/team?team_id=78284&style=bounties_posted - :alt: Bounty Source - :target: https://www.bountysource.com/teams/borgbackup +|doc| |build| |coverage| |bestpractices| .. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=stable :alt: Documentation diff --git a/README_WINDOWS.rst b/README_WINDOWS.rst deleted file mode 100644 index 520bf47c97..0000000000 --- a/README_WINDOWS.rst +++ /dev/null @@ -1,48 +0,0 @@ -Borg Native on Windows -====================== - -Running borg natively on windows is in a early alpha stage. Expect many things to fail. -Do not use the native windows build on any data which you do not want to lose! - -Build Requirements ------------------- - -- VC 14.0 Compiler -- OpenSSL Library v1.1.1c, 64bit (available at https://github.com/python/cpython-bin-deps) - Please use the `win-download-openssl.ps1` script to download and extract the library to - the correct location. See also the OpenSSL section below. -- Patience and a lot of coffee / beer - -What's working --------------- - -.. note:: - The following examples assume that the `BORG_REPO` and `BORG_PASSPHRASE` environment variables are set - if the repo or passphrase is not explicitly given. - -- Borg does not crash if called with ``borg`` -- ``borg init --encryption repokey-blake2 ./demoRepo`` runs without an error/warning. - Note that absolute paths only work if the protocol is explicitly set to file:// -- ``borg create ::backup-{now} D:\DemoData`` works as expected. -- ``borg list`` works as expected. -- ``borg extract --strip-components 1 ::backup-XXXX`` works. - If absolute paths are extracted, it's important to pass ``--strip-components 1`` as - otherwise the data is restored to the original location! - -What's NOT working ------------------- - -- Extracting a backup which was created on windows machine on a non windows machine will fail. -- And many things more. - - -OpenSSL, Windows and Python ---------------------------- -Windows does not ship OpenSSL by default, so we need to get the library from somewhere else. -However, a default python installation does include `libcrypto` which is required by borg. -The only things which are missing to build borg are the header and `*.lib` files. -Luckily the python developers provide all required files in a separate repository. -The `win-download-openssl.ps1` script can be used to download the package from -https://github.com/python/cpython-bin-deps and extract the files to the correct location. -For Anaconda, the required libraries can be installed with `conda install -c anaconda openssl`. - diff --git a/SECURITY.md b/SECURITY.md index 71776f6fae..4b8b9eba9a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,8 +5,10 @@ These borg releases are currently supported with security updates. | Version | Supported | -| ------- | ------------------ | -| 1.1.x | :white_check_mark: | +|---------|--------------------| +| 2.0.x | :x: (not released) | +| 1.2.x | :white_check_mark: | +| 1.1.x | :x: | | < 1.1 | :x: | ## Reporting a Vulnerability diff --git a/Vagrantfile b/Vagrantfile index 88c61cacbe..e1805c7573 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,8 +3,8 @@ # Automated creation of testing environments / binaries on misc. platforms -$cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus -$xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers +$cpus = Integer(ENV.fetch('VMCPUS', '8')) # create VMs with that many cpus +$xdistn = Integer(ENV.fetch('XDISTN', '8')) # dispatch tests to that many pytest workers $wmem = $xdistn * 256 # give the VM additional memory for workers [MB] def packages_debianoid(user) @@ -15,7 +15,8 @@ def packages_debianoid(user) apt-get -y -qq update apt-get -y -qq dist-upgrade # for building borgbackup and dependencies: - apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config + apt install -y pkg-config + apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true apt install -y libfuse-dev fuse || true apt install -y libfuse3-dev fuse3 || true apt install -y locales || true @@ -37,14 +38,16 @@ def packages_freebsd # install all the (security and other) updates, base system freebsd-update --not-running-from-cron fetch install # for building borgbackup and dependencies: - pkg install -y liblz4 zstd pkgconf + pkg install -y xxhash liblz4 zstd pkgconf pkg install -y fusefs-libs || true pkg install -y fusefs-libs3 || true pkg install -y git bash # fakeroot causes lots of troubles on freebsd # for building python (for the tests we use pyenv built pythons): - pkg install -y python38 py38-sqlite3 py38-virtualenv py38-pip + pkg install -y python310 py310-sqlite3 # make sure there is a python3 command - ln -sf /usr/local/bin/python3.8 /usr/local/bin/python3 + ln -sf /usr/local/bin/python3.10 /usr/local/bin/python3 + python3 -m ensurepip + pip3 install virtualenv # make bash default / work: chsh -s bash vagrant mount -t fdescfs fdesc /dev/fd @@ -52,7 +55,7 @@ def packages_freebsd # make FUSE work echo 'fuse_load="YES"' >> /boot/loader.conf echo 'vfs.usermount=1' >> /etc/sysctl.conf - kldload fuse + kldload fusefs sysctl vfs.usermount=1 pw groupmod operator -M vagrant # /dev/fuse has group operator @@ -61,6 +64,8 @@ def packages_freebsd pkg update yes | pkg upgrade echo 'export BORG_OPENSSL_PREFIX=/usr' >> ~vagrant/.bash_profile + # (re)mount / with acls + mount -o acls / EOF end @@ -68,9 +73,11 @@ def packages_openbsd return <<-EOF pkg_add bash chsh -s bash vagrant + pkg_add xxhash pkg_add lz4 pkg_add zstd pkg_add git # no fakeroot + pkg_add openssl%3.0 pkg_add py3-pip pkg_add py3-virtualenv EOF @@ -78,32 +85,28 @@ end def packages_netbsd return <<-EOF - # use the latest stuff, some packages in "9.2" are quite broken - echo 'http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.0_current/All' > /usr/pkg/etc/pkgin/repositories.conf + echo 'http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf pkgin update pkgin -y upgrade pkg_add zstd lz4 xxhash git pkg_add bash chsh -s bash vagrant - echo "export PROMPT_COMMAND=" >> ~vagrant/.bash_profile # bug in netbsd 9.2, .bash_profile broken for screen - echo "export PROMPT_COMMAND=" >> ~root/.bash_profile # bug in netbsd 9.2, .bash_profile broken for screen + echo "export PROMPT_COMMAND=" >> ~vagrant/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen + echo "export PROMPT_COMMAND=" >> ~root/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen pkg_add pkg-config # pkg_add fuse # llfuse supports netbsd, but is still buggy. # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space - pkg_add python38 py38-sqlite3 py38-pip py38-virtualenv py38-expat - ln -s /usr/pkg/lib/python3.8/_sysconfigdata_netbsd9.py /usr/pkg/lib/python3.8/_sysconfigdata__netbsd9_.py # bug in netbsd 9.2, expected filename not there. - pkg_add python39 py39-sqlite3 py39-pip py39-virtualenv py39-expat - ln -s /usr/pkg/bin/python3.9 /usr/pkg/bin/python - ln -s /usr/pkg/bin/python3.9 /usr/pkg/bin/python3 - ln -s /usr/pkg/bin/pip3.9 /usr/pkg/bin/pip - ln -s /usr/pkg/bin/pip3.9 /usr/pkg/bin/pip3 - ln -s /usr/pkg/bin/virtualenv-3.9 /usr/pkg/bin/virtualenv - ln -s /usr/pkg/bin/virtualenv-3.9 /usr/pkg/bin/virtualenv3 - ln -s /usr/pkg/lib/python3.9/_sysconfigdata_netbsd9.py /usr/pkg/lib/python3.9/_sysconfigdata__netbsd9_.py # bug in netbsd 9.2, expected filename not there. + pkg_add py311-sqlite3 py311-pip py311-virtualenv py311-expat + ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python + ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 + ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip + ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 + ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv + ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 EOF end -def packages_darwin +def packages_macos return <<-EOF # install all the (security and other) updates sudo softwareupdate --ignore iTunesX @@ -113,10 +116,13 @@ def packages_darwin sudo softwareupdate --install --all which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" brew update > /dev/null - brew install pkg-config readline openssl@1.1 zstd lz4 xz fakeroot + brew install pkg-config readline xxhash openssl@3.0 zstd lz4 xz brew install --cask macfuse # brew upgrade # upgrade everything (takes rather long) - echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@1.1/lib/pkgconfig' >> ~vagrant/.bash_profile + echo 'export LDFLAGS=-L/usr/local/opt/openssl@3.0/lib' >> ~vagrant/.bash_profile + echo 'export CPPFLAGS=-I/usr/local/opt/openssl@3.0/include' >> ~vagrant/.bash_profile + echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@3.0/lib/pkgconfig' >> ~vagrant/.bash_profile + echo 'export PYTHON_BUILD_HOMEBREW_OPENSSL_FORMULA=openssl@3.0' >> ~vagrant/.bash_profile EOF end @@ -124,17 +130,19 @@ def packages_openindiana return <<-EOF # needs separate provisioning step + reboot: #pkg update - #pkg install gcc-7 python-39 setuptools-39 + pkg install gcc-13 git pkg-config libxxhash ln -sf /usr/bin/python3.9 /usr/bin/python3 python3 -m ensurepip ln -sf /usr/bin/pip3.9 /usr/bin/pip3 pip3 install virtualenv + # let borg's pkg-config find openssl: + pfexec pkg set-mediator -V 3.1 openssl EOF end def install_pyenv(boxname) return <<-EOF - echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile + echo 'export PYTHON_CONFIGURE_OPTS="${PYTHON_CONFIGURE_OPTS} --enable-shared"' >> ~/.bash_profile echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile . ~/.bash_profile @@ -147,7 +155,7 @@ def install_pyenv(boxname) EOF end -def fix_pyenv_darwin(boxname) +def fix_pyenv_macos(boxname) return <<-EOF echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile EOF @@ -156,9 +164,11 @@ end def install_pythons(boxname) return <<-EOF . ~/.bash_profile - pyenv install 3.10.0 # tests, version supporting openssl 1.1 - pyenv install 3.9.10 # tests, version supporting openssl 1.1, binary build - pyenv install 3.8.0 # tests, version supporting openssl 1.1 + echo "PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}" + pyenv install 3.12.0 # tests + pyenv install 3.11.9 # tests, binary build + pyenv install 3.10.2 # tests + pyenv install 3.9.4 # tests pyenv rehash EOF end @@ -175,9 +185,9 @@ def build_pyenv_venv(boxname) return <<-EOF . ~/.bash_profile cd /vagrant/borg - # use the latest 3.9 release - pyenv global 3.9.10 - pyenv virtualenv 3.9.10 borg-env + # use the latest 3.11 release + pyenv global 3.11.9 + pyenv virtualenv 3.11.9 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end @@ -190,8 +200,7 @@ def install_borg(fuse) pip install -U wheel # upgrade wheel, might be too old cd borg pip install -r requirements.d/development.lock.txt - python setup.py clean - python setup.py clean2 + python3 scripts/make.py clean pip install -e .[#{fuse}] EOF end @@ -201,10 +210,7 @@ def install_pyinstaller() . ~/.bash_profile cd /vagrant/borg . borg-env/bin/activate - git clone https://github.com/thomaswaldmann/pyinstaller.git - cd pyinstaller - git checkout v4.7-maint - python setup.py install + pip install 'pyinstaller==6.7.0' EOF end @@ -227,8 +233,8 @@ def run_tests(boxname, skip_env) . ../borg-env/bin/activate if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.8.0 3.9.10 3.10.0 - pyenv local 3.8.0 3.9.10 3.10.0 + pyenv global 3.9.4 3.10.2 3.11.9 3.12.0 + pyenv local 3.9.4 3.10.2 3.11.9 3.12.0 fi # otherwise: just use the system python # some OSes can only run specific test envs, e.g. because they miss FUSE support: @@ -269,108 +275,121 @@ Vagrant.configure(2) do |config| v.cpus = $cpus end - config.vm.define "focal64" do |b| - b.vm.box = "ubuntu/focal64" + config.vm.define "noble" do |b| + b.vm.box = "ubuntu/noble64" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("focal64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("noble") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("focal64", ".*none.*") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("noble", ".*none.*") end - config.vm.define "bullseye64" do |b| - b.vm.box = "debian/bullseye64" + config.vm.define "jammy" do |b| + b.vm.box = "ubuntu/jammy64" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("bullseye64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("bullseye64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("bullseye64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jammy") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("bullseye64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bullseye64", ".*none.*") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jammy", ".*none.*") end - config.vm.define "buster64" do |b| - b.vm.box = "debian/buster64" + config.vm.define "bookworm" do |b| + b.vm.box = "debian/bookworm64" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("buster64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("buster64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("buster64") + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("bookworm") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("bookworm") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("bookworm") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("buster64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64", ".*none.*") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("bookworm") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bookworm", ".*none.*") end - config.vm.define "stretch64" do |b| - b.vm.box = "debian/stretch64" + config.vm.define "bullseye" do |b| + b.vm.box = "debian/bullseye64" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64") + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("bullseye") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("bullseye") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("bullseye") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("bullseye") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bullseye", ".*none.*") + end + + config.vm.define "freebsd13" do |b| + b.vm.box = "generic/freebsd13" + b.vm.provider :virtualbox do |v| + v.memory = 1024 + $wmem + end + b.ssh.shell = "sh" + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") + b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd13") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd13") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd13") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64", ".*(fuse3|none).*") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd13") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd13", ".*(fuse3|none).*") end - config.vm.define "freebsd64" do |b| - b.vm.box = "freebsd121-64" + config.vm.define "freebsd14" do |b| + b.vm.box = "generic/freebsd14" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.ssh.shell = "sh" b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd64") + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd14") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd14") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd14") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd64", ".*(fuse3|none).*") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd14") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd14", ".*(fuse3|none).*") end - config.vm.define "openbsd64" do |b| - b.vm.box = "generic/openbsd6" + config.vm.define "openbsd7" do |b| + b.vm.box = "generic/openbsd7" b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd7") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64", ".*fuse.*") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd7", ".*fuse.*") end - config.vm.define "netbsd64" do |b| + config.vm.define "netbsd9" do |b| b.vm.box = "generic/netbsd9" b.vm.provider :virtualbox do |v| v.memory = 4096 + $wmem # need big /tmp tmpfs in RAM! end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd9") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64", ".*fuse.*") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd9", ".*fuse.*") end - config.vm.define "darwin64" do |b| + config.vm.define "macos1012" do |b| b.vm.box = "macos-sierra" b.vm.provider :virtualbox do |v| v.memory = 4096 + $wmem @@ -385,31 +404,28 @@ Vagrant.configure(2) do |config| v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") - b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") - b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64") + b.vm.provision "packages macos", :type => :shell, :privileged => false, :inline => packages_macos + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("macos1012") + b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_macos("macos1012") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("macos1012") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("macos1012") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64", ".*(fuse3|none).*") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("macos1012") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("macos1012", ".*(fuse3|none).*") end # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again. - config.vm.define "openindiana64" do |b| - b.vm.box = "openindiana" + config.vm.define "openindiana" do |b| + b.vm.box = "openindiana/hipster" b.vm.provider :virtualbox do |v| v.memory = 2048 + $wmem end b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64", ".*fuse.*") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana", ".*fuse.*") end - - # TODO: create more VMs with python 3.8+ and openssl 1.1. - # See branch 1.1-maint for a better equipped Vagrantfile (but still on py35 and openssl 1.0). end diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 3a7c278af2..0000000000 --- a/conftest.py +++ /dev/null @@ -1,81 +0,0 @@ -import os - -import pytest - -# IMPORTANT keep this above all other borg imports to avoid inconsistent values -# for `from borg.constants import PBKDF2_ITERATIONS` (or star import) usages before -# this is executed -from borg import constants -# no fixture-based monkey-patching since star-imports are used for the constants module -constants.PBKDF2_ITERATIONS = 1 - - -# needed to get pretty assertion failures in unit tests: -if hasattr(pytest, 'register_assert_rewrite'): - pytest.register_assert_rewrite('borg.testsuite') - - -import borg.cache # noqa: E402 -from borg.logger import setup_logging # noqa: E402 - -# Ensure that the loggers exist for all tests -setup_logging() - -from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3 # noqa: E402 -from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported # noqa: E402 -from borg.testsuite.platform import fakeroot_detected # noqa: E402 - - -@pytest.fixture(autouse=True) -def clean_env(tmpdir_factory, monkeypatch): - # avoid that we access / modify the user's normal .config / .cache directory: - monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home'))) - monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home'))) - # also avoid to use anything from the outside environment: - keys = [key for key in os.environ - if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )] - for key in keys: - monkeypatch.delenv(key, raising=False) - - -def pytest_report_header(config, startdir): - tests = { - "BSD flags": has_lchflags, - "fuse2": has_llfuse, - "fuse3": has_pyfuse3, - "root": not fakeroot_detected(), - "symlinks": are_symlinks_supported(), - "hardlinks": are_hardlinks_supported(), - "atime/mtime": is_utime_fully_supported(), - "modes": "BORG_TESTS_IGNORE_MODES" not in os.environ - } - enabled = [] - disabled = [] - for test in tests: - if tests[test]: - enabled.append(test) - else: - disabled.append(test) - output = "Tests enabled: " + ", ".join(enabled) + "\n" - output += "Tests disabled: " + ", ".join(disabled) - return output - - -class DefaultPatches: - def __init__(self, request): - self.org_cache_wipe_cache = borg.cache.LocalCache.wipe_cache - - def wipe_should_not_be_called(*a, **kw): - raise AssertionError("Cache wipe was triggered, if this is part of the test add " - "@pytest.mark.allow_cache_wipe") - if 'allow_cache_wipe' not in request.keywords: - borg.cache.LocalCache.wipe_cache = wipe_should_not_be_called - request.addfinalizer(self.undo) - - def undo(self): - borg.cache.LocalCache.wipe_cache = self.org_cache_wipe_cache - - -@pytest.fixture(autouse=True) -def default_patches(request): - return DefaultPatches(request) diff --git a/docs/3rd_party/blake2/COPYING b/docs/3rd_party/blake2/COPYING deleted file mode 100644 index 6ca207ef00..0000000000 --- a/docs/3rd_party/blake2/COPYING +++ /dev/null @@ -1,122 +0,0 @@ -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - diff --git a/docs/3rd_party/blake2/README.md b/docs/3rd_party/blake2/README.md deleted file mode 100644 index 1baf979795..0000000000 --- a/docs/3rd_party/blake2/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# BLAKE2 - -This is the reference source code package of BLAKE2. - -All code is triple-licensed under the [CC0](http://creativecommons.org/publicdomain/zero/1.0), -the [OpenSSL Licence](https://www.openssl.org/source/license.html), -or the [Apache Public License 2.0](https://www.apache.org/licenses/LICENSE-2.0), -at your choosing. - -More: [https://blake2.net](https://blake2.net). [GitHub repository](https://github.com/BLAKE2/BLAKE2). - -Contact: contact@blake2.net - diff --git a/docs/3rd_party/lz4/LICENSE b/docs/3rd_party/lz4/LICENSE deleted file mode 100644 index 74c2cdd7d5..0000000000 --- a/docs/3rd_party/lz4/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/3rd_party/zstd/LICENSE b/docs/3rd_party/zstd/LICENSE deleted file mode 100644 index a793a80289..0000000000 --- a/docs/3rd_party/zstd/LICENSE +++ /dev/null @@ -1,30 +0,0 @@ -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/binaries/00_README.txt b/docs/binaries/00_README.txt new file mode 100644 index 0000000000..aad4681a27 --- /dev/null +++ b/docs/binaries/00_README.txt @@ -0,0 +1,78 @@ +Binary BorgBackup builds +======================== + +The binaries are supposed to work on the specified platform without installing +any dependencies. + + +Download the correct files +-------------------------- + +amd64 / x86_64 architecture +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +borg-linux-glibc236 Linux (built on Debian 12 "Bookworm" with glibc 2.36) +borg-linux-glibc231 Linux (built on Debian 11 "Bullseye" with glibc 2.31) + Note: you can also try them on other Linuxes with other glibc + versions - as long as the glibc is compatible, they will work. + If it doesn't work, try a borg 1.4.x or 1.2.x binary. + +borg-macos1012 macOS (built on macOS Sierra 10.12 with latest macFUSE from brew) + To avoid signing issues download the file via command line OR + remove the "quarantine" attribute after downloading: + $ xattr -dr com.apple.quarantine borg-macos.tgz + +borg-freebsd14 FreeBSD (built on FreeBSD 14) + +*.tgz similar to above, but built as a directory with files, + not as a single self-extracting binary. +*.asc GnuPG signatures for * + + +Verifying your download +----------------------- + +Please check the GPG signature to make sure you received the binary as I have +built it. + +To check the GPG signature, download both the binary and the corresponding +*.asc file and then (on the shell) type, e.g.: + + gpg --recv-keys 9F88FB52FAF7B393 + gpg --verify borg-freebsd14.asc borg-freebsd14 + +The files are signed by: + +Thomas Waldmann +GPG key fingerprint: 6D5B EF9A DD20 7580 5747 B70F 9F88 FB52 FAF7 B393 + +My fingerprint is also in the footer of all my borgbackup mailing list posts. + + +Installing +---------- + +It is suggested that you rename or symlink the binary to just "borg". + +On UNIX-like platforms, /usr/local/bin/ or ~/bin/ is a nice place for it, +but you can invoke it from every place by giving a full path to it. + +Make sure the file is readable and executable (chmod +rx borg on UNIX-like +platforms). + + +Reporting issues +---------------- +If you find issues, please open a ticket on our issue tracker: + +https://github.com/borgbackup/borg/issues/ + +There, please give: +- the version number (it is displayed if you invoke borg -V) +- the sha256sum of the binary +- a good description of what the issue is +- a good description of how to reproduce your issue +- a traceback with system info (if you have one) +- your precise platform (CPU, 32/64bit?), OS, distribution, release +- your python and (g)libc version + diff --git a/docs/borg_theme/css/borg.css b/docs/borg_theme/css/borg.css index f8f900b66b..66712b4188 100644 --- a/docs/borg_theme/css/borg.css +++ b/docs/borg_theme/css/borg.css @@ -52,8 +52,7 @@ h1 { } .container.experimental, -#debugging-facilities, -#borg-recreate { +#debugging-facilities { /* don't change text dimensions */ margin: 0 -30px; /* padding below + border width */ padding: 0 10px; /* 10 px visual margin between edge of text and the border */ diff --git a/docs/changes.rst b/docs/changes.rst index 5d4e55e426..90be2b0bb2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,4971 +1,746 @@ .. _important_notes: -Important notes -=============== +Important notes 2.x +=================== This section provides information about security and corruption issues. -.. _hashindex_set_bug: +(nothing to see here yet) -Pre-1.1.11 potential index corruption / data loss issue -------------------------------------------------------- +.. _upgradenotes2: -A bug was discovered in our hashtable code, see issue #4829. -The code is used for the client-side chunks cache and the server-side repo index. +Upgrade Notes +============= -Although borg uses the hashtables very heavily, the index corruption did not -happen too frequently, because it needed specific conditions to happen. +borg 1.2.x to borg 2.0 +---------------------- -Data loss required even more specific conditions, so it should be rare (and -also detectable via borg check). - -You might be affected if borg crashed with / complained about: - -- AssertionError: Corrupted segment reference count - corrupted index or hints -- ObjectNotFound: Object with key ... not found in repository ... -- Index mismatch for key b'...'. (..., ...) != (-1, -1) -- ValueError: stats_against: key contained in self but not in master_index. - -Advised procedure to fix any related issue in your indexes/caches: - -- install fixed borg code (on client AND server) -- for all of your clients and repos remove the cache by: - - borg delete --cache-only YOURREPO - - (later, the cache will be re-built automatically) -- for all your repos, rebuild the repo index by: - - borg check --repair YOURREPO - - This will also check all archives and detect if there is any data-loss issue. - -Affected branches / releases: - -- fd06497 introduced the bug into 1.1-maint branch - it affects all borg 1.1.x since 1.1.0b4. -- fd06497 introduced the bug into master branch - it affects all borg 1.2.0 alpha releases. -- c5cd882 introduced the bug into 1.0-maint branch - it affects all borg 1.0.x since 1.0.11rc1. - -The bug was fixed by: - -- 701159a fixes the bug in 1.1-maint branch - will be released with borg 1.1.11. -- fa63150 fixes the bug in master branch - will be released with borg 1.2.0a8. -- 7bb90b6 fixes the bug in 1.0-maint branch. Branch is EOL, no new release is planned as of now. - -.. _broken_validator: - -Pre-1.1.4 potential data corruption issue ------------------------------------------ - -A data corruption bug was discovered in borg check --repair, see issue #3444. - -This is a 1.1.x regression, releases < 1.1 (e.g. 1.0.x) are not affected. - -To avoid data loss, you must not run borg check --repair using an unfixed version -of borg 1.1.x. The first official release that has the fix is 1.1.4. - -Package maintainers may have applied the fix to updated packages of 1.1.x (x<4) -though, see the package maintainer's package changelog to make sure. - -If you never had missing item metadata chunks, the bug has not affected you -even if you did run borg check --repair with an unfixed version. - -When borg check --repair tried to repair corrupt archives that miss item metadata -chunks, the resync to valid metadata in still present item metadata chunks -malfunctioned. This was due to a broken validator that considered all (even valid) -item metadata as invalid. As they were considered invalid, borg discarded them. -Practically, that means the affected files, directories or other fs objects were -discarded from the archive. - -Due to the malfunction, the process was extremely slow, but if you let it -complete, borg would have created a "repaired" archive that has lost a lot of items. -If you interrupted borg check --repair because it was so strangely slow (killing -borg somehow, e.g. Ctrl-C) the transaction was rolled back and no corruption occurred. - -The log message indicating the precondition for the bug triggering looks like: - - item metadata chunk missing [chunk: 001056_bdee87d...a3e50d] - -If you never had that in your borg check --repair runs, you're not affected. - -But if you're unsure or you actually have seen that, better check your archives. -By just using "borg list repo::archive" you can see if all expected filesystem -items are listed. - -.. _tam_vuln: - -Pre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099) ----------------------------------------------------------- - -A flaw in the cryptographic authentication scheme in Borg allowed an attacker -to spoof the manifest. The attack requires an attacker to be able to - -1. insert files (with no additional headers) into backups -2. gain write access to the repository - -This vulnerability does not disclose plaintext to the attacker, nor does it -affect the authenticity of existing archives. - -The vulnerability allows an attacker to create a spoofed manifest (the list of archives). -Creating plausible fake archives may be feasible for small archives, but is unlikely -for large archives. - -The fix adds a separate authentication tag to the manifest. For compatibility -with prior versions this authentication tag is *not* required by default -for existing repositories. Repositories created with 1.0.9 and later require it. - -Steps you should take: - -1. Upgrade all clients to 1.0.9 or later. -2. Run ``borg upgrade --tam `` *on every client* for *each* repository. -3. This will list all archives, including archive IDs, for easy comparison with your logs. -4. Done. - -Prior versions can access and modify repositories with this measure enabled, however, -to 1.0.9 or later their modifications are indiscernible from an attack and will -raise an error until the below procedure is followed. We are aware that this can -be annoying in some circumstances, but don't see a way to fix the vulnerability -otherwise. - -In case a version prior to 1.0.9 is used to modify a repository where above procedure -was completed, and now you get an error message from other clients: - -1. ``borg upgrade --tam --force `` once with *any* client suffices. - -This attack is mitigated by: - -- Noting/logging ``borg list``, ``borg info``, or ``borg create --stats``, which - contain the archive IDs. - -We are not aware of others having discovered, disclosed or exploited this vulnerability. - -Vulnerability time line: - -* 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore) -* 2016-11-20: First patch -* 2016-12-20: Released fixed version 1.0.9 -* 2017-01-02: CVE was assigned -* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source) - -.. _attic013_check_corruption: - -Pre-1.0.9 potential data loss ------------------------------ - -If you have archives in your repository that were made with attic <= 0.13 -(and later migrated to borg), running borg check would report errors in these -archives. See issue #1837. - -The reason for this is a invalid (and useless) metadata key that was -always added due to a bug in these old attic versions. - -If you run borg check --repair, things escalate quickly: all archive items -with invalid metadata will be killed. Due to that attic bug, that means all -items in all archives made with these old attic versions. - - -Pre-1.0.4 potential repo corruption ------------------------------------ - -Some external errors (like network or disk I/O errors) could lead to -corruption of the backup repository due to issue #1138. - -A sign that this happened is if "E" status was reported for a file that can -not be explained by problems with the source file. If you still have logs from -"borg create -v --list", you can check for "E" status. - -Here is what could cause corruption and what you can do now: - -1) I/O errors (e.g. repo disk errors) while writing data to repo. - -This could lead to corrupted segment files. - -Fix:: - - # check for corrupt chunks / segments: - borg check -v --repository-only REPO - - # repair the repo: - borg check -v --repository-only --repair REPO - - # make sure everything is fixed: - borg check -v --repository-only REPO - -2) Unreliable network / unreliable connection to the repo. - -This could lead to archive metadata corruption. - -Fix:: - - # check for corrupt archives: - borg check -v --archives-only REPO - - # delete the corrupt archives: - borg delete --force REPO::CORRUPT_ARCHIVE - - # make sure everything is fixed: - borg check -v --archives-only REPO - -3) In case you want to do more intensive checking. +Compatibility notes: -The best check that everything is ok is to run a dry-run extraction:: +- this is a major "breaking" release that is not compatible with existing repos. + + We tried to put all the necessary "breaking" changes into this release, so we + hopefully do not need another breaking release in the near future. The changes + were necessary for improved security, improved speed, unblocking future + improvements, getting rid of legacy crap / design limitations, having less and + simpler code to maintain. + + You can use "borg transfer" to transfer archives from borg 1.1/1.2 repos to + a new borg 2.0 repo, but it will need some time and space. + + Before using "borg transfer", you must have upgraded to borg >= 1.2.6 (or + another borg version that was patched to fix CVE-2023-CVE-2023-36811) and + you must have followed the upgrade instructions at top of the change log + relating to manifest and archive TAMs (borg2 just requires these TAMs now). + +- command line syntax was changed, scripts and wrappers will need changes: + + - you will usually either export BORG_REPO= into your environment or + call borg like: "borg -r ". + in the docs, we usually omit "-r ..." for brevity. + - the scp-style REPO syntax was removed, please use ssh://..., #6697 + - ssh:// URLs: removed support for /~otheruser/, #6855. + If you used this, just replace it by: ssh://user@host:port/home/otheruser/ + - -P / --prefix option was removed, please use the similar -a / --match-archives. + - the archive name is always given separately from the repository + (differently than with borg 1.x you must not give repo::archive). + - the archive name is either given as a positional parameter, like: + + - borg create myarchive2 /some/path + - borg diff myarchive1 myarchive2 + - or, if the command makes sense for an arbitrary amount of archives, archives + can be selected using a glob pattern, like: + + - borg delete -a 'sh:myarchive*' + - borg recreate -a 'sh:myarchive*' + - some borg 1.x commands that supported working on a repo AND on an archive + were split into 2 commands, some others were renamed: + + - borg 2 repo commands: + + - borg rcreate # "repo create", was: borg init + - borg rlist # "repo list" + - borg rinfo # "repo info" + - borg rdelete # "repo delete" + - borg 2 archive commands: + + - borg create ARCHIVE ... + - borg list ARCHIVE + - borg extract ARCHIVE ... + - borg diff ARCH1 ARCH2 + - borg rename OLDNAME NEWNAME + - borg info -a ARCH_GLOB + - borg delete -a ARCH_GLOB + - borg recreate -a ARCH_GLOB ... + - borg mount -a ARCH_GLOB mountpoint ... + + For more details, please consult the docs or --help option output. + - create/recreate/import-tar --timestamp: defaults to local timezone + now (was: UTC) +- some deprecated options were removed: + + - removed --remote-ratelimit (use --upload-ratelimit) + - removed --numeric-owner (use --numeric-ids) + - removed --nobsdflags (use --noflags) + - removed --noatime (default now, see also --atime) + - removed --save-space option (does not change behaviour) +- using --list together with --progress is now disallowed (except with --log-json), #7219 +- the --glob-archives option was renamed to --match-archives (the short option + name -a is unchanged) and extended to support different pattern styles: + + - id: for identical string match (this is the new default!) + - sh: for shell pattern / globbing match (this was used by --glob-archives) + - re: for regular expression match + + So you might need to edit your scripts like e.g.:: + + borg 1.x: --glob-archives 'myserver-*' + borg 2.0: --match-archives 'sh:myserver-*' + +- use platformdirs 3.x.x instead of home-grown code. Due to that: + + - XDG_*_HOME is not honoured on macOS and on Windows. + - BORG_BASE_DIR can still be used to enforce some base dir + .config/ or .cache/. + - the default macOS config and cache dir will now be in ~/Library/Application Support/borg/. +- create: different included/excluded status chars, #7321 + + - dry-run: now uses "+" (was: "-") and "-" (was: "x") for included/excluded status + - non-dry-run: now uses "-" (was: "x") for excluded files + + Option --filter=... might need an update, if you filter for the status chars + that were changed. +- borg is now more strict and disallows giving some options multiple times - + if that makes no sense. Highlander options, see #6269. That might make scripts + fail now that somehow "worked" before (but maybe didn't work as intended due to + the contradicting options). - borg extract -v --dry-run REPO::ARCHIVE .. _changelog: -Change Log -========== +Change Log 2.x +============== - -Version 1.2.0 (2022-02-22 22:02:22 :-) --------------------------------------- +Version 2.0.0b9 (2024-07-20) +---------------------------- Please note: -This is the first borg 1.2 release, so be careful and read the notes below. - -Upgrade notes: - -Strictly taken, nothing special is required for upgrading to 1.2, but some -things can be recommended: - -- do you already want to upgrade? 1.1.x also will get fixes for a while. -- be careful, first upgrade your less critical / smaller repos. -- first upgrade to a recent 1.1.x release - especially if you run some older - 1.1.* or even 1.0.* borg release. -- using that, run at least one `borg create` (your normal backup), `prune` - and especially a `check` to see everything is in a good state. -- check the output of `borg check` - if there is anything special, consider - a `borg check --repair` followed by another `borg check`. -- if everything is fine so far (borg check reports no issues), you can consider - upgrading to 1.2.0. if not, please first fix any already existing issue. -- if you want to play safer, first **create a backup of your borg repository**. -- upgrade to latest borg 1.2.x release (you could use the fat binary from - github releases page) -- run `borg compact --cleanup-commits` to clean up a ton of 17 bytes long files - in your repo caused by a borg 1.1 bug -- run `borg check` again (now with borg 1.2.x) and check if there is anything - special. -- run `borg info` (with borg 1.2.x) to build the local pre12-meta cache (can - take significant time, but after that it will be fast) - for more details - see below. -- check the compatibility notes (see below) and adapt your scripts, if needed. -- if you run into any issues, please check the github issue tracker before - posting new issues there or elsewhere. - -If you follow this procedure, you can help avoiding that we get a lot of -"borg 1.2" issue reports that are not really 1.2 issues, but existed before -and maybe just were not noticed. - -Compatibility notes: - -- dropped support / testing for older Pythons, minimum requirement is 3.8. - In case your OS does not provide Python >= 3.8, consider using our binary, - which does not need an external Python interpreter. Or continue using - borg 1.1.x, which is still supported. -- freeing repository space only happens when "borg compact" is invoked. -- mount: the default for --numeric-ids is False now (same as borg extract) -- borg create --noatime is deprecated. Not storing atime is the default behaviour - now (use --atime if you want to store the atime). -- list: corrected mix-up of "isomtime" and "mtime" formats. - Previously, "isomtime" was the default but produced a verbose human format, - while "mtime" produced a ISO-8601-like format. - The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), - and the default is now "mtime". - "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). -- create/recreate --list: file status for all files used to get announced *AFTER* - the file (with borg < 1.2). Now, file status is announced *BEFORE* the file - contents are processed. If the file status changes later (e.g. due to an error - or a content change), the updated/final file status will be printed again. -- removed deprecated-since-long stuff (deprecated since): - - - command "borg change-passphrase" (2017-02), use "borg key ..." - - option "--keep-tag-files" (2017-01), use "--keep-exclude-tags" - - option "--list-format" (2017-10), use "--format" - - option "--ignore-inode" (2017-09), use "--files-cache" w/o "inode" - - option "--no-files-cache" (2017-09), use "--files-cache=disabled" -- removed BORG_HOSTNAME_IS_UNIQUE env var. - to use borg you must implement one of these 2 scenarios: - - - 1) the combination of FQDN and result of uuid.getnode() must be unique - and stable (this should be the case for almost everybody, except when - having duplicate FQDN *and* MAC address or all-zero MAC address) - - 2) if you are aware that 1) is not the case for you, you must set - BORG_HOST_ID env var to something unique. -- exit with 128 + signal number, #5161. - if you have scripts expecting rc == 2 for a signal exit, you need to update - them to check for >= 128. - -Fixes: - -- diff: reduce memory consumption, fix is_hardlink_master, #6295 -- compact: fix / improve freeable / freed space log output - - - derive really freed space from quota use before/after, #5679 - - do not say "freeable", but "maybe freeable" (based on hint, unsure) -- fix race conditions in internal SaveFile function, #6306 #6028 -- implement internal safe_unlink (was: truncate_and_unlink) function more safely: - usually it does not truncate any more, only under "disk full" circumstances - and only if there is only one hardlink. - see: https://github.com/borgbackup/borg/discussions/6286 - -Other changes: - -- info: use a pre12-meta cache to accelerate stats for borg < 1.2 archives. - the first time borg info is invoked on a borg 1.1 repo, it can take a - rather long time computing and caching some stats values for 1.1 archives, - which borg 1.2 archives have in their archive metadata structure. - be patient, esp. if you have lots of old archives. - following invocations are much faster due to the cache. - related change: add archive name to calc_stats progress display. -- docs: - - - add borg 1.2 upgrade notes, #6217 - - link to borg placeholders and borg patterns help - - init: explain the encryption modes better - - clarify usage of patternfile roots - - put import-tar docs into same file as export-tar docs - - explain the difference between a path that ends with or without a slash, - #6297 - - -Version 1.2.0rc1 (2022-02-05) ------------------------------ - -Fixes: - -- repo::archive location placeholder expansion fixes, #5826, #5998 -- repository: fix intermediate commits, shall be at end of current segment -- delete: don't commit if nothing was deleted, avoid cache sync, #6060 -- argument parsing: accept some options only once, #6026 -- disallow overwriting of existing keyfiles on init, #6036 -- if ensure_dir() fails, give more informative error message, #5952 - -New features: - -- delete --force: do not ask when deleting a repo, #5941 - -Other changes: - -- requirements: exclude broken or incompatible-with-pyinstaller setuptools -- add a requirements.d/development.lock.txt and use it for vagrant -- tests: - - - added nonce-related tests - - refactor: remove assert_true - - vagrant: macos box tuning, netbsd box fixes, #5370, #5922 -- docs: - - - update install docs / requirements docs, #6180 - - borg mount / FUSE "versions" view is not experimental any more - - --pattern* is not experimental any more, #6134 - - impact of deleting path/to/repo/nonce, #5858 - - key export: add examples, #6204 - - ~/.config/borg/keys is not used for repokey keys, #6107 - - excluded parent dir's metadata can't restore - +This is a beta release, only for testing - do not use for production repos. -Version 1.2.0b4 (2022-01-23) ----------------------------- - -Fixes: - -- create: fix passing device nodes and symlinks to --paths-from-stdin, #6009 -- create --dry-run: fix display of kept tagfile, #5834 -- check --repair: fix missing parameter in "did not consistently fail" msg, #5822 -- fix hardlinkable file type check, #6037 -- list: remove placeholders for shake_* hashes, #6082 -- prune: handle case of calling prune_split when there are no archives, #6015 -- benchmark crud: make sure cleanup of borg-test-data files/dir happens, #5630 -- do not show archive name in repository-related error msgs, #6014 -- prettier error msg (no stacktrace) if exclude file is missing, #5734 -- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979 -- fix pyinstaller detection for dir-mode, #5897 -- atomically create the CACHE_TAG file, #6028 -- deal with the SaveFile/SyncFile race, docs, see #6056 708a5853 -- avoid expanding path into LHS of formatting operation + tests, #6064 #6063 -- repository: quota / compactable computation fixes -- info: emit repo info even if repo has 0 archives + test, #6120 +For upgrade and compatibility hints, please also read the section "Upgrade Notes" +above. New features: -- check --repair: significantly speed up search for next valid object in segment, #6022 -- check: add progress indicator for archive check, #5809 -- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024 -- create: allow --files-cache=size (this is potentially dangerous, use on your own risk), #5686 -- import-tar: implement import-tar to complement export-tar, #2233 -- implement BORG_SELFTEST env variable (can be carefully used to speedup borg hosting), #5871 -- key export: print key if path is '-' or not given, #6092 -- list --format: Add command_line to format keys - -Other changes: - -- pypi metadata: alpha -> beta -- require python 3.8+, #5975 -- use pyinstaller 4.7 -- allow msgpack 1.0.3 -- upgrade to bundled xxhash to 0.8.1 -- import-tar / export-tar: tar file related changes: - - - check for short tarfile extensions - - add .lz4 and .zstd - - fix docs about extensions and decompression commands -- add github codeql analysis, #6148 -- vagrant: - - - box updates / add new boxes / remove outdated and broken boxes - - use Python 3.9.10 (incl. binary builds) and 3.10.0 - - fix pyenv initialisation, #5798 - - fix vagrant scp on macOS, #5921 - - use macfuse instead of osxfuse -- shell completions: - - - update shell completions to 1.1.17, #5923 - - remove BORG_LIBC completion, since 9914968 borg no longer uses find_library(). -- docs: - - - fixed readme.rst irc webchat link (we use libera chat now, not freenode) - - fix exceptions thrown by `setup.py build_man` - - check --repair: recommend checking hw before check --repair, #5855 - - check --verify-data: clarify and document conflict with --repository-only, #5808 - - serve: improve ssh forced commands docs, #6083 - - list: improve docs for `borg list` --format, #6061 - - list: remove --list-format from borg list - - FAQ: fix manifest-timestamp path (inside security dir) - - fix the broken link to .nix file - - document behavior for filesystems with inconsistent inodes, #5770 - - clarify user_id vs uid for fuse, #5723 - - clarify pattern usage with commands, #5176 - - clarify pp vs. pf pattern type, #5300 - - update referenced freebsd/macOS versions used for binary build, #5942 - - pull mode: add some warnings, #5827 - - clarify "you will need key and passphrase" borg init warning, #4622 - - add missing leading slashes in help patterns, #5857 - - add info on renaming repositories, #5240 - - check: add notice about defective hardware, #5753 - - mention tar --compare (compare archive to fs files), #5880 - - add note about grandfather-father-son backup retention policy / rotation scheme, #6006 - - permissions note rewritten to make it less confusing - - create github security policy - - remove leftovers of BORG_HOSTNAME_IS_UNIQUE - - excluded parent dir's metadata can't restore. (#6062) - - if parent dir is not extracted, we do not have its metadata - - clarify who starts the remote agent - - -Version 1.2.0b3 (2021-05-12) ----------------------------- - -Fixes: +- add BORG_CACHE_IMPL, default is "adhocwithfiles" to test the new cache + implementation, featuring an adhoc non-persistent chunks cache and a + persistent files cache. See the docs for other values. -- create: fix --progress --log-json, #4360#issuecomment-774580052 -- do not load files cache for commands not using it, #5673 -- fix repeated cache tag file writing bug + Requires to run "borg check --repair --archives-only" to delete orphaned + chunks before running "borg compact" to free space! These orphans are + expected due to the simplified refcounting with the AdHocFilesCache. +- make BORG_EXIT_CODES="modern" the default, #8110 +- add BORG_USE_CHUNKS_ARCHIVE env var, #8280 +- automatically rebuild cache on exception, #5213 -New features: +Bug fixes: -- create/recreate: print preliminary file status early, #5417 -- create/extract: add --noxattrs and --noacls options, #3955 -- create: verbose files cache logging via --debug-topic=files_cache, #5659 -- mount: implement --numeric-ids (default: False!), #2377 -- diff: add --json-lines option -- info / create --stats: add --iec option to print sizes in powers of 1024. +- fix Ctrl-C / SIGINT behaviour for pyinstaller-made binaries, #8155 +- delete: fix error handling with Ctrl-C +- rcompress: fix error handling with Ctrl-C +- delete: fix error handling when no archive is specified, #8256 +- setup.py: fix import error reporting for cythonize import, see #8208 +- create: deal with EBUSY, #8123 +- benchmark: inherit options --rsh --remote-path, #8099 +- benchmark: fix return value, #8113 +- key export: fix crash when no path is given, fix exception handling Other changes: -- create: add --upload-(ratelimit|buffer), deprecate --remote-* options, #5611 -- create/extract/mount: add --numeric-ids, deprecate --numeric-owner option, #5724 -- config: accept non-int value for max_segment_size / storage_quota -- use PyInstaller v4.3, #5671 -- vagrant: use Python 3.9.5 to build binaries -- tox.ini: modernize and enable execution without preinstalling deps -- cleanup code style checks -- get rid of distutils, use setuptools+packaging -- github CI: test on Python 3.10-dev -- check: missing / healed chunks: always tell chunk ID, #5704 -- docs: - - - remove bad /var/cache exclusion in example commands, #5625 - - misc. fixes and improvements, esp. for macOS - - add unsafe workaround to use an old repo copy, #5722 +- setup.py: detect noexec build fs issue, see #8208 +- improve acl_get / acl_set error handling (forward port from 1.4-maint) +- allow msgpack 1.1.0 +- vagrant: use pyinstaller 6.7.0 +- use Python 3.11.9 for binary builds +- require Cython 3.0.3 at least, #8133 +- docs: add non-root deployment strategy -Version 1.2.0b2 (2021-02-06) +Version 2.0.0b8 (2024-02-20) ---------------------------- -Fixes: - -- create: do not recurse into duplicate roots, #5603 -- create: only print stats if not ctrl-c'ed, fixes traceback, #5668 -- extract: - improve exception handling when setting xattrs, #5092. - emit a warning message giving the path, xattr key and error message. - continue trying to restore other xattrs and bsdflags of the same file - after an exception with xattr-setting happened. -- export-tar: - fix memory leak with ssh: remote repository, #5568. - fix potential memory leak with ssh: remote repository with partial extraction. -- remove empty shadowed_segments lists, #5275 -- fix bad default: manifest.archives.list(consider_checkpoints=False), - fixes tracebacks / KeyErros for missing objects in ChunkIndex, #5668 - New features: -- create: improve sparse file support - - - create --sparse (detect sparse file holes) and file map support, - only for the "fixed" chunker, #14 - - detect all-zero chunks in read data in "buzhash" and "fixed" chunkers - - cached_hash: use a small LRU cache to accelerate all-zero chunks hashing - - use cached_hash also to generate all-zero replacement chunks -- create --remote-buffer, add a upload buffer for remote repos, #5574 -- prune: keep oldest archive when retention target not met - -Other changes: - -- use blake2 from python 3.6+ hashlib - (this removes the requirement for libb2 and the bundled blake2 code) -- also accept msgpack up to 1.0.2. - exclude 1.0.1 though, which had some issues (not sure they affect borg). -- create: add repository location to --stats output, #5491 -- check: debug log the segment filename -- delete: add a --list switch to borg delete, #5116 -- borg debug dump-hints - implemented to e.g. to look at shadow_index -- Tab completion support for additional archives for 'borg delete' -- refactor: have one borg.constants.zero all-zero bytes object -- refactor shadow_index updating repo.put/delete, #5661, #5636. -- docs: - - - add another case of attempted hardlink usage - - fix description of borg upgrade hardlink usage, #5518 - - use HTTPS everywhere - - add examples for --paths-from-stdin, --paths-from-command, --paths-separator, #5644 - - fix typos/grammar - - update docs for dev environment installation instructions - - recomend running tests only on installed versions for setup - - add badge with current status of package +- create: add the slashdot hack, update docs, #4685 +- BORG_EXIT_CODES=modern: optional more specific return codes (for errors and warnings). + + The default value of this new environment variable is "legacy", which should result in + a behaviour similar to borg 1.2 and older (only using rc 0, 1 and 2). + "modern" exit codes are much more specific (see the internals/frontends docs). +- implement "borg version" (shows client and server version), #7829 + +Fixes: + +- docs: CVE-2023-36811 upgrade steps: consider checkpoint archives, #7802 +- check/compact: fix spurious reappearance of orphan chunks since borg 1.2, #6687 - + this consists of 2 fixes: + + - for existing chunks: check --repair: recreate shadow index, #7897 #6687 + - for newly created chunks: update shadow index when doing a double-put, #7896 #5661 + + If you have experienced issue #6687, you may want to run borg check --repair + after upgrading to borg 1.2.7 to recreate the shadow index and get rid of the + issue for existing chunks. +- check: fix return code for index entry value discrepancies +- LockRoster.modify: no KeyError if element was already gone, #7937 +- create --X-from-command: run subcommands with a clean environment, #7916 +- list --sort-by: support "archive" as alias of "name", #7873 +- fix rc and msg if arg parsing throws an exception, #7885 +- PATH: do not accept empty strings, #4221 +- fix invalid pattern argument error msg +- zlib legacy decompress fixes, #7883 + +Other changes: + +- replace archive/manifest TAMs by typed repo objects (ro_type), docs, #7670 +- crypto: use a one-step kdf for session keys, #7953 +- remove recreate --recompress option, use the more efficient repo-wide "rcompress". +- include unistd.h in _chunker.c (fix for Python 3.13) +- allow msgpack 1.0.7 +- allow platformdirs 4, #7950 +- use and require cython3 +- move conftest.py to src/borg/testsuite, #6386 +- use less setup.py, use pip and build +- linux: use pkgconfig to find libacl +- borg.logger: use same method params as python logging +- create and use Brewfile, document "brew bundle" install (macOS) +- blacken master branch +- prevent CLI argument issues in scripts/glibc_check.py +- pyproject.toml: exclude source files which have been compiled, #7828 +- sdist: dynamically compute readme (long_description) +- init: better borg key export instructions +- scripts/make.py: move clean, build_man, build_usage to there, + so we do not need to invoke setup.py directly, update docs - vagrant: - - use brew install --cask ..., #5557 - - use Python 3.9.1 and PyInstaller 4.1 to build the borg binary - - -Version 1.2.0b1 (2020-12-06) + - use openssl 3.0 on macOS + - add script for fetching borg binaries from VMs, #7989 + - use generic/openbsd7 box + - netbsd: test on py311 only + - remove debian 9 "stretch" box + - use freebsd 14, #6871 + - use python 3.9.4 for tests, latest python 3.11.7 for binary builds + - use pyinstaller 6.3.0 +- docs: + + - add typical PR workflow to development docs, #7495 + - improve docs for borg with-lock, add example #8024 + - create disk/partition sector backup by disk serial number + - Add "check.rebuild_refcounts" message + - not only attack/unsafe, can also be a fs issue, #7853 + - use virtualenv on Cygwin + - readthedocs: also build offline docs, #7835 + - do not refer to setup.py installation method + - how to run the testsuite using the dist package + - requirements are defined in pyproject.toml + + +Version 2.0.0b7 (2023-09-14) ---------------------------- -Fixes: - -- BORG_CACHE_DIR crashing borg if empty, atomic handling of - recursive directory creation, #5216 -- fix --dry-run and --stats coexistence, #5415 -- allow EIO with warning when trying to hardlink, #4336 -- export-tar: set tar format to GNU_FORMAT explicitly, #5274 -- use --timestamp for {utcnow} and {now} if given, #5189 -- make timestamp helper timezone-aware - New features: -- create: implement --paths-from-stdin and --paths-from-command, see #5492. - These switches read paths to archive from stdin. Delimiter can specified - by --paths-delimiter=DELIM. Paths read will be added honoring every - option but exclusion options and --one-file-system. borg won't recurse - into directories. -- 'obfuscate' pseudo compressor obfuscates compressed chunk size in repo -- add pyfuse3 (successor of llfuse) as an alternative lowlevel fuse - implementation to llfuse (deprecated), #5407. - FUSE implementation can be switched via env var BORG_FUSE_IMPL. -- allow appending to the files cache filename with BORG_FILES_CACHE_SUFFIX -- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333 - -Other changes: - -- split recursive directory walking/processing into directory walking and - item processing. -- fix warning by importing setuptools before distutils. -- debug info: include infos about FUSE implementation, #5546 -- testing: - - - add a test for the hashindex corruption bug, #5531 #4829 - - move away from travis-ci, use github actions, #5528 #5467 - - test both on fuse2 and fuse3 - - upload coverage reports to codecov - - fix spurious failure in test_cache_files, #5438 - - add tests for Location.with_timestamp - - tox: add a non-fuse env to the envlist -- vagrant: - - - use python 3.7.latest and pyinstaller 4.0 for binary creation - - pyinstaller: compute basepath from spec file location - - vagrant: updates/fixes for archlinux box, #5543 -- docs: - - - "filename with spaces" example added to exclude file, #5236 - - add a hint about sleeping computer, #5301 - - how to adjust macOS >= Catalina security settings, #5303 - - process/policy for adding new compression algorithms - - updated docs about hacked backup client, #5480 - - improve ansible deployment docs, make it more generic - - how to approach borg speed issues, give speed example, #5371 - - fix mathematical inaccuracy about chunk size, #5336 - - add example for excluding content using --pattern cli option - - clarify borg create's '--one-file-system' option, #4009 - - improve docs/FAQ about append-only remote repos, #5497 - - fix reST markup issues, labels - - add infos about contributor retirement status - - -Version 1.2.0a9 (2020-10-05) ----------------------------- +- BORG_WORKAROUNDS=authenticated_no_key to extract from authenticated repos + without having the borg key, #7700 Fixes: -- fix memory leak related to preloading, #5202 -- check --repair: fix potential data loss, #5325 -- persist shadow_index in between borg runs, #4830 -- fix hardlinked CACHEDIR.TAG processing, #4911 -- --read-special: .part files also should be regular files, #5217 -- allow server side enforcing of umask, --umask is for the local borg - process only (see docs), #4947 -- exit with 128 + signal number, #5161 -- borg config --list does not show last_segment_checked, #5159 -- locking: - - - fix ExclusiveLock race condition bug, #4923 - - fix race condition in lock migration, #4953 - - fix locking on openindiana, #5271 - -New features: - -- --content-from-command: create archive using stdout of given command, #5174 -- allow key-import + BORG_KEY_FILE to create key files -- build directory-based binary for macOS to avoid Gatekeeper delays +- archive tam verify security fix, fixes CVE-2023-36811 +- remote logging/progress: use callback to send queued records, #7662 +- make_path_safe: remove test for backslashes, #7651 +- benchmark cpu: use sanitized path, #7654 +- create: do not try to read parent dir of recursion root, #7746 Other changes: -- upgrade bundled zstd to 1.4.5 -- upgrade bundled xxhash to 0.8.0, #5362 -- if self test fails, also point to OS and hardware, #5334 -- misc. shell completions fixes/updates, rewrite zsh completion -- prettier error message when archive gets too big, #5307 -- stop relying on `false` exiting with status code 1 -- rephrase some warnings, #5164 -- parseformat: unnecessary calls removed, #5169 -- testing: - - - enable Python3.9 env for test suite and VMs, #5373 - - drop python 3.5, #5344 - - misc. vagrant fixes/updates - - misc. testing fixes, #5196 -- docs: +- always implicitly require archive TAMs (all archives have TAMs since borg 1.2.6) +- always implicitly require manifest TAMs (manifests have TAMs since borg 1.0.9) +- rlist: remove support for {tam} placeholder, archives are now always TAM-authenticated. +- support / test on Python 3.12 +- allow msgpack 1.0.6 (which has py312 wheels), #7810 +- manifest: move item_keys into config dict (manifest.version == 2 now), #7710 +- replace "datetime.utcfromtimestamp" to avoid deprecation warnings with Python 3.12 +- properly normalise paths on Windows (forward slashes, integrate drive letter into path) +- Docs: - - add ssh-agent pull backup method to doc, #5288 - - mention double --force in prune docs - - update Homebrew install instructions, #5185 - - better description of how cache and rebuilds of it work - and how the workaround applies to that - - point to borg create --list item flags in recreate usage, #5165 - - add a note to create from stdin regarding files cache, #5180 - - add security faq explaining AES-CTR crypto issues, #5254 - - clarify --exclude-if-present in recreate, #5193 - - add socat pull mode, #5150, #900 - - move content of resources doc page to community project, #2088 - - explain hash collision, #4884 - - clarify --recompress option, #5154 - - -Version 1.2.0a8 (2020-04-22) + - move upgrade / compat. notes to own section, see #7546 + - fix borg delete examples, #7759 + - improve rcreate / related repos docs + - automated-local.rst: use UUID for consistent udev rule + - rewrite `borg check` docs, #7578 + - misc. other docs updates +- Tests / CI / Vagrant: + + - major testsuite refactoring: a lot more tests now use pytest, #7626 + - freebsd: add some ACL tests, #7745 + - fix test_disk_full, #7617 + - fix failing test_get_runtime_dir test on OpenBSD, #7719 + - CI: run on ubuntu 22.04 + - CI: test building the docs + - simplify flake8 config, fix some complaints + - use pyinstaller 5.13.1 to build the borg binaries + + +Version 2.0.0b6 (2023-06-11) ---------------------------- -Fixes: - -- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829. - Please read and follow the more detailed notes close to the top of this document. -- fix crash when upgrading erroneous hints file, #4922 -- commit-time free space calc: ignore bad compact map entries, #4796 -- info: if the archive doesn't exist, print a pretty message, #4793 -- --prefix / -P: fix processing, avoid argparse issue, #4769 -- ignore EACCES (errno 13) when hardlinking, #4730 -- add a try catch when formatting the info string, #4818 -- check: do not stumble over invalid item key, #4845 -- update prevalence of env vars to set config and cache paths -- mount: fix FUSE low linear read speed on large files, #5032 -- extract: fix confusing output of borg extract --list --strip-components, #4934 -- recreate: support --timestamp option, #4745 -- fix ProgressIndicator msgids (JSON output), #4935 -- fuse: set f_namemax in statfs result, #2684 -- accept absolute paths on windows -- pyinstaller: work around issue with setuptools > 44 - New features: -- chunker speedup (plus regression test) -- added --consider-checkpoints and related test, #4788 -- added --noflags option, deprecate --nobsdflags option, #4489 -- compact: add --threshold option, #4674 -- mount: add birthtime to FUSE entries -- support platforms with no os.link, #4901 - if we don't have os.link, - we just extract another copy instead of making a hardlink. -- move sync_file_range to its own extension for better platform compatibility. -- new --bypass-lock option to bypass locking, e.g. for read-only repos -- accept absolute paths by removing leading slashes in patterns of all - sorts but re: style, #4029 -- delete: new --keep-security-info option - -Other changes: - -- support msgpack 0.6.2 and 1.0.0, #5065 -- upgrade bundled zstd to 1.4.4 -- upgrade bundled lz4 to 1.9.2 -- upgrade xxhash to 0.7.3 -- require recent enough llfuse for birthtime support, #5064 -- only store compressed data if the result actually is smaller, #4516 -- check: improve error output for matching index size, see #4829 -- ignore --stats when given with --dry-run, but continue, #4373 -- replaced usage of os.statvfs with shutil.disk_usage (better cross-platform support). -- fuse: remove unneeded version check and compat code, micro opts -- docs: - - - improve description of path variables - - document how to completely delete data, #2929 - - add FAQ about Borg config dir, #4941 - - add docs about errors not printed as JSON, #4073 - - update usage_general.rst.inc - - added "Will move with BORG_CONFIG_DIR variable unless specified." to BORG_SECURITY_DIR info. - - put BORG_SECURITY_DIR immediately below BORG_CONFIG_DIR (and moved BORG_CACHE_DIR up before them). - - add paragraph regarding cache security assumptions, #4900 - - tell about borg cache security precautions - - add FAQ describing difference between a local repo vs. repo on a server. - - document how to test exclusion patterns without performing an actual backup - - create: tell that "Calculating size" time and space needs are caused by --progress - - fix/improve documentation for @api decorator, #4674 - - add a pull backup / push restore how-to, #1552 - - fix man pages creation, #4752 - - more general FAQ for backup and retain original paths, #4532 - - explain difference between --exclude and --pattern, #4118 - - add FAQ for preventing SSH timeout in extract, #3866 - - improve password FAQ (decrease pw length, add -w 0 option to base64 to prevent line wrap), #4591 - - add note about patterns and stored paths, #4160 - - add upgrade of tools to pip installation how-to, #5090 - - document one cause of orphaned chunks in check command, #2295 - - clean up the whole check usage paragraph - - FAQ: linked recommended restrictions to ssh public keys on borg servers, #4946 - - fixed "doc downplays severity of Nonce reuse issue", #4883 - - borg repo restore instructions needed, #3428 - - new FAQ: A repo is corrupt and must be replaced with an older repo. - - clarify borg init's encryption modes -- native windows port: - - - update README_WINDOWS.rst - - updated pyinstaller spec file to support windows builds -- testing / CI: - - - improved travis config / install script, improved macOS builds - - allow osx builds to fail, #4955 - - Windows 10 build on Appveyor CI -- vagrant: - - - upgrade pyinstaller to v3.5 + patch - - use py369 for binary build, add py380 for tests - - fix issue in stretch VM hanging at grub installation - - add a debian buster and a ubuntu focal VM - - update darwin box to 10.12 - - upgrade FreeBSD box to 12.1 - - fix debianoid virtualenv packages - - use pyenv in freebsd64 VM - - remove the flake8 test - - darwin: avoid error if pkg is already installed - - debianoid: don't interactively ask questions - - -Version 1.2.0a7 (2019-09-07) +- diff: include changes in ctime and mtime, #7248 +- diff: sort JSON output alphabetically +- diff --content-only: option added to ignore metadata changes +- diff: add --format option, #4634 +- import-tar --ignore-zeros: new option to support importing concatenated tars, #7432 +- debug id-hash / parse-obj / format-obj: new debug commands, #7406 +- transfer --compression=C --recompress=M: recompress while transferring, #7529 +- extract --continue: continue a previously interrupted extraction, #1356 +- prune --list-kept/--list-pruned: only list the kept (or pruned) archives, #7511 +- prune --short/--format: enable users to format the list output, #3238 +- implement BORG__FORMAT env vars for prune, list, rlist, #5166 +- rlist: size and nfiles format keys +- implement unix domain (ipc) socket support, #6183:: + + borg serve --socket # server side (not started automatically!) + borg -r socket:///path/to/repo ... # client side +- add get_runtime_dir / BORG_RUNTIME_DIR (contains e.g. .sock and .pid file) +- support shell-style alternatives, like: sh:image.{png,jpg}, #7602 + +Fixes: + +- do not retry on permission errors (pointless) +- transfer: verify chunks we get using assert_id, #7383 +- fix config/cache dir compatibility issues, #7445 +- xattrs: fix namespace processing on FreeBSD, #6997 +- ProgressIndicatorPercent: fix space computation for wide chars, #3027 +- delete: remove --cache-only option, #7440. + for deleting the cache only, use: borg rdelete --cache-only +- borg debug get-obj/put-obj: fixed chunk id +- create: ignore empty paths, print warning, #5637 +- extract: support extraction of atime/mtime on win32 +- benchmark crud: use TemporaryDirectory below given path, #4706 +- Ensure that cli options specified with action=Highlander can only be set once, even + if the set value is a default value. Add tests for action=Highlander, #7500, #6269. +- Fix argparse error messages from misc. validators (being more specific). +- put security infos into data dir, add BORG_DATA_DIR env var, #5760 +- setup.cfg: remove setup_requires (we have a pyproject.toml for that), #7574 +- do not crash for empty archives list in borg rlist date based matching, #7522 +- sanitize paths during archive creation and extraction, #7108 #7099 +- make sure we do not get backslashes into item paths + +Other changes: + +- allow msgpack 1.0.5 also +- development.lock.txt: upgrade cython to 0.29.35, misc. other upgrades +- clarify platformdirs requirements, #7393. + 3.0.0 is only required for macOS due to breaking changes. + 2.6.0 was the last breaking change for Linux/UNIX. +- mount: improve mountpoint error msgs, see #7496 +- more Highlander options, #6269 +- Windows: simplify building (just use pip) +- refactor toplevel exception handling, #6018 +- remove nonce management, related repo methods (not needed for borg2) +- borg.remote: remove support for borg < 1.1.0 + ($LOG, logging setup, exceptions, rpc tuple data format, version) +- new remote and progress logging, #7604 +- borg.logger: add logging debugging functionality +- add function to clear empty directories at end of compact process +- unify scanning and listing of segment dirs / segment files, #7597 +- replace `LRUCache` internals with `OrderedDict` +- docs: + + - add installation instructions for Windows + - improve --one-file-system help and docs (macOS APFS), #5618 #4876 + - BORG_KEY_FILE: clarify docs, #7444 + - installation: add link to OS dependencies, #7356 + - update FAQ about locale/unicode issues, #6999 + - improve mount options rendering, #7359 + - make timestamps in manual pages reproducible. + - describe performing pull-backups via ssh remote forwarding + - suggest to use forced command when using remote-forwarding via ssh + - fix some -a / --match-archives docs issues + - incl./excl. options header, clarify --path-from-stdin exclusive control + - add note about MAX_DATA_SIZE + - update security support docs + - improve patterns help + +- CI / tests / vagrant: + + - added pre-commit for linting purposes, #7476 + - resolved mode bug and added sleep clause for darwin systems, #7470 + - "auto" compressor tests: do not assume zlib is better than lz4, #7363 + - add stretch64 VM with deps built from source + - misc. other CI / test fixes and updates + - vagrant: add lunar64 VM, fix packages_netbsd + - avoid long ids in pytest output + - tox: package = editable-legacy, #7580 + - tox under fakeroot: fix finding setup_docs, #7391 + - check buzhash chunksize distribution, #7586 + - use debian/bookworm64 box + + +Version 2.0.0b5 (2023-02-27) ---------------------------- -Fixes: - -- slave hardlinks extraction issue, see #4350 -- extract: fix KeyError for "partial" extraction, #4607 -- preload chunks for hardlink slaves w/o preloaded master, #4350 -- fix preloading for old remote servers, #4652 -- fix partial extract for hardlinked contentless file types, #4725 -- Repository.open: use stat() to check for repo dir, #4695 -- Repository.check_can_create_repository: use stat() to check, ~ #4695. -- SecurityManager.known(): check all files, #4614 -- after double-force delete, warn about necessary repair, #4704 -- cope with ANY error when importing pytest into borg.testsuite, #4652 -- fix invalid archive error message -- setup.py: fix detection of missing Cython -- filter out selinux xattrs, #4574 -- location arg - should it be optional? #4541 -- enable placeholder usage in --comment, #4559 -- use whitelist approach for borg serve, #4097 - New features: -- minimal native Windows support, see windows readme (work in progress) -- create: first ctrl-c (SIGINT) triggers checkpoint and abort, #4606 -- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710 -- remove WSL autodetection. if WSL still has this problem, you need to - set BORG_WORKAROUNDS=basesyncfile in the borg process environment to - work around it. -- support xxh64 checksum in addition to the hashlib hashes in borg list -- enable placeholder usage in all extra archive arguments -- enable placeholder usage in --comment, #4559 -- enable placeholder usage in --glob-archives, #4495 -- ability to use a system-provided version of "xxhash" -- create: - - - changed the default behaviour to not store the atime of fs items. atime is - often rather not interesting and fragile - it easily changes even if nothing - else has changed and, if stored into the archive, spoils deduplication of - the archive metadata stream. - - if you give the --noatime option, borg will output a deprecation warning - because it is currently ignored / does nothing. - Please remove the --noatime option when using borg 1.2. - - added a --atime option for storing files' atime into an archive - -Other changes: - -- argparser: always use REPOSITORY in metavar -- do not check python/libc for borg serve, #4483 -- small borg compact improvements, #4522 -- compact: log freed space at INFO level +- create: implement retries for individual fs files + (e.g. if a file changed while we read it, if a file had an OSError) +- info: add used storage quota, #7121 +- transfer: support --progress +- create/recreate/import-tar: add --checkpoint-volume option +- support date-based matching for archive selection, + add --newer/--older/--newest/--oldest options, #7062 #7296 + +Fixes: + +- disallow --list with --progress, #7219 +- create: fix --list --dry-run output for directories, #7209 +- do no assume hardlink_master=True if not present, #7175 +- fix item_ptrs orphaned chunks of checkpoint archives +- avoid orphan content chunks on BackupOSError, #6709 +- transfer: fix bug in obfuscated data upgrade code +- fs.py: fix bug in f-string (thanks mypy!) +- recreate: when --target is given, do not detect "nothing to do", #7254 +- locking (win32): deal with os.rmdir/listdir PermissionErrors +- locking: thread id must be parsed as hex from lock file name +- extract: fix mtime when ResourceFork xattr is set (macOS specific), #7234 +- recreate: without --chunker-params borg shall not rechunk, #7336 +- allow mixing --progress and --list in log-json mode +- add "files changed while reading" to Statistics class, #7354 +- fixed keys determination in Statistics.__add__(), #7355 + +Other changes: + +- use local time / local timezone to output timestamps, #7283 +- update development.lock.txt, including a setuptools security fix, #7227 +- remove --save-space option (does not change behaviour) +- remove part files from final archive +- remove --consider-part-files, related stats code, update docs +- transfer: drop part files +- check: show id of orphaned chunks +- ArchiveItem.cmdline list-of-str -> .command_line str, #7246 +- Item: symlinks: rename .source to .target, #7245 +- Item: make user/group/uid/gid optional +- create: do not store user/group for stdin data by default, #7249 +- extract: chown only if we have u/g info in archived item, #7249 +- export-tar: for items w/o uid/gid, default to 0/0, #7249 +- fix some uid/gid lookup code / tests for win32 +- cache.py: be less verbose during cache sync +- update bash completion script commands and options, #7273 +- require and use platformdirs 3.x.x package, tests +- better included/excluded status chars, docs, #7321 +- undef NDEBUG for chunker and hashindex (make assert() work) +- assert_id: better be paranoid (add back same crypto code as in old borg), #7362 +- check --verify_data: always decompress and call assert_id(), #7362 +- make hashindex_compact simpler and probably faster, minor fixes, cleanups, more tests +- hashindex minor fixes, refactor, tweaks, tests +- pyinstaller: remove icon +- validation / placeholders / JSON: + + - implement (text|binary)_to_json: key (text), key_b64 (base64(binary)) + - remove bpath, barchive, bcomment placeholders / JSON keys + - archive metadata: make sure hostname and username have no surrogate escapes + - text attributes (like archive name, comment): validate more strictly, #2290 + - transfer: validate archive names and comment before transfer + - json output: use text_to_json (path, target), #6151 +- docs: + + - docs and comments consistency, readability and spelling fixes + - fix --progress display description, #7180 + - document how borg deals with non-unicode bytes in JSON output + - document another way to get UTF-8 encoding on stdin/stdout/stderr, #2273 + - pruning interprets timestamps in the local timezone where borg prune runs + - shellpattern: add license, use copyright/license markup + - key change-passphrase: fix --encryption value in examples + - remove BORG_LIBB2_PREFIX (not used any more) + - Installation: Update Fedora in distribution list, #7357 + - add .readthedocs.yaml (use py311, use non-shallow clone) - tests: - - tox / travis: add testing on py38-dev - - fix broken test that relied on improper zlib assumptions - - pure-py msgpack warning shall not make a lot of tests fail, #4558 - - rename test_mount_hardlinks to test_fuse_mount_hardlinks (master) - - vagrant: add up-to-date openindiana box (py35, openssl10) - - get rid of confusing coverage warning, #2069 -- docs: - - - reiterate that 'file cache names are absolute' in FAQ, - mention bind mount solution, #4738 - - add restore docs, #4670 - - updated docs to cover use of temp directory on remote, #4545 - - add a push-style example to borg-create(1), #4613 - - timestamps in the files cache are now usually ctime, #4583 - - benchmark crud: clarify that space is used until compact - - update documentation of borg create, - corrects a mention of borg 1.1 as a future version. - - fix osxfuse github link in installation docs - - how to supply a passphrase, use crypto devices, #4549 - - extract: document limitation "needs empty destination", #4598 - - update macOS Brew link - - add note about software for automating backup - - compact: improve docs, - - README: new URL for funding options - - -Version 1.2.0a6 (2019-04-22) + - fix archiver tests on Windows, add running the tests to Windows CI + - fix tox4 passenv issue, #7199 + - github actions updates (fix deprecation warnings) + - add tests for borg transfer/upgrade + - fix test hanging reading FIFO when `borg create` failed + - mypy inspired fixes / updates + - fix prune tests, prune in localtime + - do not look up uid 0 / gid 0, but current process uid/gid + - safe_unlink tests: use os.link to support win32 also + - fix test_size_on_disk_accurate for large st_blksize, #7250 + - relaxed timestamp comparisons, use same_ts_ns + - add test for extracted directory mtime + - use "fail" chunker to test erroneous input file skipping + + +Version 2.0.0b4 (2022-11-27) ---------------------------- Fixes: -- delete / prune: consider part files correctly for stats, #4507 -- fix "all archives" stats considering part files, #4329 -- create: only run stat_simple_attrs() once -- create: --stats does not work with --dry-run, exit with error msg, #4373 -- give "invalid repo" error msg if repo config not found, #4411 - -New features: - -- display msgpack version as part of sysinfo (e.g. in tracebacks) +- transfer/upgrade: fix borg < 1.2 chunker_params, #7079 +- transfer/upgrade: do not access Item._dict, #7077 +- transfer/upgrade: fix crash in borg transfer, #7156 +- archive.save(): always use metadata from stats, #7072 +- benchmark: fixed TypeError in compression benchmarks, #7075 +- fix repository.scan api minimum requirement +- fix args.paths related argparsing, #6994 Other changes: -- docs: - - - sdd "SSH Configuration" section, #4493, #3988, #636, #4485 - - better document borg check --max-duration, #4473 - - sorted commands help in multiple steps, #4471 -- testing: +- tar_filter: recognize .tar.zst as zstd, #7093 +- adding performance statistics to borg create, #6991 +- docs: add rcompress to usage index +- tests: - - travis: use py 3.5.3 and 3.6.7 on macOS to get a pyenv-based python - build with openssl 1.1 - - vagrant: use py 3.5.3 and 3.6.8 on darwin64 VM to build python and - borg with openssl 1.1 - - pytest: -v and default XDISTN to 1, #4481 + - use github and MSYS2 for Windows CI, #7097 + - win32 and cygwin: test fixes / skip hanging test + - vagrant / github CI: use python 3.11.0 / 3.10.8 +- vagrant: + - upgrade pyinstaller to 5.6.2 (supports python 3.11) + - use python 3.11 to build the borg binary -Version 1.2.0a5 (2019-03-21) +Version 2.0.0b3 (2022-10-02) ---------------------------- Fixes: -- warn if a file has changed while being backed up, #1750 -- lrucache: regularly remove old FDs, #4427 -- borg command shall terminate with rc 2 for ImportErrors, #4424 -- make freebsd xattr platform code api compatible with linux, #3952 +- transfer: fix user/group == None crash with borg1 archives +- compressors: avoid memoryview related TypeError +- check: fix uninitialised variable if repo is completely empty, #7034 +- do not use version_tuple placeholder in setuptools_scm template, #7024 +- get_chunker: fix missing sparse=False argument, #7056 -Other changes: +New features: -- major setup code refactoring (especially how libraries like openssl, liblz4, - libzstd, libb2 are discovered and how it falls back to code bundled with - borg), new: uses pkg-config now (and needs python "pkgconfig" package - installed), #1925 +- rcompress: do a repo-wide (re)compression, #7037 +- implement pattern support for --match-archives, #6504 +- BORG_LOCK_WAIT=n env var to set default for --lock-wait option, #5279 - if you are a borg package maintainer, please try packaging this - (see comments in setup.py). -- Vagrantfile: add zstd, reorder, build env vars, #4444 -- travis: install script improvements -- update shell completions -- docs: +Other: - - add a sample logging.conf in docs/misc, #4380 - - fix spelling errors - - update requirements / install docs, #4374 +- repository.scan: misc. fixes / improvements +- metadata: differentiate between empty/zero and unknown, #6908 +- CI: test pyfuse3 with python 3.11 +- use more relative imports +- make borg.testsuite.archiver a package, split archiver tests into many modules +- support reading new, improved hashindex header format, #6960. + added version number and num_empty to the HashHeader, fixed alignment. +- vagrant: upgrade pyinstaller 4.10 -> 5.4.1, use python 3.9.14 for binary build +- item.pyx: use more Cython (faster, uses less memory), #5763 -Version 1.2.0a4 (2019-03-11) +Version 2.0.0b2 (2022-09-10) ---------------------------- -Fixes: +Bug fixes: -- do not use O_NONBLOCK for special files, like FIFOs, block and char devices - when using --read-special. fixes backing up FIFOs. fixes to test. #4394 -- more LibreSSL build fixes: LibreSSL has HMAC_CTX_free and HMAC_CTX_new +- xattrs / extended stat: improve exception handling, #6988 +- fix and refactor replace_placeholders, #6966 New features: -- check: incremental repo check (only checks crc32 for segment entries), #1657 - borg check --repository-only --max-duration SECONDS ... -- delete: timestamp for borg delete --info added, #4359 +- support archive timestamps with utc offsets, adapt them when using + borg transfer to transfer from borg 1.x repos (append +00:00 for UTC). +- create/recreate/import-tar --timestamp: accept giving timezone via + its utc offset. defaults to local timezone, if no utc offset is given. Other changes: -- redo stale lock handling, #3986 - drop BORG_HOSTNAME_IS_UNIQUE (please use BORG_HOST_ID if needed). - borg now always assumes it has a unique host id - either automatically - from fqdn plus uuid.getnode() or overridden via BORG_HOST_ID. -- docs: - - - added Alpine Linux to distribution list - - elaborate on append-only mode docs -- vagrant: +- chunks: have separate encrypted metadata (ctype, clevel, csize, size) - - darwin: new 10.12 box - - freebsd: new 12.0 box - - openbsd: new 6.4 box - - misc. updates / fixes + chunk = enc_meta_len16 + encrypted(msgpacked(meta)) + encrypted(compressed(data)). + this breaks repo format compatibility, you need to create fresh repos! +- repository api: flags support, #6982 +- OpenBSD only - statically link OpenSSL, #6474. + Avoid conflicting with shared libcrypto from the base OS pulled in via dependencies. +- restructured source code +- update diagrams to odg format, #6928 -Version 1.2.0a3 (2019-02-26) +Version 2.0.0b1 (2022-08-08) ---------------------------- -Fixes: - -- LibreSSL build fixes, #4403 -- dummy ACL/xattr code fixes (used by OpenBSD and others), #4403 -- create: fix openat/statat issues for root directory, #4405 - - -Version 1.2.0a2 and earlier (2019-02-24) ----------------------------------------- - New features: -- compact: "borg compact" needs to be used to free repository space by - compacting the segments (reading sparse segments, rewriting still needed - data to new segments, deleting the sparse segments). - Borg < 1.2 invoked compaction automatically at the end of each repository - writing command. - Borg >= 1.2 does not do that any more to give better speed, more control, - more segment file stability (== less stuff moving to newer segments) and - more robustness. - See the docs about "borg compact" for more details. -- "borg compact --cleanup-commits" is to cleanup the tons of 17byte long - commit-only segment files caused by borg 1.1.x issue #2850. - Invoke this once after upgrading (the server side) borg to 1.2. - Compaction now automatically removes unneeded commit-only segment files. -- prune: Show which rule was applied to keep archive, #2886 -- add fixed blocksize chunker (see --chunker-params docs), #1086 +- massively increase archive metadata stream size limit, #1473. + currently rather testing the code, scalability will improve later, see #6945. +- rcreate --copy-crypt-key: copy crypt_key from key of other repo, #6710. + default: create new, random authenticated encryption key. +- prune/delete --checkpoint-interval=1800 and ctrl-c/SIGINT support, #6284 Fixes: -- avoid stale filehandle issues, #3265 -- use more FDs, avoid race conditions on active fs, #906, #908, #1038 -- add O_NOFOLLOW to base flags, #908 -- compact: - - - require >10% freeable space in a segment, #2985 - - repository compaction now automatically removes unneeded 17byte - commit-only segments, #2850 -- make swidth available on all posix platforms, #2667 +- ctrl-c must not kill important subprocesses, #6912 +- transfer: check whether ID hash method and chunker secret are same. + add PlaintextKey and AuthenticatedKey support to uses_same_id_hash function. +- check: try harder to create the key, #5719 +- SaveFile: use a custom mkstemp with mode support, #6933, #6400 +- make setuptools happy, #6874 +- fix misc. compiler warnings +- list: fix {flags:} formatting, #6081 Other changes: -- repository: better speed and less stuff moving around by using separate - segment files for manifest DELETEs and PUTs, #3947 -- use pyinstaller v3.3.1 to build binaries -- update bundled zstd code to 1.3.8, #4210 -- update bundled lz4 code to 1.8.3, #4209 -- msgpack: - - - switch to recent "msgpack" pypi pkg name, #3890 - - wrap msgpack to avoid future compat complications, #3632, #2738 - - support msgpack 0.6.0 and 0.6.1, #4220, #4308 - -- llfuse: modernize / simplify llfuse version requirements -- code refactorings / internal improvements: - - - include size/csize/nfiles[_parts] stats into archive, #3241 - - calc_stats: use archive stats metadata, if available - - crypto: refactored crypto to use an AEAD style API - - crypto: new AES-OCB, CHACHA20-POLY1305 - - create: use less syscalls by not using a python file obj, #906, #3962 - - diff: refactor the diff functionality to new ItemDiff class, #2475 - - archive: create FilesystemObjectProcessors class - - helpers: make a package, split into smaller modules - - xattrs: move to platform package, use cython instead ctypes, #2495 - - xattrs/acls/bsdflags: misc. code/api optimizations - - FUSE: separate creation of filesystem from implementation of llfuse funcs, #3042 - - FUSE: use unpacker.tell() instead of deprecated write_bytes, #3899 - - setup.py: move build_man / build_usage code to setup_docs.py - - setup.py: update to use a newer Cython/setuptools API for compiling .pyx -> .c, #3788 - - use python 3.5's os.scandir / os.set_blocking - - multithreading preparations (not used yet): - - - item.to_optr(), Item.from_optr() - - fix chunker holding the GIL during blocking I/O - - C code portability / basic MSC compatibility, #4147, #2677 -- testing: - - - vagrant: new VMs for linux/bsd/darwin, most with OpenSSL 1.1 and py36 - - - -Version 1.1.17 (2021-07-12) ---------------------------- +- new crypto does not need to call ._assert_id(), update code and docs. + https://github.com/borgbackup/borg/pull/6463#discussion_r925436156 +- check: --verify-data does not need to decompress with new crypto modes +- Key: crypt_key instead of enc_key + enc_hmac_key, #6611 +- misc. docs updates and improvements +- CI: test on macOS 12 without fuse / fuse tests +- repository: add debug logging for issue #6687 +- _version.py: remove trailing blank, add LF at EOF (make pep8 checker happy) -Compatibility notes: - -- When upgrading from borg 1.0.x to 1.1.x, please note: - - - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. - - borg upgrade: you do not need to and you also should not run it. - - borg might ask some security-related questions once after upgrading. - You can answer them either manually or via environment variable. - One known case is if you use unencrypted repositories, then it will ask - about a unknown unencrypted repository one time. - - your first backup with 1.1.x might be significantly slower (it might - completely read, chunk, hash a lot files) - this is due to the - --files-cache mode change (and happens every time you change mode). - You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible - mode (but that is less safe for detecting changed files than the default). - See the --files-cache docs for details. -- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). - If WSL still has a problem with sync_file_range, you need to set - BORG_WORKAROUNDS=basesyncfile in the borg process environment to - work around the WSL issue. -- 1.1.14 changes return codes due to a bug fix: - In case you have scripts expecting rc == 2 for a signal exit, you need to - update them to check for >= 128 (as documented since long). -- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now. -- 1.1.17 install_requires the "packaging" pypi package now. - -Fixes: -- pyinstaller dir-mode: fix pyi detection / LIBPATH treatment, #5897 -- handle crash due to kill stale lock race, #5828 -- fix BORG_CACHE_DIR crashing borg if empty, #5216 -- create --dry-run: fix display of kept tagfile, #5834 -- fix missing parameter in "did not consistently fail" msg, #5822 -- missing / healed chunks: always tell chunk ID, #5704 -- benchmark: make sure cleanup happens even on exceptions, #5630 +Version 2.0.0a4 (2022-07-17) +---------------------------- New features: -- implement BORG_SELFTEST env variable, #5871. - this can be used to accelerate borg startup a bit. not recommended for - normal usage, but borg mass hosters with a lot of borg invocations can - save some resources with this. on my laptop, this saved ~100ms cpu time - (sys+user) per borg command invocation. -- implement BORG_LIBC env variable to give the libc filename, #5870. - you can use this if a borg does not find your libc. -- check: add progress indicator for archive check. -- allow --files-cache=size (not recommended, make sure you know what you do) +- recreate: consider level for recompression, #6698, #3622 Other changes: -- Python 3.10 now officially supported! - we test on py310-dev on github CI since a while and now also on the vagrant - machines, so it should work ok. -- github CI: test on py310 (again) -- get rid of distutils, use packaging and setuptools. - distutils is deprecated and gives warnings on py 3.10. -- setup.py: rename "clean" to "clean2" to avoid shadowing the "clean" command. -- remove libc filename fallback for the BSDs (there is no "usual" name) -- cleanup flake8 checks, fix some pep8 violations. -- docs building: replace deprecated function ".add_stylesheet()" for Sphinx 4 compatibility -- docs: - - - add a hint on sleeping computer and ssh connections, #5301 - - update the documentation on hacked backup client, #5480 - - improve docs/FAQ about append-only remote repos, #5497 - - complement the documentation for pattern files and exclude files, #5520 - - "filename with spaces" example added to exclude file, #5236 - note: no whitespace escaping needed, processed by borg. - - add info on renaming repositories, #5240 - - clarify borg check --verify-data, #5808 - - add notice about defective hardware to check documentation, #5753 - - add paragraph added in #5855 to utility documentation source - - add missing leading slashes in help patterns, #5857 - - clarify "you will need key and passphrase" borg init warning, #4622 - - pull mode: add some warnings, #5827 - - mention tar --compare (compare archive to fs files), #5880 - - fix typos, backport of #5597 -- vagrant: +- stop using libdeflate +- CI: add mypy (if we add type hints, it can do type checking) +- big changes to the source code: - - add py3.7.11 for binary build, also add 3.10-dev. - - use latest Cython 0.29.23 for py310 compat fixes. - - more RAM for openindiana upgrade plan resolver, it just hangs (swaps?) if - there is too little RAM. - - fix install_pyenv to adapt to recent changes in pyenv (same as in master now). - - use generic/netbsd9 box, copied from master branch. + - split up archiver module, transform it into a package + - use Black for automated code formatting + - remove some legacy code + - adapt/fix code for mypy +- use language_level = 3str for cython (this will be the default in cython 3) +- docs: document HardLinkManager and hlid, #2388 -Version 1.1.16 (2021-03-23) ---------------------------- +Version 2.0.0a3 (2022-07-04) +---------------------------- Fixes: -- setup.py: add special openssl prefix for Apple M1 compatibility -- do not recurse into duplicate roots, #5603 -- remove empty shadowed_segments lists, #5275, #5614 -- fix libpython load error when borg fat binary / dir-based binary is invoked - via a symlink by upgrading pyinstaller to v4.2, #5688 -- config: accept non-int value (like 500M or 100G) for max_segment_size or - storage_quota, #5639. - please note: when setting a non-int value for this in a repo config, - using the repo will require borg >= 1.1.16. +- check repo version, accept old repos only for --other-repo (e.g. rcreate/transfer). + v2 is the default repo version for borg 2.0. v1 repos must only be used in a + read-only way, e.g. for --other-repo=V1_REPO with borg init and borg transfer! New features: -- bundled msgpack: drop support for old buffer protocol to support Python 3.10 -- verbose files cache logging via --debug-topic=files_cache, #5659. - Use this if you suspect that borg does not detect unmodified files as expected. -- create/extract: add --noxattrs and --noacls option, #3955. - when given with borg create, borg will not get xattrs / ACLs from input files - (and thus, it will not archive xattrs / ACLs). when given with borg extract, - borg will not read xattrs / ACLs from archive and will not set xattrs / ACLs - on extracted files. -- diff: add --json-lines option, #3765 -- check: debug log segment filename -- borg debug dump-hints +- transfer: --upgrader=NoOp is the default. + This is to support general-purpose transfer of archives between related borg2 + repos. +- transfer: --upgrader=From12To20 must be used to transfer (and convert) archives + from borg 1.2 repos to borg 2.0 repos. Other changes: -- Tab completion support for additional archives for 'borg delete' -- repository: deduplicate code of put and delete, no functional change -- tests: fix result order issue (sporadic test failure on openindiana) -- vagrant: - - - upgrade pyinstaller to v4.2, #5671 - - avoid grub-install asking interactively for device - - remove the xenial box - - update freebsd box to 12.1 -- docs: - - - update macOS install instructions, #5677 - - use macFUSE (not osxfuse) for Apple M1 compatibility - - update docs for dev environment installation instructions, #5643 - - fix grammar in faq - - recomend running tests only on installed versions for setup - - add link back to git-installation - - remove /var/cache exclusion in example commands, #5625. - This is generally a poor idea and shouldn't be promoted through examples. - - add repology.org badge with current packaging status - - explain hash collision - - add unsafe workaround to use an old repo copy, #5722 - +- removed some deprecated options +- removed -P (aka --prefix) option, #6806. The option -a (aka --glob-archives) + can be used for same purpose and is more powerful, e.g.: -a 'PREFIX*' +- rcreate: always use argon2 kdf for new repos, #6820 +- rcreate: remove legacy encryption modes for new repos, #6490 -Version 1.1.15 (2020-12-25) ---------------------------- - -Fixes: -- extract: +Version 2.0.0a2 (2022-06-26) +---------------------------- - - improve exception handling when setting xattrs, #5092. - - emit a warning message giving the path, xattr key and error message. - - continue trying to restore other xattrs and bsdflags of the same file - after an exception with xattr-setting happened. -- export-tar: +Changes: + +- split repo and archive name into separate args, #948 + + - use -r or --repo or BORG_REPO env var to give the repository + - use --other-repo or BORG_OTHER_REPO to give another repo (e.g. borg transfer) + - use positional argument for archive name or `-a ARCH_GLOB` +- remove support for scp-style repo specification, use ssh://... +- simplify stats output: repo ops -> repo stats, archive ops -> archive stats +- repository index: add payload size (==csize) and flags to NSIndex entries +- repository index: set/query flags, iteration over flagged items (NSIndex) +- repository: sync write file in get_fd +- stats: deduplicated size now, was deduplicated compressed size in borg 1.x +- remove csize support at most places in the code (chunks index, stats, get_size, + Item.chunks) +- replace problematic/ugly hardlink_master approach of borg 1.x by: + + - symmetric hlid (all hardlinks pointing to same inode have same hlid) + - all archived hardlinked regular files have a chunks list +- borg rcreate --other-repo=OTHER_REPO: reuse key material from OTHER_REPO, #6554. + This is useful if you want to use borg transfer to transfer archives from an + existing borg 1.1/1.2 repo. If the chunker secret and the id key and algorithm + stay the same, the deduplication will also work between past and future backups. +- borg transfer: + + - efficiently copy archives from a borg 1.1/1.2 repo to a new repo. + uses deduplication and does not decompress/recompress file content data. + - does some cleanups / fixes / conversions: + + - disallow None value for .user/group/chunks/chunks_healthy + - cleanup msgpack related str/bytes mess, use new msgpack spec, #968 + - obfuscation: fix byte order for size, #6701 + - compression: use the 2 bytes for type and level, #6698 + - use version 2 for new archives + - convert timestamps int/bigint -> msgpack.Timestamp, see #2323 + - all hardlinks have chunks, maybe chunks_healthy, hlid + - remove the zlib type bytes hack + - make sure items with chunks have precomputed size + - removes the csize element from the tuples in the Item.chunks list + - clean item of attic 0.13 'acl' bug remnants +- crypto: see 1.3.0a1 log entry +- removed "borg upgrade" command (not needed any more) +- compact: removed --cleanup-commits option +- docs: fixed quickstart and usage docs with new cli command syntax +- docs: removed the parts talking about potential AES-CTR mode issues + (we will not use that any more). + + +Version 1.3.0a1 (2022-04-15) +---------------------------- - - set tar format to GNU_FORMAT explicitly, #5274 - - fix memory leak with ssh: remote repository, #5568 - - fix potential memory leak with ssh: remote repository with partial extraction -- create: fix --dry-run and --stats coexistence, #5415 -- use --timestamp for {utcnow} and {now} if given, #5189 +Although this was released as 1.3.0a1, it can be also seen as 2.0.0a1 as it was +later decided to do breaking changes and thus the major release number had to +be increased (thus, there will not be a 1.3.0 release, but 2.0.0). New features: -- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333 -- allow appending the files cache filename with BORG_FILES_CACHE_SUFFIX env var - -Other changes: - -- drop python 3.4 support, minimum requirement is 3.5 now. -- enable using libxxhash instead of bundled xxh64 code -- update llfuse requirements (1.3.8) -- set cython language_level in some files to fix warnings -- allow EIO with warning when trying to hardlink -- PropDict: fail early if internal_dict is not a dict -- update shell completions -- tests / CI - - - add a test for the hashindex corruption bug, #5531 #4829 - - fix spurious failure in test_cache_files, #5438 - - added a github ci workflow - - reduce testing on travis, no macOS, no py3x-dev, #5467 - - travis: use newer dists, native py on dist -- vagrant: - - - remove jessie and trusty boxes, #5348 #5383 - - pyinstaller 4.0, build on py379 - - binary build on stretch64, #5348 - - remove easy_install based pip installation -- docs: - - - clarify '--one-file-system' for btrfs, #5391 - - add example for excluding content using the --pattern cmd line arg - - complement the documentation for pattern files and exclude files, #5524 - - made ansible playbook more generic, use package instead of pacman. also - change state from "latest" to "present". - - complete documentation on append-only remote repos, #5497 - - internals: rather talk about target size than statistics, #5336 - - new compression algorithm policy, #1633 #5505 - - faq: add a hint on sleeping computer, #5301 - - note requirements for full disk access on macOS Catalina, #5303 - - fix/improve description of borg upgrade hardlink usage, #5518 -- modernize 1.1 code: - - - drop code/workarounds only needed to support Python 3.4 - - remove workaround for pre-release py37 argparse bug - - removed some outdated comments/docstrings - - requirements: remove some restrictions, lock on current versions - - -Version 1.1.14 (2020-10-07) ---------------------------- - -Fixes: - -- check --repair: fix potential data loss when interrupting it, #5325 -- exit with 128 + signal number (as documented) when borg is killed by a signal, #5161 -- fix hardlinked CACHEDIR.TAG processing, #4911 -- create --read-special: .part files also should be regular files, #5217 -- llfuse dependency: choose least broken 1.3.6/1.3.7. - 1.3.6 is broken on python 3.9, 1.3.7 is broken on FreeBSD. - -Other changes: - -- upgrade bundled xxhash to 0.7.4 -- self test: if it fails, also point to OS and hardware, #5334 -- pyinstaller: compute basepath from spec file location -- prettier error message when archive gets too big, #5307 -- check/recreate are not "experimental" any more (but still potentially dangerous): - - - recreate: remove extra confirmation - - rephrase some warnings, update docs, #5164 -- shell completions: - - - misc. updates / fixes - - support repositories in fish tab completion, #5256 - - complete $BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING - - rewrite zsh completion: - - - completion for almost all optional and positional arguments - - completion for Borg environment variables (parameters) -- use "allow/deny list" instead of "white/black list" wording -- declare "allow_cache_wipe" marker in setup.cfg to avoid pytest warning -- vagrant / tests: - - - misc. fixes / updates - - use python 3.5.10 for binary build - - build directory-based binaries additionally to the single file binaries - - add libffi-dev, required to build python - - use cryptography<3.0, more recent versions break the jessie box - - test on python 3.9 - - do brew update with /dev/null redirect to avoid "too much log output" on travis-ci -- docs: - - - add ssh-agent pull backup method docs, #5288 - - how to approach borg speed issues, #5371 - - mention double --force in prune docs - - update Homebrew install instructions, #5185 - - better description of how cache and rebuilds of it work - - point to borg create --list item flags in recreate usage, #5165 - - add security faq explaining AES-CTR crypto issues, #5254 - - add a note to create from stdin regarding files cache, #5180 - - fix borg.1 manpage generation regression, #5211 - - clarify how exclude options work in recreate, #5193 - - add section for retired contributors - - hint about not misusing private email addresses of contributors for borg support - - -Version 1.1.13 (2020-06-06) ---------------------------- - -Compatibility notes: - -- When upgrading from borg 1.0.x to 1.1.x, please note: - - - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. - - borg upgrade: you do not need to and you also should not run it. - - borg might ask some security-related questions once after upgrading. - You can answer them either manually or via environment variable. - One known case is if you use unencrypted repositories, then it will ask - about a unknown unencrypted repository one time. - - your first backup with 1.1.x might be significantly slower (it might - completely read, chunk, hash a lot files) - this is due to the - --files-cache mode change (and happens every time you change mode). - You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible - mode (but that is less safe for detecting changed files than the default). - See the --files-cache docs for details. -- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). - If WSL still has a problem with sync_file_range, you need to set - BORG_WORKAROUNDS=basesyncfile in the borg process environment to - work around the WSL issue. - -Fixes: - -- rebuilt using a current Cython version, compatible with python 3.8, #5214 - - -Version 1.1.12 (2020-06-06) ---------------------------- - -Fixes: - -- fix preload-related memory leak, #5202. -- mount / borgfs (FUSE filesystem): +- init: new --encryption=(repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305) - - fix FUSE low linear read speed on large files, #5067 - - fix crash on old llfuse without birthtime attrs, #5064 - accidentally - we required llfuse >= 1.3. Now also old llfuse works again. - - set f_namemax in statfs result, #2684 -- update precedence of env vars to set config and cache paths, #4894 -- correctly calculate compression ratio, taking header size into account, too + - New, better, faster crypto (see encryption-aead diagram in the docs), #6463. + - New AEAD cipher suites: AES-OCB and CHACHA20-POLY1305. + - Session keys are derived via HKDF from random session id and master key. + - Nonces/MessageIVs are counters starting from 0 for each session. + - AAD: chunk id, key type, messageIV, sessionID are now authenticated also. + - Solves the potential AES-CTR mode counter management issues of the legacy crypto. +- init: --key-algorithm=argon2 (new default KDF, older pbkdf2 also still available) -New features: + borg key change-passphrase / change-location keeps the key algorithm unchanged. +- key change-algorithm: to upgrade existing keys to argon2 or downgrade to pbkdf2. -- --bypass-lock option to bypass locking with read-only repositories + We recommend you to upgrade unless you have to keep the key compatible with older versions of borg. +- key change-location: usable for repokey <-> keyfile location change +- benchmark cpu: display benchmarks of cpu bound stuff +- export-tar: new --tar-format=PAX (default: GNU) +- import-tar/export-tar: can use PAX format for ctime and atime support +- import-tar/export-tar: --tar-format=BORG: roundtrip ALL item metadata, #5830 +- repository: create and use version 2 repos only for now +- repository: implement PUT2: header crc32, overall xxh64, #1704 Other changes: -- upgrade bundled zstd to 1.4.5 -- travis: adding comments and explanations to Travis config / install script, - improve macOS builds. -- tests: test_delete_force: avoid sporadic test setup issues, #5196 -- misc. vagrant fixes -- the binary for macOS is now built on macOS 10.12 -- the binaries for Linux are now built on Debian 8 "Jessie", #3761 -- docs: - - - PlaceholderError not printed as JSON, #4073 - - "How important is Borg config?", #4941 - - make Sphinx warnings break docs build, #4587 - - some markup / warning fixes - - add "updating borgbackup.org/releases" to release checklist, #4999 - - add "rendering docs" to release checklist, #5000 - - clarify borg init's encryption modes - - add note about patterns and stored paths, #4160 - - add upgrade of tools to pip installation how-to - - document one cause of orphaned chunks in check command, #2295 - - linked recommended restrictions to ssh public keys on borg servers in faq, #4946 - - -Version 1.1.11 (2020-03-08) ---------------------------- - -Compatibility notes: - -- When upgrading from borg 1.0.x to 1.1.x, please note: - - - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. - - borg upgrade: you do not need to and you also should not run it. - - borg might ask some security-related questions once after upgrading. - You can answer them either manually or via environment variable. - One known case is if you use unencrypted repositories, then it will ask - about a unknown unencrypted repository one time. - - your first backup with 1.1.x might be significantly slower (it might - completely read, chunk, hash a lot files) - this is due to the - --files-cache mode change (and happens every time you change mode). - You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible - mode (but that is less safe for detecting changed files than the default). - See the --files-cache docs for details. -- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). - If WSL still has a problem with sync_file_range, you need to set - BORG_WORKAROUNDS=basesyncfile in the borg process environment to - work around the WSL issue. - -Fixes: - -- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829. - Please read and follow the more detailed notes close to the top of this document. -- upgrade bundled xxhash to 0.7.3, #4891. - 0.7.2 is the minimum requirement for correct operations on ARMv6 in non-fixup - mode, where unaligned memory accesses cause bus errors. - 0.7.3 adds some speedups and libxxhash 0.7.3 even has a pkg-config file now. -- upgrade bundled lz4 to 1.9.2 -- upgrade bundled zstd to 1.4.4 -- fix crash when upgrading erroneous hints file, #4922 -- extract: - - - fix KeyError for "partial" extraction, #4607 - - fix "partial" extract for hardlinked contentless file types, #4725 - - fix preloading for old (0.xx) remote servers, #4652 - - fix confusing output of borg extract --list --strip-components, #4934 -- delete: after double-force delete, warn about necessary repair, #4704 -- create: give invalid repo error msg if repo config not found, #4411 -- mount: fix FUSE mount missing st_birthtime, #4763 #4767 -- check: do not stumble over invalid item key, #4845 -- info: if the archive doesn't exist, print a pretty message, #4793 -- SecurityManager.known(): check all files, #4614 -- Repository.open: use stat() to check for repo dir, #4695 -- Repository.check_can_create_repository: use stat() to check, #4695 -- fix invalid archive error message -- fix optional/non-optional location arg, #4541 -- commit-time free space calc: ignore bad compact map entries, #4796 -- ignore EACCES (errno 13) when hardlinking the old config, #4730 -- --prefix / -P: fix processing, avoid argparse issue, #4769 - -New features: - -- enable placeholder usage in all extra archive arguments -- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710 -- recreate: support --timestamp option, #4745 -- support platforms without os.link (e.g. Android with Termux), #4901. - if we don't have os.link, we just extract another copy instead of making a hardlink. -- support linux platforms without sync_file_range (e.g. Android 7 with Termux), #4905 - -Other: - -- ignore --stats when given with --dry-run, but continue, #4373 -- add some ProgressIndicator msgids to code / fix docs, #4935 -- elaborate on "Calculating size" message -- argparser: always use REPOSITORY in metavar, also use more consistent help phrasing. -- check: improve error output for matching index size, see #4829 -- docs: - - - changelog: add advisory about hashindex_set bug #4829 - - better describe BORG_SECURITY_DIR, BORG_CACHE_DIR, #4919 - - infos about cache security assumptions, #4900 - - add FAQ describing difference between a local repo vs. repo on a server. - - document how to test exclusion patterns without performing an actual backup - - timestamps in the files cache are now usually ctime, #4583 - - fix bad reference to borg compact (does not exist in 1.1), #4660 - - create: borg 1.1 is not future any more - - extract: document limitation "needs empty destination", #4598 - - how to supply a passphrase, use crypto devices, #4549 - - fix osxfuse github link in installation docs - - add example of exclude-norecurse rule in help patterns - - update macOS Brew link - - add note about software for automating backups, #4581 - - AUTHORS: mention copyright+license for bundled msgpack - - fix various code blocks in the docs, #4708 - - updated docs to cover use of temp directory on remote, #4545 - - add restore docs, #4670 - - add a pull backup / push restore how-to, #1552 - - add FAQ how to retain original paths, #4532 - - explain difference between --exclude and --pattern, #4118 - - add FAQs for SSH connection issues, #3866 - - improve password FAQ, #4591 - - reiterate that 'file cache names are absolute' in FAQ -- tests: - - - cope with ANY error when importing pytest into borg.testsuite, #4652 - - fix broken test that relied on improper zlib assumptions - - test_fuse: filter out selinux xattrs, #4574 -- travis / vagrant: - - - misc python versions removed / changed (due to openssl 1.1 compatibility) - or added (3.7 and 3.8, for better borg compatibility testing) - - binary building is on python 3.5.9 now -- vagrant: - - - add new boxes: ubuntu 18.04 and 20.04, debian 10 - - update boxes: openindiana, darwin, netbsd - - remove old boxes: centos 6 - - darwin: updated osxfuse to 3.10.4 - - use debian/ubuntu pip/virtualenv packages - - rather use python 3.6.2 than 3.6.0, fixes coverage/sqlite3 issue - - use requirements.d/development.lock.txt to avoid compat issues -- travis: - - - darwin: backport some install code / order from master - - remove deprecated keyword "sudo" from travis config - - allow osx builds to fail, #4955 - this is due to travis-ci frequently being so slow that the OS X builds - just fail because they exceed 50 minutes and get killed by travis. - - -Version 1.1.10 (2019-05-16) ---------------------------- - -Fixes: - -- extract: hang on partial extraction with ssh: repo, when hardlink master - is not matched/extracted and borg hangs on related slave hardlink, #4350 -- lrucache: regularly remove old FDs, #4427 -- avoid stale filehandle issues, #3265 -- freebsd: make xattr platform code api compatible with linux, #3952 -- use whitelist approach for borg serve, #4097 -- borg command shall terminate with rc 2 for ImportErrors, #4424 -- create: only run stat_simple_attrs() once, this increases - backup with lots of unchanged files performance by ~ 5%. -- prune: fix incorrect borg prune --stats output with --dry-run, #4373 -- key export: emit user-friendly error if repo key is exported to a directory, - #4348 - -New features: - -- bundle latest supported msgpack-python release (0.5.6), remove msgpack-python - from setup.py install_requires - by default we use the bundled code now. - optionally, we still support using an external msgpack (see hints in - setup.py), but this requires solid requirements management within - distributions and is not recommended. - borgbackup will break if you upgrade msgpack to an unsupported version. -- display msgpack version as part of sysinfo (e.g. in tracebacks) -- timestamp for borg delete --info added, #4359 -- enable placeholder usage in --comment and --glob-archives, #4559, #4495 - -Other: - -- serve: do not check python/libc for borg serve, #4483 -- shell completions: borg diff second archive -- release scripts: signing binaries with Qubes OS support -- testing: - - - vagrant: upgrade openbsd box to 6.4 - - travis-ci: lock test env to py 3.4 compatible versions, #4343 - - get rid of confusing coverage warning, #2069 - - rename test_mount_hardlinks to test_fuse_mount_hardlinks, - so both can be excluded by "not test_fuse". - - pure-py msgpack warning shall not make a lot of tests fail, #4558 -- docs: - - - add "SSH Configuration" section to "borg serve", #3988, #636, #4485 - - README: new URL for funding options - - add a sample logging.conf in docs/misc, #4380 - - elaborate on append-only mode docs, #3504 - - installation: added Alpine Linux to distribution list, #4415 - - usage.html: only modify window.location when redirecting, #4133 - - add msgpack license to docs/3rd_party/msgpack -- vagrant / binary builds: - - - use python 3.5.7 for builds - - use osxfuse 3.8.3 - - -Version 1.1.9 (2019-02-10) --------------------------- - -Compatibility notes: - -- When upgrading from borg 1.0.x to 1.1.x, please note: - - - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. - - borg upgrade: you do not need to and you also should not run it. - - borg might ask some security-related questions once after upgrading. - You can answer them either manually or via environment variable. - One known case is if you use unencrypted repositories, then it will ask - about a unknown unencrypted repository one time. - - your first backup with 1.1.x might be significantly slower (it might - completely read, chunk, hash a lot files) - this is due to the - --files-cache mode change (and happens every time you change mode). - You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible - mode (but that is less safe for detecting changed files than the default). - See the --files-cache docs for details. - -Fixes: - -- security fix: configure FUSE with "default_permissions", #3903 - "default_permissions" is now enforced by borg by default to let the - kernel check uid/gid/mode based permissions. - "ignore_permissions" can be given to not enforce "default_permissions". -- make "hostname" short, even on misconfigured systems, #4262 -- fix free space calculation on macOS (and others?), #4289 -- config: quit with error message when no key is provided, #4223 -- recover_segment: handle too small segment files correctly, #4272 -- correctly release memoryview, #4243 -- avoid diaper pattern in configparser by opening files, #4263 -- add "# cython: language_level=3" directive to .pyx files, #4214 -- info: consider part files for "This archive" stats, #3522 -- work around Microsoft WSL issue #645 (sync_file_range), #1961 - -New features: - -- add --rsh command line option to complement BORG_RSH env var, #1701 -- init: --make-parent-dirs parent1/parent2/repo_dir, #4235 - -Other: - -- add archive name to check --repair output, #3447 -- check for unsupported msgpack versions -- shell completions: - - - new shell completions for borg 1.1.9 - - more complete shell completions for borg mount -o - - added shell completions for borg help - - option arguments for zsh tab completion -- docs: - - - add FAQ regarding free disk space check, #3905 - - update BORG_PASSCOMMAND example and clarify variable expansion, #4249 - - FAQ regarding change of compression settings, #4222 - - add note about BSD flags to changelog, #4246 - - improve logging in example automation script - - add note about files changing during backup, #4081 - - work around the backslash issue, #4280 - - update release workflow using twine (docs, scripts), #4213 - - add warnings on repository copies to avoid future problems, #4272 -- tests: - - - fix the homebrew 1.9 issues on travis-ci, #4254 - - fix duplicate test method name, #4311 - - -Version 1.1.8 (2018-12-09) --------------------------- - -Fixes: - -- enforce storage quota if set by serve-command, #4093 -- invalid locations: give err msg containing parsed location, #4179 -- list repo: add placeholders for hostname and username, #4130 -- on linux, symlinks can't have ACLs, so don't try to set any, #4044 - -New features: - -- create: added PATH::archive output on INFO log level -- read a passphrase from a file descriptor specified in the - BORG_PASSPHRASE_FD environment variable. - -Other: - -- docs: - - - option --format is required for some expensive-to-compute values for json - - borg list by default does not compute expensive values except when - they are needed. whether they are needed is determined by the format, - in standard mode as well as in --json mode. - - tell that our binaries are x86/x64 amd/intel, bauerj has ARM - - fixed wrong archive name pattern in CRUD benchmark help - - fixed link to cachedir spec in docs, #4140 -- tests: - - - stop using fakeroot on travis, avoids sporadic EISDIR errors, #2482 - - xattr key names must start with "user." on linux - - fix code so flake8 3.6 does not complain - - explicitly convert environment variable to str, #4136 - - fix DeprecationWarning: Flags not at the start of the expression, #4137 - - support pytest4, #4172 -- vagrant: - - - use python 3.5.6 for builds - - -Version 1.1.7 (2018-08-11) --------------------------- - -Compatibility notes: - -- added support for Python 3.7 - -Fixes: - -- cache lock: use lock_wait everywhere to fix infinite wait, see #3968 -- don't archive tagged dir when recursing an excluded dir, #3991 -- py37 argparse: work around bad default in py 3.7.0a/b/rc, #3996 -- py37 remove loggerDict.clear() from tearDown method, #3805 -- some fixes for bugs which likely did not result in problems in practice: - - - fixed logic bug in platform module API version check - - fixed xattr/acl function prototypes, added missing ones - -New features: - -- init: add warning to store both key and passphrase at safe place(s) -- BORG_HOST_ID env var to work around all-zero MAC address issue, #3985 -- borg debug dump-repo-objs --ghost (dump everything from segment files, - including deleted or superseded objects or commit tags) -- borg debug search-repo-objs (search in repo objects for hex bytes or strings) - -Other changes: - -- add Python 3.7 support -- updated shell completions -- call socket.gethostname only once -- locking: better logging, add some asserts -- borg debug dump-repo-objs: - - - filename layout improvements - - use repository.scan() to get on-disk order -- docs: - - - update installation instructions for macOS - - added instructions to install fuse via homebrew - - improve diff docs - - added note that checkpoints inside files requires 1.1+ - - add link to tempfile module - - remove row/column-spanning from docs source, #4000 #3990 -- tests: - - - fetch less data via os.urandom - - add py37 env for tox - - travis: add 3.7, remove 3.6-dev (we test with -dev in master) -- vagrant / binary builds: - - - use osxfuse 3.8.2 - - use own (uptodate) openindiana box - - -Version 1.1.6 (2018-06-11) --------------------------- - -Compatibility notes: - -- 1.1.6 changes: - - - also allow msgpack-python 0.5.6. - -Fixes: - -- fix borg exception handling on ENOSPC error with xattrs, #3808 -- prune: fix/improve overall progress display -- borg config repo ... does not need cache/manifest/key, #3802 -- debug dump-repo-objs should not depend on a manifest obj -- pypi package: - - - include .coveragerc, needed by tox.ini - - fix package long description, #3854 - -New features: - -- mount: add uid, gid, umask mount options -- delete: - - - only commit once, #3823 - - implement --dry-run, #3822 -- check: - - - show progress while rebuilding missing manifest, #3787 - - more --repair output -- borg config --list , #3612 - -Other changes: - -- update msgpack requirement, #3753 -- update bundled zstd to 1.3.4, #3745 -- update bundled lz4 code to 1.8.2, #3870 -- docs: - - - describe what BORG_LIBZSTD_PREFIX does - - fix and deduplicate encryption quickstart docs, #3776 -- vagrant: - - - FUSE for macOS: upgrade 3.7.1 to 3.8.0 - - exclude macOS High Sierra upgrade on the darwin64 machine - - remove borgbackup.egg-info dir in fs_init (after rsync) - - use pyenv-based build/test on jessie32/62 - - use local 32 and 64bit debian jessie boxes - - use "vagrant" as username for new xenial box -- travis OS X: use xcode 8.3 (not broken) - - -Version 1.1.5 (2018-04-01) --------------------------- - -Compatibility notes: - -- 1.1.5 changes: - - - require msgpack-python >= 0.4.6 and < 0.5.0. - 0.5.0+ dropped python 3.4 testing and also caused some other issues because - the python package was renamed to msgpack and emitted some FutureWarning. - -Fixes: - -- create --list: fix that it was never showing M status, #3492 -- create: fix timing for first checkpoint (read files cache early, init - checkpoint timer after that), see #3394 -- extract: set rc=1 when extracting damaged files with all-zero replacement - chunks or with size inconsistencies, #3448 -- diff: consider an empty file as different to a non-existing file, #3688 -- files cache: improve exception handling, #3553 -- ignore exceptions in scandir_inorder() caused by an implicit stat(), - also remove unneeded sort, #3545 -- fixed tab completion problem where a space is always added after path even - when it shouldn't -- build: do .h file content checks in binary mode, fixes build issue for - non-ascii header files on pure-ascii locale platforms, #3544 #3639 -- borgfs: fix patterns/paths processing, #3551 -- config: add some validation, #3566 -- repository config: add validation for max_segment_size, #3592 -- set cache previous_location on load instead of save -- remove platform.uname() call which caused library mismatch issues, #3732 -- add exception handler around deprecated platform.linux_distribution() call -- use same datetime object for {now} and {utcnow}, #3548 - -New features: - -- create: implement --stdin-name, #3533 -- add chunker_params to borg archive info (--json) -- BORG_SHOW_SYSINFO=no to hide system information from exceptions - -Other changes: - -- updated zsh completions for borg 1.1.4 -- files cache related code cleanups -- be more helpful when parsing invalid --pattern values, #3575 -- be more clear in secure-erase warning message, #3591 -- improve getpass user experience, #3689 -- docs build: unicode problem fixed when using a py27-based sphinx -- docs: - - - security: explicitly note what happens OUTSIDE the attack model - - security: add note about combining compression and encryption - - security: describe chunk size / proximity issue, #3687 - - quickstart: add note about permissions, borg@localhost, #3452 - - quickstart: add introduction to repositories & archives, #3620 - - recreate --recompress: add missing metavar, clarify description, #3617 - - improve logging docs, #3549 - - add an example for --pattern usage, #3661 - - clarify path semantics when matching, #3598 - - link to offline documentation from README, #3502 - - add docs on how to verify a signed release with GPG, #3634 - - chunk seed is generated per repository (not: archive) - - better formatting of CPU usage documentation, #3554 - - extend append-only repo rollback docs, #3579 -- tests: - - - fix erroneously skipped zstd compressor tests, #3606 - - skip a test if argparse is broken, #3705 -- vagrant: - - - xenial64 box now uses username 'vagrant', #3707 - - move cleanup steps to fs_init, #3706 - - the boxcutter wheezy boxes are 404, use local ones - - update to Python 3.5.5 (for binary builds) - - -Version 1.1.4 (2017-12-31) --------------------------- - -Compatibility notes: - -- When upgrading from borg 1.0.x to 1.1.x, please note: - - - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. - - borg upgrade: you do not need to and you also should not run it. - - borg might ask some security-related questions once after upgrading. - You can answer them either manually or via environment variable. - One known case is if you use unencrypted repositories, then it will ask - about a unknown unencrypted repository one time. - - your first backup with 1.1.x might be significantly slower (it might - completely read, chunk, hash a lot files) - this is due to the - --files-cache mode change (and happens every time you change mode). - You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible - mode (but that is less safe for detecting changed files than the default). - See the --files-cache docs for details. -- borg 1.1.4 changes: - - - zstd compression is new in borg 1.1.4, older borg can't handle it. - - new minimum requirements for the compression libraries - if the required - versions (header and lib) can't be found at build time, bundled code will - be used: - - - added requirement: libzstd >= 1.3.0 (bundled: 1.3.2) - - updated requirement: liblz4 >= 1.7.0 / r129 (bundled: 1.8.0) - -Fixes: - -- check: data corruption fix: fix for borg check --repair malfunction, #3444. - See the more detailed notes close to the top of this document. -- delete: also delete security dir when deleting a repo, #3427 -- prune: fix building the "borg prune" man page, #3398 -- init: use given --storage-quota for local repo, #3470 -- init: properly quote repo path in output -- fix startup delay with dns-only own fqdn resolving, #3471 - -New features: - -- added zstd compression. try it! -- added placeholder {reverse-fqdn} for fqdn in reverse notation -- added BORG_BASE_DIR environment variable, #3338 - -Other changes: - -- list help topics when invalid topic is requested -- fix lz4 deprecation warning, requires lz4 >= 1.7.0 (r129) -- add parens for C preprocessor macro argument usages (did not cause malfunction) -- exclude broken pytest 3.3.0 release -- updated fish/bash completions -- init: more clear exception messages for borg create, #3465 -- docs: - - - add auto-generated docs for borg config - - don't generate HTML docs page for borgfs, #3404 - - docs update for lz4 b2 zstd changes - - add zstd to compression help, readme, docs - - update requirements and install docs about bundled lz4 and zstd -- refactored build of the compress and crypto.low_level extensions, #3415: - - - move some lib/build related code to setup_{zstd,lz4,b2}.py - - bundle lz4 1.8.0 (requirement: >= 1.7.0 / r129) - - bundle zstd 1.3.2 (requirement: >= 1.3.0) - - blake2 was already bundled - - rename BORG_LZ4_PREFIX env var to BORG_LIBLZ4_PREFIX for better consistency: - we also have BORG_LIBB2_PREFIX and BORG_LIBZSTD_PREFIX now. - - add prefer_system_lib* = True settings to setup.py - by default the build - will prefer a shared library over the bundled code, if library and headers - can be found and meet the minimum requirements. - - -Version 1.1.3 (2017-11-27) --------------------------- - -Fixes: - -- Security Fix for CVE-2017-15914: Incorrect implementation of access controls - allows remote users to override repository restrictions in Borg servers. - A user able to access a remote Borg SSH server is able to circumvent access - controls post-authentication. - Affected releases: 1.1.0, 1.1.1, 1.1.2. Releases 1.0.x are NOT affected. -- crc32: deal with unaligned buffer, add tests - this broke borg on older ARM - CPUs that can not deal with unaligned 32bit memory accesses and raise a bus - error in such cases. the fix might also improve performance on some CPUs as - all 32bit memory accesses by the crc32 code are properly aligned now. #3317 -- mount: fixed support of --consider-part-files and do not show .borg_part_N - files by default in the mounted FUSE filesystem. #3347 -- fixed cache/repo timestamp inconsistency message, highlight that information - is obtained from security dir (deleting the cache will not bypass this error - in case the user knows this is a legitimate repo). -- borgfs: don't show sub-command in borgfs help, #3287 -- create: show an error when --dry-run and --stats are used together, #3298 - -New features: - -- mount: added exclusion group options and paths, #2138 - - Reused some code to support similar options/paths as borg extract offers - - making good use of these to only mount a smaller subset of dirs/files can - speed up mounting a lot and also will consume way less memory. - - borg mount [options] repo_or_archive mountpoint path [paths...] - - paths: you can just give some "root paths" (like for borg extract) to - only partially populate the FUSE filesystem. - - new options: --exclude[-from], --pattern[s-from], --strip-components -- create/extract: support st_birthtime on platforms supporting it, #3272 -- add "borg config" command for querying/setting/deleting config values, #3304 - -Other changes: - -- clean up and simplify packaging (only package committed files, do not install - .c/.h/.pyx files) -- docs: - - - point out tuning options for borg create, #3239 - - add instructions for using ntfsclone, zerofree, #81 - - move image backup-related FAQ entries to a new page - - clarify key aliases for borg list --format, #3111 - - mention break-lock in checkpointing FAQ entry, #3328 - - document sshfs rename workaround, #3315 - - add FAQ about removing files from existing archives - - add FAQ about different prune policies - - usage and man page for borgfs, #3216 - - clarify create --stats duration vs. wall time, #3301 - - clarify encrypted key format for borg key export, #3296 - - update release checklist about security fixes - - document good and problematic option placements, fix examples, #3356 - - add note about using --nobsdflags to avoid speed penalty related to - bsdflags, #3239 - - move most of support section to www.borgbackup.org - - -Version 1.1.2 (2017-11-05) --------------------------- - -Fixes: - -- fix KeyError crash when talking to borg server < 1.0.7, #3244 -- extract: set bsdflags last (include immutable flag), #3263 -- create: don't do stat() call on excluded-norecurse directory, fix exception - handling for stat() call, #3209 -- create --stats: do not count data volume twice when checkpointing, #3224 -- recreate: move chunks_healthy when excluding hardlink master, #3228 -- recreate: get rid of chunks_healthy when rechunking (does not match), #3218 -- check: get rid of already existing not matching chunks_healthy metadata, #3218 -- list: fix stdout broken pipe handling, #3245 -- list/diff: remove tag-file options (not used), #3226 - -New features: - -- bash, zsh and fish shell auto-completions, see scripts/shell_completions/ -- added BORG_CONFIG_DIR env var, #3083 - -Other changes: - -- docs: - - - clarify using a blank passphrase in keyfile mode - - mention "!" (exclude-norecurse) type in "patterns" help - - document to first heal before running borg recreate to re-chunk stuff, - because that will have to get rid of chunks_healthy metadata. - - more than 23 is not supported for CHUNK_MAX_EXP, #3115 - - borg does not respect nodump flag by default any more - - clarify same-filesystem requirement for borg upgrade, #2083 - - update / rephrase cygwin / WSL status, #3174 - - improve docs about --stats, #3260 -- vagrant: openindiana new clang package - -Already contained in 1.1.1 (last minute fix): - -- arg parsing: fix fallback function, refactor, #3205. This is a fixup - for #3155, which was broken on at least python <= 3.4.2. - - -Version 1.1.1 (2017-10-22) --------------------------- - -Compatibility notes: - -- The deprecated --no-files-cache is not a global/common option any more, - but only available for borg create (it is not needed for anything else). - Use --files-cache=disabled instead of --no-files-cache. -- The nodump flag ("do not backup this file") is not honoured any more by - default because this functionality (esp. if it happened by error or - unexpected) was rather confusing and unexplainable at first to users. - If you want that "do not backup NODUMP-flagged files" behaviour, use: - borg create --exclude-nodump ... -- If you are on Linux and do not need bsdflags archived, consider using - ``--nobsdflags`` with ``borg create`` to avoid additional syscalls and - speed up backup creation. - -Fixes: - -- borg recreate: correctly compute part file sizes. fixes cosmetic, but - annoying issue as borg check complains about size inconsistencies of part - files in affected archives. you can solve that by running borg recreate on - these archives, see also #3157. -- bsdflags support: do not open BLK/CHR/LNK files, avoid crashes and - slowness, #3130 -- recreate: don't crash on attic archives w/o time_end, #3109 -- don't crash on repository filesystems w/o hardlink support, #3107 -- don't crash in first part of truncate_and_unlink, #3117 -- fix server-side IndexError crash with clients < 1.0.7, #3192 -- don't show traceback if only a global option is given, show help, #3142 -- cache: use SaveFile for more safety, #3158 -- init: fix wrong encryption choices in command line parser, fix missing - "authenticated-blake2", #3103 -- move --no-files-cache from common to borg create options, #3146 -- fix detection of non-local path (failed on ..filename), #3108 -- logging with fileConfig: set json attr on "borg" logger, #3114 -- fix crash with relative BORG_KEY_FILE, #3197 -- show excluded dir with "x" for tagged dirs / caches, #3189 - -New features: - -- create: --nobsdflags and --exclude-nodump options, #3160 -- extract: --nobsdflags option, #3160 - -Other changes: - -- remove annoying hardlinked symlinks warning, #3175 -- vagrant: use self-made FreeBSD 10.3 box, #3022 -- travis: don't brew update, hopefully fixes #2532 -- docs: - - - readme: -e option is required in borg 1.1 - - add example showing --show-version --show-rc - - use --format rather than --list-format (deprecated) in example - - update docs about hardlinked symlinks limitation - - -Version 1.1.0 (2017-10-07) --------------------------- - -Compatibility notes: - -- borg command line: do not put options in between positional arguments - - This sometimes works (e.g. it worked in borg 1.0.x), but can easily stop - working if we make positional arguments optional (like it happened for - borg create's "paths" argument in 1.1). There are also places in borg 1.0 - where we do that, so it doesn't work there in general either. #3356 - - Good: borg create -v --stats repo::archive path - Good: borg create repo::archive path -v --stats - Bad: borg create repo::archive -v --stats path - -Fixes: - -- fix LD_LIBRARY_PATH restoration for subprocesses, #3077 -- "auto" compression: make sure expensive compression is actually better, - otherwise store lz4 compressed data we already computed. - -Other changes: - -- docs: - - - FAQ: we do not implement futile attempts of ETA / progress displays - - manpage: fix typos, update homepage - - implement simple "issue" role for manpage generation, #3075 - - -Version 1.1.0rc4 (2017-10-01) ------------------------------ - -Compatibility notes: - -- A borg server >= 1.1.0rc4 does not support borg clients 1.1.0b3-b5. #3033 -- The files cache is now controlled differently and has a new default mode: - - - the files cache now uses ctime by default for improved file change - detection safety. You can still use mtime for more speed and less safety. - - --ignore-inode is deprecated (use --files-cache=... without "inode") - - --no-files-cache is deprecated (use --files-cache=disabled) - -New features: - -- --files-cache - implement files cache mode control, #911 - You can now control the files cache mode using this option: - --files-cache={ctime,mtime,size,inode,rechunk,disabled} - (only some combinations are supported). See the docs for details. - -Fixes: - -- remote progress/logging: deal with partial lines, #2637 -- remote progress: flush json mode output -- fix subprocess environments, #3050 (and more) - -Other changes: - -- remove client_supports_log_v3 flag, #3033 -- exclude broken Cython 0.27(.0) in requirements, #3066 -- vagrant: - - - upgrade to FUSE for macOS 3.7.1 - - use Python 3.5.4 to build the binaries -- docs: - - - security: change-passphrase only changes the passphrase, #2990 - - fixed/improved borg create --compression examples, #3034 - - add note about metadata dedup and --no[ac]time, #2518 - - twitter account @borgbackup now, better visible, #2948 - - simplified rate limiting wrapper in FAQ - - -Version 1.1.0rc3 (2017-09-10) ------------------------------ - -New features: - -- delete: support naming multiple archives, #2958 - -Fixes: - -- repo cleanup/write: invalidate cached FDs, #2982 -- fix datetime.isoformat() microseconds issues, #2994 -- recover_segment: use mmap(), lower memory needs, #2987 - -Other changes: - -- with-lock: close segment file before invoking subprocess -- keymanager: don't depend on optional readline module, #2976 -- docs: - - - fix macOS keychain integration command - - show/link new screencasts in README, #2936 - - document utf-8 locale requirement for json mode, #2273 -- vagrant: clean up shell profile init, user name, #2977 -- test_detect_attic_repo: don't test mount, #2975 -- add debug logging for repository cleanup - - -Version 1.1.0rc2 (2017-08-28) ------------------------------ - -Compatibility notes: - -- list: corrected mix-up of "isomtime" and "mtime" formats. Previously, - "isomtime" was the default but produced a verbose human format, - while "mtime" produced a ISO-8601-like format. - The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), - and the default is now "mtime". - "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). - -New features: - -- None. - -Fixes: - -- list: fix weird mixup of mtime/isomtime -- create --timestamp: set start time, #2957 -- ignore corrupt files cache, #2939 -- migrate locks to child PID when daemonize is used -- fix exitcode of borg serve, #2910 -- only compare contents when chunker params match, #2899 -- umount: try fusermount, then try umount, #2863 - -Other changes: - -- JSON: use a more standard ISO 8601 datetime format, #2376 -- cache: write_archive_index: truncate_and_unlink on error, #2628 -- detect non-upgraded Attic repositories, #1933 -- delete various nogil and threading related lines -- coala / pylint related improvements -- docs: - - - renew asciinema/screencasts, #669 - - create: document exclusion through nodump, #2949 - - minor formatting fixes - - tar: tarpipe example - - improve "with-lock" and "info" docs, #2869 - - detail how to use macOS/GNOME/KDE keyrings for repo passwords, #392 -- travis: only short-circuit docs-only changes for pull requests -- vagrant: - - - netbsd: bash is already installed - - fix netbsd version in PKG_PATH - - add exe location to PATH when we build an exe - - -Version 1.1.0rc1 (2017-07-24) ------------------------------ - -Compatibility notes: - -- delete: removed short option for --cache-only - -New features: - -- support borg list repo --format {comment} {bcomment} {end}, #2081 -- key import: allow reading from stdin, #2760 - -Fixes: - -- with-lock: avoid creating segment files that might be overwritten later, #1867 -- prune: fix checkpoints processing with --glob-archives -- FUSE: versions view: keep original file extension at end, #2769 -- fix --last, --first: do not accept values <= 0, - fix reversed archive ordering with --last -- include testsuite data (attic.tar.gz) when installing the package -- use limited unpacker for outer key, for manifest (both security precautions), - #2174 #2175 -- fix bashism in shell scripts, #2820, #2816 -- cleanup endianness detection, create _endian.h, - fixes build on alpine linux, #2809 -- fix crash with --no-cache-sync (give known chunk size to chunk_incref), #2853 - -Other changes: - -- FUSE: versions view: linear numbering by archive time -- split up interval parsing from filtering for --keep-within, #2610 -- add a basic .editorconfig, #2734 -- use archive creation time as mtime for FUSE mount, #2834 -- upgrade FUSE for macOS (osxfuse) from 3.5.8 to 3.6.3, #2706 -- hashindex: speed up by replacing modulo with "if" to check for wraparound -- coala checker / pylint: fixed requirements and .coafile, more ignores -- borg upgrade: name backup directories as 'before-upgrade', #2811 -- add .mailmap -- some minor changes suggested by lgtm.com -- docs: - - - better explanation of the --ignore-inode option relevance, #2800 - - fix openSUSE command and add openSUSE section - - simplify ssh authorized_keys file using "restrict", add legacy note, #2121 - - mount: show usage of archive filters - - mount: add repository example, #2462 - - info: update and add examples, #2765 - - prune: include example - - improved style / formatting - - improved/fixed segments_per_dir docs - - recreate: fix wrong "remove unwanted files" example - - reference list of status chars in borg recreate --filter description - - update source-install docs about doc build dependencies, #2795 - - cleanup installation docs - - file system requirements, update segs per dir - - fix checkpoints/parts reference in FAQ, #2859 -- code: - - - hashindex: don't pass side effect into macro - - crypto low_level: don't mutate local bytes() - - use dash_open function to open file or "-" for stdin/stdout - - archiver: argparse cleanup / refactoring - - shellpattern: add match_end arg -- tests: added some additional unit tests, some fixes, #2700 #2710 -- vagrant: fix setup of cygwin, add Debian 9 "stretch" -- travis: don't perform full travis build on docs-only changes, #2531 - - -Version 1.1.0b6 (2017-06-18) ----------------------------- - -Compatibility notes: - -- Running "borg init" via a "borg serve --append-only" server will *not* create - an append-only repository anymore. Use "borg init --append-only" to initialize - an append-only repository. - -- Repositories in the "repokey" and "repokey-blake2" modes with an empty passphrase - are now treated as unencrypted repositories for security checks (e.g. - BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK). - - Previously there would be no prompts nor messages if an unknown repository - in one of these modes with an empty passphrase was encountered. This would - allow an attacker to swap a repository, if one assumed that the lack of - password prompts was due to a set BORG_PASSPHRASE. - - Since the "trick" does not work if BORG_PASSPHRASE is set, this does generally - not affect scripts. - -- Repositories in the "authenticated" mode are now treated as the unencrypted - repositories they are. - -- The client-side temporary repository cache now holds unencrypted data for better speed. - -- borg init: removed the short form of --append-only (-a). - -- borg upgrade: removed the short form of --inplace (-i). - -New features: - -- reimplemented the RepositoryCache, size-limited caching of decrypted repo - contents, integrity checked via xxh64. #2515 -- reduced space usage of chunks.archive.d. Existing caches are migrated during - a cache sync. #235 #2638 -- integrity checking using xxh64 for important files used by borg, #1101: - - - repository: index and hints files - - cache: chunks and files caches, chunks.archive.d -- improve cache sync speed, #1729 -- create: new --no-cache-sync option -- add repository mandatory feature flags infrastructure, #1806 -- Verify most operations against SecurityManager. Location, manifest timestamp - and key types are now checked for almost all non-debug commands. #2487 -- implement storage quotas, #2517 -- serve: add --restrict-to-repository, #2589 -- BORG_PASSCOMMAND: use external tool providing the key passphrase, #2573 -- borg export-tar, #2519 -- list: --json-lines instead of --json for archive contents, #2439 -- add --debug-profile option (and also "borg debug convert-profile"), #2473 -- implement --glob-archives/-a, #2448 -- normalize authenticated key modes for better naming consistency: - - - rename "authenticated" to "authenticated-blake2" (uses blake2b) - - implement "authenticated" mode (uses hmac-sha256) - -Fixes: - -- hashindex: read/write indices >2 GiB on 32bit systems, better error - reporting, #2496 -- repository URLs: implement IPv6 address support and also more informative - error message when parsing fails. -- mount: check whether llfuse is installed before asking for passphrase, #2540 -- mount: do pre-mount checks before opening repository, #2541 -- FUSE: - - - fix crash if empty (None) xattr is read, #2534 - - fix read(2) caching data in metadata cache - - fix negative uid/gid crash (fix crash when mounting archives - of external drives made on cygwin), #2674 - - redo ItemCache, on top of object cache - - use decrypted cache - - remove unnecessary normpaths -- serve: ignore --append-only when initializing a repository (borg init), #2501 -- serve: fix incorrect type of exception_short for Errors, #2513 -- fix --exclude and --exclude-from recursing into directories, #2469 -- init: don't allow creating nested repositories, #2563 -- --json: fix encryption[mode] not being the cmdline name -- remote: propagate Error.traceback correctly -- fix remote logging and progress, #2241 - - - implement --debug-topic for remote servers - - remote: restore "Remote:" prefix (as used in 1.0.x) - - rpc negotiate: enable v3 log protocol only for supported clients - - fix --progress and logging in general for remote -- fix parse_version, add tests, #2556 -- repository: truncate segments (and also some other files) before unlinking, #2557 -- recreate: keep timestamps as in original archive, #2384 -- recreate: if single archive is not processed, exit 2 -- patterns: don't recurse with ! / --exclude for pf:, #2509 -- cache sync: fix n^2 behaviour in lookup_name -- extract: don't write to disk with --stdout (affected non-regular-file items), #2645 -- hashindex: implement KeyError, more tests - -Other changes: - -- remote: show path in PathNotAllowed -- consider repokey w/o passphrase == unencrypted, #2169 -- consider authenticated mode == unencrypted, #2503 -- restrict key file names, #2560 -- document follow_symlinks requirements, check libc, use stat and chown - with follow_symlinks=False, #2507 -- support common options on the main command, #2508 -- support common options on mid-level commands (e.g. borg *key* export) -- make --progress a common option -- increase DEFAULT_SEGMENTS_PER_DIR to 1000 -- chunker: fix invalid use of types (function only used by tests) -- chunker: don't do uint32_t >> 32 -- FUSE: - - - add instrumentation (--debug and SIGUSR1/SIGINFO) - - reduced memory usage for repository mounts by lazily instantiating archives - - improved archive load times -- info: use CacheSynchronizer & HashIndex.stats_against (better performance) -- docs: - - - init: document --encryption as required - - security: OpenSSL usage - - security: used implementations; note python libraries - - security: security track record of OpenSSL and msgpack - - patterns: document denial of service (regex, wildcards) - - init: note possible denial of service with "none" mode - - init: document SHA extension is supported in OpenSSL and thus SHA is - faster on AMD Ryzen than blake2b. - - book: use A4 format, new builder option format. - - book: create appendices - - data structures: explain repository compaction - - data structures: add chunk layout diagram - - data structures: integrity checking - - data structures: demingle cache and repo index - - Attic FAQ: separate section for attic stuff - - FAQ: I get an IntegrityError or similar - what now? - - FAQ: Can I use Borg on SMR hard drives?, #2252 - - FAQ: specify "using inline shell scripts" - - add systemd warning regarding placeholders, #2543 - - xattr: document API - - add docs/misc/borg-data-flow data flow chart - - debugging facilities - - README: how to help the project, #2550 - - README: add bountysource badge, #2558 - - fresh new theme + tweaking - - logo: vectorized (PDF and SVG) versions - - frontends: use headlines - you can link to them - - mark --pattern, --patterns-from as experimental - - highlight experimental features in online docs - - remove regex based pattern examples, #2458 - - nanorst for "borg help TOPIC" and --help - - split deployment - - deployment: hosting repositories - - deployment: automated backups to a local hard drive - - development: vagrant, windows10 requirements - - development: update docs remarks - - split usage docs, #2627 - - usage: avoid bash highlight, [options] instead of - - usage: add benchmark page - - helpers: truncate_and_unlink doc - - don't suggest to leak BORG_PASSPHRASE - - internals: columnize rather long ToC [webkit fixup] - internals: manifest & feature flags - - internals: more HashIndex details - - internals: fix ASCII art equations - - internals: edited obj graph related sections a bit - - internals: layers image + description - - fix way too small figures in pdf - - index: disable syntax highlight (bash) - - improve options formatting, fix accidental block quotes - -- testing / checking: - - - add support for using coala, #1366 - - testsuite: add ArchiverCorruptionTestCase - - do not test logger name, #2504 - - call setup_logging after destroying logging config - - testsuite.archiver: normalise pytest.raises vs. assert_raises - - add test for preserved intermediate folder permissions, #2477 - - key: add round-trip test - - remove attic dependency of the tests, #2505 - - enable remote tests on cygwin - - tests: suppress tar's future timestamp warning - - cache sync: add more refcount tests - - repository: add tests, including corruption tests - -- vagrant: - - - control VM cpus and pytest workers via env vars VMCPUS and XDISTN - - update cleaning workdir - - fix openbsd shell - - add OpenIndiana - -- packaging: - - - binaries: don't bundle libssl - - setup.py clean to remove compiled files - - fail in borg package if version metadata is very broken (setuptools_scm) - -- repo / code structure: - - - create borg.algorithms and borg.crypto packages - - algorithms: rename crc32 to checksums - - move patterns to module, #2469 - - gitignore: complete paths for src/ excludes - - cache: extract CacheConfig class - - implement IntegrityCheckedFile + Detached variant, #2502 #1688 - - introduce popen_with_error_handling to handle common user errors - - -Version 1.1.0b5 (2017-04-30) ----------------------------- - -Compatibility notes: - -- BORG_HOSTNAME_IS_UNIQUE is now on by default. -- removed --compression-from feature -- recreate: add --recompress flag, unify --always-recompress and - --recompress - -Fixes: - -- catch exception for os.link when hardlinks are not supported, #2405 -- borg rename / recreate: expand placeholders, #2386 -- generic support for hardlinks (files, devices, FIFOs), #2324 -- extract: also create parent dir for device files, if needed, #2358 -- extract: if a hardlink master is not in the to-be-extracted subset, - the "x" status was not displayed for it, #2351 -- embrace y2038 issue to support 32bit platforms: clamp timestamps to int32, - #2347 -- verify_data: fix IntegrityError handling for defect chunks, #2442 -- allow excluding parent and including child, #2314 - -Other changes: - -- refactor compression decision stuff -- change global compression default to lz4 as well, to be consistent - with --compression defaults. -- placeholders: deny access to internals and other unspecified stuff -- clearer error message for unrecognized placeholder -- more clear exception if borg check does not help, #2427 -- vagrant: upgrade FUSE for macOS to 3.5.8, #2346 -- linux binary builds: get rid of glibc 2.13 dependency, #2430 -- docs: - - - placeholders: document escaping - - serve: env vars in original commands are ignored - - tell what kind of hardlinks we support - - more docs about compression - - LICENSE: use canonical formulation - ("copyright holders and contributors" instead of "author") - - document borg init behaviour via append-only borg serve, #2440 - - be clear about what buzhash is used for, #2390 - - add hint about chunker params, #2421 - - clarify borg upgrade docs, #2436 - - FAQ to explain warning when running borg check --repair, #2341 - - repository file system requirements, #2080 - - pre-install considerations - - misc. formatting / crossref fixes -- tests: - - - enhance travis setuptools_scm situation - - add extra test for the hashindex - - fix invalid param issue in benchmarks - -These belong to 1.1.0b4 release, but did not make it into changelog by then: - -- vagrant: increase memory for parallel testing -- lz4 compress: lower max. buffer size, exception handling -- add docstring to do_benchmark_crud -- patterns help: mention path full-match in intro - - -Version 1.1.0b4 (2017-03-27) ----------------------------- - -Compatibility notes: - -- init: the --encryption argument is mandatory now (there are several choices) -- moved "borg migrate-to-repokey" to "borg key migrate-to-repokey". -- "borg change-passphrase" is deprecated, use "borg key change-passphrase" - instead. -- the --exclude-if-present option now supports tagging a folder with any - filesystem object type (file, folder, etc), instead of expecting only files - as tags, #1999 -- the --keep-tag-files option has been deprecated in favor of the new - --keep-exclude-tags, to account for the change mentioned above. -- use lz4 compression by default, #2179 - -New features: - -- JSON API to make developing frontends and automation easier - (see :ref:`json_output`) - - - add JSON output to commands: `borg create/list/info --json ...`. - - add --log-json option for structured logging output. - - add JSON progress information, JSON support for confirmations (yes()). -- add two new options --pattern and --patterns-from as discussed in #1406 -- new path full match pattern style (pf:) for very fast matching, #2334 -- add 'debug dump-manifest' and 'debug dump-archive' commands -- add 'borg benchmark crud' command, #1788 -- new 'borg delete --force --force' to delete severely corrupted archives, #1975 -- info: show utilization of maximum archive size, #1452 -- list: add dsize and dcsize keys, #2164 -- paperkey.html: Add interactive html template for printing key backups. -- key export: add qr html export mode -- securely erase config file (which might have old encryption key), #2257 -- archived file items: add size to metadata, 'borg extract' and 'borg check' do - check the file size for consistency, FUSE uses precomputed size from Item. - -Fixes: - -- fix remote speed regression introduced in 1.1.0b3, #2185 -- fix regression handling timestamps beyond 2262 (revert bigint removal), - introduced in 1.1.0b3, #2321 -- clamp (nano)second values to unproblematic range, #2304 -- hashindex: rebuild hashtable if we have too little empty buckets - (performance fix), #2246 -- Location regex: fix bad parsing of wrong syntax -- ignore posix_fadvise errors in repository.py, #2095 -- borg rpc: use limited msgpack.Unpacker (security precaution), #2139 -- Manifest: Make sure manifest timestamp is strictly monotonically increasing. -- create: handle BackupOSError on a per-path level in one spot -- create: clarify -x option / meaning of "same filesystem" -- create: don't create hard link refs to failed files -- archive check: detect and fix missing all-zero replacement chunks, #2180 -- files cache: update inode number when --ignore-inode is used, #2226 -- fix decompression exceptions crashing ``check --verify-data`` and others - instead of reporting integrity error, #2224 #2221 -- extract: warning for unextracted big extended attributes, #2258, #2161 -- mount: umount on SIGINT/^C when in foreground -- mount: handle invalid hard link refs -- mount: fix huge RAM consumption when mounting a repository (saves number of - archives * 8 MiB), #2308 -- hashindex: detect mingw byte order #2073 -- hashindex: fix wrong skip_hint on hashindex_set when encountering tombstones, - the regression was introduced in #1748 -- fix ChunkIndex.__contains__ assertion for big-endian archs -- fix borg key/debug/benchmark crashing without subcommand, #2240 -- Location: accept //servername/share/path -- correct/refactor calculation of unique/non-unique chunks -- extract: fix missing call to ProgressIndicator.finish -- prune: fix error msg, it is --keep-within, not --within -- fix "auto" compression mode bug (not compressing), #2331 -- fix symlink item fs size computation, #2344 - -Other changes: - -- remote repository: improved async exception processing, #2255 #2225 -- with --compression auto,C, only use C if lz4 achieves at least 3% compression -- PatternMatcher: only normalize path once, #2338 -- hashindex: separate endian-dependent defs from endian detection -- migrate-to-repokey: ask using canonical_path() as we do everywhere else. -- SyncFile: fix use of fd object after close -- make LoggedIO.close_segment reentrant -- creating a new segment: use "xb" mode, #2099 -- redo key_creator, key_factory, centralise key knowledge, #2272 -- add return code functions, #2199 -- list: only load cache if needed -- list: files->items, clarifications -- list: add "name" key for consistency with info cmd -- ArchiveFormatter: add "start" key for compatibility with "info" -- RemoteRepository: account rx/tx bytes -- setup.py build_usage/build_man/build_api fixes -- Manifest.in: simplify, exclude .so, .dll and .orig, #2066 -- FUSE: get rid of chunk accounting, st_blocks = ceil(size / blocksize). -- tests: - - - help python development by testing 3.6-dev - - test for borg delete --force -- vagrant: - - - freebsd: some fixes, #2067 - - darwin64: use osxfuse 3.5.4 for tests / to build binaries - - darwin64: improve VM settings - - use python 3.5.3 to build binaries, #2078 - - upgrade pyinstaller from 3.1.1+ to 3.2.1 - - pyinstaller: use fixed AND freshly compiled bootloader, #2002 - - pyinstaller: automatically builds bootloader if missing -- docs: - - - create really nice man pages - - faq: mention --remote-ratelimit in bandwidth limit question - - fix caskroom link, #2299 - - docs/security: reiterate that RPC in Borg does no networking - - docs/security: counter tracking, #2266 - - docs/development: update merge remarks - - address SSH batch mode in docs, #2202 #2270 - - add warning about running build_usage on Python >3.4, #2123 - - one link per distro in the installation page - - improve --exclude-if-present and --keep-exclude-tags, #2268 - - improve automated backup script in doc, #2214 - - improve remote-path description - - update docs for create -C default change (lz4) - - document relative path usage, #1868 - - document snapshot usage, #2178 - - corrected some stuff in internals+security - - internals: move toctree to after the introduction text - - clarify metadata kind, manifest ops - - key enc: correct / clarify some stuff, link to internals/security - - datas: enc: 1.1.x mas different MACs - - datas: enc: correct factual error -- no nonce involved there. - - make internals.rst an index page and edit it a bit - - add "Cryptography in Borg" and "Remote RPC protocol security" sections - - document BORG_HOSTNAME_IS_UNIQUE, #2087 - - FAQ by categories as proposed by @anarcat in #1802 - - FAQ: update Which file types, attributes, etc. are *not* preserved? - - development: new branching model for git repository - - development: define "ours" merge strategy for auto-generated files - - create: move --exclude note to main doc - - create: move item flags to main doc - - fix examples using borg init without -e/--encryption - - list: don't print key listings in fat (html + man) - - remove Python API docs (were very incomplete, build problems on RTFD) - - added FAQ section about backing up root partition - - -Version 1.1.0b3 (2017-01-15) ----------------------------- - -Compatibility notes: - -- borg init: removed the default of "--encryption/-e", #1979 - This was done so users do a informed decision about -e mode. - -Bug fixes: - -- borg recreate: don't rechunkify unless explicitly told so -- borg info: fixed bug when called without arguments, #1914 -- borg init: fix free space check crashing if disk is full, #1821 -- borg debug delete/get obj: fix wrong reference to exception -- fix processing of remote ~/ and ~user/ paths (regressed since 1.1.0b1), #1759 -- posix platform module: only build / import on non-win32 platforms, #2041 - -New features: - -- new CRC32 implementations that are much faster than the zlib one used previously, #1970 -- add blake2b key modes (use blake2b as MAC). This links against system libb2, - if possible, otherwise uses bundled code -- automatically remove stale locks - set BORG_HOSTNAME_IS_UNIQUE env var - to enable stale lock killing. If set, stale locks in both cache and - repository are deleted. #562 #1253 -- borg info : print general repo information, #1680 -- borg check --first / --last / --sort / --prefix, #1663 -- borg mount --first / --last / --sort / --prefix, #1542 -- implement "health" item formatter key, #1749 -- BORG_SECURITY_DIR to remember security related infos outside the cache. - Key type, location and manifest timestamp checks now survive cache - deletion. This also means that you can now delete your cache and avoid - previous warnings, since Borg can still tell it's safe. -- implement BORG_NEW_PASSPHRASE, #1768 - -Other changes: - -- borg recreate: - - - remove special-cased --dry-run - - update --help - - remove bloat: interruption blah, autocommit blah, resuming blah - - re-use existing checkpoint functionality - - archiver tests: add check_cache tool - lints refcounts - -- fixed cache sync performance regression from 1.1.0b1 onwards, #1940 -- syncing the cache without chunks.archive.d (see :ref:`disable_archive_chunks`) - now avoids any merges and is thus faster, #1940 -- borg check --verify-data: faster due to linear on-disk-order scan -- borg debug-xxx commands removed, we use "debug xxx" subcommands now, #1627 -- improve metadata handling speed -- shortcut hashindex_set by having hashindex_lookup hint about address -- improve / add progress displays, #1721 -- check for index vs. segment files object count mismatch -- make RPC protocol more extensible: use named parameters. -- RemoteRepository: misc. code cleanups / refactors -- clarify cache/repository README file - -- docs: - - - quickstart: add a comment about other (remote) filesystems - - quickstart: only give one possible ssh url syntax, all others are - documented in usage chapter. - - mention file:// - - document repo URLs / archive location - - clarify borg diff help, #980 - - deployment: synthesize alternative --restrict-to-path example - - improve cache / index docs, esp. files cache docs, #1825 - - document using "git merge 1.0-maint -s recursive -X rename-threshold=20%" - for avoiding troubles when merging the 1.0-maint branch into master. - -- tests: - - - FUSE tests: catch ENOTSUP on freebsd - - FUSE tests: test troublesome xattrs last - - fix byte range error in test, #1740 - - use monkeypatch to set env vars, but only on pytest based tests. - - point XDG_*_HOME to temp dirs for tests, #1714 - - remove all BORG_* env vars from the outer environment - - -Version 1.1.0b2 (2016-10-01) ----------------------------- - -Bug fixes: - -- fix incorrect preservation of delete tags, leading to "object count mismatch" - on borg check, #1598. This only occurred with 1.1.0b1 (not with 1.0.x) and is - normally fixed by running another borg create/delete/prune. -- fix broken --progress for double-cell paths (e.g. CJK), #1624 -- borg recreate: also catch SIGHUP -- FUSE: - - - fix hardlinks in versions view, #1599 - - add parameter check to ItemCache.get to make potential failures more clear - -New features: - -- Archiver, RemoteRepository: add --remote-ratelimit (send data) -- borg help compression, #1582 -- borg check: delete chunks with integrity errors, #1575, so they can be - "repaired" immediately and maybe healed later. -- archives filters concept (refactoring/unifying older code) - - - covers --first/--last/--prefix/--sort-by options - - currently used for borg list/info/delete - -Other changes: - -- borg check --verify-data slightly tuned (use get_many()) -- change {utcnow} and {now} to ISO-8601 format ("T" date/time separator) -- repo check: log transaction IDs, improve object count mismatch diagnostic -- Vagrantfile: use TW's fresh-bootloader pyinstaller branch -- fix module names in api.rst -- hashindex: bump api_version - - -Version 1.1.0b1 (2016-08-28) ----------------------------- - -New features: - -- new commands: - - - borg recreate: re-create existing archives, #787 #686 #630 #70, also see - #757, #770. - - - selectively remove files/dirs from old archives - - re-compress data - - re-chunkify data, e.g. to have upgraded Attic / Borg 0.xx archives - deduplicate with Borg 1.x archives or to experiment with chunker-params. - - borg diff: show differences between archives - - borg with-lock: execute a command with the repository locked, #990 -- borg create: - - - Flexible compression with pattern matching on path/filename, - and LZ4 heuristic for deciding compressibility, #810, #1007 - - visit files in inode order (better speed, esp. for large directories and rotating disks) - - in-file checkpoints, #1217 - - increased default checkpoint interval to 30 minutes (was 5 minutes), #896 - - added uuid archive format tag, #1151 - - save mountpoint directories with --one-file-system, makes system restore easier, #1033 - - Linux: added support for some BSD flags, #1050 - - add 'x' status for excluded paths, #814 - - - also means files excluded via UF_NODUMP, #1080 -- borg check: - - - will not produce the "Checking segments" output unless new --progress option is passed, #824. - - --verify-data to verify data cryptographically on the client, #975 -- borg list, #751, #1179 - - - removed {formatkeys}, see "borg list --help" - - --list-format is deprecated, use --format instead - - --format now also applies to listing archives, not only archive contents, #1179 - - now supports the usual [PATH [PATHS…]] syntax and excludes - - new keys: csize, num_chunks, unique_chunks, NUL - - supports guaranteed_available hashlib hashes - (to avoid varying functionality depending on environment), - which includes the SHA1 and SHA2 family as well as MD5 -- borg prune: - - - to better visualize the "thinning out", we now list all archives in - reverse time order. rephrase and reorder help text. - - implement --keep-last N via --keep-secondly N, also --keep-minutely. - assuming that there is not more than 1 backup archive made in 1s, - --keep-last N and --keep-secondly N are equivalent, #537 - - cleanup checkpoints except the latest, #1008 -- borg extract: - - - added --progress, #1449 - - Linux: limited support for BSD flags, #1050 -- borg info: - - - output is now more similar to borg create --stats, #977 -- borg mount: - - - provide "borgfs" wrapper for borg mount, enables usage via fstab, #743 - - "versions" mount option - when used with a repository mount, this gives - a merged, versioned view of the files in all archives, #729 -- repository: - - - added progress information to commit/compaction phase (often takes some time when deleting/pruning), #1519 - - automatic recovery for some forms of repository inconsistency, #858 - - check free space before going forward with a commit, #1336 - - improved write performance (esp. for rotating media), #985 - - - new IO code for Linux - - raised default segment size to approx 512 MiB - - improved compaction performance, #1041 - - reduced client CPU load and improved performance for remote repositories, #940 - -- options that imply output (--show-rc, --show-version, --list, --stats, - --progress) don't need -v/--info to have that output displayed, #865 -- add archive comments (via borg (re)create --comment), #842 -- borg list/prune/delete: also output archive id, #731 -- --show-version: shows/logs the borg version, #725 -- added --debug-topic for granular debug logging, #1447 -- use atomic file writing/updating for configuration and key files, #1377 -- BORG_KEY_FILE environment variable, #1001 -- self-testing module, #970 - - -Bug fixes: - -- list: fixed default output being produced if --format is given with empty parameter, #1489 -- create: fixed overflowing progress line with CJK and similar characters, #1051 -- prune: fixed crash if --prefix resulted in no matches, #1029 -- init: clean up partial repo if passphrase input is aborted, #850 -- info: quote cmdline arguments that have spaces in them -- fix hardlinks failing in some cases for extracting subtrees, #761 - -Other changes: - -- replace stdlib hmac with OpenSSL, zero-copy decrypt (10-15% increase in - performance of hash-lists and extract). -- improved chunker performance, #1021 -- open repository segment files in exclusive mode (fail-safe), #1134 -- improved error logging, #1440 -- Source: - - - pass meta-data around, #765 - - move some constants to new constants module - - better readability and fewer errors with namedtuples, #823 - - moved source tree into src/ subdirectory, #1016 - - made borg.platform a package, #1113 - - removed dead crypto code, #1032 - - improved and ported parts of the test suite to py.test, #912 - - created data classes instead of passing dictionaries around, #981, #1158, #1161 - - cleaned up imports, #1112 -- Docs: - - - better help texts and sphinx reproduction of usage help: - - - Group options - - Nicer list of options in Sphinx - - Deduplicate 'Common options' (including --help) - - chunker: added some insights by "Voltara", #903 - - clarify what "deduplicated size" means - - fix / update / add package list entries - - added a SaltStack usage example, #956 - - expanded FAQ - - new contributors in AUTHORS! -- Tests: - - - vagrant: add ubuntu/xenial 64bit - this box has still some issues - - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 - - -Version 1.0.13 (2019-02-15) ---------------------------- - -Please note: this is very likely the last 1.0.x release, please upgrade to 1.1.x. - -Bug fixes: - -- security fix: configure FUSE with "default_permissions", #3903. - "default_permissions" is now enforced by borg by default to let the - kernel check uid/gid/mode based permissions. - "ignore_permissions" can be given to not enforce "default_permissions". -- xattrs: fix borg exception handling on ENOSPC error, #3808. - -New features: - -- Read a passphrase from a file descriptor specified in the - BORG_PASSPHRASE_FD environment variable. - -Other changes: - -- acl platform code: fix acl set return type -- xattr: - - - add linux {list,get,set}xattr ctypes prototypes - - fix darwin flistxattr ctypes prototype -- testing / travis-ci: - - - fix the homebrew 1.9 issues on travis-ci, #4254 - - travis OS X: use xcode 8.3 (not broken) - - tox.ini: lock requirements - - unbreak 1.0-maint on travis, fixes #4123 -- vagrant: - - - misc. fixes - - FUSE for macOS: upgrade 3.7.1 to 3.8.3 - - Python: upgrade 3.5.5 to 3.5.6 -- docs: - - - Update installation instructions for macOS - - update release workflow using twine (docs, scripts), #4213 - -Version 1.0.12 (2018-04-08) ---------------------------- - -Bug fixes: - -- repository: cleanup/write: invalidate cached FDs, tests -- serve: fix exitcode, #2910 -- extract: set bsdflags last (include immutable flag), #3263 -- create --timestamp: set start time, #2957 -- create: show excluded dir with "x" for tagged dirs / caches, #3189 -- migrate locks to child PID when daemonize is used -- Buffer: fix wrong thread-local storage use, #2951 -- fix detection of non-local path, #3108 -- fix LDLP restoration for subprocesses, #3077 -- fix subprocess environments (xattr module's fakeroot version check, - borg umount, BORG_PASSCOMMAND), #3050 -- remote: deal with partial lines, #2637 -- get rid of datetime.isoformat, use safe parse_timestamp to parse - timestamps, #2994 -- build: do .h file content checks in binary mode, fixes build issue for - non-ascii header files on pure-ascii locale platforms, #3544 #3639 -- remove platform.uname() call which caused library mismatch issues, #3732 -- add exception handler around deprecated platform.linux_distribution() call - -Other changes: - -- require msgpack-python >= 0.4.6 and < 0.5.0, see #3753 -- add parens for C preprocessor macro argument usages (did not cause - malfunction) -- ignore corrupt files cache, #2939 -- replace "modulo" with "if" to check for wraparound in hashmap -- keymanager: don't depend on optional readline module, #2980 -- exclude broken pytest 3.3.0 release -- exclude broken Cython 0.27(.0) release, #3066 -- flake8: add some ignores -- docs: - - - create: document exclusion through nodump - - document good and problematic option placements, fix examples, #3356 - - update docs about hardlinked symlinks limitation - - faq: we do not implement futile attempts of ETA / progress displays - - simplified rate limiting wrapper in FAQ - - twitter account @borgbackup, #2948 - - add note about metadata dedup and --no[ac]time, #2518 - - change-passphrase only changes the passphrase, #2990 - - clarify encrypted key format for borg key export, #3296 - - document sshfs rename workaround, #3315 - - update release checklist about security fixes - - docs about how to verify a signed release, #3634 - - chunk seed is generated per /repository/ -- vagrant: - - - use FUSE for macOS 3.7.1 to build the macOS binary - - use python 3.5.5 to build the binaries - - add exe location to PATH when we build an exe - - use https pypi url for wheezy - - netbsd: bash is already installed - - netbsd: fix netbsd version in PKG_PATH - - use self-made FreeBSD 10.3 box, #3022 - - backport fs_init (including related updates) from 1.1 - - the boxcutter wheezy boxes are 404, use local ones -- travis: - - - don't perform full Travis build on docs-only changes, #2531 - - only short-circuit docs-only changes for pull requests - - -Version 1.0.11 (2017-07-21) ---------------------------- - -Bug fixes: - -- use limited unpacker for outer key (security precaution), #2174 -- fix paperkey import bug - -Other changes: - -- change --checkpoint-interval default from 600s to 1800s, #2841. - this improves efficiency for big repositories a lot. -- docs: fix OpenSUSE command and add OpenSUSE section -- tests: add tests for split_lstring and paperkey -- vagrant: - - - fix openbsd shell - - backport cpu/ram setup from master - - add stretch64 VM - -Version 1.0.11rc1 (2017-06-27) ------------------------------- - -Bug fixes: - -- performance: rebuild hashtable if we have too few empty buckets, #2246. - this fixes some sporadic, but severe performance breakdowns. -- Archive: allocate zeros when needed, #2308 - fixes huge memory usage of mount (8 MiB × number of archives) -- IPv6 address support - also: Location: more informative exception when parsing fails -- borg single-file binary: use pyinstaller v3.2.1, #2396 - this fixes that the prelink cronjob on some distros kills the - borg binary by stripping away parts of it. -- extract: - - - warning for unextracted big extended attributes, #2258 - - also create parent dir for device files, if needed. - - don't write to disk with --stdout, #2645 -- archive check: detect and fix missing all-zero replacement chunks, #2180 -- fix (de)compression exceptions, #2224 #2221 -- files cache: update inode number, #2226 -- borg rpc: use limited msgpack.Unpacker (security precaution), #2139 -- Manifest: use limited msgpack.Unpacker (security precaution), #2175 -- Location: accept //servername/share/path -- fix ChunkIndex.__contains__ assertion for big-endian archs (harmless) -- create: handle BackupOSError on a per-path level in one spot -- fix error msg, there is no --keep-last in borg 1.0.x, #2282 -- clamp (nano)second values to unproblematic range, #2304 -- fuse / borg mount: - - - fix st_blocks to be an integer (not float) value - - fix negative uid/gid crash (they could come into archives e.g. when - backing up external drives under cygwin), #2674 - - fix crash if empty (None) xattr is read - - do pre-mount checks before opening repository - - check llfuse is installed before asking for passphrase -- borg rename: expand placeholders, #2386 -- borg serve: fix forced command lines containing BORG_* env vars -- fix error msg, it is --keep-within, not --within -- fix borg key/debug/benchmark crashing without subcommand, #2240 -- chunker: fix invalid use of types, don't do uint32_t >> 32 -- document follow_symlinks requirements, check libc, #2507 - -New features: - -- added BORG_PASSCOMMAND environment variable, #2573 -- add minimal version of in repository mandatory feature flags, #2134 - - This should allow us to make sure older borg versions can be cleanly - prevented from doing operations that are no longer safe because of - repository format evolution. This allows more fine grained control than - just incrementing the manifest version. So for example a change that - still allows new archives to be created but would corrupt the repository - when an old version tries to delete an archive or check the repository - would add the new feature to the check and delete set but leave it out - of the write set. -- borg delete --force --force to delete severely corrupted archives, #1975 - -Other changes: - -- embrace y2038 issue to support 32bit platforms -- be more clear that this is a "beyond repair" case, #2427 -- key file names: limit to 100 characters and remove colons from host name -- upgrade FUSE for macOS to 3.5.8, #2346 -- split up parsing and filtering for --keep-within, better error message, #2610 -- docs: - - - fix caskroom link, #2299 - - address SSH batch mode, #2202 #2270 - - improve remote-path description - - document snapshot usage, #2178 - - document relative path usage, #1868 - - one link per distro in the installation page - - development: new branching model in git repository - - kill api page - - added FAQ section about backing up root partition - - add bountysource badge, #2558 - - create empty docs.txt reequirements, #2694 - - README: how to help the project - - note -v/--verbose requirement on affected options, #2542 - - document borg init behaviour via append-only borg serve, #2440 - - be clear about what buzhash is used for (chunking) and want it is not - used for (deduplication)- also say already in the readme that we use a - cryptohash for dedupe, so people don't worry, #2390 - - add hint about chunker params to borg upgrade docs, #2421 - - clarify borg upgrade docs, #2436 - - quickstart: delete problematic BORG_PASSPRHASE use, #2623 - - faq: specify "using inline shell scripts" - - document pattern denial of service, #2624 -- tests: - - - remove attic dependency of the tests, #2505 - - travis: - - - enhance travis setuptools_scm situation - - install fakeroot for Linux - - add test for borg delete --force - - enable remote tests on cygwin (the cygwin issue that caused these tests - to break was fixed in cygwin at least since cygwin 2.8, maybe even since - 2.7.0). - - remove skipping the noatime tests on GNU/Hurd, #2710 - - fix borg import issue, add comment, #2718 - - include attic.tar.gz when installing the package - also: add include_package_data=True - -Version 1.0.10 (2017-02-13) ---------------------------- - -Bug fixes: - -- Manifest timestamps are now monotonically increasing, - this fixes issues when the system clock jumps backwards - or is set inconsistently across computers accessing the same repository, #2115 -- Fixed testing regression in 1.0.10rc1 that lead to a hard dependency on - py.test >= 3.0, #2112 - -New features: - -- "key export" can now generate a printable HTML page with both a QR code and - a human-readable "paperkey" representation (and custom text) through the - ``--qr-html`` option. - - The same functionality is also available through `paperkey.html `_, - which is the same HTML page generated by ``--qr-html``. It works with existing - "key export" files and key files. - -Other changes: - -- docs: - - - language clarification - "borg create --one-file-system" option does not respect - mount points, but considers different file systems instead, #2141 -- setup.py: build_api: sort file list for determinism - - -Version 1.0.10rc1 (2017-01-29) ------------------------------- - -Bug fixes: - -- borg serve: fix transmission data loss of pipe writes, #1268 - This affects only the cygwin platform (not Linux, BSD, OS X). -- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992 -- When running out of buffer memory when reading xattrs, only skip the - current file, #1993 -- Fixed "borg upgrade --tam" crashing with unencrypted repositories. Since - :ref:`the issue ` is not relevant for unencrypted repositories, - it now does nothing and prints an error, #1981. -- Fixed change-passphrase crashing with unencrypted repositories, #1978 -- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997 -- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled -- fix bad parsing of wrong repo location syntax -- create: don't create hard link refs to failed files, - mount: handle invalid hard link refs, #2092 -- detect mingw byte order, #2073 -- creating a new segment: use "xb" mode, #2099 -- mount: umount on SIGINT/^C when in foreground, #2082 - -Other changes: - -- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002 -- xattr: ignore empty names returned by llistxattr(2) et al -- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT, - SIGBUS and SIGILL signals to dump the Python traceback. -- Also print a traceback on SIGUSR2. -- borg change-passphrase: print key location (simplify making a backup of it) -- officially support Python 3.6 (setup.py: add Python 3.6 qualifier) -- tests: - - - vagrant / travis / tox: add Python 3.6 based testing - - vagrant: fix openbsd repo, #2042 - - vagrant: fix the freebsd64 machine, #2037 #2067 - - vagrant: use python 3.5.3 to build binaries, #2078 - - vagrant: use osxfuse 3.5.4 for tests / to build binaries - vagrant: improve darwin64 VM settings - - travis: fix osxfuse install (fixes OS X testing on Travis CI) - - travis: require succeeding OS X tests, #2028 - - travis: use latest pythons for OS X based testing - - use pytest-xdist to parallelize testing - - fix xattr test race condition, #2047 - - setup.cfg: fix pytest deprecation warning, #2050 -- docs: - - - language clarification - VM backup FAQ - - borg create: document how to backup stdin, #2013 - - borg upgrade: fix incorrect title levels - - add CVE numbers for issues fixed in 1.0.9, #2106 -- fix typos (taken from Debian package patch) -- remote: include data hexdump in "unexpected RPC data" error message -- remote: log SSH command line at debug level -- API_VERSION: use numberspaces, #2023 -- remove .github from pypi package, #2051 -- add pip and setuptools to requirements file, #2030 -- SyncFile: fix use of fd object after close (cosmetic) -- Manifest.in: simplify, exclude \*.{so,dll,orig}, #2066 -- ignore posix_fadvise errors in repository.py, #2095 - (works around issues with docker on ARM) -- make LoggedIO.close_segment reentrant, avoid reentrance - - -Version 1.0.9 (2016-12-20) --------------------------- - -Security fixes: - -- A flaw in the cryptographic authentication scheme in Borg allowed an attacker - to spoof the manifest. See :ref:`tam_vuln` above for the steps you should - take. - - CVE-2016-10099 was assigned to this vulnerability. -- borg check: When rebuilding the manifest (which should only be needed very rarely) - duplicate archive names would be handled on a "first come first serve" basis, allowing - an attacker to apparently replace archives. - - CVE-2016-10100 was assigned to this vulnerability. - -Bug fixes: - -- borg check: - - - rebuild manifest if it's corrupted - - skip corrupted chunks during manifest rebuild -- fix TypeError in integrity error handler, #1903, #1894 -- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 -- fix wrong duration/timestamps if system clock jumped during a create -- fix progress display not updating if system clock jumps backwards -- fix checkpoint interval being incorrect if system clock jumps - -Other changes: - -- docs: - - - add python3-devel as a dependency for cygwin-based installation - - clarify extract is relative to current directory - - FAQ: fix link to changelog - - markup fixes -- tests: - - - test_get\_(cache|keys)_dir: clean env state, #1897 - - get back pytest's pretty assertion failures, #1938 -- setup.py build_usage: - - - fixed build_usage not processing all commands - - fixed build_usage not generating includes for debug commands - - -Version 1.0.9rc1 (2016-11-27) ------------------------------ - -Bug fixes: - -- files cache: fix determination of newest mtime in backup set (which is - used in cache cleanup and led to wrong "A" [added] status for unchanged - files in next backup), #1860. - -- borg check: - - - fix incorrectly reporting attic 0.13 and earlier archives as corrupt - - handle repo w/o objects gracefully and also bail out early if repo is - *completely* empty, #1815. -- fix tox/pybuild in 1.0-maint -- at xattr module import time, loggers are not initialized yet - -New features: - -- borg umount - exposed already existing umount code via the CLI api, so users can use it, - which is more consistent than using borg to mount and fusermount -u (or - umount) to un-mount, #1855. -- implement borg create --noatime --noctime, fixes #1853 - -Other changes: - -- docs: - - - display README correctly on PyPI - - improve cache / index docs, esp. files cache docs, fixes #1825 - - different pattern matching for --exclude, #1779 - - datetime formatting examples for {now} placeholder, #1822 - - clarify passphrase mode attic repo upgrade, #1854 - - clarify --umask usage, #1859 - - clarify how to choose PR target branch - - clarify prune behavior for different archive contents, #1824 - - fix PDF issues, add logo, fix authors, headings, TOC - - move security verification to support section - - fix links in standalone README (:ref: tags) - - add link to security contact in README - - add FAQ about security - - move fork differences to FAQ - - add more details about resource usage -- tests: skip remote tests on cygwin, #1268 -- travis: - - - allow OS X failures until the brew cask osxfuse issue is fixed - - caskroom osxfuse-beta gone, it's osxfuse now (3.5.3) -- vagrant: - - - upgrade OSXfuse / FUSE for macOS to 3.5.3 - - remove llfuse from tox.ini at a central place - - do not try to install llfuse on centos6 - - fix FUSE test for darwin, #1546 - - add windows virtual machine with cygwin - - Vagrantfile cleanup / code deduplication - - -Version 1.0.8 (2016-10-29) --------------------------- - -Bug fixes: - -- RemoteRepository: Fix busy wait in call_many, #940 - -New features: - -- implement borgmajor/borgminor/borgpatch placeholders, #1694 - {borgversion} was already there (full version string). With the new - placeholders you can now also get e.g. 1 or 1.0 or 1.0.8. - -Other changes: - -- avoid previous_location mismatch, #1741 - - due to the changed canonicalization for relative paths in PR #1711 / #1655 - (implement /./ relpath hack), there would be a changed repo location warning - and the user would be asked if this is ok. this would break automation and - require manual intervention, which is unwanted. - - thus, we automatically fix the previous_location config entry, if it only - changed in the expected way, but still means the same location. - -- docs: - - - deployment.rst: do not use bare variables in ansible snippet - - add clarification about append-only mode, #1689 - - setup.py: add comment about requiring llfuse, #1726 - - update usage.rst / api.rst - - repo url / archive location docs + typo fix - - quickstart: add a comment about other (remote) filesystems - -- vagrant / tests: - - - no chown when rsyncing (fixes boxes w/o vagrant group) - - fix FUSE permission issues on linux/freebsd, #1544 - - skip FUSE test for borg binary + fakeroot - - ignore security.selinux xattrs, fixes tests on centos, #1735 - - -Version 1.0.8rc1 (2016-10-17) ------------------------------ - -Bug fixes: - -- fix signal handling (SIGINT, SIGTERM, SIGHUP), #1620 #1593 - Fixes e.g. leftover lock files for quickly repeated signals (e.g. Ctrl-C - Ctrl-C) or lost connections or systemd sending SIGHUP. -- progress display: adapt formatting to narrow screens, do not crash, #1628 -- borg create --read-special - fix crash on broken symlink, #1584. - also correctly processes broken symlinks. before this regressed to a crash - (5b45385) a broken symlink would've been skipped. -- process_symlink: fix missing backup_io() - Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting - dirents and dispatching to process_symlink. -- yes(): abort on wrong answers, saying so, #1622 -- fixed exception borg serve raised when connection was closed before repository - was opened. Add an error message for this. -- fix read-from-closed-FD issue, #1551 - (this seems not to get triggered in 1.0.x, but was discovered in master) -- hashindex: fix iterators (always raise StopIteration when exhausted) - (this seems not to get triggered in 1.0.x, but was discovered in master) -- enable relative paths in ssh:// repo URLs, via /./relpath hack, #1655 -- allow repo paths with colons, #1705 -- update changed repo location immediately after acceptance, #1524 -- fix debug get-obj / delete-obj crash if object not found and remote repo, - #1684 -- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib - on Mac OS X (avoids mismatch lib <-> driver), #1619 - -New features: - -- add "borg key export" / "borg key import" commands, #1555, so users are able - to backup / restore their encryption keys more easily. - - Supported formats are the keyfile format used by borg internally and a - special "paper" format with by line checksums for printed backups. For the - paper format, the import is an interactive process which checks each line as - soon as it is input. -- add "borg debug-refcount-obj" to determine a repo objects' referrer counts, - #1352 - -Other changes: - -- add "borg debug ..." subcommands - (borg debug-* still works, but will be removed in borg 1.1) -- setup.py: Add subcommand support to build_usage. -- remote: change exception message for unexpected RPC data format to indicate - dataflow direction. -- improved messages / error reporting: - - - IntegrityError: add placeholder for message, so that the message we give - appears not only in the traceback, but also in the (short) error message, - #1572 - - borg.key: include chunk id in exception msgs, #1571 - - better messages for cache newer than repo, #1700 -- vagrant (testing/build VMs): - - - upgrade OSXfuse / FUSE for macOS to 3.5.2 - - update Debian Wheezy boxes, #1686 - - openbsd / netbsd: use own boxes, fixes misc rsync installation and - FUSE/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728 -- docs: - - - add docs for "key export" and "key import" commands, #1641 - - fix inconsistency in FAQ (pv-wrapper). - - fix second block in "Easy to use" section not showing on GitHub, #1576 - - add bestpractices badge - - link reference docs and faq about BORG_FILES_CACHE_TTL, #1561 - - improve borg info --help, explain size infos, #1532 - - add release signing key / security contact to README, #1560 - - add contribution guidelines for developers - - development.rst: add sphinx_rtd_theme to the sphinx install command - - adjust border color in borg.css - - add debug-info usage help file - - internals.rst: fix typos - - setup.py: fix build_usage to always process all commands - - added docs explaining multiple --restrict-to-path flags, #1602 - - add more specific warning about write-access debug commands, #1587 - - clarify FAQ regarding backup of virtual machines, #1672 -- tests: - - - work around FUSE xattr test issue with recent fakeroot - - simplify repo/hashindex tests - - travis: test FUSE-enabled borg, use trusty to have a recent FUSE - - re-enable FUSE tests for RemoteArchiver (no deadlocks any more) - - clean env for pytest based tests, #1714 - - fuse_mount contextmanager: accept any options - - -Version 1.0.7 (2016-08-19) --------------------------- - -Security fixes: - -- borg serve: fix security issue with remote repository access, #1428 - If you used e.g. --restrict-to-path /path/client1/ (with or without trailing - slash does not make a difference), it acted like a path prefix match using - /path/client1 (note the missing trailing slash) - the code then also allowed - working in e.g. /path/client13 or /path/client1000. - - As this could accidentally lead to major security/privacy issues depending on - the paths you use, the behaviour was changed to be a strict directory match. - That means --restrict-to-path /path/client1 (with or without trailing slash - does not make a difference) now uses /path/client1/ internally (note the - trailing slash here!) for matching and allows precisely that path AND any - path below it. So, /path/client1 is allowed, /path/client1/repo1 is allowed, - but not /path/client13 or /path/client1000. - - If you willingly used the undocumented (dangerous) previous behaviour, you - may need to rearrange your --restrict-to-path paths now. We are sorry if - that causes work for you, but we did not want a potentially dangerous - behaviour in the software (not even using a for-backwards-compat option). - -Bug fixes: - -- fixed repeated LockTimeout exceptions when borg serve tried to write into - a already write-locked repo (e.g. by a borg mount), #502 part b) - This was solved by the fix for #1220 in 1.0.7rc1 already. -- fix cosmetics + file leftover for "not a valid borg repository", #1490 -- Cache: release lock if cache is invalid, #1501 -- borg extract --strip-components: fix leak of preloaded chunk contents -- Repository, when a InvalidRepository exception happens: - - - fix spurious, empty lock.roster - - fix repo not closed cleanly - -New features: - -- implement borg debug-info, fixes #1122 - (just calls already existing code via cli, same output as below tracebacks) - -Other changes: - -- skip the O_NOATIME test on GNU Hurd, fixes #1315 - (this is a very minor issue and the GNU Hurd project knows the bug) -- document using a clean repo to test / build the release - - -Version 1.0.7rc2 (2016-08-13) ------------------------------ - -Bug fixes: - -- do not write objects to repository that are bigger than the allowed size, - borg will reject reading them, #1451. - - Important: if you created archives with many millions of files or - directories, please verify if you can open them successfully, - e.g. try a "borg list REPO::ARCHIVE". -- lz4 compression: dynamically enlarge the (de)compression buffer, the static - buffer was not big enough for archives with extremely many items, #1453 -- larger item metadata stream chunks, raise archive item limit by 8x, #1452 -- fix untracked segments made by moved DELETEs, #1442 - - Impact: Previously (metadata) segments could become untracked when deleting data, - these would never be cleaned up. -- extended attributes (xattrs) related fixes: - - - fixed a race condition in xattrs querying that led to the entire file not - being backed up (while logging the error, exit code = 1), #1469 - - fixed a race condition in xattrs querying that led to a crash, #1462 - - raise OSError including the error message derived from errno, deal with - path being a integer FD - -Other changes: - -- print active env var override by default, #1467 -- xattr module: refactor code, deduplicate, clean up -- repository: split object size check into too small and too big -- add a transaction_id assertion, so borg init on a broken (inconsistent) - filesystem does not look like a coding error in borg, but points to the - real problem. -- explain confusing TypeError caused by compat support for old servers, #1456 -- add forgotten usage help file from build_usage -- refactor/unify buffer code into helpers.Buffer class, add tests -- docs: - - - document archive limitation, #1452 - - improve prune examples - - -Version 1.0.7rc1 (2016-08-05) ------------------------------ - -Bug fixes: - -- fix repo lock deadlocks (related to lock upgrade), #1220 -- catch unpacker exceptions, resync, #1351 -- fix borg break-lock ignoring BORG_REPO env var, #1324 -- files cache performance fixes (fixes unnecessary re-reading/chunking/ - hashing of unmodified files for some use cases): - - - fix unintended file cache eviction, #1430 - - implement BORG_FILES_CACHE_TTL, update FAQ, raise default TTL from 10 - to 20, #1338 -- FUSE: - - - cache partially read data chunks (performance), #965, #966 - - always create a root dir, #1125 -- use an OrderedDict for helptext, making the build reproducible, #1346 -- RemoteRepository init: always call close on exceptions, #1370 (cosmetic) -- ignore stdout/stderr broken pipe errors (cosmetic), #1116 - -New features: - -- better borg versions management support (useful esp. for borg servers - wanting to offer multiple borg versions and for clients wanting to choose - a specific server borg version), #1392: - - - add BORG_VERSION environment variable before executing "borg serve" via ssh - - add new placeholder {borgversion} - - substitute placeholders in --remote-path - -- borg init --append-only option (makes using the more secure append-only mode - more convenient. when used remotely, this requires 1.0.7+ also on the borg - server), #1291. - -Other changes: - -- Vagrantfile: - - - darwin64: upgrade to FUSE for macOS 3.4.1 (aka osxfuse), #1378 - - xenial64: use user "ubuntu", not "vagrant" (as usual), #1331 -- tests: - - - fix FUSE tests on OS X, #1433 -- docs: - - - FAQ: add backup using stable filesystem names recommendation - - FAQ about glibc compatibility added, #491, glibc-check improved - - FAQ: 'A' unchanged file; remove ambiguous entry age sentence. - - OS X: install pkg-config to build with FUSE support, fixes #1400 - - add notes about shell/sudo pitfalls with env. vars, #1380 - - added platform feature matrix -- implement borg debug-dump-repo-objs - - -Version 1.0.6 (2016-07-12) --------------------------- - -Bug fixes: - -- Linux: handle multiple LD_PRELOAD entries correctly, #1314, #1111 -- Fix crash with unclear message if the libc is not found, #1314, #1111 - -Other changes: - -- tests: - - - Fixed O_NOATIME tests for Solaris and GNU Hurd, #1315 - - Fixed sparse file tests for (file) systems not supporting it, #1310 -- docs: - - - Fixed syntax highlighting, #1313 - - misc docs: added data processing overview picture - - -Version 1.0.6rc1 (2016-07-10) ------------------------------ - -New features: - -- borg check --repair: heal damaged files if missing chunks re-appear (e.g. if - the previously missing chunk was added again in a later backup archive), - #148. (*) Also improved logging. - -Bug fixes: - -- sync_dir: silence fsync() failing with EINVAL, #1287 - Some network filesystems (like smbfs) don't support this and we use this in - repository code. -- borg mount (FUSE): - - - fix directories being shadowed when contained paths were also specified, - #1295 - - raise I/O Error (EIO) on damaged files (unless -o allow_damaged_files is - used), #1302. (*) -- borg extract: warn if a damaged file is extracted, #1299. (*) -- Added some missing return code checks (ChunkIndex._add, hashindex_resize). -- borg check: fix/optimize initial hash table size, avoids resize of the table. - -Other changes: - -- tests: - - - add more FUSE tests, #1284 - - deduplicate FUSE (u)mount code - - fix borg binary test issues, #862 -- docs: - - - changelog: added release dates to older borg releases - - fix some sphinx (docs generator) warnings, #881 - -Notes: - -(*) Some features depend on information (chunks_healthy list) added to item -metadata when a file with missing chunks was "repaired" using all-zero -replacement chunks. The chunks_healthy list is generated since borg 1.0.4, -thus borg can't recognize such "repaired" (but content-damaged) files if the -repair was done with an older borg version. - - -Version 1.0.5 (2016-07-07) --------------------------- - -Bug fixes: - -- borg mount: fix FUSE crash in xattr code on Linux introduced in 1.0.4, #1282 - -Other changes: - -- backport some FAQ entries from master branch -- add release helper scripts -- Vagrantfile: - - - centos6: no FUSE, don't build binary - - add xz for redhat-like dists - - -Version 1.0.4 (2016-07-07) --------------------------- - -New features: - -- borg serve --append-only, #1168 - This was included because it was a simple change (append-only functionality - was already present via repository config file) and makes better security now - practically usable. -- BORG_REMOTE_PATH environment variable, #1258 - This was included because it was a simple change (--remote-path cli option - was already present) and makes borg much easier to use if you need it. -- Repository: cleanup incomplete transaction on "no space left" condition. - In many cases, this can avoid a 100% full repo filesystem (which is very - problematic as borg always needs free space - even to delete archives). - -Bug fixes: - -- Fix wrong handling and reporting of OSErrors in borg create, #1138. - This was a serious issue: in the context of "borg create", errors like - repository I/O errors (e.g. disk I/O errors, ssh repo connection errors) - were handled badly and did not lead to a crash (which would be good for this - case, because the repo transaction would be incomplete and trigger a - transaction rollback to clean up). - Now, error handling for source files is cleanly separated from every other - error handling, so only problematic input files are logged and skipped. -- Implement fail-safe error handling for borg extract. - Note that this isn't nearly as critical as the borg create error handling - bug, since nothing is written to the repo. So this was "merely" misleading - error reporting. -- Add missing error handler in directory attr restore loop. -- repo: make sure write data hits disk before the commit tag (#1236) and also - sync the containing directory. -- FUSE: getxattr fail must use errno.ENOATTR, #1126 - (fixes Mac OS X Finder malfunction: "zero bytes" file length, access denied) -- borg check --repair: do not lose information about the good/original chunks. - If we do not lose the original chunk IDs list when "repairing" a file - (replacing missing chunks with all-zero chunks), we have a chance to "heal" - the file back into its original state later, in case the chunks re-appear - (e.g. in a fresh backup). Healing is not implemented yet, see #148. -- fixes for --read-special mode: - - - ignore known files cache, #1241 - - fake regular file mode, #1214 - - improve symlinks handling, #1215 -- remove passphrase from subprocess environment, #1105 -- Ignore empty index file (will trigger index rebuild), #1195 -- add missing placeholder support for --prefix, #1027 -- improve exception handling for placeholder replacement -- catch and format exceptions in arg parsing -- helpers: fix "undefined name 'e'" in exception handler -- better error handling for missing repo manifest, #1043 -- borg delete: - - - make it possible to delete a repo without manifest - - borg delete --forced allows one to delete corrupted archives, #1139 -- borg check: - - - make borg check work for empty repo - - fix resync and msgpacked item qualifier, #1135 - - rebuild_manifest: fix crash if 'name' or 'time' key were missing. - - better validation of item metadata dicts, #1130 - - better validation of archive metadata dicts -- close the repo on exit - even if rollback did not work, #1197. - This is rather cosmetic, it avoids repo closing in the destructor. - -- tests: - - - fix sparse file test, #1170 - - flake8: ignore new F405, #1185 - - catch "invalid argument" on cygwin, #257 - - fix sparseness assertion in test prep, #1264 - -Other changes: - -- make borg build/work on OpenSSL 1.0 and 1.1, #1187 -- docs / help: - - - fix / clarify prune help, #1143 - - fix "patterns" help formatting - - add missing docs / help about placeholders - - resources: rename atticmatic to borgmatic - - document sshd settings, #545 - - more details about checkpoints, add split trick, #1171 - - support docs: add freenode web chat link, #1175 - - add prune visualization / example, #723 - - add note that Fnmatch is default, #1247 - - make clear that lzma levels > 6 are a waste of cpu cycles - - add a "do not edit" note to auto-generated files, #1250 - - update cygwin installation docs -- repository interoperability with borg master (1.1dev) branch: - - - borg check: read item metadata keys from manifest, #1147 - - read v2 hints files, #1235 - - fix hints file "unknown version" error handling bug -- tests: add tests for format_line -- llfuse: update version requirement for freebsd -- Vagrantfile: - - - use openbsd 5.9, #716 - - do not install llfuse on netbsd (broken) - - update OSXfuse to version 3.3.3 - - use Python 3.5.2 to build the binaries -- glibc compatibility checker: scripts/glibc_check.py -- add .eggs to .gitignore - - -Version 1.0.3 (2016-05-20) --------------------------- - -Bug fixes: - -- prune: avoid that checkpoints are kept and completed archives are deleted in - a prune run), #997 -- prune: fix commandline argument validation - some valid command lines were - considered invalid (annoying, but harmless), #942 -- fix capabilities extraction on Linux (set xattrs last, after chown()), #1069 -- repository: fix commit tags being seen in data -- when probing key files, do binary reads. avoids crash when non-borg binary - files are located in borg's key files directory. -- handle SIGTERM and make a clean exit - avoids orphan lock files. -- repository cache: don't cache large objects (avoid using lots of temp. disk - space), #1063 - -Other changes: - -- Vagrantfile: OS X: update osxfuse / install lzma package, #933 -- setup.py: add check for platform_darwin.c -- setup.py: on freebsd, use a llfuse release that builds ok -- docs / help: - - - update readthedocs URLs, #991 - - add missing docs for "borg break-lock", #992 - - borg create help: add some words to about the archive name - - borg create help: document format tags, #894 - - -Version 1.0.2 (2016-04-16) --------------------------- - -Bug fixes: - -- fix malfunction and potential corruption on (nowadays rather rare) big-endian - architectures or bi-endian archs in (rare) BE mode. #886, #889 - - cache resync / index merge was malfunctioning due to this, potentially - leading to data loss. borg info had cosmetic issues (displayed wrong values). - - note: all (widespread) little-endian archs (like x86/x64) or bi-endian archs - in (widespread) LE mode (like ARMEL, MIPSEL, ...) were NOT affected. -- add overflow and range checks for 1st (special) uint32 of the hashindex - values, switch from int32 to uint32. -- fix so that refcount will never overflow, but just stick to max. value after - a overflow would have occurred. -- borg delete: fix --cache-only for broken caches, #874 - - Makes --cache-only idempotent: it won't fail if the cache is already deleted. -- fixed borg create --one-file-system erroneously traversing into other - filesystems (if starting fs device number was 0), #873 -- workround a bug in Linux fadvise FADV_DONTNEED, #907 - -Other changes: - -- better test coverage for hashindex, incl. overflow testing, checking correct - computations so endianness issues would be discovered. -- reproducible doc for ProgressIndicator*, make the build reproducible. -- use latest llfuse for vagrant machines -- docs: - - - use /path/to/repo in examples, fixes #901 - - fix confusing usage of "repo" as archive name (use "arch") - - -Version 1.0.1 (2016-04-08) --------------------------- - -New features: - -Usually there are no new features in a bugfix release, but these were added -due to their high impact on security/safety/speed or because they are fixes -also: - -- append-only mode for repositories, #809, #36 (see docs) -- borg create: add --ignore-inode option to make borg detect unmodified files - even if your filesystem does not have stable inode numbers (like sshfs and - possibly CIFS). -- add options --warning, --error, --critical for missing log levels, #826. - it's not recommended to suppress warnings or errors, but the user may decide - this on his own. - note: --warning is not given to borg serve so a <= 1.0.0 borg will still - work as server (it is not needed as it is the default). - do not use --error or --critical when using a <= 1.0.0 borg server. - -Bug fixes: - -- fix silently skipping EIO, #748 -- add context manager for Repository (avoid orphan repository locks), #285 -- do not sleep for >60s while waiting for lock, #773 -- unpack file stats before passing to FUSE -- fix build on illumos -- don't try to backup doors or event ports (Solaris and derivatives) -- remove useless/misleading libc version display, #738 -- test suite: reset exit code of persistent archiver, #844 -- RemoteRepository: clean up pipe if remote open() fails -- Remote: don't print tracebacks for Error exceptions handled downstream, #792 -- if BORG_PASSPHRASE is present but wrong, don't prompt for password, but fail - instead, #791 -- ArchiveChecker: move "orphaned objects check skipped" to INFO log level, #826 -- fix capitalization, add ellipses, change log level to debug for 2 messages, - #798 - -Other changes: - -- update llfuse requirement, llfuse 1.0 works -- update OS / dist packages on build machines, #717 -- prefer showing --info over -v in usage help, #859 -- docs: - - - fix cygwin requirements (gcc-g++) - - document how to debug / file filesystem issues, #664 - - fix reproducible build of api docs - - RTD theme: CSS !important overwrite, #727 - - Document logo font. Recreate logo png. Remove GIMP logo file. - - -Version 1.0.0 (2016-03-05) --------------------------- - -The major release number change (0.x -> 1.x) indicates bigger incompatible -changes, please read the compatibility notes, adapt / test your scripts and -check your backup logs. - -Compatibility notes: - -- drop support for python 3.2 and 3.3, require 3.4 or 3.5, #221 #65 #490 - note: we provide binaries that include python 3.5.1 and everything else - needed. they are an option in case you are stuck with < 3.4 otherwise. -- change encryption to be on by default (using "repokey" mode) -- moved keyfile keys from ~/.borg/keys to ~/.config/borg/keys, - you can either move them manually or run "borg upgrade " -- remove support for --encryption=passphrase, - use borg migrate-to-repokey to switch to repokey mode, #97 -- remove deprecated --compression , - use --compression zlib, instead - in case of 0, you could also use --compression none -- remove deprecated --hourly/daily/weekly/monthly/yearly - use --keep-hourly/daily/weekly/monthly/yearly instead -- remove deprecated --do-not-cross-mountpoints, - use --one-file-system instead -- disambiguate -p option, #563: - - - -p now is same as --progress - - -P now is same as --prefix -- remove deprecated "borg verify", - use "borg extract --dry-run" instead -- cleanup environment variable semantics, #355 - the environment variables used to be "yes sayers" when set, this was - conceptually generalized to "automatic answerers" and they just give their - value as answer (as if you typed in that value when being asked). - See the "usage" / "Environment Variables" section of the docs for details. -- change the builtin default for --chunker-params, create 2MiB chunks, #343 - --chunker-params new default: 19,23,21,4095 - old default: 10,23,16,4095 - - one of the biggest issues with borg < 1.0 (and also attic) was that it had a - default target chunk size of 64kiB, thus it created a lot of chunks and thus - also a huge chunk management overhead (high RAM and disk usage). - - please note that the new default won't change the chunks that you already - have in your repository. the new big chunks do not deduplicate with the old - small chunks, so expect your repo to grow at least by the size of every - changed file and in the worst case (e.g. if your files cache was lost / is - not used) by the size of every file (minus any compression you might use). - - in case you want to immediately see a much lower resource usage (RAM / disk) - for chunks management, it might be better to start with a new repo than - continuing in the existing repo (with an existing repo, you'ld have to wait - until all archives with small chunks got pruned to see a lower resource - usage). - - if you used the old --chunker-params default value (or if you did not use - --chunker-params option at all) and you'ld like to continue using small - chunks (and you accept the huge resource usage that comes with that), just - explicitly use borg create --chunker-params=10,23,16,4095. -- archive timestamps: the 'time' timestamp now refers to archive creation - start time (was: end time), the new 'time_end' timestamp refers to archive - creation end time. This might affect prune if your backups take rather long. - if you give a timestamp via cli this is stored into 'time', therefore it now - needs to mean archive creation start time. - -New features: - -- implement password roundtrip, #695 - -Bug fixes: - -- remote end does not need cache nor keys directories, do not create them, #701 -- added retry counter for passwords, #703 - -Other changes: - -- fix compiler warnings, #697 -- docs: - - - update README.rst to new changelog location in docs/changes.rst - - add Teemu to AUTHORS - - changes.rst: fix old chunker params, #698 - - FAQ: how to limit bandwidth - - -Version 1.0.0rc2 (2016-02-28) ------------------------------ - -New features: - -- format options for location: user, pid, fqdn, hostname, now, utcnow, user -- borg list --list-format -- borg prune -v --list enables the keep/prune list output, #658 - -Bug fixes: - -- fix _open_rb noatime handling, #657 -- add a simple archivename validator, #680 -- borg create --stats: show timestamps in localtime, use same labels/formatting - as borg info, #651 -- llfuse compatibility fixes (now compatible with: 0.40, 0.41, 0.42) - -Other changes: - -- it is now possible to use "pip install borgbackup[fuse]" to automatically - install the llfuse dependency using the correct version requirement - for it. you still need to care about having installed the FUSE / build - related OS package first, though, so that building llfuse can succeed. -- Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4 -- Vagrant: use pyinstaller v3.1.1 to build binaries -- docs: - - - borg upgrade: add to docs that only LOCAL repos are supported - - borg upgrade also handles borg 0.xx -> 1.0 - - use pip extras or requirements file to install llfuse - - fix order in release process - - updated usage docs and other minor / cosmetic fixes - - verified borg examples in docs, #644 - - freebsd dependency installation and FUSE configuration, #649 - - add example how to restore a raw device, #671 - - add a hint about the dev headers needed when installing from source - - add examples for delete (and handle delete after list, before prune), #656 - - update example for borg create -v --stats (use iso datetime format), #663 - - added example to BORG_RSH docs - - "connection closed by remote": add FAQ entry and point to issue #636 - - -Version 1.0.0rc1 (2016-02-07) ------------------------------ - -New features: - -- borg migrate-to-repokey ("passphrase" -> "repokey" encryption key mode) -- implement --short for borg list REPO, #611 -- implement --list for borg extract (consistency with borg create) -- borg serve: overwrite client's --restrict-to-path with ssh forced command's - option value (but keep everything else from the client commandline), #544 -- use $XDG_CONFIG_HOME/keys for keyfile keys (~/.config/borg/keys), #515 -- "borg upgrade" moves the keyfile keys to the new location -- display both archive creation start and end time in "borg info", #627 - - -Bug fixes: - -- normalize trailing slashes for the repository path, #606 -- Cache: fix exception handling in __init__, release lock, #610 - -Other changes: - -- suppress unneeded exception context (PEP 409), simpler tracebacks -- removed special code needed to deal with imperfections / incompatibilities / - missing stuff in py 3.2/3.3, simplify code that can be done simpler in 3.4 -- removed some version requirements that were kept on old versions because - newer did not support py 3.2 any more -- use some py 3.4+ stdlib code instead of own/openssl/pypi code: - - - use os.urandom instead of own cython openssl RAND_bytes wrapper, #493 - - use hashlib.pbkdf2_hmac from py stdlib instead of own openssl wrapper - - use hmac.compare_digest instead of == operator (constant time comparison) - - use stat.filemode instead of homegrown code - - use "mock" library from stdlib, #145 - - remove borg.support (with non-broken argparse copy), it is ok in 3.4+, #358 -- Vagrant: copy CHANGES.rst as symlink, #592 -- cosmetic code cleanups, add flake8 to tox/travis, #4 -- docs / help: - - - make "borg -h" output prettier, #591 - - slightly rephrase prune help - - add missing example for --list option of borg create - - quote exclude line that includes an asterisk to prevent shell expansion - - fix dead link to license - - delete Ubuntu Vivid, it is not supported anymore (EOL) - - OS X binary does not work for older OS X releases, #629 - - borg serve's special support for forced/original ssh commands, #544 - - misc. updates and fixes - - -Version 0.30.0 (2016-01-23) ---------------------------- - -Compatibility notes: - -- you may need to use -v (or --info) more often to actually see output emitted - at INFO log level (because it is suppressed at the default WARNING log level). - See the "general" section in the usage docs. -- for borg create, you need --list (additionally to -v) to see the long file - list (was needed so you can have e.g. --stats alone without the long list) -- see below about BORG_DELETE_I_KNOW_WHAT_I_AM_DOING (was: - BORG_CHECK_I_KNOW_WHAT_I_AM_DOING) - -Bug fixes: - -- fix crash when using borg create --dry-run --keep-tag-files, #570 -- make sure teardown with cleanup happens for Cache and RepositoryCache, - avoiding leftover locks and TEMP dir contents, #285 (partially), #548 -- fix locking KeyError, partial fix for #502 -- log stats consistently, #526 -- add abbreviated weekday to timestamp format, fixes #496 -- strip whitespace when loading exclusions from file -- unset LD_LIBRARY_PATH before invoking ssh, fixes strange OpenSSL library - version warning when using the borg binary, #514 -- add some error handling/fallback for C library loading, #494 -- added BORG_DELETE_I_KNOW_WHAT_I_AM_DOING for check in "borg delete", #503 -- remove unused "repair" rpc method name - -New features: - -- borg create: implement exclusions using regular expression patterns. -- borg create: implement inclusions using patterns. -- borg extract: support patterns, #361 -- support different styles for patterns: - - - fnmatch (`fm:` prefix, default when omitted), like borg <= 0.29. - - shell (`sh:` prefix) with `*` not matching directory separators and - `**/` matching 0..n directories - - path prefix (`pp:` prefix, for unifying borg create pp1 pp2 into the - patterns system), semantics like in borg <= 0.29 - - regular expression (`re:`), new! -- --progress option for borg upgrade (#291) and borg delete -- update progress indication more often (e.g. for borg create within big - files or for borg check repo), #500 -- finer chunker granularity for items metadata stream, #547, #487 -- borg create --list now used (additionally to -v) to enable the verbose - file list output -- display borg version below tracebacks, #532 - -Other changes: - -- hashtable size (and thus: RAM and disk consumption) follows a growth policy: - grows fast while small, grows slower when getting bigger, #527 -- Vagrantfile: use pyinstaller 3.1 to build binaries, freebsd sqlite3 fix, - fixes #569 -- no separate binaries for centos6 any more because the generic linux binaries - also work on centos6 (or in general: on systems with a slightly older glibc - than debian7 -- dev environment: require virtualenv<14.0 so we get a py32 compatible pip -- docs: - - - add space-saving chunks.archive.d trick to FAQ - - important: clarify -v and log levels in usage -> general, please read! - - sphinx configuration: create a simple man page from usage docs - - add a repo server setup example - - disable unneeded SSH features in authorized_keys examples for security. - - borg prune only knows "--keep-within" and not "--within" - - add gource video to resources docs, #507 - - add netbsd install instructions - - authors: make it more clear what refers to borg and what to attic - - document standalone binary requirements, #499 - - rephrase the mailing list section - - development docs: run build_api and build_usage before tagging release - - internals docs: hash table max. load factor is 0.75 now - - markup, typo, grammar, phrasing, clarifications and other fixes. - - add gcc gcc-c++ to redhat/fedora/corora install docs, fixes #583 - - -Version 0.29.0 (2015-12-13) ---------------------------- - -Compatibility notes: - -- when upgrading to 0.29.0 you need to upgrade client as well as server - installations due to the locking and commandline interface changes otherwise - you'll get an error msg about a RPC protocol mismatch or a wrong commandline - option. - if you run a server that needs to support both old and new clients, it is - suggested that you have a "borg-0.28.2" and a "borg-0.29.0" command. - clients then can choose via e.g. "borg --remote-path=borg-0.29.0 ...". -- the default waiting time for a lock changed from infinity to 1 second for a - better interactive user experience. if the repo you want to access is - currently locked, borg will now terminate after 1s with an error message. - if you have scripts that shall wait for the lock for a longer time, use - --lock-wait N (with N being the maximum wait time in seconds). - -Bug fixes: - -- hash table tuning (better chosen hashtable load factor 0.75 and prime initial - size of 1031 gave ~1000x speedup in some scenarios) -- avoid creation of an orphan lock for one case, #285 -- --keep-tag-files: fix file mode and multiple tag files in one directory, #432 -- fixes for "borg upgrade" (attic repo converter), #466 -- remove --progress isatty magic (and also --no-progress option) again, #476 -- borg init: display proper repo URL -- fix format of umask in help pages, #463 - -New features: - -- implement --lock-wait, support timeout for UpgradableLock, #210 -- implement borg break-lock command, #157 -- include system info below traceback, #324 -- sane remote logging, remote stderr, #461: - - - remote log output: intercept it and log it via local logging system, - with "Remote: " prefixed to message. log remote tracebacks. - - remote stderr: output it to local stderr with "Remote: " prefixed. -- add --debug and --info (same as --verbose) to set the log level of the - builtin logging configuration (which otherwise defaults to warning), #426 - note: there are few messages emitted at DEBUG level currently. -- optionally configure logging via env var BORG_LOGGING_CONF -- add --filter option for status characters: e.g. to show only the added - or modified files (and also errors), use "borg create -v --filter=AME ...". -- more progress indicators, #394 -- use ISO-8601 date and time format, #375 -- "borg check --prefix" to restrict archive checking to that name prefix, #206 - -Other changes: - -- hashindex_add C implementation (speed up cache re-sync for new archives) -- increase FUSE read_size to 1024 (speed up metadata operations) -- check/delete/prune --save-space: free unused segments quickly, #239 -- increase rpc protocol version to 2 (see also Compatibility notes), #458 -- silence borg by default (via default log level WARNING) -- get rid of C compiler warnings, #391 -- upgrade OS X FUSE to 3.0.9 on the OS X binary build system -- use python 3.5.1 to build binaries -- docs: - - - new mailing list borgbackup@python.org, #468 - - readthedocs: color and logo improvements - - load coverage icons over SSL (avoids mixed content) - - more precise binary installation steps - - update release procedure docs about OS X FUSE - - FAQ entry about unexpected 'A' status for unchanged file(s), #403 - - add docs about 'E' file status - - add "borg upgrade" docs, #464 - - add developer docs about output and logging - - clarify encryption, add note about client-side encryption - - add resources section, with videos, talks, presentations, #149 - - Borg moved to Arch Linux [community] - - fix wrong installation instructions for archlinux - - -Version 0.28.2 (2015-11-15) ---------------------------- - -New features: - -- borg create --exclude-if-present TAGFILE - exclude directories that have the - given file from the backup. You can additionally give --keep-tag-files to - preserve just the directory roots and the tag-files (but not backup other - directory contents), #395, attic #128, attic #142 - -Other changes: - -- do not create docs sources at build time (just have them in the repo), - completely remove have_cython() hack, do not use the "mock" library at build - time, #384 -- avoid hidden import, make it easier for PyInstaller, easier fix for #218 -- docs: - - - add description of item flags / status output, fixes #402 - - explain how to regenerate usage and API files (build_api or - build_usage) and when to commit usage files directly into git, #384 - - minor install docs improvements - - -Version 0.28.1 (2015-11-08) ---------------------------- - -Bug fixes: - -- do not try to build api / usage docs for production install, - fixes unexpected "mock" build dependency, #384 - -Other changes: - -- avoid using msgpack.packb at import time -- fix formatting issue in changes.rst -- fix build on readthedocs - - -Version 0.28.0 (2015-11-08) ---------------------------- - -Compatibility notes: - -- changed return codes (exit codes), see docs. in short: - old: 0 = ok, 1 = error. now: 0 = ok, 1 = warning, 2 = error - -New features: - -- refactor return codes (exit codes), fixes #61 -- add --show-rc option enable "terminating with X status, rc N" output, fixes 58, #351 -- borg create backups atime and ctime additionally to mtime, fixes #317 - - extract: support atime additionally to mtime - - FUSE: support ctime and atime additionally to mtime -- support borg --version -- emit a warning if we have a slow msgpack installed -- borg list --prefix=thishostname- REPO, fixes #205 -- Debug commands (do not use except if you know what you do: debug-get-obj, - debug-put-obj, debug-delete-obj, debug-dump-archive-items. - -Bug fixes: - -- setup.py: fix bug related to BORG_LZ4_PREFIX processing -- fix "check" for repos that have incomplete chunks, fixes #364 -- borg mount: fix unlocking of repository at umount time, fixes #331 -- fix reading files without touching their atime, #334 -- non-ascii ACL fixes for Linux, FreeBSD and OS X, #277 -- fix acl_use_local_uid_gid() and add a test for it, attic #359 -- borg upgrade: do not upgrade repositories in place by default, #299 -- fix cascading failure with the index conversion code, #269 -- borg check: implement 'cmdline' archive metadata value decoding, #311 -- fix RobustUnpacker, it missed some metadata keys (new atime and ctime keys - were missing, but also bsdflags). add check for unknown metadata keys. -- create from stdin: also save atime, ctime (cosmetic) -- use default_notty=False for confirmations, fixes #345 -- vagrant: fix msgpack installation on centos, fixes #342 -- deal with unicode errors for symlinks in same way as for regular files and - have a helpful warning message about how to fix wrong locale setup, fixes #382 -- add ACL keys the RobustUnpacker must know about - -Other changes: - -- improve file size displays, more flexible size formatters -- explicitly commit to the units standard, #289 -- archiver: add E status (means that an error occurred when processing this - (single) item -- do binary releases via "github releases", closes #214 -- create: use -x and --one-file-system (was: --do-not-cross-mountpoints), #296 -- a lot of changes related to using "logging" module and screen output, #233 -- show progress display if on a tty, output more progress information, #303 -- factor out status output so it is consistent, fix surrogates removal, - maybe fixes #309 -- move away from RawConfigParser to ConfigParser -- archive checker: better error logging, give chunk_id and sequence numbers - (can be used together with borg debug-dump-archive-items). -- do not mention the deprecated passphrase mode -- emit a deprecation warning for --compression N (giving a just a number) -- misc .coverragerc fixes (and coverage measurement improvements), fixes #319 -- refactor confirmation code, reduce code duplication, add tests -- prettier error messages, fixes #307, #57 -- tests: - - - add a test to find disk-full issues, #327 - - travis: also run tests on Python 3.5 - - travis: use tox -r so it rebuilds the tox environments - - test the generated pyinstaller-based binary by archiver unit tests, #215 - - vagrant: tests: announce whether fakeroot is used or not - - vagrant: add vagrant user to fuse group for debianoid systems also - - vagrant: llfuse install on darwin needs pkgconfig installed - - vagrant: use pyinstaller from develop branch, fixes #336 - - benchmarks: test create, extract, list, delete, info, check, help, fixes #146 - - benchmarks: test with both the binary and the python code - - archiver tests: test with both the binary and the python code, fixes #215 - - make basic test more robust -- docs: - - - moved docs to borgbackup.readthedocs.org, #155 - - a lot of fixes and improvements, use mobile-friendly RTD standard theme - - use zlib,6 compression in some examples, fixes #275 - - add missing rename usage to docs, closes #279 - - include the help offered by borg help in the usage docs, fixes #293 - - include a list of major changes compared to attic into README, fixes #224 - - add OS X install instructions, #197 - - more details about the release process, #260 - - fix linux glibc requirement (binaries built on debian7 now) - - build: move usage and API generation to setup.py - - update docs about return codes, #61 - - remove api docs (too much breakage on rtd) - - borgbackup install + basics presentation (asciinema) - - describe the current style guide in documentation - - add section about debug commands - - warn about not running out of space - - add example for rename - - improve chunker params docs, fixes #362 - - minor development docs update - - -Version 0.27.0 (2015-10-07) ---------------------------- - -New features: - -- "borg upgrade" command - attic -> borg one time converter / migration, #21 -- temporary hack to avoid using lots of disk space for chunks.archive.d, #235: - To use it: rm -rf chunks.archive.d ; touch chunks.archive.d -- respect XDG_CACHE_HOME, attic #181 -- add support for arbitrary SSH commands, attic #99 -- borg delete --cache-only REPO (only delete cache, not REPO), attic #123 - - -Bug fixes: - -- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow - down observed when running the Centos6-built binary on Ubuntu, #222 -- do not crash on empty lock.roster, fixes #232 -- fix multiple issues with the cache config version check, #234 -- fix segment entry header size check, attic #352 - plus other error handling improvements / code deduplication there. -- always give segment and offset in repo IntegrityErrors - - -Other changes: - -- stop producing binary wheels, remove docs about it, #147 -- docs: - - add warning about prune - - generate usage include files only as needed - - development docs: add Vagrant section - - update / improve / reformat FAQ - - hint to single-file pyinstaller binaries from README - - -Version 0.26.1 (2015-09-28) ---------------------------- - -This is a minor update, just docs and new pyinstaller binaries. - -- docs update about python and binary requirements -- better docs for --read-special, fix #220 -- re-built the binaries, fix #218 and #213 (glibc version issue) -- update web site about single-file pyinstaller binaries - -Note: if you did a python-based installation, there is no need to upgrade. - - -Version 0.26.0 (2015-09-19) ---------------------------- - -New features: - -- Faster cache sync (do all in one pass, remove tar/compression stuff), #163 -- BORG_REPO env var to specify the default repo, #168 -- read special files as if they were regular files, #79 -- implement borg create --dry-run, attic issue #267 -- Normalize paths before pattern matching on OS X, #143 -- support OpenBSD and NetBSD (except xattrs/ACLs) -- support / run tests on Python 3.5 - -Bug fixes: - -- borg mount repo: use absolute path, attic #200, attic #137 -- chunker: use off_t to get 64bit on 32bit platform, #178 -- initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0) -- fix reaction to "no" answer at delete repo prompt, #182 -- setup.py: detect lz4.h header file location -- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194) -- fix for obtaining ``char *`` from temporary Python value (old code causes - a compile error on Mint 17.2) -- llfuse 0.41 install troubles on some platforms, require < 0.41 - (UnicodeDecodeError exception due to non-ascii llfuse setup.py) -- cython code: add some int types to get rid of unspecific python add / - subtract operations (avoid ``undefined symbol FPE_``... error on some platforms) -- fix verbose mode display of stdin backup -- extract: warn if a include pattern never matched, fixes #209, - implement counters for Include/ExcludePatterns -- archive names with slashes are invalid, attic issue #180 -- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined - - fixes building on OpenBSD. - -Other changes: - -- detect inconsistency / corruption / hash collision, #170 -- replace versioneer with setuptools_scm, #106 -- docs: - - - pkg-config is needed for llfuse installation - - be more clear about pruning, attic issue #132 -- unit tests: - - - xattr: ignore security.selinux attribute showing up - - ext3 seems to need a bit more space for a sparse file - - do not test lzma level 9 compression (avoid MemoryError) - - work around strange mtime granularity issue on netbsd, fixes #204 - - ignore st_rdev if file is not a block/char device, fixes #203 - - stay away from the setgid and sticky mode bits -- use Vagrant to do easy cross-platform testing (#196), currently: - - - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit - - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit - - Centos 7 64bit - - FreeBSD 10.2 64bit - - OpenBSD 5.7 64bit - - NetBSD 6.1.5 64bit - - Darwin (OS X Yosemite) - - -Version 0.25.0 (2015-08-29) ---------------------------- - -Compatibility notes: - -- lz4 compression library (liblz4) is a new requirement (#156) -- the new compression code is very compatible: as long as you stay with zlib - compression, older borg releases will still be able to read data from a - repo/archive made with the new code (note: this is not the case for the - default "none" compression, use "zlib,0" if you want a "no compression" mode - that can be read by older borg). Also the new code is able to read repos and - archives made with older borg versions (for all zlib levels 0..9). - -Deprecations: - -- --compression N (with N being a number, as in 0.24) is deprecated. - We keep the --compression 0..9 for now to not break scripts, but it is - deprecated and will be removed later, so better fix your scripts now: - --compression 0 (as in 0.24) is the same as --compression zlib,0 (now). - BUT: if you do not want compression, you rather want --compression none - (which is the default). - --compression 1 (in 0.24) is the same as --compression zlib,1 (now) - --compression 9 (in 0.24) is the same as --compression zlib,9 (now) - -New features: - -- create --compression none (default, means: do not compress, just pass through - data "as is". this is more efficient than zlib level 0 as used in borg 0.24) -- create --compression lz4 (super-fast, but not very high compression) -- create --compression zlib,N (slower, higher compression, default for N is 6) -- create --compression lzma,N (slowest, highest compression, default N is 6) -- honor the nodump flag (UF_NODUMP) and do not backup such items -- list --short just outputs a simple list of the files/directories in an archive - -Bug fixes: - -- fixed --chunker-params parameter order confusion / malfunction, fixes #154 -- close fds of segments we delete (during compaction) -- close files which fell out the lrucache -- fadvise DONTNEED now is only called for the byte range actually read, not for - the whole file, fixes #158. -- fix issue with negative "all archives" size, fixes #165 -- restore_xattrs: ignore if setxattr fails with EACCES, fixes #162 - -Other changes: - -- remove fakeroot requirement for tests, tests run faster without fakeroot - (test setup does not fail any more without fakeroot, so you can run with or - without fakeroot), fixes #151 and #91. -- more tests for archiver -- recover_segment(): don't assume we have an fd for segment -- lrucache refactoring / cleanup, add dispose function, py.test tests -- generalize hashindex code for any key length (less hardcoding) -- lock roster: catch file not found in remove() method and ignore it -- travis CI: use requirements file -- improved docs: - - - replace hack for llfuse with proper solution (install libfuse-dev) - - update docs about compression - - update development docs about fakeroot - - internals: add some words about lock files / locking system - - support: mention BountySource and for what it can be used - - theme: use a lighter green - - add pypi, wheel, dist package based install docs - - split install docs into system-specific preparations and generic instructions - - -Version 0.24.0 (2015-08-09) ---------------------------- - -Incompatible changes (compared to 0.23): - -- borg now always issues --umask NNN option when invoking another borg via ssh - on the repository server. By that, it's making sure it uses the same umask - for remote repos as for local ones. Because of this, you must upgrade both - server and client(s) to 0.24. -- the default umask is 077 now (if you do not specify via --umask) which might - be a different one as you used previously. The default umask avoids that - you accidentally give access permissions for group and/or others to files - created by borg (e.g. the repository). - -Deprecations: - -- "--encryption passphrase" mode is deprecated, see #85 and #97. - See the new "--encryption repokey" mode for a replacement. - -New features: - -- borg create --chunker-params ... to configure the chunker, fixes #16 - (attic #302, attic #300, and somehow also #41). - This can be used to reduce memory usage caused by chunk management overhead, - so borg does not create a huge chunks index/repo index and eats all your RAM - if you back up lots of data in huge files (like VM disk images). - See docs/misc/create_chunker-params.txt for more information. -- borg info now reports chunk counts in the chunk index. -- borg create --compression 0..9 to select zlib compression level, fixes #66 - (attic #295). -- borg init --encryption repokey (to store the encryption key into the repo), - fixes #85 -- improve at-end error logging, always log exceptions and set exit_code=1 -- LoggedIO: better error checks / exceptions / exception handling -- implement --remote-path to allow non-default-path borg locations, #125 -- implement --umask M and use 077 as default umask for better security, #117 -- borg check: give a named single archive to it, fixes #139 -- cache sync: show progress indication -- cache sync: reimplement the chunk index merging in C - -Bug fixes: - -- fix segfault that happened for unreadable files (chunker: n needs to be a - signed size_t), #116 -- fix the repair mode, #144 -- repo delete: add destroy to allowed rpc methods, fixes issue #114 -- more compatible repository locking code (based on mkdir), maybe fixes #92 - (attic #317, attic #201). -- better Exception msg if no Borg is installed on the remote repo server, #56 -- create a RepositoryCache implementation that can cope with >2GiB, - fixes attic #326. -- fix Traceback when running check --repair, attic #232 -- clarify help text, fixes #73. -- add help string for --no-files-cache, fixes #140 - -Other changes: - -- improved docs: - - - added docs/misc directory for misc. writeups that won't be included - "as is" into the html docs. - - document environment variables and return codes (attic #324, attic #52) - - web site: add related projects, fix web site url, IRC #borgbackup - - Fedora/Fedora-based install instructions added to docs - - Cygwin-based install instructions added to docs - - updated AUTHORS - - add FAQ entries about redundancy / integrity - - clarify that borg extract uses the cwd as extraction target - - update internals doc about chunker params, memory usage and compression - - added docs about development - - add some words about resource usage in general - - document how to backup a raw disk - - add note about how to run borg from virtual env - - add solutions for (ll)fuse installation problems - - document what borg check does, fixes #138 - - reorganize borgbackup.github.io sidebar, prev/next at top - - deduplicate and refactor the docs / README.rst - -- use borg-tmp as prefix for temporary files / directories -- short prune options without "keep-" are deprecated, do not suggest them -- improved tox configuration -- remove usage of unittest.mock, always use mock from pypi -- use entrypoints instead of scripts, for better use of the wheel format and - modern installs -- add requirements.d/development.txt and modify tox.ini -- use travis-ci for testing based on Linux and (new) OS X -- use coverage.py, pytest-cov and codecov.io for test coverage support - -I forgot to list some stuff already implemented in 0.23.0, here they are: - -New features: - -- efficient archive list from manifest, meaning a big speedup for slow - repo connections and "list ", "delete ", "prune" (attic #242, - attic #167) -- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18 -- hashindex: improve error messages - -Other changes: - -- explicitly specify binary mode to open binary files -- some easy micro optimizations - - -Version 0.23.0 (2015-06-11) ---------------------------- - -Incompatible changes (compared to attic, fork related): - -- changed sw name and cli command to "borg", updated docs -- package name (and name in urls) uses "borgbackup" to have fewer collisions -- changed repo / cache internal magic strings from ATTIC* to BORG*, - changed cache location to .cache/borg/ - this means that it currently won't - accept attic repos (see issue #21 about improving that) - -Bug fixes: - -- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185 -- fix traceback when trying to do unsupported passphrase change, fixes attic #189 -- datetime does not like the year 10.000, fixes attic #139 -- fix "info" all archives stats, fixes attic #183 -- fix parsing with missing microseconds, fixes attic #282 -- fix misleading hint the fuse ImportError handler gave, fixes attic #237 -- check unpacked data from RPC for tuple type and correct length, fixes attic #127 -- fix Repository._active_txn state when lock upgrade fails -- give specific path to xattr.is_enabled(), disable symlink setattr call that - always fails -- fix test setup for 32bit platforms, partial fix for attic #196 -- upgraded versioneer, PEP440 compliance, fixes attic #257 - -New features: - -- less memory usage: add global option --no-cache-files -- check --last N (only check the last N archives) -- check: sort archives in reverse time order -- rename repo::oldname newname (rename repository) -- create -v output more informative -- create --progress (backup progress indicator) -- create --timestamp (utc string or reference file/dir) -- create: if "-" is given as path, read binary from stdin -- extract: if --stdout is given, write all extracted binary data to stdout -- extract --sparse (simple sparse file support) -- extra debug information for 'fread failed' -- delete (deletes whole repo + local cache) -- FUSE: reflect deduplication in allocated blocks -- only allow whitelisted RPC calls in server mode -- normalize source/exclude paths before matching -- use posix_fadvise to not spoil the OS cache, fixes attic #252 -- toplevel error handler: show tracebacks for better error analysis -- sigusr1 / sigint handler to print current file infos - attic PR #286 -- RPCError: include the exception args we get from remote - -Other changes: - -- source: misc. cleanups, pep8, style -- docs and faq improvements, fixes, updates -- cleanup crypto.pyx, make it easier to adapt to other AES modes -- do os.fsync like recommended in the python docs -- source: Let chunker optionally work with os-level file descriptor. -- source: Linux: remove duplicate os.fsencode calls -- source: refactor _open_rb code a bit, so it is more consistent / regular -- source: refactor indicator (status) and item processing -- source: use py.test for better testing, flake8 for code style checks -- source: fix tox >=2.0 compatibility (test runner) -- pypi package: add python version classifiers, add FreeBSD to platforms - - -Attic Changelog ---------------- - -Here you can see the full list of changes between each Attic release until Borg -forked from Attic: - -Version 0.17 -~~~~~~~~~~~~ - -(bugfix release, released on X) - -- Fix hashindex ARM memory alignment issue (#309) -- Improve hashindex error messages (#298) - -Version 0.16 -~~~~~~~~~~~~ - -(bugfix release, released on May 16, 2015) - -- Fix typo preventing the security confirmation prompt from working (#303) -- Improve handling of systems with improperly configured file system encoding (#289) -- Fix "All archives" output for attic info. (#183) -- More user friendly error message when repository key file is not found (#236) -- Fix parsing of iso 8601 timestamps with zero microseconds (#282) - -Version 0.15 -~~~~~~~~~~~~ - -(bugfix release, released on Apr 15, 2015) - -- xattr: Be less strict about unknown/unsupported platforms (#239) -- Reduce repository listing memory usage (#163). -- Fix BrokenPipeError for remote repositories (#233) -- Fix incorrect behavior with two character directory names (#265, #268) -- Require approval before accessing relocated/moved repository (#271) -- Require approval before accessing previously unknown unencrypted repositories (#271) -- Fix issue with hash index files larger than 2GB. -- Fix Python 3.2 compatibility issue with noatime open() (#164) -- Include missing pyx files in dist files (#168) - -Version 0.14 -~~~~~~~~~~~~ - -(feature release, released on Dec 17, 2014) - -- Added support for stripping leading path segments (#95) - "attic extract --strip-segments X" -- Add workaround for old Linux systems without acl_extended_file_no_follow (#96) -- Add MacPorts' path to the default openssl search path (#101) -- HashIndex improvements, eliminates unnecessary IO on low memory systems. -- Fix "Number of files" output for attic info. (#124) -- limit create file permissions so files aren't read while restoring -- Fix issue with empty xattr values (#106) - -Version 0.13 -~~~~~~~~~~~~ - -(feature release, released on Jun 29, 2014) - -- Fix sporadic "Resource temporarily unavailable" when using remote repositories -- Reduce file cache memory usage (#90) -- Faster AES encryption (utilizing AES-NI when available) -- Experimental Linux, OS X and FreeBSD ACL support (#66) -- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56) -- Fix bug where xattrs on symlinks were not correctly restored -- Added cachedir support. CACHEDIR.TAG compatible cache directories - can now be excluded using ``--exclude-caches`` (#74) -- Fix crash on extreme mtime timestamps (year 2400+) (#81) -- Fix Python 3.2 specific lockf issue (EDEADLK) - -Version 0.12 -~~~~~~~~~~~~ - -(feature release, released on April 7, 2014) - -- Python 3.4 support (#62) -- Various documentation improvements a new style -- ``attic mount`` now supports mounting an entire repository not only - individual archives (#59) -- Added option to restrict remote repository access to specific path(s): - ``attic serve --restrict-to-path X`` (#51) -- Include "all archives" size information in "--stats" output. (#54) -- Added ``--stats`` option to ``attic delete`` and ``attic prune`` -- Fixed bug where ``attic prune`` used UTC instead of the local time zone - when determining which archives to keep. -- Switch to SI units (Power of 1000 instead 1024) when printing file sizes - -Version 0.11 -~~~~~~~~~~~~ - -(feature release, released on March 7, 2014) - -- New "check" command for repository consistency checking (#24) -- Documentation improvements -- Fix exception during "attic create" with repeated files (#39) -- New "--exclude-from" option for attic create/extract/verify. -- Improved archive metadata deduplication. -- "attic verify" has been deprecated. Use "attic extract --dry-run" instead. -- "attic prune --hourly|daily|..." has been deprecated. - Use "attic prune --keep-hourly|daily|..." instead. -- Ignore xattr errors during "extract" if not supported by the filesystem. (#46) - -Version 0.10 -~~~~~~~~~~~~ - -(bugfix release, released on Jan 30, 2014) - -- Fix deadlock when extracting 0 sized files from remote repositories -- "--exclude" wildcard patterns are now properly applied to the full path - not just the file name part (#5). -- Make source code endianness agnostic (#1) - -Version 0.9 -~~~~~~~~~~~ - -(feature release, released on Jan 23, 2014) - -- Remote repository speed and reliability improvements. -- Fix sorting of segment names to ignore NFS left over files. (#17) -- Fix incorrect display of time (#13) -- Improved error handling / reporting. (#12) -- Use fcntl() instead of flock() when locking repository/cache. (#15) -- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9) -- Improved libcrypto path detection (#23). - -Version 0.8.1 -~~~~~~~~~~~~~ - -(bugfix release, released on Oct 4, 2013) - -- Fix segmentation fault issue. - -Version 0.8 -~~~~~~~~~~~ - -(feature release, released on Oct 3, 2013) - -- Fix xattr issue when backing up sshfs filesystems (#4) -- Fix issue with excessive index file size (#6) -- Support access of read only repositories. -- New syntax to enable repository encryption: - attic init --encryption="none|passphrase|keyfile". -- Detect and abort if repository is older than the cache. - - -Version 0.7 -~~~~~~~~~~~ - -(feature release, released on Aug 5, 2013) - -- Ported to FreeBSD -- Improved documentation -- Experimental: Archives mountable as FUSE filesystems. -- The "user." prefix is no longer stripped from xattrs on Linux - - -Version 0.6.1 -~~~~~~~~~~~~~ - -(bugfix release, released on July 19, 2013) - -- Fixed an issue where mtime was not always correctly restored. - - -Version 0.6 -~~~~~~~~~~~ - -First public release on July 9, 2013 +- require python >= 3.9, #6315 +- simplify libs setup, #6482 +- unbundle most bundled 3rd party code, use libs, #6316 +- use libdeflate.crc32 (Linux and all others) or zlib.crc32 (macOS) +- repository: code cleanups / simplifications +- internal crypto api: speedups / cleanups / refactorings / modernisation +- remove "borg upgrade" support for "attic backup" repos +- remove PassphraseKey code and borg key migrate-to-repokey command +- OpenBSD: build borg with OpenSSL (not: LibreSSL), #6474 +- remove support for LibreSSL, #6474 +- remove support for OpenSSL < 1.1.1 diff --git a/docs/changes_0.x.rst b/docs/changes_0.x.rst new file mode 100644 index 0000000000..39290a1105 --- /dev/null +++ b/docs/changes_0.x.rst @@ -0,0 +1,807 @@ +.. _changelog_0x: + +Change Log 0.x +============== + +Version 0.30.0 (2016-01-23) +--------------------------- + +Compatibility notes: + +- The new default logging level is WARNING. Previously, it was INFO, which was + more verbose. Use -v (or --info) to show once again log level INFO messages. + See the "general" section in the usage docs. +- for borg create, you need --list (additionally to -v) to see the long file + list (was needed so you can have e.g. --stats alone without the long list) +- see below about BORG_DELETE_I_KNOW_WHAT_I_AM_DOING (was: + BORG_CHECK_I_KNOW_WHAT_I_AM_DOING) + +Bug fixes: + +- fix crash when using borg create --dry-run --keep-tag-files, #570 +- make sure teardown with cleanup happens for Cache and RepositoryCache, + avoiding leftover locks and TEMP dir contents, #285 (partially), #548 +- fix locking KeyError, partial fix for #502 +- log stats consistently, #526 +- add abbreviated weekday to timestamp format, fixes #496 +- strip whitespace when loading exclusions from file +- unset LD_LIBRARY_PATH before invoking ssh, fixes strange OpenSSL library + version warning when using the borg binary, #514 +- add some error handling/fallback for C library loading, #494 +- added BORG_DELETE_I_KNOW_WHAT_I_AM_DOING for check in "borg delete", #503 +- remove unused "repair" rpc method name + +New features: + +- borg create: implement exclusions using regular expression patterns. +- borg create: implement inclusions using patterns. +- borg extract: support patterns, #361 +- support different styles for patterns: + + - fnmatch (`fm:` prefix, default when omitted), like borg <= 0.29. + - shell (`sh:` prefix) with `*` not matching directory separators and + `**/` matching 0..n directories + - path prefix (`pp:` prefix, for unifying borg create pp1 pp2 into the + patterns system), semantics like in borg <= 0.29 + - regular expression (`re:`), new! +- --progress option for borg upgrade (#291) and borg delete +- update progress indication more often (e.g. for borg create within big + files or for borg check repo), #500 +- finer chunker granularity for items metadata stream, #547, #487 +- borg create --list now used (additionally to -v) to enable the verbose + file list output +- display borg version below tracebacks, #532 + +Other changes: + +- hashtable size (and thus: RAM and disk consumption) follows a growth policy: + grows fast while small, grows slower when getting bigger, #527 +- Vagrantfile: use pyinstaller 3.1 to build binaries, freebsd sqlite3 fix, + fixes #569 +- no separate binaries for centos6 any more because the generic linux binaries + also work on centos6 (or in general: on systems with a slightly older glibc + than debian7 +- dev environment: require virtualenv<14.0 so we get a py32 compatible pip +- docs: + + - add space-saving chunks.archive.d trick to FAQ + - important: clarify -v and log levels in usage -> general, please read! + - sphinx configuration: create a simple man page from usage docs + - add a repo server setup example + - disable unneeded SSH features in authorized_keys examples for security. + - borg prune only knows "--keep-within" and not "--within" + - add gource video to resources docs, #507 + - add netbsd install instructions + - authors: make it more clear what refers to borg and what to attic + - document standalone binary requirements, #499 + - rephrase the mailing list section + - development docs: run build_api and build_usage before tagging release + - internals docs: hash table max. load factor is 0.75 now + - markup, typo, grammar, phrasing, clarifications and other fixes. + - add gcc gcc-c++ to redhat/fedora/corora install docs, fixes #583 + + +Version 0.29.0 (2015-12-13) +--------------------------- + +Compatibility notes: + +- when upgrading to 0.29.0 you need to upgrade client as well as server + installations due to the locking and commandline interface changes otherwise + you'll get an error msg about a RPC protocol mismatch or a wrong commandline + option. + if you run a server that needs to support both old and new clients, it is + suggested that you have a "borg-0.28.2" and a "borg-0.29.0" command. + clients then can choose via e.g. "borg --remote-path=borg-0.29.0 ...". +- the default waiting time for a lock changed from infinity to 1 second for a + better interactive user experience. if the repo you want to access is + currently locked, borg will now terminate after 1s with an error message. + if you have scripts that shall wait for the lock for a longer time, use + --lock-wait N (with N being the maximum wait time in seconds). + +Bug fixes: + +- hash table tuning (better chosen hashtable load factor 0.75 and prime initial + size of 1031 gave ~1000x speedup in some scenarios) +- avoid creation of an orphan lock for one case, #285 +- --keep-tag-files: fix file mode and multiple tag files in one directory, #432 +- fixes for "borg upgrade" (attic repo converter), #466 +- remove --progress isatty magic (and also --no-progress option) again, #476 +- borg init: display proper repo URL +- fix format of umask in help pages, #463 + +New features: + +- implement --lock-wait, support timeout for UpgradableLock, #210 +- implement borg break-lock command, #157 +- include system info below traceback, #324 +- sane remote logging, remote stderr, #461: + + - remote log output: intercept it and log it via local logging system, + with "Remote: " prefixed to message. log remote tracebacks. + - remote stderr: output it to local stderr with "Remote: " prefixed. +- add --debug and --info (same as --verbose) to set the log level of the + builtin logging configuration (which otherwise defaults to warning), #426 + note: there are few messages emitted at DEBUG level currently. +- optionally configure logging via env var BORG_LOGGING_CONF +- add --filter option for status characters: e.g. to show only the added + or modified files (and also errors), use "borg create -v --filter=AME ...". +- more progress indicators, #394 +- use ISO-8601 date and time format, #375 +- "borg check --prefix" to restrict archive checking to that name prefix, #206 + +Other changes: + +- hashindex_add C implementation (speed up cache re-sync for new archives) +- increase FUSE read_size to 1024 (speed up metadata operations) +- check/delete/prune --save-space: free unused segments quickly, #239 +- increase rpc protocol version to 2 (see also Compatibility notes), #458 +- silence borg by default (via default log level WARNING) +- get rid of C compiler warnings, #391 +- upgrade OS X FUSE to 3.0.9 on the OS X binary build system +- use python 3.5.1 to build binaries +- docs: + + - new mailing list borgbackup@python.org, #468 + - readthedocs: color and logo improvements + - load coverage icons over SSL (avoids mixed content) + - more precise binary installation steps + - update release procedure docs about OS X FUSE + - FAQ entry about unexpected 'A' status for unchanged file(s), #403 + - add docs about 'E' file status + - add "borg upgrade" docs, #464 + - add developer docs about output and logging + - clarify encryption, add note about client-side encryption + - add resources section, with videos, talks, presentations, #149 + - Borg moved to Arch Linux [community] + - fix wrong installation instructions for archlinux + + +Version 0.28.2 (2015-11-15) +--------------------------- + +New features: + +- borg create --exclude-if-present TAGFILE - exclude directories that have the + given file from the backup. You can additionally give --keep-tag-files to + preserve just the directory roots and the tag-files (but not back up other + directory contents), #395, attic #128, attic #142 + +Other changes: + +- do not create docs sources at build time (just have them in the repo), + completely remove have_cython() hack, do not use the "mock" library at build + time, #384 +- avoid hidden import, make it easier for PyInstaller, easier fix for #218 +- docs: + + - add description of item flags / status output, fixes #402 + - explain how to regenerate usage and API files (build_api or + build_usage) and when to commit usage files directly into git, #384 + - minor install docs improvements + + +Version 0.28.1 (2015-11-08) +--------------------------- + +Bug fixes: + +- do not try to build api / usage docs for production install, + fixes unexpected "mock" build dependency, #384 + +Other changes: + +- avoid using msgpack.packb at import time +- fix formatting issue in changes.rst +- fix build on readthedocs + + +Version 0.28.0 (2015-11-08) +--------------------------- + +Compatibility notes: + +- changed return codes (exit codes), see docs. in short: + old: 0 = ok, 1 = error. now: 0 = ok, 1 = warning, 2 = error + +New features: + +- refactor return codes (exit codes), fixes #61 +- add --show-rc option enable "terminating with X status, rc N" output, fixes 58, #351 +- borg create backups atime and ctime additionally to mtime, fixes #317 + - extract: support atime additionally to mtime + - FUSE: support ctime and atime additionally to mtime +- support borg --version +- emit a warning if we have a slow msgpack installed +- borg list --prefix=thishostname- REPO, fixes #205 +- Debug commands (do not use except if you know what you do: debug-get-obj, + debug-put-obj, debug-delete-obj, debug-dump-archive-items. + +Bug fixes: + +- setup.py: fix bug related to BORG_LZ4_PREFIX processing +- fix "check" for repos that have incomplete chunks, fixes #364 +- borg mount: fix unlocking of repository at umount time, fixes #331 +- fix reading files without touching their atime, #334 +- non-ascii ACL fixes for Linux, FreeBSD and OS X, #277 +- fix acl_use_local_uid_gid() and add a test for it, attic #359 +- borg upgrade: do not upgrade repositories in place by default, #299 +- fix cascading failure with the index conversion code, #269 +- borg check: implement 'cmdline' archive metadata value decoding, #311 +- fix RobustUnpacker, it missed some metadata keys (new atime and ctime keys + were missing, but also bsdflags). add check for unknown metadata keys. +- create from stdin: also save atime, ctime (cosmetic) +- use default_notty=False for confirmations, fixes #345 +- vagrant: fix msgpack installation on centos, fixes #342 +- deal with unicode errors for symlinks in same way as for regular files and + have a helpful warning message about how to fix wrong locale setup, fixes #382 +- add ACL keys the RobustUnpacker must know about + +Other changes: + +- improve file size displays, more flexible size formatters +- explicitly commit to the units standard, #289 +- archiver: add E status (means that an error occurred when processing this + (single) item +- do binary releases via "github releases", closes #214 +- create: use -x and --one-file-system (was: --do-not-cross-mountpoints), #296 +- a lot of changes related to using "logging" module and screen output, #233 +- show progress display if on a tty, output more progress information, #303 +- factor out status output so it is consistent, fix surrogates removal, + maybe fixes #309 +- move away from RawConfigParser to ConfigParser +- archive checker: better error logging, give chunk_id and sequence numbers + (can be used together with borg debug-dump-archive-items). +- do not mention the deprecated passphrase mode +- emit a deprecation warning for --compression N (giving a just a number) +- misc .coverragerc fixes (and coverage measurement improvements), fixes #319 +- refactor confirmation code, reduce code duplication, add tests +- prettier error messages, fixes #307, #57 +- tests: + + - add a test to find disk-full issues, #327 + - travis: also run tests on Python 3.5 + - travis: use tox -r so it rebuilds the tox environments + - test the generated pyinstaller-based binary by archiver unit tests, #215 + - vagrant: tests: announce whether fakeroot is used or not + - vagrant: add vagrant user to fuse group for debianoid systems also + - vagrant: llfuse install on darwin needs pkgconfig installed + - vagrant: use pyinstaller from develop branch, fixes #336 + - benchmarks: test create, extract, list, delete, info, check, help, fixes #146 + - benchmarks: test with both the binary and the python code + - archiver tests: test with both the binary and the python code, fixes #215 + - make basic test more robust +- docs: + + - moved docs to borgbackup.readthedocs.org, #155 + - a lot of fixes and improvements, use mobile-friendly RTD standard theme + - use zlib,6 compression in some examples, fixes #275 + - add missing rename usage to docs, closes #279 + - include the help offered by borg help in the usage docs, fixes #293 + - include a list of major changes compared to attic into README, fixes #224 + - add OS X install instructions, #197 + - more details about the release process, #260 + - fix linux glibc requirement (binaries built on debian7 now) + - build: move usage and API generation to setup.py + - update docs about return codes, #61 + - remove api docs (too much breakage on rtd) + - borgbackup install + basics presentation (asciinema) + - describe the current style guide in documentation + - add section about debug commands + - warn about not running out of space + - add example for rename + - improve chunker params docs, fixes #362 + - minor development docs update + + +Version 0.27.0 (2015-10-07) +--------------------------- + +New features: + +- "borg upgrade" command - attic -> borg one time converter / migration, #21 +- temporary hack to avoid using lots of disk space for chunks.archive.d, #235: + To use it: rm -rf chunks.archive.d ; touch chunks.archive.d +- respect XDG_CACHE_HOME, attic #181 +- add support for arbitrary SSH commands, attic #99 +- borg delete --cache-only REPO (only delete cache, not REPO), attic #123 + + +Bug fixes: + +- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow + down observed when running the Centos6-built binary on Ubuntu, #222 +- do not crash on empty lock.roster, fixes #232 +- fix multiple issues with the cache config version check, #234 +- fix segment entry header size check, attic #352 + plus other error handling improvements / code deduplication there. +- always give segment and offset in repo IntegrityErrors + + +Other changes: + +- stop producing binary wheels, remove docs about it, #147 +- docs: + - add warning about prune + - generate usage include files only as needed + - development docs: add Vagrant section + - update / improve / reformat FAQ + - hint to single-file pyinstaller binaries from README + + +Version 0.26.1 (2015-09-28) +--------------------------- + +This is a minor update, just docs and new pyinstaller binaries. + +- docs update about python and binary requirements +- better docs for --read-special, fix #220 +- re-built the binaries, fix #218 and #213 (glibc version issue) +- update web site about single-file pyinstaller binaries + +Note: if you did a python-based installation, there is no need to upgrade. + + +Version 0.26.0 (2015-09-19) +--------------------------- + +New features: + +- Faster cache sync (do all in one pass, remove tar/compression stuff), #163 +- BORG_REPO env var to specify the default repo, #168 +- read special files as if they were regular files, #79 +- implement borg create --dry-run, attic issue #267 +- Normalize paths before pattern matching on OS X, #143 +- support OpenBSD and NetBSD (except xattrs/ACLs) +- support / run tests on Python 3.5 + +Bug fixes: + +- borg mount repo: use absolute path, attic #200, attic #137 +- chunker: use off_t to get 64bit on 32bit platform, #178 +- initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0) +- fix reaction to "no" answer at delete repo prompt, #182 +- setup.py: detect lz4.h header file location +- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194) +- fix for obtaining ``char *`` from temporary Python value (old code causes + a compile error on Mint 17.2) +- llfuse 0.41 install troubles on some platforms, require < 0.41 + (UnicodeDecodeError exception due to non-ascii llfuse setup.py) +- cython code: add some int types to get rid of unspecific python add / + subtract operations (avoid ``undefined symbol FPE_``... error on some platforms) +- fix verbose mode display of stdin backup +- extract: warn if a include pattern never matched, fixes #209, + implement counters for Include/ExcludePatterns +- archive names with slashes are invalid, attic issue #180 +- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined - + fixes building on OpenBSD. + +Other changes: + +- detect inconsistency / corruption / hash collision, #170 +- replace versioneer with setuptools_scm, #106 +- docs: + + - pkg-config is needed for llfuse installation + - be more clear about pruning, attic issue #132 +- unit tests: + + - xattr: ignore security.selinux attribute showing up + - ext3 seems to need a bit more space for a sparse file + - do not test lzma level 9 compression (avoid MemoryError) + - work around strange mtime granularity issue on netbsd, fixes #204 + - ignore st_rdev if file is not a block/char device, fixes #203 + - stay away from the setgid and sticky mode bits +- use Vagrant to do easy cross-platform testing (#196), currently: + + - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit + - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit + - Centos 7 64bit + - FreeBSD 10.2 64bit + - OpenBSD 5.7 64bit + - NetBSD 6.1.5 64bit + - Darwin (OS X Yosemite) + + +Version 0.25.0 (2015-08-29) +--------------------------- + +Compatibility notes: + +- lz4 compression library (liblz4) is a new requirement (#156) +- the new compression code is very compatible: as long as you stay with zlib + compression, older borg releases will still be able to read data from a + repo/archive made with the new code (note: this is not the case for the + default "none" compression, use "zlib,0" if you want a "no compression" mode + that can be read by older borg). Also the new code is able to read repos and + archives made with older borg versions (for all zlib levels 0..9). + +Deprecations: + +- --compression N (with N being a number, as in 0.24) is deprecated. + We keep the --compression 0..9 for now not to break scripts, but it is + deprecated and will be removed later, so better fix your scripts now: + --compression 0 (as in 0.24) is the same as --compression zlib,0 (now). + BUT: if you do not want compression, use --compression none + (which is the default). + --compression 1 (in 0.24) is the same as --compression zlib,1 (now) + --compression 9 (in 0.24) is the same as --compression zlib,9 (now) + +New features: + +- create --compression none (default, means: do not compress, just pass through + data "as is". this is more efficient than zlib level 0 as used in borg 0.24) +- create --compression lz4 (super-fast, but not very high compression) +- create --compression zlib,N (slower, higher compression, default for N is 6) +- create --compression lzma,N (slowest, highest compression, default N is 6) +- honor the nodump flag (UF_NODUMP) and do not back up such items +- list --short just outputs a simple list of the files/directories in an archive + +Bug fixes: + +- fixed --chunker-params parameter order confusion / malfunction, fixes #154 +- close fds of segments we delete (during compaction) +- close files which fell out the lrucache +- fadvise DONTNEED now is only called for the byte range actually read, not for + the whole file, fixes #158. +- fix issue with negative "all archives" size, fixes #165 +- restore_xattrs: ignore if setxattr fails with EACCES, fixes #162 + +Other changes: + +- remove fakeroot requirement for tests, tests run faster without fakeroot + (test setup does not fail any more without fakeroot, so you can run with or + without fakeroot), fixes #151 and #91. +- more tests for archiver +- recover_segment(): don't assume we have an fd for segment +- lrucache refactoring / cleanup, add dispose function, py.test tests +- generalize hashindex code for any key length (less hardcoding) +- lock roster: catch file not found in remove() method and ignore it +- travis CI: use requirements file +- improved docs: + + - replace hack for llfuse with proper solution (install libfuse-dev) + - update docs about compression + - update development docs about fakeroot + - internals: add some words about lock files / locking system + - support: mention BountySource and for what it can be used + - theme: use a lighter green + - add pypi, wheel, dist package based install docs + - split install docs into system-specific preparations and generic instructions + + +Version 0.24.0 (2015-08-09) +--------------------------- + +Incompatible changes (compared to 0.23): + +- borg now always issues --umask NNN option when invoking another borg via ssh + on the repository server. By that, it's making sure it uses the same umask + for remote repos as for local ones. Because of this, you must upgrade both + server and client(s) to 0.24. +- the default umask is 077 now (if you do not specify via --umask) which might + be a different one as you used previously. The default umask avoids that + you accidentally give access permissions for group and/or others to files + created by borg (e.g. the repository). + +Deprecations: + +- "--encryption passphrase" mode is deprecated, see #85 and #97. + See the new "--encryption repokey" mode for a replacement. + +New features: + +- borg create --chunker-params ... to configure the chunker, fixes #16 + (attic #302, attic #300, and somehow also #41). + This can be used to reduce memory usage caused by chunk management overhead, + so borg does not create a huge chunks index/repo index and eats all your RAM + if you back up lots of data in huge files (like VM disk images). + See docs/misc/create_chunker-params.txt for more information. +- borg info now reports chunk counts in the chunk index. +- borg create --compression 0..9 to select zlib compression level, fixes #66 + (attic #295). +- borg init --encryption repokey (to store the encryption key into the repo), + fixes #85 +- improve at-end error logging, always log exceptions and set exit_code=1 +- LoggedIO: better error checks / exceptions / exception handling +- implement --remote-path to allow non-default-path borg locations, #125 +- implement --umask M and use 077 as default umask for better security, #117 +- borg check: give a named single archive to it, fixes #139 +- cache sync: show progress indication +- cache sync: reimplement the chunk index merging in C + +Bug fixes: + +- fix segfault that happened for unreadable files (chunker: n needs to be a + signed size_t), #116 +- fix the repair mode, #144 +- repo delete: add destroy to allowed rpc methods, fixes issue #114 +- more compatible repository locking code (based on mkdir), maybe fixes #92 + (attic #317, attic #201). +- better Exception msg if no Borg is installed on the remote repo server, #56 +- create a RepositoryCache implementation that can cope with >2GiB, + fixes attic #326. +- fix Traceback when running check --repair, attic #232 +- clarify help text, fixes #73. +- add help string for --no-files-cache, fixes #140 + +Other changes: + +- improved docs: + + - added docs/misc directory for misc. writeups that won't be included + "as is" into the html docs. + - document environment variables and return codes (attic #324, attic #52) + - web site: add related projects, fix web site url, IRC #borgbackup + - Fedora/Fedora-based install instructions added to docs + - Cygwin-based install instructions added to docs + - updated AUTHORS + - add FAQ entries about redundancy / integrity + - clarify that borg extract uses the cwd as extraction target + - update internals doc about chunker params, memory usage and compression + - added docs about development + - add some words about resource usage in general + - document how to back up a raw disk + - add note about how to run borg from virtual env + - add solutions for (ll)fuse installation problems + - document what borg check does, fixes #138 + - reorganize borgbackup.github.io sidebar, prev/next at top + - deduplicate and refactor the docs / README.rst + +- use borg-tmp as prefix for temporary files / directories +- short prune options without "keep-" are deprecated, do not suggest them +- improved tox configuration +- remove usage of unittest.mock, always use mock from pypi +- use entrypoints instead of scripts, for better use of the wheel format and + modern installs +- add requirements.d/development.txt and modify tox.ini +- use travis-ci for testing based on Linux and (new) OS X +- use coverage.py, pytest-cov and codecov.io for test coverage support + +I forgot to list some stuff already implemented in 0.23.0, here they are: + +New features: + +- efficient archive list from manifest, meaning a big speedup for slow + repo connections and "list ", "delete ", "prune" (attic #242, + attic #167) +- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18 +- hashindex: improve error messages + +Other changes: + +- explicitly specify binary mode to open binary files +- some easy micro optimizations + + +Version 0.23.0 (2015-06-11) +--------------------------- + +Incompatible changes (compared to attic, fork related): + +- changed sw name and cli command to "borg", updated docs +- package name (and name in urls) uses "borgbackup" to have fewer collisions +- changed repo / cache internal magic strings from ATTIC* to BORG*, + changed cache location to .cache/borg/ - this means that it currently won't + accept attic repos (see issue #21 about improving that) + +Bug fixes: + +- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185 +- fix traceback when trying to do unsupported passphrase change, fixes attic #189 +- datetime does not like the year 10.000, fixes attic #139 +- fix "info" all archives stats, fixes attic #183 +- fix parsing with missing microseconds, fixes attic #282 +- fix misleading hint the fuse ImportError handler gave, fixes attic #237 +- check unpacked data from RPC for tuple type and correct length, fixes attic #127 +- fix Repository._active_txn state when lock upgrade fails +- give specific path to xattr.is_enabled(), disable symlink setattr call that + always fails +- fix test setup for 32bit platforms, partial fix for attic #196 +- upgraded versioneer, PEP440 compliance, fixes attic #257 + +New features: + +- less memory usage: add global option --no-cache-files +- check --last N (only check the last N archives) +- check: sort archives in reverse time order +- rename repo::oldname newname (rename repository) +- create -v output more informative +- create --progress (backup progress indicator) +- create --timestamp (utc string or reference file/dir) +- create: if "-" is given as path, read binary from stdin +- extract: if --stdout is given, write all extracted binary data to stdout +- extract --sparse (simple sparse file support) +- extra debug information for 'fread failed' +- delete (deletes whole repo + local cache) +- FUSE: reflect deduplication in allocated blocks +- only allow whitelisted RPC calls in server mode +- normalize source/exclude paths before matching +- use posix_fadvise not to spoil the OS cache, fixes attic #252 +- toplevel error handler: show tracebacks for better error analysis +- sigusr1 / sigint handler to print current file infos - attic PR #286 +- RPCError: include the exception args we get from remote + +Other changes: + +- source: misc. cleanups, pep8, style +- docs and faq improvements, fixes, updates +- cleanup crypto.pyx, make it easier to adapt to other AES modes +- do os.fsync like recommended in the python docs +- source: Let chunker optionally work with os-level file descriptor. +- source: Linux: remove duplicate os.fsencode calls +- source: refactor _open_rb code a bit, so it is more consistent / regular +- source: refactor indicator (status) and item processing +- source: use py.test for better testing, flake8 for code style checks +- source: fix tox >=2.0 compatibility (test runner) +- pypi package: add python version classifiers, add FreeBSD to platforms + + +Attic Changelog +--------------- + +Here you can see the full list of changes between each Attic release until Borg +forked from Attic: + +Version 0.17 +~~~~~~~~~~~~ + +(bugfix release, released on X) + +- Fix hashindex ARM memory alignment issue (#309) +- Improve hashindex error messages (#298) + +Version 0.16 +~~~~~~~~~~~~ + +(bugfix release, released on May 16, 2015) + +- Fix typo preventing the security confirmation prompt from working (#303) +- Improve handling of systems with improperly configured file system encoding (#289) +- Fix "All archives" output for attic info. (#183) +- More user friendly error message when repository key file is not found (#236) +- Fix parsing of iso 8601 timestamps with zero microseconds (#282) + +Version 0.15 +~~~~~~~~~~~~ + +(bugfix release, released on Apr 15, 2015) + +- xattr: Be less strict about unknown/unsupported platforms (#239) +- Reduce repository listing memory usage (#163). +- Fix BrokenPipeError for remote repositories (#233) +- Fix incorrect behavior with two character directory names (#265, #268) +- Require approval before accessing relocated/moved repository (#271) +- Require approval before accessing previously unknown unencrypted repositories (#271) +- Fix issue with hash index files larger than 2GB. +- Fix Python 3.2 compatibility issue with noatime open() (#164) +- Include missing pyx files in dist files (#168) + +Version 0.14 +~~~~~~~~~~~~ + +(feature release, released on Dec 17, 2014) + +- Added support for stripping leading path segments (#95) + "attic extract --strip-segments X" +- Add workaround for old Linux systems without acl_extended_file_no_follow (#96) +- Add MacPorts' path to the default openssl search path (#101) +- HashIndex improvements, eliminates unnecessary IO on low memory systems. +- Fix "Number of files" output for attic info. (#124) +- limit create file permissions so files aren't read while restoring +- Fix issue with empty xattr values (#106) + +Version 0.13 +~~~~~~~~~~~~ + +(feature release, released on Jun 29, 2014) + +- Fix sporadic "Resource temporarily unavailable" when using remote repositories +- Reduce file cache memory usage (#90) +- Faster AES encryption (utilizing AES-NI when available) +- Experimental Linux, OS X and FreeBSD ACL support (#66) +- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56) +- Fix bug where xattrs on symlinks were not correctly restored +- Added cachedir support. CACHEDIR.TAG compatible cache directories + can now be excluded using ``--exclude-caches`` (#74) +- Fix crash on extreme mtime timestamps (year 2400+) (#81) +- Fix Python 3.2 specific lockf issue (EDEADLK) + +Version 0.12 +~~~~~~~~~~~~ + +(feature release, released on April 7, 2014) + +- Python 3.4 support (#62) +- Various documentation improvements a new style +- ``attic mount`` now supports mounting an entire repository not only + individual archives (#59) +- Added option to restrict remote repository access to specific path(s): + ``attic serve --restrict-to-path X`` (#51) +- Include "all archives" size information in "--stats" output. (#54) +- Added ``--stats`` option to ``attic delete`` and ``attic prune`` +- Fixed bug where ``attic prune`` used UTC instead of the local time zone + when determining which archives to keep. +- Switch to SI units (Power of 1000 instead 1024) when printing file sizes + +Version 0.11 +~~~~~~~~~~~~ + +(feature release, released on March 7, 2014) + +- New "check" command for repository consistency checking (#24) +- Documentation improvements +- Fix exception during "attic create" with repeated files (#39) +- New "--exclude-from" option for attic create/extract/verify. +- Improved archive metadata deduplication. +- "attic verify" has been deprecated. Use "attic extract --dry-run" instead. +- "attic prune --hourly|daily|..." has been deprecated. + Use "attic prune --keep-hourly|daily|..." instead. +- Ignore xattr errors during "extract" if not supported by the filesystem. (#46) + +Version 0.10 +~~~~~~~~~~~~ + +(bugfix release, released on Jan 30, 2014) + +- Fix deadlock when extracting 0 sized files from remote repositories +- "--exclude" wildcard patterns are now properly applied to the full path + not just the file name part (#5). +- Make source code endianness agnostic (#1) + +Version 0.9 +~~~~~~~~~~~ + +(feature release, released on Jan 23, 2014) + +- Remote repository speed and reliability improvements. +- Fix sorting of segment names to ignore NFS left over files. (#17) +- Fix incorrect display of time (#13) +- Improved error handling / reporting. (#12) +- Use fcntl() instead of flock() when locking repository/cache. (#15) +- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9) +- Improved libcrypto path detection (#23). + +Version 0.8.1 +~~~~~~~~~~~~~ + +(bugfix release, released on Oct 4, 2013) + +- Fix segmentation fault issue. + +Version 0.8 +~~~~~~~~~~~ + +(feature release, released on Oct 3, 2013) + +- Fix xattr issue when backing up sshfs filesystems (#4) +- Fix issue with excessive index file size (#6) +- Support access of read only repositories. +- New syntax to enable repository encryption: + attic init --encryption="none|passphrase|keyfile". +- Detect and abort if repository is older than the cache. + + +Version 0.7 +~~~~~~~~~~~ + +(feature release, released on Aug 5, 2013) + +- Ported to FreeBSD +- Improved documentation +- Experimental: Archives mountable as FUSE filesystems. +- The "user." prefix is no longer stripped from xattrs on Linux + + +Version 0.6.1 +~~~~~~~~~~~~~ + +(bugfix release, released on July 19, 2013) + +- Fixed an issue where mtime was not always correctly restored. + + +Version 0.6 +~~~~~~~~~~~ + +First public release on July 9, 2013 diff --git a/docs/changes_1.x.rst b/docs/changes_1.x.rst new file mode 100644 index 0000000000..3366a90b8c --- /dev/null +++ b/docs/changes_1.x.rst @@ -0,0 +1,4879 @@ +.. _important_notes_1x: + +Important notes 1.x +=================== + +This section provides information about security and corruption issues. + +.. _archives_tam_vuln: + +Pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811) +---------------------------------------------------------- + +A flaw in the cryptographic authentication scheme in Borg allowed an attacker to +fake archives and potentially indirectly cause backup data loss in the repository. + +The attack requires an attacker to be able to + +1. insert files (with no additional headers) into backups +2. gain write access to the repository + +This vulnerability does not disclose plaintext to the attacker, nor does it +affect the authenticity of existing archives. + +Creating plausible fake archives may be feasible for empty or small archives, +but is unlikely for large archives. + +The fix enforces checking the TAM authentication tag of archives at critical +places. Borg now considers archives without TAM as garbage or an attack. + +We are not aware of others having discovered, disclosed or exploited this vulnerability. + +Below, if we speak of borg 1.2.6, we mean a borg version >= 1.2.6 **or** a +borg version that has the relevant security patches for this vulnerability applied +(could be also an older version in that case). + +Steps you must take to upgrade a repository (this applies to all kinds of repos +no matter what encryption mode they use, including "none"): + +1. Upgrade all clients using this repository to borg 1.2.6. + Note: it is not required to upgrade a server, except if the server-side borg + is also used as a client (and not just for "borg serve"). + + Do **not** run ``borg check`` with borg 1.2.6 before completing the upgrade steps: + + - ``borg check`` would complain about archives without a valid archive TAM. + - ``borg check --repair`` would remove such archives! +2. Run: ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg info --debug 2>&1 | grep TAM | grep -i manifest`` + + a) If you get "TAM-verified manifest", continue with 3. + b) If you get "Manifest TAM not found and not required", run + ``borg upgrade --tam --force `` *on every client*. + +3. Run: ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg list --consider-checkpoints --format='{name} {time} tam:{tam}{NL}' `` + + "tam:verified" means that the archive has a valid TAM authentication. + "tam:none" is expected as output for archives created by borg <1.0.9. + "tam:none" is also expected for archives resulting from a borg rename + or borg recreate operation (see #7791). + "tam:none" could also come from archives created by an attacker. + You should verify that "tam:none" archives are authentic and not malicious + (== have good content, have correct timestamp, can be extracted successfully). + In case you find crappy/malicious archives, you must delete them before proceeding. + In low-risk, trusted environments, you may decide on your own risk to skip step 3 + and just trust in everything being OK. + +4. If there are no tam:none archives left at this point, you can skip this step. + Run ``BORG_WORKAROUNDS=ignore_invalid_archive_tam borg upgrade --archives-tam ``. + This will unconditionally add a correct archive TAM to all archives not having one. + ``borg check`` would consider TAM-less or invalid-TAM archives as garbage or a potential attack. + To see that all archives now are "tam:verified" run: ``borg list --consider-checkpoints --format='{name} {time} tam:{tam}{NL}' `` + +5. Please note that you should never use BORG_WORKAROUNDS=ignore_invalid_archive_tam + for normal production operations - it is only needed once to get the archives in a + repository into a good state. All archives have a valid TAM now. + +Vulnerability time line: + +* 2023-06-13: Vulnerability discovered during code review by Thomas Waldmann +* 2023-06-13...: Work on fixing the issue, upgrade procedure, docs. +* 2023-06-30: CVE was assigned via Github CNA +* 2023-06-30 .. 2023-08-29: Fixed issue, code review, docs, testing. +* 2023-08-30: Released fixed version 1.2.5 (broken upgrade procedure for some repos) +* 2023-08-31: Released fixed version 1.2.6 (fixes upgrade procedure) + +.. _hashindex_set_bug: + +Pre-1.1.11 potential index corruption / data loss issue +------------------------------------------------------- + +A bug was discovered in our hashtable code, see issue #4829. +The code is used for the client-side chunks cache and the server-side repo index. + +Although borg uses the hashtables very heavily, the index corruption did not +happen too frequently, because it needed specific conditions to happen. + +Data loss required even more specific conditions, so it should be rare (and +also detectable via borg check). + +You might be affected if borg crashed with / complained about: + +- AssertionError: Corrupted segment reference count - corrupted index or hints +- ObjectNotFound: Object with key ... not found in repository ... +- Index mismatch for key b'...'. (..., ...) != (-1, -1) +- ValueError: stats_against: key contained in self but not in master_index. + +Advised procedure to fix any related issue in your indexes/caches: + +- install fixed borg code (on client AND server) +- for all of your clients and repos remove the cache by: + + borg delete --cache-only YOURREPO + + (later, the cache will be re-built automatically) +- for all your repos, rebuild the repo index by: + + borg check --repair YOURREPO + + This will also check all archives and detect if there is any data-loss issue. + +Affected branches / releases: + +- fd06497 introduced the bug into 1.1-maint branch - it affects all borg 1.1.x since 1.1.0b4. +- fd06497 introduced the bug into master branch - it affects all borg 1.2.0 alpha releases. +- c5cd882 introduced the bug into 1.0-maint branch - it affects all borg 1.0.x since 1.0.11rc1. + +The bug was fixed by: + +- 701159a fixes the bug in 1.1-maint branch - will be released with borg 1.1.11. +- fa63150 fixes the bug in master branch - will be released with borg 1.2.0a8. +- 7bb90b6 fixes the bug in 1.0-maint branch. Branch is EOL, no new release is planned as of now. + +.. _broken_validator: + +Pre-1.1.4 potential data corruption issue +----------------------------------------- + +A data corruption bug was discovered in borg check --repair, see issue #3444. + +This is a 1.1.x regression, releases < 1.1 (e.g. 1.0.x) are not affected. + +To avoid data loss, you must not run borg check --repair using an unfixed version +of borg 1.1.x. The first official release that has the fix is 1.1.4. + +Package maintainers may have applied the fix to updated packages of 1.1.x (x<4) +though, see the package maintainer's package changelog to make sure. + +If you never had missing item metadata chunks, the bug has not affected you +even if you did run borg check --repair with an unfixed version. + +When borg check --repair tried to repair corrupt archives that miss item metadata +chunks, the resync to valid metadata in still present item metadata chunks +malfunctioned. This was due to a broken validator that considered all (even valid) +item metadata as invalid. As they were considered invalid, borg discarded them. +Practically, that means the affected files, directories or other fs objects were +discarded from the archive. + +Due to the malfunction, the process was extremely slow, but if you let it +complete, borg would have created a "repaired" archive that has lost a lot of items. +If you interrupted borg check --repair because it was so strangely slow (killing +borg somehow, e.g. Ctrl-C) the transaction was rolled back and no corruption occurred. + +The log message indicating the precondition for the bug triggering looks like: + + item metadata chunk missing [chunk: 001056_bdee87d...a3e50d] + +If you never had that in your borg check --repair runs, you're not affected. + +But if you're unsure or you actually have seen that, better check your archives. +By just using "borg list repo::archive" you can see if all expected filesystem +items are listed. + +.. _tam_vuln: + +Pre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099) +---------------------------------------------------------- + +A flaw in the cryptographic authentication scheme in Borg allowed an attacker +to spoof the manifest. The attack requires an attacker to be able to + +1. insert files (with no additional headers) into backups +2. gain write access to the repository + +This vulnerability does not disclose plaintext to the attacker, nor does it +affect the authenticity of existing archives. + +The vulnerability allows an attacker to create a spoofed manifest (the list of archives). +Creating plausible fake archives may be feasible for small archives, but is unlikely +for large archives. + +The fix adds a separate authentication tag to the manifest. For compatibility +with prior versions this authentication tag is *not* required by default +for existing repositories. Repositories created with 1.0.9 and later require it. + +Steps you should take: + +1. Upgrade all clients to 1.0.9 or later. +2. Run ``borg upgrade --tam `` *on every client* for *each* repository. +3. This will list all archives, including archive IDs, for easy comparison with your logs. +4. Done. + +Prior versions can access and modify repositories with this measure enabled, however, +to 1.0.9 or later their modifications are indiscernible from an attack and will +raise an error until the below procedure is followed. We are aware that this can +be annoying in some circumstances, but don't see a way to fix the vulnerability +otherwise. + +In case a version prior to 1.0.9 is used to modify a repository where above procedure +was completed, and now you get an error message from other clients: + +1. ``borg upgrade --tam --force `` once with *any* client suffices. + +This attack is mitigated by: + +- Noting/logging ``borg list``, ``borg info``, or ``borg create --stats``, which + contain the archive IDs. + +We are not aware of others having discovered, disclosed or exploited this vulnerability. + +Vulnerability time line: + +* 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore) +* 2016-11-20: First patch +* 2016-12-20: Released fixed version 1.0.9 +* 2017-01-02: CVE was assigned +* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source) + +.. _attic013_check_corruption: + +Pre-1.0.9 potential data loss +----------------------------- + +If you have archives in your repository that were made with attic <= 0.13 +(and later migrated to borg), running borg check would report errors in these +archives. See issue #1837. + +The reason for this is a invalid (and useless) metadata key that was +always added due to a bug in these old attic versions. + +If you run borg check --repair, things escalate quickly: all archive items +with invalid metadata will be killed. Due to that attic bug, that means all +items in all archives made with these old attic versions. + + +Pre-1.0.4 potential repo corruption +----------------------------------- + +Some external errors (like network or disk I/O errors) could lead to +corruption of the backup repository due to issue #1138. + +A sign that this happened is if "E" status was reported for a file that can +not be explained by problems with the source file. If you still have logs from +"borg create -v --list", you can check for "E" status. + +Here is what could cause corruption and what you can do now: + +1) I/O errors (e.g. repo disk errors) while writing data to repo. + +This could lead to corrupted segment files. + +Fix:: + + # check for corrupt chunks / segments: + borg check -v --repository-only REPO + + # repair the repo: + borg check -v --repository-only --repair REPO + + # make sure everything is fixed: + borg check -v --repository-only REPO + +2) Unreliable network / unreliable connection to the repo. + +This could lead to archive metadata corruption. + +Fix:: + + # check for corrupt archives: + borg check -v --archives-only REPO + + # delete the corrupt archives: + borg delete --force REPO::CORRUPT_ARCHIVE + + # make sure everything is fixed: + borg check -v --archives-only REPO + +3) In case you want to do more intensive checking. + +The best check that everything is ok is to run a dry-run extraction:: + + borg extract -v --dry-run REPO::ARCHIVE + + +.. _upgradenotes: + +Upgrade Notes +============= + +borg 1.1.x to 1.2.x +------------------- + +Some things can be recommended for the upgrade process from borg 1.1.x +(please also read the important compatibility notes below): + +- first upgrade to a recent 1.1.x release - especially if you run some older + 1.1.* or even 1.0.* borg release. +- using that, run at least one `borg create` (your normal backup), `prune` + and especially a `check` to see everything is in a good state. +- check the output of `borg check` - if there is anything special, consider + a `borg check --repair` followed by another `borg check`. +- if everything is fine so far (borg check reports no issues), you can consider + upgrading to 1.2.x. if not, please first fix any already existing issue. +- if you want to play safer, first **create a backup of your borg repository**. +- upgrade to latest borg 1.2.x release (you could use the fat binary from + github releases page) +- borg 1.2.6 has a security fix for the pre-1.2.5 archives spoofing vulnerability + (CVE-2023-36811), see details and necessary upgrade procedure described above. +- run `borg compact --cleanup-commits` to clean up a ton of 17 bytes long files + in your repo caused by a borg 1.1 bug +- run `borg check` again (now with borg 1.2.x) and check if there is anything + special. +- run `borg info` (with borg 1.2.x) to build the local pre12-meta cache (can + take significant time, but after that it will be fast) - for more details + see below. +- check the compatibility notes (see below) and adapt your scripts, if needed. +- if you run into any issues, please check the github issue tracker before + posting new issues there or elsewhere. + +If you follow this procedure, you can help avoiding that we get a lot of +"borg 1.2" issue reports that are not really 1.2 issues, but existed before +and maybe just were not noticed. + +Compatibility notes: + +- matching of path patterns has been aligned with borg storing relative paths. + Borg archives file paths without leading slashes. Previously, include/exclude + patterns could contain leading slashes. You should check your patterns and + remove leading slashes. +- dropped support / testing for older Pythons, minimum requirement is 3.8. + In case your OS does not provide Python >= 3.8, consider using our binary, + which does not need an external Python interpreter. Or continue using + borg 1.1.x, which is still supported. +- freeing repository space only happens when "borg compact" is invoked. +- mount: the default for --numeric-ids is False now (same as borg extract) +- borg create --noatime is deprecated. Not storing atime is the default behaviour + now (use --atime if you want to store the atime). +- --prefix is deprecated, use -a / --glob-archives, see #6806 +- list: corrected mix-up of "isomtime" and "mtime" formats. + Previously, "isomtime" was the default but produced a verbose human format, + while "mtime" produced a ISO-8601-like format. + The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), + and the default is now "mtime". + "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). +- create/recreate --list: file status for all files used to get announced *AFTER* + the file (with borg < 1.2). Now, file status is announced *BEFORE* the file + contents are processed. If the file status changes later (e.g. due to an error + or a content change), the updated/final file status will be printed again. +- removed deprecated-since-long stuff (deprecated since): + + - command "borg change-passphrase" (2017-02), use "borg key ..." + - option "--keep-tag-files" (2017-01), use "--keep-exclude-tags" + - option "--list-format" (2017-10), use "--format" + - option "--ignore-inode" (2017-09), use "--files-cache" w/o "inode" + - option "--no-files-cache" (2017-09), use "--files-cache=disabled" +- removed BORG_HOSTNAME_IS_UNIQUE env var. + to use borg you must implement one of these 2 scenarios: + + - 1) the combination of FQDN and result of uuid.getnode() must be unique + and stable (this should be the case for almost everybody, except when + having duplicate FQDN *and* MAC address or all-zero MAC address) + - 2) if you are aware that 1) is not the case for you, you must set + BORG_HOST_ID env var to something unique. +- exit with 128 + signal number, #5161. + if you have scripts expecting rc == 2 for a signal exit, you need to update + them to check for >= 128. + + +.. _changelog_1x: + +Change Log 1.x +============== + +Version 1.3.0a1 (2022-04-15) +---------------------------- + +Please note: + +This is an alpha release, only for testing - do not use this with production repos. + +New features: + +- init: new --encryption=(repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305) + + - New, better, faster crypto (see encryption-aead diagram in the docs), #6463. + - New AEAD cipher suites: AES-OCB and CHACHA20-POLY1305. + - Session keys are derived via HKDF from random session id and master key. + - Nonces/MessageIVs are counters starting from 0 for each session. + - AAD: chunk id, key type, messageIV, sessionID are now authenticated also. + - Solves the potential AES-CTR mode counter management issues of the legacy crypto. +- init: --key-algorithm=argon2 (new default KDF, older pbkdf2 also still available) + + borg key change-passphrase / change-location keeps the key algorithm unchanged. +- key change-algorithm: to upgrade existing keys to argon2 or downgrade to pbkdf2. + + We recommend you to upgrade unless you have to keep the key compatible with older versions of borg. +- key change-location: usable for repokey <-> keyfile location change +- benchmark cpu: display benchmarks of cpu bound stuff +- export-tar: new --tar-format=PAX (default: GNU) +- import-tar/export-tar: can use PAX format for ctime and atime support +- import-tar/export-tar: --tar-format=BORG: roundtrip ALL item metadata, #5830 +- repository: create and use version 2 repos only for now +- repository: implement PUT2: header crc32, overall xxh64, #1704 + +Other changes: + +- require python >= 3.9, #6315 +- simplify libs setup, #6482 +- unbundle most bundled 3rd party code, use libs, #6316 +- use libdeflate.crc32 (Linux and all others) or zlib.crc32 (macOS) +- repository: code cleanups / simplifications +- internal crypto api: speedups / cleanups / refactorings / modernisation +- remove "borg upgrade" support for "attic backup" repos +- remove PassphraseKey code and borg key migrate-to-repokey command +- OpenBSD: build borg with OpenSSL (not: LibreSSL), #6474 +- remove support for LibreSSL, #6474 +- remove support for OpenSSL < 1.1.1 + + +Version 1.2.7 (2023-12-02) +-------------------------- + +For upgrade and compatibility hints, please also read the section "Upgrade Notes" +above. + +Fixes: + +- docs: CVE-2023-36811 upgrade steps: consider checkpoint archives, #7802 +- check/compact: fix spurious reappearance of orphan chunks since borg 1.2, #6687 - + this consists of 2 fixes: + + - for existing chunks: check --repair: recreate shadow index, #7897 #6687 + - for newly created chunks: update shadow index when doing a double-put, #7896 #5661 + + If you have experienced issue #6687, you may want to run borg check --repair + after upgrading to borg 1.2.7 to recreate the shadow index and get rid of the + issue for existing chunks. +- LockRoster.modify: no KeyError if element was already gone, #7937 +- create --X-from-command: run subcommands with a clean environment, #7916 +- list --sort-by: support "archive" as alias of "name", #7873 +- fix rc and msg if arg parsing throws an exception, #7885 + +Other changes: + +- support and test on Python 3.12 +- include unistd.h in _chunker.c (fix for Python 3.13) +- allow msgpack 1.0.6 and 1.0.7 +- TAM issues: show tracebacks, improve borg check logging, #7797 +- replace "datetime.utcfromtimestamp" with custom helper to avoid + deprecation warnings when using Python 3.12 +- vagrant: + + - use generic/debian9 box, fixes #7579 + - add VM with debian bookworm / test on OpenSSL 3.0.x. +- docs: + + - not only attack/unsafe, can also be a fs issue, #7853 + - point to CVE-2023-36811 upgrade steps from borg 1.1 to 1.2 upgrade steps, #7899 + - upgrade steps needed for all kinds of repos (including "none" encryption mode), #7813 + - upgrade steps: talk about consequences of borg check, #7816 + - upgrade steps: remove period that could be interpreted as part of the command + - automated-local.rst: use GPT UUID for consistent udev rule + - create disk/partition sector backup by disk serial number, #7934 + - update macOS hint about full disk access + - clarify borg prune -a option description, #7871 + - readthedocs: also build offline docs (HTMLzip), #7835 + - frontends: add "check.rebuild_refcounts" message + + +Version 1.2.6 (2023-08-31) +-------------------------- + +Fixes: + +- The upgrade procedure docs as published with borg 1.2.5 did not work, if the + repository had archives resulting from a borg rename or borg recreate operation. + + The updated docs now use BORG_WORKAROUNDS=ignore_invalid_archive_tam at some + places to avoid that issue, #7791. + + See: fix pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811), + details and necessary upgrade procedure described above. + +Other changes: + +- updated 1.2.5 changelog entry: 1.2.5 already has the fix for rename/recreate. +- remove cython restrictions. recommended is to build with cython 0.29.latest, + because borg 1.2.x uses this since years and it is very stable. + you can also try to build with cython 3.0.x, there is a good chance that it works. + as a 3rd option, we also bundle the `*.c` files cython outputs in the release + pypi package, so you can also just use these and not need cython at all. + + +Version 1.2.5 (2023-08-30) +-------------------------- + +Fixes: + +- Security: fix pre-1.2.5 archives spoofing vulnerability (CVE-2023-36811), + see details and necessary upgrade procedure described above. +- rename/recreate: correctly update resulting archive's TAM, see #7791 +- create: do not try to read parent dir of recursion root, #7746 +- extract: fix false warning about pattern never matching, #4110 +- diff: remove surrogates before output, #7535 +- compact: clear empty directories at end of compact process, #6823 +- create --files-cache=size: fix crash, #7658 +- keyfiles: improve key sanity check, #7561 +- only warn about "invalid" chunker params, #7590 +- ProgressIndicatorPercent: fix space computation for wide chars, #3027 +- improve argparse validator error messages + +New features: + +- mount: make up volname if not given (macOS), #7690. + macFUSE supports a volname mount option to give what finder displays on the + desktop / in the directory view. if the user did not specify it, we make + something up, because otherwise it would be "macFUSE Volume 0 (Python)" and + hide the mountpoint directory name. +- BORG_WORKAROUNDS=authenticated_no_key to extract from authenticated repos + without key, #7700 + +Other changes: + +- add `utcnow()` helper function to avoid deprecated `datetime.utcnow()` +- stay on latest Cython 0.29 (0.29.36) for borg 1.2.x (do not use Cython 3.0 yet) +- docs: + + - move upgrade notes to own section, see #7546 + - mount -olocal: how to show mount in finder's sidebar, #5321 + - list: fix --pattern examples, #7611 + - improve patterns help + - incl./excl. options, path-from-stdin exclusiveness + - obfuscation docs: markup fix, note about MAX_DATA_SIZE + - --one-file-system: add macOS apfs notes, #4876 + - improve --one-file-system help string, #5618 + - rewrite borg check docs + - improve the docs for --keep-within, #7687 + - fix borg init command in environment.rst.inc + - 1.1.x upgrade notes: more precise borg upgrade instructions, #3396 + +- tests: + + - fix repo reopen + - avoid long ids in pytest output + - check buzhash chunksize distribution, see #7586 + + +Version 1.2.4 (2023-03-24) +-------------------------- + +New features: + +- import-tar: add --ignore-zeros to process concatenated tars, #7432. +- debug id-hash: computes file/chunk content id-hash, #7406 +- diff: --content-only does not show mode/ctime/mtime changes, #7248 +- diff: JSON strings in diff output are now sorted alphabetically + +Bug fixes: + +- xattrs: fix namespace processing on FreeBSD, #6997 +- diff: fix path related bug seen when addressing deferred items. +- debug get-obj/put-obj: always give chunkid as cli param, see #7290 + (this is an incompatible change, see also borg debug id-hash) +- extract: fix mtime when ResourceFork xattr is set (macOS specific), #7234 +- recreate: without --chunker-params, do not re-chunk, #7337 +- recreate: when --target is given, do not detect "nothing to do". + use case: borg recreate -a src --target dst can be used to make a copy + of an archive inside the same repository, #7254. +- set .hardlink_master for ALL hardlinkable items, #7175 +- locking: fix host, pid, tid order. + tid (thread id) must be parsed as hex from lock file name. +- update development.lock.txt, including a setuptools security fix, #7227 + +Other changes: + +- requirements: allow msgpack 1.0.5 also +- upgrade Cython to 0.29.33 +- hashindex minor fixes, refactor, tweaks, tests +- use os.replace not os.rename +- remove BORG_LIBB2_PREFIX (not used any more) +- docs: + + - BORG_KEY_FILE: clarify docs, #7444 + - update FAQ about locale/unicode issues, #6999 + - improve mount options rendering, #7359 + - make timestamps in manual pages reproducible + - installation: update Fedora in distribution list, #7357 +- tests: + + - fix test_size_on_disk_accurate for large st_blksize, #7250 + - add same_ts_ns function and use it for relaxed timestamp comparisons + - "auto" compressor tests: don't assume a specific size, + do not assume zlib is better than lz4, #7363 + - add test for extracted directory mtime +- vagrant: + + - upgrade local freebsd 12.1 box -> generic/freebsd13 box (13.1) + - use pythons > 3.8 which work on freebsd 13.1 + - pyenv: also install python 3.11.1 for testing + - pyenv: use python 3.10.1, 3.10.0 build is broken on freebsd + + +Version 1.2.3 (2022-12-24) +-------------------------- + +Fixes: + +- create: fix --list --dry-run output for directories, #7209 +- diff/recreate: normalize chunker params before comparing them, #7079 +- check: fix uninitialised variable if repo is completely empty, #7034 +- xattrs: improve error handling, #6988 +- fix args.paths related argparsing, #6994 +- archive.save(): always use metadata from stats (e.g. nfiles, size, ...), #7072 +- tar_filter: recognize .tar.zst as zstd, #7093 +- get_chunker: fix missing sparse=False argument, #7056 +- file_integrity.py: make sure file_fd is always closed on exit +- repository: cleanup(): close segment before unlinking +- repository: use os.replace instead of os.rename + +Other changes: + +- remove python < 3.7 compatibility code +- do not use version_tuple placeholder in setuptools_scm template +- CI: fix tox4 passenv issue, #7199 +- vagrant: update to python 3.9.16, use the openbsd 7.1 box +- misc. test suite and docs fixes / improvements +- remove deprecated --prefix from docs, #7109 +- Windows: use MSYS2 for Github CI, remove Appveyor CI + + +Version 1.2.2 (2022-08-20) +-------------------------- + +New features: + +- prune/delete --checkpoint-interval=1800 and ctrl-c/SIGINT support, #6284 + +Fixes: + +- SaveFile: use a custom mkstemp with mode support, #6933, #6400, #6786. + This fixes umask/mode/ACL issues (and also "chmod not supported" exceptions + seen in 1.2.1) of files updated using SaveFile, e.g. the repo config. +- hashindex_compact: fix eval order (check idx before use), #5899 +- create --paths-from-(stdin|command): normalize paths, #6778 +- secure_erase: avoid collateral damage, #6768. + If a hardlink copy of a repo was made and a new repo config shall be saved, + do NOT fill in random garbage before deleting the previous repo config, + because that would damage the hardlink copy. +- list: fix {flags:} formatting, #6081 +- check: try harder to create the key, #5719 +- misc commands: ctrl-c must not kill other subprocesses, #6912 + + - borg create with a remote repo via ssh + - borg create --content-from-command + - borg create --paths-from-command + - (de)compression filter process of import-tar / export-tar + +Other changes: + +- deprecate --prefix, use -a / --glob-archives, see #6806 +- make setuptools happy ("package would be ignored"), #6874 +- fix pyproject.toml to create a fixed _version.py file, compatible with both + old and new setuptools_scm version, #6875 +- automate asciinema screencasts +- CI: test on macOS 12 without fuse / fuse tests + (too troublesome on github CI due to kernel extensions needed by macFUSE) +- tests: fix test_obfuscate byte accounting +- repository: add debug logging for issue #6687 +- _chunker.c: fix warnings on macOS +- requirements.lock.txt: use the latest cython 0.29.32 +- docs: + + - add info on man page installation, #6894 + - update archive_progress json description about "finished", #6570 + - json progress_percent: some values are optional, #4074 + - FAQ: full quota / full disk, #5960 + - correct shell syntax for installation using git + + +Version 1.2.1 (2022-06-06) +-------------------------- + +Fixes: + +- create: skip with warning if opening the parent dir of recursion root fails, #6374 +- create: fix crash. metadata stream can produce all-zero chunks, #6587 +- fix crash when computing stats, escape % chars in archive name, #6500 +- fix transaction rollback: use files cache filename as found in txn.active/, #6353 +- import-tar: kill filter process in case of borg exceptions, #6401 #6681 +- import-tar: fix mtime type bug +- ensure_dir: respect umask for created directory modes, #6400 +- SaveFile: respect umask for final file mode, #6400 +- check archive: improve error handling for corrupt archive metadata block, make + robust_iterator more robust, #4777 +- pre12-meta cache: do not use the cache if want_unique is True, #6612 +- fix scp-style repo url parsing for ip v6 address, #6526 +- mount -o versions: give clear error msg instead of crashing. + it does not make sense to request versions view if you only look at 1 archive, + but the code shall not crash in that case as it did, but give a clear error msg. +- show_progress: add finished=true/false to archive_progress json, #6570 +- delete/prune: fix --iec mode output (decimal vs. binary units), #6606 +- info: fix authenticated mode repo to show "Encrypted: No", #6462 +- diff: support presence change for blkdev, chrdev and fifo items, #6615 + +New features: + +- delete: add repository id and location to prompt, #6453 +- borg debug dump-repo-objs --ghost: new --segment=S --offset=O options + +Other changes: + +- support python 3.11 +- allow msgpack 1.0.4, #6716 +- load_key: no key is same as empty key, #6441 +- give a more helpful error msg for unsupported key formats, #6561 +- better error msg for defect or unsupported repo configs, #6566 +- docs: + + - document borg 1.2 pattern matching behavior change, #6407 + Make clear that absolute paths always go into the matcher as if they are + relative (without leading slash). Adapt all examples accordingly. + - authentication primitives: improved security and performance infos + - mention BORG_FILES_CACHE_SUFFIX as alternative to BORG_FILES_CACHE_TTL, #5602 + - FAQ: add a hint about --debug-topic=files_cache + - improve borg check --max-duration description + - fix values of TAG bytes, #6515 + - borg compact --cleanup-commits also runs a normal compaction, #6324 + - virtualization speed tips + - recommend umask for passphrase file perms + - borg 1.2 is security supported + - update link to ubuntu packages, #6485 + - use --numeric-ids in pull mode docs + - remove blake2 docs, blake2 code not bundled any more, #6371 + - clarify on-disk order and size of segment file log entry fields, #6357 + - docs building: do not transform --/--- to unicode dashes +- tests: + + - check that borg does not require pytest for normal usage, fixes #6563 + - fix OpenBSD symlink mode test failure, #2055 +- vagrant: + + - darwin64: remove fakeroot, #6314 + - update development.lock.txt + - use pyinstaller 4.10 and python 3.9.13 for binary build + - upgrade VMCPUS and xdistn from 4 to 16, maybe this speeds up the tests +- crypto: + + - use hmac.compare_digest instead of ==, #6470 + - hmac_sha256: replace own cython wrapper code by hmac.digest python stdlib (since py38) + - hmac and blake2b minor optimizations and cleanups + - removed some unused crypto related code, #6472 + - avoid losing the key (potential use-after-free). this never could happen in + 1.2 due to the way we use the code. The issue was discovered in master after + other changes, so we also "fixed" it here before it bites us. +- setup / build: + + - add pyproject.toml, fix sys.path, #6466 + - setuptools_scm: also require it via pyproject.toml + - allow extra compiler flags for every extension build + - fix misc. C / Cython compiler warnings, deprecation warnings + - fix zstd.h include for bundled zstd, #6369 +- source using python 3.8 features: ``pyupgrade --py38-plus ./**/*.py`` + + +Version 1.2.0 (2022-02-22 22:02:22 :-) +-------------------------------------- + +Please note: + +This is the first borg 1.2 release, so be careful and read the notes below. + +Upgrade notes: + +Strictly taken, nothing special is required for upgrading to 1.2, but some +things can be recommended: + +- do you already want to upgrade? 1.1.x also will get fixes for a while. +- be careful, first upgrade your less critical / smaller repos. +- first upgrade to a recent 1.1.x release - especially if you run some older + 1.1.* or even 1.0.* borg release. +- using that, run at least one `borg create` (your normal backup), `prune` + and especially a `check` to see everything is in a good state. +- check the output of `borg check` - if there is anything special, consider + a `borg check --repair` followed by another `borg check`. +- if everything is fine so far (borg check reports no issues), you can consider + upgrading to 1.2.0. if not, please first fix any already existing issue. +- if you want to play safer, first **create a backup of your borg repository**. +- upgrade to latest borg 1.2.x release (you could use the fat binary from + github releases page) +- run `borg compact --cleanup-commits` to clean up a ton of 17 bytes long files + in your repo caused by a borg 1.1 bug +- run `borg check` again (now with borg 1.2.x) and check if there is anything + special. +- run `borg info` (with borg 1.2.x) to build the local pre12-meta cache (can + take significant time, but after that it will be fast) - for more details + see below. +- check the compatibility notes (see below) and adapt your scripts, if needed. +- if you run into any issues, please check the github issue tracker before + posting new issues there or elsewhere. + +If you follow this procedure, you can help avoiding that we get a lot of +"borg 1.2" issue reports that are not really 1.2 issues, but existed before +and maybe just were not noticed. + +Compatibility notes: + +- matching of path patterns has been aligned with borg storing relative paths. + Borg archives file paths without leading slashes. Previously, include/exclude + patterns could contain leading slashes. You should check your patterns and + remove leading slashes. +- dropped support / testing for older Pythons, minimum requirement is 3.8. + In case your OS does not provide Python >= 3.8, consider using our binary, + which does not need an external Python interpreter. Or continue using + borg 1.1.x, which is still supported. +- freeing repository space only happens when "borg compact" is invoked. +- mount: the default for --numeric-ids is False now (same as borg extract) +- borg create --noatime is deprecated. Not storing atime is the default behaviour + now (use --atime if you want to store the atime). +- list: corrected mix-up of "isomtime" and "mtime" formats. + Previously, "isomtime" was the default but produced a verbose human format, + while "mtime" produced a ISO-8601-like format. + The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), + and the default is now "mtime". + "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). +- create/recreate --list: file status for all files used to get announced *AFTER* + the file (with borg < 1.2). Now, file status is announced *BEFORE* the file + contents are processed. If the file status changes later (e.g. due to an error + or a content change), the updated/final file status will be printed again. +- removed deprecated-since-long stuff (deprecated since): + + - command "borg change-passphrase" (2017-02), use "borg key ..." + - option "--keep-tag-files" (2017-01), use "--keep-exclude-tags" + - option "--list-format" (2017-10), use "--format" + - option "--ignore-inode" (2017-09), use "--files-cache" w/o "inode" + - option "--no-files-cache" (2017-09), use "--files-cache=disabled" +- removed BORG_HOSTNAME_IS_UNIQUE env var. + to use borg you must implement one of these 2 scenarios: + + - 1) the combination of FQDN and result of uuid.getnode() must be unique + and stable (this should be the case for almost everybody, except when + having duplicate FQDN *and* MAC address or all-zero MAC address) + - 2) if you are aware that 1) is not the case for you, you must set + BORG_HOST_ID env var to something unique. +- exit with 128 + signal number, #5161. + if you have scripts expecting rc == 2 for a signal exit, you need to update + them to check for >= 128. + +Fixes: + +- diff: reduce memory consumption, fix is_hardlink_master, #6295 +- compact: fix / improve freeable / freed space log output + + - derive really freed space from quota use before/after, #5679 + - do not say "freeable", but "maybe freeable" (based on hint, unsure) +- fix race conditions in internal SaveFile function, #6306 #6028 +- implement internal safe_unlink (was: truncate_and_unlink) function more safely: + usually it does not truncate any more, only under "disk full" circumstances + and only if there is only one hardlink. + see: https://github.com/borgbackup/borg/discussions/6286 + +Other changes: + +- info: use a pre12-meta cache to accelerate stats for borg < 1.2 archives. + the first time borg info is invoked on a borg 1.1 repo, it can take a + rather long time computing and caching some stats values for 1.1 archives, + which borg 1.2 archives have in their archive metadata structure. + be patient, esp. if you have lots of old archives. + following invocations are much faster due to the cache. + related change: add archive name to calc_stats progress display. +- docs: + + - add borg 1.2 upgrade notes, #6217 + - link to borg placeholders and borg patterns help + - init: explain the encryption modes better + - clarify usage of patternfile roots + - put import-tar docs into same file as export-tar docs + - explain the difference between a path that ends with or without a slash, + #6297 + + +Version 1.2.0rc1 (2022-02-05) +----------------------------- + +Fixes: + +- repo::archive location placeholder expansion fixes, #5826, #5998 +- repository: fix intermediate commits, shall be at end of current segment +- delete: don't commit if nothing was deleted, avoid cache sync, #6060 +- argument parsing: accept some options only once, #6026 +- disallow overwriting of existing keyfiles on init, #6036 +- if ensure_dir() fails, give more informative error message, #5952 + +New features: + +- delete --force: do not ask when deleting a repo, #5941 + +Other changes: + +- requirements: exclude broken or incompatible-with-pyinstaller setuptools +- add a requirements.d/development.lock.txt and use it for vagrant +- tests: + + - added nonce-related tests + - refactor: remove assert_true + - vagrant: macos box tuning, netbsd box fixes, #5370, #5922 +- docs: + + - update install docs / requirements docs, #6180 + - borg mount / FUSE "versions" view is not experimental any more + - --pattern* is not experimental any more, #6134 + - impact of deleting path/to/repo/nonce, #5858 + - key export: add examples, #6204 + - ~/.config/borg/keys is not used for repokey keys, #6107 + - excluded parent dir's metadata can't restore + + +Version 1.2.0b4 (2022-01-23) +---------------------------- + +Fixes: + +- create: fix passing device nodes and symlinks to --paths-from-stdin, #6009 +- create --dry-run: fix display of kept tagfile, #5834 +- check --repair: fix missing parameter in "did not consistently fail" msg, #5822 +- fix hardlinkable file type check, #6037 +- list: remove placeholders for shake_* hashes, #6082 +- prune: handle case of calling prune_split when there are no archives, #6015 +- benchmark crud: make sure cleanup of borg-test-data files/dir happens, #5630 +- do not show archive name in repository-related error msgs, #6014 +- prettier error msg (no stacktrace) if exclude file is missing, #5734 +- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979 +- fix pyinstaller detection for dir-mode, #5897 +- atomically create the CACHE_TAG file, #6028 +- deal with the SaveFile/SyncFile race, docs, see #6056 708a5853 +- avoid expanding path into LHS of formatting operation + tests, #6064 #6063 +- repository: quota / compactable computation fixes +- info: emit repo info even if repo has 0 archives + test, #6120 + +New features: + +- check --repair: significantly speed up search for next valid object in segment, #6022 +- check: add progress indicator for archive check, #5809 +- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024 +- create: allow --files-cache=size (this is potentially dangerous, use on your own risk), #5686 +- import-tar: implement import-tar to complement export-tar, #2233 +- implement BORG_SELFTEST env variable (can be carefully used to speedup borg hosting), #5871 +- key export: print key if path is '-' or not given, #6092 +- list --format: Add command_line to format keys + +Other changes: + +- pypi metadata: alpha -> beta +- require python 3.8+, #5975 +- use pyinstaller 4.7 +- allow msgpack 1.0.3 +- upgrade to bundled xxhash to 0.8.1 +- import-tar / export-tar: tar file related changes: + + - check for short tarfile extensions + - add .lz4 and .zstd + - fix docs about extensions and decompression commands +- add github codeql analysis, #6148 +- vagrant: + + - box updates / add new boxes / remove outdated and broken boxes + - use Python 3.9.10 (incl. binary builds) and 3.10.0 + - fix pyenv initialisation, #5798 + - fix vagrant scp on macOS, #5921 + - use macfuse instead of osxfuse +- shell completions: + + - update shell completions to 1.1.17, #5923 + - remove BORG_LIBC completion, since 9914968 borg no longer uses find_library(). +- docs: + + - fixed readme.rst irc webchat link (we use libera chat now, not freenode) + - fix exceptions thrown by `setup.py build_man` + - check --repair: recommend checking hw before check --repair, #5855 + - check --verify-data: clarify and document conflict with --repository-only, #5808 + - serve: improve ssh forced commands docs, #6083 + - list: improve docs for `borg list` --format, #6061 + - list: remove --list-format from borg list + - FAQ: fix manifest-timestamp path (inside security dir) + - fix the broken link to .nix file + - document behavior for filesystems with inconsistent inodes, #5770 + - clarify user_id vs uid for fuse, #5723 + - clarify pattern usage with commands, #5176 + - clarify pp vs. pf pattern type, #5300 + - update referenced freebsd/macOS versions used for binary build, #5942 + - pull mode: add some warnings, #5827 + - clarify "you will need key and passphrase" borg init warning, #4622 + - add missing leading slashes in help patterns, #5857 + - add info on renaming repositories, #5240 + - check: add notice about defective hardware, #5753 + - mention tar --compare (compare archive to fs files), #5880 + - add note about grandfather-father-son backup retention policy / rotation scheme, #6006 + - permissions note rewritten to make it less confusing + - create github security policy + - remove leftovers of BORG_HOSTNAME_IS_UNIQUE + - excluded parent dir's metadata can't restore. (#6062) + - if parent dir is not extracted, we do not have its metadata + - clarify who starts the remote agent + + +Version 1.2.0b3 (2021-05-12) +---------------------------- + +Fixes: + +- create: fix --progress --log-json, #4360#issuecomment-774580052 +- do not load files cache for commands not using it, #5673 +- fix repeated cache tag file writing bug + +New features: + +- create/recreate: print preliminary file status early, #5417 +- create/extract: add --noxattrs and --noacls options, #3955 +- create: verbose files cache logging via --debug-topic=files_cache, #5659 +- mount: implement --numeric-ids (default: False!), #2377 +- diff: add --json-lines option +- info / create --stats: add --iec option to print sizes in powers of 1024. + +Other changes: + +- create: add --upload-(ratelimit|buffer), deprecate --remote-* options, #5611 +- create/extract/mount: add --numeric-ids, deprecate --numeric-owner option, #5724 +- config: accept non-int value for max_segment_size / storage_quota +- use PyInstaller v4.3, #5671 +- vagrant: use Python 3.9.5 to build binaries +- tox.ini: modernize and enable execution without preinstalling deps +- cleanup code style checks +- get rid of distutils, use setuptools+packaging +- github CI: test on Python 3.10-dev +- check: missing / healed chunks: always tell chunk ID, #5704 +- docs: + + - remove bad /var/cache exclusion in example commands, #5625 + - misc. fixes and improvements, esp. for macOS + - add unsafe workaround to use an old repo copy, #5722 + + +Version 1.2.0b2 (2021-02-06) +---------------------------- + +Fixes: + +- create: do not recurse into duplicate roots, #5603 +- create: only print stats if not ctrl-c'ed, fixes traceback, #5668 +- extract: + improve exception handling when setting xattrs, #5092. + emit a warning message giving the path, xattr key and error message. + continue trying to restore other xattrs and bsdflags of the same file + after an exception with xattr-setting happened. +- export-tar: + fix memory leak with ssh: remote repository, #5568. + fix potential memory leak with ssh: remote repository with partial extraction. +- remove empty shadowed_segments lists, #5275 +- fix bad default: manifest.archives.list(consider_checkpoints=False), + fixes tracebacks / KeyErros for missing objects in ChunkIndex, #5668 + +New features: + +- create: improve sparse file support + + - create --sparse (detect sparse file holes) and file map support, + only for the "fixed" chunker, #14 + - detect all-zero chunks in read data in "buzhash" and "fixed" chunkers + - cached_hash: use a small LRU cache to accelerate all-zero chunks hashing + - use cached_hash also to generate all-zero replacement chunks +- create --remote-buffer, add a upload buffer for remote repos, #5574 +- prune: keep oldest archive when retention target not met + +Other changes: + +- use blake2 from python 3.6+ hashlib + (this removes the requirement for libb2 and the bundled blake2 code) +- also accept msgpack up to 1.0.2. + exclude 1.0.1 though, which had some issues (not sure they affect borg). +- create: add repository location to --stats output, #5491 +- check: debug log the segment filename +- delete: add a --list switch to borg delete, #5116 +- borg debug dump-hints - implemented to e.g. to look at shadow_index +- Tab completion support for additional archives for 'borg delete' +- refactor: have one borg.constants.zero all-zero bytes object +- refactor shadow_index updating repo.put/delete, #5661, #5636. +- docs: + + - add another case of attempted hardlink usage + - fix description of borg upgrade hardlink usage, #5518 + - use HTTPS everywhere + - add examples for --paths-from-stdin, --paths-from-command, --paths-separator, #5644 + - fix typos/grammar + - update docs for dev environment installation instructions + - recommend running tests only on installed versions for setup + - add badge with current status of package +- vagrant: + + - use brew install --cask ..., #5557 + - use Python 3.9.1 and PyInstaller 4.1 to build the borg binary + + +Version 1.2.0b1 (2020-12-06) +---------------------------- + +Fixes: + +- BORG_CACHE_DIR crashing borg if empty, atomic handling of + recursive directory creation, #5216 +- fix --dry-run and --stats coexistence, #5415 +- allow EIO with warning when trying to hardlink, #4336 +- export-tar: set tar format to GNU_FORMAT explicitly, #5274 +- use --timestamp for {utcnow} and {now} if given, #5189 +- make timestamp helper timezone-aware + +New features: + +- create: implement --paths-from-stdin and --paths-from-command, see #5492. + These switches read paths to archive from stdin. Delimiter can specified + by --paths-delimiter=DELIM. Paths read will be added honoring every + option but exclusion options and --one-file-system. borg won't recurse + into directories. +- 'obfuscate' pseudo compressor obfuscates compressed chunk size in repo +- add pyfuse3 (successor of llfuse) as an alternative lowlevel fuse + implementation to llfuse (deprecated), #5407. + FUSE implementation can be switched via env var BORG_FUSE_IMPL. +- allow appending to the files cache filename with BORG_FILES_CACHE_SUFFIX +- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333 + +Other changes: + +- split recursive directory walking/processing into directory walking and + item processing. +- fix warning by importing setuptools before distutils. +- debug info: include infos about FUSE implementation, #5546 +- testing: + + - add a test for the hashindex corruption bug, #5531 #4829 + - move away from travis-ci, use github actions, #5528 #5467 + - test both on fuse2 and fuse3 + - upload coverage reports to codecov + - fix spurious failure in test_cache_files, #5438 + - add tests for Location.with_timestamp + - tox: add a non-fuse env to the envlist +- vagrant: + + - use python 3.7.latest and pyinstaller 4.0 for binary creation + - pyinstaller: compute basepath from spec file location + - vagrant: updates/fixes for archlinux box, #5543 +- docs: + + - "filename with spaces" example added to exclude file, #5236 + - add a hint about sleeping computer, #5301 + - how to adjust macOS >= Catalina security settings, #5303 + - process/policy for adding new compression algorithms + - updated docs about hacked backup client, #5480 + - improve ansible deployment docs, make it more generic + - how to approach borg speed issues, give speed example, #5371 + - fix mathematical inaccuracy about chunk size, #5336 + - add example for excluding content using --pattern cli option + - clarify borg create's '--one-file-system' option, #4009 + - improve docs/FAQ about append-only remote repos, #5497 + - fix reST markup issues, labels + - add infos about contributor retirement status + + +Version 1.2.0a9 (2020-10-05) +---------------------------- + +Fixes: + +- fix memory leak related to preloading, #5202 +- check --repair: fix potential data loss, #5325 +- persist shadow_index in between borg runs, #4830 +- fix hardlinked CACHEDIR.TAG processing, #4911 +- --read-special: .part files also should be regular files, #5217 +- allow server side enforcing of umask, --umask is for the local borg + process only (see docs), #4947 +- exit with 128 + signal number, #5161 +- borg config --list does not show last_segment_checked, #5159 +- locking: + + - fix ExclusiveLock race condition bug, #4923 + - fix race condition in lock migration, #4953 + - fix locking on openindiana, #5271 + +New features: + +- --content-from-command: create archive using stdout of given command, #5174 +- allow key-import + BORG_KEY_FILE to create key files +- build directory-based binary for macOS to avoid Gatekeeper delays + +Other changes: + +- upgrade bundled zstd to 1.4.5 +- upgrade bundled xxhash to 0.8.0, #5362 +- if self test fails, also point to OS and hardware, #5334 +- misc. shell completions fixes/updates, rewrite zsh completion +- prettier error message when archive gets too big, #5307 +- stop relying on `false` exiting with status code 1 +- rephrase some warnings, #5164 +- parseformat: unnecessary calls removed, #5169 +- testing: + + - enable Python3.9 env for test suite and VMs, #5373 + - drop python 3.5, #5344 + - misc. vagrant fixes/updates + - misc. testing fixes, #5196 +- docs: + + - add ssh-agent pull backup method to doc, #5288 + - mention double --force in prune docs + - update Homebrew install instructions, #5185 + - better description of how cache and rebuilds of it work + and how the workaround applies to that + - point to borg create --list item flags in recreate usage, #5165 + - add a note to create from stdin regarding files cache, #5180 + - add security faq explaining AES-CTR crypto issues, #5254 + - clarify --exclude-if-present in recreate, #5193 + - add socat pull mode, #5150, #900 + - move content of resources doc page to community project, #2088 + - explain hash collision, #4884 + - clarify --recompress option, #5154 + + +Version 1.2.0a8 (2020-04-22) +---------------------------- + +Fixes: + +- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829. + Please read and follow the more detailed notes close to the top of this document. +- fix crash when upgrading erroneous hints file, #4922 +- commit-time free space calc: ignore bad compact map entries, #4796 +- info: if the archive doesn't exist, print a pretty message, #4793 +- --prefix / -P: fix processing, avoid argparse issue, #4769 +- ignore EACCES (errno 13) when hardlinking, #4730 +- add a try catch when formatting the info string, #4818 +- check: do not stumble over invalid item key, #4845 +- update prevalence of env vars to set config and cache paths +- mount: fix FUSE low linear read speed on large files, #5032 +- extract: fix confusing output of borg extract --list --strip-components, #4934 +- recreate: support --timestamp option, #4745 +- fix ProgressIndicator msgids (JSON output), #4935 +- fuse: set f_namemax in statfs result, #2684 +- accept absolute paths on windows +- pyinstaller: work around issue with setuptools > 44 + +New features: + +- chunker speedup (plus regression test) +- added --consider-checkpoints and related test, #4788 +- added --noflags option, deprecate --nobsdflags option, #4489 +- compact: add --threshold option, #4674 +- mount: add birthtime to FUSE entries +- support platforms with no os.link, #4901 - if we don't have os.link, + we just extract another copy instead of making a hardlink. +- move sync_file_range to its own extension for better platform compatibility. +- new --bypass-lock option to bypass locking, e.g. for read-only repos +- accept absolute paths by removing leading slashes in patterns of all + sorts but re: style, #4029 +- delete: new --keep-security-info option + +Other changes: + +- support msgpack 0.6.2 and 1.0.0, #5065 +- upgrade bundled zstd to 1.4.4 +- upgrade bundled lz4 to 1.9.2 +- upgrade xxhash to 0.7.3 +- require recent enough llfuse for birthtime support, #5064 +- only store compressed data if the result actually is smaller, #4516 +- check: improve error output for matching index size, see #4829 +- ignore --stats when given with --dry-run, but continue, #4373 +- replaced usage of os.statvfs with shutil.disk_usage (better cross-platform support). +- fuse: remove unneeded version check and compat code, micro opts +- docs: + + - improve description of path variables + - document how to delete data completely, #2929 + - add FAQ about Borg config dir, #4941 + - add docs about errors not printed as JSON, #4073 + - update usage_general.rst.inc + - added "Will move with BORG_CONFIG_DIR variable unless specified." to BORG_SECURITY_DIR info. + - put BORG_SECURITY_DIR immediately below BORG_CONFIG_DIR (and moved BORG_CACHE_DIR up before them). + - add paragraph regarding cache security assumptions, #4900 + - tell about borg cache security precautions + - add FAQ describing difference between a local repo vs. repo on a server. + - document how to test exclusion patterns without performing an actual backup + - create: tell that "Calculating size" time and space needs are caused by --progress + - fix/improve documentation for @api decorator, #4674 + - add a pull backup / push restore how-to, #1552 + - fix man pages creation, #4752 + - more general FAQ for backup and retain original paths, #4532 + - explain difference between --exclude and --pattern, #4118 + - add FAQ for preventing SSH timeout in extract, #3866 + - improve password FAQ (decrease pw length, add -w 0 option to base64 to prevent line wrap), #4591 + - add note about patterns and stored paths, #4160 + - add upgrade of tools to pip installation how-to, #5090 + - document one cause of orphaned chunks in check command, #2295 + - clean up the whole check usage paragraph + - FAQ: linked recommended restrictions to ssh public keys on borg servers, #4946 + - fixed "doc downplays severity of Nonce reuse issue", #4883 + - borg repo restore instructions needed, #3428 + - new FAQ: A repo is corrupt and must be replaced with an older repo. + - clarify borg init's encryption modes +- native windows port: + + - update README_WINDOWS.rst + - updated pyinstaller spec file to support windows builds +- testing / CI: + + - improved travis config / install script, improved macOS builds + - allow osx builds to fail, #4955 + - Windows 10 build on Appveyor CI +- vagrant: + + - upgrade pyinstaller to v3.5 + patch + - use py369 for binary build, add py380 for tests + - fix issue in stretch VM hanging at grub installation + - add a debian buster and a ubuntu focal VM + - update darwin box to 10.12 + - upgrade FreeBSD box to 12.1 + - fix debianoid virtualenv packages + - use pyenv in freebsd64 VM + - remove the flake8 test + - darwin: avoid error if pkg is already installed + - debianoid: don't interactively ask questions + + +Version 1.2.0a7 (2019-09-07) +---------------------------- + +Fixes: + +- slave hardlinks extraction issue, see #4350 +- extract: fix KeyError for "partial" extraction, #4607 +- preload chunks for hardlink slaves w/o preloaded master, #4350 +- fix preloading for old remote servers, #4652 +- fix partial extract for hardlinked contentless file types, #4725 +- Repository.open: use stat() to check for repo dir, #4695 +- Repository.check_can_create_repository: use stat() to check, ~ #4695. +- SecurityManager.known(): check all files, #4614 +- after double-force delete, warn about necessary repair, #4704 +- cope with ANY error when importing pytest into borg.testsuite, #4652 +- fix invalid archive error message +- setup.py: fix detection of missing Cython +- filter out selinux xattrs, #4574 +- location arg - should it be optional? #4541 +- enable placeholder usage in --comment, #4559 +- use whitelist approach for borg serve, #4097 + +New features: + +- minimal native Windows support, see windows readme (work in progress) +- create: first ctrl-c (SIGINT) triggers checkpoint and abort, #4606 +- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710 +- remove WSL autodetection. if WSL still has this problem, you need to + set BORG_WORKAROUNDS=basesyncfile in the borg process environment to + work around it. +- support xxh64 checksum in addition to the hashlib hashes in borg list +- enable placeholder usage in all extra archive arguments +- enable placeholder usage in --comment, #4559 +- enable placeholder usage in --glob-archives, #4495 +- ability to use a system-provided version of "xxhash" +- create: + + - changed the default behaviour not to store the atime of fs items. atime is + often rather not interesting and fragile - it easily changes even if nothing + else has changed and, if stored into the archive, spoils deduplication of + the archive metadata stream. + - if you give the --noatime option, borg will output a deprecation warning + because it is currently ignored / does nothing. + Please remove the --noatime option when using borg 1.2. + - added a --atime option for storing files' atime into an archive + +Other changes: + +- argparser: always use REPOSITORY in metavar +- do not check python/libc for borg serve, #4483 +- small borg compact improvements, #4522 +- compact: log freed space at INFO level +- tests: + + - tox / travis: add testing on py38-dev + - fix broken test that relied on improper zlib assumptions + - pure-py msgpack warning shall not make a lot of tests fail, #4558 + - rename test_mount_hardlinks to test_fuse_mount_hardlinks (master) + - vagrant: add up-to-date openindiana box (py35, openssl10) + - get rid of confusing coverage warning, #2069 +- docs: + + - reiterate that 'file cache names are absolute' in FAQ, + mention bind mount solution, #4738 + - add restore docs, #4670 + - updated docs to cover use of temp directory on remote, #4545 + - add a push-style example to borg-create(1), #4613 + - timestamps in the files cache are now usually ctime, #4583 + - benchmark crud: clarify that space is used until compact + - update documentation of borg create, + corrects a mention of borg 1.1 as a future version. + - fix osxfuse github link in installation docs + - how to supply a passphrase, use crypto devices, #4549 + - extract: document limitation "needs empty destination", #4598 + - update macOS Brew link + - add note about software for automating backup + - compact: improve docs, + - README: new URL for funding options + + +Version 1.2.0a6 (2019-04-22) +---------------------------- + +Fixes: + +- delete / prune: consider part files correctly for stats, #4507 +- fix "all archives" stats considering part files, #4329 +- create: only run stat_simple_attrs() once +- create: --stats does not work with --dry-run, exit with error msg, #4373 +- give "invalid repo" error msg if repo config not found, #4411 + +New features: + +- display msgpack version as part of sysinfo (e.g. in tracebacks) + +Other changes: + +- docs: + + - sdd "SSH Configuration" section, #4493, #3988, #636, #4485 + - better document borg check --max-duration, #4473 + - sorted commands help in multiple steps, #4471 +- testing: + + - travis: use py 3.5.3 and 3.6.7 on macOS to get a pyenv-based python + build with openssl 1.1 + - vagrant: use py 3.5.3 and 3.6.8 on darwin64 VM to build python and + borg with openssl 1.1 + - pytest: -v and default XDISTN to 1, #4481 + + +Version 1.2.0a5 (2019-03-21) +---------------------------- + +Fixes: + +- warn if a file has changed while being backed up, #1750 +- lrucache: regularly remove old FDs, #4427 +- borg command shall terminate with rc 2 for ImportErrors, #4424 +- make freebsd xattr platform code api compatible with linux, #3952 + +Other changes: + +- major setup code refactoring (especially how libraries like openssl, liblz4, + libzstd, libb2 are discovered and how it falls back to code bundled with + borg), new: uses pkg-config now (and needs python "pkgconfig" package + installed), #1925 + + if you are a borg package maintainer, please try packaging this + (see comments in setup.py). +- Vagrantfile: add zstd, reorder, build env vars, #4444 +- travis: install script improvements +- update shell completions +- docs: + + - add a sample logging.conf in docs/misc, #4380 + - fix spelling errors + - update requirements / install docs, #4374 + + +Version 1.2.0a4 (2019-03-11) +---------------------------- + +Fixes: + +- do not use O_NONBLOCK for special files, like FIFOs, block and char devices + when using --read-special. fixes backing up FIFOs. fixes to test. #4394 +- more LibreSSL build fixes: LibreSSL has HMAC_CTX_free and HMAC_CTX_new + +New features: + +- check: incremental repo check (only checks crc32 for segment entries), #1657 + borg check --repository-only --max-duration SECONDS ... +- delete: timestamp for borg delete --info added, #4359 + +Other changes: + +- redo stale lock handling, #3986 + drop BORG_HOSTNAME_IS_UNIQUE (please use BORG_HOST_ID if needed). + borg now always assumes it has a unique host id - either automatically + from fqdn plus uuid.getnode() or overridden via BORG_HOST_ID. +- docs: + + - added Alpine Linux to distribution list + - elaborate on append-only mode docs +- vagrant: + + - darwin: new 10.12 box + - freebsd: new 12.0 box + - openbsd: new 6.4 box + - misc. updates / fixes + + +Version 1.2.0a3 (2019-02-26) +---------------------------- + +Fixes: + +- LibreSSL build fixes, #4403 +- dummy ACL/xattr code fixes (used by OpenBSD and others), #4403 +- create: fix openat/statat issues for root directory, #4405 + + +Version 1.2.0a2 and earlier (2019-02-24) +---------------------------------------- + +New features: + +- compact: "borg compact" needs to be used to free repository space by + compacting the segments (reading sparse segments, rewriting still needed + data to new segments, deleting the sparse segments). + Borg < 1.2 invoked compaction automatically at the end of each repository + writing command. + Borg >= 1.2 does not do that any more to give better speed, more control, + more segment file stability (== less stuff moving to newer segments) and + more robustness. + See the docs about "borg compact" for more details. +- "borg compact --cleanup-commits" is to cleanup the tons of 17byte long + commit-only segment files caused by borg 1.1.x issue #2850. + Invoke this once after upgrading (the server side) borg to 1.2. + Compaction now automatically removes unneeded commit-only segment files. +- prune: Show which rule was applied to keep archive, #2886 +- add fixed blocksize chunker (see --chunker-params docs), #1086 + +Fixes: + +- avoid stale filehandle issues, #3265 +- use more FDs, avoid race conditions on active fs, #906, #908, #1038 +- add O_NOFOLLOW to base flags, #908 +- compact: + + - require >10% freeable space in a segment, #2985 + - repository compaction now automatically removes unneeded 17byte + commit-only segments, #2850 +- make swidth available on all posix platforms, #2667 + +Other changes: + +- repository: better speed and less stuff moving around by using separate + segment files for manifest DELETEs and PUTs, #3947 +- use pyinstaller v3.3.1 to build binaries +- update bundled zstd code to 1.3.8, #4210 +- update bundled lz4 code to 1.8.3, #4209 +- msgpack: + + - switch to recent "msgpack" pypi pkg name, #3890 + - wrap msgpack to avoid future compat complications, #3632, #2738 + - support msgpack 0.6.0 and 0.6.1, #4220, #4308 + +- llfuse: modernize / simplify llfuse version requirements +- code refactorings / internal improvements: + + - include size/csize/nfiles[_parts] stats into archive, #3241 + - calc_stats: use archive stats metadata, if available + - crypto: refactored crypto to use an AEAD style API + - crypto: new AES-OCB, CHACHA20-POLY1305 + - create: use less syscalls by not using a python file obj, #906, #3962 + - diff: refactor the diff functionality to new ItemDiff class, #2475 + - archive: create FilesystemObjectProcessors class + - helpers: make a package, split into smaller modules + - xattrs: move to platform package, use cython instead ctypes, #2495 + - xattrs/acls/bsdflags: misc. code/api optimizations + - FUSE: separate creation of filesystem from implementation of llfuse funcs, #3042 + - FUSE: use unpacker.tell() instead of deprecated write_bytes, #3899 + - setup.py: move build_man / build_usage code to setup_docs.py + - setup.py: update to use a newer Cython/setuptools API for compiling .pyx -> .c, #3788 + - use python 3.5's os.scandir / os.set_blocking + - multithreading preparations (not used yet): + + - item.to_optr(), Item.from_optr() + - fix chunker holding the GIL during blocking I/O + - C code portability / basic MSC compatibility, #4147, #2677 +- testing: + + - vagrant: new VMs for linux/bsd/darwin, most with OpenSSL 1.1 and py36 + + +Version 1.1.18 (2022-06-05) +--------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). + If WSL still has a problem with sync_file_range, you need to set + BORG_WORKAROUNDS=basesyncfile in the borg process environment to + work around the WSL issue. +- 1.1.14 changes return codes due to a bug fix: + In case you have scripts expecting rc == 2 for a signal exit, you need to + update them to check for >= 128 (as documented since long). +- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now. +- 1.1.17 install_requires the "packaging" pypi package now. + +New features: + +- check --repair: significantly speed up search for next valid object in segment, #6022 +- create: add retry_erofs workaround for O_NOATIME issue on volume shadow copies in WSL1, #6024 +- key export: display key if path is '-' or not given, #6092 +- list --format: add command_line to format keys, #6108 + +Fixes: + +- check: improve error handling for corrupt archive metadata block, + make robust_iterator more robust, #4777 +- diff: support presence change for blkdev, chrdev and fifo items, #6483 +- diff: reduce memory consumption, fix is_hardlink_master +- init: disallow overwriting of existing keyfiles +- info: fix authenticated mode repo to show "Encrypted: No", #6462 +- info: emit repo info even if repo has 0 archives, #6120 +- list: remove placeholders for shake_* hashes, #6082 +- mount -o versions: give clear error msg instead of crashing +- show_progress: add finished=true/false to archive_progress json, #6570 +- fix hardlinkable file type check, #6037 +- do not show archive name in error msgs referring to the repository, #6023 +- prettier error msg (no stacktrace) if exclude file is missing, #5734 +- do not require BORG_CONFIG_DIR if BORG_{SECURITY,KEYS}_DIR are set, #5979 +- atomically create the CACHE_TAG file, #6028 +- deal with the SaveFile/SyncFile race, docs, see #6176 5c5b59bc9 +- avoid expanding path into LHS of formatting operation + tests, #6064 #6063 +- repository: quota / compactable computation fixes, #6119. + This is mainly to keep the repo code in sync with borg 1.2. As borg 1.1 + compacts immediately, there was not really an issue with this in 1.1. +- fix transaction rollback: use files cache filename as found in txn.active, #6353 +- do not load files cache for commands not using it, fixes #5673 +- fix scp repo url parsing for ip v6 addrs, #6526 +- repo::archive location placeholder expansion fixes, #5826, #5998 + + - use expanded location for log output + - support placeholder expansion for BORG_REPO env var +- respect umask for created directory and file modes, #6400 +- safer truncate_and_unlink implementation + +Other changes: + +- upgrade bundled xxhash code to 0.8.1 +- fix xxh64 related build (setup.py and post-0.8.1 patch for static_assert). + The patch was required to build the bundled xxhash code on FreeBSD, see + https://github.com/Cyan4973/xxHash/pull/670 +- msgpack build: remove endianness macro, #6105 +- update and fix shell completions +- fuse: remove unneeded version check and compat code +- delete --force: do not ask when deleting a repo, #5941 +- delete: don't commit if nothing was deleted, avoid cache sync, #6060 +- delete: add repository id and location to prompt +- compact segments: improve freeable / freed space log output, #5679 +- if ensure_dir() fails, give more informative error message, #5952 +- load_key: no key is same as empty key, #6441 +- better error msg for defect or unsupported repo configs, #6566 +- use hmac.compare_digest instead of ==, #6470 +- implement more standard hashindex.setdefault behaviour +- remove stray punctuation from secure-erase message +- add development.lock.txt, use a real python 3.5 to generate frozen reqs +- setuptools 60.7.0 breaks pyinstaller, #6246 +- setup.py clean2 was added to work around some setuptools customizability limitation. +- allow extra compiler flags for every extension build +- C code: make switch fallthrough explicit +- Cython code: fix "useless trailing comma" cython warnings +- requirements.lock.txt: use the latest cython 0.29.30 +- fix compilation warnings: ‘PyUnicode_AsUnicode’ is deprecated +- docs: + + - ~/.config/borg/keys is not used for repokey keys, #6107 + - excluded parent dir's metadata can't restore, #6062 + - permissions note rewritten to make it less confusing, #5490 + - add note about grandfather-father-son backup retention policy / rotation scheme + - clarify who starts the remote agent (borg serve) + - test/improve pull backup docs, #5903 + - document the socat pull mode described in #900 #515ß + - borg serve: improve ssh forced commands docs, #6083 + - improve docs for borg list --format, #6080 + - fix the broken link to .nix file + - clarify pattern usage with commands, #5176 + - clarify user_id vs uid for fuse, #5723 + - fix binary build freebsd/macOS version, #5942 + - FAQ: fix manifest-timestamp path, #6016 + - remove duplicate faq entries, #5926 + - fix sphinx warnings, #5919 + - virtualisation speed tips + - fix values of TAG bytes, #6515 + - recommend umask for passphrase file perms + - update link to ubuntu packages, #6485 + - clarify on-disk order and size of log entry fields, #6357 + - do not transform --/--- to unicode dashes + - improve linking inside docs, link to borg_placeholders, link to borg_patterns + - use same phrasing in misc. help texts + - borg init: explain the encryption modes better + - explain the difference between a path that ends with or without a slash, #6297 + - clarify usage of patternfile roots, #6242 + - borg key export: add examples + - updates about features not experimental any more: FUSE "versions" view, --pattern*, #6134 + - fix/update cygwin package requirements + - impact of deleting path/to/repo/nonce, #5858 + - warn about tampered server nonce + - mention BORG_FILES_CACHE_SUFFIX as alternative to BORG_FILES_CACHE_TTL, #5602 + - add a troubleshooting note about "is not a valid repository" to the FAQ +- vagrant / CI / testing: + + - misc. fixes and updates, new python versions + - macOS on github: re-enable fuse2 testing by downgrading to older macOS, #6099 + - fix OpenBSD symlink mode test failure, #2055 + - use the generic/openbsd6 box + - strengthen the test: we can read data w/o nonces + - add tests for path/to/repo/nonce deletion + - darwin64: backport some tunings from master + - darwin64: remove fakeroot, #6314 + - darwin64: fix vagrant scp, #5921 + - darwin64: use macfuse instead of osxfuse + - add ubuntu "jammy" 22.04 LTS VM + - adapt memory for openindiana64 and darwin64 + + +Version 1.1.17 (2021-07-12) +--------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). + If WSL still has a problem with sync_file_range, you need to set + BORG_WORKAROUNDS=basesyncfile in the borg process environment to + work around the WSL issue. +- 1.1.14 changes return codes due to a bug fix: + In case you have scripts expecting rc == 2 for a signal exit, you need to + update them to check for >= 128 (as documented since long). +- 1.1.15 drops python 3.4 support, minimum requirement is 3.5 now. +- 1.1.17 install_requires the "packaging" pypi package now. + +Fixes: + +- pyinstaller dir-mode: fix pyi detection / LIBPATH treatment, #5897 +- handle crash due to kill stale lock race, #5828 +- fix BORG_CACHE_DIR crashing borg if empty, #5216 +- create --dry-run: fix display of kept tagfile, #5834 +- fix missing parameter in "did not consistently fail" msg, #5822 +- missing / healed chunks: always tell chunk ID, #5704 +- benchmark: make sure cleanup happens even on exceptions, #5630 + +New features: + +- implement BORG_SELFTEST env variable, #5871. + this can be used to accelerate borg startup a bit. not recommended for + normal usage, but borg mass hosters with a lot of borg invocations can + save some resources with this. on my laptop, this saved ~100ms cpu time + (sys+user) per borg command invocation. +- implement BORG_LIBC env variable to give the libc filename, #5870. + you can use this if a borg does not find your libc. +- check: add progress indicator for archive check. +- allow --files-cache=size (not recommended, make sure you know what you do) + +Other changes: + +- Python 3.10 now officially supported! + we test on py310-dev on github CI since a while and now also on the vagrant + machines, so it should work ok. +- github CI: test on py310 (again) +- get rid of distutils, use packaging and setuptools. + distutils is deprecated and gives warnings on py 3.10. +- setup.py: rename "clean" to "clean2" to avoid shadowing the "clean" command. +- remove libc filename fallback for the BSDs (there is no "usual" name) +- cleanup flake8 checks, fix some pep8 violations. +- docs building: replace deprecated function ".add_stylesheet()" for Sphinx 4 compatibility +- docs: + + - add a hint on sleeping computer and ssh connections, #5301 + - update the documentation on hacked backup client, #5480 + - improve docs/FAQ about append-only remote repos, #5497 + - complement the documentation for pattern files and exclude files, #5520 + - "filename with spaces" example added to exclude file, #5236 + note: no whitespace escaping needed, processed by borg. + - add info on renaming repositories, #5240 + - clarify borg check --verify-data, #5808 + - add notice about defective hardware to check documentation, #5753 + - add paragraph added in #5855 to utility documentation source + - add missing leading slashes in help patterns, #5857 + - clarify "you will need key and passphrase" borg init warning, #4622 + - pull mode: add some warnings, #5827 + - mention tar --compare (compare archive to fs files), #5880 + - fix typos, backport of #5597 +- vagrant: + + - add py3.7.11 for binary build, also add 3.10-dev. + - use latest Cython 0.29.23 for py310 compat fixes. + - more RAM for openindiana upgrade plan resolver, it just hangs (swaps?) if + there is too little RAM. + - fix install_pyenv to adapt to recent changes in pyenv (same as in master now). + - use generic/netbsd9 box, copied from master branch. + + +Version 1.1.16 (2021-03-23) +--------------------------- + +Fixes: + +- setup.py: add special openssl prefix for Apple M1 compatibility +- do not recurse into duplicate roots, #5603 +- remove empty shadowed_segments lists, #5275, #5614 +- fix libpython load error when borg fat binary / dir-based binary is invoked + via a symlink by upgrading pyinstaller to v4.2, #5688 +- config: accept non-int value (like 500M or 100G) for max_segment_size or + storage_quota, #5639. + please note: when setting a non-int value for this in a repo config, + using the repo will require borg >= 1.1.16. + +New features: + +- bundled msgpack: drop support for old buffer protocol to support Python 3.10 +- verbose files cache logging via --debug-topic=files_cache, #5659. + Use this if you suspect that borg does not detect unmodified files as expected. +- create/extract: add --noxattrs and --noacls option, #3955. + when given with borg create, borg will not get xattrs / ACLs from input files + (and thus, it will not archive xattrs / ACLs). when given with borg extract, + borg will not read xattrs / ACLs from archive and will not set xattrs / ACLs + on extracted files. +- diff: add --json-lines option, #3765 +- check: debug log segment filename +- borg debug dump-hints + +Other changes: + +- Tab completion support for additional archives for 'borg delete' +- repository: deduplicate code of put and delete, no functional change +- tests: fix result order issue (sporadic test failure on openindiana) +- vagrant: + + - upgrade pyinstaller to v4.2, #5671 + - avoid grub-install asking interactively for device + - remove the xenial box + - update freebsd box to 12.1 +- docs: + + - update macOS install instructions, #5677 + - use macFUSE (not osxfuse) for Apple M1 compatibility + - update docs for dev environment installation instructions, #5643 + - fix grammar in faq + - recommend running tests only on installed versions for setup + - add link back to git-installation + - remove /var/cache exclusion in example commands, #5625. + This is generally a poor idea and shouldn't be promoted through examples. + - add repology.org badge with current packaging status + - explain hash collision + - add unsafe workaround to use an old repo copy, #5722 + + +Version 1.1.15 (2020-12-25) +--------------------------- + +Fixes: + +- extract: + + - improve exception handling when setting xattrs, #5092. + - emit a warning message giving the path, xattr key and error message. + - continue trying to restore other xattrs and bsdflags of the same file + after an exception with xattr-setting happened. +- export-tar: + + - set tar format to GNU_FORMAT explicitly, #5274 + - fix memory leak with ssh: remote repository, #5568 + - fix potential memory leak with ssh: remote repository with partial extraction +- create: fix --dry-run and --stats coexistence, #5415 +- use --timestamp for {utcnow} and {now} if given, #5189 + +New features: + +- create: implement --stdin-mode, --stdin-user and --stdin-group, #5333 +- allow appending the files cache filename with BORG_FILES_CACHE_SUFFIX env var + +Other changes: + +- drop python 3.4 support, minimum requirement is 3.5 now. +- enable using libxxhash instead of bundled xxh64 code +- update llfuse requirements (1.3.8) +- set cython language_level in some files to fix warnings +- allow EIO with warning when trying to hardlink +- PropDict: fail early if internal_dict is not a dict +- update shell completions +- tests / CI + + - add a test for the hashindex corruption bug, #5531 #4829 + - fix spurious failure in test_cache_files, #5438 + - added a github ci workflow + - reduce testing on travis, no macOS, no py3x-dev, #5467 + - travis: use newer dists, native py on dist +- vagrant: + + - remove jessie and trusty boxes, #5348 #5383 + - pyinstaller 4.0, build on py379 + - binary build on stretch64, #5348 + - remove easy_install based pip installation +- docs: + + - clarify '--one-file-system' for btrfs, #5391 + - add example for excluding content using the --pattern cmd line arg + - complement the documentation for pattern files and exclude files, #5524 + - made ansible playbook more generic, use package instead of pacman. also + change state from "latest" to "present". + - complete documentation on append-only remote repos, #5497 + - internals: rather talk about target size than statistics, #5336 + - new compression algorithm policy, #1633 #5505 + - faq: add a hint on sleeping computer, #5301 + - note requirements for full disk access on macOS Catalina, #5303 + - fix/improve description of borg upgrade hardlink usage, #5518 +- modernize 1.1 code: + + - drop code/workarounds only needed to support Python 3.4 + - remove workaround for pre-release py37 argparse bug + - removed some outdated comments/docstrings + - requirements: remove some restrictions, lock on current versions + + +Version 1.1.14 (2020-10-07) +--------------------------- + +Fixes: + +- check --repair: fix potential data loss when interrupting it, #5325 +- exit with 128 + signal number (as documented) when borg is killed by a signal, #5161 +- fix hardlinked CACHEDIR.TAG processing, #4911 +- create --read-special: .part files also should be regular files, #5217 +- llfuse dependency: choose least broken 1.3.6/1.3.7. + 1.3.6 is broken on python 3.9, 1.3.7 is broken on FreeBSD. + +Other changes: + +- upgrade bundled xxhash to 0.7.4 +- self test: if it fails, also point to OS and hardware, #5334 +- pyinstaller: compute basepath from spec file location +- prettier error message when archive gets too big, #5307 +- check/recreate are not "experimental" any more (but still potentially dangerous): + + - recreate: remove extra confirmation + - rephrase some warnings, update docs, #5164 +- shell completions: + + - misc. updates / fixes + - support repositories in fish tab completion, #5256 + - complete $BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING + - rewrite zsh completion: + + - completion for almost all optional and positional arguments + - completion for Borg environment variables (parameters) +- use "allow/deny list" instead of "white/black list" wording +- declare "allow_cache_wipe" marker in setup.cfg to avoid pytest warning +- vagrant / tests: + + - misc. fixes / updates + - use python 3.5.10 for binary build + - build directory-based binaries additionally to the single file binaries + - add libffi-dev, required to build python + - use cryptography<3.0, more recent versions break the jessie box + - test on python 3.9 + - do brew update with /dev/null redirect to avoid "too much log output" on travis-ci +- docs: + + - add ssh-agent pull backup method docs, #5288 + - how to approach borg speed issues, #5371 + - mention double --force in prune docs + - update Homebrew install instructions, #5185 + - better description of how cache and rebuilds of it work + - point to borg create --list item flags in recreate usage, #5165 + - add security faq explaining AES-CTR crypto issues, #5254 + - add a note to create from stdin regarding files cache, #5180 + - fix borg.1 manpage generation regression, #5211 + - clarify how exclude options work in recreate, #5193 + - add section for retired contributors + - hint about not misusing private email addresses of contributors for borg support + + +Version 1.1.13 (2020-06-06) +--------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). + If WSL still has a problem with sync_file_range, you need to set + BORG_WORKAROUNDS=basesyncfile in the borg process environment to + work around the WSL issue. + +Fixes: + +- rebuilt using a current Cython version, compatible with python 3.8, #5214 + + +Version 1.1.12 (2020-06-06) +--------------------------- + +Fixes: + +- fix preload-related memory leak, #5202. +- mount / borgfs (FUSE filesystem): + + - fix FUSE low linear read speed on large files, #5067 + - fix crash on old llfuse without birthtime attrs, #5064 - accidentally + we required llfuse >= 1.3. Now also old llfuse works again. + - set f_namemax in statfs result, #2684 +- update precedence of env vars to set config and cache paths, #4894 +- correctly calculate compression ratio, taking header size into account, too + +New features: + +- --bypass-lock option to bypass locking with read-only repositories + +Other changes: + +- upgrade bundled zstd to 1.4.5 +- travis: adding comments and explanations to Travis config / install script, + improve macOS builds. +- tests: test_delete_force: avoid sporadic test setup issues, #5196 +- misc. vagrant fixes +- the binary for macOS is now built on macOS 10.12 +- the binaries for Linux are now built on Debian 8 "Jessie", #3761 +- docs: + + - PlaceholderError not printed as JSON, #4073 + - "How important is Borg config?", #4941 + - make Sphinx warnings break docs build, #4587 + - some markup / warning fixes + - add "updating borgbackup.org/releases" to release checklist, #4999 + - add "rendering docs" to release checklist, #5000 + - clarify borg init's encryption modes + - add note about patterns and stored paths, #4160 + - add upgrade of tools to pip installation how-to + - document one cause of orphaned chunks in check command, #2295 + - linked recommended restrictions to ssh public keys on borg servers in faq, #4946 + + +Version 1.1.11 (2020-03-08) +--------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- 1.1.11 removes WSL autodetection (Windows 10 Subsystem for Linux). + If WSL still has a problem with sync_file_range, you need to set + BORG_WORKAROUNDS=basesyncfile in the borg process environment to + work around the WSL issue. + +Fixes: + +- fixed potential index corruption / data loss issue due to bug in hashindex_set, #4829. + Please read and follow the more detailed notes close to the top of this document. +- upgrade bundled xxhash to 0.7.3, #4891. + 0.7.2 is the minimum requirement for correct operations on ARMv6 in non-fixup + mode, where unaligned memory accesses cause bus errors. + 0.7.3 adds some speedups and libxxhash 0.7.3 even has a pkg-config file now. +- upgrade bundled lz4 to 1.9.2 +- upgrade bundled zstd to 1.4.4 +- fix crash when upgrading erroneous hints file, #4922 +- extract: + + - fix KeyError for "partial" extraction, #4607 + - fix "partial" extract for hardlinked contentless file types, #4725 + - fix preloading for old (0.xx) remote servers, #4652 + - fix confusing output of borg extract --list --strip-components, #4934 +- delete: after double-force delete, warn about necessary repair, #4704 +- create: give invalid repo error msg if repo config not found, #4411 +- mount: fix FUSE mount missing st_birthtime, #4763 #4767 +- check: do not stumble over invalid item key, #4845 +- info: if the archive doesn't exist, print a pretty message, #4793 +- SecurityManager.known(): check all files, #4614 +- Repository.open: use stat() to check for repo dir, #4695 +- Repository.check_can_create_repository: use stat() to check, #4695 +- fix invalid archive error message +- fix optional/non-optional location arg, #4541 +- commit-time free space calc: ignore bad compact map entries, #4796 +- ignore EACCES (errno 13) when hardlinking the old config, #4730 +- --prefix / -P: fix processing, avoid argparse issue, #4769 + +New features: + +- enable placeholder usage in all extra archive arguments +- new BORG_WORKAROUNDS mechanism, basesyncfile, #4710 +- recreate: support --timestamp option, #4745 +- support platforms without os.link (e.g. Android with Termux), #4901. + if we don't have os.link, we just extract another copy instead of making a hardlink. +- support linux platforms without sync_file_range (e.g. Android 7 with Termux), #4905 + +Other: + +- ignore --stats when given with --dry-run, but continue, #4373 +- add some ProgressIndicator msgids to code / fix docs, #4935 +- elaborate on "Calculating size" message +- argparser: always use REPOSITORY in metavar, also use more consistent help phrasing. +- check: improve error output for matching index size, see #4829 +- docs: + + - changelog: add advisory about hashindex_set bug #4829 + - better describe BORG_SECURITY_DIR, BORG_CACHE_DIR, #4919 + - infos about cache security assumptions, #4900 + - add FAQ describing difference between a local repo vs. repo on a server. + - document how to test exclusion patterns without performing an actual backup + - timestamps in the files cache are now usually ctime, #4583 + - fix bad reference to borg compact (does not exist in 1.1), #4660 + - create: borg 1.1 is not future any more + - extract: document limitation "needs empty destination", #4598 + - how to supply a passphrase, use crypto devices, #4549 + - fix osxfuse github link in installation docs + - add example of exclude-norecurse rule in help patterns + - update macOS Brew link + - add note about software for automating backups, #4581 + - AUTHORS: mention copyright+license for bundled msgpack + - fix various code blocks in the docs, #4708 + - updated docs to cover use of temp directory on remote, #4545 + - add restore docs, #4670 + - add a pull backup / push restore how-to, #1552 + - add FAQ how to retain original paths, #4532 + - explain difference between --exclude and --pattern, #4118 + - add FAQs for SSH connection issues, #3866 + - improve password FAQ, #4591 + - reiterate that 'file cache names are absolute' in FAQ +- tests: + + - cope with ANY error when importing pytest into borg.testsuite, #4652 + - fix broken test that relied on improper zlib assumptions + - test_fuse: filter out selinux xattrs, #4574 +- travis / vagrant: + + - misc python versions removed / changed (due to openssl 1.1 compatibility) + or added (3.7 and 3.8, for better borg compatibility testing) + - binary building is on python 3.5.9 now +- vagrant: + + - add new boxes: ubuntu 18.04 and 20.04, debian 10 + - update boxes: openindiana, darwin, netbsd + - remove old boxes: centos 6 + - darwin: updated osxfuse to 3.10.4 + - use debian/ubuntu pip/virtualenv packages + - rather use python 3.6.2 than 3.6.0, fixes coverage/sqlite3 issue + - use requirements.d/development.lock.txt to avoid compat issues +- travis: + + - darwin: backport some install code / order from master + - remove deprecated keyword "sudo" from travis config + - allow osx builds to fail, #4955 + this is due to travis-ci frequently being so slow that the OS X builds + just fail because they exceed 50 minutes and get killed by travis. + + +Version 1.1.10 (2019-05-16) +--------------------------- + +Fixes: + +- extract: hang on partial extraction with ssh: repo, when hardlink master + is not matched/extracted and borg hangs on related slave hardlink, #4350 +- lrucache: regularly remove old FDs, #4427 +- avoid stale filehandle issues, #3265 +- freebsd: make xattr platform code api compatible with linux, #3952 +- use whitelist approach for borg serve, #4097 +- borg command shall terminate with rc 2 for ImportErrors, #4424 +- create: only run stat_simple_attrs() once, this increases + backup with lots of unchanged files performance by ~ 5%. +- prune: fix incorrect borg prune --stats output with --dry-run, #4373 +- key export: emit user-friendly error if repo key is exported to a directory, + #4348 + +New features: + +- bundle latest supported msgpack-python release (0.5.6), remove msgpack-python + from setup.py install_requires - by default we use the bundled code now. + optionally, we still support using an external msgpack (see hints in + setup.py), but this requires solid requirements management within + distributions and is not recommended. + borgbackup will break if you upgrade msgpack to an unsupported version. +- display msgpack version as part of sysinfo (e.g. in tracebacks) +- timestamp for borg delete --info added, #4359 +- enable placeholder usage in --comment and --glob-archives, #4559, #4495 + +Other: + +- serve: do not check python/libc for borg serve, #4483 +- shell completions: borg diff second archive +- release scripts: signing binaries with Qubes OS support +- testing: + + - vagrant: upgrade openbsd box to 6.4 + - travis-ci: lock test env to py 3.4 compatible versions, #4343 + - get rid of confusing coverage warning, #2069 + - rename test_mount_hardlinks to test_fuse_mount_hardlinks, + so both can be excluded by "not test_fuse". + - pure-py msgpack warning shall not make a lot of tests fail, #4558 +- docs: + + - add "SSH Configuration" section to "borg serve", #3988, #636, #4485 + - README: new URL for funding options + - add a sample logging.conf in docs/misc, #4380 + - elaborate on append-only mode docs, #3504 + - installation: added Alpine Linux to distribution list, #4415 + - usage.html: only modify window.location when redirecting, #4133 + - add msgpack license to docs/3rd_party/msgpack +- vagrant / binary builds: + + - use python 3.5.7 for builds + - use osxfuse 3.8.3 + + +Version 1.1.9 (2019-02-10) +-------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. + +Fixes: + +- security fix: configure FUSE with "default_permissions", #3903 + "default_permissions" is now enforced by borg by default to let the + kernel check uid/gid/mode based permissions. + "ignore_permissions" can be given not to enforce "default_permissions". +- make "hostname" short, even on misconfigured systems, #4262 +- fix free space calculation on macOS (and others?), #4289 +- config: quit with error message when no key is provided, #4223 +- recover_segment: handle too small segment files correctly, #4272 +- correctly release memoryview, #4243 +- avoid diaper pattern in configparser by opening files, #4263 +- add "# cython: language_level=3" directive to .pyx files, #4214 +- info: consider part files for "This archive" stats, #3522 +- work around Microsoft WSL issue #645 (sync_file_range), #1961 + +New features: + +- add --rsh command line option to complement BORG_RSH env var, #1701 +- init: --make-parent-dirs parent1/parent2/repo_dir, #4235 + +Other: + +- add archive name to check --repair output, #3447 +- check for unsupported msgpack versions +- shell completions: + + - new shell completions for borg 1.1.9 + - more complete shell completions for borg mount -o + - added shell completions for borg help + - option arguments for zsh tab completion +- docs: + + - add FAQ regarding free disk space check, #3905 + - update BORG_PASSCOMMAND example and clarify variable expansion, #4249 + - FAQ regarding change of compression settings, #4222 + - add note about BSD flags to changelog, #4246 + - improve logging in example automation script + - add note about files changing during backup, #4081 + - work around the backslash issue, #4280 + - update release workflow using twine (docs, scripts), #4213 + - add warnings on repository copies to avoid future problems, #4272 +- tests: + + - fix the homebrew 1.9 issues on travis-ci, #4254 + - fix duplicate test method name, #4311 + + +Version 1.1.8 (2018-12-09) +-------------------------- + +Fixes: + +- enforce storage quota if set by serve-command, #4093 +- invalid locations: give err msg containing parsed location, #4179 +- list repo: add placeholders for hostname and username, #4130 +- on linux, symlinks can't have ACLs, so don't try to set any, #4044 + +New features: + +- create: added PATH::archive output on INFO log level +- read a passphrase from a file descriptor specified in the + BORG_PASSPHRASE_FD environment variable. + +Other: + +- docs: + + - option --format is required for some expensive-to-compute values for json + + borg list by default does not compute expensive values except when + they are needed. whether they are needed is determined by the format, + in standard mode as well as in --json mode. + - tell that our binaries are x86/x64 amd/intel, bauerj has ARM + - fixed wrong archive name pattern in CRUD benchmark help + - fixed link to cachedir spec in docs, #4140 +- tests: + + - stop using fakeroot on travis, avoids sporadic EISDIR errors, #2482 + - xattr key names must start with "user." on linux + - fix code so flake8 3.6 does not complain + - explicitly convert environment variable to str, #4136 + - fix DeprecationWarning: Flags not at the start of the expression, #4137 + - support pytest4, #4172 +- vagrant: + + - use python 3.5.6 for builds + + +Version 1.1.7 (2018-08-11) +-------------------------- + +Compatibility notes: + +- added support for Python 3.7 + +Fixes: + +- cache lock: use lock_wait everywhere to fix infinite wait, see #3968 +- don't archive tagged dir when recursing an excluded dir, #3991 +- py37 argparse: work around bad default in py 3.7.0a/b/rc, #3996 +- py37 remove loggerDict.clear() from tearDown method, #3805 +- some fixes for bugs which likely did not result in problems in practice: + + - fixed logic bug in platform module API version check + - fixed xattr/acl function prototypes, added missing ones + +New features: + +- init: add warning to store both key and passphrase at safe place(s) +- BORG_HOST_ID env var to work around all-zero MAC address issue, #3985 +- borg debug dump-repo-objs --ghost (dump everything from segment files, + including deleted or superseded objects or commit tags) +- borg debug search-repo-objs (search in repo objects for hex bytes or strings) + +Other changes: + +- add Python 3.7 support +- updated shell completions +- call socket.gethostname only once +- locking: better logging, add some asserts +- borg debug dump-repo-objs: + + - filename layout improvements + - use repository.scan() to get on-disk order +- docs: + + - update installation instructions for macOS + - added instructions to install fuse via homebrew + - improve diff docs + - added note that checkpoints inside files requires 1.1+ + - add link to tempfile module + - remove row/column-spanning from docs source, #4000 #3990 +- tests: + + - fetch less data via os.urandom + - add py37 env for tox + - travis: add 3.7, remove 3.6-dev (we test with -dev in master) +- vagrant / binary builds: + + - use osxfuse 3.8.2 + - use own (uptodate) openindiana box + + +Version 1.1.6 (2018-06-11) +-------------------------- + +Compatibility notes: + +- 1.1.6 changes: + + - also allow msgpack-python 0.5.6. + +Fixes: + +- fix borg exception handling on ENOSPC error with xattrs, #3808 +- prune: fix/improve overall progress display +- borg config repo ... does not need cache/manifest/key, #3802 +- debug dump-repo-objs should not depend on a manifest obj +- pypi package: + + - include .coveragerc, needed by tox.ini + - fix package long description, #3854 + +New features: + +- mount: add uid, gid, umask mount options +- delete: + + - only commit once, #3823 + - implement --dry-run, #3822 +- check: + + - show progress while rebuilding missing manifest, #3787 + - more --repair output +- borg config --list , #3612 + +Other changes: + +- update msgpack requirement, #3753 +- update bundled zstd to 1.3.4, #3745 +- update bundled lz4 code to 1.8.2, #3870 +- docs: + + - describe what BORG_LIBZSTD_PREFIX does + - fix and deduplicate encryption quickstart docs, #3776 +- vagrant: + + - FUSE for macOS: upgrade 3.7.1 to 3.8.0 + - exclude macOS High Sierra upgrade on the darwin64 machine + - remove borgbackup.egg-info dir in fs_init (after rsync) + - use pyenv-based build/test on jessie32/62 + - use local 32 and 64bit debian jessie boxes + - use "vagrant" as username for new xenial box +- travis OS X: use xcode 8.3 (not broken) + + +Version 1.1.5 (2018-04-01) +-------------------------- + +Compatibility notes: + +- 1.1.5 changes: + + - require msgpack-python >= 0.4.6 and < 0.5.0. + 0.5.0+ dropped python 3.4 testing and also caused some other issues because + the python package was renamed to msgpack and emitted some FutureWarning. + +Fixes: + +- create --list: fix that it was never showing M status, #3492 +- create: fix timing for first checkpoint (read files cache early, init + checkpoint timer after that), see #3394 +- extract: set rc=1 when extracting damaged files with all-zero replacement + chunks or with size inconsistencies, #3448 +- diff: consider an empty file as different to a non-existing file, #3688 +- files cache: improve exception handling, #3553 +- ignore exceptions in scandir_inorder() caused by an implicit stat(), + also remove unneeded sort, #3545 +- fixed tab completion problem where a space is always added after path even + when it shouldn't +- build: do .h file content checks in binary mode, fixes build issue for + non-ascii header files on pure-ascii locale platforms, #3544 #3639 +- borgfs: fix patterns/paths processing, #3551 +- config: add some validation, #3566 +- repository config: add validation for max_segment_size, #3592 +- set cache previous_location on load instead of save +- remove platform.uname() call which caused library mismatch issues, #3732 +- add exception handler around deprecated platform.linux_distribution() call +- use same datetime object for {now} and {utcnow}, #3548 + +New features: + +- create: implement --stdin-name, #3533 +- add chunker_params to borg archive info (--json) +- BORG_SHOW_SYSINFO=no to hide system information from exceptions + +Other changes: + +- updated zsh completions for borg 1.1.4 +- files cache related code cleanups +- be more helpful when parsing invalid --pattern values, #3575 +- be more clear in secure-erase warning message, #3591 +- improve getpass user experience, #3689 +- docs build: unicode problem fixed when using a py27-based sphinx +- docs: + + - security: explicitly note what happens OUTSIDE the attack model + - security: add note about combining compression and encryption + - security: describe chunk size / proximity issue, #3687 + - quickstart: add note about permissions, borg@localhost, #3452 + - quickstart: add introduction to repositories & archives, #3620 + - recreate --recompress: add missing metavar, clarify description, #3617 + - improve logging docs, #3549 + - add an example for --pattern usage, #3661 + - clarify path semantics when matching, #3598 + - link to offline documentation from README, #3502 + - add docs on how to verify a signed release with GPG, #3634 + - chunk seed is generated per repository (not: archive) + - better formatting of CPU usage documentation, #3554 + - extend append-only repo rollback docs, #3579 +- tests: + + - fix erroneously skipped zstd compressor tests, #3606 + - skip a test if argparse is broken, #3705 +- vagrant: + + - xenial64 box now uses username 'vagrant', #3707 + - move cleanup steps to fs_init, #3706 + - the boxcutter wheezy boxes are 404, use local ones + - update to Python 3.5.5 (for binary builds) + + +Version 1.1.4 (2017-12-31) +-------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- borg 1.1.4 changes: + + - zstd compression is new in borg 1.1.4, older borg can't handle it. + - new minimum requirements for the compression libraries - if the required + versions (header and lib) can't be found at build time, bundled code will + be used: + + - added requirement: libzstd >= 1.3.0 (bundled: 1.3.2) + - updated requirement: liblz4 >= 1.7.0 / r129 (bundled: 1.8.0) + +Fixes: + +- check: data corruption fix: fix for borg check --repair malfunction, #3444. + See the more detailed notes close to the top of this document. +- delete: also delete security dir when deleting a repo, #3427 +- prune: fix building the "borg prune" man page, #3398 +- init: use given --storage-quota for local repo, #3470 +- init: properly quote repo path in output +- fix startup delay with dns-only own fqdn resolving, #3471 + +New features: + +- added zstd compression. try it! +- added placeholder {reverse-fqdn} for fqdn in reverse notation +- added BORG_BASE_DIR environment variable, #3338 + +Other changes: + +- list help topics when invalid topic is requested +- fix lz4 deprecation warning, requires lz4 >= 1.7.0 (r129) +- add parens for C preprocessor macro argument usages (did not cause malfunction) +- exclude broken pytest 3.3.0 release +- updated fish/bash completions +- init: more clear exception messages for borg create, #3465 +- docs: + + - add auto-generated docs for borg config + - don't generate HTML docs page for borgfs, #3404 + - docs update for lz4 b2 zstd changes + - add zstd to compression help, readme, docs + - update requirements and install docs about bundled lz4 and zstd +- refactored build of the compress and crypto.low_level extensions, #3415: + + - move some lib/build related code to setup_{zstd,lz4,b2}.py + - bundle lz4 1.8.0 (requirement: >= 1.7.0 / r129) + - bundle zstd 1.3.2 (requirement: >= 1.3.0) + - blake2 was already bundled + - rename BORG_LZ4_PREFIX env var to BORG_LIBLZ4_PREFIX for better consistency: + we also have BORG_LIBB2_PREFIX and BORG_LIBZSTD_PREFIX now. + - add prefer_system_lib* = True settings to setup.py - by default the build + will prefer a shared library over the bundled code, if library and headers + can be found and meet the minimum requirements. + + +Version 1.1.3 (2017-11-27) +-------------------------- + +Fixes: + +- Security Fix for CVE-2017-15914: Incorrect implementation of access controls + allows remote users to override repository restrictions in Borg servers. + A user able to access a remote Borg SSH server is able to circumvent access + controls post-authentication. + Affected releases: 1.1.0, 1.1.1, 1.1.2. Releases 1.0.x are NOT affected. +- crc32: deal with unaligned buffer, add tests - this broke borg on older ARM + CPUs that can not deal with unaligned 32bit memory accesses and raise a bus + error in such cases. the fix might also improve performance on some CPUs as + all 32bit memory accesses by the crc32 code are properly aligned now. #3317 +- mount: fixed support of --consider-part-files and do not show .borg_part_N + files by default in the mounted FUSE filesystem. #3347 +- fixed cache/repo timestamp inconsistency message, highlight that information + is obtained from security dir (deleting the cache will not bypass this error + in case the user knows this is a legitimate repo). +- borgfs: don't show sub-command in borgfs help, #3287 +- create: show an error when --dry-run and --stats are used together, #3298 + +New features: + +- mount: added exclusion group options and paths, #2138 + + Reused some code to support similar options/paths as borg extract offers - + making good use of these to mount only a smaller subset of dirs/files can + speed up mounting a lot and also will consume way less memory. + + borg mount [options] repo_or_archive mountpoint path [paths...] + + paths: you can just give some "root paths" (like for borg extract) to + only partially populate the FUSE filesystem. + + new options: --exclude[-from], --pattern[s-from], --strip-components +- create/extract: support st_birthtime on platforms supporting it, #3272 +- add "borg config" command for querying/setting/deleting config values, #3304 + +Other changes: + +- clean up and simplify packaging (only package committed files, do not install + .c/.h/.pyx files) +- docs: + + - point out tuning options for borg create, #3239 + - add instructions for using ntfsclone, zerofree, #81 + - move image backup-related FAQ entries to a new page + - clarify key aliases for borg list --format, #3111 + - mention break-lock in checkpointing FAQ entry, #3328 + - document sshfs rename workaround, #3315 + - add FAQ about removing files from existing archives + - add FAQ about different prune policies + - usage and man page for borgfs, #3216 + - clarify create --stats duration vs. wall time, #3301 + - clarify encrypted key format for borg key export, #3296 + - update release checklist about security fixes + - document good and problematic option placements, fix examples, #3356 + - add note about using --nobsdflags to avoid speed penalty related to + bsdflags, #3239 + - move most of support section to www.borgbackup.org + + +Version 1.1.2 (2017-11-05) +-------------------------- + +Fixes: + +- fix KeyError crash when talking to borg server < 1.0.7, #3244 +- extract: set bsdflags last (include immutable flag), #3263 +- create: don't do stat() call on excluded-norecurse directory, fix exception + handling for stat() call, #3209 +- create --stats: do not count data volume twice when checkpointing, #3224 +- recreate: move chunks_healthy when excluding hardlink master, #3228 +- recreate: get rid of chunks_healthy when rechunking (does not match), #3218 +- check: get rid of already existing not matching chunks_healthy metadata, #3218 +- list: fix stdout broken pipe handling, #3245 +- list/diff: remove tag-file options (not used), #3226 + +New features: + +- bash, zsh and fish shell auto-completions, see scripts/shell_completions/ +- added BORG_CONFIG_DIR env var, #3083 + +Other changes: + +- docs: + + - clarify using a blank passphrase in keyfile mode + - mention "!" (exclude-norecurse) type in "patterns" help + - document to first heal before running borg recreate to re-chunk stuff, + because that will have to get rid of chunks_healthy metadata. + - more than 23 is not supported for CHUNK_MAX_EXP, #3115 + - borg does not respect nodump flag by default any more + - clarify same-filesystem requirement for borg upgrade, #2083 + - update / rephrase cygwin / WSL status, #3174 + - improve docs about --stats, #3260 +- vagrant: openindiana new clang package + +Already contained in 1.1.1 (last minute fix): + +- arg parsing: fix fallback function, refactor, #3205. This is a fixup + for #3155, which was broken on at least python <= 3.4.2. + + +Version 1.1.1 (2017-10-22) +-------------------------- + +Compatibility notes: + +- The deprecated --no-files-cache is not a global/common option any more, + but only available for borg create (it is not needed for anything else). + Use --files-cache=disabled instead of --no-files-cache. +- The nodump flag ("do not back up this file") is not honoured any more by + default because this functionality (esp. if it happened by error or + unexpected) was rather confusing and unexplainable at first to users. + If you want that "do not back up NODUMP-flagged files" behaviour, use: + borg create --exclude-nodump ... +- If you are on Linux and do not need bsdflags archived, consider using + ``--nobsdflags`` with ``borg create`` to avoid additional syscalls and + speed up backup creation. + +Fixes: + +- borg recreate: correctly compute part file sizes. fixes cosmetic, but + annoying issue as borg check complains about size inconsistencies of part + files in affected archives. you can solve that by running borg recreate on + these archives, see also #3157. +- bsdflags support: do not open BLK/CHR/LNK files, avoid crashes and + slowness, #3130 +- recreate: don't crash on attic archives w/o time_end, #3109 +- don't crash on repository filesystems w/o hardlink support, #3107 +- don't crash in first part of truncate_and_unlink, #3117 +- fix server-side IndexError crash with clients < 1.0.7, #3192 +- don't show traceback if only a global option is given, show help, #3142 +- cache: use SaveFile for more safety, #3158 +- init: fix wrong encryption choices in command line parser, fix missing + "authenticated-blake2", #3103 +- move --no-files-cache from common to borg create options, #3146 +- fix detection of non-local path (failed on ..filename), #3108 +- logging with fileConfig: set json attr on "borg" logger, #3114 +- fix crash with relative BORG_KEY_FILE, #3197 +- show excluded dir with "x" for tagged dirs / caches, #3189 + +New features: + +- create: --nobsdflags and --exclude-nodump options, #3160 +- extract: --nobsdflags option, #3160 + +Other changes: + +- remove annoying hardlinked symlinks warning, #3175 +- vagrant: use self-made FreeBSD 10.3 box, #3022 +- travis: don't brew update, hopefully fixes #2532 +- docs: + + - readme: -e option is required in borg 1.1 + - add example showing --show-version --show-rc + - use --format rather than --list-format (deprecated) in example + - update docs about hardlinked symlinks limitation + + +Version 1.1.0 (2017-10-07) +-------------------------- + +Compatibility notes: + +- borg command line: do not put options in between positional arguments + + This sometimes works (e.g. it worked in borg 1.0.x), but can easily stop + working if we make positional arguments optional (like it happened for + borg create's "paths" argument in 1.1). There are also places in borg 1.0 + where we do that, so it doesn't work there in general either. #3356 + + Good: borg create -v --stats repo::archive path + Good: borg create repo::archive path -v --stats + Bad: borg create repo::archive -v --stats path + +Fixes: + +- fix LD_LIBRARY_PATH restoration for subprocesses, #3077 +- "auto" compression: make sure expensive compression is actually better, + otherwise store lz4 compressed data we already computed. + +Other changes: + +- docs: + + - FAQ: we do not implement futile attempts of ETA / progress displays + - manpage: fix typos, update homepage + - implement simple "issue" role for manpage generation, #3075 + + +Version 1.1.0rc4 (2017-10-01) +----------------------------- + +Compatibility notes: + +- A borg server >= 1.1.0rc4 does not support borg clients 1.1.0b3-b5. #3033 +- The files cache is now controlled differently and has a new default mode: + + - the files cache now uses ctime by default for improved file change + detection safety. You can still use mtime for more speed and less safety. + - --ignore-inode is deprecated (use --files-cache=... without "inode") + - --no-files-cache is deprecated (use --files-cache=disabled) + +New features: + +- --files-cache - implement files cache mode control, #911 + You can now control the files cache mode using this option: + --files-cache={ctime,mtime,size,inode,rechunk,disabled} + (only some combinations are supported). See the docs for details. + +Fixes: + +- remote progress/logging: deal with partial lines, #2637 +- remote progress: flush json mode output +- fix subprocess environments, #3050 (and more) + +Other changes: + +- remove client_supports_log_v3 flag, #3033 +- exclude broken Cython 0.27(.0) in requirements, #3066 +- vagrant: + + - upgrade to FUSE for macOS 3.7.1 + - use Python 3.5.4 to build the binaries +- docs: + + - security: change-passphrase only changes the passphrase, #2990 + - fixed/improved borg create --compression examples, #3034 + - add note about metadata dedup and --no[ac]time, #2518 + - twitter account @borgbackup now, better visible, #2948 + - simplified rate limiting wrapper in FAQ + + +Version 1.1.0rc3 (2017-09-10) +----------------------------- + +New features: + +- delete: support naming multiple archives, #2958 + +Fixes: + +- repo cleanup/write: invalidate cached FDs, #2982 +- fix datetime.isoformat() microseconds issues, #2994 +- recover_segment: use mmap(), lower memory needs, #2987 + +Other changes: + +- with-lock: close segment file before invoking subprocess +- keymanager: don't depend on optional readline module, #2976 +- docs: + + - fix macOS keychain integration command + - show/link new screencasts in README, #2936 + - document utf-8 locale requirement for json mode, #2273 +- vagrant: clean up shell profile init, user name, #2977 +- test_detect_attic_repo: don't test mount, #2975 +- add debug logging for repository cleanup + + +Version 1.1.0rc2 (2017-08-28) +----------------------------- + +Compatibility notes: + +- list: corrected mix-up of "isomtime" and "mtime" formats. Previously, + "isomtime" was the default but produced a verbose human format, + while "mtime" produced a ISO-8601-like format. + The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), + and the default is now "mtime". + "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). + +New features: + +- None. + +Fixes: + +- list: fix weird mixup of mtime/isomtime +- create --timestamp: set start time, #2957 +- ignore corrupt files cache, #2939 +- migrate locks to child PID when daemonize is used +- fix exitcode of borg serve, #2910 +- only compare contents when chunker params match, #2899 +- umount: try fusermount, then try umount, #2863 + +Other changes: + +- JSON: use a more standard ISO 8601 datetime format, #2376 +- cache: write_archive_index: truncate_and_unlink on error, #2628 +- detect non-upgraded Attic repositories, #1933 +- delete various nogil and threading related lines +- coala / pylint related improvements +- docs: + + - renew asciinema/screencasts, #669 + - create: document exclusion through nodump, #2949 + - minor formatting fixes + - tar: tarpipe example + - improve "with-lock" and "info" docs, #2869 + - detail how to use macOS/GNOME/KDE keyrings for repo passwords, #392 +- travis: only short-circuit docs-only changes for pull requests +- vagrant: + + - netbsd: bash is already installed + - fix netbsd version in PKG_PATH + - add exe location to PATH when we build an exe + + +Version 1.1.0rc1 (2017-07-24) +----------------------------- + +Compatibility notes: + +- delete: removed short option for --cache-only + +New features: + +- support borg list repo --format {comment} {bcomment} {end}, #2081 +- key import: allow reading from stdin, #2760 + +Fixes: + +- with-lock: avoid creating segment files that might be overwritten later, #1867 +- prune: fix checkpoints processing with --glob-archives +- FUSE: versions view: keep original file extension at end, #2769 +- fix --last, --first: do not accept values <= 0, + fix reversed archive ordering with --last +- include testsuite data (attic.tar.gz) when installing the package +- use limited unpacker for outer key, for manifest (both security precautions), + #2174 #2175 +- fix bashism in shell scripts, #2820, #2816 +- cleanup endianness detection, create _endian.h, + fixes build on alpine linux, #2809 +- fix crash with --no-cache-sync (give known chunk size to chunk_incref), #2853 + +Other changes: + +- FUSE: versions view: linear numbering by archive time +- split up interval parsing from filtering for --keep-within, #2610 +- add a basic .editorconfig, #2734 +- use archive creation time as mtime for FUSE mount, #2834 +- upgrade FUSE for macOS (osxfuse) from 3.5.8 to 3.6.3, #2706 +- hashindex: speed up by replacing modulo with "if" to check for wraparound +- coala checker / pylint: fixed requirements and .coafile, more ignores +- borg upgrade: name backup directories as 'before-upgrade', #2811 +- add .mailmap +- some minor changes suggested by lgtm.com +- docs: + + - better explanation of the --ignore-inode option relevance, #2800 + - fix openSUSE command and add openSUSE section + - simplify ssh authorized_keys file using "restrict", add legacy note, #2121 + - mount: show usage of archive filters + - mount: add repository example, #2462 + - info: update and add examples, #2765 + - prune: include example + - improved style / formatting + - improved/fixed segments_per_dir docs + - recreate: fix wrong "remove unwanted files" example + - reference list of status chars in borg recreate --filter description + - update source-install docs about doc build dependencies, #2795 + - cleanup installation docs + - file system requirements, update segs per dir + - fix checkpoints/parts reference in FAQ, #2859 +- code: + + - hashindex: don't pass side effect into macro + - crypto low_level: don't mutate local bytes() + - use dash_open function to open file or "-" for stdin/stdout + - archiver: argparse cleanup / refactoring + - shellpattern: add match_end arg +- tests: added some additional unit tests, some fixes, #2700 #2710 +- vagrant: fix setup of cygwin, add Debian 9 "stretch" +- travis: don't perform full travis build on docs-only changes, #2531 + + +Version 1.1.0b6 (2017-06-18) +---------------------------- + +Compatibility notes: + +- Running "borg init" via a "borg serve --append-only" server will *not* create + an append-only repository anymore. Use "borg init --append-only" to initialize + an append-only repository. + +- Repositories in the "repokey" and "repokey-blake2" modes with an empty passphrase + are now treated as unencrypted repositories for security checks (e.g. + BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK). + + Previously there would be no prompts nor messages if an unknown repository + in one of these modes with an empty passphrase was encountered. This would + allow an attacker to swap a repository, if one assumed that the lack of + password prompts was due to a set BORG_PASSPHRASE. + + Since the "trick" does not work if BORG_PASSPHRASE is set, this does generally + not affect scripts. + +- Repositories in the "authenticated" mode are now treated as the unencrypted + repositories they are. + +- The client-side temporary repository cache now holds unencrypted data for better speed. + +- borg init: removed the short form of --append-only (-a). + +- borg upgrade: removed the short form of --inplace (-i). + +New features: + +- reimplemented the RepositoryCache, size-limited caching of decrypted repo + contents, integrity checked via xxh64. #2515 +- reduced space usage of chunks.archive.d. Existing caches are migrated during + a cache sync. #235 #2638 +- integrity checking using xxh64 for important files used by borg, #1101: + + - repository: index and hints files + - cache: chunks and files caches, chunks.archive.d +- improve cache sync speed, #1729 +- create: new --no-cache-sync option +- add repository mandatory feature flags infrastructure, #1806 +- Verify most operations against SecurityManager. Location, manifest timestamp + and key types are now checked for almost all non-debug commands. #2487 +- implement storage quotas, #2517 +- serve: add --restrict-to-repository, #2589 +- BORG_PASSCOMMAND: use external tool providing the key passphrase, #2573 +- borg export-tar, #2519 +- list: --json-lines instead of --json for archive contents, #2439 +- add --debug-profile option (and also "borg debug convert-profile"), #2473 +- implement --glob-archives/-a, #2448 +- normalize authenticated key modes for better naming consistency: + + - rename "authenticated" to "authenticated-blake2" (uses blake2b) + - implement "authenticated" mode (uses hmac-sha256) + +Fixes: + +- hashindex: read/write indices >2 GiB on 32bit systems, better error + reporting, #2496 +- repository URLs: implement IPv6 address support and also more informative + error message when parsing fails. +- mount: check whether llfuse is installed before asking for passphrase, #2540 +- mount: do pre-mount checks before opening repository, #2541 +- FUSE: + + - fix crash if empty (None) xattr is read, #2534 + - fix read(2) caching data in metadata cache + - fix negative uid/gid crash (fix crash when mounting archives + of external drives made on cygwin), #2674 + - redo ItemCache, on top of object cache + - use decrypted cache + - remove unnecessary normpaths +- serve: ignore --append-only when initializing a repository (borg init), #2501 +- serve: fix incorrect type of exception_short for Errors, #2513 +- fix --exclude and --exclude-from recursing into directories, #2469 +- init: don't allow creating nested repositories, #2563 +- --json: fix encryption[mode] not being the cmdline name +- remote: propagate Error.traceback correctly +- fix remote logging and progress, #2241 + + - implement --debug-topic for remote servers + - remote: restore "Remote:" prefix (as used in 1.0.x) + - rpc negotiate: enable v3 log protocol only for supported clients + - fix --progress and logging in general for remote +- fix parse_version, add tests, #2556 +- repository: truncate segments (and also some other files) before unlinking, #2557 +- recreate: keep timestamps as in original archive, #2384 +- recreate: if single archive is not processed, exit 2 +- patterns: don't recurse with ! / --exclude for pf:, #2509 +- cache sync: fix n^2 behaviour in lookup_name +- extract: don't write to disk with --stdout (affected non-regular-file items), #2645 +- hashindex: implement KeyError, more tests + +Other changes: + +- remote: show path in PathNotAllowed +- consider repokey w/o passphrase == unencrypted, #2169 +- consider authenticated mode == unencrypted, #2503 +- restrict key file names, #2560 +- document follow_symlinks requirements, check libc, use stat and chown + with follow_symlinks=False, #2507 +- support common options on the main command, #2508 +- support common options on mid-level commands (e.g. borg *key* export) +- make --progress a common option +- increase DEFAULT_SEGMENTS_PER_DIR to 1000 +- chunker: fix invalid use of types (function only used by tests) +- chunker: don't do uint32_t >> 32 +- FUSE: + + - add instrumentation (--debug and SIGUSR1/SIGINFO) + - reduced memory usage for repository mounts by lazily instantiating archives + - improved archive load times +- info: use CacheSynchronizer & HashIndex.stats_against (better performance) +- docs: + + - init: document --encryption as required + - security: OpenSSL usage + - security: used implementations; note python libraries + - security: security track record of OpenSSL and msgpack + - patterns: document denial of service (regex, wildcards) + - init: note possible denial of service with "none" mode + - init: document SHA extension is supported in OpenSSL and thus SHA is + faster on AMD Ryzen than blake2b. + - book: use A4 format, new builder option format. + - book: create appendices + - data structures: explain repository compaction + - data structures: add chunk layout diagram + - data structures: integrity checking + - data structures: demingle cache and repo index + - Attic FAQ: separate section for attic stuff + - FAQ: I get an IntegrityError or similar - what now? + - FAQ: Can I use Borg on SMR hard drives?, #2252 + - FAQ: specify "using inline shell scripts" + - add systemd warning regarding placeholders, #2543 + - xattr: document API + - add docs/misc/borg-data-flow data flow chart + - debugging facilities + - README: how to help the project, #2550 + - README: add bountysource badge, #2558 + - fresh new theme + tweaking + - logo: vectorized (PDF and SVG) versions + - frontends: use headlines - you can link to them + - mark --pattern, --patterns-from as experimental + - highlight experimental features in online docs + - remove regex based pattern examples, #2458 + - nanorst for "borg help TOPIC" and --help + - split deployment + - deployment: hosting repositories + - deployment: automated backups to a local hard drive + - development: vagrant, windows10 requirements + - development: update docs remarks + - split usage docs, #2627 + - usage: avoid bash highlight, [options] instead of + - usage: add benchmark page + - helpers: truncate_and_unlink doc + - don't suggest to leak BORG_PASSPHRASE + - internals: columnize rather long ToC [webkit fixup] + internals: manifest & feature flags + - internals: more HashIndex details + - internals: fix ASCII art equations + - internals: edited obj graph related sections a bit + - internals: layers image + description + - fix way too small figures in pdf + - index: disable syntax highlight (bash) + - improve options formatting, fix accidental block quotes + +- testing / checking: + + - add support for using coala, #1366 + - testsuite: add ArchiverCorruptionTestCase + - do not test logger name, #2504 + - call setup_logging after destroying logging config + - testsuite.archiver: normalise pytest.raises vs. assert_raises + - add test for preserved intermediate folder permissions, #2477 + - key: add round-trip test + - remove attic dependency of the tests, #2505 + - enable remote tests on cygwin + - tests: suppress tar's future timestamp warning + - cache sync: add more refcount tests + - repository: add tests, including corruption tests + +- vagrant: + + - control VM cpus and pytest workers via env vars VMCPUS and XDISTN + - update cleaning workdir + - fix openbsd shell + - add OpenIndiana + +- packaging: + + - binaries: don't bundle libssl + - setup.py clean to remove compiled files + - fail in borg package if version metadata is very broken (setuptools_scm) + +- repo / code structure: + + - create borg.algorithms and borg.crypto packages + - algorithms: rename crc32 to checksums + - move patterns to module, #2469 + - gitignore: complete paths for src/ excludes + - cache: extract CacheConfig class + - implement IntegrityCheckedFile + Detached variant, #2502 #1688 + - introduce popen_with_error_handling to handle common user errors + + +Version 1.1.0b5 (2017-04-30) +---------------------------- + +Compatibility notes: + +- BORG_HOSTNAME_IS_UNIQUE is now on by default. +- removed --compression-from feature +- recreate: add --recompress flag, unify --always-recompress and + --recompress + +Fixes: + +- catch exception for os.link when hardlinks are not supported, #2405 +- borg rename / recreate: expand placeholders, #2386 +- generic support for hardlinks (files, devices, FIFOs), #2324 +- extract: also create parent dir for device files, if needed, #2358 +- extract: if a hardlink master is not in the to-be-extracted subset, + the "x" status was not displayed for it, #2351 +- embrace y2038 issue to support 32bit platforms: clamp timestamps to int32, + #2347 +- verify_data: fix IntegrityError handling for defect chunks, #2442 +- allow excluding parent and including child, #2314 + +Other changes: + +- refactor compression decision stuff +- change global compression default to lz4 as well, to be consistent + with --compression defaults. +- placeholders: deny access to internals and other unspecified stuff +- clearer error message for unrecognized placeholder +- more clear exception if borg check does not help, #2427 +- vagrant: upgrade FUSE for macOS to 3.5.8, #2346 +- linux binary builds: get rid of glibc 2.13 dependency, #2430 +- docs: + + - placeholders: document escaping + - serve: env vars in original commands are ignored + - tell what kind of hardlinks we support + - more docs about compression + - LICENSE: use canonical formulation + ("copyright holders and contributors" instead of "author") + - document borg init behaviour via append-only borg serve, #2440 + - be clear about what buzhash is used for, #2390 + - add hint about chunker params, #2421 + - clarify borg upgrade docs, #2436 + - FAQ to explain warning when running borg check --repair, #2341 + - repository file system requirements, #2080 + - pre-install considerations + - misc. formatting / crossref fixes +- tests: + + - enhance travis setuptools_scm situation + - add extra test for the hashindex + - fix invalid param issue in benchmarks + +These belong to 1.1.0b4 release, but did not make it into changelog by then: + +- vagrant: increase memory for parallel testing +- lz4 compress: lower max. buffer size, exception handling +- add docstring to do_benchmark_crud +- patterns help: mention path full-match in intro + + +Version 1.1.0b4 (2017-03-27) +---------------------------- + +Compatibility notes: + +- init: the --encryption argument is mandatory now (there are several choices) +- moved "borg migrate-to-repokey" to "borg key migrate-to-repokey". +- "borg change-passphrase" is deprecated, use "borg key change-passphrase" + instead. +- the --exclude-if-present option now supports tagging a folder with any + filesystem object type (file, folder, etc), instead of expecting only files + as tags, #1999 +- the --keep-tag-files option has been deprecated in favor of the new + --keep-exclude-tags, to account for the change mentioned above. +- use lz4 compression by default, #2179 + +New features: + +- JSON API to make developing frontends and automation easier + (see :ref:`json_output`) + + - add JSON output to commands: `borg create/list/info --json ...`. + - add --log-json option for structured logging output. + - add JSON progress information, JSON support for confirmations (yes()). +- add two new options --pattern and --patterns-from as discussed in #1406 +- new path full match pattern style (pf:) for very fast matching, #2334 +- add 'debug dump-manifest' and 'debug dump-archive' commands +- add 'borg benchmark crud' command, #1788 +- new 'borg delete --force --force' to delete severely corrupted archives, #1975 +- info: show utilization of maximum archive size, #1452 +- list: add dsize and dcsize keys, #2164 +- paperkey.html: Add interactive html template for printing key backups. +- key export: add qr html export mode +- securely erase config file (which might have old encryption key), #2257 +- archived file items: add size to metadata, 'borg extract' and 'borg check' do + check the file size for consistency, FUSE uses precomputed size from Item. + +Fixes: + +- fix remote speed regression introduced in 1.1.0b3, #2185 +- fix regression handling timestamps beyond 2262 (revert bigint removal), + introduced in 1.1.0b3, #2321 +- clamp (nano)second values to unproblematic range, #2304 +- hashindex: rebuild hashtable if we have too little empty buckets + (performance fix), #2246 +- Location regex: fix bad parsing of wrong syntax +- ignore posix_fadvise errors in repository.py, #2095 +- borg rpc: use limited msgpack.Unpacker (security precaution), #2139 +- Manifest: Make sure manifest timestamp is strictly monotonically increasing. +- create: handle BackupOSError on a per-path level in one spot +- create: clarify -x option / meaning of "same filesystem" +- create: don't create hard link refs to failed files +- archive check: detect and fix missing all-zero replacement chunks, #2180 +- files cache: update inode number when --ignore-inode is used, #2226 +- fix decompression exceptions crashing ``check --verify-data`` and others + instead of reporting integrity error, #2224 #2221 +- extract: warning for unextracted big extended attributes, #2258, #2161 +- mount: umount on SIGINT/^C when in foreground +- mount: handle invalid hard link refs +- mount: fix huge RAM consumption when mounting a repository (saves number of + archives * 8 MiB), #2308 +- hashindex: detect mingw byte order #2073 +- hashindex: fix wrong skip_hint on hashindex_set when encountering tombstones, + the regression was introduced in #1748 +- fix ChunkIndex.__contains__ assertion for big-endian archs +- fix borg key/debug/benchmark crashing without subcommand, #2240 +- Location: accept //servername/share/path +- correct/refactor calculation of unique/non-unique chunks +- extract: fix missing call to ProgressIndicator.finish +- prune: fix error msg, it is --keep-within, not --within +- fix "auto" compression mode bug (not compressing), #2331 +- fix symlink item fs size computation, #2344 + +Other changes: + +- remote repository: improved async exception processing, #2255 #2225 +- with --compression auto,C, only use C if lz4 achieves at least 3% compression +- PatternMatcher: only normalize path once, #2338 +- hashindex: separate endian-dependent defs from endian detection +- migrate-to-repokey: ask using canonical_path() as we do everywhere else. +- SyncFile: fix use of fd object after close +- make LoggedIO.close_segment reentrant +- creating a new segment: use "xb" mode, #2099 +- redo key_creator, key_factory, centralise key knowledge, #2272 +- add return code functions, #2199 +- list: only load cache if needed +- list: files->items, clarifications +- list: add "name" key for consistency with info cmd +- ArchiveFormatter: add "start" key for compatibility with "info" +- RemoteRepository: account rx/tx bytes +- setup.py build_usage/build_man/build_api fixes +- Manifest.in: simplify, exclude .so, .dll and .orig, #2066 +- FUSE: get rid of chunk accounting, st_blocks = ceil(size / blocksize). +- tests: + + - help python development by testing 3.6-dev + - test for borg delete --force +- vagrant: + + - freebsd: some fixes, #2067 + - darwin64: use osxfuse 3.5.4 for tests / to build binaries + - darwin64: improve VM settings + - use python 3.5.3 to build binaries, #2078 + - upgrade pyinstaller from 3.1.1+ to 3.2.1 + - pyinstaller: use fixed AND freshly compiled bootloader, #2002 + - pyinstaller: automatically builds bootloader if missing +- docs: + + - create really nice man pages + - faq: mention --remote-ratelimit in bandwidth limit question + - fix caskroom link, #2299 + - docs/security: reiterate that RPC in Borg does no networking + - docs/security: counter tracking, #2266 + - docs/development: update merge remarks + - address SSH batch mode in docs, #2202 #2270 + - add warning about running build_usage on Python >3.4, #2123 + - one link per distro in the installation page + - improve --exclude-if-present and --keep-exclude-tags, #2268 + - improve automated backup script in doc, #2214 + - improve remote-path description + - update docs for create -C default change (lz4) + - document relative path usage, #1868 + - document snapshot usage, #2178 + - corrected some stuff in internals+security + - internals: move toctree to after the introduction text + - clarify metadata kind, manifest ops + - key enc: correct / clarify some stuff, link to internals/security + - datas: enc: 1.1.x mas different MACs + - datas: enc: correct factual error -- no nonce involved there. + - make internals.rst an index page and edit it a bit + - add "Cryptography in Borg" and "Remote RPC protocol security" sections + - document BORG_HOSTNAME_IS_UNIQUE, #2087 + - FAQ by categories as proposed by @anarcat in #1802 + - FAQ: update Which file types, attributes, etc. are *not* preserved? + - development: new branching model for git repository + - development: define "ours" merge strategy for auto-generated files + - create: move --exclude note to main doc + - create: move item flags to main doc + - fix examples using borg init without -e/--encryption + - list: don't print key listings in fat (html + man) + - remove Python API docs (were very incomplete, build problems on RTFD) + - added FAQ section about backing up root partition + + +Version 1.1.0b3 (2017-01-15) +---------------------------- + +Compatibility notes: + +- borg init: removed the default of "--encryption/-e", #1979 + This was done so users do a informed decision about -e mode. + +Bug fixes: + +- borg recreate: don't rechunkify unless explicitly told so +- borg info: fixed bug when called without arguments, #1914 +- borg init: fix free space check crashing if disk is full, #1821 +- borg debug delete/get obj: fix wrong reference to exception +- fix processing of remote ~/ and ~user/ paths (regressed since 1.1.0b1), #1759 +- posix platform module: only build / import on non-win32 platforms, #2041 + +New features: + +- new CRC32 implementations that are much faster than the zlib one used previously, #1970 +- add blake2b key modes (use blake2b as MAC). This links against system libb2, + if possible, otherwise uses bundled code +- automatically remove stale locks - set BORG_HOSTNAME_IS_UNIQUE env var + to enable stale lock killing. If set, stale locks in both cache and + repository are deleted. #562 #1253 +- borg info : print general repo information, #1680 +- borg check --first / --last / --sort / --prefix, #1663 +- borg mount --first / --last / --sort / --prefix, #1542 +- implement "health" item formatter key, #1749 +- BORG_SECURITY_DIR to remember security related infos outside the cache. + Key type, location and manifest timestamp checks now survive cache + deletion. This also means that you can now delete your cache and avoid + previous warnings, since Borg can still tell it's safe. +- implement BORG_NEW_PASSPHRASE, #1768 + +Other changes: + +- borg recreate: + + - remove special-cased --dry-run + - update --help + - remove bloat: interruption blah, autocommit blah, resuming blah + - re-use existing checkpoint functionality + - archiver tests: add check_cache tool - lints refcounts + +- fixed cache sync performance regression from 1.1.0b1 onwards, #1940 +- syncing the cache without chunks.archive.d (see :ref:`disable_archive_chunks`) + now avoids any merges and is thus faster, #1940 +- borg check --verify-data: faster due to linear on-disk-order scan +- borg debug-xxx commands removed, we use "debug xxx" subcommands now, #1627 +- improve metadata handling speed +- shortcut hashindex_set by having hashindex_lookup hint about address +- improve / add progress displays, #1721 +- check for index vs. segment files object count mismatch +- make RPC protocol more extensible: use named parameters. +- RemoteRepository: misc. code cleanups / refactors +- clarify cache/repository README file + +- docs: + + - quickstart: add a comment about other (remote) filesystems + - quickstart: only give one possible ssh url syntax, all others are + documented in usage chapter. + - mention file:// + - document repo URLs / archive location + - clarify borg diff help, #980 + - deployment: synthesize alternative --restrict-to-path example + - improve cache / index docs, esp. files cache docs, #1825 + - document using "git merge 1.0-maint -s recursive -X rename-threshold=20%" + for avoiding troubles when merging the 1.0-maint branch into master. + +- tests: + + - FUSE tests: catch ENOTSUP on freebsd + - FUSE tests: test troublesome xattrs last + - fix byte range error in test, #1740 + - use monkeypatch to set env vars, but only on pytest based tests. + - point XDG_*_HOME to temp dirs for tests, #1714 + - remove all BORG_* env vars from the outer environment + + +Version 1.1.0b2 (2016-10-01) +---------------------------- + +Bug fixes: + +- fix incorrect preservation of delete tags, leading to "object count mismatch" + on borg check, #1598. This only occurred with 1.1.0b1 (not with 1.0.x) and is + normally fixed by running another borg create/delete/prune. +- fix broken --progress for double-cell paths (e.g. CJK), #1624 +- borg recreate: also catch SIGHUP +- FUSE: + + - fix hardlinks in versions view, #1599 + - add parameter check to ItemCache.get to make potential failures more clear + +New features: + +- Archiver, RemoteRepository: add --remote-ratelimit (send data) +- borg help compression, #1582 +- borg check: delete chunks with integrity errors, #1575, so they can be + "repaired" immediately and maybe healed later. +- archives filters concept (refactoring/unifying older code) + + - covers --first/--last/--prefix/--sort-by options + - currently used for borg list/info/delete + +Other changes: + +- borg check --verify-data slightly tuned (use get_many()) +- change {utcnow} and {now} to ISO-8601 format ("T" date/time separator) +- repo check: log transaction IDs, improve object count mismatch diagnostic +- Vagrantfile: use TW's fresh-bootloader pyinstaller branch +- fix module names in api.rst +- hashindex: bump api_version + + +Version 1.1.0b1 (2016-08-28) +---------------------------- + +New features: + +- new commands: + + - borg recreate: re-create existing archives, #787 #686 #630 #70, also see + #757, #770. + + - selectively remove files/dirs from old archives + - re-compress data + - re-chunkify data, e.g. to have upgraded Attic / Borg 0.xx archives + deduplicate with Borg 1.x archives or to experiment with chunker-params. + - borg diff: show differences between archives + - borg with-lock: execute a command with the repository locked, #990 +- borg create: + + - Flexible compression with pattern matching on path/filename, + and LZ4 heuristic for deciding compressibility, #810, #1007 + - visit files in inode order (better speed, esp. for large directories and rotating disks) + - in-file checkpoints, #1217 + - increased default checkpoint interval to 30 minutes (was 5 minutes), #896 + - added uuid archive format tag, #1151 + - save mountpoint directories with --one-file-system, makes system restore easier, #1033 + - Linux: added support for some BSD flags, #1050 + - add 'x' status for excluded paths, #814 + + - also means files excluded via UF_NODUMP, #1080 +- borg check: + + - will not produce the "Checking segments" output unless new --progress option is passed, #824. + - --verify-data to verify data cryptographically on the client, #975 +- borg list, #751, #1179 + + - removed {formatkeys}, see "borg list --help" + - --list-format is deprecated, use --format instead + - --format now also applies to listing archives, not only archive contents, #1179 + - now supports the usual [PATH [PATHS…]] syntax and excludes + - new keys: csize, num_chunks, unique_chunks, NUL + - supports guaranteed_available hashlib hashes + (to avoid varying functionality depending on environment), + which includes the SHA1 and SHA2 family as well as MD5 +- borg prune: + + - to visualize the "thinning out" better, we now list all archives in + reverse time order. rephrase and reorder help text. + - implement --keep-last N via --keep-secondly N, also --keep-minutely. + assuming that there is not more than 1 backup archive made in 1s, + --keep-last N and --keep-secondly N are equivalent, #537 + - cleanup checkpoints except the latest, #1008 +- borg extract: + + - added --progress, #1449 + - Linux: limited support for BSD flags, #1050 +- borg info: + + - output is now more similar to borg create --stats, #977 +- borg mount: + + - provide "borgfs" wrapper for borg mount, enables usage via fstab, #743 + - "versions" mount option - when used with a repository mount, this gives + a merged, versioned view of the files in all archives, #729 +- repository: + + - added progress information to commit/compaction phase (often takes some time when deleting/pruning), #1519 + - automatic recovery for some forms of repository inconsistency, #858 + - check free space before going forward with a commit, #1336 + - improved write performance (esp. for rotating media), #985 + + - new IO code for Linux + - raised default segment size to approx 512 MiB + - improved compaction performance, #1041 + - reduced client CPU load and improved performance for remote repositories, #940 + +- options that imply output (--show-rc, --show-version, --list, --stats, + --progress) don't need -v/--info to have that output displayed, #865 +- add archive comments (via borg (re)create --comment), #842 +- borg list/prune/delete: also output archive id, #731 +- --show-version: shows/logs the borg version, #725 +- added --debug-topic for granular debug logging, #1447 +- use atomic file writing/updating for configuration and key files, #1377 +- BORG_KEY_FILE environment variable, #1001 +- self-testing module, #970 + + +Bug fixes: + +- list: fixed default output being produced if --format is given with empty parameter, #1489 +- create: fixed overflowing progress line with CJK and similar characters, #1051 +- prune: fixed crash if --prefix resulted in no matches, #1029 +- init: clean up partial repo if passphrase input is aborted, #850 +- info: quote cmdline arguments that have spaces in them +- fix hardlinks failing in some cases for extracting subtrees, #761 + +Other changes: + +- replace stdlib hmac with OpenSSL, zero-copy decrypt (10-15% increase in + performance of hash-lists and extract). +- improved chunker performance, #1021 +- open repository segment files in exclusive mode (fail-safe), #1134 +- improved error logging, #1440 +- Source: + + - pass meta-data around, #765 + - move some constants to new constants module + - better readability and fewer errors with namedtuples, #823 + - moved source tree into src/ subdirectory, #1016 + - made borg.platform a package, #1113 + - removed dead crypto code, #1032 + - improved and ported parts of the test suite to py.test, #912 + - created data classes instead of passing dictionaries around, #981, #1158, #1161 + - cleaned up imports, #1112 +- Docs: + + - better help texts and sphinx reproduction of usage help: + + - Group options + - Nicer list of options in Sphinx + - Deduplicate 'Common options' (including --help) + - chunker: added some insights by "Voltara", #903 + - clarify what "deduplicated size" means + - fix / update / add package list entries + - added a SaltStack usage example, #956 + - expanded FAQ + - new contributors in AUTHORS! +- Tests: + + - vagrant: add ubuntu/xenial 64bit - this box has still some issues + - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 + + +Version 1.0.13 (2019-02-15) +--------------------------- + +Please note: this is very likely the last 1.0.x release, please upgrade to 1.1.x. + +Bug fixes: + +- security fix: configure FUSE with "default_permissions", #3903. + "default_permissions" is now enforced by borg by default to let the + kernel check uid/gid/mode based permissions. + "ignore_permissions" can be given not to enforce "default_permissions". +- xattrs: fix borg exception handling on ENOSPC error, #3808. + +New features: + +- Read a passphrase from a file descriptor specified in the + BORG_PASSPHRASE_FD environment variable. + +Other changes: + +- acl platform code: fix acl set return type +- xattr: + + - add linux {list,get,set}xattr ctypes prototypes + - fix darwin flistxattr ctypes prototype +- testing / travis-ci: + + - fix the homebrew 1.9 issues on travis-ci, #4254 + - travis OS X: use xcode 8.3 (not broken) + - tox.ini: lock requirements + - unbreak 1.0-maint on travis, fixes #4123 +- vagrant: + + - misc. fixes + - FUSE for macOS: upgrade 3.7.1 to 3.8.3 + - Python: upgrade 3.5.5 to 3.5.6 +- docs: + + - Update installation instructions for macOS + - update release workflow using twine (docs, scripts), #4213 + +Version 1.0.12 (2018-04-08) +--------------------------- + +Bug fixes: + +- repository: cleanup/write: invalidate cached FDs, tests +- serve: fix exitcode, #2910 +- extract: set bsdflags last (include immutable flag), #3263 +- create --timestamp: set start time, #2957 +- create: show excluded dir with "x" for tagged dirs / caches, #3189 +- migrate locks to child PID when daemonize is used +- Buffer: fix wrong thread-local storage use, #2951 +- fix detection of non-local path, #3108 +- fix LDLP restoration for subprocesses, #3077 +- fix subprocess environments (xattr module's fakeroot version check, + borg umount, BORG_PASSCOMMAND), #3050 +- remote: deal with partial lines, #2637 +- get rid of datetime.isoformat, use safe parse_timestamp to parse + timestamps, #2994 +- build: do .h file content checks in binary mode, fixes build issue for + non-ascii header files on pure-ascii locale platforms, #3544 #3639 +- remove platform.uname() call which caused library mismatch issues, #3732 +- add exception handler around deprecated platform.linux_distribution() call + +Other changes: + +- require msgpack-python >= 0.4.6 and < 0.5.0, see #3753 +- add parens for C preprocessor macro argument usages (did not cause + malfunction) +- ignore corrupt files cache, #2939 +- replace "modulo" with "if" to check for wraparound in hashmap +- keymanager: don't depend on optional readline module, #2980 +- exclude broken pytest 3.3.0 release +- exclude broken Cython 0.27(.0) release, #3066 +- flake8: add some ignores +- docs: + + - create: document exclusion through nodump + - document good and problematic option placements, fix examples, #3356 + - update docs about hardlinked symlinks limitation + - faq: we do not implement futile attempts of ETA / progress displays + - simplified rate limiting wrapper in FAQ + - twitter account @borgbackup, #2948 + - add note about metadata dedup and --no[ac]time, #2518 + - change-passphrase only changes the passphrase, #2990 + - clarify encrypted key format for borg key export, #3296 + - document sshfs rename workaround, #3315 + - update release checklist about security fixes + - docs about how to verify a signed release, #3634 + - chunk seed is generated per /repository/ +- vagrant: + + - use FUSE for macOS 3.7.1 to build the macOS binary + - use python 3.5.5 to build the binaries + - add exe location to PATH when we build an exe + - use https pypi url for wheezy + - netbsd: bash is already installed + - netbsd: fix netbsd version in PKG_PATH + - use self-made FreeBSD 10.3 box, #3022 + - backport fs_init (including related updates) from 1.1 + - the boxcutter wheezy boxes are 404, use local ones +- travis: + + - don't perform full Travis build on docs-only changes, #2531 + - only short-circuit docs-only changes for pull requests + + +Version 1.0.11 (2017-07-21) +--------------------------- + +Bug fixes: + +- use limited unpacker for outer key (security precaution), #2174 +- fix paperkey import bug + +Other changes: + +- change --checkpoint-interval default from 600s to 1800s, #2841. + this improves efficiency for big repositories a lot. +- docs: fix OpenSUSE command and add OpenSUSE section +- tests: add tests for split_lstring and paperkey +- vagrant: + + - fix openbsd shell + - backport cpu/ram setup from master + - add stretch64 VM + +Version 1.0.11rc1 (2017-06-27) +------------------------------ + +Bug fixes: + +- performance: rebuild hashtable if we have too few empty buckets, #2246. + this fixes some sporadic, but severe performance breakdowns. +- Archive: allocate zeros when needed, #2308 + fixes huge memory usage of mount (8 MiB × number of archives) +- IPv6 address support + also: Location: more informative exception when parsing fails +- borg single-file binary: use pyinstaller v3.2.1, #2396 + this fixes that the prelink cronjob on some distros kills the + borg binary by stripping away parts of it. +- extract: + + - warning for unextracted big extended attributes, #2258 + - also create parent dir for device files, if needed. + - don't write to disk with --stdout, #2645 +- archive check: detect and fix missing all-zero replacement chunks, #2180 +- fix (de)compression exceptions, #2224 #2221 +- files cache: update inode number, #2226 +- borg rpc: use limited msgpack.Unpacker (security precaution), #2139 +- Manifest: use limited msgpack.Unpacker (security precaution), #2175 +- Location: accept //servername/share/path +- fix ChunkIndex.__contains__ assertion for big-endian archs (harmless) +- create: handle BackupOSError on a per-path level in one spot +- fix error msg, there is no --keep-last in borg 1.0.x, #2282 +- clamp (nano)second values to unproblematic range, #2304 +- fuse / borg mount: + + - fix st_blocks to be an integer (not float) value + - fix negative uid/gid crash (they could come into archives e.g. when + backing up external drives under cygwin), #2674 + - fix crash if empty (None) xattr is read + - do pre-mount checks before opening repository + - check llfuse is installed before asking for passphrase +- borg rename: expand placeholders, #2386 +- borg serve: fix forced command lines containing BORG_* env vars +- fix error msg, it is --keep-within, not --within +- fix borg key/debug/benchmark crashing without subcommand, #2240 +- chunker: fix invalid use of types, don't do uint32_t >> 32 +- document follow_symlinks requirements, check libc, #2507 + +New features: + +- added BORG_PASSCOMMAND environment variable, #2573 +- add minimal version of in repository mandatory feature flags, #2134 + + This should allow us to make sure older borg versions can be cleanly + prevented from doing operations that are no longer safe because of + repository format evolution. This allows more fine grained control than + just incrementing the manifest version. So for example a change that + still allows new archives to be created but would corrupt the repository + when an old version tries to delete an archive or check the repository + would add the new feature to the check and delete set but leave it out + of the write set. +- borg delete --force --force to delete severely corrupted archives, #1975 + +Other changes: + +- embrace y2038 issue to support 32bit platforms +- be more clear that this is a "beyond repair" case, #2427 +- key file names: limit to 100 characters and remove colons from host name +- upgrade FUSE for macOS to 3.5.8, #2346 +- split up parsing and filtering for --keep-within, better error message, #2610 +- docs: + + - fix caskroom link, #2299 + - address SSH batch mode, #2202 #2270 + - improve remote-path description + - document snapshot usage, #2178 + - document relative path usage, #1868 + - one link per distro in the installation page + - development: new branching model in git repository + - kill api page + - added FAQ section about backing up root partition + - add bountysource badge, #2558 + - create empty docs.txt requirements, #2694 + - README: how to help the project + - note -v/--verbose requirement on affected options, #2542 + - document borg init behaviour via append-only borg serve, #2440 + - be clear about what buzhash is used for (chunking) and want it is not + used for (deduplication)- also say already in the readme that we use a + cryptohash for dedupe, so people don't worry, #2390 + - add hint about chunker params to borg upgrade docs, #2421 + - clarify borg upgrade docs, #2436 + - quickstart: delete problematic BORG_PASSPHRASE use, #2623 + - faq: specify "using inline shell scripts" + - document pattern denial of service, #2624 +- tests: + + - remove attic dependency of the tests, #2505 + - travis: + + - enhance travis setuptools_scm situation + - install fakeroot for Linux + - add test for borg delete --force + - enable remote tests on cygwin (the cygwin issue that caused these tests + to break was fixed in cygwin at least since cygwin 2.8, maybe even since + 2.7.0). + - remove skipping the noatime tests on GNU/Hurd, #2710 + - fix borg import issue, add comment, #2718 + - include attic.tar.gz when installing the package + also: add include_package_data=True + +Version 1.0.10 (2017-02-13) +--------------------------- + +Bug fixes: + +- Manifest timestamps are now monotonically increasing, + this fixes issues when the system clock jumps backwards + or is set inconsistently across computers accessing the same repository, #2115 +- Fixed testing regression in 1.0.10rc1 that lead to a hard dependency on + py.test >= 3.0, #2112 + +New features: + +- "key export" can now generate a printable HTML page with both a QR code and + a human-readable "paperkey" representation (and custom text) through the + ``--qr-html`` option. + + The same functionality is also available through `paperkey.html `_, + which is the same HTML page generated by ``--qr-html``. It works with existing + "key export" files and key files. + +Other changes: + +- docs: + + - language clarification - "borg create --one-file-system" option does not respect + mount points, but considers different file systems instead, #2141 +- setup.py: build_api: sort file list for determinism + + +Version 1.0.10rc1 (2017-01-29) +------------------------------ + +Bug fixes: + +- borg serve: fix transmission data loss of pipe writes, #1268 + This affects only the cygwin platform (not Linux, BSD, OS X). +- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992 +- When running out of buffer memory when reading xattrs, only skip the + current file, #1993 +- Fixed "borg upgrade --tam" crashing with unencrypted repositories. Since + :ref:`the issue ` is not relevant for unencrypted repositories, + it now does nothing and prints an error, #1981. +- Fixed change-passphrase crashing with unencrypted repositories, #1978 +- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997 +- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled +- fix bad parsing of wrong repo location syntax +- create: don't create hard link refs to failed files, + mount: handle invalid hard link refs, #2092 +- detect mingw byte order, #2073 +- creating a new segment: use "xb" mode, #2099 +- mount: umount on SIGINT/^C when in foreground, #2082 + +Other changes: + +- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002 +- xattr: ignore empty names returned by llistxattr(2) et al +- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT, + SIGBUS and SIGILL signals to dump the Python traceback. +- Also print a traceback on SIGUSR2. +- borg change-passphrase: print key location (simplify making a backup of it) +- officially support Python 3.6 (setup.py: add Python 3.6 qualifier) +- tests: + + - vagrant / travis / tox: add Python 3.6 based testing + - vagrant: fix openbsd repo, #2042 + - vagrant: fix the freebsd64 machine, #2037 #2067 + - vagrant: use python 3.5.3 to build binaries, #2078 + - vagrant: use osxfuse 3.5.4 for tests / to build binaries + vagrant: improve darwin64 VM settings + - travis: fix osxfuse install (fixes OS X testing on Travis CI) + - travis: require succeeding OS X tests, #2028 + - travis: use latest pythons for OS X based testing + - use pytest-xdist to parallelize testing + - fix xattr test race condition, #2047 + - setup.cfg: fix pytest deprecation warning, #2050 +- docs: + + - language clarification - VM backup FAQ + - borg create: document how to back up stdin, #2013 + - borg upgrade: fix incorrect title levels + - add CVE numbers for issues fixed in 1.0.9, #2106 +- fix typos (taken from Debian package patch) +- remote: include data hexdump in "unexpected RPC data" error message +- remote: log SSH command line at debug level +- API_VERSION: use numberspaces, #2023 +- remove .github from pypi package, #2051 +- add pip and setuptools to requirements file, #2030 +- SyncFile: fix use of fd object after close (cosmetic) +- Manifest.in: simplify, exclude \*.{so,dll,orig}, #2066 +- ignore posix_fadvise errors in repository.py, #2095 + (works around issues with docker on ARM) +- make LoggedIO.close_segment reentrant, avoid reentrance + + +Version 1.0.9 (2016-12-20) +-------------------------- + +Security fixes: + +- A flaw in the cryptographic authentication scheme in Borg allowed an attacker + to spoof the manifest. See :ref:`tam_vuln` above for the steps you should + take. + + CVE-2016-10099 was assigned to this vulnerability. +- borg check: When rebuilding the manifest (which should only be needed very rarely) + duplicate archive names would be handled on a "first come first serve" basis, + potentially opening an attack vector to replace archives. + + Example: were there 2 archives named "foo" in a repo (which can not happen + under normal circumstances, because borg checks if the name is already used) + and a "borg check" recreated a (previously lost) manifest, the first of the + archives it encountered would be in the manifest. The second archive is also + still in the repo, but not referenced in the manifest, in this case. If the + second archive is the "correct" one (and was previously referenced from the + manifest), it looks like it got replaced by the first one. In the manifest, + it actually got replaced. Both remain in the repo but the "correct" one is no + longer accessible via normal means - the manifest. + + CVE-2016-10100 was assigned to this vulnerability. + +Bug fixes: + +- borg check: + + - rebuild manifest if it's corrupted + - skip corrupted chunks during manifest rebuild +- fix TypeError in integrity error handler, #1903, #1894 +- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 +- fix wrong duration/timestamps if system clock jumped during a create +- fix progress display not updating if system clock jumps backwards +- fix checkpoint interval being incorrect if system clock jumps + +Other changes: + +- docs: + + - add python3-devel as a dependency for cygwin-based installation + - clarify extract is relative to current directory + - FAQ: fix link to changelog + - markup fixes +- tests: + + - test_get\_(cache|keys)_dir: clean env state, #1897 + - get back pytest's pretty assertion failures, #1938 +- setup.py build_usage: + + - fixed build_usage not processing all commands + - fixed build_usage not generating includes for debug commands + + +Version 1.0.9rc1 (2016-11-27) +----------------------------- + +Bug fixes: + +- files cache: fix determination of newest mtime in backup set (which is + used in cache cleanup and led to wrong "A" [added] status for unchanged + files in next backup), #1860. + +- borg check: + + - fix incorrectly reporting attic 0.13 and earlier archives as corrupt + - handle repo w/o objects gracefully and also bail out early if repo is + *completely* empty, #1815. +- fix tox/pybuild in 1.0-maint +- at xattr module import time, loggers are not initialized yet + +New features: + +- borg umount + exposed already existing umount code via the CLI api, so users can use it, + which is more consistent than using borg to mount and fusermount -u (or + umount) to un-mount, #1855. +- implement borg create --noatime --noctime, fixes #1853 + +Other changes: + +- docs: + + - display README correctly on PyPI + - improve cache / index docs, esp. files cache docs, fixes #1825 + - different pattern matching for --exclude, #1779 + - datetime formatting examples for {now} placeholder, #1822 + - clarify passphrase mode attic repo upgrade, #1854 + - clarify --umask usage, #1859 + - clarify how to choose PR target branch + - clarify prune behavior for different archive contents, #1824 + - fix PDF issues, add logo, fix authors, headings, TOC + - move security verification to support section + - fix links in standalone README (:ref: tags) + - add link to security contact in README + - add FAQ about security + - move fork differences to FAQ + - add more details about resource usage +- tests: skip remote tests on cygwin, #1268 +- travis: + + - allow OS X failures until the brew cask osxfuse issue is fixed + - caskroom osxfuse-beta gone, it's osxfuse now (3.5.3) +- vagrant: + + - upgrade OSXfuse / FUSE for macOS to 3.5.3 + - remove llfuse from tox.ini at a central place + - do not try to install llfuse on centos6 + - fix FUSE test for darwin, #1546 + - add windows virtual machine with cygwin + - Vagrantfile cleanup / code deduplication + + +Version 1.0.8 (2016-10-29) +-------------------------- + +Bug fixes: + +- RemoteRepository: Fix busy wait in call_many, #940 + +New features: + +- implement borgmajor/borgminor/borgpatch placeholders, #1694 + {borgversion} was already there (full version string). With the new + placeholders you can now also get e.g. 1 or 1.0 or 1.0.8. + +Other changes: + +- avoid previous_location mismatch, #1741 + + due to the changed canonicalization for relative paths in PR #1711 / #1655 + (implement /./ relpath hack), there would be a changed repo location warning + and the user would be asked if this is ok. this would break automation and + require manual intervention, which is unwanted. + + thus, we automatically fix the previous_location config entry, if it only + changed in the expected way, but still means the same location. + +- docs: + + - deployment.rst: do not use bare variables in ansible snippet + - add clarification about append-only mode, #1689 + - setup.py: add comment about requiring llfuse, #1726 + - update usage.rst / api.rst + - repo url / archive location docs + typo fix + - quickstart: add a comment about other (remote) filesystems + +- vagrant / tests: + + - no chown when rsyncing (fixes boxes w/o vagrant group) + - fix FUSE permission issues on linux/freebsd, #1544 + - skip FUSE test for borg binary + fakeroot + - ignore security.selinux xattrs, fixes tests on centos, #1735 + + +Version 1.0.8rc1 (2016-10-17) +----------------------------- + +Bug fixes: + +- fix signal handling (SIGINT, SIGTERM, SIGHUP), #1620 #1593 + Fixes e.g. leftover lock files for quickly repeated signals (e.g. Ctrl-C + Ctrl-C) or lost connections or systemd sending SIGHUP. +- progress display: adapt formatting to narrow screens, do not crash, #1628 +- borg create --read-special - fix crash on broken symlink, #1584. + also correctly processes broken symlinks. before this regressed to a crash + (5b45385) a broken symlink would've been skipped. +- process_symlink: fix missing backup_io() + Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting + dirents and dispatching to process_symlink. +- yes(): abort on wrong answers, saying so, #1622 +- fixed exception borg serve raised when connection was closed before repository + was opened. Add an error message for this. +- fix read-from-closed-FD issue, #1551 + (this seems not to get triggered in 1.0.x, but was discovered in master) +- hashindex: fix iterators (always raise StopIteration when exhausted) + (this seems not to get triggered in 1.0.x, but was discovered in master) +- enable relative paths in ssh:// repo URLs, via /./relpath hack, #1655 +- allow repo paths with colons, #1705 +- update changed repo location immediately after acceptance, #1524 +- fix debug get-obj / delete-obj crash if object not found and remote repo, + #1684 +- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib + on Mac OS X (avoids mismatch lib <-> driver), #1619 + +New features: + +- add "borg key export" / "borg key import" commands, #1555, so users are able + to back up / restore their encryption keys more easily. + + Supported formats are the keyfile format used by borg internally and a + special "paper" format with by line checksums for printed backups. For the + paper format, the import is an interactive process which checks each line as + soon as it is input. +- add "borg debug-refcount-obj" to determine a repo objects' referrer counts, + #1352 + +Other changes: + +- add "borg debug ..." subcommands + (borg debug-* still works, but will be removed in borg 1.1) +- setup.py: Add subcommand support to build_usage. +- remote: change exception message for unexpected RPC data format to indicate + dataflow direction. +- improved messages / error reporting: + + - IntegrityError: add placeholder for message, so that the message we give + appears not only in the traceback, but also in the (short) error message, + #1572 + - borg.key: include chunk id in exception msgs, #1571 + - better messages for cache newer than repo, #1700 +- vagrant (testing/build VMs): + + - upgrade OSXfuse / FUSE for macOS to 3.5.2 + - update Debian Wheezy boxes, #1686 + - openbsd / netbsd: use own boxes, fixes misc rsync installation and + FUSE/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728 +- docs: + + - add docs for "key export" and "key import" commands, #1641 + - fix inconsistency in FAQ (pv-wrapper). + - fix second block in "Easy to use" section not showing on GitHub, #1576 + - add bestpractices badge + - link reference docs and faq about BORG_FILES_CACHE_TTL, #1561 + - improve borg info --help, explain size infos, #1532 + - add release signing key / security contact to README, #1560 + - add contribution guidelines for developers + - development.rst: add sphinx_rtd_theme to the sphinx install command + - adjust border color in borg.css + - add debug-info usage help file + - internals.rst: fix typos + - setup.py: fix build_usage to always process all commands + - added docs explaining multiple --restrict-to-path flags, #1602 + - add more specific warning about write-access debug commands, #1587 + - clarify FAQ regarding backup of virtual machines, #1672 +- tests: + + - work around FUSE xattr test issue with recent fakeroot + - simplify repo/hashindex tests + - travis: test FUSE-enabled borg, use trusty to have a recent FUSE + - re-enable FUSE tests for RemoteArchiver (no deadlocks any more) + - clean env for pytest based tests, #1714 + - fuse_mount contextmanager: accept any options + + +Version 1.0.7 (2016-08-19) +-------------------------- + +Security fixes: + +- borg serve: fix security issue with remote repository access, #1428 + If you used e.g. --restrict-to-path /path/client1/ (with or without trailing + slash does not make a difference), it acted like a path prefix match using + /path/client1 (note the missing trailing slash) - the code then also allowed + working in e.g. /path/client13 or /path/client1000. + + As this could accidentally lead to major security/privacy issues depending on + the paths you use, the behaviour was changed to be a strict directory match. + That means --restrict-to-path /path/client1 (with or without trailing slash + does not make a difference) now uses /path/client1/ internally (note the + trailing slash here!) for matching and allows precisely that path AND any + path below it. So, /path/client1 is allowed, /path/client1/repo1 is allowed, + but not /path/client13 or /path/client1000. + + If you willingly used the undocumented (dangerous) previous behaviour, you + may need to rearrange your --restrict-to-path paths now. We are sorry if + that causes work for you, but we did not want a potentially dangerous + behaviour in the software (not even using a for-backwards-compat option). + +Bug fixes: + +- fixed repeated LockTimeout exceptions when borg serve tried to write into + a already write-locked repo (e.g. by a borg mount), #502 part b) + This was solved by the fix for #1220 in 1.0.7rc1 already. +- fix cosmetics + file leftover for "not a valid borg repository", #1490 +- Cache: release lock if cache is invalid, #1501 +- borg extract --strip-components: fix leak of preloaded chunk contents +- Repository, when a InvalidRepository exception happens: + + - fix spurious, empty lock.roster + - fix repo not closed cleanly + +New features: + +- implement borg debug-info, fixes #1122 + (just calls already existing code via cli, same output as below tracebacks) + +Other changes: + +- skip the O_NOATIME test on GNU Hurd, fixes #1315 + (this is a very minor issue and the GNU Hurd project knows the bug) +- document using a clean repo to test / build the release + + +Version 1.0.7rc2 (2016-08-13) +----------------------------- + +Bug fixes: + +- do not write objects to repository that are bigger than the allowed size, + borg will reject reading them, #1451. + + Important: if you created archives with many millions of files or + directories, please verify if you can open them successfully, + e.g. try a "borg list REPO::ARCHIVE". +- lz4 compression: dynamically enlarge the (de)compression buffer, the static + buffer was not big enough for archives with extremely many items, #1453 +- larger item metadata stream chunks, raise archive item limit by 8x, #1452 +- fix untracked segments made by moved DELETEs, #1442 + + Impact: Previously (metadata) segments could become untracked when deleting data, + these would never be cleaned up. +- extended attributes (xattrs) related fixes: + + - fixed a race condition in xattrs querying that led to the entire file not + being backed up (while logging the error, exit code = 1), #1469 + - fixed a race condition in xattrs querying that led to a crash, #1462 + - raise OSError including the error message derived from errno, deal with + path being a integer FD + +Other changes: + +- print active env var override by default, #1467 +- xattr module: refactor code, deduplicate, clean up +- repository: split object size check into too small and too big +- add a transaction_id assertion, so borg init on a broken (inconsistent) + filesystem does not look like a coding error in borg, but points to the + real problem. +- explain confusing TypeError caused by compat support for old servers, #1456 +- add forgotten usage help file from build_usage +- refactor/unify buffer code into helpers.Buffer class, add tests +- docs: + + - document archive limitation, #1452 + - improve prune examples + + +Version 1.0.7rc1 (2016-08-05) +----------------------------- + +Bug fixes: + +- fix repo lock deadlocks (related to lock upgrade), #1220 +- catch unpacker exceptions, resync, #1351 +- fix borg break-lock ignoring BORG_REPO env var, #1324 +- files cache performance fixes (fixes unnecessary re-reading/chunking/ + hashing of unmodified files for some use cases): + + - fix unintended file cache eviction, #1430 + - implement BORG_FILES_CACHE_TTL, update FAQ, raise default TTL from 10 + to 20, #1338 +- FUSE: + + - cache partially read data chunks (performance), #965, #966 + - always create a root dir, #1125 +- use an OrderedDict for helptext, making the build reproducible, #1346 +- RemoteRepository init: always call close on exceptions, #1370 (cosmetic) +- ignore stdout/stderr broken pipe errors (cosmetic), #1116 + +New features: + +- better borg versions management support (useful esp. for borg servers + wanting to offer multiple borg versions and for clients wanting to choose + a specific server borg version), #1392: + + - add BORG_VERSION environment variable before executing "borg serve" via ssh + - add new placeholder {borgversion} + - substitute placeholders in --remote-path + +- borg init --append-only option (makes using the more secure append-only mode + more convenient. when used remotely, this requires 1.0.7+ also on the borg + server), #1291. + +Other changes: + +- Vagrantfile: + + - darwin64: upgrade to FUSE for macOS 3.4.1 (aka osxfuse), #1378 + - xenial64: use user "ubuntu", not "vagrant" (as usual), #1331 +- tests: + + - fix FUSE tests on OS X, #1433 +- docs: + + - FAQ: add backup using stable filesystem names recommendation + - FAQ about glibc compatibility added, #491, glibc-check improved + - FAQ: 'A' unchanged file; remove ambiguous entry age sentence. + - OS X: install pkg-config to build with FUSE support, fixes #1400 + - add notes about shell/sudo pitfalls with env. vars, #1380 + - added platform feature matrix +- implement borg debug-dump-repo-objs + + +Version 1.0.6 (2016-07-12) +-------------------------- + +Bug fixes: + +- Linux: handle multiple LD_PRELOAD entries correctly, #1314, #1111 +- Fix crash with unclear message if the libc is not found, #1314, #1111 + +Other changes: + +- tests: + + - Fixed O_NOATIME tests for Solaris and GNU Hurd, #1315 + - Fixed sparse file tests for (file) systems not supporting it, #1310 +- docs: + + - Fixed syntax highlighting, #1313 + - misc docs: added data processing overview picture + + +Version 1.0.6rc1 (2016-07-10) +----------------------------- + +New features: + +- borg check --repair: heal damaged files if missing chunks re-appear (e.g. if + the previously missing chunk was added again in a later backup archive), + #148. (*) Also improved logging. + +Bug fixes: + +- sync_dir: silence fsync() failing with EINVAL, #1287 + Some network filesystems (like smbfs) don't support this and we use this in + repository code. +- borg mount (FUSE): + + - fix directories being shadowed when contained paths were also specified, + #1295 + - raise I/O Error (EIO) on damaged files (unless -o allow_damaged_files is + used), #1302. (*) +- borg extract: warn if a damaged file is extracted, #1299. (*) +- Added some missing return code checks (ChunkIndex._add, hashindex_resize). +- borg check: fix/optimize initial hash table size, avoids resize of the table. + +Other changes: + +- tests: + + - add more FUSE tests, #1284 + - deduplicate FUSE (u)mount code + - fix borg binary test issues, #862 +- docs: + + - changelog: added release dates to older borg releases + - fix some sphinx (docs generator) warnings, #881 + +Notes: + +(*) Some features depend on information (chunks_healthy list) added to item +metadata when a file with missing chunks was "repaired" using all-zero +replacement chunks. The chunks_healthy list is generated since borg 1.0.4, +thus borg can't recognize such "repaired" (but content-damaged) files if the +repair was done with an older borg version. + + +Version 1.0.5 (2016-07-07) +-------------------------- + +Bug fixes: + +- borg mount: fix FUSE crash in xattr code on Linux introduced in 1.0.4, #1282 + +Other changes: + +- backport some FAQ entries from master branch +- add release helper scripts +- Vagrantfile: + + - centos6: no FUSE, don't build binary + - add xz for redhat-like dists + + +Version 1.0.4 (2016-07-07) +-------------------------- + +New features: + +- borg serve --append-only, #1168 + This was included because it was a simple change (append-only functionality + was already present via repository config file) and makes better security now + practically usable. +- BORG_REMOTE_PATH environment variable, #1258 + This was included because it was a simple change (--remote-path cli option + was already present) and makes borg much easier to use if you need it. +- Repository: cleanup incomplete transaction on "no space left" condition. + In many cases, this can avoid a 100% full repo filesystem (which is very + problematic as borg always needs free space - even to delete archives). + +Bug fixes: + +- Fix wrong handling and reporting of OSErrors in borg create, #1138. + This was a serious issue: in the context of "borg create", errors like + repository I/O errors (e.g. disk I/O errors, ssh repo connection errors) + were handled badly and did not lead to a crash (which would be good for this + case, because the repo transaction would be incomplete and trigger a + transaction rollback to clean up). + Now, error handling for source files is cleanly separated from every other + error handling, so only problematic input files are logged and skipped. +- Implement fail-safe error handling for borg extract. + Note that this isn't nearly as critical as the borg create error handling + bug, since nothing is written to the repo. So this was "merely" misleading + error reporting. +- Add missing error handler in directory attr restore loop. +- repo: make sure write data hits disk before the commit tag (#1236) and also + sync the containing directory. +- FUSE: getxattr fail must use errno.ENOATTR, #1126 + (fixes Mac OS X Finder malfunction: "zero bytes" file length, access denied) +- borg check --repair: do not lose information about the good/original chunks. + If we do not lose the original chunk IDs list when "repairing" a file + (replacing missing chunks with all-zero chunks), we have a chance to "heal" + the file back into its original state later, in case the chunks re-appear + (e.g. in a fresh backup). Healing is not implemented yet, see #148. +- fixes for --read-special mode: + + - ignore known files cache, #1241 + - fake regular file mode, #1214 + - improve symlinks handling, #1215 +- remove passphrase from subprocess environment, #1105 +- Ignore empty index file (will trigger index rebuild), #1195 +- add missing placeholder support for --prefix, #1027 +- improve exception handling for placeholder replacement +- catch and format exceptions in arg parsing +- helpers: fix "undefined name 'e'" in exception handler +- better error handling for missing repo manifest, #1043 +- borg delete: + + - make it possible to delete a repo without manifest + - borg delete --forced allows one to delete corrupted archives, #1139 +- borg check: + + - make borg check work for empty repo + - fix resync and msgpacked item qualifier, #1135 + - rebuild_manifest: fix crash if 'name' or 'time' key were missing. + - better validation of item metadata dicts, #1130 + - better validation of archive metadata dicts +- close the repo on exit - even if rollback did not work, #1197. + This is rather cosmetic, it avoids repo closing in the destructor. + +- tests: + + - fix sparse file test, #1170 + - flake8: ignore new F405, #1185 + - catch "invalid argument" on cygwin, #257 + - fix sparseness assertion in test prep, #1264 + +Other changes: + +- make borg build/work on OpenSSL 1.0 and 1.1, #1187 +- docs / help: + + - fix / clarify prune help, #1143 + - fix "patterns" help formatting + - add missing docs / help about placeholders + - resources: rename atticmatic to borgmatic + - document sshd settings, #545 + - more details about checkpoints, add split trick, #1171 + - support docs: add freenode web chat link, #1175 + - add prune visualization / example, #723 + - add note that Fnmatch is default, #1247 + - make clear that lzma levels > 6 are a waste of cpu cycles + - add a "do not edit" note to auto-generated files, #1250 + - update cygwin installation docs +- repository interoperability with borg master (1.1dev) branch: + + - borg check: read item metadata keys from manifest, #1147 + - read v2 hints files, #1235 + - fix hints file "unknown version" error handling bug +- tests: add tests for format_line +- llfuse: update version requirement for freebsd +- Vagrantfile: + + - use openbsd 5.9, #716 + - do not install llfuse on netbsd (broken) + - update OSXfuse to version 3.3.3 + - use Python 3.5.2 to build the binaries +- glibc compatibility checker: scripts/glibc_check.py +- add .eggs to .gitignore + + +Version 1.0.3 (2016-05-20) +-------------------------- + +Bug fixes: + +- prune: avoid that checkpoints are kept and completed archives are deleted in + a prune run), #997 +- prune: fix commandline argument validation - some valid command lines were + considered invalid (annoying, but harmless), #942 +- fix capabilities extraction on Linux (set xattrs last, after chown()), #1069 +- repository: fix commit tags being seen in data +- when probing key files, do binary reads. avoids crash when non-borg binary + files are located in borg's key files directory. +- handle SIGTERM and make a clean exit - avoids orphan lock files. +- repository cache: don't cache large objects (avoid using lots of temp. disk + space), #1063 + +Other changes: + +- Vagrantfile: OS X: update osxfuse / install lzma package, #933 +- setup.py: add check for platform_darwin.c +- setup.py: on freebsd, use a llfuse release that builds ok +- docs / help: + + - update readthedocs URLs, #991 + - add missing docs for "borg break-lock", #992 + - borg create help: add some words to about the archive name + - borg create help: document format tags, #894 + + +Version 1.0.2 (2016-04-16) +-------------------------- + +Bug fixes: + +- fix malfunction and potential corruption on (nowadays rather rare) big-endian + architectures or bi-endian archs in (rare) BE mode. #886, #889 + + cache resync / index merge was malfunctioning due to this, potentially + leading to data loss. borg info had cosmetic issues (displayed wrong values). + + note: all (widespread) little-endian archs (like x86/x64) or bi-endian archs + in (widespread) LE mode (like ARMEL, MIPSEL, ...) were NOT affected. +- add overflow and range checks for 1st (special) uint32 of the hashindex + values, switch from int32 to uint32. +- fix so that refcount will never overflow, but just stick to max. value after + a overflow would have occurred. +- borg delete: fix --cache-only for broken caches, #874 + + Makes --cache-only idempotent: it won't fail if the cache is already deleted. +- fixed borg create --one-file-system erroneously traversing into other + filesystems (if starting fs device number was 0), #873 +- workaround a bug in Linux fadvise FADV_DONTNEED, #907 + +Other changes: + +- better test coverage for hashindex, incl. overflow testing, checking correct + computations so endianness issues would be discovered. +- reproducible doc for ProgressIndicator*, make the build reproducible. +- use latest llfuse for vagrant machines +- docs: + + - use /path/to/repo in examples, fixes #901 + - fix confusing usage of "repo" as archive name (use "arch") + + +Version 1.0.1 (2016-04-08) +-------------------------- + +New features: + +Usually there are no new features in a bugfix release, but these were added +due to their high impact on security/safety/speed or because they are fixes +also: + +- append-only mode for repositories, #809, #36 (see docs) +- borg create: add --ignore-inode option to make borg detect unmodified files + even if your filesystem does not have stable inode numbers (like sshfs and + possibly CIFS). +- add options --warning, --error, --critical for missing log levels, #826. + it's not recommended to suppress warnings or errors, but the user may decide + this on his own. + note: --warning is not given to borg serve so a <= 1.0.0 borg will still + work as server (it is not needed as it is the default). + do not use --error or --critical when using a <= 1.0.0 borg server. + +Bug fixes: + +- fix silently skipping EIO, #748 +- add context manager for Repository (avoid orphan repository locks), #285 +- do not sleep for >60s while waiting for lock, #773 +- unpack file stats before passing to FUSE +- fix build on illumos +- don't try to back up doors or event ports (Solaris and derivatives) +- remove useless/misleading libc version display, #738 +- test suite: reset exit code of persistent archiver, #844 +- RemoteRepository: clean up pipe if remote open() fails +- Remote: don't print tracebacks for Error exceptions handled downstream, #792 +- if BORG_PASSPHRASE is present but wrong, don't prompt for password, but fail + instead, #791 +- ArchiveChecker: move "orphaned objects check skipped" to INFO log level, #826 +- fix capitalization, add ellipses, change log level to debug for 2 messages, + #798 + +Other changes: + +- update llfuse requirement, llfuse 1.0 works +- update OS / dist packages on build machines, #717 +- prefer showing --info over -v in usage help, #859 +- docs: + + - fix cygwin requirements (gcc-g++) + - document how to debug / file filesystem issues, #664 + - fix reproducible build of api docs + - RTD theme: CSS !important overwrite, #727 + - Document logo font. Recreate logo png. Remove GIMP logo file. + + +Version 1.0.0 (2016-03-05) +-------------------------- + +The major release number change (0.x -> 1.x) indicates bigger incompatible +changes, please read the compatibility notes, adapt / test your scripts and +check your backup logs. + +Compatibility notes: + +- drop support for python 3.2 and 3.3, require 3.4 or 3.5, #221 #65 #490 + note: we provide binaries that include python 3.5.1 and everything else + needed. they are an option in case you are stuck with < 3.4 otherwise. +- change encryption to be on by default (using "repokey" mode) +- moved keyfile keys from ~/.borg/keys to ~/.config/borg/keys, + you can either move them manually or run "borg upgrade " +- remove support for --encryption=passphrase, + use borg migrate-to-repokey to switch to repokey mode, #97 +- remove deprecated --compression , + use --compression zlib, instead + in case of 0, you could also use --compression none +- remove deprecated --hourly/daily/weekly/monthly/yearly + use --keep-hourly/daily/weekly/monthly/yearly instead +- remove deprecated --do-not-cross-mountpoints, + use --one-file-system instead +- disambiguate -p option, #563: + + - -p now is same as --progress + - -P now is same as --prefix +- remove deprecated "borg verify", + use "borg extract --dry-run" instead +- cleanup environment variable semantics, #355 + the environment variables used to be "yes sayers" when set, this was + conceptually generalized to "automatic answerers" and they just give their + value as answer (as if you typed in that value when being asked). + See the "usage" / "Environment Variables" section of the docs for details. +- change the builtin default for --chunker-params, create 2MiB chunks, #343 + --chunker-params new default: 19,23,21,4095 - old default: 10,23,16,4095 + + one of the biggest issues with borg < 1.0 (and also attic) was that it had a + default target chunk size of 64kiB, thus it created a lot of chunks and thus + also a huge chunk management overhead (high RAM and disk usage). + + please note that the new default won't change the chunks that you already + have in your repository. the new big chunks do not deduplicate with the old + small chunks, so expect your repo to grow at least by the size of every + changed file and in the worst case (e.g. if your files cache was lost / is + not used) by the size of every file (minus any compression you might use). + + in case you want to see a much lower resource usage immediately (RAM / disk) + for chunks management, it might be better to start with a new repo than + to continue in the existing repo (with an existing repo, you have to wait + until all archives with small chunks get pruned to see a lower resource + usage). + + if you used the old --chunker-params default value (or if you did not use + --chunker-params option at all) and you'd like to continue using small + chunks (and you accept the huge resource usage that comes with that), just + use explicitly borg create --chunker-params=10,23,16,4095. +- archive timestamps: the 'time' timestamp now refers to archive creation + start time (was: end time), the new 'time_end' timestamp refers to archive + creation end time. This might affect prune if your backups take a long time. + if you give a timestamp via cli, this is stored into 'time'. therefore it now + needs to mean archive creation start time. + +New features: + +- implement password roundtrip, #695 + +Bug fixes: + +- remote end does not need cache nor keys directories, do not create them, #701 +- added retry counter for passwords, #703 + +Other changes: + +- fix compiler warnings, #697 +- docs: + + - update README.rst to new changelog location in docs/changes.rst + - add Teemu to AUTHORS + - changes.rst: fix old chunker params, #698 + - FAQ: how to limit bandwidth + + +Version 1.0.0rc2 (2016-02-28) +----------------------------- + +New features: + +- format options for location: user, pid, fqdn, hostname, now, utcnow, user +- borg list --list-format +- borg prune -v --list enables the keep/prune list output, #658 + +Bug fixes: + +- fix _open_rb noatime handling, #657 +- add a simple archivename validator, #680 +- borg create --stats: show timestamps in localtime, use same labels/formatting + as borg info, #651 +- llfuse compatibility fixes (now compatible with: 0.40, 0.41, 0.42) + +Other changes: + +- it is now possible to use "pip install borgbackup[fuse]" to + install the llfuse dependency automatically, using the correct version requirement + for it. you still need to care about having installed the FUSE / build + related OS package first, though, so that building llfuse can succeed. +- Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4 +- Vagrant: use pyinstaller v3.1.1 to build binaries +- docs: + + - borg upgrade: add to docs that only LOCAL repos are supported + - borg upgrade also handles borg 0.xx -> 1.0 + - use pip extras or requirements file to install llfuse + - fix order in release process + - updated usage docs and other minor / cosmetic fixes + - verified borg examples in docs, #644 + - freebsd dependency installation and FUSE configuration, #649 + - add example how to restore a raw device, #671 + - add a hint about the dev headers needed when installing from source + - add examples for delete (and handle delete after list, before prune), #656 + - update example for borg create -v --stats (use iso datetime format), #663 + - added example to BORG_RSH docs + - "connection closed by remote": add FAQ entry and point to issue #636 + + +Version 1.0.0rc1 (2016-02-07) +----------------------------- + +New features: + +- borg migrate-to-repokey ("passphrase" -> "repokey" encryption key mode) +- implement --short for borg list REPO, #611 +- implement --list for borg extract (consistency with borg create) +- borg serve: overwrite client's --restrict-to-path with ssh forced command's + option value (but keep everything else from the client commandline), #544 +- use $XDG_CONFIG_HOME/keys for keyfile keys (~/.config/borg/keys), #515 +- "borg upgrade" moves the keyfile keys to the new location +- display both archive creation start and end time in "borg info", #627 + + +Bug fixes: + +- normalize trailing slashes for the repository path, #606 +- Cache: fix exception handling in __init__, release lock, #610 + +Other changes: + +- suppress unneeded exception context (PEP 409), simpler tracebacks +- removed special code needed to deal with imperfections / incompatibilities / + missing stuff in py 3.2/3.3, simplify code that can be done simpler in 3.4 +- removed some version requirements that were kept on old versions because + newer did not support py 3.2 any more +- use some py 3.4+ stdlib code instead of own/openssl/pypi code: + + - use os.urandom instead of own cython openssl RAND_bytes wrapper, #493 + - use hashlib.pbkdf2_hmac from py stdlib instead of own openssl wrapper + - use hmac.compare_digest instead of == operator (constant time comparison) + - use stat.filemode instead of homegrown code + - use "mock" library from stdlib, #145 + - remove borg.support (with non-broken argparse copy), it is ok in 3.4+, #358 +- Vagrant: copy CHANGES.rst as symlink, #592 +- cosmetic code cleanups, add flake8 to tox/travis, #4 +- docs / help: + + - make "borg -h" output prettier, #591 + - slightly rephrase prune help + - add missing example for --list option of borg create + - quote exclude line that includes an asterisk to prevent shell expansion + - fix dead link to license + - delete Ubuntu Vivid, it is not supported anymore (EOL) + - OS X binary does not work for older OS X releases, #629 + - borg serve's special support for forced/original ssh commands, #544 + - misc. updates and fixes diff --git a/docs/conf.py b/docs/conf.py index 76e0ad8d6e..4e5a6e17f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,85 +12,87 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -import sys, os -sys.path.insert(0, os.path.abspath('../src')) +import sys +import os + +sys.path.insert(0, os.path.abspath("../src")) from borg import __version__ as sw_version # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Borg - Deduplicating Archiver' -copyright = u'2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)' +project = "Borg - Deduplicating Archiver" +copyright = "2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -split_char = '+' if '+' in sw_version else '-' +split_char = "+" if "+" in sw_version else "-" version = sw_version.split(split_char)[0] # The full version, including alpha/beta/rc tags. release = version -suppress_warnings = ['image.nonlocal_uri'] +suppress_warnings = ["image.nonlocal_uri"] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -today_fmt = '%Y-%m-%d' +today_fmt = "%Y-%m-%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # The Borg docs contain no or very little Python docs. # Thus, the primary domain is rst. -primary_domain = 'rst' +primary_domain = "rst" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -100,78 +102,73 @@ import guzzle_sphinx_theme html_theme_path = guzzle_sphinx_theme.html_theme_path() -html_theme = 'guzzle_sphinx_theme' +html_theme = "guzzle_sphinx_theme" def set_rst_settings(app): - app.env.settings.update({ - 'field_name_limit': 0, - 'option_limit': 0, - }) + app.env.settings.update({"field_name_limit": 0, "option_limit": 0}) def setup(app): - app.add_css_file('css/borg.css') - app.connect('builder-inited', set_rst_settings) + app.add_css_file("css/borg.css") + app.connect("builder-inited", set_rst_settings) + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { - 'project_nav_name': 'Borg %s' % version, -} +html_theme_options = {"project_nav_name": "Borg %s" % version} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = ['_themes'] +# html_theme_path = ['_themes'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/logo.svg' +html_logo = "_static/logo.svg" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = '_static/favicon.ico' +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['borg_theme'] +html_static_path = ["borg_theme"] -html_extra_path = ['../src/borg/paperkey.html'] +html_extra_path = ["../src/borg/paperkey.html"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d' +html_last_updated_fmt = "%Y-%m-%d" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. html_use_smartypants = True +smartquotes_action = "qe" # no D in there means "do not transform -- and ---" # Custom sidebar templates, maps document names to template names. -html_sidebars = { - '**': ['logo-text.html', 'searchbox.html', 'globaltoc.html'], -} +html_sidebars = {"**": ["logo-text.html", "searchbox.html", "globaltoc.html"]} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False @@ -185,57 +182,45 @@ def setup(app): # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'borgdoc' +htmlhelp_basename = "borgdoc" # -- Options for LaTeX output -------------------------------------------------- # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('book', 'Borg.tex', 'Borg Documentation', - 'The Borg Collective', 'manual'), -] +latex_documents = [("book", "Borg.tex", "Borg Documentation", "The Borg Collective", "manual")] # The name of an image file (relative to this directory) to place at the top of # the title page. -latex_logo = '_static/logo.pdf' +latex_logo = "_static/logo.pdf" -latex_elements = { - 'papersize': 'a4paper', - 'pointsize': '10pt', - 'figure_align': 'H', -} +latex_elements = {"papersize": "a4paper", "pointsize": "10pt", "figure_align": "H"} # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -latex_show_urls = 'footnote' +latex_show_urls = "footnote" # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -latex_appendices = [ - 'support', - 'resources', - 'changes', - 'authors', -] +latex_appendices = ["support", "resources", "changes", "authors"] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -243,21 +228,24 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('usage', 'borg', - 'BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.', - ['The Borg Collective (see AUTHORS file)'], - 1), + ( + "usage", + "borg", + "BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.", + ["The Borg Collective (see AUTHORS file)"], + 1, + ) ] extensions = [ - 'sphinx.ext.extlinks', - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", ] extlinks = { - 'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'), - 'targz_url': ('https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz' % version, None), + "issue": ("https://github.com/borgbackup/borg/issues/%s", "#%s"), + "targz_url": ("https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz" % version, None), } diff --git a/docs/deployment.rst b/docs/deployment.rst index 3d4605b2bb..edb7849bac 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -14,3 +14,4 @@ This chapter details deployment strategies for the following scenarios. deployment/automated-local deployment/image-backup deployment/pull-backup + deployment/non-root-user diff --git a/docs/deployment/automated-local.rst b/docs/deployment/automated-local.rst index b22ce60ab9..d34a70a7f6 100644 --- a/docs/deployment/automated-local.rst +++ b/docs/deployment/automated-local.rst @@ -29,26 +29,12 @@ Configuring the system First, create the ``/etc/backups`` directory (as root). All configuration goes into this directory. -Then, create ``/etc/backups/40-backup.rules`` with the following content (all on one line):: - - ACTION=="add", SUBSYSTEM=="bdi", DEVPATH=="/devices/virtual/bdi/*", - TAG+="systemd", ENV{SYSTEMD_WANTS}="automatic-backup.service" - -.. topic:: Finding a more precise udev rule +Find out the ID of the partition table of your backup disk (here assumed to be /dev/sdz): + lsblk --fs -o +PTUUID /dev/sdz - If you always connect the drive(s) to the same physical hardware path, e.g. the same - eSATA port, then you can make a more precise udev rule. - - Execute ``udevadm monitor`` and connect a drive to the port you intend to use. - You should see a flurry of events, find those regarding the `block` subsystem. - Pick the event whose device path ends in something similar to a device file name, - typically`sdX/sdXY`. Use the event's device path and replace `sdX/sdXY` after the - `/block/` part in the path with a star (\*). For example: - `DEVPATH=="/devices/pci0000:00/0000:00:11.0/ata3/host2/target2:0:0/2:0:0:0/block/*"`. +Then, create ``/etc/backups/40-backup.rules`` with the following content (all on one line):: - Reboot a few times to ensure that the hardware path does not change: on some motherboards - components of it can be random. In these cases you cannot use a more accurate rule, - or need to insert additional stars for matching the path. + ACTION=="add", SUBSYSTEM=="block", ENV{ID_PART_TABLE_UUID}=="", TAG+="systemd", ENV{SYSTEMD_WANTS}="automatic-backup.service" The "systemd" tag in conjunction with the SYSTEMD_WANTS environment variable has systemd launch the "automatic-backup" service, which we will create next, as the @@ -110,7 +96,7 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.). # Mount file system if not already done. This assumes that if something is already # mounted at $MOUNTPOINT, it is the backup drive. It won't find the drive if # it was mounted somewhere else. - (mount | grep $MOUNTPOINT) || mount $partition_path $MOUNTPOINT + findmnt $MOUNTPOINT >/dev/null || mount $partition_path $MOUNTPOINT drive=$(lsblk --inverse --noheadings --list --paths --output name $partition_path | head --lines 1) echo "Drive path: $drive" @@ -136,8 +122,8 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.). # This is just an example, change it however you see fit borg create $BORG_OPTS \ - --exclude /root/.cache \ - --exclude /var/lib/docker/devicemapper \ + --exclude root/.cache \ + --exclude var/lib/docker/devicemapper \ $TARGET::$DATE-$$-system \ / /boot @@ -145,7 +131,7 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.). # Even if it isn't (add --exclude /home above), it probably makes sense # to have /home in a separate archive. borg create $BORG_OPTS \ - --exclude 'sh:/home/*/.cache' \ + --exclude 'sh:home/*/.cache' \ $TARGET::$DATE-$$-home \ /home/ @@ -164,17 +150,17 @@ modify it to suit your needs (e.g. more backup sets, dumping databases etc.). fi Create the ``/etc/backups/autoeject`` file to have the script automatically eject the drive -after creating the backup. Rename the file to something else (e.g. ``/etc/backup/autoeject-no``) +after creating the backup. Rename the file to something else (e.g. ``/etc/backups/autoeject-no``) when you want to do something with the drive after creating backups (e.g running check). Create the ``/etc/backups/backup-suspend`` file if the machine should suspend after completing -the backup. Don't forget to physically disconnect the device before resuming, +the backup. Don't forget to disconnect the device physically before resuming, otherwise you'll enter a cycle. You can also add an option to power down instead. Create an empty ``/etc/backups/backup.disks`` file, you'll register your backup drives there. -The last part is to actually enable the udev rules and services: +The last part is actually to enable the udev rules and services: .. code-block:: bash @@ -191,7 +177,7 @@ Find the UUID of the file system that backups should be stored on:: lsblk -o+uuid,label -Note the UUID into the ``/etc/backup/backup.disks`` file. +Note the UUID into the ``/etc/backups/backup.disks`` file. Mount the drive to /mnt/backup. @@ -212,7 +198,7 @@ Security considerations ----------------------- The script as shown above will mount any file system with an UUID listed in -``/etc/backup/backup.disks``. The UUID check is a safety / annoyance-reduction +``/etc/backups/backup.disks``. The UUID check is a safety / annoyance-reduction mechanism to keep the script from blowing up whenever a random USB thumb drive is connected. It is not meant as a security mechanism. Mounting file systems and reading repository data exposes additional attack surfaces (kernel file system drivers, diff --git a/docs/deployment/central-backup-server.rst b/docs/deployment/central-backup-server.rst index 431bd531e9..a63618f9a6 100644 --- a/docs/deployment/central-backup-server.rst +++ b/docs/deployment/central-backup-server.rst @@ -1,10 +1,11 @@ .. include:: ../global.rst.inc .. highlight:: none +.. _central-backup-server: Central repository server with Ansible or Salt ============================================== -This section will give an example how to setup a borg repository server for multiple +This section will give an example how to set up a borg repository server for multiple clients. Machines @@ -95,7 +96,7 @@ The client needs to initialize the `pictures` repository like this: borg init backup@backup01.srv.local:pictures -Or with the full path (should actually never be used, as only for demonstrational purposes). +Or with the full path (should actually never be used, as only for demonstration purposes). The server should automatically change the current working directory to the `` folder. :: @@ -103,7 +104,7 @@ The server should automatically change the current working directory to the ``_ within a systemd +service. + +Linux capabilities allow us to give parts of the privileges the root user has to +a non-root user. This works on a per-thread level and does not give the permission +to the non-root user as a whole. + +For this we need to run our backup script from a systemd service and use the `AmbientCapabilities +`_ +option added in systemd 229. + +A very basic unit file would look like this: + +:: + + [Unit] + Description=Borg Backup + + [Service] + Type=oneshot + User=borg + ExecStart=/usr/local/sbin/backup.sh + + AmbientCapabilities=CAP_DAC_READ_SEARCH + +The ``CAP_DAC_READ_SEARCH`` capability gives borg read-only access to all files and directories on the system. + +This service can then be started manually using ``systemctl start``, a systemd timer or other methods. + +Restore considerations +====================== + +When restoring files, the root user should be used. When using the non-root user, borg extract will +change all files to be owned by the non-root user. Using borg mount will not allow the non-root user +to access files that it would not have access to on the system itself. + +Other than that, the same restore process, that would be used when running the backup as root, can be used. + +.. warning:: + + When using a local repo and running borg commands as root, make sure to only use commands that do not + modify the repo itself, like extract or mount. Modifying the repo using the root user will break + the repo for the non-root user, since some files inside the repo will now be owned by root. diff --git a/docs/deployment/pull-backup.rst b/docs/deployment/pull-backup.rst index f516e0ae3c..5532e2fcac 100644 --- a/docs/deployment/pull-backup.rst +++ b/docs/deployment/pull-backup.rst @@ -13,7 +13,7 @@ If you however require the backup server to initiate the connection or prefer it to initiate the backup run, one of the following workarounds is required to allow such a pull mode setup. -A common use case for pull mode is to backup a remote server to a local personal +A common use case for pull mode is to back up a remote server to a local personal computer. SSHFS @@ -58,7 +58,7 @@ completely in every aspect from such a backup. mappings, assuming they only come from files (/etc/passwd and group). This assumption might be wrong, e.g. if users/groups also come from ldap or other providers. - Thus, it might be better to use ``--numeric-owner`` and not archive any + Thus, it might be better to use ``--numeric-ids`` and not archive any user or group names (but just the numeric IDs) and not use chroot. Creating a backup @@ -98,9 +98,9 @@ create the backup, retaining the original paths, excluding the repository: :: - borg create --exclude /borgrepo --files-cache ctime,size /borgrepo::archive / + borg create --exclude borgrepo --files-cache ctime,size /borgrepo::archive / -For the sake of simplicity only ``/borgrepo`` is excluded here. You may want to +For the sake of simplicity only ``borgrepo`` is excluded here. You may want to set up an exclude file with additional files and folders to be excluded. Also note that we have to modify Borg's file change detection behaviour – SSHFS cannot guarantee stable inode numbers, so we have to supply the @@ -161,7 +161,7 @@ Now we can run borg extract /borgrepo::archive PATH -to partially restore whatever we like. Finally, do the clean-up: +to restore whatever we like partially. Finally, do the clean-up: :: @@ -181,13 +181,13 @@ When doing a full restore, we restore all files (including the ones containing the ID-to-name mapping, ``/etc/passwd`` and ``/etc/group``). Everything will be consistent automatically if we restore the numeric IDs stored in the archive. So there is no need for a chroot environment; we just mount the client file system -and extract a backup, utilizing the ``--numeric-owner`` option: +and extract a backup, utilizing the ``--numeric-ids`` option: :: sshfs root@host:/ /mnt/sshfs cd /mnt/sshfs - borg extract --numeric-owner /path/to/repo::archive + borg extract --numeric-ids /path/to/repo::archive cd ~ umount /mnt/sshfs @@ -209,8 +209,8 @@ socat ===== In this setup a SSH connection from the backup server to the client is -established that uses SSH reverse port forwarding to transparently -tunnel data between UNIX domain sockets on the client and server and the socat +established that uses SSH reverse port forwarding to tunnel data +transparently between UNIX domain sockets on the client and server and the socat tool to connect these with the borg client and server processes, respectively. The program socat has to be available on the backup server and on the client @@ -277,7 +277,7 @@ forwarding can do this for us:: Warning: remote port forwarding failed for listen path /run/borg/reponame.sock - When you are done, you have to manually remove the socket file, otherwise + When you are done, you have to remove the socket file manually, otherwise you may see an error like this when trying to execute borg commands:: Remote: YYYY/MM/DD HH:MM:SS socat[XXX] E connect(5, AF=1 "/run/borg/reponame.sock", 13): Connection refused @@ -417,8 +417,88 @@ Parentheses are not needed when using a dedicated bash process. *ssh://borgs@borg-server/~/repo* refers to the repository *repo* within borgs's home directory on *borg-server*. - *StrictHostKeyChecking=no* is used to automatically add host keys to *~/.ssh/known_hosts* without user intervention. + *StrictHostKeyChecking=no* is used to add host keys automatically to *~/.ssh/known_hosts* without user intervention. ``kill "${SSH_AGENT_PID}"`` Kill ssh-agent with loaded keys when it is not needed anymore. + +Remote forwarding +================= + +The standard ssh client allows to create tunnels to forward local ports to a remote server (local forwarding) and also +to allow remote ports to be forwarded to local ports (remote forwarding). + +This remote forwarding can be used to allow remote backup clients to access the backup server even if the backup server +cannot be reached by the backup client. + +This can even be used in cases where neither the backup server can reach the backup client and the backup client cannot +reach the backup server, but some intermediate host can access both. + +A schematic approach is as follows + +:: + + Backup Server (backup@mybackup) Intermediate Machine (john@myinter) Backup Client (bob@myclient) + + 1. Establish SSH remote forwarding -----------> SSH listen on local port + + 2. Starting ``borg create`` establishes + 3. SSH forwards to intermediate machine <------- SSH connection to the local port + 4. Receives backup connection <------- and further on to backup server + via SSH + +So for the backup client the backup is done via SSH to a local port and for the backup server there is a normal backup +performed via ssh. + +In order to achieve this, the following commands can be used to create the remote port forwarding: + +1. On machine ``myinter`` + +``ssh bob@myclient -v -C -R 8022:mybackup:22 -N`` + +This will listen for ssh-connections on port ``8022`` on ``myclient`` and forward connections to port 22 on ``mybackup``. + +You can also remove the need for machine ``myinter`` and create the port forwarding on the backup server directly by +using ``localhost`` instead of ``mybackup`` + +2. On machine ``myclient`` + +``borg create -v --progress --stats ssh://backup@localhost:8022/home/backup/repos/myclient /`` + +Make sure to use port ``8022`` and ``localhost`` for the repository as this instructs borg on ``myclient`` to use the +remote forwarded ssh connection. + +SSH Keys +-------- + +If you want to automate backups when using this method, the ssh ``known_hosts`` and ``authorized_keys`` need to be set up +to allow connections. + +Security Considerations +----------------------- + +Opening up SSH access this way can pose a security risk as it effectively opens remote access to your +backup server on the client even if it is located outside of your company network. + +To reduce the chances of compromise, you should configure a forced command in ``authorized_keys`` to prevent +anyone from performing any other action on the backup server. + +This can be done e.g. by adding the following in ``$HOME/.ssh/authorized_keys`` on ``mybackup`` with proper +path and client-fqdn: + +:: + + command="cd /home/backup/repos/;borg serve --restrict-to-path /home/backup/repos/" + + +All the additional security considerations for borg should be applied, see :ref:`central-backup-server` for some additional +hints. + +More information +---------------- + +See `remote forwarding`_ and the `ssh man page`_ for more information about remote forwarding. + + .. _remote forwarding: https://linuxize.com/post/how-to-setup-ssh-tunneling/ + .. _ssh man page: https://manpages.debian.org/testing/manpages-de/ssh.1.de.html diff --git a/docs/development.rst b/docs/development.rst index d54c6c4e90..8c389262bf 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -53,7 +53,7 @@ requests (if you don't have GitHub or don't want to use it you can send smaller patches via the borgbackup mailing list to the maintainers). Stable releases are maintained on maintenance branches named ``x.y-maint``, eg. -the maintenance branch of the 1.0.x series is ``1.0-maint``. +the maintenance branch of the 1.2.x series is ``1.2-maint``. Most PRs should be filed against the ``master`` branch. Only if an issue affects **only** a particular maintenance branch a PR should be @@ -110,6 +110,100 @@ most minor changes and fixes where committed to a maintenance branch back into the main development branch. This became more and more troublesome due to merges growing more conflict-heavy and error-prone. +How to submit a pull request +---------------------------- + +In order to contribute to Borg, you will need to fork the ``borgbackup/borg`` +main repository to your own Github repository. Then clone your Github repository +to your local machine. The instructions for forking and cloning a repository +can be found there: +``_ . + +To work on your contribution, you first need to decide which branch your pull +request should be against. Often, this might be master branch (esp. for big / +risky contributions), but it could be also a maintenance branch like e.g. +1.4-maint (esp. for small fixes that should go into next maintenance release, +e.g. 1.4.x). + +Start by checking out the appropriate branch: +:: + + git checkout master + +It is best practice for a developer to keep local ``master`` branch as an +uptodate copy of the upstream ``master`` branch and always do own work in a +separate feature or bugfix branch. +This is useful to be able to rebase own branches onto the upstream branches +they were branched from, if necessary. + +This also applies to other upstream branches (like e.g. ``1.4-maint``), not +only to ``master``. + +Thus, create a new branch now: +:: + + git checkout -b MYCONTRIB-master # choose an appropriate own branch name + +Now, work on your contribution in that branch. Use these git commands: +:: + + git status # is there anything that needs to be added? + git add ... # if so, add it + git commit # finally, commit it. use a descriptive comment. + +Then push the changes to your Github repository: +:: + + git push --set-upstream origin MYCONTRIB-master + +Finally, make a pull request on ``borgbackup/borg`` Github repository against +the appropriate branch (e.g. ``master``) so that your changes can be reviewed. + +What to do if work was accidentally started in wrong branch +----------------------------------------------------------- + +If you accidentally worked in ``master`` branch, check out the ``master`` +branch and make sure there are no uncommitted changes. Then, create a feature +branch from that, so that your contribution is in a feature branch. +:: + + git checkout master + git checkout -b MYCONTRIB-master + +Next, check out the ``master`` branch again. Find the commit hash of the last +commit that was made before you started working on your contribution and perform +a hard reset. +:: + + git checkout master + git log + git reset --hard THATHASH + +Then, update the local ``master`` branch with changes made in the upstream +repository. +:: + + git pull borg master + +Rebase feature branch onto updated master branch +------------------------------------------------ + +After updating the local ``master`` branch from upstream, the feature branch +can be checked out and rebased onto (the now uptodate) ``master`` branch. +:: + + git checkout MYCONTRIB-master + git rebase -i master + +Next, check if there are any commits that exist in the feature branch +but not in the ``master`` branch and vice versa. If there are no +conflicts or after resolving them, push your changes to your Github repository. +:: + + git log + git diff master + git push -f + Code and issues --------------- @@ -119,24 +213,36 @@ Code is stored on GitHub, in the `Borgbackup organization `_ should be sent there as well. See also the :ref:`support` section for more details. -Style guide ------------ +Style guide / Automated Code Formatting +--------------------------------------- + +We use `black`_ for automatically formatting the code. + +If you work on the code, it is recommended that you run black **before each commit** +(so that new code is always using the desired formatting and no additional commits +are required to fix the formatting). + +:: + + pip install -r requirements.d/codestyle.txt # everybody use same black version + black --check . # only check, don't change + black . # reformat the code -We generally follow `pep8 -`_, with 120 columns -instead of 79. We do *not* use form-feed (``^L``) characters to -separate sections either. Compliance is tested automatically when -you run the tests. + +The CI workflows will check the code formatting and will fail if it is not formatted correctly. + +When (mass-)reformatting existing code, we need to avoid ruining `git blame`, so please +follow their `guide about avoiding ruining git blame`_: + +.. _black: https://black.readthedocs.io/ +.. _guide about avoiding ruining git blame: https://black.readthedocs.io/en/stable/guides/introducing_black_to_your_project.html#avoiding-ruining-git-blame Continuous Integration ---------------------- -All pull requests go through `GitHub Actions`_, which runs the tests on Linux -and Mac OS X as well as the flake8 style checker. Windows builds run on AppVeyor_, -while additional Unix-like platforms are tested on Golem_. +All pull requests go through `GitHub Actions`_, which runs the tests on misc. +Python versions and on misc. platforms as well as some additional checks. -.. _AppVeyor: https://ci.appveyor.com/project/borgbackup/borg/ -.. _Golem: https://golem.enkore.de/view/Borg/ .. _GitHub Actions: https://github.com/borgbackup/borg/actions Output and Logging @@ -164,6 +270,13 @@ virtual env and run:: pip install -r requirements.d/development.txt +This project utilizes pre-commit to format and lint code before it is committed. +Although pre-commit is installed when running the command above, the pre-commit hooks +will have to be installed separately. Run this command to install the pre-commit hooks:: + + pre-commit install + + Running the tests ----------------- @@ -182,7 +295,7 @@ Some more advanced examples:: # verify a changed tox.ini (run this after any change to tox.ini): fakeroot -u tox --recreate - fakeroot -u tox -e py38 # run all tests, but only on python 3.8 + fakeroot -u tox -e py39 # run all tests, but only on python 3.9 fakeroot -u tox borg.testsuite.locking # only run 1 test module @@ -195,24 +308,35 @@ Important notes: - When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``. -Running more checks using coala -------------------------------- +Running the tests (using the pypi package) +------------------------------------------ -First install coala and some checkers ("bears"): +Since borg 1.4, it is also possible to run the tests without a development +environment, using the borgbackup dist package (downloaded from pypi.org or +github releases page):: -:: + # optional: create and use a virtual env: + python3 -m venv env + . env/bin/activate - pip install -r requirements.d/coala.txt + # install packages + pip install borgbackup + pip install pytest pytest-benchmark -You can now run coala from the toplevel directory; it will read its settings -from ``.coafile`` there: + # method A: use a pytest.ini -:: + cat >pytest.ini <<`_ with the @@ -366,9 +498,10 @@ Checklist: - Create a GitHub release, include: + * pypi dist package and signature * Standalone binaries (see above for how to create them). - + For OS X, document the OS X Fuse version in the README of the binaries. - OS X FUSE uses a kernel extension that needs to be compatible with the + + For macOS, document the macFUSE version in the README of the binaries. + macFUSE uses a kernel extension that needs to be compatible with the code contained in the binary. * A link to ``CHANGES.rst``. diff --git a/docs/faq.rst b/docs/faq.rst index ef9d356562..0daa226ca6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -24,16 +24,10 @@ SSHFS, the Borg client only can do file system operations and has no agent running on the remote side, so *every* operation needs to go over the network, which is slower. -Can I backup from multiple servers into a single repository? ------------------------------------------------------------- - -Yes, this is *possible* from the technical standpoint, but it is -*not recommended* from the security perspective. BorgBackup is -built upon a defined :ref:`attack_model` that cannot provide its -guarantees for multiple clients using the same repository. See -:ref:`borg_security_critique` for a detailed explanation. +Can I back up from multiple servers into a single repository? +------------------------------------------------------------- -Also, in order for the deduplication used by Borg to work, it +In order for the deduplication used by Borg to work, it needs to keep a local cache containing checksums of all file chunks already stored in the repository. This cache is stored in ``~/.cache/borg/``. If Borg detects that a repository has been @@ -45,53 +39,38 @@ repository is only modified from one place. Also keep in mind that Borg will keep an exclusive lock on the repository while creating or deleting archives, which may make *simultaneous* backups fail. +Can I back up to multiple, swapped backup targets? +-------------------------------------------------- + +It is possible to swap your backup disks if each backup medium is assigned its +own repository by creating a new one with :ref:`borg_rcreate`. + Can I copy or synchronize my repo to another location? ------------------------------------------------------ If you want to have redundant backup repositories (preferably at separate locations), the recommended way to do that is like this: -- ``borg init repo1`` -- ``borg init repo2`` +- ``borg rcreate repo1 --encryption=X`` +- ``borg rcreate repo2 --encryption=X --other-repo=repo1`` +- maybe do a snapshot to have stable and same input data for both borg create. - client machine ---borg create---> repo1 - client machine ---borg create---> repo2 -This will create distinct repositories (separate repo ID, separate -keys) and nothing bad happening in repo1 will influence repo2. - -Some people decide against above recommendation and create identical -copies of a repo (using some copy / sync / clone tool). - -While this might be better than having no redundancy at all, you have -to be very careful about how you do that and what you may / must not -do with the result (if you decide against our recommendation). - -What you would get with this is: +This will create distinct (different repo ID), but related repositories. +Related means using the same chunker secret and the same id_key, thus producing +the same chunks / the same chunk ids if the input data is the same. -- client machine ---borg create---> repo -- repo ---copy/sync---> copy-of-repo +The 2 independent borg create invocations mean that there is no error propagation +from repo1 to repo2 when done like that. -There is no special borg command to do the copying, you could just -use any reliable tool that creates an identical copy (cp, rsync, rclone -might be options). +An alternative way would be to use ``borg transfer`` to copy backup archives +from repo1 to repo2. Likely a bit more efficient and the archives would be identical, +but suffering from potential error propagation. -But think about whether that is really what you want. If something goes -wrong in repo, you will have the same issue in copy-of-repo. - -Make sure you do the copy/sync while no backup is running, see -:ref:`borg_with-lock` about how to do that. - -Also, you must not run borg against multiple instances of the same repo -(like repo and copy-of-repo) as that would create severe issues: - -- Data loss: they have the same repository ID, so the borg client will - think they are identical and e.g. use the same local cache for them - (which is an issue if they happen to be not the same). - See :issue:`4272` for an example. -- Encryption security issues if you would update repo and copy-of-repo - independently, due to AES counter reuse. - -See also: :ref:`faq_corrupt_repo` +Warning: using borg with multiple repositories with identical repository ID (like when +creating 1:1 repository copies) is not supported and can lead to all sorts of issues, +like e.g. cache coherency issues, malfunction, data corruption. "this is either an attack or unsafe" warning -------------------------------------------- @@ -107,14 +86,17 @@ run into this by yourself by restoring an older copy of your repository. "attack": maybe an attacker has replaced your repo by an older copy, trying to trick you into AES counter reuse, trying to break your repo encryption. -If you'ld decide to ignore this and accept unsafe operation for this repository, +Borg users have also reported that fs issues (like hw issues / I/O errors causing +the fs to become read-only) can cause this warning, see :issue:`7853`. + +If you decide to ignore this and accept unsafe operation for this repository, you could delete the manifest-timestamp and the local cache: :: - borg config repo id # shows the REPO_ID + borg config id # shows the REPO_ID rm ~/.config/borg/security/REPO_ID/manifest-timestamp - borg delete --cache-only REPO + borg rdelete --cache-only This is an unsafe and unsupported way to use borg, you have been warned. @@ -132,24 +114,12 @@ Which file types, attributes, etc. are *not* preserved? Archive extraction has optional support to extract all-zero chunks as holes in a sparse file. * Some filesystem specific attributes, like btrfs NOCOW, see :ref:`platforms`. - * For hardlinked symlinks, the hardlinking can not be archived (and thus, - the hardlinking will not be done at extraction time). The symlinks will - be archived and extracted as non-hardlinked symlinks, see :issue:`2379`. Are there other known limitations? ---------------------------------- -- A single archive can only reference a limited volume of file/dir metadata, - usually corresponding to tens or hundreds of millions of files/dirs. - When trying to go beyond that limit, you will get a fatal IntegrityError - exception telling that the (archive) object is too big. - An easy workaround is to create multiple archives with fewer items each. - See also the :ref:`archive_limitation` and :issue:`1452`. - - :ref:`borg_info` shows how large (relative to the maximum size) existing - archives are. -- borg extract only supports restoring into an empty destination. After that, - the destination will exactly have the contents of the extracted archive. +- borg extract supports restoring only into an empty destination. After extraction, + the destination will have exactly the contents of the extracted archive. If you extract into a non-empty destination, borg will (for example) not remove files which are in the destination, but not in the archive. See :issue:`4598` for a workaround and more details. @@ -161,14 +131,14 @@ If a backup stops mid-way, does the already-backed-up data stay there? Yes, Borg supports resuming backups. -During a backup a special checkpoint archive named ``.checkpoint`` -is saved every checkpoint interval (the default value for this is 30 +During a backup, a special checkpoint archive named ``.checkpoint`` +is saved at every checkpoint interval (the default value for this is 30 minutes) containing all the data backed-up until that point. -This checkpoint archive is a valid archive, -but it is only a partial backup (not all files that you wanted to backup are -contained in it). Having it in the repo until a successful, full backup is -completed is useful because it references all the transmitted chunks up +This checkpoint archive is a valid archive, but it is only a partial backup +(not all files that you wanted to back up are contained in it and the last file +in it might be a partial file). Having it in the repo until a successful, full +backup is completed is useful because it references all the transmitted chunks up to the checkpoint. This means that in case of an interruption, you only need to retransfer the data since the last checkpoint. @@ -187,26 +157,23 @@ Once your backup has finished successfully, you can delete all ``.checkpoint`` archives. If you run ``borg prune``, it will also care for deleting unneeded checkpoints. -Note: the checkpointing mechanism creates hidden, partial files in an archive, -so that checkpoints even work while a big file is being processed. -They are named ``.borg_part_`` and all operations usually ignore -these files, but you can make them considered by giving the option -``--consider-part-files``. You usually only need that option if you are -really desperate (e.g. if you have no completed backup of that file and you'ld -rather get a partial file extracted than nothing). You do **not** want to give -that option under any normal circumstances. - -Note that checkpoints inside files are created only since version 1.1, make -sure you have an up-to-date version of borgbackup if you want to continue -instead of retransferring a huge file. In some cases, there is only an outdated -version shipped with your distribution (e.g. Debian). See :ref:`installation`. - -How can I backup huge file(s) over a unstable connection? ---------------------------------------------------------- +Note: the checkpointing mechanism may create a partial (truncated) last file +in a checkpoint archive named ``.borg_part``. Such partial files +won't be contained in the final archive. +This is done so that checkpoints work cleanly and promptly while a big +file is being processed. -This is not a problem anymore. -For more details, see :ref:`checkpoints_parts`. +How can I back up huge file(s) over a unstable connection? +---------------------------------------------------------- + +Yes. For more details, see :ref:`checkpoints_parts`. + +How can I restore huge file(s) over an unstable connection? +----------------------------------------------------------- + +Try using ``borg mount`` and ``rsync`` (or a similar tool that supports +resuming a partial file copy from what's already copied). How can I switch append-only mode on and off? ----------------------------------------------------------------------------------------------------------------------------------- @@ -215,21 +182,12 @@ You could do that (via borg config REPO append_only 0/1), but using different ssh keys and different entries in ``authorized_keys`` is much easier and also maybe has less potential of things going wrong somehow. - My machine goes to sleep causing `Broken pipe` ---------------------------------------------- -When backing up your data over the network, your machine should not go to sleep. +While backing up your data over the network, your machine should not go to sleep. On macOS you can use `caffeinate` to avoid that. -How can I restore huge file(s) over an unstable connection? ------------------------------------------------------------ - -If you cannot manage to extract the whole big file in one go, you can extract -all the part files and manually concatenate them together. - -For more details, see :ref:`checkpoints_parts`. - How can I compare contents of an archive to my local filesystem? ----------------------------------------------------------------- @@ -238,30 +196,8 @@ then use ``tar`` to perform the comparison: :: - borg export-tar /path/to/repo::archive-name - | tar --compare -f - -C /path/to/compare/to - - -.. _faq_corrupt_repo: - -My repository is corrupt, how can I restore from an older copy of it? ---------------------------------------------------------------------- - -If your repositories are encrypted and have the same ID, the recommended method -is to delete the corrupted repository, but keep its security info, and then copy -the working repository to the same location: - -:: + borg export-tar archive-name - | tar --compare -f - -C /path/to/compare/to - borg delete --keep-security-info /path/to/repo - rsync -aH /path/to/repo-working/ /path/to/repo # Note the trailing slash. - -A plain delete command would remove the security info in -``~/.config/borg/security``, including the nonce value. In BorgBackup -:ref:`security_encryption` is AES-CTR, where the nonce is a counter. When the -working repo was used later for creating new archives, Borg would re-use nonce -values due to starting from a lower counter value given by the older copy of the -repository. To prevent this, the ``keep-security-info`` option is applied so -that the client-side nonce counter is kept. Can Borg add redundancy to the backup data to deal with hardware malfunction? ----------------------------------------------------------------------------- @@ -291,7 +227,7 @@ SMR (shingled magnetic recording) hard drives are very different from regular hard drives. Applications have to behave in certain ways or performance will be heavily degraded. -Borg 1.1 ships with default settings suitable for SMR drives, +Borg ships with default settings suitable for SMR drives, and has been successfully tested on *Seagate Archive v2* drives using the ext4 file system. @@ -397,7 +333,7 @@ Assuming that all your chunks have a size of :math:`2^{21}` bytes (approximately and we have a "perfect" hash algorithm, we can think that the probability of collision would be of :math:`p^2/2^{n+1}` then, using SHA-256 (:math:`n=256`) and for example we have 1000 million chunks (:math:`p=10^9`) (1000 million chunks would be about 2100TB). -The probability would be around to 0.0000000000000000000000000000000000000000000000000000000000043. +The probability would be around 0.0000000000000000000000000000000000000000000000000000000000043. A mass-murderer space rock happens about once every 30 million years on average. This leads to a probability of such an event occurring in the next second to about :math:`10^{-15}`. @@ -405,9 +341,9 @@ That's **45** orders of magnitude more probable than the SHA-256 collision. Brie if you find SHA-256 collisions scary then your priorities are wrong. This example was grabbed from `this SO answer `_, it's great honestly. -Still, the real question is if Borg tries to not make this happen? +Still, the real question is whether Borg tries not to make this happen? -Well... it used to not check anything but there was a feature added which saves the size +Well... previously it did not check anything until there was a feature added which saves the size of the chunks too, so the size of the chunks is compared to the size that you got with the hash and if the check says there is a mismatch it will raise an exception instead of corrupting the file. This doesn't save us from everything but reduces the chances of corruption. @@ -419,35 +355,34 @@ Why is the time elapsed in the archive stats different from wall clock time? ---------------------------------------------------------------------------- Borg needs to write the time elapsed into the archive metadata before finalizing -the archive, compacting the segments, and committing the repo & cache. This means -when Borg is run with e.g. the ``time`` command, the duration shown in the archive -stats may be shorter than the full time the command runs for. +the archive and committing the repo & cache. +This means when Borg is run with e.g. the ``time`` command, the duration shown +in the archive stats may be shorter than the full time the command runs for. How do I configure different prune policies for different directories? ---------------------------------------------------------------------- Say you want to prune ``/var/log`` faster than the rest of -``/``. How do we implement that? The answer is to backup to different +``/``. How do we implement that? The answer is to back up to different archive *names* and then implement different prune policies for different prefixes. For example, you could have a script that does:: - borg create --exclude /var/log $REPOSITORY:main-$(date +%Y-%m-%d) / - borg create $REPOSITORY:logs-$(date +%Y-%m-%d) /var/log + borg create --exclude var/log main-$(date +%Y-%m-%d) / + borg create logs-$(date +%Y-%m-%d) /var/log Then you would have two different prune calls with different policies:: - borg prune --verbose --list -d 30 --prefix main- "$REPOSITORY" - borg prune --verbose --list -d 7 --prefix logs- "$REPOSITORY" + borg prune --verbose --list -d 30 -a 'sh:main-*' + borg prune --verbose --list -d 7 -a 'sh:logs-*' -This will keep 7 days of logs and 30 days of everything else. Borg 1.1 -also supports the ``--glob-archives`` parameter. +This will keep 7 days of logs and 30 days of everything else. How do I remove files from an existing backup? ---------------------------------------------- A file is only removed from a BorgBackup repository if all archives that contain the file are deleted and the corresponding data chunks are removed from the -repository There are two ways how to remove files from a repository. +repository. There are two ways how to remove files from a repository. 1. Use :ref:`borg_delete` to remove all archives that contain the files. This will of course delete everything in the archive, not only some files. @@ -471,34 +406,6 @@ to change them. Security ######## -.. _borg_security_critique: - -Isn't BorgBackup's AES-CTR crypto broken? ------------------------------------------ - -If a nonce (counter) value is reused, AES-CTR mode crypto is broken. - -To exploit the AES counter management issue, an attacker would need to have -access to the borg repository. - -By tampering with the repo, the attacker could bring the repo into a state so -that it reports a lower "highest used counter value" than the one that actually -was used. The client would usually notice that, because it rather trusts the -clientside stored "highest used counter value" than trusting the server. - -But there are situations, where this is simply not possible: - -- If clients A and B used the repo, the client A can only know its own highest - CTR value, but not the one produced by B. That is only known to (B and) the - server (the repo) and thus the client A needs to trust the server about the - value produced by B in that situation. You can't do much about this except - not having multiple clients per repo. - -- Even if there is only one client, if client-side information is completely - lost (e.g. due to disk defect), the client also needs to trust the value from - server side. You can avoid this by not continuing to write to the repository - after you have lost clientside borg information. - .. _home_config_borg: How important is the $HOME/.config/borg directory? @@ -506,13 +413,6 @@ How important is the $HOME/.config/borg directory? The Borg config directory has content that you should take care of: -``security`` subdirectory - Each directory here represents one Borg repository by its ID and contains the last known status. - If a repository's status is different from this information at the beginning of BorgBackup - operation, Borg outputs warning messages and asks for confirmation, so make sure you do not lose - or manipulate these files. However, apart from those warnings, a loss of these files can be - recovered. - ``keys`` subdirectory All your borg keyfile keys are stored in this directory. Please note that borg repokey keys are stored inside the repository. You MUST make sure to have an @@ -522,6 +422,22 @@ The Borg config directory has content that you should take care of: Make sure that only you have access to the Borg config directory. +.. _home_data_borg: + +How important is the $HOME/.local/share/borg directory? +------------------------------------------------------- + +The Borg data directory has content that you should take care of: + +``security`` subdirectory + Each directory here represents one Borg repository by its ID and contains the last known status. + If a repository's status is different from this information at the beginning of BorgBackup + operation, Borg outputs warning messages and asks for confirmation, so make sure you do not lose + or manipulate these files. However, apart from those warnings, a loss of these files can be + recovered. + +Make sure that only you have access to the Borg data directory. + .. _cache_security: Do I need to take security precautions regarding the cache? @@ -559,13 +475,12 @@ Setting ``BORG_PASSPHRASE`` user `_. -Using ``BORG_PASSCOMMAND`` with a properly permissioned file +Using ``BORG_PASSCOMMAND`` with a file of proper permissions Another option is to create a file with a password in it in your home directory and use permissions to keep anyone else from reading it. For example, first create a key:: - head -c 32 /dev/urandom | base64 -w 0 > ~/.borg-passphrase - chmod 400 ~/.borg-passphrase + (umask 0077; head -c 32 /dev/urandom | base64 -w 0 > ~/.borg-passphrase) Then in an automated script one can put:: @@ -576,13 +491,13 @@ Using ``BORG_PASSCOMMAND`` with a properly permissioned file Using keyfile-based encryption with a blank passphrase It is possible to encrypt your repository in ``keyfile`` mode instead of the default ``repokey`` mode and use a blank passphrase for the key file (simply press Enter twice - when ``borg init`` asks for the password). See :ref:`encrypted_repos` + when ``borg rcreate`` asks for the password). See :ref:`encrypted_repos` for more details. Using ``BORG_PASSCOMMAND`` with macOS Keychain macOS has a native manager for secrets (such as passphrases) which is safer than just using a file as it is encrypted at rest and unlocked manually - (fortunately, the login keyring automatically unlocks when you login). With + (fortunately, the login keyring automatically unlocks when you log in). With the built-in ``security`` command, you can access it from the command line, making it useful for ``BORG_PASSCOMMAND``. @@ -617,7 +532,7 @@ Using ``BORG_PASSCOMMAND`` with GNOME Keyring export BORG_PASSCOMMAND="secret-tool lookup borg-repository repo-name" - .. note:: For this to automatically unlock the keychain it must be run + .. note:: For this to unlock the keychain automatically it must be run in the ``dbus`` session of an unlocked terminal; for example, running a backup script as a ``cron`` job might not work unless you also ``export DISPLAY=:0`` so ``secret-tool`` can pick up your open session. `It gets even more complicated`__ @@ -660,13 +575,13 @@ otherwise make unavailable) all your backups. How can I protect against a hacked backup client? ------------------------------------------------- -Assume you backup your backup client machine C to the backup server S and +Assume you back up your backup client machine C to the backup server S and C gets hacked. In a simple push setup, the attacker could then use borg on C to delete all backups residing on S. These are your options to protect against that: -- Do not allow to permanently delete data from the repo, see :ref:`append_only_mode`. +- Do not allow to delete data permanently from the repo, see :ref:`append_only_mode`. - Use a pull-mode setup using ``ssh -R``, see :ref:`pull_backup` for more information. - Mount C's filesystem on another machine and then create a backup of it. - Do not give C filesystem-level access to S. @@ -710,34 +625,17 @@ Send a private email to the :ref:`security contact ` if you think you have discovered a security issue. Please disclose security issues responsibly. -How important are the nonce files? ------------------------------------- - -Borg uses :ref:`AES-CTR encryption `. An -essential part of AES-CTR is a sequential counter that must **never** -repeat. If the same value of the counter is used twice in the same repository, -an attacker can decrypt the data. The counter is stored in the home directory -of each user ($HOME/.config/borg/security/$REPO_ID/nonce) as well as -in the repository (/path/to/repo/nonce). When creating a new archive borg uses -the highest of the two values. The value of the counter in the repository may be -higher than your local value if another user has created an archive more recently -than you did. - -Since the nonce is not necessary to read the data that is already encrypted, -``borg info``, ``borg list``, ``borg extract`` and ``borg mount`` should work -just fine without it. +Common issues +############# -If the the nonce file stored in the repo is lost, but you still have your local copy, -borg will recreate the repository nonce file the next time you run ``borg create``. -This should be safe for repositories that are only used from one user account -on one machine. +/path/to/repo is not a valid repository. Check repo config. +----------------------------------------------------------- -For repositories that are used by multiple users and/or from multiple machines -it is safest to avoid running *any* commands that modify the repository after -the nonce is deleted or if you suspect it may have been tampered with. See :ref:`attack_model`. +There can be many causes of this error. E.g. you have incorrectly specified the repository path. -Common issues -############# +You will also get this error if you try to access a repository with a key that uses the argon2 key algorithm using an old version of borg. +We recommend upgrading to the latest stable version and trying again. We are sorry. We should have thought about forward +compatibility and implemented a more helpful error message. Why does Borg extract hang after some time? ------------------------------------------- @@ -771,9 +669,9 @@ How can I deal with my very unstable SSH connection? If you have issues with lost connections during long-running borg commands, you could try to work around: -- Make partial extracts like ``borg extract REPO PATTERN`` to do multiple +- Make partial extracts like ``borg extract PATTERN`` to do multiple smaller extraction runs that complete before your connection has issues. -- Try using ``borg mount REPO MOUNTPOINT`` and ``rsync -avH`` from +- Try using ``borg mount MOUNTPOINT`` and ``rsync -avH`` from ``MOUNTPOINT`` to your desired extraction directory. If the connection breaks down, just repeat that over and over again until rsync does not find anything to do any more. Due to the way borg mount works, this might be less efficient @@ -822,18 +720,8 @@ will make the subsequent rebuilds faster (because it needs to transfer less data from the repository). While being faster, the cache needs quite some disk space, which might be unwanted. -There is a temporary (but maybe long lived) hack to avoid using lots of disk -space for chunks.archive.d (see :issue:`235` for details): - -:: - - # this assumes you are working with the same user as the backup. - cd ~/.cache/borg/$(borg config /path/to/repo id) - rm -rf chunks.archive.d ; touch chunks.archive.d - -This deletes all the cached archive chunk indexes and replaces the directory -that kept them with a file, so borg won't be able to store anything "in" there -in future. +You can disable the cached archive chunk indexes by setting the environment +variable ``BORG_USE_CHUNKS_ARCHIVE`` to ``no``. This has some pros and cons, though: @@ -848,13 +736,13 @@ This has some pros and cons, though: The long term plan to improve this is called "borgception", see :issue:`474`. -Can I backup my root partition (/) with Borg? ---------------------------------------------- +Can I back up my root partition (/) with Borg? +---------------------------------------------- Backing up your entire root partition works just fine, but remember to -exclude directories that make no sense to backup, such as /dev, /proc, +exclude directories that make no sense to back up, such as /dev, /proc, /sys, /tmp and /run, and to use ``--one-file-system`` if you only want to -backup the root partition (and not any mounted devices e.g.). +back up the root partition (and not any mounted devices e.g.). If it crashes with a UnicodeError, what can I do? ------------------------------------------------- @@ -863,6 +751,12 @@ Check if your encoding is set correctly. For most POSIX-like systems, try:: export LANG=en_US.UTF-8 # or similar, important is correct charset +If that does not help: + +- check for typos, check if you really used ``export``. +- check if you have set ``LC_ALL`` - if so, try not setting it. +- check if you generated the respective locale via ``locale-gen``. + I can't extract non-ascii filenames by giving them on the commandline!? ----------------------------------------------------------------------- @@ -880,13 +774,29 @@ If you run into that, try this: What's the expected backup performance? --------------------------------------- -A first backup will usually be somehow "slow" because there is a lot of data -to process. Performance here depends on a lot of factors, so it is hard to -give specific numbers. +Compared to simply copying files (e.g. with ``rsync``), Borg has more work to do. +This can make creation of the first archive slower, but saves time +and disk space on subsequent runs. Here what Borg does when you run ``borg create``: + +- Borg chunks the file (using the relatively expensive buzhash algorithm) +- It then computes the "id" of the chunk (hmac-sha256 (often slow, except + if your CPU has sha256 acceleration) or blake2b (fast, in software)) +- Then it checks whether this chunk is already in the repo (local hashtable lookup, + fast). If so, the processing of the chunk is completed here. Otherwise it needs to + process the chunk: +- Compresses (the default lz4 is super fast) +- Encrypts and authenticates (AES-OCB, usually fast if your CPU has AES acceleration as usual + since about 10y, or chacha20-poly1305, fast pure-software crypto) +- Transmits to repo. If the repo is remote, this usually involves an SSH connection + (does its own encryption / authentication). +- Stores the chunk into a key/value store (the key is the chunk id, the value + is the data). While doing that, it computes CRC32 / XXH64 of the data (repo low-level + checksum, used by borg check --repository) and also updates the repo index + (another hashtable). Subsequent backups are usually very fast if most files are unchanged and only a few are new or modified. The high performance on unchanged files primarily depends -only on a few factors (like fs recursion + metadata reading performance and the +only on a few factors (like FS recursion + metadata reading performance and the files cache working as expected) and much less on other factors. E.g., for this setup: @@ -904,14 +814,30 @@ few FAQ entries below. .. _slow_backup: -Why is backup slow for me? +Why is my backup so slow? -------------------------- -So, if you feel your Borg backup is too slow somehow, you should find out why. - -The usual way to approach this is to add ``--list --filter=AME --stats`` to your -``borg create`` call to produce more log output, including a file list (with file status -characters) and also some statistics at the end of the backup. +If you feel your Borg backup is too slow somehow, here is what you can do: + +- Make sure Borg has enough RAM (depends on how big your repo is / how many + files you have) +- Use one of the blake2 modes for --encryption except if you positively know + your CPU (and openssl) accelerates sha256 (then stay with hmac-sha256). +- Don't use any expensive compression. The default is lz4 and super fast. + Uncompressed is often slower than lz4. +- Just wait. You can also interrupt it and start it again as often as you like, + it will converge against a valid "completed" state (see ``--checkpoint-interval``, + maybe use the default, but in any case don't make it too short). It is starting + from the beginning each time, but it is still faster then as it does not store + data into the repo which it already has there from last checkpoint. +- If you don’t need additional file attributes, you can disable them with ``--noflags``, + ``--noacls``, ``--noxattrs``. This can lead to noticeable performance improvements + when your backup consists of many small files. + +To see what files have changed and take more time processing, you can also add +``--list --filter=AME --stats`` to your ``borg create`` call to produce more log output, +including a file list (with file status characters) and also some statistics at +the end of the backup. Then you do the backup and look at the log output: @@ -931,8 +857,26 @@ Then you do the backup and look at the log output: The metadata values used in this comparison are determined by the ``--files-cache`` option and could be e.g. size, ctime and inode number (see the ``borg create`` docs for more details and potential issues). - You can use the ``stat`` command on files to manually look at fs metadata to debug if + You can use the ``stat`` command on files to look at fs metadata manually to debug if there is any unexpected change triggering the ``M`` status. + Also, the ``--debug-topic=files_cache`` option of ``borg create`` provides a lot of debug + output helping to analyse why the files cache does not give its expected high performance. + +When borg runs inside a virtual machine, there are some more things to look at: + +Some hypervisors (e.g. kvm on proxmox) give some broadly compatible CPU type to the +VM (usually to ease migration between VM hosts of potentially different hardware CPUs). + +It is broadly compatible because they leave away modern CPU features that could be +not present in older or other CPUs, e.g. hardware acceleration for AES crypto, for +sha2 hashes, for (P)CLMUL(QDQ) computations useful for crc32. + +So, basically you pay for compatibility with bad performance. If you prefer better +performance, you should try to expose the host CPU's misc. hw acceleration features +to the VM which runs borg. + +On Linux, check ``/proc/cpuinfo`` for the CPU flags inside the VM. +For kvm check the docs about "Host model" and "Host passthrough". See also the next few FAQ entries for more details. @@ -1007,11 +951,15 @@ will be slow because it would chunk all the files each time. If you set BORG_FILES_CACHE_TTL to at least 26 (or maybe even a small multiple of that), it would be much faster. +Besides using a higher BORG_FILES_CACHE_TTL (which also increases memory usage), +there is also BORG_FILES_CACHE_SUFFIX which can be used to have separate (smaller) +files caches for each backup set instead of the default one (big) unified files cache. + Another possible reason is that files don't always have the same path, for example if you mount a filesystem without stable mount points for each backup or if you are running the backup from a filesystem snapshot whose name is not stable. If the directory where you mount a filesystem is different every time, -Borg assumes they are different files. This is true even if you backup these +Borg assumes they are different files. This is true even if you back up these files with relative pathnames - borg uses full pathnames in files cache regardless. It is possible for some filesystems, such as ``mergerfs`` or network filesystems, @@ -1062,7 +1010,7 @@ How can I avoid unwanted base directories getting stored into archives? Possible use cases: -- Another file system is mounted and you want to backup it with original paths. +- Another file system is mounted and you want to back it up with original paths. - You have created a BTRFS snapshot in a ``/.snapshots`` directory for backup. To achieve this, run ``borg create`` within the mountpoint/snapshot directory: @@ -1071,7 +1019,7 @@ To achieve this, run ``borg create`` within the mountpoint/snapshot directory: # Example: Some file system mounted in /mnt/rootfs. cd /mnt/rootfs - borg create /path/to/repo::rootfs_backup . + borg create rootfs_backup . I am having troubles with some network/FUSE/special filesystem, why? @@ -1160,7 +1108,7 @@ This can happen for CIFS- or FUSE shares. If you are sure that your target volum will always have enough disk space, you can use the following workaround to disable checking for free disk space:: - borg config -- $REPO_LOCATION additional_free_space -2T + borg config -- additional_free_space -2T How do I rename a repository? ----------------------------- @@ -1178,133 +1126,72 @@ It may be useful to set ``BORG_RELOCATED_REPO_ACCESS_IS_OK=yes`` to avoid the prompts when renaming multiple repositories or in a non-interactive context such as a script. See :doc:`deployment` for an example. +The repository quota size is reached, what can I do? +---------------------------------------------------- -Miscellaneous -############# - -Requirements for the borg single-file binary, esp. (g)libc? ------------------------------------------------------------ - -We try to build the binary on old, but still supported systems - to keep the -minimum requirement for the (g)libc low. The (g)libc can't be bundled into -the binary as it needs to fit your kernel and OS, but Python and all other -required libraries will be bundled into the binary. - -If your system fulfills the minimum (g)libc requirement (see the README that -is released with the binary), there should be no problem. If you are slightly -below the required version, maybe just try. Due to the dynamic loading (or not -loading) of some shared libraries, it might still work depending on what -libraries are actually loaded and used. - -In the borg git repository, there is scripts/glibc_check.py that can determine -(based on the symbols' versions they want to link to) whether a set of given -(Linux) binaries works with a given glibc version. - - -Why was Borg forked from Attic? -------------------------------- - -Borg was created in May 2015 in response to the difficulty of getting new -code or larger changes incorporated into Attic and establishing a bigger -developer community / more open development. - -More details can be found in `ticket 217 -`_ that led to the fork. - -Borg intends to be: - -* simple: - - * as simple as possible, but no simpler - * do the right thing by default, but offer options -* open: +The simplest solution is to increase or disable the quota and resume the backup: - * welcome feature requests - * accept pull requests of good quality and coding style - * give feedback on PRs that can't be accepted "as is" - * discuss openly, don't work in the dark -* changing: +:: - * Borg is not compatible with Attic - * do not break compatibility accidentally, without a good reason - or without warning. allow compatibility breaking for other cases. - * if major version number changes, it may have incompatible changes + borg config /path/to/repo storage_quota 0 -Migrating from Attic -#################### +If you are bound to the quota, you have to free repository space. The first to +try is running :ref:`borg_compact` to free unused backup space (see also +:ref:`separate_compaction`): -What are the differences between Attic and Borg? ------------------------------------------------- +:: -Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_". + borg compact /path/to/repo -.. _Attic: https://github.com/jborg/attic -.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html +If your repository is already compacted, run :ref:`borg_prune` or +:ref:`borg_delete` to delete archives that you do not need anymore, and then run +``borg compact`` again. -Here's a (incomplete) list of some major changes: +My backup disk is full, what can I do? +-------------------------------------- -* lots of attic issues fixed - (see `issue #5 `_), - including critical data corruption bugs and security issues. -* more open, faster paced development - (see `issue #1 `_) -* less chunk management overhead (less memory and disk usage for chunks index) -* faster remote cache resync (useful when backing up multiple machines into same repo) -* compression: no, lz4, zstd, zlib or lzma compression, adjustable compression levels -* repokey replaces problematic passphrase mode (you can't change the passphrase - nor the pbkdf2 iteration count in "passphrase" mode) -* simple sparse file support, great for virtual machine disk files -* can read special files (e.g. block devices) or from stdin, write to stdout -* rename-based locking is more compatible than attic's posix locking -* uses fadvise to not spoil / blow up the fs cache -* better error messages / exception handling -* better logging, screen output, progress indication -* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, macOS +Borg cannot work if you really have zero free space on the backup disk, so the +first thing you must do is deleting some files to regain free disk space. See +:ref:`about_free_space` for further details. -Please read the :ref:`changelog` (or ``docs/changes.rst`` in the source distribution) for more -information. +Some Borg commands that do not change the repository might work under disk-full +conditions, but generally this should be avoided. If your backup disk is already +full when Borg starts a write command like `borg create`, it will abort +immediately and the repository will stay as-is. -Borg is not compatible with original Attic (but there is a one-way conversion). +If you run a backup that stops due to a disk running full, Borg will roll back, +delete the new segment file and thus freeing disk space automatically. There +may be a checkpoint archive left that has been saved before the disk got full. +You can keep it to speed up the next backup or delete it to get back more disk +space. -How do I migrate from Attic to Borg? ------------------------------------- +Miscellaneous +############# -Use :ref:`borg_upgrade`. This is a one-way process that cannot be reversed. +macOS: borg mounts not shown in Finder's side bar +------------------------------------------------- -There are some caveats: +https://github.com/osxfuse/osxfuse/wiki/Mount-options#local -- The upgrade can only be performed on local repositories. - It cannot be performed on remote repositories. +Read the above first and use this on your own risk:: -- If the repository is in "keyfile" encryption mode, the keyfile must - exist locally or it must be manually moved after performing the upgrade: + borg mount -olocal REPO MOUNTPOINT - 1. Get the repository ID with ``borg config /path/to/repo id``. - 2. Locate the attic key file at ``~/.attic/keys/``. The correct key for the - repository starts with the line ``ATTIC_KEY ``. - 3. Copy the attic key file to ``~/.config/borg/keys/`` - 4. Change the first line from ``ATTIC_KEY ...`` to ``BORG_KEY ...``. - 5. Verify that the repository is now accessible (e.g. ``borg list ``). -- Attic and Borg use different :ref:`"chunker params" `. - This means that data added by Borg won't deduplicate with the existing data - stored by Attic. The effect is lessened if the files cache is used with Borg. -- Repositories in "passphrase" mode *must* be migrated to "repokey" mode using - :ref:`borg_key_migrate-to-repokey`. Borg does not support the "passphrase" mode - any other way. -Why is my backup bigger than with attic? ----------------------------------------- +Requirements for the borg single-file binary, esp. (g)libc? +----------------------------------------------------------- -Attic was rather unflexible when it comes to compression, it always -compressed using zlib level 6 (no way to switch compression off or -adjust the level or algorithm). +We try to build the binary on old, but still supported systems - to keep the +minimum requirement for the (g)libc low. The (g)libc can't be bundled into +the binary as it needs to fit your kernel and OS, but Python and all other +required libraries will be bundled into the binary. -The default in Borg is lz4, which is fast enough to not use significant CPU time -in most cases, but can only achieve modest compression. It still compresses -easily compressed data fairly well. +If your system fulfills the minimum (g)libc requirement (see the README that +is released with the binary), there should be no problem. If you are slightly +below the required version, maybe just try. Due to the dynamic loading (or not +loading) of some shared libraries, it might still work depending on what +libraries are actually loaded and used. -Borg also offers zstd, zlib and lzma compression, choose wisely. - -Which choice is the best option depends on a number of factors, like -bandwidth to the repository, how well the data compresses, available CPU -power and so on. +In the borg git repository, there is scripts/glibc_check.py that can determine +(based on the symbols' versions they want to link to) whether a set of given +(Linux) binaries works with a given glibc version. diff --git a/docs/global.rst.inc b/docs/global.rst.inc index 14725f7312..11e8ccec17 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -10,12 +10,12 @@ .. _HMAC-SHA256: https://en.wikipedia.org/wiki/HMAC .. _SHA256: https://en.wikipedia.org/wiki/SHA-256 .. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2 +.. _argon2: https://en.wikipedia.org/wiki/Argon2 .. _ACL: https://en.wikipedia.org/wiki/Access_control_list .. _libacl: https://savannah.nongnu.org/projects/acl/ .. _libattr: https://savannah.nongnu.org/projects/attr/ .. _liblz4: https://github.com/Cyan4973/lz4 .. _libzstd: https://github.com/facebook/zstd -.. _libb2: https://github.com/BLAKE2/libb2 .. _OpenSSL: https://www.openssl.org/ .. _`Python 3`: https://www.python.org/ .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash diff --git a/docs/index.rst b/docs/index.rst index 9675ed989a..6fc3bb6a52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,8 @@ Borg Documentation faq support changes + changes_1.x + changes_0.x internals development authors diff --git a/docs/installation.rst b/docs/installation.rst index 46a640c4d8..fcc12ffd54 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,6 +13,7 @@ There are different ways to install Borg: that comes bundled with all dependencies. - :ref:`source-install`, either: + - :ref:`windows-binary` - builds a binary file for Windows using MSYS2. - :ref:`pip-installation` - installing a source package with pip needs more installation steps and requires all dependencies with development headers and a compiler. @@ -42,7 +43,7 @@ package which can be installed with the package manager. Distribution Source Command ============ ============================================= ======= Alpine Linux `Alpine repository`_ ``apk add borgbackup`` -Arch Linux `[community]`_ ``pacman -S borg`` +Arch Linux `[extra]`_ ``pacman -S borg`` Debian `Debian packages`_ ``apt install borgbackup`` Gentoo `ebuild`_ ``emerge borgbackup`` GNU Guix `GNU Guix`_ ``guix package --install borg`` @@ -63,9 +64,9 @@ Ubuntu `Ubuntu packages`_, `Ubuntu PPA`_ ``apt install borgbac ============ ============================================= ======= .. _Alpine repository: https://pkgs.alpinelinux.org/packages?name=borgbackup -.. _[community]: https://www.archlinux.org/packages/?name=borg +.. _[extra]: https://www.archlinux.org/packages/?name=borg .. _Debian packages: https://packages.debian.org/search?keywords=borgbackup&searchon=names&exact=1&suite=all§ion=all -.. _Fedora official repository: https://apps.fedoraproject.org/packages/borgbackup +.. _Fedora official repository: https://packages.fedoraproject.org/pkgs/borgbackup/borgbackup/ .. _FreeBSD ports: https://www.freshports.org/archivers/py-borgbackup/ .. _ebuild: https://packages.gentoo.org/packages/app-backup/borgbackup .. _GNU Guix: https://www.gnu.org/software/guix/package-list.html#borg @@ -78,7 +79,7 @@ Ubuntu `Ubuntu packages`_, `Ubuntu PPA`_ ``apt install borgbac .. _Homebrew: https://formulae.brew.sh/formula/borgbackup .. _private Tap: https://github.com/borgbackup/homebrew-tap .. _Raspbian testing: https://archive.raspbian.org/raspbian/pool/main/b/borgbackup/ -.. _Ubuntu packages: https://packages.ubuntu.com/xenial/borgbackup +.. _Ubuntu packages: https://launchpad.net/ubuntu/+source/borgbackup .. _Ubuntu PPA: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup Please ask package maintainers to build a package or, if you can package / @@ -134,7 +135,7 @@ fail if /tmp has not enough free space or is mounted with the ``noexec`` option. You can change the temporary directory by setting the ``TEMP`` environment variable before running Borg. -If a new version is released, you will have to manually download it and replace +If a new version is released, you will have to download it manually and replace the old version using the same steps as shown above. .. _pyinstaller: http://www.pyinstaller.org @@ -159,28 +160,26 @@ Dependencies To install Borg from a source package (including pip), you have to install the following dependencies first: -* `Python 3`_ >= 3.8.0, plus development headers. -* OpenSSL_ >= 1.0.0, plus development headers. -* libacl_ (which depends on libattr_), both plus development headers. -* We have bundled code of the following packages, but borg by default (see - setup.py if you want to change that) prefers a shared library if it can - be found on the system (lib + dev headers) at build time: +* `Python 3`_ >= 3.9.0, plus development headers. +* Libraries (library plus development headers): + - OpenSSL_ >= 1.1.1 (LibreSSL will not work) + - libacl_ (which depends on libattr_) - liblz4_ >= 1.7.0 (r129) - libzstd_ >= 1.3.0 - libxxhash >= 0.8.1 (0.8.0 might work also) * pkg-config (cli tool) and pkgconfig python package (borg uses these to discover header and library location - if it can't import pkgconfig and is not pointed to header/library locations via env vars [see setup.py], - it will fall back to using the bundled code, see above). + it will raise a fatal error). **These must be present before invoking setup.py!** * some other Python dependencies, pip will automatically install them for you. * optionally, if you wish to mount an archive as a FUSE filesystem, you need a FUSE implementation for Python: - - Either pyfuse3_ (preferably, newer and maintained) or llfuse_ (older, - unmaintained now). See also the BORG_FUSE_IMPL env variable. - - See setup.py about the version requirements. + - Either pyfuse3_ (preferably, newer) or llfuse_ (older). + See also the BORG_FUSE_IMPL env variable. + - See pyproject.toml about the version requirements. If you have troubles finding the right package names, have a look at the distribution specific sections below or the Vagrantfile in the git repository, @@ -237,8 +236,7 @@ Install the dependencies automatically using zypper:: Alternatively, you can enumerate all build dependencies in the command line:: sudo zypper install python3 python3-devel \ - libacl-devel openssl-devel \ - libxxhash-devel \ + libacl-devel openssl-devel xxhash-devel libzstd-devel liblz4-devel \ python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \ python3-pytest python3-setuptools python3-setuptools_scm \ python3-sphinx_rtd_theme gcc gcc-c++ @@ -247,16 +245,10 @@ Alternatively, you can enumerate all build dependencies in the command line:: macOS +++++ -When installing via Homebrew_, dependencies are installed automatically. To install -dependencies manually:: +When installing borgbackup via Homebrew_, the basic dependencies are installed automatically. - brew install python3 openssl zstd lz4 xxhash - brew install pkg-config - pip3 install virtualenv pkgconfig - -For FUSE support to mount the backup archives, you need at least version 3.0 of -macFUSE, which is available via `github -`__, or Homebrew:: +For FUSE support to mount the backup archives, you need macFUSE, which is available +via `github `__, or Homebrew:: brew install --cask macfuse @@ -266,7 +258,14 @@ the installed ``openssl`` formula, point pkg-config to the correct path:: PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" pip install borgbackup[llfuse] -For OS X Catalina and later, be aware that you must authorize full disk access. +When working from a borg git repo workdir, you can install dependencies using the +Brewfile:: + + brew install python@3.11 # can be any supported python3 version + brew bundle install # install requirements from borg repo's ./Brewfile + pip3 install virtualenv pkgconfig + +Be aware that for all recent macOS releases you must authorize full disk access. It is no longer sufficient to run borg backups as root. If you have not yet granted full disk access, and you run Borg backup from cron, you will see messages such as:: @@ -297,6 +296,20 @@ and commands to make FUSE work for using the mount command. kldload fuse sysctl vfs.usermount=1 +.. _windows_deps: + +Windows ++++++++ + +.. note:: + Running under Windows is experimental. + +.. warning:: + This script needs to be run in the UCRT64 environment in MSYS2. + +Install the dependencies with the provided script:: + + ./scripts/msys2-install-deps Windows 10's Linux Subsystem ++++++++++++++++++++++++++++ @@ -315,11 +328,33 @@ Cygwin Use the Cygwin installer to install the dependencies:: - python38 python38-devel python38-pkgconfig - python38-setuptools python38-pip python38-wheel python38-virtualenv + python39 python39-devel python39-pkgconfig + python39-setuptools python39-pip python39-wheel python39-virtualenv libssl-devel libxxhash-devel liblz4-devel libzstd-devel binutils gcc-g++ git make openssh +Make sure to use a virtual environment to avoid confusions with any Python installed on Windows. + +.. _windows-binary: + +Building a binary on Windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + This is experimental. + +.. warning:: + This needs to be run in the UCRT64 environment in MSYS2. + +Ensure to install the dependencies as described within :ref:`Dependencies: Windows `. + +:: + + export SETUPTOOLS_USE_DISTUTILS=stdlib # Needed for pip to work - https://www.msys2.org/docs/python/#known-issues + pip install -e . + pyinstaller -y scripts/borg.exe.spec + +A standalone executable will be created in ``dist/borg.exe``. .. _pip-installation: @@ -330,11 +365,13 @@ Virtualenv_ can be used to build and install Borg without affecting the system Python or requiring root access. Using a virtual environment is optional, but recommended except for the most simple use cases. +Ensure to install the dependencies as described within :ref:`source-install`. + .. note:: If you install into a virtual environment, you need to **activate** it first (``source borg-env/bin/activate``), before running ``borg``. Alternatively, symlink ``borg-env/bin/borg`` into some directory that is in - your ``PATH`` so you can just run ``borg``. + your ``PATH`` so you can run ``borg``. This will use ``pip`` to install the latest release from PyPi:: @@ -358,6 +395,19 @@ activating your virtual environment:: pip install -U borgbackup # or ... borgbackup[llfuse/pyfuse3] +When doing manual pip installation, man pages are not automatically +installed. You can run these commands to install the man pages +locally:: + + # get borg from github + git clone https://github.com/borgbackup/borg.git borg + + # Install the files with proper permissions + install -D -m 0644 borg/docs/man/borg*.1* $HOME/.local/share/man/man1/borg.1 + + # Update the man page cache + mandb + .. _git-installation: Using git @@ -366,13 +416,15 @@ Using git This uses latest, unreleased development code from git. While we try not to break master, there are no guarantees on anything. +Ensure to install the dependencies as described within :ref:`source-install`. + :: # get borg from github git clone https://github.com/borgbackup/borg.git # create a virtual environment - virtualenv --python=${which python3} borg-env + virtualenv --python=$(which python3) borg-env source borg-env/bin/activate # always before using! # install borg + dependencies into virtualenv @@ -397,11 +449,11 @@ If you need to use a different version of Python you can install this using ``py ... # create a virtual environment - pyenv install 3.8.0 # minimum, preferably use something more recent! - pyenv global 3.8.0 - pyenv local 3.8.0 + pyenv install 3.9.0 # minimum, preferably use something more recent! + pyenv global 3.9.0 + pyenv local 3.9.0 virtualenv --python=${pyenv which python} borg-env source borg-env/bin/activate # always before using! ... -.. note:: As a developer or power user, you always want to use a virtual environment. +.. note:: As a developer or power user, you should always use a virtual environment. diff --git a/docs/internals.rst b/docs/internals.rst index d11f0bfedd..e587803cbc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -22,7 +22,7 @@ metadata, using :ref:`chunks` created by the chunker using the Buzhash_ algorithm ("buzhash" chunker) or a simpler fixed blocksize algorithm ("fixed" chunker). -To actually perform the repository-wide deduplication, a hash of each +To perform the repository-wide deduplication, a hash of each chunk is checked against the :ref:`chunks cache `, which is a hash-table of all chunks that already exist. diff --git a/docs/internals/compaction.odg b/docs/internals/compaction.odg new file mode 100644 index 0000000000..8d193e009e Binary files /dev/null and b/docs/internals/compaction.odg differ diff --git a/docs/internals/compaction.png b/docs/internals/compaction.png index 927ae0b215..d5c53a680e 100644 Binary files a/docs/internals/compaction.png and b/docs/internals/compaction.png differ diff --git a/docs/internals/compaction.vsd b/docs/internals/compaction.vsd deleted file mode 100644 index 73cc0b061a..0000000000 Binary files a/docs/internals/compaction.vsd and /dev/null differ diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index 6d1b4ab07a..8d1562ff23 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -59,7 +59,7 @@ Each repository has a ``config`` file which is a ``INI``-style file and looks like this:: [repository] - version = 1 + version = 2 segments_per_dir = 1000 max_segment_size = 524288000 id = 57d6c1d52ce76a836b532b0e42e677dec6af9fca3673db511279358828a21ed6 @@ -77,9 +77,9 @@ don't have a particular meaning (except for the Manifest_). Normally the keys are computed like this:: - key = id = id_hash(unencrypted_data) + key = id = id_hash(plaintext_data) # plain = not encrypted, not compressed, not obfuscated -The id_hash function depends on the :ref:`encryption mode `. +The id_hash function depends on the :ref:`encryption mode `. As the id / key is used for deduplication, id_hash must be a cryptographically strong hash or MAC. @@ -94,20 +94,29 @@ this value in a non-empty repository, you may also need to relocate the segment files manually. A segment starts with a magic number (``BORG_SEG`` as an eight byte ASCII string), -followed by a number of log entries. Each log entry consists of: - -* 32-bit size of the entry -* CRC32 of the entire entry (for a PUT this includes the data) -* entry tag: PUT, DELETE or COMMIT -* PUT and DELETE follow this with the 32 byte key -* PUT follow the key with the data +followed by a number of log entries. Each log entry consists of (in this order): + +* crc32 checksum (uint32): + - for PUT2: CRC32(size + tag + key + digest) + - for PUT: CRC32(size + tag + key + payload) + - for DELETE: CRC32(size + tag + key) + - for COMMIT: CRC32(size + tag) +* size (uint32) of the entry (including the whole header) +* tag (uint8): PUT(0), DELETE(1), COMMIT(2) or PUT2(3) +* key (256 bit) - only for PUT/PUT2/DELETE +* payload (size - 41 bytes) - only for PUT +* xxh64 digest (64 bit) = XXH64(size + tag + key + payload) - only for PUT2 +* payload (size - 41 - 8 bytes) - only for PUT2 + +PUT2 is new since repository version 2. For new log entries PUT2 is used. +PUT is still supported to read version 1 repositories, but not generated any more. +If we talk about ``PUT`` in general, it shall usually mean PUT2 for repository +version 2+. Those files are strictly append-only and modified only once. -Tag is either ``PUT``, ``DELETE``, or ``COMMIT``. - When an object is written to the repository a ``PUT`` entry is written -to the file containing the object id and data. If an object is deleted +to the file containing the object id and payload. If an object is deleted a ``DELETE`` entry is appended with the object id. A ``COMMIT`` tag is written when a repository transaction is @@ -121,14 +130,56 @@ partial/uncommitted transaction. The size of individual segments is limited to 4 GiB, since the offset of entries within segments is stored in a 32-bit unsigned integer in the repository index. +Objects / Payload structure +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All data (the manifest, archives, archive item stream chunks and file data +chunks) is compressed, optionally obfuscated and encrypted. This produces some +additional metadata (size and compression information), which is separately +serialized and also encrypted. + +See :ref:`data-encryption` for a graphic outlining the anatomy of the encryption in Borg. +What you see at the bottom there is done twice: once for the data and once for the metadata. + +An object (the payload part of a segment file log entry) must be like: + +- length of encrypted metadata (16bit unsigned int) +- encrypted metadata (incl. encryption header), when decrypted: + + - msgpacked dict with: + + - ctype (compression type 0..255) + - clevel (compression level 0..255) + - csize (overall compressed (and maybe obfuscated) data size) + - psize (only when obfuscated: payload size without the obfuscation trailer) + - size (uncompressed size of the data) +- encrypted data (incl. encryption header), when decrypted: + + - compressed data (with an optional all-zero-bytes obfuscation trailer) + +This new, more complex repo v2 object format was implemented to be able to query the +metadata efficiently without having to read, transfer and decrypt the (usually much bigger) +data part. + +The metadata is encrypted not to disclose potentially sensitive information that could be +used for e.g. fingerprinting attacks. + +The compression `ctype` and `clevel` is explained in :ref:`data-compression`. + + Index, hints and integrity ~~~~~~~~~~~~~~~~~~~~~~~~~~ The **repository index** is stored in ``index.`` and is used to determine an object's location in the repository. It is a HashIndex_, -a hash table using open addressing. It maps object keys_ to two -unsigned 32-bit integers; the first integer gives the segment number, -the second indicates the offset of the object's entry within the segment. +a hash table using open addressing. + +It maps object keys_ to: + +* segment number (unit32) +* offset of the object's entry within the segment (uint32) +* size of the payload, not including the entry header (uint32) +* flags (uint32) The **hints file** is a msgpacked file named ``hints.``. It contains: @@ -136,6 +187,8 @@ It contains: * version * list of segments * compact +* shadow_index +* storage_quota_use The **integrity file** is a msgpacked file named ``integrity.``. It contains checksums of the index and hints files and is described in the @@ -159,17 +212,9 @@ Since writing a ``DELETE`` tag does not actually delete any data and thus does not free disk space any log-based data store will need a compaction strategy (somewhat analogous to a garbage collector). -Borg uses a simple forward compacting algorithm, -which avoids modifying existing segments. +Borg uses a simple forward compacting algorithm, which avoids modifying existing segments. Compaction runs when a commit is issued with ``compact=True`` parameter, e.g. by the ``borg compact`` command (unless the :ref:`append_only_mode` is active). -One client transaction can manifest as multiple physical transactions, -since compaction is transacted, too, and Borg does not distinguish between the two:: - - Perspective| Time --> - -----------+-------------- - Client | Begin transaction - Modify Data - Commit | (done) - Repository | Begin transaction - Modify Data - Commit | Compact segments - Commit | (done) The compaction algorithm requires two inputs in addition to the segments themselves: @@ -181,9 +226,6 @@ The compaction algorithm requires two inputs in addition to the segments themsel to be stored as well. Therefore, Borg stores a mapping ``(segment id,) -> (number of sparse bytes,)``. - The 1.0.x series used a simpler non-conditional algorithm, - which only required the list of sparse segments. Thus, - it only stored a list, not the mapping described above. (ii) Each segment's reference count, which indicates how many live objects are in a segment. This is not strictly required to perform the algorithm. Rather, it is used to validate that a segment is unused before deleting it. If the algorithm is incorrect, or the reference @@ -192,14 +234,7 @@ The compaction algorithm requires two inputs in addition to the segments themsel These two pieces of information are stored in the hints file (`hints.N`) next to the index (`index.N`). -When loading a hints file, Borg checks the version contained in the file. -The 1.0.x series writes version 1 of the format (with the segments list instead -of the mapping, mentioned above). Since Borg 1.0.4, version 2 is read as well. -The 1.1.x series writes version 2 of the format and reads either version. -When reading a version 1 hints file, Borg 1.1.x will -read all sparse segments to determine their sparsity. - -This process may take some time if a repository has been kept in append-only mode +Compaction may take some time if a repository has been kept in append-only mode or ``borg compact`` has not been used for a longer time, which both has caused the number of sparse segments to grow. @@ -320,17 +355,16 @@ or modified. It looks like this: .. code-block:: python { - b'version': 1, - b'timestamp': b'2017-05-05T12:42:23.042864', - b'item_keys': [b'acl_access', b'acl_default', ...], - b'config': {}, - b'archives': { - b'2017-05-05-system-backup': { - b'id': b'<32 byte binary object ID>', - b'time': b'2017-05-05T12:42:22.942864', + 'version': 1, + 'timestamp': '2017-05-05T12:42:23.042864', + 'item_keys': ['acl_access', 'acl_default', ...], + 'config': {}, + 'archives': { + '2017-05-05-system-backup': { + 'id': b'<32 byte binary object ID>', + 'time': '2017-05-05T12:42:22.942864', }, }, - b'tam': ..., } The *version* field can be either 1 or 2. The versions differ in the @@ -344,13 +378,8 @@ the repository. It is used by *borg check*, which verifies that all keys in all items are a subset of these keys. Thus, an older version of *borg check* supporting this mechanism can correctly detect keys introduced in later versions. -The *tam* key is part of the :ref:`tertiary authentication mechanism ` -(formerly known as "tertiary authentication for metadata") and authenticates -the manifest, since an ID check is not possible. - *config* is a general-purpose location for additional metadata. All versions -of Borg preserve its contents (it may have been a better place for *item_keys*, -which is not preserved by unaware Borg versions, releases predating 1.0.4). +of Borg preserve its contents. Feature flags +++++++++++++ @@ -384,15 +413,15 @@ The *config* key stores the feature flags enabled on a repository: .. code-block:: python config = { - b'feature_flags': { - b'read': { - b'mandatory': [b'some_feature'], + 'feature_flags': { + 'read': { + 'mandatory': ['some_feature'], }, - b'check': { - b'mandatory': [b'other_feature'], + 'check': { + 'mandatory': ['other_feature'], } - b'write': ..., - b'delete': ... + 'write': ..., + 'delete': ... }, } @@ -505,8 +534,9 @@ The archive object itself further contains some metadata: When :ref:`borg_check` rebuilds the manifest (e.g. if it was corrupted) and finds more than one archive object with the same name, it adds a counter to the name in the manifest, but leaves the *name* field of the archives as it was. -* *items*, a list of chunk IDs containing item metadata (size: count * ~34B) -* *cmdline*, the command line which was used to create the archive +* *item_ptrs*, a list of "pointer chunk" IDs. + Each "pointer chunk" contains a list of chunk IDs of item metadata. +* *command_line*, the command line which was used to create the archive * *hostname* * *username* * *time* and *time_end* are the start and end timestamps, respectively @@ -515,34 +545,6 @@ The archive object itself further contains some metadata: This is used by :ref:`borg_recreate` to determine whether a given archive needs rechunking. * Some other pieces of information related to recreate. -.. _archive_limitation: - -.. rubric:: Note about archive limitations - -The archive is currently stored as a single object in the repository -and thus limited in size to MAX_OBJECT_SIZE (20MiB). - -As one chunk list entry is ~40B, that means we can reference ~500.000 item -metadata stream chunks per archive. - -Each item metadata stream chunk is ~128kiB (see hardcoded ITEMS_CHUNKER_PARAMS). - -So that means the whole item metadata stream is limited to ~64GiB chunks. -If compression is used, the amount of storable metadata is bigger - by the -compression factor. - -If the medium size of an item entry is 100B (small size file, no ACLs/xattrs), -that means a limit of ~640 million files/directories per archive. - -If the medium size of an item entry is 2kB (~100MB size files or more -ACLs/xattrs), the limit will be ~32 million files/directories per archive. - -If one tries to create an archive object bigger than MAX_OBJECT_SIZE, a fatal -IntegrityError will be raised. - -A workaround is to create multiple archives with fewer items each, see -also :issue:`1452`. - .. _item: Items @@ -558,9 +560,10 @@ dictionary created by the ``Item`` class that contains: * uid * gid * mode (item type + permissions) -* source (for symlinks, and for hardlinks within one archive) +* source (for symlinks) +* hlid (for hardlinks) * rdev (for device files) -* mtime, atime, ctime in nanoseconds +* mtime, atime, ctime, birthtime in nanoseconds * xattrs * acl (various OS-dependent fields) * flags @@ -671,15 +674,15 @@ In memory, the files cache is a key -> value mapping (a Python *dict*) and conta - file inode number - file size - - file mtime_ns + - file ctime_ns (or mtime_ns) - age (0 [newest], 1, 2, 3, ..., BORG_FILES_CACHE_TTL - 1) - list of chunk ids representing the file's contents To determine whether a file has not changed, cached values are looked up via the key in the mapping and compared to the current file attribute values. -If the file's size, mtime_ns and inode number is still the same, it is -considered to not have changed. In that case, we check that all file content +If the file's size, timestamp and inode number is still the same, it is +considered not to have changed. In that case, we check that all file content chunks are (still) present in the repository (we check that via the chunks cache). @@ -696,7 +699,7 @@ different files, as a single path may not be unique across different archives in different setups. Not all filesystems have stable inode numbers. If that is the case, borg can -be told to ignore the inode number in the check via --ignore-inode. +be told to ignore the inode number in the check via --files-cache. The age value is used for cache management. If a file is "seen" in a backup run, its age is reset to 0, otherwise its age is incremented by one. @@ -727,7 +730,6 @@ The chunks cache is a key -> value mapping and contains: - reference count - size - - encrypted/compressed size The chunks cache is a HashIndex_. Due to some restrictions of HashIndex, the reference count of each given chunk is limited to a constant, MAX_VALUE @@ -745,9 +747,9 @@ Here is the estimated memory usage of Borg - it's complicated:: chunk_size ~= 2 ^ HASH_MASK_BITS (for buzhash chunker, BLOCK_SIZE for fixed chunker) chunk_count ~= total_file_size / chunk_size - repo_index_usage = chunk_count * 40 + repo_index_usage = chunk_count * 48 - chunks_cache_usage = chunk_count * 44 + chunks_cache_usage = chunk_count * 40 files_cache_usage = total_file_count * 240 + chunk_count * 80 @@ -785,7 +787,7 @@ For small hash tables, we start with a growth factor of 2, which comes down to E.g. backing up a total count of 1 Mi (IEC binary prefix i.e. 2^20) files with a total size of 1TiB. -a) with ``create --chunker-params buzhash,10,23,16,4095`` (custom, like borg < 1.0 or attic): +a) with ``create --chunker-params buzhash,10,23,16,4095`` (custom): mem_usage = 2.8GiB @@ -810,7 +812,7 @@ bucket is reached. This particular mode of operation is open addressing with linear probing. When the hash table is filled to 75%, its size is grown. When it's -emptied to 25%, its size is shrinked. Operations on it have a variable +emptied to 25%, its size is shrunken. Operations on it have a variable complexity between constant and linear with low factor, and memory overhead varies between 33% and 300%. @@ -860,29 +862,47 @@ HashIndex is implemented in C and wrapped with Cython in a class-based interface The Cython wrapper checks every passed value against these reserved values and raises an AssertionError if they are used. +.. _data-encryption: + Encryption ---------- .. seealso:: The :ref:`borgcrypto` section for an in-depth review. -AES_-256 is used in CTR mode (so no need for padding). A 64 bit initialization -vector is used, a MAC is computed on the encrypted chunk -and both are stored in the chunk. Encryption and MAC use two different keys. -Each chunk consists of ``TYPE(1)`` + ``MAC(32)`` + ``NONCE(8)`` + ``CIPHERTEXT``: +AEAD modes +~~~~~~~~~~ + +For new repositories, borg only uses modern AEAD ciphers: AES-OCB or CHACHA20-POLY1305. -.. figure:: encryption.png +For each borg invocation, a new sessionkey is derived from the borg key material +and the 48bit IV starts from 0 again (both ciphers internally add a 32bit counter +to our IV, so we'll just count up by 1 per chunk). + +The encryption layout is best seen at the bottom of this diagram: + +.. figure:: encryption-aead.png :figwidth: 100% :width: 100% -In AES-CTR mode you can think of the IV as the start value for the counter. -The counter itself is incremented by one after each 16 byte block. -The IV/counter is not required to be random but it must NEVER be reused. -So to accomplish this Borg initializes the encryption counter to be -higher than any previously used counter value before encrypting new data. +No special IV/counter management is needed here due to the use of session keys. -To reduce payload size, only 8 bytes of the 16 bytes nonce is saved in the -payload, the first 8 bytes are always zeros. This does not affect security but -limits the maximum repository capacity to only 295 exabytes (2**64 * 16 bytes). +A 48 bit IV is way more than needed: If you only backed up 4kiB chunks (2^12B), +the IV would "limit" the data encrypted in one session to 2^(12+48)B == 2.3 exabytes, +meaning you would run against other limitations (RAM, storage, time) way before that. +In practice, chunks are usually bigger, for big files even much bigger, giving an +even higher limit. + +Legacy modes +~~~~~~~~~~~~ + +Old repositories (which used AES-CTR mode) are supported read-only to be able to +``borg transfer`` their archives to new repositories (which use AEAD modes). + +AES-CTR mode is not supported for new repositories and the related code will be +removed in a future release. + +Both modes +~~~~~~~~~~ Encryption keys (and other secrets) are kept either in a key file on the client ('keyfile' mode) or in the repository config on the server ('repokey' mode). @@ -900,40 +920,34 @@ Key files .. seealso:: The :ref:`key_encryption` section for an in-depth review of the key encryption. -When initialized with the ``init -e keyfile`` command, Borg -needs an associated file in ``$HOME/.config/borg/keys`` to read and write -the repository. The format is based on msgpack_, base64 encoding and -PBKDF2_ SHA256 hashing, which is then encoded again in a msgpack_. +When initializing a repository with one of the "keyfile" encryption modes, +Borg creates an associated key file in ``$HOME/.config/borg/keys``. -The same data structure is also used in the "repokey" modes, which store -it in the repository in the configuration file. +The same key is also used in the "repokey" modes, which store it in the repository +in the configuration file. The internal data structure is as follows: version - currently always an integer, 1 + currently always an integer, 2 repository_id the ``id`` field in the ``config`` ``INI`` file of the repository. -enc_key - the key used to encrypt data with AES (256 bits) - -enc_hmac_key - the key used to HMAC the encrypted data (256 bits) +crypt_key + the initial key material used for the AEAD crypto (512 bits) id_key - the key used to HMAC the plaintext chunk data to compute the chunk's id + the key used to MAC the plaintext chunk data to compute the chunk's id chunk_seed the seed for the buzhash chunking table (signed 32 bit integer) These fields are packed using msgpack_. The utf-8 encoded passphrase -is processed with PBKDF2_ (SHA256_, 100000 iterations, random 256 bit salt) -to derive a 256 bit key encryption key (KEK). +is processed with argon2_ to derive a 256 bit key encryption key (KEK). -A `HMAC-SHA256`_ checksum of the packed fields is generated with the KEK, -then the KEK is also used to encrypt the same packed fields using AES-CTR. +Then the KEK is used to encrypt and authenticate the packed data using +the chacha20-poly1305 AEAD cipher. The result is stored in a another msgpack_ formatted as follows: @@ -943,15 +957,12 @@ version salt random 256 bits salt used to process the passphrase -iterations - number of iterations used to process the passphrase (currently 100000) +argon2_* + some parameters for the argon2 kdf algorithm - the hashing algorithm used to process the passphrase and do the HMAC - checksum (currently the string ``sha256``) - -hash - HMAC-SHA256 of the *plaintext* of the packed fields. + the algorithms used to process the passphrase + (currently the string ``argon2 chacha20-poly1305``) data The encrypted, packed fields. @@ -961,18 +972,25 @@ key file, wrapped using the standard ``textwrap`` module with a header. The header is a single line with a MAGIC string, a space and a hexadecimal representation of the repository id. +.. _data-compression: + Compression ----------- -Borg supports the following compression methods: +Borg supports the following compression methods, each identified by a ctype value +in the range between 0 and 255 (and augmented by a clevel 0..255 value for the +compression level): -- none (no compression, pass through data 1:1) -- lz4 (low compression, but super fast) +- none (no compression, pass through data 1:1), identified by 0x00 +- lz4 (low compression, but super fast), identified by 0x01 - zstd (level 1-22 offering a wide range: level 1 is lower compression and high - speed, level 22 is higher compression and lower speed) - since borg 1.1.4 + speed, level 22 is higher compression and lower speed) - identified by 0x03 - zlib (level 0-9, level 0 is no compression [but still adding zlib overhead], - level 1 is low, level 9 is high compression) -- lzma (level 0-9, level 0 is low, level 9 is high compression). + level 1 is low, level 9 is high compression), identified by 0x05 +- lzma (level 0-9, level 0 is low, level 9 is high compression), identified + by 0x02. + +The type byte is followed by a byte indicating the compression level. Speed: none > lz4 > zlib > lzma, lz4 > zstd Compression: lzma > zlib > lz4 > none, zstd > lz4 @@ -989,7 +1007,7 @@ while doing no compression at all (none) is a operation that takes no time, it likely will need to store more data to the storage compared to using lz4. The time needed to transfer and store the additional data might be much more than if you had used lz4 (which is super fast, but still might compress your -data about 2:1). This is assuming your data is compressible (if you backup +data about 2:1). This is assuming your data is compressible (if you back up already compressed data, trying to compress them at backup time is usually pointless). @@ -1183,9 +1201,9 @@ transaction ID in the file names. Integrity data is stored in a third file .. code-block:: python { - b'version': 2, - b'hints': b'{"algorithm": "XXH64", "digests": {"final": "411208db2aa13f1a"}}', - b'index': b'{"algorithm": "XXH64", "digests": {"HashHeader": "846b7315f91b8e48", "final": "cb3e26cadc173e40"}}' + 'version': 2, + 'hints': '{"algorithm": "XXH64", "digests": {"final": "411208db2aa13f1a"}}', + 'index': '{"algorithm": "XXH64", "digests": {"HashHeader": "846b7315f91b8e48", "final": "cb3e26cadc173e40"}}' } The *version* key started at 2, the same version used for the hints. Since Borg has @@ -1210,3 +1228,28 @@ For example, using 1.1 on a repository, noticing corruption or similar issues an ``borg-1.0 check --repair``, which rewrites the index and hints, results in this situation. Borg 1.1 would erroneously report checksum errors in the hints and/or index files and trigger an automatic rebuild of these files. + +HardLinkManager and the hlid concept +------------------------------------ + +Dealing with hard links needs some extra care, implemented in borg within the HardLinkManager +class: + +- At archive creation time, fs items with st_nlink > 1 indicate that they are a member of + a group of hardlinks all pointing to the same inode. For such fs items, the archived item + includes a hlid attribute (hardlink id), which is computed like H(st_dev, st_ino). Thus, + if archived items have the same hlid value, they pointed to the same inode and form a + group of hardlinks. Besides that, nothing special is done for any member of the group + of hardlinks, meaning that e.g. for regular files, each archived item will have a + chunks list. +- At extraction time, the presence of a hlid attribute indicates that there might be more + hardlinks coming, pointing to the same content (inode), thus borg will remember the "hlid + to extracted path" mapping, so it will know the correct path for extracting (hardlinking) + the next hardlink of that group / with the same hlid. +- This symmetric approach (each item has all the information, e.g. the chunks list) + simplifies dealing with such items a lot, especially for partial extraction, for the + FUSE filesystem, etc. +- This is different from the asymmetric approach of old borg versions (< 2.0) and also from + tar which have the concept of a main item (first hardlink, has the content) and content-less + secondary items with by-name back references for each subsequent hardlink, causing lots + of complications when dealing with them. diff --git a/docs/internals/encryption-aead.odg b/docs/internals/encryption-aead.odg new file mode 100644 index 0000000000..5caa55afdf Binary files /dev/null and b/docs/internals/encryption-aead.odg differ diff --git a/docs/internals/encryption-aead.png b/docs/internals/encryption-aead.png new file mode 100644 index 0000000000..771afd4a2b Binary files /dev/null and b/docs/internals/encryption-aead.png differ diff --git a/docs/internals/encryption.png b/docs/internals/encryption.png deleted file mode 100644 index e35120748a..0000000000 Binary files a/docs/internals/encryption.png and /dev/null differ diff --git a/docs/internals/encryption.vsd b/docs/internals/encryption.vsd deleted file mode 100644 index 22507a1792..0000000000 Binary files a/docs/internals/encryption.vsd and /dev/null differ diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 9e093c6b0c..08e2f4f4a1 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -10,7 +10,7 @@ Borg does not have a public API on the Python level. That does not keep you from but does mean that there are no release-to-release guarantees on what you might find in that package, not even for point releases (1.1.x), and there is no documentation beyond the code and the internals documents. -Borg does on the other hand provide an API on a command-line level. In other words, a frontend should to +Borg does on the other hand provide an API on a command-line level. In other words, a frontend should (for example) create a backup archive just invoke :ref:`borg_create`, give commandline parameters/options as needed and parse JSON output from borg. @@ -18,9 +18,6 @@ Important: JSON output is expected to be UTF-8, but currently borg depends on th for that (must be a UTF-8 locale and *not* "C" or "ascii"), so that Python will choose to encode to UTF-8. The same applies to any inputs read by borg, they are expected to be UTF-8 encoded also. -We consider this a bug (see :issue:`2273`) and might fix it later, so borg will use UTF-8 independent of -the locale. - On POSIX systems, you can usually set environment vars to choose a UTF-8 locale: :: @@ -29,6 +26,53 @@ On POSIX systems, you can usually set environment vars to choose a UTF-8 locale: export LC_CTYPE=en_US.UTF-8 +Another way to get Python's stdin/stdout/stderr streams to use UTF-8 encoding (without having +a UTF-8 locale / LANG / LC_CTYPE) is: + +:: + + export PYTHONIOENCODING=utf-8 + + +See :issue:`2273` for more details. + + +Dealing with non-unicode byte sequences and JSON limitations +------------------------------------------------------------ + +Paths on POSIX systems can have arbitrary bytes in them (except 0x00 which is used as string terminator in C). + +Nowadays, UTF-8 encoded paths (which decode to valid unicode) are the usual thing, but a lot of systems +still have paths from the past, when other, non-unicode codings were used. Especially old Samba shares often +have wild mixtures of misc. encodings, sometimes even very broken stuff. + +borg deals with such non-unicode paths ("with funny/broken characters") by decoding such byte sequences using +UTF-8 coding and "surrogateescape" error handling mode, which maps invalid bytes to special unicode code points +(surrogate escapes). When encoding such a unicode string back to a byte sequence, the original byte sequence +will be reproduced exactly. + +JSON should only contain valid unicode text without any surrogate escapes, so we can't just directly have a +surrogate-escaped path in JSON ("path" is only one example, this also affects other text-like content). + +Borg deals with this situation like this (since borg 2.0): + +For a valid unicode path (no surrogate escapes), the JSON will only have "path": path. + +For a non-unicode path (with surrogate escapes), the JSON will have 2 entries: + +- "path": path_approximation (pure valid unicode, all invalid bytes will show up as "?") +- "path_b64": path_bytes_base64_encoded (if you decode the base64, you get the original path byte string) + +JSON users need to pick whatever suits their needs best. The suggested procedure (shown for "path") is: + +- check if there is a "path_b64" key. +- if it is there, you will know that the original bytes path did not cleanly UTF-8-decode into unicode (has + some invalid bytes) and that the string given by the "path" key is only an approximation, but not the precise + path. if you need precision, you must base64-decode the value of "path_b64" and deal with the arbitrary byte + string you'll get. if an approximation is fine, use the value of the "path" key. +- if it is not there, the value of the "path" key is all you need (the original bytes path is its UTF-8 encoding). + + Logging ------- @@ -40,8 +84,6 @@ where each line is a JSON object. The *type* key of the object determines its ot parsing error will be printed in plain text, because logging set-up happens after all arguments are parsed. -Since JSON can only encode text, any string representing a file system path may miss non-text parts. - The following types are in use. Progress information is governed by the usual rules for progress information, it is not produced unless ``--progress`` is specified. @@ -50,17 +92,20 @@ archive_progress The following keys exist, each represents the current progress. original_size - Original size of data processed so far (before compression and deduplication) + Original size of data processed so far (before compression and deduplication, may be empty/absent) compressed_size - Compressed size + Compressed size (may be empty/absent) deduplicated_size - Deduplicated size + Deduplicated size (may be empty/absent) nfiles - Number of (regular) files processed so far + Number of (regular) files processed so far (may be empty/absent) path - Current path + Current path (may be empty/absent) time Unix timestamp (float) + finished + boolean indicating whether the operation has finished, only the last object for an *operation* + can have this property set to *true*. progress_message A message-based progress information with no concrete progress information, just a message @@ -90,12 +135,14 @@ progress_percent can have this property set to *true*. message A formatted progress message, this will include the percentage and perhaps other information + (absent for finished == true) current - Current value (always less-or-equal to *total*) + Current value (always less-or-equal to *total*, absent for finished == true) info Array that describes the current item, may be *null*, contents depend on *msgid* + (absent for finished == true) total - Total value + Total value (absent for finished == true) time Unix timestamp (float) @@ -208,7 +255,7 @@ Passphrase prompts should be handled differently. Use the environment variables and *BORG_NEW_PASSPHRASE* (see :ref:`env_vars` for reference) to pass passphrases to Borg, don't use the interactive passphrase prompts. -When setting a new passphrase (:ref:`borg_init`, :ref:`borg_key_change-passphrase`) normally +When setting a new passphrase (:ref:`borg_rcreate`, :ref:`borg_key_change-passphrase`) normally Borg prompts whether it should display the passphrase. This can be suppressed by setting the environment variable *BORG_DISPLAY_PASSPHRASE* to *no*. @@ -252,7 +299,7 @@ last_modified The *encryption* key, if present, contains: mode - Textual encryption mode name (same as :ref:`borg_init` ``--encryption`` names) + Textual encryption mode name (same as :ref:`borg_rcreate` ``--encryption`` names) keyfile Path to the local key file used for access. Depending on *mode* this key may be absent. @@ -269,12 +316,8 @@ stats Number of unique chunks total_size Total uncompressed size of all chunks multiplied with their reference counts - total_csize - Total compressed and encrypted size of all chunks multiplied with their reference counts unique_size Uncompressed size of all chunks - unique_csize - Compressed and encrypted size of all chunks .. highlight: json @@ -285,10 +328,8 @@ Example *borg info* output:: "path": "/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", "stats": { "total_chunks": 511533, - "total_csize": 17948017540, "total_size": 22635749792, "total_unique_chunks": 54892, - "unique_csize": 1920405405, "unique_size": 2449675468 } }, @@ -332,11 +373,6 @@ stats Deduplicated size (against the current repository, not when the archive was created) nfiles Number of regular files in the archive -limits - Object describing the utilization of Borg limits - - max_archive_size - Float between 0 and 1 describing how large this archive is relative to the maximum size allowed by Borg command_line Array of strings of the command line that created the archive @@ -406,9 +442,6 @@ The same archive with more information (``borg info --last 1 --json``):: "end": "2017-02-27T12:27:20.789123", "hostname": "host", "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", - "limits": { - "max_archive_size": 0.0001330855110409714 - }, "name": "host-system-backup-2017-02-27", "start": "2017-02-27T12:27:20.789123", "stats": { @@ -424,10 +457,8 @@ The same archive with more information (``borg info --last 1 --json``):: "path": "/home/user/.cache/borg/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", "stats": { "total_chunks": 511533, - "total_csize": 17948017540, "total_size": 22635749792, "total_unique_chunks": 54892, - "unique_csize": 1920405405, "unique_size": 2449675468 } }, @@ -449,14 +480,15 @@ Refer to the *borg list* documentation for the available keys and their meaning. Example (excerpt) of ``borg list --json-lines``:: - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0} - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "target": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "target": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0} + Archive Differencing ++++++++++++++++++++ Each archive difference item (file contents, user/group/mode) output by :ref:`borg_diff` is represented by an *ItemDiff* object. -The propertiese of an *ItemDiff* object are: +The properties of an *ItemDiff* object are: path: The filename/path of the *Item* (file, directory, symlink). @@ -465,13 +497,13 @@ changes: A list of *Change* objects describing the changes made to the item in the two archives. For example, there will be two changes if the contents of a file are changed, and its ownership are changed. -The *Change* object can contain a number of properties depending on the type of change that occured. +The *Change* object can contain a number of properties depending on the type of change that occurred. If a 'property' is not required for the type of change, it is not output. The possible properties of a *Change* object are: type: The **type** property is always present. It identifies the type of change and will be one of these values: - + - *modified* - file contents changed. - *added* - the file was added. - *removed* - the file was removed. @@ -495,26 +527,26 @@ added: removed: See **added** property. - + old_mode: If **type** == '*mode*', then **old_mode** and **new_mode** provide the mode and permissions changes. new_mode: See **old_mode** property. - + old_user: If **type** == '*owner*', then **old_user**, **new_user**, **old_group** and **new_group** provide the user and group ownership changes. old_group: See **old_user** property. - + new_user: See **old_user** property. - + new_group: See **old_user** property. - + Example (excerpt) of ``borg diff --json-lines``:: @@ -533,92 +565,173 @@ Message IDs are strings that essentially give a log message or operation a name, full text, since texts change more frequently. Message IDs are unambiguous and reduce the need to parse log messages. -Assigned message IDs are: +Assigned message IDs and related error RCs (exit codes) are: .. See scripts/errorlist.py; this is slightly edited. Errors - Archive.AlreadyExists + Error rc: 2 traceback: no + Error: {} + ErrorWithTraceback rc: 2 traceback: yes + Error: {} + + Buffer.MemoryLimitExceeded rc: 2 traceback: no + Requested buffer size {} is above the limit of {}. + EfficientCollectionQueue.SizeUnderflow rc: 2 traceback: no + Could not pop_front first {} elements, collection only has {} elements.. + RTError rc: 2 traceback: no + Runtime Error: {} + + CancelledByUser rc: 3 traceback: no + Cancelled by user. + + CommandError rc: 4 traceback: no + Command Error: {} + PlaceholderError rc: 5 traceback: no + Formatting Error: "{}".format({}): {}({}) + InvalidPlaceholder rc: 6 traceback: no + Invalid placeholder "{}" in string: {} + + Repository.AlreadyExists rc: 10 traceback: no + A repository already exists at {}. + Repository.CheckNeeded rc: 12 traceback: yes + Inconsistency detected. Please run "borg check {}". + Repository.DoesNotExist rc: 13 traceback: no + Repository {} does not exist. + Repository.InsufficientFreeSpaceError rc: 14 traceback: no + Insufficient free space to complete transaction (required: {}, available: {}). + Repository.InvalidRepository rc: 15 traceback: no + {} is not a valid repository. Check repo config. + Repository.InvalidRepositoryConfig rc: 16 traceback: no + {} does not have a valid configuration. Check repo config [{}]. + Repository.ObjectNotFound rc: 17 traceback: yes + Object with key {} not found in repository {}. + Repository.ParentPathDoesNotExist rc: 18 traceback: no + The parent path of the repo directory [{}] does not exist. + Repository.PathAlreadyExists rc: 19 traceback: no + There is already something at {}. + Repository.StorageQuotaExceeded rc: 20 traceback: no + The storage quota ({}) has been exceeded ({}). Try deleting some archives. + Repository.PathPermissionDenied rc: 21 traceback: no + Permission denied to {}. + + MandatoryFeatureUnsupported rc: 25 traceback: no + Unsupported repository feature(s) {}. A newer version of borg is required to access this repository. + NoManifestError rc: 26 traceback: no + Repository has no manifest. + UnsupportedManifestError rc: 27 traceback: no + Unsupported manifest envelope. A newer version is required to access this repository. + + Archive.AlreadyExists rc: 30 traceback: no Archive {} already exists - Archive.DoesNotExist + Archive.DoesNotExist rc: 31 traceback: no Archive {} does not exist - Archive.IncompatibleFilesystemEncodingError + Archive.IncompatibleFilesystemEncodingError rc: 32 traceback: no Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable. - Cache.CacheInitAbortedError - Cache initialization aborted - Cache.EncryptionMethodMismatch - Repository encryption method changed since last access, refusing to continue - Cache.RepositoryAccessAborted - Repository access aborted - Cache.RepositoryIDNotUnique - Cache is newer than repository - do you have multiple, independently updated repos with same ID? - Cache.RepositoryReplay - Cache is newer than repository - this is either an attack or unsafe (multiple repos with same ID) - Buffer.MemoryLimitExceeded - Requested buffer size {} is above the limit of {}. - ExtensionModuleError - The Borg binary extension modules do not seem to be properly installed - IntegrityError - Data integrity error: {} - NoManifestError - Repository has no manifest. - PlaceholderError - Formatting Error: "{}".format({}): {}({}) - KeyfileInvalidError - Invalid key file for repository {} found in {}. - KeyfileMismatchError + + KeyfileInvalidError rc: 40 traceback: no + Invalid key data for repository {} found in {}. + KeyfileMismatchError rc: 41 traceback: no Mismatch between repository {} and key file {}. - KeyfileNotFoundError + KeyfileNotFoundError rc: 42 traceback: no No key file for repository {} found in {}. - PassphraseWrong - passphrase supplied in BORG_PASSPHRASE is incorrect - PasswordRetriesExceeded - exceeded the maximum password retries - RepoKeyNotFoundError - No key entry found in the config of repository {}. - UnsupportedManifestError - Unsupported manifest envelope. A newer version is required to access this repository. - UnsupportedPayloadError - Unsupported payload type {}. A newer version is required to access this repository. - NotABorgKeyFile + NotABorgKeyFile rc: 43 traceback: no This file is not a borg key backup, aborting. - RepoIdMismatch + RepoKeyNotFoundError rc: 44 traceback: no + No key entry found in the config of repository {}. + RepoIdMismatch rc: 45 traceback: no This key backup seems to be for a different backup repository, aborting. - UnencryptedRepo - Keymanagement not available for unencrypted repositories. - UnknownKeyType - Keytype {0} is unknown. - LockError + UnencryptedRepo rc: 46 traceback: no + Key management not available for unencrypted repositories. + UnknownKeyType rc: 47 traceback: no + Key type {0} is unknown. + UnsupportedPayloadError rc: 48 traceback: no + Unsupported payload type {}. A newer version is required to access this repository. + UnsupportedKeyFormatError rc: 49 traceback:no + Your borg key is stored in an unsupported format. Try using a newer version of borg. + + + NoPassphraseFailure rc: 50 traceback: no + can not acquire a passphrase: {} + PasscommandFailure rc: 51 traceback: no + passcommand supplied in BORG_PASSCOMMAND failed: {} + PassphraseWrong rc: 52 traceback: no + passphrase supplied in BORG_PASSPHRASE, by BORG_PASSCOMMAND or via BORG_PASSPHRASE_FD is incorrect. + PasswordRetriesExceeded rc: 53 traceback: no + exceeded the maximum password retries + + Cache.CacheInitAbortedError rc: 60 traceback: no + Cache initialization aborted + Cache.EncryptionMethodMismatch rc: 61 traceback: no + Repository encryption method changed since last access, refusing to continue + Cache.RepositoryAccessAborted rc: 62 traceback: no + Repository access aborted + Cache.RepositoryIDNotUnique rc: 63 traceback: no + Cache is newer than repository - do you have multiple, independently updated repos with same ID? + Cache.RepositoryReplay rc: 64 traceback: no + Cache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID) + + LockError rc: 70 traceback: no Failed to acquire the lock {}. - LockErrorT + LockErrorT rc: 71 traceback: yes Failed to acquire the lock {}. - ConnectionClosed + LockFailed rc: 72 traceback: yes + Failed to create/acquire the lock {} ({}). + LockTimeout rc: 73 traceback: no + Failed to create/acquire the lock {} (timeout). + NotLocked rc: 74 traceback: yes + Failed to release the lock {} (was not locked). + NotMyLock rc: 75 traceback: yes + Failed to release the lock {} (was/is locked, but not by me). + + ConnectionClosed rc: 80 traceback: no Connection closed by remote host - InvalidRPCMethod + ConnectionClosedWithHint rc: 81 traceback: no + Connection closed by remote host. {} + InvalidRPCMethod rc: 82 traceback: no RPC method {} is not valid - PathNotAllowed - Repository path not allowed - RemoteRepository.RPCServerOutdated + PathNotAllowed rc: 83 traceback: no + Repository path not allowed: {} + RemoteRepository.RPCServerOutdated rc: 84 traceback: no Borg server is too old for {}. Required version {} - UnexpectedRPCDataFormatFromClient + UnexpectedRPCDataFormatFromClient rc: 85 traceback: no Borg {}: Got unexpected RPC data format from client. - UnexpectedRPCDataFormatFromServer + UnexpectedRPCDataFormatFromServer rc: 86 traceback: no Got unexpected RPC data format from server: {} - Repository.AlreadyExists - Repository {} already exists. - Repository.CheckNeeded - Inconsistency detected. Please run "borg check {}". - Repository.DoesNotExist - Repository {} does not exist. - Repository.InsufficientFreeSpaceError - Insufficient free space to complete transaction (required: {}, available: {}). - Repository.InvalidRepository - {} is not a valid repository. Check repo config. - Repository.AtticRepository - Attic repository detected. Please run "borg upgrade {}". - Repository.ObjectNotFound - Object with key {} not found in repository {}. + ConnectionBrokenWithHint rc: 87 traceback: no + Connection to remote host is broken. {} + + IntegrityError rc: 90 traceback: yes + Data integrity error: {} + FileIntegrityError rc: 91 traceback: yes + File failed integrity check: {} + DecompressionError rc: 92 traceback: yes + Decompression error: {} + + +Warnings + BorgWarning rc: 1 + Warning: {} + BackupWarning rc: 1 + {}: {} + + FileChangedWarning rc: 100 + {}: file changed while we backed it up + IncludePatternNeverMatchedWarning rc: 101 + Include pattern '{}' never matched. + BackupError rc: 102 + {}: backup error + BackupRaceConditionError rc: 103 + {}: file type or inode changed while we backed it up (race condition, skipped file) + BackupOSError rc: 104 + {}: {} + BackupPermissionError rc: 105 + {}: {} + BackupIOError rc: 106 + {}: {} + BackupFileNotFoundError rc: 107 + {}: {} Operations - cache.begin_transaction @@ -632,6 +745,7 @@ Operations - repository.check - check.verify_data - check.rebuild_manifest + - check.rebuild_refcounts - extract *info* is one string element, the name of the path currently extracted. @@ -649,4 +763,4 @@ Prompts BORG_CHECK_I_KNOW_WHAT_I_AM_DOING For "This is a potentially dangerous function..." (check --repair) BORG_DELETE_I_KNOW_WHAT_I_AM_DOING - For "You requested to completely DELETE the repository *including* all archives it contains:" + For "You requested to DELETE the repository completely *including* all archives it contains:" diff --git a/docs/internals/object-graph.odg b/docs/internals/object-graph.odg new file mode 100644 index 0000000000..c4060e6ee4 Binary files /dev/null and b/docs/internals/object-graph.odg differ diff --git a/docs/internals/object-graph.png b/docs/internals/object-graph.png index 8a153428cd..f0d1f0001c 100644 Binary files a/docs/internals/object-graph.png and b/docs/internals/object-graph.png differ diff --git a/docs/internals/object-graph.vsd b/docs/internals/object-graph.vsd deleted file mode 100644 index 70989e1dd3..0000000000 Binary files a/docs/internals/object-graph.vsd and /dev/null differ diff --git a/docs/internals/security.rst b/docs/internals/security.rst index 3f79e66bd7..68bd647b29 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -1,3 +1,5 @@ +.. include:: ../global.rst.inc + .. somewhat surprisingly the "bash" highlighter gives nice results with the pseudo-code notation used in the "Encryption" section. @@ -38,9 +40,6 @@ Under these circumstances Borg guarantees that the attacker cannot The attacker can always impose a denial of service per definition (he could forbid connections to the repository, or delete it entirely). -When the above attack model is extended to include multiple clients -independently updating the same repository, then Borg fails to provide -confidentiality (i.e. guarantees 3) and 4) do not apply any more). .. _security_structural_auth: @@ -61,50 +60,33 @@ In other words, the object ID itself only authenticates the plaintext of the object and not its context or meaning. The latter is established by a different object referring to an object ID, thereby assigning a particular meaning to an object. For example, an archive item contains a list of object IDs that -represent packed file metadata. On their own it's not clear that these objects +represent packed file metadata. On their own, it's not clear that these objects would represent what they do, but by the archive item referring to them in a particular part of its own data structure assigns this meaning. This results in a directed acyclic graph of authentication from the manifest to the data chunks of individual files. -.. _tam_description: - -.. rubric:: Authenticating the manifest - -Since the manifest has a fixed ID (000...000) the aforementioned authentication -does not apply to it, indeed, cannot apply to it; it is impossible to authenticate -the root node of a DAG through its edges, since the root node has no incoming edges. - -With the scheme as described so far an attacker could easily replace the manifest, -therefore Borg includes a tertiary authentication mechanism (TAM) that is applied -to the manifest since version 1.0.9 (see :ref:`tam_vuln`). +Above used to be all for borg 1.x and was the reason why it needed the +tertiary authentication mechanism (TAM) for manifest and archives. -TAM works by deriving a separate key through HKDF_ from the other encryption and -authentication keys and calculating the HMAC of the metadata to authenticate [#]_:: +borg 2 now stores the ro_type ("meaning") of a repo object's data into that +object's metadata (like e.g.: manifest vs. archive vs. user file content data). +When loading data from the repo, borg verifies that the type of object it got +matches the type it wanted. borg 2 does not use TAMs any more. - # RANDOM(n) returns n random bytes - salt = RANDOM(64) +As both the object's metadata and data are AEAD encrypted and also bound to +the object ID (via giving the ID as AAD), there is no way an attacker (without +access to the borg key) could change the type of the object or move content +to a different object ID. - ikm = id_key || enc_key || enc_hmac_key - # *context* depends on the operation, for manifest authentication it is - # the ASCII string "borg-metadata-authentication-manifest". - tam_key = HKDF-SHA-512(ikm, salt, context) +This effectively 'anchors' the manifest (and also other metadata, like archives) +to the key, which is controlled by the client, thereby anchoring the entire DAG, +making it impossible for an attacker to add, remove or modify any part of the +DAG without Borg being able to detect the tampering. - # *data* is a dict-like structure - data[hmac] = zeroes - packed = pack(data) - data[hmac] = HMAC(tam_key, packed) - packed_authenticated = pack(data) - -Since an attacker cannot gain access to this key and also cannot make the -client authenticate arbitrary data using this mechanism, the attacker is unable -to forge the authentication. - -This effectively 'anchors' the manifest to the key, which is controlled by the -client, thereby anchoring the entire DAG, making it impossible for an attacker -to add, remove or modify any part of the DAG without Borg being able to detect -the tampering. +Passphrase notes +---------------- Note that when using BORG_PASSPHRASE the attacker cannot swap the *entire* repository against a new repository with e.g. repokey mode and no passphrase, @@ -114,106 +96,95 @@ However, interactively a user might not notice this kind of attack immediately, if she assumes that the reason for the absent passphrase prompt is a set BORG_PASSPHRASE. See issue :issue:`2169` for details. -.. [#] The reason why the authentication tag is stored in the packed - data itself is that older Borg versions can still read the - manifest this way, while a changed layout would have broken - compatibility. - .. _security_encryption: Encryption ---------- -Encryption is currently based on the Encrypt-then-MAC construction, -which is generally seen as the most robust way to create an authenticated -encryption scheme from encryption and message authentication primitives. +AEAD modes +~~~~~~~~~~ + +Modes: --encryption (repokey|keyfile)-[blake2-](aes-ocb|chacha20-poly1305) -Every operation (encryption, MAC / authentication, chunk ID derivation) -uses independent, random keys generated by `os.urandom`_ [#]_. +Supported: borg 2.0+ -Borg does not support unauthenticated encryption -- only authenticated encryption -schemes are supported. No unauthenticated encryption schemes will be added -in the future. +Encryption with these modes is based on AEAD ciphers (authenticated encryption +with associated data) and session keys. -Depending on the chosen mode (see :ref:`borg_init`) different primitives are used: +Depending on the chosen mode (see :ref:`borg_rcreate`) different AEAD ciphers are used: -- The actual encryption is currently always AES-256 in CTR mode. The - counter is added in plaintext, since it is needed for decryption, - and is also tracked locally on the client to avoid counter reuse. +- AES-256-OCB - super fast, single-pass algorithm IF you have hw accelerated AES. +- chacha20-poly1305 - very fast, purely software based AEAD cipher. -- The authentication primitive is either HMAC-SHA-256 or BLAKE2b-256 - in a keyed mode. HMAC-SHA-256 uses 256 bit keys, while BLAKE2b-256 - uses 512 bit keys. +The chunk ID is derived via a MAC over the plaintext (mac key taken from borg key): - The latter is secure not only because BLAKE2b itself is not - susceptible to `length extension`_, but also since it truncates the - hash output from 512 bits to 256 bits, which would make the - construction safe even if BLAKE2b were broken regarding length - extension or similar attacks. +- HMAC-SHA256 - super fast IF you have hw accelerated SHA256 (see section "Encryption" below). +- Blake2b - very fast, purely software based algorithm. -- The primitive used for authentication is always the same primitive - that is used for deriving the chunk ID, but they are always - used with independent keys. +For each borg invocation, a new session id is generated by `os.urandom`_. + +From that session id, the initial key material (ikm, taken from the borg key) +and an application and cipher specific salt, borg derives a session key using a +"one-step KDF" based on just sha256. + +For each session key, IVs (nonces) are generated by a counter which increments for +each encrypted message. + +Session:: + + sessionid = os.urandom(24) + domain = "borg-session-key-CIPHERNAME" + sessionkey = sha256(crypt_key + sessionid + domain) + message_iv = 0 Encryption:: - id = AUTHENTICATOR(id_key, data) + id = MAC(id_key, data) compressed = compress(data) - iv = reserve_iv() - encrypted = AES-256-CTR(enc_key, 8-null-bytes || iv, compressed) - authenticated = type-byte || AUTHENTICATOR(enc_hmac_key, encrypted) || iv || encrypted - + header = type-byte || 00h || message_iv || sessionid + aad = id || header + message_iv++ + encrypted, auth_tag = AEAD_encrypt(session_key, message_iv, compressed, aad) + authenticated = header || auth_tag || encrypted Decryption:: - # Given: input *authenticated* data, possibly a *chunk-id* to assert - type-byte, mac, iv, encrypted = SPLIT(authenticated) + # Given: input *authenticated* data and a *chunk-id* to assert + type-byte, past_message_iv, past_sessionid, auth_tag, encrypted = SPLIT(authenticated) ASSERT(type-byte is correct) - ASSERT( CONSTANT-TIME-COMPARISON( mac, AUTHENTICATOR(enc_hmac_key, encrypted) ) ) - decrypted = AES-256-CTR(enc_key, 8-null-bytes || iv, encrypted) + domain = "borg-session-key-CIPHERNAME" + past_key = sha256(crypt_key + past_sessionid + domain) + + decrypted = AEAD_decrypt(past_key, past_message_iv, authenticated) + decompressed = decompress(decrypted) - ASSERT( CONSTANT-TIME-COMPARISON( chunk-id, AUTHENTICATOR(id_key, decompressed) ) ) - -The client needs to track which counter values have been used, since -encrypting a chunk requires a starting counter value and no two chunks -may have overlapping counter ranges (otherwise the bitwise XOR of the -overlapping plaintexts is revealed). - -The client does not directly track the counter value, because it -changes often (with each encrypted chunk), instead it commits a -"reservation" to the security database and the repository by taking -the current counter value and adding 4 GiB / 16 bytes (the block size) -to the counter. Thus the client only needs to commit a new reservation -every few gigabytes of encrypted data. - -This mechanism also avoids reusing counter values in case the client -crashes or the connection to the repository is severed, since any -reservation would have been committed to both the security database -and the repository before any data is encrypted. Borg uses its -standard mechanism (SaveFile) to ensure that reservations are durable -(on most hardware / storage systems), therefore a crash of the -client's host would not impact tracking of reservations. - -However, this design is not infallible, and requires synchronization -between clients, which is handled through the repository. Therefore in -a multiple-client scenario a repository can trick a client into -reusing counter values by ignoring counter reservations and replaying -the manifest (which will fail if the client has seen a more recent -manifest or has a more recent nonce reservation). If the repository is -untrusted, but a trusted synchronization channel exists between -clients, the security database could be synchronized between them over -said trusted channel. This is not part of Borg's functionality. - -.. [#] Using the :ref:`borg key migrate-to-repokey ` - command a user can convert repositories created using Attic in "passphrase" - mode to "repokey" mode. In this case the keys were directly derived from - the user's passphrase at some point using PBKDF2. - - Borg does not support "passphrase" mode otherwise any more. +Notable: + +- More modern and often faster AEAD ciphers instead of self-assembled stuff. +- Due to the usage of session keys, IVs (nonces) do not need special care here as + they did for the legacy encryption modes. +- The id is now also input into the authentication tag computation. + This strongly associates the id with the written data (== associates the key with + the value). When later reading the data for some id, authentication will only + succeed if what we get was really written by us for that id. + + +Legacy modes +~~~~~~~~~~~~ + +Modes: --encryption (repokey|keyfile)-[blake2] + +Supported: borg < 2.0 + +These were the AES-CTR based modes in previous borg versions. + +borg 2.0 does not support creating new repos using these modes, +but ``borg transfer`` can still read such existing repos. + .. _key_encryption: @@ -227,32 +198,23 @@ For offline storage of the encryption keys they are encrypted with a user-chosen passphrase. A 256 bit key encryption key (KEK) is derived from the passphrase -using PBKDF2-HMAC-SHA256 with a random 256 bit salt which is then used -to Encrypt-*and*-MAC (unlike the Encrypt-*then*-MAC approach used -otherwise) a packed representation of the keys with AES-256-CTR with a -constant initialization vector of 0. A HMAC-SHA256 of the plaintext is -generated using the same KEK and is stored alongside the ciphertext, -which is converted to base64 in its entirety. +using argon2_ with a random 256 bit salt. The KEK is then used +to Encrypt-*then*-MAC a packed representation of the keys using the +chacha20-poly1305 AEAD cipher and a constant IV == 0. +The ciphertext is then converted to base64. This base64 blob (commonly referred to as *keyblob*) is then stored in the key file or in the repository config (keyfile and repokey modes respectively). -This scheme, and specifically the use of a constant IV with the CTR -mode, is secure because an identical passphrase will result in a -different derived KEK for every key encryption due to the salt. - -The use of Encrypt-and-MAC instead of Encrypt-then-MAC is seen as -uncritical (but not ideal) here, since it is combined with AES-CTR mode, -which is not vulnerable to padding attacks. +The use of a constant IV is secure because an identical passphrase will +result in a different derived KEK for every key encryption due to the salt. .. seealso:: Refer to the :ref:`key_files` section for details on the format. - Refer to issue :issue:`747` for suggested improvements of the encryption - scheme and password-based key derivation. Implementations used -------------------- @@ -260,29 +222,16 @@ Implementations used We do not implement cryptographic primitives ourselves, but rely on widely used libraries providing them: -- AES-CTR and HMAC-SHA-256 from OpenSSL 1.0 / 1.1 are used, +- AES-OCB and CHACHA20-POLY1305 from OpenSSL 1.1 are used, which is also linked into the static binaries we provide. We think this is not an additional risk, since we don't ever use OpenSSL's networking, TLS or X.509 code, but only their primitives implemented in libcrypto. - SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used. - Borg requires a Python built with OpenSSL support (due to PBKDF2), therefore - these functions are delegated to OpenSSL by Python. -- HMAC, PBKDF2 and a constant-time comparison from Python's hmac_ standard - library module is used. While the HMAC implementation is written in Python, - the PBKDF2 implementation is provided by OpenSSL. The constant-time comparison - (``compare_digest``) is written in C and part of Python. - -Implemented cryptographic constructions are: - -- Encrypt-then-MAC based on AES-256-CTR and either HMAC-SHA-256 - or keyed BLAKE2b256 as described above under Encryption_. -- Encrypt-and-MAC based on AES-256-CTR and HMAC-SHA-256 - as described above under `Offline key security`_. -- HKDF_-SHA-512 +- HMAC and a constant-time comparison from Python's hmac_ standard library module are used. +- argon2 is used via argon2-cffi. .. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle -.. _HKDF: https://tools.ietf.org/html/rfc5869 .. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack .. _hashlib: https://docs.python.org/3/library/hashlib.html .. _hmac: https://docs.python.org/3/library/hmac.html @@ -303,7 +252,7 @@ SSH server -- Borg RPC does not contain *any* networking code. Networking is done by the SSH client running in a separate process, Borg only communicates over the standard pipes (stdout, stderr and stdin) with this process. This also means that Borg doesn't -have to directly use a SSH client (or SSH at all). For example, +have to use a SSH client directly (or SSH at all). For example, ``sudo`` or ``qrexec`` could be used as an intermediary. By using the system's SSH client and not implementing a @@ -380,13 +329,12 @@ Compression and Encryption Combining encryption with compression can be insecure in some contexts (e.g. online protocols). -There was some discussion about this in `github issue #1040`_ and for Borg some developers +There was some discussion about this in :issue:`1040` and for Borg some developers concluded this is no problem at all, some concluded this is hard and extremely slow to exploit and thus no problem in practice. No matter what, there is always the option not to use compression if you are worried about this. -.. _github issue #1040: https://github.com/borgbackup/borg/issues/1040 Fingerprinting ============== @@ -402,7 +350,7 @@ output of the chunker. The sizes of these stored chunks are influenced by the compression, encryption and authentication. buzhash chunker -+++++++++++++++ +~~~~~~~~~~~~~~~ The buzhash chunker chunks according to the input data, the chunker's parameters and the secret chunker seed (which all influence the chunk boundary @@ -413,7 +361,7 @@ chunk (identical content / size as the original file), bigger files result in multiple chunks. fixed chunker -+++++++++++++ +~~~~~~~~~~~~~ This chunker yields fixed sized chunks, with optional support of a differently sized header chunk. The last chunk is not required to have the full block size @@ -445,6 +393,24 @@ To summarize, this is making size-based fingerprinting difficult: - optional ``obfuscate`` pseudo compressor with different choices of algorithm and parameters +Secret key usage against fingerprinting +--------------------------------------- + +Borg uses the borg key also for chunking and chunk ID generation to protect against fingerprinting. +As usual for borg's attack model, the attacker is assumed to have access to a borg repository. + +The borg key includes a secret random chunk_seed which (together with the chunking algorithm) +determines the cutting places and thereby the length of the chunks cut. Because the attacker trying +a chunk length fingerprinting attack would use a different chunker secret than the borg setup being +attacked, they would not be able to determine the set of chunk lengths for a known set of files. + +The borg key also includes a secret random id_key. The chunk ID generation is not just using a simple +cryptographic hash like sha256 (because that would be insecure as an attacker could see the hashes of +small files that result only in 1 chunk in the repository). Instead, borg uses keyed hash (a MAC, +e.g. HMAC-SHA256) to compute the chunk ID from the content and the secret id_key. Thus, an attacker +can't compute the same chunk IDs for a known set of small files to determine whether these are stored +in the attacked repository. + Stored chunk proximity ---------------------- diff --git a/docs/internals/structure.odg b/docs/internals/structure.odg new file mode 100644 index 0000000000..d68e89d028 Binary files /dev/null and b/docs/internals/structure.odg differ diff --git a/docs/internals/structure.png b/docs/internals/structure.png index 69566003a8..5c17db6d5e 100644 Binary files a/docs/internals/structure.png and b/docs/internals/structure.png differ diff --git a/docs/internals/structure.vsd b/docs/internals/structure.vsd deleted file mode 100644 index 3c7ce0cfd4..0000000000 Binary files a/docs/internals/structure.vsd and /dev/null differ diff --git a/docs/man/borg-benchmark-cpu.1 b/docs/man/borg-benchmark-cpu.1 new file mode 100644 index 0000000000..50cca65ace --- /dev/null +++ b/docs/man/borg-benchmark-cpu.1 @@ -0,0 +1,57 @@ +.\" Man page generated from reStructuredText. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "BORG-BENCHMARK-CPU" 1 "2024-07-19" "" "borg backup tool" +.SH NAME +borg-benchmark-cpu \- Benchmark CPU bound operations. +.SH SYNOPSIS +.sp +borg [common options] benchmark cpu [options] +.SH DESCRIPTION +.sp +This command benchmarks misc. CPU bound borg operations. +.sp +It creates input data in memory, runs the operation and then displays throughput. +To reduce outside influence on the timings, please make sure to run this with: +.INDENT 0.0 +.IP \(bu 2 +an otherwise as idle as possible machine +.IP \(bu 2 +enough free memory so there will be no slow down due to paging activity +.UNINDENT +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index 041a88a218..e4efc752fa 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK-CRUD" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-BENCHMARK-CRUD" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. .SH SYNOPSIS .sp -borg [common options] benchmark crud [options] REPOSITORY PATH +borg [common options] benchmark crud [options] PATH .SH DESCRIPTION .sp This command benchmarks borg CRUD (create, read, update, delete) operations. @@ -88,9 +88,6 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B REPOSITORY -repository to use for benchmark (must exist) -.TP .B PATH path were to create benchmark input data .UNINDENT diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 92b5515ef7..4dd0fcb5f4 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -27,12 +27,13 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BENCHMARK" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-BENCHMARK" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command .SH SYNOPSIS .nf borg [common options] benchmark crud ... +borg [common options] benchmark cpu ... .fi .sp .SH DESCRIPTION @@ -40,7 +41,7 @@ borg [common options] benchmark crud ... These commands do various benchmarks. .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-benchmark\-crud(1)\fP +\fIborg\-common(1)\fP, \fIborg\-benchmark\-crud(1)\fP, \fIborg\-benchmark\-cpu(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 31bc50bbab..3bb5d24419 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-BREAK-LOCK" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-BREAK-LOCK" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. .SH SYNOPSIS .sp -borg [common options] break\-lock [options] [REPOSITORY] +borg [common options] break\-lock [options] .SH DESCRIPTION .sp This command breaks the repository and cache locks. @@ -41,12 +41,6 @@ trying to access the Cache or the Repository. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY -repository for which to break the locks -.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 29265c0eab..b4040573c4 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -27,108 +27,134 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CHECK" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-CHECK" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency .SH SYNOPSIS .sp -borg [common options] check [options] [REPOSITORY_OR_ARCHIVE] +borg [common options] check [options] .SH DESCRIPTION .sp -The check command verifies the consistency of a repository and the corresponding archives. +The check command verifies the consistency of a repository and its archives. +It consists of two major steps: +.INDENT 0.0 +.IP 1. 3 +Checking the consistency of the repository itself. This includes checking +the segment magic headers, and both the metadata and data of all objects in +the segments. The read data is checked by size and CRC. Bit rot and other +types of accidental damage can be detected this way. Running the repository +check can be split into multiple partial checks using \fB\-\-max\-duration\fP\&. +When checking a remote repository, please note that the checks run on the +server and do not cause significant network traffic. +.IP 2. 3 +Checking consistency and correctness of the archive metadata and optionally +archive data (requires \fB\-\-verify\-data\fP). This includes ensuring that the +repository manifest exists, the archive metadata chunk is present, and that +all chunks referencing files (items) in the archive exist. This requires +reading archive and file metadata, but not data. To cryptographically verify +the file (content) data integrity pass \fB\-\-verify\-data\fP, but keep in mind +that this requires reading all data and is hence very time consuming. When +checking archives of a remote repository, archive checks run on the client +machine because they require decrypting data and therefore the encryption +key. +.UNINDENT .sp -check \-\-repair is a potentially dangerous function and might lead to data loss -(for kinds of corruption it is not capable of dealing with). BE VERY CAREFUL! +Both steps can also be run independently. Pass \fB\-\-repository\-only\fP to run the +repository checks only, or pass \fB\-\-archives\-only\fP to run the archive checks +only. .sp -Pursuant to the previous warning it is also highly recommended to test the -reliability of the hardware running this software with stress testing software -such as memory testers. Unreliable hardware can also lead to data loss especially -when this command is run in repair mode. +The \fB\-\-max\-duration\fP option can be used to split a long\-running repository +check into multiple partial checks. After the given number of seconds the check +is interrupted. The next partial check will continue where the previous one +stopped, until the full repository has been checked. Assuming a complete check +would take 7 hours, then running a daily check with \fB\-\-max\-duration=3600\fP +(1 hour) would result in one full repository check per week. Doing a full +repository check aborts any previous partial check; the next partial check will +restart from the beginning. With partial repository checks you can run neither +archive checks, nor enable repair mode. Consequently, if you want to use +\fB\-\-max\-duration\fP you must also pass \fB\-\-repository\-only\fP, and must not pass +\fB\-\-archives\-only\fP, nor \fB\-\-repair\fP\&. +.sp +\fBWarning:\fP Please note that partial repository checks (i.e. running it with +\fB\-\-max\-duration\fP) can only perform non\-cryptographic checksum checks on the +segment files. A full repository check (i.e. without \fB\-\-max\-duration\fP) can +also do a repository index check. Enabling partial repository checks excepts +archive checks for the same reason. Therefore partial checks may be useful with +very large repositories only where a full check would take too long. +.sp +The \fB\-\-verify\-data\fP option will perform a full integrity verification (as +opposed to checking the CRC32 of the segment) of data, which means reading the +data from the repository, decrypting and decompressing it. It is a complete +cryptographic verification and hence very time consuming, but will detect any +accidental and malicious corruption. Tamper\-resistance is only guaranteed for +encrypted repositories against attackers without access to the keys. You can +not use \fB\-\-verify\-data\fP with \fB\-\-repository\-only\fP\&. +.SS About repair mode +.sp +The check command is a readonly task by default. If any corruption is found, +Borg will report the issue and proceed with checking. To actually repair the +issues found, pass \fB\-\-repair\fP\&. .sp -First, the underlying repository data files are checked: +\fBNOTE:\fP .INDENT 0.0 -.IP \(bu 2 -For all segments, the segment magic header is checked. -.IP \(bu 2 -For all objects stored in the segments, all metadata (e.g. CRC and size) and -all data is read. The read data is checked by size and CRC. Bit rot and other -types of accidental damage can be detected this way. -.IP \(bu 2 -In repair mode, if an integrity error is detected in a segment, try to recover -as many objects from the segment as possible. -.IP \(bu 2 -In repair mode, make sure that the index is consistent with the data stored in -the segments. -.IP \(bu 2 -If checking a remote repo via \fBssh:\fP, the repo check is executed on the server -without causing significant network traffic. -.IP \(bu 2 -The repository check can be skipped using the \fB\-\-archives\-only\fP option. -.IP \(bu 2 -A repository check can be time consuming. Partial checks are possible with the -\fB\-\-max\-duration\fP option. +.INDENT 3.5 +\fB\-\-repair\fP is a \fBPOTENTIALLY DANGEROUS FEATURE\fP and might lead to data +loss! This does not just include data that was previously lost anyway, but +might include more data for kinds of corruption it is not capable of +dealing with. \fBBE VERY CAREFUL!\fP .UNINDENT +.UNINDENT +.sp +Pursuant to the previous warning it is also highly recommended to test the +reliability of the hardware running Borg with stress testing software. This +especially includes storage and memory testers. Unreliable hardware might lead +to additional data loss. +.sp +It is highly recommended to create a backup of your repository before running +in repair mode (i.e. running it with \fB\-\-repair\fP). +.sp +Repair mode will attempt to fix any corruptions found. Fixing corruptions does +not mean recovering lost data: Borg can not magically restore data lost due to +e.g. a hardware failure. Repairing a repository means sacrificing some data +for the sake of the repository as a whole and the remaining data. Hence it is, +by definition, a potentially lossy task. .sp -Second, the consistency and correctness of the archive metadata is verified: +In practice, repair mode hooks into both the repository and archive checks: .INDENT 0.0 -.IP \(bu 2 -Is the repo manifest present? If not, it is rebuilt from archive metadata -chunks (this requires reading and decrypting of all metadata and data). -.IP \(bu 2 -Check if archive metadata chunk is present; if not, remove archive from manifest. -.IP \(bu 2 -For all files (items) in the archive, for all chunks referenced by these -files, check if chunk is present. In repair mode, if a chunk is not present, -replace it with a same\-size replacement chunk of zeroes. If a previously lost -chunk reappears (e.g. via a later backup), in repair mode the all\-zero replacement -chunk will be replaced by the correct chunk. This requires reading of archive and -file metadata, but not data. -.IP \(bu 2 -In repair mode, when all the archives were checked, orphaned chunks are deleted -from the repo. One cause of orphaned chunks are input file related errors (like -read errors) in the archive creation process. -.IP \(bu 2 -In verify\-data mode, a complete cryptographic verification of the archive data -integrity is performed. This conflicts with \fB\-\-repository\-only\fP as this mode -only makes sense if the archive checks are enabled. The full details of this mode -are documented below. -.IP \(bu 2 -If checking a remote repo via \fBssh:\fP, the archive check is executed on the -client machine because it requires decryption, and this is always done client\-side -as key access is needed. -.IP \(bu 2 -The archive checks can be time consuming; they can be skipped using the -\fB\-\-repository\-only\fP option. +.IP 1. 3 +When checking the repository\(aqs consistency, repair mode will try to recover +as many objects from segments with integrity errors as possible, and ensure +that the index is consistent with the data stored in the segments. +.IP 2. 3 +When checking the consistency and correctness of archives, repair mode might +remove whole archives from the manifest if their archive metadata chunk is +corrupt or lost. On a chunk level (i.e. the contents of files), repair mode +will replace corrupt or lost chunks with a same\-size replacement chunk of +zeroes. If a previously zeroed chunk reappears, repair mode will restore +this lost chunk using the new chunk. Lastly, repair mode will also delete +orphaned chunks (e.g. caused by read errors while creating the archive). .UNINDENT .sp -The \fB\-\-max\-duration\fP option can be used to split a long\-running repository check -into multiple partial checks. After the given number of seconds the check is -interrupted. The next partial check will continue where the previous one stopped, -until the complete repository has been checked. Example: Assuming a full check took 7 -hours, then running a daily check with \-\-max\-duration=3600 (1 hour) resulted in one -full check per week. -.sp -Attention: Partial checks can only do way less checking than a full check (only the -CRC32 checks on segment file entries are done), and cannot be combined with the -\fB\-\-repair\fP option. Partial checks may therefore be useful only with very large -repositories where a full check took too long. Doing a full repository check aborts a -partial check; the next partial check will restart from the beginning. -.sp -The \fB\-\-verify\-data\fP option will perform a full integrity verification (as opposed to -checking the CRC32 of the segment) of data, which means reading the data from the -repository, decrypting and decompressing it. This is a cryptographic verification, -which will detect (accidental) corruption. For encrypted repositories it is -tamper\-resistant as well, unless the attacker has access to the keys. It is also very -slow. +Most steps taken by repair mode have a one\-time effect on the repository, like +removing a lost archive from the repository. However, replacing a corrupt or +lost chunk with an all\-zero replacement will have an ongoing effect on the +repository: When attempting to extract a file referencing an all\-zero chunk, +the \fBextract\fP command will distinctly warn about it. The FUSE filesystem +created by the \fBmount\fP command will reject reading such a \(dqzero\-patched\(dq +file unless a special mount option is given. +.sp +As mentioned earlier, Borg might be able to \(dqheal\(dq a \(dqzero\-patched\(dq file in +repair mode, if all its previously lost chunks reappear (e.g. via a later +backup). This is achieved by Borg not only keeping track of the all\-zero +replacement chunks, but also by keeping metadata about the lost chunks. In +repair mode Borg will check whether a previously lost chunk reappeared and will +replace the all\-zero replacement chunk by the reappeared chunk. If all lost +chunks of a \(dqzero\-patched\(dq file reappear, this effectively \(dqheals\(dq the file. +Consequently, if lost chunks were repaired earlier, it is advised to run +\fB\-\-repair\fP a second time after creating some new backups. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY_OR_ARCHIVE -repository or archive to check consistency of -.UNINDENT .SS optional arguments .INDENT 0.0 .TP @@ -144,29 +170,35 @@ perform cryptographic archive data integrity verification (conflicts with \fB\-\ .B \-\-repair attempt to repair any inconsistencies found .TP -.B \-\-save\-space -work slower, but using less space -.TP .BI \-\-max\-duration \ SECONDS do only a partial repo check for max. SECONDS seconds (Default: unlimited) .UNINDENT .SS Archive filters .INDENT 0.0 .TP -.BI \-P \ PREFIX\fR,\fB \ \-\-prefix \ PREFIX -only consider archive names starting with this prefix. -.TP -.BI \-a \ GLOB\fR,\fB \ \-\-glob\-archives \ GLOB -only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. +.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN +only consider archive names matching the pattern. see \(dqborg help match\-archives\(dq. .TP .BI \-\-sort\-by \ KEYS -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id; default is: timestamp .TP .BI \-\-first \ N consider first N archives after other filters were applied .TP .BI \-\-last \ N consider last N archives after other filters were applied +.TP +.BI \-\-oldest \ TIMESPAN +consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newest \ TIMESPAN +consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-older \ TIMESPAN +consider archives older than (now \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newer \ TIMESPAN +consider archives newer than (now \- TIMESPAN), e.g. 7d or 12m. .UNINDENT .SH SEE ALSO .sp diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 9b6f8dfa99..ad10ac4a9c 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMMON" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-COMMON" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands .SH SYNOPSIS @@ -79,28 +79,25 @@ show/log the return code (rc) set umask to M (local only, default: 0077) .TP .BI \-\-remote\-path \ PATH -use PATH as borg executable on the remote (default: "borg") -.TP -.BI \-\-remote\-ratelimit \ RATE -deprecated, use \fB\-\-upload\-ratelimit\fP instead +use PATH as borg executable on the remote (default: \(dqborg\(dq) .TP .BI \-\-upload\-ratelimit \ RATE set network upload rate limit in kiByte/s (default: 0=unlimited) .TP -.BI \-\-remote\-buffer \ UPLOAD_BUFFER -deprecated, use \fB\-\-upload\-buffer\fP instead -.TP .BI \-\-upload\-buffer \ UPLOAD_BUFFER set network upload buffer size in MiB. (default: 0=no buffer) .TP -.B \-\-consider\-part\-files -treat part files like normal files (e.g. to list/extract them) -.TP .BI \-\-debug\-profile \ FILE -Write execution profile in Borg format into FILE. For local use a Python\-compatible file can be generated by suffixing FILE with ".pyprof". +Write execution profile in Borg format into FILE. For local use a Python\-compatible file can be generated by suffixing FILE with \(dq.pyprof\(dq. .TP .BI \-\-rsh \ RSH Use this command to connect to the \(aqborg serve\(aq process (default: \(aqssh\(aq) +.TP +.BI \-\-socket \ PATH +Use UNIX DOMAIN (IPC) socket at PATH for client/server communication with socket: protocol. +.TP +.BI \-r \ REPO\fR,\fB \ \-\-repo \ REPO +repository to use .UNINDENT .SH SEE ALSO .sp diff --git a/docs/man/borg-compact.1 b/docs/man/borg-compact.1 index f159a0875f..31c5d1cdb2 100644 --- a/docs/man/borg-compact.1 +++ b/docs/man/borg-compact.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPACT" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-COMPACT" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-compact \- compact segment files in the repository .SH SYNOPSIS .sp -borg [common options] compact [options] [REPOSITORY] +borg [common options] compact [options] .SH DESCRIPTION .sp This command frees repository space by compacting segments. @@ -51,26 +51,13 @@ A segment is compacted if the amount of saved space is above the percentage valu given by the \fB\-\-threshold\fP option. If omitted, a threshold of 10% is used. When using \fB\-\-verbose\fP, borg will output an estimate of the freed space. .sp -After upgrading borg (server) to 1.2+, you can use \fBborg compact \-\-cleanup\-commits\fP -to clean up the numerous 17byte commit\-only segments that borg 1.1 did not clean up -due to a bug. It is enough to do that once per repository. -.sp See \fIseparate_compaction\fP in Additional Notes for more details. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY -repository to compact -.UNINDENT .SS optional arguments .INDENT 0.0 .TP -.B \-\-cleanup\-commits -cleanup commit\-only 17\-byte segment files -.TP .BI \-\-threshold \ PERCENT set minimum threshold for saved space in PERCENT (Default: 10) .UNINDENT @@ -81,10 +68,7 @@ set minimum threshold for saved space in PERCENT (Default: 10) .nf .ft C # compact segments and free repo disk space -$ borg compact /path/to/repo - -# same as above plus clean up 17byte commit\-only segments -$ borg compact \-\-cleanup\-commits /path/to/repo +$ borg compact .ft P .fi .UNINDENT diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 274aa86ed8..22c69ce710 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-COMPRESSION" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-COMPRESSION" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression .SH DESCRIPTION @@ -53,20 +53,19 @@ Do not compress. Use lz4 compression. Very high speed, very low compression. (default) .TP .B zstd[,L] -Use zstd ("zstandard") compression, a modern wide\-range algorithm. +Use zstd (\(dqzstandard\(dq) compression, a modern wide\-range algorithm. If you do not explicitly give the compression level L (ranging from 1 to 22), it will use level 3. -Archives compressed with zstd are not compatible with borg < 1.1.4. .TP .B zlib[,L] -Use zlib ("gz") compression. Medium speed, medium compression. +Use zlib (\(dqgz\(dq) compression. Medium speed, medium compression. If you do not explicitly give the compression level L (ranging from 0 to 9), it will use level 6. -Giving level 0 (means "no compression", but still has zlib protocol -overhead) is usually pointless, you better use "none" compression. +Giving level 0 (means \(dqno compression\(dq, but still has zlib protocol +overhead) is usually pointless, you better use \(dqnone\(dq compression. .TP .B lzma[,L] -Use lzma ("xz") compression. Low speed, high compression. +Use lzma (\(dqxz\(dq) compression. Low speed, high compression. If you do not explicitly give the compression level L (ranging from 0 to 9), it will use level 6. Giving levels above 6 is pointless and counterproductive because it does @@ -76,35 +75,76 @@ lots of CPU cycles and RAM. .B auto,C[,L] Use a built\-in heuristic to decide per chunk whether to compress or not. The heuristic tries with lz4 whether the data is compressible. -For incompressible data, it will not use compression (uses "none"). +For incompressible data, it will not use compression (uses \(dqnone\(dq). For compressible data, it uses the given C[,L] compression \- with C[,L] being any valid compression specifier. .TP .B obfuscate,SPEC,C[,L] Use compressed\-size obfuscation to make fingerprinting attacks based on -the observable stored chunk size more difficult. -Note: -\- you must combine this with encryption or it won\(aqt make any sense. -\- your repo size will be bigger, of course. +the observable stored chunk size more difficult. Note: +.INDENT 7.0 +.IP \(bu 2 +You must combine this with encryption, or it won\(aqt make any sense. +.IP \(bu 2 +Your repo size will be bigger, of course. +.IP \(bu 2 +A chunk is limited by the constant \fBMAX_DATA_SIZE\fP (cur. ~20MiB). +.UNINDENT +.sp +The SPEC value determines how the size obfuscation works: .sp -The SPEC value will determine how the size obfuscation will work: +\fIRelative random reciprocal size variation\fP (multiplicative) .sp -Relative random reciprocal size variation: Size will increase by a factor, relative to the compressed data size. -Smaller factors are often used, larger factors rarely. -1: factor 0.01 .. 100.0 -2: factor 0.1 .. 1000.0 -3: factor 1.0 .. 10000.0 -4: factor 10.0 .. 100000.0 -5: factor 100.0 .. 1000000.0 -6: factor 1000.0 .. 10000000.0 +Smaller factors are used often, larger factors rarely. +.sp +Available factors: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +1: 0.01 .. 100 +2: 0.1 .. 1,000 +3: 1 .. 10,000 +4: 10 .. 100,000 +5: 100 .. 1,000,000 +6: 1,000 .. 10,000,000 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Example probabilities for SPEC \fB1\fP: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +90 % 0.01 .. 0.1 + 9 % 0.1 .. 1 + 0.9 % 1 .. 10 + 0.09% 10 .. 100 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fIRandomly sized padding up to the given size\fP (additive) +.INDENT 7.0 +.INDENT 3.5 .sp -Add a randomly sized padding up to the given size: -110: 1kiB +.nf +.ft C +110: 1kiB (2 ^ (SPEC \- 100)) \&... 120: 1MiB \&... 123: 8MiB (max.) +.ft P +.fi +.UNINDENT +.UNINDENT .UNINDENT .sp Examples: @@ -120,7 +160,7 @@ borg create \-\-compression zlib REPO::ARCHIVE data borg create \-\-compression zlib,1 REPO::ARCHIVE data borg create \-\-compression auto,lzma,6 REPO::ARCHIVE data borg create \-\-compression auto,lzma ... -borg create \-\-compression obfuscate,3,none ... +borg create \-\-compression obfuscate,110,none ... borg create \-\-compression obfuscate,3,auto,zstd,10 ... borg create \-\-compression obfuscate,2,zstd,6 ... .ft P diff --git a/docs/man/borg-config.1 b/docs/man/borg-config.1 index 24e0d708ca..6cdac59f00 100644 --- a/docs/man/borg-config.1 +++ b/docs/man/borg-config.1 @@ -27,22 +27,22 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CONFIG" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-CONFIG" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-config \- get, set, and delete values in a repository or cache config file .SH SYNOPSIS .sp -borg [common options] config [options] [REPOSITORY] [NAME] [VALUE] +borg [common options] config [options] [NAME] [VALUE] .SH DESCRIPTION .sp This command gets and sets options in a local repository or cache config file. For security reasons, this command only works on local repositories. .sp To delete a config value entirely, use \fB\-\-delete\fP\&. To list the values -of the configuration file or the default values, use \fB\-\-list\fP\&. To get and existing +of the configuration file or the default values, use \fB\-\-list\fP\&. To get an existing key, pass only the key name. To set a key, pass both the key name and -the new value. Keys can be specified in the format "section.name" or -simply "name"; the section will default to "repository" and "cache" for +the new value. Keys can be specified in the format \(dqsection.name\(dq or +simply \(dqname\(dq; the section will default to \(dqrepository\(dq and \(dqcache\(dq for the repo and cache configs, respectively. .sp By default, borg config manipulates the repository config file. Using \fB\-\-cache\fP @@ -53,9 +53,6 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B REPOSITORY -repository to configure -.TP .B NAME name of config key .TP @@ -90,13 +87,13 @@ making changes! .nf .ft C # find cache directory -$ cd ~/.cache/borg/$(borg config /path/to/repo id) +$ cd ~/.cache/borg/$(borg config id) # reserve some space -$ borg config /path/to/repo additional_free_space 2G +$ borg config additional_free_space 2G # make a repo append\-only -$ borg config /path/to/repo append_only 1 +$ borg config append_only 1 .ft P .fi .UNINDENT diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 3815b178c3..d0616c639d 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-CREATE" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-CREATE" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-create \- Create new archive .SH SYNOPSIS .sp -borg [common options] create [options] ARCHIVE [PATH...] +borg [common options] create [options] NAME [PATH...] .SH DESCRIPTION .sp This command creates a backup archive containing all files found while recursively @@ -40,6 +40,11 @@ traversing all paths specified. Paths are added to the archive as they are given that means if relative paths are desired, the command has to be run from the correct directory. .sp +The slashdot hack in paths (recursion roots) is triggered by using \fB/./\fP: +\fB/this/gets/stripped/./this/gets/archived\fP means to process that fs object, but +strip the prefix on the left side of \fB\&./\fP from the archived items (in this case, +\fBthis/gets/archived\fP will be the path in the archived item). +.sp When giving \(aq\-\(aq as path, borg will read data from standard input and create a file \(aqstdin\(aq in the created archive from that data. In some cases it\(aqs more appropriate to use \-\-content\-from\-command, however. See section \fIReading from @@ -107,13 +112,13 @@ creation of a new archive to ensure fast operation. This is because the file cac is used to determine changed files quickly uses absolute filenames. If this is not possible, consider creating a bind mount to a stable location. .sp -The \fB\-\-progress\fP option shows (from left to right) Original, Compressed and Deduplicated -(O, C and D, respectively), then the Number of files (N) processed so far, followed by -the currently processed path. +The \fB\-\-progress\fP option shows (from left to right) Original and (uncompressed) +deduplicated size (O and U respectively), then the Number of files (N) processed so far, +followed by the currently processed path. .sp When using \fB\-\-stats\fP, you will get some statistics about how much data was -added \- the "This Archive" deduplicated size there is most interesting as that is -how much your repository will grow. Please note that the "All archives" stats refer to +added \- the \(dqThis Archive\(dq deduplicated size there is most interesting as that is +how much your repository will grow. Please note that the \(dqAll archives\(dq stats refer to the state after creation. Also, the \fB\-\-stats\fP and \fB\-\-dry\-run\fP options are mutually exclusive because the data is not actually compressed and deduplicated during a dry run. .sp @@ -126,8 +131,8 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B ARCHIVE -name of archive to create (must be also a valid directory name) +.B NAME +specify the archive name .TP .B PATH paths to archive @@ -151,16 +156,22 @@ only display items with the given status characters (see description) output stats as JSON. Implies \fB\-\-stats\fP\&. .TP .B \-\-no\-cache\-sync -experimental: do not synchronize the cache. Implies not using the files cache. +experimental: do not synchronize the chunks cache. +.TP +.B \-\-no\-cache\-sync\-forced +experimental: do not synchronize the chunks cache (forced). +.TP +.B \-\-prefer\-adhoc\-cache +experimental: prefer AdHocCache (w/o files cache) over AdHocWithFilesCache (with files cache). .TP .BI \-\-stdin\-name \ NAME use NAME in archive for stdin data (default: \(aqstdin\(aq) .TP .BI \-\-stdin\-user \ USER -set user USER in archive for stdin data (default: \(aqroot\(aq) +set user USER in archive for stdin data (default: do not store user/uid) .TP .BI \-\-stdin\-group \ GROUP -set group GROUP in archive for stdin data (default: \(aqwheel\(aq) +set group GROUP in archive for stdin data (default: do not store group/gid) .TP .BI \-\-stdin\-mode \ M set mode to M in archive for stdin data (default: 0660) @@ -169,15 +180,15 @@ set mode to M in archive for stdin data (default: 0660) interpret PATH as command and store its stdout. See also section Reading from stdin below. .TP .B \-\-paths\-from\-stdin -read DELIM\-separated list of paths to backup from stdin. Will not recurse into directories. +read DELIM\-separated list of paths to back up from stdin. All control is external: it will back up all files given \- no more, no less. .TP .B \-\-paths\-from\-command interpret PATH as command and treat its output as \fB\-\-paths\-from\-stdin\fP .TP .BI \-\-paths\-delimiter \ DELIM -set path delimiter for \fB\-\-paths\-from\-stdin\fP and \fB\-\-paths\-from\-command\fP (default: n) +set path delimiter for \fB\-\-paths\-from\-stdin\fP and \fB\-\-paths\-from\-command\fP (default: \fB\en\fP) .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN @@ -208,17 +219,11 @@ exclude files flagged NODUMP .INDENT 0.0 .TP .B \-x\fP,\fB \-\-one\-file\-system -stay in the same file system and do not store mount points of other file systems. This might behave different from your expectations, see the docs. -.TP -.B \-\-numeric\-owner -deprecated, use \fB\-\-numeric\-ids\fP instead +stay in the same file system and do not store mount points of other file systems \- this might behave different from your expectations, see the description below. .TP .B \-\-numeric\-ids only store numeric user and group identifiers .TP -.B \-\-noatime -do not store atime into archive -.TP .B \-\-atime do store atime into archive .TP @@ -228,9 +233,6 @@ do not store ctime into archive .B \-\-nobirthtime do not store birthtime (creation date) into archive .TP -.B \-\-nobsdflags -deprecated, use \fB\-\-noflags\fP instead -.TP .B \-\-noflags do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive .TP @@ -256,16 +258,19 @@ open and read block and char device files as well as FIFOs as if they were regul add a comment text to the archive .TP .BI \-\-timestamp \ TIMESTAMP -manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). Alternatively, give a reference file/directory. +manually specify the archive creation date/time (yyyy\-mm\-ddThh:mm:ss[(+|\-)HH:MM] format, (+|\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory. .TP .BI \-c \ SECONDS\fR,\fB \ \-\-checkpoint\-interval \ SECONDS write checkpoint every SECONDS seconds (Default: 1800) .TP +.BI \-\-checkpoint\-volume \ BYTES +write checkpoint every BYTES bytes (Default: 0, meaning no volume based checkpointing) +.TP .BI \-\-chunker\-params \ PARAMS specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095 .TP .BI \-C \ COMPRESSION\fR,\fB \ \-\-compression \ COMPRESSION -select compression algorithm, see the output of the "borg help compression" command for details. +select compression algorithm, see the output of the \(dqborg help compression\(dq command for details. .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -273,87 +278,87 @@ select compression algorithm, see the output of the "borg help compression" comm .sp .nf .ft C -# Backup ~/Documents into an archive named "my\-documents" -$ borg create /path/to/repo::my\-documents ~/Documents +# Backup ~/Documents into an archive named \(dqmy\-documents\(dq +$ borg create my\-documents ~/Documents # same, but list all files as we process them -$ borg create \-\-list /path/to/repo::my\-documents ~/Documents +$ borg create \-\-list my\-documents ~/Documents + +# Backup /mnt/disk/docs, but strip path prefix using the slashdot hack +$ borg create /path/to/repo::docs /mnt/disk/./docs # Backup ~/Documents and ~/src but exclude pyc files -$ borg create /path/to/repo::my\-files \e +$ borg create my\-files \e ~/Documents \e ~/src \e \-\-exclude \(aq*.pyc\(aq # Backup home directories excluding image thumbnails (i.e. only # /home//.thumbnails is excluded, not /home/*/*/.thumbnails etc.) -$ borg create /path/to/repo::my\-files /home \e - \-\-exclude \(aqsh:/home/*/.thumbnails\(aq +$ borg create my\-files /home \-\-exclude \(aqsh:home/*/.thumbnails\(aq -# Backup the root filesystem into an archive named "root\-YYYY\-MM\-DD" +# Backup the root filesystem into an archive named \(dqroot\-YYYY\-MM\-DD\(dq # use zlib compression (good, but slow) \- default is lz4 (fast, low compression ratio) -$ borg create \-C zlib,6 \-\-one\-file\-system /path/to/repo::root\-{now:%Y\-%m\-%d} / +$ borg create \-C zlib,6 \-\-one\-file\-system root\-{now:%Y\-%m\-%d} / -# Backup onto a remote host ("push" style) via ssh to port 2222, -# logging in as user "borg" and storing into /path/to/repo -$ borg create ssh://borg@backup.example.org:2222/path/to/repo::{fqdn}\-root\-{now} / +# Backup into an archive name like FQDN\-root\-TIMESTAMP +$ borg create \(aq{fqdn}\-root\-{now}\(aq / -# Backup a remote host locally ("pull" style) using sshfs +# Backup a remote host locally (\(dqpull\(dq style) using sshfs $ mkdir sshfs\-mount $ sshfs root@example.com:/ sshfs\-mount $ cd sshfs\-mount -$ borg create /path/to/repo::example.com\-root\-{now:%Y\-%m\-%d} . +$ borg create example.com\-root\-{now:%Y\-%m\-%d} . $ cd .. $ fusermount \-u sshfs\-mount # Make a big effort in fine granular deduplication (big chunk management -# overhead, needs a lot of RAM and disk space, see formula in internals -# docs \- same parameters as borg < 1.0 or attic): -$ borg create \-\-chunker\-params buzhash,10,23,16,4095 /path/to/repo::small /smallstuff +# overhead, needs a lot of RAM and disk space, see formula in internals docs): +$ borg create \-\-chunker\-params buzhash,10,23,16,4095 small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) -$ borg create \-\-read\-special \-\-chunker\-params fixed,4194304 /path/to/repo::my\-sdx /dev/sdX +$ borg create \-\-read\-special \-\-chunker\-params fixed,4194304 my\-sdx /dev/sdX # Backup a sparse disk image (must not be active/in use/mounted at that time) -$ borg create \-\-sparse \-\-chunker\-params fixed,4194304 /path/to/repo::my\-disk my\-disk.raw +$ borg create \-\-sparse \-\-chunker\-params fixed,4194304 my\-disk my\-disk.raw # No compression (none) -$ borg create \-\-compression none /path/to/repo::arch ~ +$ borg create \-\-compression none arch ~ # Super fast, low compression (lz4, default) -$ borg create /path/to/repo::arch ~ +$ borg create arch ~ # Less fast, higher compression (zlib, N = 0..9) -$ borg create \-\-compression zlib,N /path/to/repo::arch ~ +$ borg create \-\-compression zlib,N arch ~ # Even slower, even higher compression (lzma, N = 0..9) -$ borg create \-\-compression lzma,N /path/to/repo::arch ~ +$ borg create \-\-compression lzma,N arch ~ # Only compress compressible data with lzma,N (N = 0..9) -$ borg create \-\-compression auto,lzma,N /path/to/repo::arch ~ +$ borg create \-\-compression auto,lzma,N arch ~ # Use short hostname, user name and current time in archive name -$ borg create /path/to/repo::{hostname}\-{user}\-{now} ~ +$ borg create \(aq{hostname}\-{user}\-{now}\(aq ~ # Similar, use the same datetime format that is default as of borg 1.1 -$ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S} ~ +$ borg create \(aq{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S}\(aq ~ # As above, but add nanoseconds -$ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f} ~ +$ borg create \(aq{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f}\(aq ~ # Backing up relative paths by moving into the correct directory first $ cd /home/user/Documents -# The root directory of the archive will be "projectA" -$ borg create /path/to/repo::daily\-projectA\-{now:%Y\-%m\-%d} projectA +# The root directory of the archive will be \(dqprojectA\(dq +$ borg create \(aqdaily\-projectA\-{now:%Y\-%m\-%d}\(aq projectA # Use external command to determine files to archive -# Use \-\-paths\-from\-stdin with find to only backup files less than 1MB in size -$ find ~ \-size \-1000k | borg create \-\-paths\-from\-stdin /path/to/repo::small\-files\-only -# Use \-\-paths\-from\-command with find to only backup files from a given user -$ borg create \-\-paths\-from\-command /path/to/repo::joes\-files \-\- find /srv/samba/shared \-user joe +# Use \-\-paths\-from\-stdin with find to back up only files less than 1MB in size +$ find ~ \-size \-1000k | borg create \-\-paths\-from\-stdin small\-files\-only +# Use \-\-paths\-from\-command with find to back up files from only a given user +$ borg create \-\-paths\-from\-command joes\-files \-\- find /srv/samba/shared \-user joe # Use \-\-paths\-from\-stdin with \-\-paths\-delimiter (for example, for filenames with newlines in them) $ find ~ \-size \-1000k \-print0 | borg create \e \-\-paths\-from\-stdin \e - \-\-paths\-delimiter "\e0" \e - /path/to/repo::smallfiles\-handle\-newline + \-\-paths\-delimiter \(dq\e0\(dq \e + smallfiles\-handle\-newline .ft P .fi .UNINDENT @@ -376,13 +381,13 @@ through using the \fB\-\-keep\-exclude\-tags\fP option. The \fB\-x\fP or \fB\-\-one\-file\-system\fP option excludes directories, that are mountpoints (and everything in them). It detects mountpoints by comparing the device number from the output of \fBstat()\fP of the directory and its parent directory. Specifically, it excludes directories for which \fBstat()\fP reports a device number different -from the device number of their parent. Be aware that in Linux (and possibly elsewhere) there are directories -with device number different from their parent, which the kernel does not consider a mountpoint and also the -other way around. Examples are bind mounts (possibly same device number, but always a mountpoint) and ALL -subvolumes of a btrfs (different device number from parent but not necessarily a mountpoint). Therefore when -using \fB\-\-one\-file\-system\fP, one should make doubly sure that the backup works as intended especially when using -btrfs. This is even more important, if the btrfs layout was created by someone else, e.g. a distribution -installer. +from the device number of their parent. +In general: be aware that there are directories with device number different from their parent, which the kernel +does not consider a mountpoint and also the other way around. +Linux examples for this are bind mounts (possibly same device number, but always a mountpoint) and ALL +subvolumes of a btrfs (different device number from parent but not necessarily a mountpoint). +macOS examples are the apfs mounts of a typical macOS installation. +Therefore, when using \fB\-\-one\-file\-system\fP, you should double\-check that the backup works as intended. .SS Item flags .sp \fB\-\-list\fP outputs a list of all files, directories and other @@ -395,7 +400,7 @@ If you are interested only in a subset of that output, you can give e.g. below). .sp A uppercase character represents the status of a regular file relative to the -"files" cache (not relative to the repo \-\- this is an issue if the files cache +\(dqfiles\(dq cache (not relative to the repo \-\- this is an issue if the files cache is not used). Metadata is stored in any case and for \(aqA\(aq and \(aqM\(aq also new data chunks are stored. For \(aqU\(aq all data chunks refer to already existing chunks. .INDENT 0.0 @@ -431,15 +436,15 @@ borg usually just stores their metadata: Other flags used include: .INDENT 0.0 .IP \(bu 2 -\(aqi\(aq = backup data was read from standard input (stdin) +\(aq+\(aq = included, item would be backed up (if not in dry\-run mode) .IP \(bu 2 -\(aq\-\(aq = dry run, item was \fInot\fP backed up +\(aq\-\(aq = excluded, item would not be / was not backed up .IP \(bu 2 -\(aqx\(aq = excluded, item was \fInot\fP backed up +\(aqi\(aq = backup data was read from standard input (stdin) .IP \(bu 2 \(aq?\(aq = missing status code (if you see this, please file a bug report!) .UNINDENT -.SS Reading from stdin +.SS Reading backup data from stdin .sp There are two methods to read from stdin. Either specify \fB\-\fP as path and pipe directly to borg: @@ -486,9 +491,22 @@ creation a bit. .sp By default, the content read from stdin is stored in a file called \(aqstdin\(aq. Use \fB\-\-stdin\-name\fP to change the name. +.SS Feeding all file paths from externally +.sp +Usually, you give a starting path (recursion root) to borg and then borg +automatically recurses, finds and backs up all fs objects contained in +there (optionally considering include/exclude rules). +.sp +If you need more control and you want to give every single fs object path +to borg (maybe implementing your own recursion or your own rules), you can use +\fB\-\-paths\-from\-stdin\fP or \fB\-\-paths\-from\-command\fP (with the latter, borg will +fail to create an archive should the command fail). +.sp +Borg supports paths with the slashdot hack to strip path prefixes here also. +So, be careful not to unintentionally trigger that. .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP +\fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP, \fIborg\-rcreate(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 8435c20ff0..9e8baf0f44 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -27,52 +27,34 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DELETE" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-DELETE" 1 "2024-07-19" "" "borg backup tool" .SH NAME -borg-delete \- Delete an existing repository or archives +borg-delete \- Delete archives .SH SYNOPSIS .sp -borg [common options] delete [options] [REPOSITORY_OR_ARCHIVE] [ARCHIVE...] +borg [common options] delete [options] .SH DESCRIPTION .sp -This command deletes an archive from the repository or the complete repository. +This command deletes archives from the repository. .sp Important: When deleting archives, repository disk space is \fBnot\fP freed until you run \fBborg compact\fP\&. .sp -When you delete a complete repository, the security info and local cache for it -(if any) are also deleted. Alternatively, you can delete just the local cache -with the \fB\-\-cache\-only\fP option, or keep the security info with the -\fB\-\-keep\-security\-info\fP option. -.sp When in doubt, use \fB\-\-dry\-run \-\-list\fP to see what would be deleted. .sp When using \fB\-\-stats\fP, you will get some statistics about how much data was -deleted \- the "Deleted data" deduplicated size there is most interesting as +deleted \- the \(dqDeleted data\(dq deduplicated size there is most interesting as that is how much your repository will shrink. -Please note that the "All archives" stats refer to the state after deletion. +Please note that the \(dqAll archives\(dq stats refer to the state after deletion. .sp -You can delete multiple archives by specifying their common prefix, if they -have one, using the \fB\-\-prefix PREFIX\fP option. You can also specify a shell -pattern to match multiple archives using the \fB\-\-glob\-archives GLOB\fP option -(for more info on these patterns, see \fIborg_patterns\fP). Note that these -two options are mutually exclusive. +You can delete multiple archives by specifying a matching pattern, +using the \fB\-\-match\-archives PATTERN\fP option (for more info on these patterns, +see \fIborg_patterns\fP). .sp -To avoid accidentally deleting archives, especially when using glob patterns, -it might be helpful to use the \fB\-\-dry\-run\fP to test out the command without -actually making any changes to the repository. +Always first use \fB\-\-dry\-run \-\-list\fP to see what would be deleted. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY_OR_ARCHIVE -repository or archive to delete -.TP -.B ARCHIVE -archives to delete -.UNINDENT .SS optional arguments .INDENT 0.0 .TP @@ -82,38 +64,44 @@ do not change repository .B \-\-list output verbose list of archives .TP +.B \-\-consider\-checkpoints +consider checkpoint archives for deletion (default: not considered). +.TP .B \-s\fP,\fB \-\-stats print statistics for the deleted archive .TP -.B \-\-cache\-only -delete only the local cache for the given repository -.TP .B \-\-force force deletion of corrupted archives, use \fB\-\-force \-\-force\fP in case \fB\-\-force\fP does not work. .TP -.B \-\-keep\-security\-info -keep the local security info when deleting a repository -.TP -.B \-\-save\-space -work slower, but using less space +.BI \-c \ SECONDS\fR,\fB \ \-\-checkpoint\-interval \ SECONDS +write checkpoint every SECONDS seconds (Default: 1800) .UNINDENT .SS Archive filters .INDENT 0.0 .TP -.BI \-P \ PREFIX\fR,\fB \ \-\-prefix \ PREFIX -only consider archive names starting with this prefix. -.TP -.BI \-a \ GLOB\fR,\fB \ \-\-glob\-archives \ GLOB -only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. +.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN +only consider archive names matching the pattern. see \(dqborg help match\-archives\(dq. .TP .BI \-\-sort\-by \ KEYS -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id; default is: timestamp .TP .BI \-\-first \ N consider first N archives after other filters were applied .TP .BI \-\-last \ N consider last N archives after other filters were applied +.TP +.BI \-\-oldest \ TIMESPAN +consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newest \ TIMESPAN +consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-older \ TIMESPAN +consider archives older than (now \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newer \ TIMESPAN +consider archives newer than (now \- TIMESPAN), e.g. 7d or 12m. .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -122,33 +110,25 @@ consider last N archives after other filters were applied .nf .ft C # delete a single backup archive: -$ borg delete /path/to/repo::Monday +$ borg delete \-a Monday # actually free disk space: -$ borg compact /path/to/repo +$ borg compact -# delete all archives whose names begin with the machine\(aqs hostname followed by "\-" -$ borg delete \-\-prefix \(aq{hostname}\-\(aq /path/to/repo +# delete all archives whose names begin with the machine\(aqs hostname followed by \(dq\-\(dq +$ borg delete \-a \(aqsh:{hostname}\-*\(aq -# delete all archives whose names contain "\-2012\-" -$ borg delete \-\-glob\-archives \(aq*\-2012\-*\(aq /path/to/repo +# delete all archives whose names contain \(dq\-2012\-\(dq +$ borg delete \-a \(aqsh:*\-2012\-*\(aq # see what would be deleted if delete was run without \-\-dry\-run -$ borg delete \-\-list \-\-dry\-run \-a \(aq*\-May\-*\(aq /path/to/repo - -# delete the whole repository and the related local cache: -$ borg delete /path/to/repo -You requested to completely DELETE the repository *including* all archives it contains: -repo Mon, 2016\-02\-15 19:26:54 -root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29 -newname Mon, 2016\-02\-15 19:50:19 -Type \(aqYES\(aq if you understand this and want to continue: YES +$ borg delete \-\-list \-\-dry\-run \-a \(aqsh:*\-May\-*\(aq .ft P .fi .UNINDENT .UNINDENT .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-compact(1)\fP +\fIborg\-common(1)\fP, \fIborg\-compact(1)\fP, \fIborg\-rdelete(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 6cce20acd8..ab9e24f779 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -27,28 +27,15 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-DIFF" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-DIFF" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives .SH SYNOPSIS .sp -borg [common options] diff [options] REPO::ARCHIVE1 ARCHIVE2 [PATH...] +borg [common options] diff [options] ARCHIVE1 ARCHIVE2 [PATH...] .SH DESCRIPTION .sp -This command finds differences (file contents, user/group/mode) between archives. -.sp -A repository location and an archive name must be specified for REPO::ARCHIVE1. -ARCHIVE2 is just another archive name in same repository (no repository location -allowed). -.sp -For archives created with Borg 1.1 or newer diff automatically detects whether -the archives are created with the same chunker params. If so, only chunk IDs -are compared, which is very fast. -.sp -For archives prior to Borg 1.1 chunk contents are compared by default. -If you did not create the archives with different chunker params, -pass \fB\-\-same\-chunker\-params\fP\&. -Note that the chunker params changed from Borg 0.xx to 1.0. +This command finds differences (file contents, metadata) between ARCHIVE1 and ARCHIVE2. .sp For more help on include/exclude patterns, see the \fIborg_patterns\fP command output. .SH OPTIONS @@ -57,11 +44,11 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B REPO::ARCHIVE1 -repository location and ARCHIVE1 name +.B ARCHIVE1 +ARCHIVE1 name .TP .B ARCHIVE2 -ARCHIVE2 name (no repository location allowed) +ARCHIVE2 name .TP .B PATH paths of items inside the archives to compare; patterns are supported @@ -69,9 +56,6 @@ paths of items inside the archives to compare; patterns are supported .SS optional arguments .INDENT 0.0 .TP -.B \-\-numeric\-owner -deprecated, use \fB\-\-numeric\-ids\fP instead -.TP .B \-\-numeric\-ids only consider numeric user and group identifiers .TP @@ -81,10 +65,16 @@ Override check of chunker parameters. .B \-\-sort Sort the output lines by file path. .TP +.BI \-\-format \ FORMAT +specify format for differences between archives (default: \(dq{change} {path}{NL}\(dq) +.TP .B \-\-json\-lines Format output as JSON Lines. +.TP +.B \-\-content\-only +Only compare differences in content (exclude metadata differences) .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN @@ -105,48 +95,101 @@ read include/exclude patterns from PATTERNFILE, one per line .sp .nf .ft C -$ borg init \-e=none testrepo -$ mkdir testdir -$ cd testdir -$ echo asdf > file1 -$ dd if=/dev/urandom bs=1M count=4 > file2 -$ touch file3 -$ borg create ../testrepo::archive1 . - -$ chmod a+x file1 -$ echo "something" >> file2 -$ borg create ../testrepo::archive2 . - -$ echo "testing 123" >> file1 -$ rm file3 -$ touch file4 -$ borg create ../testrepo::archive3 . - -$ cd .. -$ borg diff testrepo::archive1 archive2 -[\-rw\-r\-\-r\-\- \-> \-rwxr\-xr\-x] file1 - +135 B \-252 B file2 - -$ borg diff testrepo::archive2 archive3 - +17 B \-5 B file1 -added 0 B file4 -removed 0 B file3 - -$ borg diff testrepo::archive1 archive3 +$ borg diff archive1 archive2 +17 B \-5 B [\-rw\-r\-\-r\-\- \-> \-rwxr\-xr\-x] file1 +135 B \-252 B file2 added 0 B file4 removed 0 B file3 -$ borg diff \-\-json\-lines testrepo::archive1 archive3 -{"path": "file1", "changes": [{"type": "modified", "added": 17, "removed": 5}, {"type": "mode", "old_mode": "\-rw\-r\-\-r\-\-", "new_mode": "\-rwxr\-xr\-x"}]} -{"path": "file2", "changes": [{"type": "modified", "added": 135, "removed": 252}]} -{"path": "file4", "changes": [{"type": "added", "size": 0}]} -{"path": "file3", "changes": [{"type": "removed", "size": 0}] +$ borg diff archive1 archive2 +{\(dqpath\(dq: \(dqfile1\(dq, \(dqchanges\(dq: [{\(dqtype\(dq: \(dqmodified\(dq, \(dqadded\(dq: 17, \(dqremoved\(dq: 5}, {\(dqtype\(dq: \(dqmode\(dq, \(dqold_mode\(dq: \(dq\-rw\-r\-\-r\-\-\(dq, \(dqnew_mode\(dq: \(dq\-rwxr\-xr\-x\(dq}]} +{\(dqpath\(dq: \(dqfile2\(dq, \(dqchanges\(dq: [{\(dqtype\(dq: \(dqmodified\(dq, \(dqadded\(dq: 135, \(dqremoved\(dq: 252}]} +{\(dqpath\(dq: \(dqfile4\(dq, \(dqchanges\(dq: [{\(dqtype\(dq: \(dqadded\(dq, \(dqsize\(dq: 0}]} +{\(dqpath\(dq: \(dqfile3\(dq, \(dqchanges\(dq: [{\(dqtype\(dq: \(dqremoved\(dq, \(dqsize\(dq: 0}]} +.ft P +.fi +.UNINDENT +.UNINDENT +.SH NOTES +.SS The FORMAT specifier syntax +.sp +The \fB\-\-format\fP option uses python\(aqs \fI\%format string syntax\fP\&. +.sp +Examples: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ borg diff \-\-format \(aq{content:30} {path}{NL}\(aq ArchiveFoo ArchiveBar +modified: +4.1 kB \-1.0 kB file\-diff +\&... + +# {VAR:NUMBER} \- pad to NUMBER columns right\-aligned. +$ borg diff \-\-format \(aq{content:>30} {path}{NL}\(aq ArchiveFoo ArchiveBar + modified: +4.1 kB \-1.0 kB file\-diff +\&... .ft P .fi .UNINDENT .UNINDENT +.sp +The following keys are always available: +.INDENT 0.0 +.IP \(bu 2 +NEWLINE: OS dependent line separator +.IP \(bu 2 +NL: alias of NEWLINE +.IP \(bu 2 +NUL: NUL character for creating print0 / xargs \-0 like output +.IP \(bu 2 +SPACE: space character +.IP \(bu 2 +TAB: tab character +.IP \(bu 2 +CR: carriage return character +.IP \(bu 2 +LF: line feed character +.UNINDENT +.sp +Keys available only when showing differences between archives: +.INDENT 0.0 +.IP \(bu 2 +path: archived file path +.IP \(bu 2 +change: all available changes +.IP \(bu 2 +content: file content change +.IP \(bu 2 +mode: file mode change +.IP \(bu 2 +type: file type change +.IP \(bu 2 +owner: file owner (user/group) change +.IP \(bu 2 +group: file group change +.IP \(bu 2 +user: file user change +.IP \(bu 2 +link: file link change +.IP \(bu 2 +directory: file directory change +.IP \(bu 2 +blkdev: file block device change +.IP \(bu 2 +chrdev: file character device change +.IP \(bu 2 +fifo: file fifo change +.IP \(bu 2 +mtime: file modification time change +.IP \(bu 2 +ctime: file change time change +.IP \(bu 2 +isomtime: file modification time change (ISO 8601) +.IP \(bu 2 +isoctime: file creation time change (ISO 8601) +.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 4a26acb6b6..53632e1712 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXPORT-TAR" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-EXPORT-TAR" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball .SH SYNOPSIS .sp -borg [common options] export\-tar [options] ARCHIVE FILE [PATH...] +borg [common options] export\-tar [options] NAME FILE [PATH...] .SH DESCRIPTION .sp This command creates a tarball from an archive. @@ -50,7 +50,7 @@ before writing it to FILE: .IP \(bu 2 \&.tar.xz or .txz: xz .IP \(bu 2 -\&.tar.zstd: zstd +\&.tar.zstd or .tar.zst: zstd .IP \(bu 2 \&.tar.lz4: lz4 .UNINDENT @@ -59,12 +59,45 @@ Alternatively, a \fB\-\-tar\-filter\fP program may be explicitly specified. It s read the uncompressed tar stream from stdin and write a compressed/filtered tar stream to stdout. .sp -The generated tarball uses the GNU tar format. -.sp -export\-tar is a lossy conversion: -BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported. -Timestamp resolution is limited to whole seconds, not the nanosecond resolution -otherwise supported by Borg. +Depending on the \fB\-tar\-format\fP option, these formats are created: +.TS +center; +|l|l|l|. +_ +T{ +\-\-tar\-format +T} T{ +Specification +T} T{ +Metadata +T} +_ +T{ +BORG +T} T{ +BORG specific, like PAX +T} T{ +all as supported by borg +T} +_ +T{ +PAX +T} T{ +POSIX.1\-2001 (pax) format +T} T{ +GNU + atime/ctime/mtime ns +T} +_ +T{ +GNU +T} T{ +GNU tar format +T} T{ +mtime s, no atime/ctime, +no ACLs/xattrs/bsdflags +T} +_ +.TE .sp A \fB\-\-sparse\fP option (as found in borg extract) is not supported. .sp @@ -82,11 +115,11 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B ARCHIVE -archive to export +.B NAME +specify the archive name .TP .B FILE -output tar file. "\-" to write to stdout instead. +output tar file. \(dq\-\(dq to write to stdout instead. .TP .B PATH paths to extract; patterns are supported @@ -99,8 +132,11 @@ filter program to pipe data through .TP .B \-\-list output verbose list of items (files, dirs, ...) +.TP +.BI \-\-tar\-format \ FMT +select tar format: BORG, PAX or GNU .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN @@ -118,31 +154,6 @@ read include/exclude patterns from PATTERNFILE, one per line .BI \-\-strip\-components \ NUMBER Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. .UNINDENT -.SH EXAMPLES -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -# export as uncompressed tar -$ borg export\-tar /path/to/repo::Monday Monday.tar - -# exclude some types, compress using gzip -$ borg export\-tar /path/to/repo::Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq - -# use higher compression level with gzip -$ borg export\-tar \-\-tar\-filter="gzip \-9" testrepo::linux Monday.tar.gz - -# export a tar, but instead of storing it on disk, -# upload it to a remote site using curl. -$ borg export\-tar /path/to/repo::Monday \- | curl \-\-data\-binary @\- https://somewhere/to/POST - -# remote extraction via "tarpipe" -$ borg export\-tar /path/to/repo::Monday \- | ssh somewhere "cd extracted; tar x" -.ft P -.fi -.UNINDENT -.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 9d155d2e98..33585fd97e 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-EXTRACT" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-EXTRACT" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents .SH SYNOPSIS .sp -borg [common options] extract [options] ARCHIVE [PATH...] +borg [common options] extract [options] NAME [PATH...] .SH DESCRIPTION .sp This command extracts the contents of an archive. By default the entire @@ -52,7 +52,7 @@ pass over the archive metadata. \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Currently, extract always writes into the current working directory ("."), +Currently, extract always writes into the current working directory (\(dq.\(dq), so make sure you \fBcd\fP to the right place before calling \fBborg extract\fP\&. .sp When parent directories are not extracted (because of using file/directory selection @@ -66,8 +66,8 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B ARCHIVE -archive to extract +.B NAME +specify the archive name .TP .B PATH paths to extract; patterns are supported @@ -81,15 +81,9 @@ output verbose list of items (files, dirs, ...) .B \-n\fP,\fB \-\-dry\-run do not actually change any files .TP -.B \-\-numeric\-owner -deprecated, use \fB\-\-numeric\-ids\fP instead -.TP .B \-\-numeric\-ids only obey numeric user and group identifiers .TP -.B \-\-nobsdflags -deprecated, use \fB\-\-noflags\fP instead -.TP .B \-\-noflags do not extract/set flags (e.g. NODUMP, IMMUTABLE) .TP @@ -104,8 +98,11 @@ write all extracted data to stdout .TP .B \-\-sparse create holes in output sparse file from all\-zero chunks +.TP +.B \-\-continue +continue a previously interrupted extraction of same archive .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN @@ -130,22 +127,22 @@ Remove the specified number of leading path elements. Paths with fewer elements .nf .ft C # Extract entire archive -$ borg extract /path/to/repo::my\-files +$ borg extract my\-files # Extract entire archive and list files while processing -$ borg extract \-\-list /path/to/repo::my\-files +$ borg extract \-\-list my\-files # Verify whether an archive could be successfully extracted, but do not write files to disk -$ borg extract \-\-dry\-run /path/to/repo::my\-files +$ borg extract \-\-dry\-run my\-files -# Extract the "src" directory -$ borg extract /path/to/repo::my\-files home/USERNAME/src +# Extract the \(dqsrc\(dq directory +$ borg extract my\-files home/USERNAME/src -# Extract the "src" directory but exclude object files -$ borg extract /path/to/repo::my\-files home/USERNAME/src \-\-exclude \(aq*.o\(aq +# Extract the \(dqsrc\(dq directory but exclude object files +$ borg extract my\-files home/USERNAME/src \-\-exclude \(aq*.o\(aq # Restore a raw device (must not be active/in use/mounted at that time) -$ borg extract \-\-stdout /path/to/repo::my\-sdx | dd of=/dev/sdx bs=10M +$ borg extract \-\-stdout my\-sdx | dd of=/dev/sdx bs=10M .ft P .fi .UNINDENT diff --git a/docs/man/borg-import-tar.1 b/docs/man/borg-import-tar.1 index 9c9f2a11a9..1b0be536d0 100644 --- a/docs/man/borg-import-tar.1 +++ b/docs/man/borg-import-tar.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-IMPORT-TAR" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-IMPORT-TAR" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-import-tar \- Create a backup archive from a tarball .SH SYNOPSIS .sp -borg [common options] import\-tar [options] ARCHIVE TARFILE +borg [common options] import\-tar [options] NAME TARFILE .SH DESCRIPTION .sp This command creates a backup archive from a tarball. @@ -49,7 +49,7 @@ based on its file extension and pipe the file through an appropriate filter: .IP \(bu 2 \&.tar.xz or .txz: xz \-d .IP \(bu 2 -\&.tar.zstd: zstd \-d +\&.tar.zstd or .tar.zst: zstd \-d .IP \(bu 2 \&.tar.lz4: lz4 \-d .UNINDENT @@ -61,26 +61,40 @@ stdout. Most documentation of borg create applies. Note that this command does not support excluding files. .sp -import\-tar is a lossy conversion: -BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported. -Timestamp resolution is limited to whole seconds, not the nanosecond resolution -otherwise supported by Borg. -.sp A \fB\-\-sparse\fP option (as found in borg create) is not supported. .sp -import\-tar reads POSIX.1\-1988 (ustar), POSIX.1\-2001 (pax), GNU tar, UNIX V7 tar -and SunOS tar with extended attributes. +About tar formats and metadata conservation or loss, please see \fBborg export\-tar\fP\&. +.sp +import\-tar reads these tar formats: +.INDENT 0.0 +.IP \(bu 2 +BORG: borg specific (PAX\-based) +.IP \(bu 2 +PAX: POSIX.1\-2001 +.IP \(bu 2 +GNU: GNU tar +.IP \(bu 2 +POSIX.1\-1988 (ustar) +.IP \(bu 2 +UNIX V7 tar +.IP \(bu 2 +SunOS tar with extended attributes +.UNINDENT +.sp +To import multiple tarballs into a single archive, they can be simply +concatenated (e.g. using \(dqcat\(dq) into a single file, and imported with an +\fB\-\-ignore\-zeros\fP option to skip through the stop markers between them. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B ARCHIVE -name of archive to create (must be also a valid directory name) +.B NAME +specify the archive name .TP .B TARFILE -input tar file. "\-" to read from stdin instead. +input tar file. \(dq\-\(dq to read from stdin instead. .UNINDENT .SS optional arguments .INDENT 0.0 @@ -99,6 +113,9 @@ only display items with the given status characters .TP .B \-\-json output stats as JSON (implies \-\-stats) +.TP +.B \-\-ignore\-zeros +ignore zero\-filled blocks in the input tarball .UNINDENT .SS Archive options .INDENT 0.0 @@ -107,16 +124,93 @@ output stats as JSON (implies \-\-stats) add a comment text to the archive .TP .BI \-\-timestamp \ TIMESTAMP -manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). alternatively, give a reference file/directory. +manually specify the archive creation date/time (yyyy\-mm\-ddThh:mm:ss[(+|\-)HH:MM] format, (+|\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory. .TP .BI \-c \ SECONDS\fR,\fB \ \-\-checkpoint\-interval \ SECONDS write checkpoint every SECONDS seconds (Default: 1800) .TP +.BI \-\-checkpoint\-volume \ BYTES +write checkpoint every BYTES bytes (Default: 0, meaning no volume based checkpointing) +.TP .BI \-\-chunker\-params \ PARAMS specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095 .TP .BI \-C \ COMPRESSION\fR,\fB \ \-\-compression \ COMPRESSION -select compression algorithm, see the output of the "borg help compression" command for details. +select compression algorithm, see the output of the \(dqborg help compression\(dq command for details. +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# export as uncompressed tar +$ borg export\-tar Monday Monday.tar + +# import an uncompressed tar +$ borg import\-tar Monday Monday.tar + +# exclude some file types, compress using gzip +$ borg export\-tar Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq + +# use higher compression level with gzip +$ borg export\-tar \-\-tar\-filter=\(dqgzip \-9\(dq Monday Monday.tar.gz + +# copy an archive from repoA to repoB +$ borg \-r repoA export\-tar \-\-tar\-format=BORG archive \- | borg \-r repoB import\-tar archive \- + +# export a tar, but instead of storing it on disk, upload it to remote site using curl +$ borg export\-tar Monday \- | curl \-\-data\-binary @\- https://somewhere/to/POST + +# remote extraction via \(dqtarpipe\(dq +$ borg export\-tar Monday \- | ssh somewhere \(dqcd extracted; tar x\(dq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Archives transfer script +.sp +Outputs a script that copies all archives from repo1 to repo2: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +for A T in \(gaborg list \-\-format=\(aq{archive} {time:%Y\-%m\-%dT%H:%M:%S}{NL}\(aq\(ga +do + echo \(dqborg \-r repo1 export\-tar \-\-tar\-format=BORG $A \- | borg \-r repo2 import\-tar \-\-timestamp=$T $A \-\(dq +done +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Kept: +.INDENT 0.0 +.IP \(bu 2 +archive name, archive timestamp +.IP \(bu 2 +archive contents (all items with metadata and data) +.UNINDENT +.sp +Lost: +.INDENT 0.0 +.IP \(bu 2 +some archive metadata (like the original commandline, execution time, etc.) +.UNINDENT +.sp +Please note: +.INDENT 0.0 +.IP \(bu 2 +all data goes over that pipe, again and again for every archive +.IP \(bu 2 +the pipe is dumb, there is no data or transfer time reduction there due to deduplication +.IP \(bu 2 +maybe add compression +.IP \(bu 2 +pipe over ssh for remote transfer +.IP \(bu 2 +no special sparse file support .UNINDENT .SH SEE ALSO .sp diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 92f516eb60..a9812b154b 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -27,38 +27,27 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-INFO" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-INFO" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used .SH SYNOPSIS .sp -borg [common options] info [options] [REPOSITORY_OR_ARCHIVE] +borg [common options] info [options] .SH DESCRIPTION .sp -This command displays detailed information about the specified archive or repository. +This command displays detailed information about the specified archive. .sp Please note that the deduplicated sizes of the individual archives do not add -up to the deduplicated size of the repository ("all archives"), because the two +up to the deduplicated size of the repository (\(dqall archives\(dq), because the two are meaning different things: .sp This archive / deduplicated size = amount of data stored ONLY for this archive = unique chunks of this archive. All archives / deduplicated size = amount of data stored in the repo = all chunks in the repository. -.sp -Borg archives can only contain a limited amount of file metadata. -The size of an archive relative to this limit depends on a number of factors, -mainly the number of files, the lengths of paths and other metadata stored for files. -This is shown as \fIutilization of maximum supported archive size\fP\&. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY_OR_ARCHIVE -repository or archive to display information about -.UNINDENT .SS optional arguments .INDENT 0.0 .TP @@ -68,20 +57,29 @@ format output as JSON .SS Archive filters .INDENT 0.0 .TP -.BI \-P \ PREFIX\fR,\fB \ \-\-prefix \ PREFIX -only consider archive names starting with this prefix. -.TP -.BI \-a \ GLOB\fR,\fB \ \-\-glob\-archives \ GLOB -only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. +.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN +only consider archive names matching the pattern. see \(dqborg help match\-archives\(dq. .TP .BI \-\-sort\-by \ KEYS -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id; default is: timestamp .TP .BI \-\-first \ N consider first N archives after other filters were applied .TP .BI \-\-last \ N consider last N archives after other filters were applied +.TP +.BI \-\-oldest \ TIMESPAN +consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newest \ TIMESPAN +consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-older \ TIMESPAN +consider archives older than (now \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newer \ TIMESPAN +consider archives newer than (now \- TIMESPAN), e.g. 7d or 12m. .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -89,65 +87,27 @@ consider last N archives after other filters were applied .sp .nf .ft C -$ borg info /path/to/repo::2017\-06\-29T11:00\-srv -Archive name: 2017\-06\-29T11:00\-srv -Archive fingerprint: b2f1beac2bd553b34e06358afa45a3c1689320d39163890c5bbbd49125f00fe5 -Comment: -Hostname: myhostname -Username: root -Time (start): Thu, 2017\-06\-29 11:03:07 -Time (end): Thu, 2017\-06\-29 11:03:13 -Duration: 5.66 seconds -Number of files: 17037 -Command line: /usr/sbin/borg create /path/to/repo::2017\-06\-29T11:00\-srv /srv -Utilization of max. archive size: 0% -\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- - Original size Compressed size Deduplicated size -This archive: 12.53 GB 12.49 GB 1.62 kB -All archives: 121.82 TB 112.41 TB 215.42 GB - - Unique chunks Total chunks -Chunk index: 1015213 626934122 - -$ borg info /path/to/repo \-\-last 1 -Archive name: 2017\-06\-29T11:00\-srv -Archive fingerprint: b2f1beac2bd553b34e06358afa45a3c1689320d39163890c5bbbd49125f00fe5 +$ borg info Tuesday2022\-06\-25T20:51:39 +Archive name: Tuesday2022\-06\-25T20:51:39 +Archive fingerprint: f7dea0788dfc026cc2be1c0f5b94beb4e4084eb3402fc40c38d8719b1bf2d943 Comment: -Hostname: myhostname -Username: root -Time (start): Thu, 2017\-06\-29 11:03:07 -Time (end): Thu, 2017\-06\-29 11:03:13 -Duration: 5.66 seconds -Number of files: 17037 -Command line: /usr/sbin/borg create /path/to/repo::2017\-06\-29T11:00\-srv /srv -Utilization of max. archive size: 0% -\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- - Original size Compressed size Deduplicated size -This archive: 12.53 GB 12.49 GB 1.62 kB -All archives: 121.82 TB 112.41 TB 215.42 GB - - Unique chunks Total chunks -Chunk index: 1015213 626934122 - -$ borg info /path/to/repo -Repository ID: d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 -Location: /path/to/repo -Encrypted: Yes (repokey) -Cache: /root/.cache/borg/d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 -Security dir: /root/.config/borg/security/d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 -\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- - Original size Compressed size Deduplicated size -All archives: 121.82 TB 112.41 TB 215.42 GB - - Unique chunks Total chunks -Chunk index: 1015213 626934122 +Hostname: mba2020 +Username: tw +Time (start): Sat, 2022\-06\-25 20:51:40 +Time (end): Sat, 2022\-06\-25 20:51:40 +Duration: 0.03 seconds +Command line: /Users/tw/w/borg\-env/bin/borg \-r path/to/repo create \-\-stats \(aqTuesday{now}\(aq src \-\-progress +Utilization of maximum supported archive size: 0% +Number of files: 244 +Original size: 13.80 MB +Deduplicated size: 531 B .ft P .fi .UNINDENT .UNINDENT .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-list(1)\fP, \fIborg\-diff(1)\fP +\fIborg\-common(1)\fP, \fIborg\-list(1)\fP, \fIborg\-diff(1)\fP, \fIborg\-rinfo(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 deleted file mode 100644 index 64e68c1f6f..0000000000 --- a/docs/man/borg-init.1 +++ /dev/null @@ -1,270 +0,0 @@ -.\" Man page generated from reStructuredText. -. -. -.nr rst2man-indent-level 0 -. -.de1 rstReportMargin -\\$1 \\n[an-margin] -level \\n[rst2man-indent-level] -level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] -- -\\n[rst2man-indent0] -\\n[rst2man-indent1] -\\n[rst2man-indent2] -.. -.de1 INDENT -.\" .rstReportMargin pre: -. RS \\$1 -. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] -. nr rst2man-indent-level +1 -.\" .rstReportMargin post: -.. -.de UNINDENT -. RE -.\" indent \\n[an-margin] -.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] -.nr rst2man-indent-level -1 -.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] -.in \\n[rst2man-indent\\n[rst2man-indent-level]]u -.. -.TH "BORG-INIT" 1 "2022-02-19" "" "borg backup tool" -.SH NAME -borg-init \- Initialize an empty repository -.SH SYNOPSIS -.sp -borg [common options] init [options] [REPOSITORY] -.SH DESCRIPTION -.sp -This command initializes an empty repository. A repository is a filesystem -directory containing the deduplicated data from zero or more archives. -.SS Encryption mode TLDR -.sp -The encryption mode can only be configured when creating a new repository \- -you can neither configure it on a per\-archive basis nor change the -encryption mode of an existing repository. -.sp -Use \fBrepokey\fP: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -borg init \-\-encryption repokey /path/to/repo -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Or \fBrepokey\-blake2\fP depending on which is faster on your client machines (see below): -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -borg init \-\-encryption repokey\-blake2 /path/to/repo -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Borg will: -.INDENT 0.0 -.IP 1. 3 -Ask you to come up with a passphrase. -.IP 2. 3 -Create a borg key (which contains 3 random secrets. See \fIkey_files\fP). -.IP 3. 3 -Encrypt the key with your passphrase. -.IP 4. 3 -Store the encrypted borg key inside the repository directory (in the repo config). -This is why it is essential to use a secure passphrase. -.IP 5. 3 -Encrypt and sign your backups to prevent anyone from reading or forging them unless they -have the key and know the passphrase. Make sure to keep a backup of -your key \fBoutside\fP the repository \- do not lock yourself out by -"leaving your keys inside your car" (see \fIborg_key_export\fP). -For remote backups the encryption is done locally \- the remote machine -never sees your passphrase, your unencrypted key or your unencrypted files. -Chunking and id generation are also based on your key to improve -your privacy. -.IP 6. 3 -Use the key when extracting files to decrypt them and to verify that the contents of -the backups have not been accidentally or maliciously altered. -.UNINDENT -.SS Picking a passphrase -.sp -Make sure you use a good passphrase. Not too short, not too simple. The real -encryption / decryption key is encrypted with / locked by your passphrase. -If an attacker gets your key, he can\(aqt unlock and use it without knowing the -passphrase. -.sp -Be careful with special or non\-ascii characters in your passphrase: -.INDENT 0.0 -.IP \(bu 2 -Borg processes the passphrase as unicode (and encodes it as utf\-8), -so it does not have problems dealing with even the strangest characters. -.IP \(bu 2 -BUT: that does not necessarily apply to your OS / VM / keyboard configuration. -.UNINDENT -.sp -So better use a long passphrase made from simple ascii chars than one that -includes non\-ascii stuff or characters that are hard/impossible to enter on -a different keyboard layout. -.sp -You can change your passphrase for existing repos at any time, it won\(aqt affect -the encryption/decryption key or other secrets. -.SS More encryption modes -.sp -Only use \fB\-\-encryption none\fP if you are OK with anyone who has access to -your repository being able to read your backups and tamper with their -contents without you noticing. -.sp -If you want "passphrase and having\-the\-key" security, use \fB\-\-encryption keyfile\fP\&. -The key will be stored in your home directory (in \fB~/.config/borg/keys\fP). -.sp -If you do \fBnot\fP want to encrypt the contents of your backups, but still -want to detect malicious tampering use \fB\-\-encryption authenticated\fP\&. -.sp -If \fBBLAKE2b\fP is faster than \fBSHA\-256\fP on your hardware, use \fB\-\-encryption authenticated\-blake2\fP, -\fB\-\-encryption repokey\-blake2\fP or \fB\-\-encryption keyfile\-blake2\fP\&. Note: for remote backups -the hashing is done on your local machine. -.\" nanorst: inline-fill -. -.TS -center; -|l|l|l|l|. -_ -T{ -Hash/MAC -T} T{ -Not encrypted -no auth -T} T{ -Not encrypted, -but authenticated -T} T{ -Encrypted (AEAD w/ AES) -and authenticated -T} -_ -T{ -SHA\-256 -T} T{ -none -T} T{ -\fIauthenticated\fP -T} T{ -repokey -keyfile -T} -_ -T{ -BLAKE2b -T} T{ -n/a -T} T{ -\fIauthenticated\-blake2\fP -T} T{ -\fIrepokey\-blake2\fP -\fIkeyfile\-blake2\fP -T} -_ -.TE -.\" nanorst: inline-replace -. -.sp -Modes \fImarked like this\fP in the above table are new in Borg 1.1 and are not -backwards\-compatible with Borg 1.0.x. -.sp -On modern Intel/AMD CPUs (except very cheap ones), AES is usually -hardware\-accelerated. -BLAKE2b is faster than SHA256 on Intel/AMD 64\-bit CPUs -(except AMD Ryzen and future CPUs with SHA extensions), -which makes \fIauthenticated\-blake2\fP faster than \fInone\fP and \fIauthenticated\fP\&. -.sp -On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster -than BLAKE2b\-256 there. NEON accelerates AES as well. -.sp -Hardware acceleration is always used automatically when available. -.sp -\fIrepokey\fP and \fIkeyfile\fP use AES\-CTR\-256 for encryption and HMAC\-SHA256 for -authentication in an encrypt\-then\-MAC (EtM) construction. The chunk ID hash -is HMAC\-SHA256 as well (with a separate key). -These modes are compatible with Borg 1.0.x. -.sp -\fIrepokey\-blake2\fP and \fIkeyfile\-blake2\fP are also authenticated encryption modes, -but use BLAKE2b\-256 instead of HMAC\-SHA256 for authentication. The chunk ID -hash is a keyed BLAKE2b\-256 hash. -These modes are new and \fInot\fP compatible with Borg 1.0.x. -.sp -\fIauthenticated\fP mode uses no encryption, but authenticates repository contents -through the same HMAC\-SHA256 hash as the \fIrepokey\fP and \fIkeyfile\fP modes (it uses it -as the chunk ID hash). The key is stored like \fIrepokey\fP\&. -This mode is new and \fInot\fP compatible with Borg 1.0.x. -.sp -\fIauthenticated\-blake2\fP is like \fIauthenticated\fP, but uses the keyed BLAKE2b\-256 hash -from the other blake2 modes. -This mode is new and \fInot\fP compatible with Borg 1.0.x. -.sp -\fInone\fP mode uses no encryption and no authentication. It uses SHA256 as chunk -ID hash. This mode is not recommended, you should rather consider using an authenticated -or authenticated/encrypted mode. This mode has possible denial\-of\-service issues -when running \fBborg create\fP on contents controlled by an attacker. -Use it only for new repositories where no encryption is wanted \fBand\fP when compatibility -with 1.0.x is important. If compatibility with 1.0.x is not important, use -\fIauthenticated\-blake2\fP or \fIauthenticated\fP instead. -This mode is compatible with Borg 1.0.x. -.SH OPTIONS -.sp -See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.INDENT 0.0 -.TP -.B REPOSITORY -repository to create -.UNINDENT -.SS optional arguments -.INDENT 0.0 -.TP -.BI \-e \ MODE\fR,\fB \ \-\-encryption \ MODE -select encryption key mode \fB(required)\fP -.TP -.B \-\-append\-only -create an append\-only mode repository. Note that this only affects the low level structure of the repository, and running \fIdelete\fP or \fIprune\fP will still be allowed. See \fIappend_only_mode\fP in Additional Notes for more details. -.TP -.BI \-\-storage\-quota \ QUOTA -Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota. -.TP -.B \-\-make\-parent\-dirs -create the parent directories of the repository directory, if they are missing. -.UNINDENT -.SH EXAMPLES -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -# Local repository, repokey encryption, BLAKE2b (often faster, since Borg 1.1) -$ borg init \-\-encryption=repokey\-blake2 /path/to/repo - -# Local repository (no encryption) -$ borg init \-\-encryption=none /path/to/repo - -# Remote repository (accesses a remote borg via ssh) -# repokey: stores the (encrypted) key into /config -$ borg init \-\-encryption=repokey\-blake2 user@hostname:backup - -# Remote repository (accesses a remote borg via ssh) -# keyfile: stores the (encrypted) key into ~/.config/borg/keys/ -$ borg init \-\-encryption=keyfile user@hostname:backup -.ft P -.fi -.UNINDENT -.UNINDENT -.SH SEE ALSO -.sp -\fIborg\-common(1)\fP, \fIborg\-create(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-check(1)\fP, \fIborg\-list(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP -.SH AUTHOR -The Borg Collective -.\" Generated by docutils manpage writer. -. diff --git a/docs/man/borg-key-change-algorithm.1 b/docs/man/borg-key-change-algorithm.1 new file mode 100644 index 0000000000..8178a92b2d --- /dev/null +++ b/docs/man/borg-key-change-algorithm.1 @@ -0,0 +1,91 @@ +.\" Man page generated from reStructuredText. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "BORG-KEY-CHANGE-ALGORITHM" 1 "2022-06-26" "" "borg backup tool" +.SH NAME +borg-key-change-algorithm \- Change repository key algorithm +.SH SYNOPSIS +.sp +borg [common options] key change\-algorithm [options] ALGORITHM +.SH DESCRIPTION +.sp +Change the algorithm we use to encrypt and authenticate the borg key. +.sp +Important: In a \fIrepokey\fP mode (e.g. repokey\-blake2) all users share the same key. +In this mode upgrading to \fIargon2\fP will make it impossible to access the repo for users who use an old version of borg. +We recommend upgrading to the latest stable version. +.sp +Important: In a \fIkeyfile\fP mode (e.g. keyfile\-blake2) each user has their own key (in \fB~/.config/borg/keys\fP). +In this mode this command will only change the key used by the current user. +If you want to upgrade to \fIargon2\fP to strengthen security, you will have to upgrade each user\(aqs key individually. +.sp +Your repository is encrypted and authenticated with a key that is randomly generated by \fBborg init\fP\&. +The key is encrypted and authenticated with your passphrase. +.sp +We currently support two choices: +.INDENT 0.0 +.IP 1. 3 +argon2 \- recommended. This algorithm is used by default when initialising a new repository. +The key encryption key is derived from your passphrase via argon2\-id. +Argon2 is considered more modern and secure than pbkdf2. +.IP 2. 3 +pbkdf2 \- the legacy algorithm. Use this if you want to access your repo via old versions of borg. +The key encryption key is derived from your passphrase via PBKDF2\-HMAC\-SHA256. +.UNINDENT +.sp +Examples: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# Upgrade an existing key to argon2 +borg key change\-algorithm /path/to/repo argon2 +# Downgrade to pbkdf2 \- use this if upgrading borg is not an option +borg key change\-algorithm /path/to/repo pbkdf2 +.ft P +.fi +.UNINDENT +.UNINDENT +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B ALGORITHM +select key algorithm +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-key-change-location.1 b/docs/man/borg-key-change-location.1 new file mode 100644 index 0000000000..7ffb2f185f --- /dev/null +++ b/docs/man/borg-key-change-location.1 @@ -0,0 +1,71 @@ +.\" Man page generated from reStructuredText. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "BORG-KEY-CHANGE-LOCATION" 1 "2024-07-19" "" "borg backup tool" +.SH NAME +borg-key-change-location \- Change repository key location +.SH SYNOPSIS +.sp +borg [common options] key change\-location [options] KEY_LOCATION +.SH DESCRIPTION +.sp +Change the location of a borg key. The key can be stored at different locations: +.INDENT 0.0 +.IP \(bu 2 +keyfile: locally, usually in the home directory +.IP \(bu 2 +repokey: inside the repo (in the repo config) +.UNINDENT +.sp +Please note: +.sp +This command does NOT change the crypto algorithms, just the key location, +thus you must ONLY give the key location (keyfile or repokey). +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B KEY_LOCATION +select key location +.UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-\-keep +keep the key also at the current location (default: remove it) +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 9cde3c97f6..a1c6c6b2c0 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-KEY-CHANGE-PASSPHRASE" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase .SH SYNOPSIS .sp -borg [common options] key change\-passphrase [options] [REPOSITORY] +borg [common options] key change\-passphrase [options] .SH DESCRIPTION .sp The key files used for repository encryption are optionally passphrase @@ -45,9 +45,6 @@ does not protect future (nor past) backups to the same repository. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. -.SS arguments -.sp -REPOSITORY .SH EXAMPLES .INDENT 0.0 .INDENT 3.5 @@ -55,19 +52,19 @@ REPOSITORY .nf .ft C # Create a key file protected repository -$ borg init \-\-encryption=keyfile \-v /path/to/repo -Initializing repository at "/path/to/repo" +$ borg rcreate \-\-encryption=keyfile\-aes\-ocb \-v +Initializing repository at \(dq/path/to/repo\(dq Enter new passphrase: Enter same passphrase again: Remember your passphrase. Your data will be inaccessible without it. -Key in "/root/.config/borg/keys/mnt_backup" created. +Key in \(dq/root/.config/borg/keys/mnt_backup\(dq created. Keep this key safe. Your data will be inaccessible without it. Synchronizing chunks cache... Archives: 0, w/ cached Idx: 0, w/ outdated Idx: 0, w/o cached Idx: 0. Done. # Change key file passphrase -$ borg key change\-passphrase \-v /path/to/repo +$ borg key change\-passphrase \-v Enter passphrase for key /root/.config/borg/keys/mnt_backup: Enter new passphrase: Enter same passphrase again: @@ -77,7 +74,7 @@ Key updated # Import a previously\-exported key into the specified # key file (creating or overwriting the output key) # (keyfile repositories only) -$ BORG_KEY_FILE=/path/to/output\-key borg key import /path/to/repo /path/to/exported +$ BORG_KEY_FILE=/path/to/output\-key borg key import /path/to/exported .ft P .fi .UNINDENT @@ -89,10 +86,10 @@ Fully automated using environment variables: .sp .nf .ft C -$ BORG_NEW_PASSPHRASE=old borg init \-e=repokey repo -# now "old" is the current passphrase. -$ BORG_PASSPHRASE=old BORG_NEW_PASSPHRASE=new borg key change\-passphrase repo -# now "new" is the current passphrase. +$ BORG_NEW_PASSPHRASE=old borg rcreate \-\-encryption=repokey\-aes\-ocb +# now \(dqold\(dq is the current passphrase. +$ BORG_PASSPHRASE=old BORG_NEW_PASSPHRASE=new borg key change\-passphrase +# now \(dqnew\(dq is the current passphrase. .ft P .fi .UNINDENT diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 42e963f0eb..c202b3a87d 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -27,16 +27,16 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-EXPORT" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-KEY-EXPORT" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup .SH SYNOPSIS .sp -borg [common options] key export [options] [REPOSITORY] [PATH] +borg [common options] key export [options] [PATH] .SH DESCRIPTION .sp If repository encryption is used, the repository is inaccessible -without the key. This command allows one to backup this essential key. +without the key. This command allows one to back up this essential key. Note that the backup produced does not include the passphrase itself (i.e. the exported key stays encrypted). In order to regain access to a repository, one needs both the exported key and the original passphrase. @@ -56,30 +56,10 @@ For repositories using the repokey encryption the key is saved in the repository in the config file. A backup is thus not strictly needed, but guards against the repository becoming inaccessible if the file is damaged for some reason. -.sp -Examples: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -borg key export /path/to/repo > encrypted\-key\-backup -borg key export \-\-paper /path/to/repo > encrypted\-key\-backup.txt -borg key export \-\-qr\-html /path/to/repo > encrypted\-key\-backup.html -# Or pass the output file as an argument instead of redirecting stdout: -borg key export /path/to/repo encrypted\-key\-backup -borg key export \-\-paper /path/to/repo encrypted\-key\-backup.txt -borg key export \-\-qr\-html /path/to/repo encrypted\-key\-backup.html -.ft P -.fi -.UNINDENT -.UNINDENT .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments -.sp -REPOSITORY .INDENT 0.0 .TP .B PATH @@ -94,6 +74,23 @@ Create an export suitable for printing and later type\-in .B \-\-qr\-html Create an html file suitable for printing and later type\-in or qr scan .UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +borg key export > encrypted\-key\-backup +borg key export \-\-paper > encrypted\-key\-backup.txt +borg key export \-\-qr\-html > encrypted\-key\-backup.html +# Or pass the output file as an argument instead of redirecting stdout: +borg key export encrypted\-key\-backup +borg key export \-\-paper encrypted\-key\-backup.txt +borg key export \-\-qr\-html encrypted\-key\-backup.html +.ft P +.fi +.UNINDENT +.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-key\-import(1)\fP diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 589d19f545..4240086786 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -27,12 +27,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY-IMPORT" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-KEY-IMPORT" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup .SH SYNOPSIS .sp -borg [common options] key import [options] [REPOSITORY] [PATH] +borg [common options] key import [options] [PATH] .SH DESCRIPTION .sp This command restores a key previously backed up with the export command. @@ -53,8 +53,6 @@ key import\fP creates a new key file in \fB$BORG_KEYS_DIR\fP\&. .sp See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments -.sp -REPOSITORY .INDENT 0.0 .TP .B PATH diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 4f898d7c73..7ce6eb6e93 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-KEY" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-KEY" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository .SH SYNOPSIS @@ -35,12 +35,12 @@ borg-key \- Manage a keyfile or repokey of a repository borg [common options] key export ... borg [common options] key import ... borg [common options] key change\-passphrase ... -borg [common options] key migrate\-to\-repokey ... +borg [common options] key change\-location ... .fi .sp .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP, \fIborg\-key\-migrate\-to\-repokey(1)\fP +\fIborg\-common(1)\fP, \fIborg\-key\-export(1)\fP, \fIborg\-key\-import(1)\fP, \fIborg\-key\-change\-passphrase(1)\fP, \fIborg\-key\-change\-location(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index ccea21110b..9d997585b7 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -27,15 +27,15 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-LIST" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-LIST" 1 "2024-07-19" "" "borg backup tool" .SH NAME -borg-list \- List archive or repository contents +borg-list \- List archive contents .SH SYNOPSIS .sp -borg [common options] list [options] [REPOSITORY_OR_ARCHIVE] [PATH...] +borg [common options] list [options] NAME [PATH...] .SH DESCRIPTION .sp -This command lists the contents of a repository or an archive. +This command lists the contents of an archive. .sp For more help on include/exclude patterns, see the \fIborg_patterns\fP command output. .SH OPTIONS @@ -44,8 +44,8 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B REPOSITORY_OR_ARCHIVE -repository or archive to list contents of +.B NAME +specify the archive name .TP .B PATH paths to list; patterns are supported @@ -53,40 +53,16 @@ paths to list; patterns are supported .SS optional arguments .INDENT 0.0 .TP -.B \-\-consider\-checkpoints -Show checkpoint archives in the repository contents list (default: hidden). -.TP .B \-\-short only print file/directory names, nothing else .TP .BI \-\-format \ FORMAT -specify format for file or archive listing (default for files: "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}"; for archives: "{archive:<36} {time} [{id}]{NL}") -.TP -.B \-\-json -Only valid for listing repository contents. Format output as JSON. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available. +specify format for file listing (default: \(dq{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\(dq) .TP .B \-\-json\-lines -Only valid for listing archive contents. Format output as JSON Lines. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available. -.UNINDENT -.SS Archive filters -.INDENT 0.0 -.TP -.BI \-P \ PREFIX\fR,\fB \ \-\-prefix \ PREFIX -only consider archive names starting with this prefix. -.TP -.BI \-a \ GLOB\fR,\fB \ \-\-glob\-archives \ GLOB -only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. -.TP -.BI \-\-sort\-by \ KEYS -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp -.TP -.BI \-\-first \ N -consider first N archives after other filters were applied -.TP -.BI \-\-last \ N -consider last N archives after other filters were applied +Format output as JSON Lines. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN @@ -107,14 +83,7 @@ read include/exclude patterns from PATTERNFILE, one per line .sp .nf .ft C -$ borg list /path/to/repo -Monday Mon, 2016\-02\-15 19:15:11 -repo Mon, 2016\-02\-15 19:26:54 -root\-2016\-02\-15 Mon, 2016\-02\-15 19:36:29 -newname Mon, 2016\-02\-15 19:50:19 -\&... - -$ borg list /path/to/repo::root\-2016\-02\-15 +$ borg list root\-2016\-02\-15 drwxr\-xr\-x root root 0 Mon, 2016\-02\-15 17:44:27 . drwxrwxr\-x root root 0 Mon, 2016\-02\-15 19:04:49 bin \-rwxr\-xr\-x root root 1029624 Thu, 2014\-11\-13 00:08:51 bin/bash @@ -122,14 +91,14 @@ lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdif \-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff \&... -$ borg list /path/to/repo::root\-2016\-02\-15 \-\-pattern "\- bin/ba*" +$ borg list root\-2016\-02\-15 \-\-pattern \(dq\- bin/ba*\(dq drwxr\-xr\-x root root 0 Mon, 2016\-02\-15 17:44:27 . drwxrwxr\-x root root 0 Mon, 2016\-02\-15 19:04:49 bin lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdiff \-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff \&... -$ borg list /path/to/repo::archiveA \-\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" +$ borg list archiveA \-\-format=\(dq{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}\(dq drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 . drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject @@ -137,11 +106,11 @@ drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.text \&... -$ borg list /path/to/repo/::archiveA \-\-pattern \(aqre:\e.ext$\(aq +$ borg list archiveA \-\-pattern \(aq+ re:\e.ext$\(aq \-\-pattern \(aq\- re:^.*$\(aq \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext \&... -$ borg list /path/to/repo/::archiveA \-\-pattern \(aqre:.ext$\(aq +$ borg list archiveA \-\-pattern \(aq+ re:.ext$\(aq \-\-pattern \(aq\- re:^.*$\(aq \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.ext \-rw\-rw\-r\-\- user user 1416192 Sun, 2015\-02\-01 11:00:00 code/myproject/file.text \&... @@ -160,23 +129,13 @@ Examples: .sp .nf .ft C -$ borg list \-\-format \(aq{archive}{NL}\(aq /path/to/repo -ArchiveFoo -ArchiveBar -\&... - -# {VAR:NUMBER} \- pad to NUMBER columns. -# Strings are left\-aligned, numbers are right\-aligned. -# Note: time columns except \(ga\(gaisomtime\(ga\(ga, \(ga\(gaisoctime\(ga\(ga and \(ga\(gaisoatime\(ga\(ga cannot be padded. -$ borg list \-\-format \(aq{archive:36} {time} [{id}]{NL}\(aq /path/to/repo -ArchiveFoo Thu, 2021\-12\-09 10:22:28 [0b8e9a312bef3f2f6e2d0fc110c196827786c15eba0188738e81697a7fa3b274] -$ borg list \-\-format \(aq{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\(aq /path/to/repo::ArchiveFoo +$ borg list \-\-format \(aq{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\(aq ArchiveFoo \-rw\-rw\-r\-\- user user 1024 Thu, 2021\-12\-09 10:22:17 file\-foo \&... # {VAR:NUMBER} \- pad to NUMBER columns right\-aligned. -$ borg list \-\-format \(aq{mode} {user:>6} {group:>6} {size:<8} {mtime} {path}{extra}{NL}\(aq /path/to/repo::ArchiveFoo +$ borg list \-\-format \(aq{mode} {user:>6} {group:>6} {size:<8} {mtime} {path}{extra}{NL}\(aq ArchiveFoo \-rw\-rw\-r\-\- user user 1024 Thu, 2021\-12\-09 10:22:17 file\-foo \&... .ft P @@ -191,93 +150,59 @@ NEWLINE: OS dependent line separator .IP \(bu 2 NL: alias of NEWLINE .IP \(bu 2 -NUL: NUL character for creating print0 / xargs \-0 like output, see barchive and bpath keys below +NUL: NUL character for creating print0 / xargs \-0 like output .IP \(bu 2 -SPACE +SPACE: space character .IP \(bu 2 -TAB +TAB: tab character .IP \(bu 2 -CR +CR: carriage return character .IP \(bu 2 -LF -.UNINDENT -.sp -Keys available only when listing archives in a repository: -.INDENT 0.0 -.IP \(bu 2 -archive: archive name interpreted as text (might be missing non\-text characters, see barchive) -.IP \(bu 2 -name: alias of "archive" -.IP \(bu 2 -barchive: verbatim archive name, can contain any character except NUL -.IP \(bu 2 -comment: archive comment interpreted as text (might be missing non\-text characters, see bcomment) -.IP \(bu 2 -bcomment: verbatim archive comment, can contain any character except NUL -.IP \(bu 2 -id: internal ID of the archive -.IP \(bu 2 -start: time (start) of creation of the archive -.IP \(bu 2 -time: alias of "start" -.IP \(bu 2 -end: time (end) of creation of the archive -.IP \(bu 2 -command_line: command line which was used to create the archive -.IP \(bu 2 -hostname: hostname of host on which this archive was created -.IP \(bu 2 -username: username of user who created this archive +LF: line feed character .UNINDENT .sp Keys available only when listing files in an archive: .INDENT 0.0 .IP \(bu 2 -type +type: file type (file, dir, symlink, ...) .IP \(bu 2 -mode +mode: file mode (as in stat) .IP \(bu 2 -uid +uid: user id of file owner .IP \(bu 2 -gid +gid: group id of file owner .IP \(bu 2 -user +user: user name of file owner .IP \(bu 2 -group +group: group name of file owner .IP \(bu 2 -path: path interpreted as text (might be missing non\-text characters, see bpath) +path: file path .IP \(bu 2 -bpath: verbatim POSIX path, can contain any character except NUL +target: link target for symlinks .IP \(bu 2 -source: link target for links (identical to linktarget) +hlid: hard link identity (same if hardlinking same fs object) .IP \(bu 2 -linktarget +flags: file flags .IP \(bu 2 -flags -.IP \(bu 2 -size -.IP \(bu 2 -csize: compressed size +size: file size .IP \(bu 2 dsize: deduplicated size .IP \(bu 2 -dcsize: deduplicated compressed size -.IP \(bu 2 num_chunks: number of chunks in this file .IP \(bu 2 unique_chunks: number of unique chunks in this file .IP \(bu 2 -mtime +mtime: file modification time .IP \(bu 2 -ctime +ctime: file change time .IP \(bu 2 -atime +atime: file access time .IP \(bu 2 -isomtime +isomtime: file modification time (ISO 8601 format) .IP \(bu 2 -isoctime +isoctime: file change time (ISO 8601 format) .IP \(bu 2 -isoatime +isoatime: file access time (ISO 8601 format) .IP \(bu 2 blake2b .IP \(bu 2 @@ -305,17 +230,17 @@ sha512 .IP \(bu 2 xxh64: XXH64 checksum of this file (note: this is NOT a cryptographic hash!) .IP \(bu 2 -archiveid +archiveid: internal ID of the archive .IP \(bu 2 -archivename +archivename: name of the archive .IP \(bu 2 -extra: prepends {source} with " \-> " for soft links and " link to " for hard links +extra: prepends {target} with \(dq \-> \(dq for soft links and \(dq link to \(dq for hard links .IP \(bu 2 -health: either "healthy" (file ok) or "broken" (if file has all\-zero replacement chunks) +health: either \(dqhealthy\(dq (file ok) or \(dqbroken\(dq (if file has all\-zero replacement chunks) .UNINDENT .SH SEE ALSO .sp -\fIborg\-common(1)\fP, \fIborg\-info(1)\fP, \fIborg\-diff(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-patterns(1)\fP +\fIborg\-common(1)\fP, \fIborg\-info(1)\fP, \fIborg\-diff(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-rlist(1)\fP .SH AUTHOR The Borg Collective .\" Generated by docutils manpage writer. diff --git a/docs/man/borg-match-archives.1 b/docs/man/borg-match-archives.1 new file mode 100644 index 0000000000..6d5b39ef71 --- /dev/null +++ b/docs/man/borg-match-archives.1 @@ -0,0 +1,75 @@ +.\" Man page generated from reStructuredText. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "BORG-MATCH-ARCHIVES" 1 "2024-07-19" "" "borg backup tool" +.SH NAME +borg-match-archives \- Details regarding match-archives +.SH DESCRIPTION +.sp +The \fB\-\-match\-archives\fP option matches a given pattern against the list of all archive +names in the repository. +.sp +It uses pattern styles similar to the ones described by \fBborg help patterns\fP: +.INDENT 0.0 +.TP +.B Identical match pattern, selector \fBid:\fP (default) +Simple string match, must fully match exactly as given. +.TP +.B Shell\-style patterns, selector \fBsh:\fP +Match like on the shell, wildcards like \fI*\fP and \fI?\fP work. +.TP +.B \fI\%Regular expressions\fP, selector \fBre:\fP +Full regular expression support. +This is very powerful, but can also get rather complicated. +.UNINDENT +.sp +Examples: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# id: style +borg delete \-\-match\-archives \(aqid:archive\-with\-crap\(aq +borg delete \-a \(aqid:archive\-with\-crap\(aq # same, using short option +borg delete \-a \(aqarchive\-with\-crap\(aq # same, because \(aqid:\(aq is the default + +# sh: style +borg delete \-a \(aqsh:home\-kenny\-*\(aq + +# re: style +borg delete \-a \(aqre:pc[123]\-home\-(user1|user2)\-2022\-09\-.*\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index f989d1edc6..81b7fc218d 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -27,18 +27,21 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-MOUNT" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-MOUNT" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem .SH SYNOPSIS .sp -borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT [PATH...] +borg [common options] mount [options] MOUNTPOINT [PATH...] .SH DESCRIPTION .sp -This command mounts an archive as a FUSE filesystem. This can be useful for -browsing an archive or restoring individual files. Unless the \fB\-\-foreground\fP -option is given the command will run in the background until the filesystem -is \fBumounted\fP\&. +This command mounts an archive as a FUSE filesystem. This can be useful +for browsing an archive or restoring individual files. When restoring, +take into account that the current FUSE implementation does not support +special fs flags and ACLs. +.sp +Unless the \fB\-\-foreground\fP option is given the command will run in the +background until the filesystem is \fBumounted\fP\&. .sp The command \fBborgfs\fP provides a wrapper for \fBborg mount\fP\&. This can also be used in fstab entries: @@ -69,16 +72,16 @@ manually. Unlike the \fBuid\fP and \fBgid\fP mount options which affect all file Additional mount options supported by borg: .INDENT 0.0 .IP \(bu 2 -versions: when used with a repository mount, this gives a merged, versioned +\fBversions\fP: when used with a repository mount, this gives a merged, versioned view of the files in the archives. EXPERIMENTAL, layout may change in future. .IP \(bu 2 -allow_damaged_files: by default damaged files (where missing chunks were -replaced with runs of zeros by borg check \fB\-\-repair\fP) are not readable and +\fBallow_damaged_files\fP: by default damaged files (where missing chunks were +replaced with runs of zeros by \fBborg check \-\-repair\fP) are not readable and return EIO (I/O error). Set this option to read such files. .IP \(bu 2 -ignore_permissions: for security reasons the "default_permissions" mount -option is internally enforced by borg. "ignore_permissions" can be given to -not enforce "default_permissions". +\fBignore_permissions\fP: for security reasons the \fBdefault_permissions\fP mount +option is internally enforced by borg. \fBignore_permissions\fP can be given to +not enforce \fBdefault_permissions\fP\&. .UNINDENT .sp The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users @@ -88,7 +91,7 @@ of CPU cores. .sp When the daemonized process receives a signal or crashes, it does not unmount. Unmounting in these cases could cause an active rsync or similar process -to unintentionally delete data. +to delete data unintentionally. .sp When running in the foreground ^C/SIGINT unmounts cleanly, but other signals or crashes do not. @@ -98,9 +101,6 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .SS arguments .INDENT 0.0 .TP -.B REPOSITORY_OR_ARCHIVE -repository or archive to mount -.TP .B MOUNTPOINT where to mount filesystem .TP @@ -119,31 +119,37 @@ stay in foreground, do not daemonize .B \-o Extra mount options .TP -.B \-\-numeric\-owner -deprecated, use \fB\-\-numeric\-ids\fP instead -.TP .B \-\-numeric\-ids use numeric user and group identifiers from archive(s) .UNINDENT .SS Archive filters .INDENT 0.0 .TP -.BI \-P \ PREFIX\fR,\fB \ \-\-prefix \ PREFIX -only consider archive names starting with this prefix. -.TP -.BI \-a \ GLOB\fR,\fB \ \-\-glob\-archives \ GLOB -only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. +.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN +only consider archive names matching the pattern. see \(dqborg help match\-archives\(dq. .TP .BI \-\-sort\-by \ KEYS -Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id; default is: timestamp .TP .BI \-\-first \ N consider first N archives after other filters were applied .TP .BI \-\-last \ N consider last N archives after other filters were applied +.TP +.BI \-\-oldest \ TIMESPAN +consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newest \ TIMESPAN +consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-older \ TIMESPAN +consider archives older than (now \- TIMESPAN), e.g. 7d or 12m. +.TP +.BI \-\-newer \ TIMESPAN +consider archives newer than (now \- TIMESPAN), e.g. 7d or 12m. .UNINDENT -.SS Exclusion options +.SS Include/Exclude options .INDENT 0.0 .TP .BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 9488d77b08..6850b83ed8 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -27,48 +27,45 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "BORG-PATTERNS" 1 "2022-02-19" "" "borg backup tool" +.TH "BORG-PATTERNS" 1 "2024-07-19" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns .SH DESCRIPTION .sp -The path/filenames used as input for the pattern matching start from the -currently active recursion root. You usually give the recursion root(s) -when invoking borg and these can be either relative or absolute paths. -.sp -So, when you give \fIrelative/\fP as root, the paths going into the matcher -will look like \fIrelative/.../file.ext\fP\&. When you give \fI/absolute/\fP as -root, they will look like \fI/absolute/.../file.ext\fP\&. -.sp -File paths in Borg archives are always stored normalized and relative. -This means that e.g. \fBborg create /path/to/repo ../some/path\fP will -store all files as \fIsome/path/.../file.ext\fP and \fBborg create -/path/to/repo /home/user\fP will store all files as -\fIhome/user/.../file.ext\fP\&. -.sp -A directory exclusion pattern can end either with or without a slash (\(aq/\(aq). -If it ends with a slash, such as \fIsome/path/\fP, the directory will be -included but not its content. If it does not end with a slash, such as -\fIsome/path\fP, both the directory and content will be excluded. +When specifying one or more file paths in a Borg command that supports +patterns for the respective option or argument, you can apply the +patterns described here to include only desired files and/or exclude +unwanted ones. Patterns can be used +.INDENT 0.0 +.IP \(bu 2 +for \fB\-\-exclude\fP option, +.IP \(bu 2 +in the file given with \fB\-\-exclude\-from\fP option, +.IP \(bu 2 +for \fB\-\-pattern\fP option, +.IP \(bu 2 +in the file given with \fB\-\-patterns\-from\fP option and +.IP \(bu 2 +for \fBPATH\fP arguments that explicitly support them. +.UNINDENT .sp -File patterns support these styles: fnmatch, shell, regular expressions, -path prefixes and path full\-matches. By default, fnmatch is used for -\fB\-\-exclude\fP patterns and shell\-style is used for the \fB\-\-pattern\fP -option. For commands that support patterns in their \fBPATH\fP argument -like (\fBborg list\fP), the default pattern is path prefix. +Borg always stores all file paths normalized and relative to the +current recursion root. The recursion root is also named \fBPATH\fP in +Borg commands like \fIborg create\fP that do a file discovery, so do not +confuse the root with the \fBPATH\fP argument of e.g. \fIborg extract\fP\&. .sp -Starting with Borg 1.2, for all but regular expression pattern matching -styles, all paths are treated as relative, meaning that a leading path -separator is removed after normalizing and before matching. This allows -you to use absolute or relative patterns arbitrarily. +Starting with Borg 1.2, paths that are matched against patterns always +appear relative. If you give \fB/absolute/\fP as root, the paths going +into the matcher will start with \fBabsolute/\fP\&. +If you give \fB\&../../relative\fP as root, the paths will be normalized +as \fBrelative/\fP\&. .sp -If followed by a colon (\(aq:\(aq) the first two characters of a pattern are -used as a style selector. Explicit style selection is necessary when a -non\-default style is desired or when the desired pattern starts with -two alphanumeric characters followed by a colon (i.e. \fIaa:something/*\fP). +Borg supports different pattern styles. To define a non\-default +style for a specific pattern, prefix it with two characters followed +by a colon \(aq:\(aq (i.e. \fBfm:path/*\fP, \fBsh:path/**\fP). .INDENT 0.0 .TP -.B \fI\%Fnmatch\fP, selector \fIfm:\fP +.B \fI\%Fnmatch\fP, selector \fBfm:\fP This is the default style for \fB\-\-exclude\fP and \fB\-\-exclude\-from\fP\&. These patterns use a variant of shell pattern syntax, with \(aq*\(aq matching any number of characters, \(aq?\(aq matching any single character, \(aq[...]\(aq @@ -76,7 +73,7 @@ matching any single character specified, including ranges, and \(aq[!...]\(aq matching any character not specified. For the purpose of these patterns, the path separator (backslash for Windows and \(aq/\(aq on other systems) is not treated specially. Wrap meta\-characters in brackets for a literal -match (i.e. \fI[?]\fP to match the literal character \fI?\fP). For a path +match (i.e. \fB[?]\fP to match the literal character \(aq?\(aq). For a path to match a pattern, the full path must match, or it must match from the start of the full path to just before a path separator. Except for the root path, paths will never end in the path separator when @@ -84,33 +81,32 @@ matching is attempted. Thus, if a given pattern ends in a path separator, a \(aq*\(aq is appended before matching is attempted. A leading path separator is always removed. .TP -.B Shell\-style patterns, selector \fIsh:\fP +.B Shell\-style patterns, selector \fBsh:\fP This is the default style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP\&. Like fnmatch patterns these are similar to shell patterns. The difference -is that the pattern may include \fI**/\fP for matching zero or more directory -levels, \fI*\fP for matching zero or more arbitrary characters with the -exception of any path separator. A leading path separator is always removed. +is that the pattern may include \fB**/\fP for matching zero or more directory +levels, \fB*\fP for matching zero or more arbitrary characters with the +exception of any path separator, \fB{}\fP containing comma\-separated +alternative patterns. A leading path separator is always removed. .TP -.B Regular expressions, selector \fIre:\fP -Regular expressions similar to those found in Perl are supported. Unlike -shell patterns regular expressions are not required to match the full +.B \fI\%Regular expressions\fP, selector \fBre:\fP +Unlike shell patterns, regular expressions are not required to match the full path and any substring match is sufficient. It is strongly recommended to anchor patterns to the start (\(aq^\(aq), to the end (\(aq$\(aq) or both. Path separators (backslash for Windows and \(aq/\(aq on other systems) in paths are -always normalized to a forward slash (\(aq/\(aq) before applying a pattern. The -regular expression syntax is described in the \fI\%Python documentation for -the re module\fP\&. +always normalized to a forward slash \(aq/\(aq before applying a pattern. .TP -.B Path prefix, selector \fIpp:\fP +.B Path prefix, selector \fBpp:\fP This pattern style is useful to match whole sub\-directories. The pattern -\fIpp:root/somedir\fP matches \fIroot/somedir\fP and everything therein. A leading -path separator is always removed. +\fBpp:root/somedir\fP matches \fBroot/somedir\fP and everything therein. +A leading path separator is always removed. .TP -.B Path full\-match, selector \fIpf:\fP +.B Path full\-match, selector \fBpf:\fP This pattern style is (only) useful to match full paths. This is kind of a pseudo pattern as it can not have any variable or -unspecified parts \- the full path must be given. \fIpf:root/file.ext\fP matches -\fIroot/file.ext\fP only. A leading path separator is always removed. +unspecified parts \- the full path must be given. \fBpf:root/file.ext\fP +matches \fBroot/file.ext\fP only. A leading path separator is always +removed. .sp Implementation note: this is implemented via very time\-efficient O(1) hashtable lookups (this means you can have huge amounts of such patterns @@ -125,12 +121,12 @@ Same logic applies for exclude. \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -\fIre:\fP, \fIsh:\fP and \fIfm:\fP patterns are all implemented on top of the Python SRE -engine. It is very easy to formulate patterns for each of these types which -requires an inordinate amount of time to match paths. If untrusted users -are able to supply patterns, ensure they cannot supply \fIre:\fP patterns. -Further, ensure that \fIsh:\fP and \fIfm:\fP patterns only contain a handful of -wildcards at most. +\fBre:\fP, \fBsh:\fP and \fBfm:\fP patterns are all implemented on top of +the Python SRE engine. It is very easy to formulate patterns for each +of these types which requires an inordinate amount of time to match +paths. If untrusted users are able to supply patterns, ensure they +cannot supply \fBre:\fP patterns. Further, ensure that \fBsh:\fP and +\fBfm:\fP patterns only contain a handful of wildcards at most. .UNINDENT .UNINDENT .sp @@ -139,8 +135,8 @@ from within a shell, the patterns should be quoted to protect them from expansion. .sp The \fB\-\-exclude\-from\fP option permits loading exclusion patterns from a text -file with one pattern per line. Lines empty or starting with the number sign -(\(aq#\(aq) after removing whitespace on both ends are ignored. The optional style +file with one pattern per line. Lines empty or starting with the hash sign +\(aq#\(aq after removing whitespace on both ends are ignored. The optional style selector prefix is also supported for patterns loaded from a file. Due to whitespace removal, paths with whitespace at the beginning or end can only be excluded using regular expressions. @@ -155,74 +151,128 @@ Examples: .nf .ft C # Exclude \(aq/home/user/file.o\(aq but not \(aq/home/user/file.odt\(aq: -$ borg create \-e \(aq*.o\(aq backup / +$ borg create \-e \(aq*.o\(aq archive / # Exclude \(aq/home/user/junk\(aq and \(aq/home/user/subdir/junk\(aq but # not \(aq/home/user/importantjunk\(aq or \(aq/etc/junk\(aq: -$ borg create \-e \(aq/home/*/junk\(aq backup / +$ borg create \-e \(aqhome/*/junk\(aq archive / # Exclude the contents of \(aq/home/user/cache\(aq but not the directory itself: -$ borg create \-e home/user/cache/ backup / +$ borg create \-e home/user/cache/ archive / # The file \(aq/home/user/cache/important\(aq is *not* backed up: -$ borg create \-e /home/user/cache/ backup / /home/user/cache/important +$ borg create \-e home/user/cache/ archive / /home/user/cache/important # The contents of directories in \(aq/home\(aq are not backed up when their name # ends in \(aq.tmp\(aq -$ borg create \-\-exclude \(aqre:^/home/[^/]+\e.tmp/\(aq backup / +$ borg create \-\-exclude \(aqre:^home/[^/]+\e.tmp/\(aq archive / # Load exclusions from file $ cat >exclude.txt <