From c7369ef2d6fea025391b364c21ef156852086f25 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 14 Jan 2026 16:36:42 -0500 Subject: [PATCH 01/47] fixed some local variable definitions, empty list, and RECV_DEFAULT --- share/zelta/zelta-args.awk | 19 +++++++++++-------- share/zelta/zelta-backup.awk | 11 +++++++---- share/zelta/zelta-match.awk | 7 +++---- share/zelta/zelta-policy.awk | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/share/zelta/zelta-args.awk b/share/zelta/zelta-args.awk index ee61781..cee126b 100644 --- a/share/zelta/zelta-args.awk +++ b/share/zelta/zelta-args.awk @@ -74,15 +74,18 @@ function set_arg(flag, subopt, _type, key) { } # Handle 'set' and 'list' action logic -function get_subopt(flag, m, _subopt) { +function get_subopt(flag, m, _subopt, _has_equals) { + # Check if the original input had an '=' (for --key= or --key=value) + _has_equals = index($0, "=") + # If a key=value is given out of context, stop - if ($2 && (!(OptListType[flag] in SUBOPT_TYPES) || OptListValue[flag])) { + if (_has_equals && (!(OptListType[flag] in SUBOPT_TYPES) || OptListValue[flag])) { stop(1, "invalid option assignment '"$0"'") } # Not a subopt type else if (!(OptListType[flag] in SUBOPT_TYPES)) return "" - # --key=value - else if ($2) return $2 + # --key=value or --key= (empty value is valid) + else if (_has_equals) return $2 # Value is defined upstream else if (OptListValue[flag]) return OptListValue[flag] # Single dash option @@ -93,11 +96,11 @@ function get_subopt(flag, m, _subopt) { } # Note, we're modifying ARGV's 'Idx' as a global because the logic to reconcile # this otherwise would be gnarly and inefficient. - # Find '--key value' or '-k 1' - _subopt = ARGV[++Idx] - if (!_subopt) { + # Find '--key value' or '-k 1' (where value may be empty string) + if (++Idx >= ARGC) { stop(1, "option '"$1"' requires an argument") - } else return _subopt + } + return ARGV[Idx] } function get_args( _i, _flag, _arg, _m, _subopt, _opts_done) { diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 914e8a2..9b2a70f 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -19,6 +19,7 @@ # Opt: User settings (see the 'zelta' sh script and zelta-opts.tsv) # NumDS: Number of datasets in the tree # DSList: List of "ds_suffix" elements in replication order +# FailedProps: List of "permission denied" messages received during sync # Dataset: Properties of each dataset, indexed by: ("ENDPOINT", ds_suffix, element) # [zfsprops]: ZFS properties from the property-source 'local' or 'none' # exists @@ -293,7 +294,7 @@ function explain_sync_status(ds_suffix, _src_idx, _tgt_idx, _src_ds, _tgt_ds) } # Ensure source snapshots are avialable and load snapshot relationship data -function validate_snapshots( _i, _ds_suffix, _src_idx, _match, _src_latest) { +function validate_snapshots( _i, _ds_suffix, _src_idx, _match, _src_exists, _src_latest) { create_source_snapshot() load_snapshot_deltas() for (_i in DSList) { @@ -516,8 +517,8 @@ function dataset_exists(ep, ds, _cmd_arr, _cmd, _ds_exists, _remote) { # so this is a perfect test and it's a requirement if the CHECK_PARENT option is given. Unfortunately, # a nasty ZFS bug means that 'zfs create' won't work with readonly datasets, or datasets the user doesn't # have access to. Thus, we cannot avoid the following gnarly logic. -function validate_target_parent_dataset( _parent, _cmd, _cmd_arr, _depth, - _ds_exists, _retry, _null_arr) { +function validate_target_parent_dataset( _parent, _cmd, _cmd_arr, _depth, _hit_readonly, + _attempts, _ds_exists, _retry, _success, _null_arr) { if (DSTree["target_exists"]) return 1 _parent = Opt["TGT_DS"] sub(/\/[^\/]*$/, "", _parent) # Strip last child element @@ -712,6 +713,8 @@ function create_send_command(ds_suffix, idx, remote_ep, _cmd_arr, _cmd, _ds_sn function get_recv_command_flags(ds_suffix, src_idx, remote_ep, _flag_arr, _flags, _i, _origin) { if (Opt["RECV_OVERRIDE"]) return Opt["RECV_OVERRIDE"] + if (Opt["RECV_DEFAULT"]) + _flag_arr[++_i] = Opt["RECV_DEFAULT"] # If this is a full backup, rotate, or rebase, set contextual properties if (!DSPair[ds_suffix, "match"] || DSPair[ds_suffix, "target_origin"]) { if (ds_suffix == "") @@ -1063,7 +1066,7 @@ function run_backup( _i, _ds_suffix, _syncable) { } } -function print_summary( _i, _ds_suffix, _num_streams) { +function print_summary( _status, _i, _ds_suffix, _num_streams) { if(Summary["failed_props"]) report(LOG_WARNING, "missing `zfs allow` permissions: " Summary["failed_props"]) if (DSTree["up_to_date"] == NumDS) { diff --git a/share/zelta/zelta-match.awk b/share/zelta/zelta-match.awk index 902c573..82dc402 100644 --- a/share/zelta/zelta-match.awk +++ b/share/zelta/zelta-match.awk @@ -221,18 +221,17 @@ function process_row(ep, _name, _guid, _written, _name_suffix, _ds_suffix, _sav } # Check for exceptions or time(1) output, or process the row -function load_zfs_list_row(ep) { +function load_zfs_list_row(ep, _time_arr) { IGNORE_ZFS_LIST_OUTPUT="(sys|user)[ \t]+[0-9]|dataset does not exist" if ($0 ~ IGNORE_ZFS_LIST_OUTPUT) return if (/^real[ \t]+[0-9]/) { - split($0, time_arr, /[ \t]+/) - ep["list_time"] += time_arr[2] + split($0, _time_arr, /[ \t]+/) + ep["list_time"] += _time_arr[2] } else if ($2 ~ /^[0-9]+$/) { process_row(ep) } else { report(LOG_WARNING, "stream output unexpected: "$0) - exit_code = 1 } } diff --git a/share/zelta/zelta-policy.awk b/share/zelta/zelta-policy.awk index 5b727e5..59bd35a 100644 --- a/share/zelta/zelta-policy.awk +++ b/share/zelta/zelta-policy.awk @@ -108,7 +108,7 @@ function set_var(option_list, var, val) { } # Load policy options from TSV file into global arrays for scope and type tracking -function load_option_list( _tsv, _key, _idx, _flags, _flag_arr) { +function load_option_list( _tsv, _key, _idx, _flags, _flag_arr, _legacy_arr) { _tsv = Opt["SHARE"]"/zelta-opts.tsv" # TO-DO: Complain if TSV doesn't load FS="\t" From 64bf0fd806e8d077e522fe6a34c5f9dc5132428c Mon Sep 17 00:00:00 2001 From: rlogwood Date: Wed, 14 Jan 2026 21:24:36 -0500 Subject: [PATCH 02/47] Feature/zelta test (#59) add shellspec testing, see spec/README.md for details --- .editorconfig | 3 + .gitignore | 11 +- .shellspec | 17 + share/zelta/zelta-backup.awk | 9 +- share/zelta/zelta-match.awk | 7 +- share/zelta/zelta-policy.awk | 2 +- spec/.gitignore | 1 + spec/README.md | 228 ++++++++++++ spec/banner | 6 + spec/bin/all_tests_setup/all_tests_setup.sh | 66 ++++ spec/bin/all_tests_setup/common_test_env.sh | 79 +++++ .../create_file_backed_zfs_test_pools.sh | 272 +++++++++++++++ spec/bin/all_tests_setup/env_constants.sh | 33 ++ .../all_tests_setup/install_local_zelta.sh | 8 + .../bin/divergent_test/divergent_snap_tree.sh | 69 ++++ spec/bin/divergent_test/divergent_test_env.sh | 6 + .../bin/divergent_test/divergent_test_spec.sh | 327 ++++++++++++++++++ spec/bin/hello_example.sh | 23 ++ spec/bin/one_time_setup/setup_sudoers.sh | 88 +++++ .../setup_remote_host_test_env.sh | 38 ++ spec/bin/standard_test/standard_snap_tree.sh | 137 ++++++++ spec/bin/standard_test/standard_test_env.sh | 6 + spec/bin/standard_test/standard_test_spec.sh | 153 ++++++++ spec/doc/vm/README.md | 5 + spec/doc/vm/creation.md | 41 +++ spec/doc/vm/installing-kvm.md | 55 +++ spec/doc/vm/running.md | 0 spec/doc/vm/zfs-configuration.md | 13 + spec/lib/common.sh | 206 +++++++++++ spec/lib/hello.sh | 4 + spec/lib/script_util.sh | 41 +++ spec/spec_helper.sh | 111 ++++++ spec/util/README.md | 47 +++ spec/util/generate_case_stmt_func.awk | 40 +++ spec/util/matcher_func_generator.sh | 20 ++ .../zelta_backup_after_rotate_output.txt | 3 + .../zelta_match_after_backup_output.txt | 10 + spec/util/test_data/zelta_match_output.txt | 11 + test/test_runner.sh | 87 +++++ 39 files changed, 2273 insertions(+), 10 deletions(-) create mode 100644 .editorconfig create mode 100644 .shellspec create mode 100644 spec/.gitignore create mode 100644 spec/README.md create mode 100644 spec/banner create mode 100755 spec/bin/all_tests_setup/all_tests_setup.sh create mode 100755 spec/bin/all_tests_setup/common_test_env.sh create mode 100755 spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh create mode 100644 spec/bin/all_tests_setup/env_constants.sh create mode 100755 spec/bin/all_tests_setup/install_local_zelta.sh create mode 100755 spec/bin/divergent_test/divergent_snap_tree.sh create mode 100644 spec/bin/divergent_test/divergent_test_env.sh create mode 100644 spec/bin/divergent_test/divergent_test_spec.sh create mode 100644 spec/bin/hello_example.sh create mode 100755 spec/bin/one_time_setup/setup_sudoers.sh create mode 100755 spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh create mode 100755 spec/bin/standard_test/standard_snap_tree.sh create mode 100755 spec/bin/standard_test/standard_test_env.sh create mode 100644 spec/bin/standard_test/standard_test_spec.sh create mode 100644 spec/doc/vm/README.md create mode 100644 spec/doc/vm/creation.md create mode 100644 spec/doc/vm/installing-kvm.md create mode 100644 spec/doc/vm/running.md create mode 100644 spec/doc/vm/zfs-configuration.md create mode 100644 spec/lib/common.sh create mode 100644 spec/lib/hello.sh create mode 100644 spec/lib/script_util.sh create mode 100644 spec/spec_helper.sh create mode 100644 spec/util/README.md create mode 100644 spec/util/generate_case_stmt_func.awk create mode 100755 spec/util/matcher_func_generator.sh create mode 100644 spec/util/test_data/zelta_backup_after_rotate_output.txt create mode 100644 spec/util/test_data/zelta_match_after_backup_output.txt create mode 100644 spec/util/test_data/zelta_match_output.txt create mode 100755 test/test_runner.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..47ff004 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.sh] +indent_style = space +indent_size = 4 diff --git a/.gitignore b/.gitignore index a0636aa..f3aefbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,14 @@ +!.gitignore +!.shellspec *.swp *.swo .* doc/man? -!.gitignore +tmp +spec/tmp +hide.* +*~ +\#*# +TAGS +.#* +retired/ diff --git a/.shellspec b/.shellspec new file mode 100644 index 0000000..d0d9e43 --- /dev/null +++ b/.shellspec @@ -0,0 +1,17 @@ +--require spec_helper + +# Set test environment variables +#--env-from spec/initialize/test_env.sh + +## Default kcov (coverage) options +# --kcov-options "--include-path=. --path-strip-level=1" +# --kcov-options "--include-pattern=.sh" +# --kcov-options "--exclude-pattern=/.shellspec,/spec/,/coverage/,/report/" + +## Example: Include script "myprog" with no extension +# --kcov-options "--include-pattern=.sh,myprog" + +## Example: Only specified files/directories +# --kcov-options "--include-pattern=myprog,/lib/" + + diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 914e8a2..3c5772e 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -19,6 +19,7 @@ # Opt: User settings (see the 'zelta' sh script and zelta-opts.tsv) # NumDS: Number of datasets in the tree # DSList: List of "ds_suffix" elements in replication order +# FailedProps: List of "permission denied" messages received during sync # Dataset: Properties of each dataset, indexed by: ("ENDPOINT", ds_suffix, element) # [zfsprops]: ZFS properties from the property-source 'local' or 'none' # exists @@ -293,7 +294,7 @@ function explain_sync_status(ds_suffix, _src_idx, _tgt_idx, _src_ds, _tgt_ds) } # Ensure source snapshots are avialable and load snapshot relationship data -function validate_snapshots( _i, _ds_suffix, _src_idx, _match, _src_latest) { +function validate_snapshots( _i, _ds_suffix, _src_idx, _match, _src_exists, _src_latest) { create_source_snapshot() load_snapshot_deltas() for (_i in DSList) { @@ -516,8 +517,8 @@ function dataset_exists(ep, ds, _cmd_arr, _cmd, _ds_exists, _remote) { # so this is a perfect test and it's a requirement if the CHECK_PARENT option is given. Unfortunately, # a nasty ZFS bug means that 'zfs create' won't work with readonly datasets, or datasets the user doesn't # have access to. Thus, we cannot avoid the following gnarly logic. -function validate_target_parent_dataset( _parent, _cmd, _cmd_arr, _depth, - _ds_exists, _retry, _null_arr) { +function validate_target_parent_dataset( _parent, _cmd, _cmd_arr, _depth, _hit_readonly, + _attempts, _ds_exists, _retry, _success, _null_arr) { if (DSTree["target_exists"]) return 1 _parent = Opt["TGT_DS"] sub(/\/[^\/]*$/, "", _parent) # Strip last child element @@ -1063,7 +1064,7 @@ function run_backup( _i, _ds_suffix, _syncable) { } } -function print_summary( _i, _ds_suffix, _num_streams) { +function print_summary( _status, _i, _ds_suffix, _num_streams) { if(Summary["failed_props"]) report(LOG_WARNING, "missing `zfs allow` permissions: " Summary["failed_props"]) if (DSTree["up_to_date"] == NumDS) { diff --git a/share/zelta/zelta-match.awk b/share/zelta/zelta-match.awk index 902c573..82dc402 100644 --- a/share/zelta/zelta-match.awk +++ b/share/zelta/zelta-match.awk @@ -221,18 +221,17 @@ function process_row(ep, _name, _guid, _written, _name_suffix, _ds_suffix, _sav } # Check for exceptions or time(1) output, or process the row -function load_zfs_list_row(ep) { +function load_zfs_list_row(ep, _time_arr) { IGNORE_ZFS_LIST_OUTPUT="(sys|user)[ \t]+[0-9]|dataset does not exist" if ($0 ~ IGNORE_ZFS_LIST_OUTPUT) return if (/^real[ \t]+[0-9]/) { - split($0, time_arr, /[ \t]+/) - ep["list_time"] += time_arr[2] + split($0, _time_arr, /[ \t]+/) + ep["list_time"] += _time_arr[2] } else if ($2 ~ /^[0-9]+$/) { process_row(ep) } else { report(LOG_WARNING, "stream output unexpected: "$0) - exit_code = 1 } } diff --git a/share/zelta/zelta-policy.awk b/share/zelta/zelta-policy.awk index 5b727e5..59bd35a 100644 --- a/share/zelta/zelta-policy.awk +++ b/share/zelta/zelta-policy.awk @@ -108,7 +108,7 @@ function set_var(option_list, var, val) { } # Load policy options from TSV file into global arrays for scope and type tracking -function load_option_list( _tsv, _key, _idx, _flags, _flag_arr) { +function load_option_list( _tsv, _key, _idx, _flags, _flag_arr, _legacy_arr) { _tsv = Opt["SHARE"]"/zelta-opts.tsv" # TO-DO: Complain if TSV doesn't load FS="\t" diff --git a/spec/.gitignore b/spec/.gitignore new file mode 100644 index 0000000..a9a5aec --- /dev/null +++ b/spec/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..af49785 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,228 @@ +# Shellspec Testing +* * * + +* [Shellspec Testing](#shellspec-testing) + * [Overview](#overview) + * [Installing ShellSpec](#installing-shellspec) + * [🔰 Making your first test - a simple Example](#-making-your-first-test---a-simple-example) + * [Setting up a local development environment](#setting-up-a-local-development-environment) + * [Testing `zelta`](#testing-zelta) + * [:zap: To run the standard Zelta test](#zap-to-run-the-standard-zelta-test-) + * [:zap: To run all tests](#zap-to-run-all-tests) + * [🔰 shellspec examples](#-shellspec-examples) + +* * * +## Overview + +[zelta](https://zelta.space/) uses [ShellSpec](https://github.com/shellspec/shellspec) for testing. If you're new to ShellSpec the +the following links are helpful: +- https://github.com/shellspec/shellspec +- https://shellspec.info/ +- :star: https://deepwiki.com/shellspec/shellspec :heart: + +### Installing ShellSpec + +See the [ShellSpec installation guide](https://github.com/shellspec/shellspec#installation) for instructions. +- The following works on FreeBSD and Ubuntu +- `curl -fsSL https://git.io/shellspec | sh` +- Add `$HOME/.local/bin` to your `PATH` + +#### 🔰 Making your first test - a simple Example +Use the hello_spec.sh file as a template for your first test. +- [hello_example.sh](./bin/hello_example.sh) +```shell +shellspec -f d spec/bin/hello_example.sh +``` + +### Setting up a local development environment +- [Ubuntu VM](./doc/vm/README.md) + + +## Testing `zelta` +> 🔑 zfs must be installed on your system. +> +> â„šī¸ sudo is required because root privilege is needed for zfs and zelta commands +> +> â›‘ī¸ Only temporary file backed zfs pools are used during testing +> +> đŸĻē Installs are local to a temporary directory +* * * +### :star: Using the test helper +```shell +% ~/src/repos/zelta$ test/test_runner.sh +Error: Expected 2 arguments: +Usage: test/test_runner.sh +``` +- For remote tests setup your server and backup user +- Export the following env vars before running +```shell +# for example +# TODO: use the same server, multiple servers is a WIP +# NOTE: different servers for SRC and TGT is a WIP +export SRC_SVR="backupuser@server" +export TGT_SVR="backupuser@server" +``` + +- Typical test secnarios for local testing + - TODO: encrypted trees aren't implemented yet +``` +test/test_runner.sh local standard +test/test_runner.sh local divergent +test/test_runner.sh remote standard +test/test_runner.sh remote divergent +``` + +### Example test_runner.sh output +- Recommendation: test locally first, before trying remote + +
+ +remote standard run + +```shell +% test/test_runner.sh remote standard + +# setup output omitted +# respond to install prompt and sudo password for pool setup + _____ _ _ _____ _ +|__ /___| | |_ __ _ |_ _|__ ___| |_ + / // _ \ | __/ _` | | |/ _ \/ __| __| + / /| __/ | || (_| | | | __/\__ \ |_ +/____\___|_|\__\__,_| |_|\___||___/\__| + +[info] specshell precheck: version:0.28.1 shell: sh +[info] *** TREE_NAME is {standard} +[info] *** RUNNING_MODE is {remote} +[info] *** +[info] *** Running Remotely +[info] *** Source Server is SRC_SVR:{dever@fzfsdev} +[info] *** Target Server is TGT_SVR:{dever@fzfsdev} +[info] *** +Settings OS specific environment for {Linux} +OS_TYPE: Linux: set POOL_TYPE={2} +Running: /bin/sh [sh] + +confirm zfs setup + has good initial SRC_POOL:{apool} simple snap tree + has good initial TGT_POOL:{bpool} simple snap tree +try backup + backs up the initial tree + has valid backup + has 8 snapshots on dever@fzfsdev matching pattern '^(apool|bpool)' + has 4 snapshots on dever@fzfsdev matching pattern 'apool/treetop' + has 4 snapshots on dever@fzfsdev matching pattern 'bpool/backups/treetop' +zelta rotate + rotates the backed up tree + has 16 snapshots on dever@fzfsdev matching pattern '^(apool|bpool)' + has 8 snapshots on dever@fzfsdev matching pattern 'apool/treetop' + has 8 snapshots on dever@fzfsdev matching pattern 'bpool/backups/treetop' + +Finished in 7.30 seconds (user 1.60 seconds, sys 0.12 seconds) +11 examples, 0 failures + + +✓ Tests complete + +``` + +
+ + +
+ +remote divergent run + +```shell +% test/test_runner.sh remote divergent + +# setup output omitted +# respond to install prompt and sudo password for pool setup + + _____ _ _ _____ _ +|__ /___| | |_ __ _ |_ _|__ ___| |_ + / // _ \ | __/ _` | | |/ _ \/ __| __| + / /| __/ | || (_| | | | __/\__ \ |_ +/____\___|_|\__\__,_| |_|\___||___/\__| + +[info] specshell precheck: version:0.28.1 shell: sh +[info] *** TREE_NAME is {divergent} +[info] *** RUNNING_MODE is {remote} +[info] *** +[info] *** Running Remotely +[info] *** Source Server is SRC_SVR:{dever@fzfsdev} +[info] *** Target Server is TGT_SVR:{dever@fzfsdev} +[info] *** +Settings OS specific environment for {Linux} +OS_TYPE: Linux: set POOL_TYPE={2} +Running: /bin/sh [sh] + +confirm zfs setup + zfs list output validation + matches expected pattern for each line + check initial zelta match state + initial match has 5 up-to-date, 1 syncable, 3 blocked, with 9 total datasets compared + add incremental source snapshot + adds dever@fzfsdev:apool/treetop/sub3@two snapshot + add divergent snapshots of same name + adds divergent snapshots for dever@fzfsdev:apool/treetop/sub2@two and dever@fzfsdev:bpool/backups/treetop/sub2@two + check zelta match after divergent snapshots + after divergent snapshot match has 2 up-to-date, 2 syncable, 5 blocked, with 9 total datasets compared +Divergent match, rotate, match + shows current match for divergent dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop + rotate divergent dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop + match dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop after divergent rotate +Divergent backup, then match + backup divergent dever@fzfsdev:apool/treetop to dever@fzfsdev:bpool/backups/treetop + match after backup + +Finished in 7.95 seconds (user 2.09 seconds, sys 0.22 seconds) +10 examples, 0 failures + + +✓ Tests complete +``` + +
+ +* * * +### :zap: To run the standard Zelta test +[zelta_standard_test_spec.sh](./bin/zelta_standard_test_spec.sh) + + ``` + sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh + ``` + 🔎 This test will create a standard setup via [initialize_testing_setup.sh](initialize/initialize_testing_setup.sh) +* * * +### :zap: To run all tests +> â„šī¸ Tests will run in the order they are listed in the spec directory +> use `-P, --pattern PATTERN` to filter tests by pattern +> the default pattern is `"*_spec.sh"` +```shell +sudo -E env "PATH=$PATH" shellspec -f d +``` + +* * * +### 🔰 shellspec examples +- Run all files matching a pattern [default: "*_spec.sh"] +`sudo -E env "PATH=$PATH" shellspec -f d -P "*_setup_*"` +- List all Groups (`Describe`) and Examples (`It`) + ```shell + # shellspec --list examples (directory/file) + $ shellspec --list examples spec/bin + spec/bin/zelta_standard_test_spec.sh:@1-1 + spec/bin/zelta_standard_test_spec.sh:@1-2 + spec/bin/zelta_standard_test_spec.sh:@2-1 + spec/bin/zelta_standard_test_spec.sh:@2-2 + ``` +- `:@1` 🟰 Run all examples in group @1 + ```shell + sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh:@1 + ``` +- `:@-1` 🟰 Run only example #1 in group @1 + ```shell + sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh:@1-1 + ``` +- use options `--xtrace --shell bash` to show a trace with expectation evaluation + ```shell + shellspec -f d --xtrace --shell bash spec/bin/standard_test/standard_test_spec.sh:@2-2 + ``` diff --git a/spec/banner b/spec/banner new file mode 100644 index 0000000..42edab5 --- /dev/null +++ b/spec/banner @@ -0,0 +1,6 @@ + _____ _ _ _____ _ +|__ /___| | |_ __ _ |_ _|__ ___| |_ + / // _ \ | __/ _` | | |/ _ \/ __| __| + / /| __/ | || (_| | | | __/\__ \ |_ +/____\___|_|\__\__,_| |_|\___||___/\__| + diff --git a/spec/bin/all_tests_setup/all_tests_setup.sh b/spec/bin/all_tests_setup/all_tests_setup.sh new file mode 100755 index 0000000..f79559f --- /dev/null +++ b/spec/bin/all_tests_setup/all_tests_setup.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +echo "check SRC_SVR:{$SRC_SVR}" +echo "check TGT_SVR:{$SRC_SVR}" + +set -x +. spec/bin/all_tests_setup/common_test_env.sh +set +x + +. spec/lib/common.sh + + +verify_root() { + # Check if running as root + if [ "$(id -u)" -ne 0 ]; then + echo "Error: You must run as root or with sudo" >&2 + return 1 + fi +} + +initialize_zelta_test() { + echo "-- BeforeAll setup" + + echo "-- installing zelta" + "${ALL_TESTS_SETUP_DIR}"/install_local_zelta.sh + INSTALL_STATUS=$? + if [ $INSTALL_STATUS -ne 0 ]; then + echo "** Error: zelta install failed" + fi + + #"${INITIALIZE_DIR}"/create_device_backed_zfs_test_pools.sh + #"${INITIALIZE_DIR}"/create_file_backed_zfs_test_pools.sh + #TREE_STATUS=$? + +# echo "-- creating test pools" + if "${ALL_TESTS_SETUP_DIR}"/create_file_backed_zfs_test_pools.sh; then + #if "${INITIALIZE_DIR}"/create_device_backed_zfs_test_pools.sh; then + #echo "-- setting up snap tree" + #"${INITIALIZE_DIR}"/setup_simple_snap_tree.sh + TREE_STATUS=$? + # NOTE: moving create snap tree into test specs, we'll have different kinds of trees + #TREE_STATUS=0 + else + echo "** Error: failed to setup zfs pool" >&2 + TREE_STATUS=1 + fi + + #CREATE_STATUS=$? + + #echo "-- Create pool status: {$CREATE_STATUS}" + echo "-- Install Zelta status: {$INSTALL_STATUS}" + echo "-- Make snap tree status: {$TREE_STATUS}" + + #SETUP_STATUS=$((CREATE_STATUS || INSTALL_STATUS || TREE_STATUS)) + SETUP_STATUS=$((INSTALL_STATUS || TREE_STATUS)) + echo "-- returning SETUP_STATUS:{$SETUP_STATUS}" + + if [ $SETUP_STATUS -ne 0 ]; then + echo "** Error: zfs pool and/or zelta install failed!" >&2 + fi + + return $SETUP_STATUS +} + +# NOTE: root is no longer required, unless there is a sloppy state left by improper sudo use +initialize_zelta_test diff --git a/spec/bin/all_tests_setup/common_test_env.sh b/spec/bin/all_tests_setup/common_test_env.sh new file mode 100755 index 0000000..8ceedc7 --- /dev/null +++ b/spec/bin/all_tests_setup/common_test_env.sh @@ -0,0 +1,79 @@ +#!/bin/sh + +. spec/lib/common.sh + +CUR_DIR=$(pwd) + +. spec/bin/all_tests_setup/env_constants.sh + +# backup username, should be configured for ssh access on remotes +export BACKUP_USER="${BACKUP_USER:-dever}" +#export BACKUP_USER="${SUDO_USER:-$(whoami)}" + +# location for git pulls of source for testing +export ZELTA_GIT_CLONE_DIR="${ZELTA_GIT_CLONE_DIR:-/tmp/zelta-dev}" +export GIT_TEST_BRANCH=${GIT_TEST_BRANCH:-feature/zelta-test} + +# remote test host is the machine we'll setup for remote testing +# when a target server is specified, it should be the REMOTE_TEST_HOST + +# TODO: consider eliminating REMOTE_TEST_HOST, it looks redundant, consider TGT_SVR, updated dependencies, review, test + +export REMOTE_TEST_HOST=${REMOTE_TEST_HOST:-fzfsdev} + +# Zelta supports remote commands, by default SRC and TGT servers are the current host +export SRC_SVR="${SRC_SVR:-}" +export TGT_SVR="${TGT_SVR:-}" + + +if [ -z "$SRC_SVR" ]; then + export ZELTA_SRC_POOL="${SRC_POOL}" +else + export ZELTA_SRC_POOL="${SRC_SVR}:${SRC_POOL}" +fi + +if [ -z "$TGT_SVR" ]; then + export ZELTA_TGT_POOL="${TGT_POOL}" +else + export ZELTA_TGT_POOL="${TGT_SVR}:${TGT_POOL}" +fi + +ALL_TESTS_SETUP_DIR=${CUR_DIR}/spec/bin/all_tests_setup + +export LOCAL_TMP="${CUR_DIR}/spec/tmp" +export TEST_INSTALL="${LOCAL_TMP}/test_install" +export ZELTA_BIN="$TEST_INSTALL/bin" +export ZELTA_SHARE="$TEST_INSTALL/share/zelta" +export ZELTA_ETC="$TEST_INSTALL/zelta" +export ZELTA_MAN8="$TEST_INSTALL/share/man/man8" + +# TODO: remove device support completely or clean it up, currently using image files for pools +# TODO: if keeping it, clean up the tested code for this and support the POOL_TYPE FLAG that selects it +# Default devices if not set +: ${SRC_POOL_DEVICES:="/dev/nvme1n1"} +: ${TGT_POOL_DEVICES:="/dev/nvme2n1"} + +export SRC_POOL_DEVICES +export TGT_POOL_DEVICES + +export ZFS_MOUNT_BASE="${LOCAL_TMP}/zfs-test-mounts" +export ZELTA_ZFS_STORE_TEST_DIR="${LOCAL_TMP}/zelta-zfs-store-test" +export ZELTA_ZFS_TEST_POOL_SIZE="20G" + + +# set default pool type +export POOL_TYPE=$FILE_IMG_POOL + +setup_os_specific_env +#echo "Using POOL_TYPE: {$POOL_TYPE}" + +check_zfs_installed + +# If you need to modify the version of awk used +#export ZELTA_AWK=mawk + +export PATH="${ZELTA_BIN}:$PATH" + +# make exec_cmd silent +# export EXEC_CMD_QUIET=1 + diff --git a/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh b/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh new file mode 100755 index 0000000..39033f1 --- /dev/null +++ b/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh @@ -0,0 +1,272 @@ +#!/bin/sh + +. spec/lib/common.sh + +pool_image_file() { + pool_name=$1 + echo "${ZELTA_ZFS_STORE_TEST_DIR}/${pool_name}.img" # return value is output to stdout +} + + +md_unit_file() { + pool_name=$1 + echo "${ZELTA_ZFS_STORE_TEST_DIR}/${pool_name}.md" # return value is output to stdout +} + + +create_freebsd_mem_disk_pool() { + pool_name=$1 + img_file=$(pool_image_file $pool_name) + create_file_img $img_file + + # Attach as memory disk + md_unit=$(mdconfig -a -t vnode -f "$img_file") + echo "Created $md_unit for $pool_name" + + # Create pool on the device + exec_cmd sudo zpool create "$pool_name" "/dev/$md_unit" + + # Store md unit for cleanup + md_file=$(md_unit_file "$pool_name") + echo "$md_unit" > "$md_file" +} + +## Destroy image-backed pool on FreeBSD +# NOTE: the pool has already been destroyed +destroy_freebsd_mem_disk_md_device_and_img() { + pool_name=$1 + img_file=$(pool_image_file "$pool_name") + + md_file=$(md_unit_file "$pool_name") + + # Detach md device + if [ -f "$md_file" ]; then + md_unit=$(cat "$md_file") + mdconfig -d -u "${md_unit#md}" + rm "$md_file" + fi + + # Remove image file + rm -f "$img_file" +} + +create_freebsd_test_pool() { + pool_name=$1 + echo_alert "running create_freebsd_test_pool - pool_name {$pool_name}" + destroy_freebsd_mem_disk_md_device_and_img $pool_name + create_freebsd_mem_disk_pool $pool_name +} + + + +check_pool_exists() { + pool_name="$1" + if [ -z "$pool_name" ]; then + echo "** Error: Pool name required" >&2 + return 1 + fi + exec_cmd sudo zpool list "$pool_name" >/dev/null 2>&1 +} + + + +destroy_pool() { + pool_name=$1 + echo "Destroying pool '$pool_name'..." + if exec_cmd sudo zpool export -f "$pool_name"; then + # TODO: the export seems to remove the pool and then zpool destory fails + # TODO: research this + exec_cmd sudo zpool destroy -f "$pool_name" + + fi + + # since the above isn't working as expected, we check if the pool + # still exists and return an error if it does + if check_pool_exists $pool_name; then + return 1 + fi + + # forcing this to return 0 because of the above + #return 0 + +# if ! exec_cmd sudo zpool destroy -f "$pool_name"; then +# echo "Destroy for pool '$pool_name' failed, trying export then destroy" +# # Export is only needed when the pool is busy/imported but destroy can't complete +# exec_cmd sudo zpool export -f "$pool_name" && exec_cmd sudo zpool destroy -f "$pool_name" +# fi +} + +## 1. Export (destroy) the pool +#zfs destroy -r poolname # destroys all datasets (optional, if you want to be thorough) +#zpool destroy poolname # destroys the pool itself +# +## 2. Detach the loop device +#sudo losetup -d /dev/loop0 # replace loop0 with your actual loop device + + + + + +destroy_pool_if_exists() { + pool_name="$1" + if check_pool_exists "$pool_name"; then + destroy_pool "$pool_name" + else + echo "Pool '$pool_name' does not exist, no need to destroy" + fi +} + +rm_img_and_its_loop_devices() { + img=$1 + echo "removing loop devices associated with image file: {$img}" + + # Remove all loop devices at once + sudo losetup -j "$img" | cut -d: -f1 | xargs -r -n1 sudo losetup -d + + echo "removing image file: {$img}" + exec_cmd sudo rm -f "$img" +} + +#x2rm_img_and_its_loop_devices() { +# img=$1 +# echo "removing loop devices associated with image file: {$img}" +# +# # Get all loop devices for this image +# while IFS= read -r loop_device; do +# if [[ -n "$loop_device" ]]; then +# echo "removing loop device {$loop_device}" +# exec_cmd sudo losetup -d "$loop_device" +# fi +# done < <(sudo losetup -j "$img" | cut -d: -f1) +# +# echo "removing image file: {$img}" +# exec_cmd sudo rm -f "$img" +#} + +#xxrm_img_and_its_loop_devices() { +# img=$1 +# echo "removing loop devices associated with image file: {$img}" +# #exec_cmd sudo losetup -j "$img" | cut -d: -f1 | xargs -r exec_cmd sudo losetup -d +# +# loop_device=$(sudo losetup -j "$img" | cut -d: -f1) +# echo "removing loop device {$loop_device}" +# exec_cmd sudo losetup -d $loop_device +# echo "removing image file: {$img}" +# exec_cmd sudo rm -f "$img" +#} + +create_file_img() { + img_file=$1 + + # NOTE: important to remove the image file first so that all zfs metadata is removed. + # truncate on an existing file will resize it and leave the metadata in place leading to crashes + rm -f "${img_file}" + + echo "Creating ${ZELTA_ZFS_TEST_POOL_SIZE}" "${img_file}" + truncate -s "${ZELTA_ZFS_TEST_POOL_SIZE}" "${img_file}" + + echo "showing created file image:" + ls -lh "${img_file}" +} +# old name +#create_pool_from_loop_device() + +create_linux_loop_device_pool() { + pool_name=$1 + + echo_alert "running create_linux_loop_device_pool - pool {$pool_name}" + + img_file=$(pool_image_file $pool_name) + + rm_img_and_its_loop_devices "$img_file" + + create_file_img "$img_file" + + echo "create loop device for file image: {$img_file}" + exec_cmd sudo losetup -f "$img_file" + + loop_device=$(losetup --list --noheadings --output NAME --associated "$img_file") + echo "created loop_device:{$loop_device} for image file:{$img_file}" + + echo "create pool {$pool_name} for loop device {$loop_device}" + exec_cmd sudo zpool create -f -m "/${pool_name}" "${pool_name}" "${loop_device}" +} + +create_pool_from_image_file() { + pool_name=$1 + img_file=$(pool_image_file $pool_name) + + create_file_img "$img_file" + echo "Creating zfs pool {$pool_name} from image file {$img_file}" + exec_cmd sudo zpool create -f -m "/${pool_name}" "${pool_name}" "${img_file}" +} + +create_test_pool() { + pool_name="$1" + if ! destroy_pool_if_exists "${pool_name}"; then + echo "** Error: Can't delete pool {$pool_name}" >&2 + return 1 + fi + + if [ "$POOL_TYPE" = "$MEMORY_DISK_POOL" ]; then + create_freebsd_test_pool $pool_name + elif [ "$POOL_TYPE" = "$LOOP_DEV_POOL" ]; then + create_linux_loop_device_pool "$pool_name" + elif [ "$POOL_TYPE" = "$FILE_IMG_POOL" ]; then + create_pool_from_image_file $pool_name + else + echo "Can't create pools for unsupported POOL_TYPE: {$POOL_TYPE}" >&2 + return 1 + fi + + echo "Created ${pool_name}" + exec_cmd sudo zpool list -v "${pool_name}" +} + + +verify_pool_creation() { + pool_name="$1" + expected_size="$2" + + if check_pool_exists "$pool_name"; then + actual_size=$(exec_cmd sudo zpool list -H -o size "$pool_name") + echo "Success: Pool '$pool_name' created successfully. Size: $actual_size (Expected: $expected_size)" + else + echo "Error: Pool '$pool_name' was NOT created." + return 1 + fi +} + +create_pools() { + echo "" + echo "=== create pool ${SRC_POOL} ===" + create_test_pool "${SRC_POOL}" + SRC_STATUS=$? + echo "" + echo "=== create pool ${TGT_POOL} ===" + create_test_pool "${TGT_POOL}" + TGT_STATUS=$? + + echo "SRC_STATUS:{$SRC_STATUS}" + echo "TGT_STATUS:{$TGT_STATUS}" + + return $((SRC_STATUS || TGT_STATUS)) +} +#set -x +rm -fR "${ZFS_MOUNT_BASE}" +mkdir -p "${ZFS_MOUNT_BASE}" +chmod 777 "${ZFS_MOUNT_BASE}" +chown ${BACKUP_USER} "${ZFS_MOUNT_BASE}" +chgrp ${BACKUP_USER} "${ZFS_MOUNT_BASE}" + +ls -ld "${ZFS_MOUNT_BASE}" +mkdir -p "${ZELTA_ZFS_STORE_TEST_DIR}" + +create_pools +setup_zfs_allow + +#set +x +#setup_loop_img "${SRC_POOL}" +#setup_loop_img "${TGT_POOL}" + + diff --git a/spec/bin/all_tests_setup/env_constants.sh b/spec/bin/all_tests_setup/env_constants.sh new file mode 100644 index 0000000..f871cee --- /dev/null +++ b/spec/bin/all_tests_setup/env_constants.sh @@ -0,0 +1,33 @@ +# Snap tree types, these correspond to different types of tests +export STANDARD_TREE=standard +export DIVERGENT_TREE=divergent +export ENCRYPTED_TREE=encrypted + +# Running modes +export RUN_REMOTELY=remote +export RUN_LOCALLY=local + +# we use a a/b pool naming convention, were a is the starting point +# and b is used for backups or perturbations to a +export SRC_POOL="apool" +export TGT_POOL="bpool" + +# zfs pool creation strategy types +export FILE_IMG_POOL=1 + +# On Ubuntu we use a loop device backed by a file +export LOOP_DEV_POOL=2 + +# On FreeBSD we use a memory disk backed by a file +export MEMORY_DISK_POOL=3 + +export TREETOP_DSN='treetop' +export BACKUPS_DSN='backups' + +# zelta version for pool names will include the remote +export SOURCE="${ZELTA_SRC_POOL}/${TREETOP_DSN}" +export TARGET="${ZELTA_TGT_POOL}/${BACKUPS_DSN}/${TREETOP_DSN}" + +# zfs versions for pool names do not include th remote +export SRC_TREE="$SRC_POOL/$TREETOP_DSN" +export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" diff --git a/spec/bin/all_tests_setup/install_local_zelta.sh b/spec/bin/all_tests_setup/install_local_zelta.sh new file mode 100755 index 0000000..4c68a95 --- /dev/null +++ b/spec/bin/all_tests_setup/install_local_zelta.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +rm -fr "${TEST_INSTALL}" +mkdir -p "${TEST_INSTALL}" +mkdir -p "$ZELTA_MAN8" +# After setting the needed environment variables we +# can use the standard install script +. ./install.sh diff --git a/spec/bin/divergent_test/divergent_snap_tree.sh b/spec/bin/divergent_test/divergent_snap_tree.sh new file mode 100755 index 0000000..aca61a2 --- /dev/null +++ b/spec/bin/divergent_test/divergent_snap_tree.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +. spec/bin/divergent_test/divergent_test_env.sh + +# Add deterministic data based on snapshot name +etch () { + zfs list -Hro name -t filesystem $SRC_TREE | tr '\n' '\0' | xargs -0 -I% -n1 \ + dd if=/dev/random of='/%/file' bs=64k count=1 > /dev/null 2>&1 + zfs list -Hro name -t volume $SRC_TREE | tr '\n' '\0' | xargs -0 -I% -n1 \ + dd if=/dev/random of='/dev/zvol/%' bs=64k count=1 > /dev/null 2>&1 + zfs snapshot -r "$SRC_TREE"@snap$1 +} + +set -x +# Clean house +zfs destroy -vR "$SRC_POOL" +zfs destroy -vR "$TGT_POOL" + +# Create the setup tree +zelta backup "$SRC_POOL" "$TGT_SETUP"/sub1 +zelta backup "$SRC_POOL" "$TGT_SETUP"/sub2/orphan +zelta backup "$SRC_POOL" "${TGT_SETUP}/sub3/space name" +zfs create -vsV 16G -o volmode=dev $TGT_SETUP'/vol1' +# TO-DO: Add encrypted dataset + +# Sync the temp tree to $SRC_TREE +zelta snapshot "$TGT_SETUP"@set +zelta revert --snap-name "go" "$TGT_SETUP" +zelta backup --snap-name "one" "$TGT_SETUP" "$SRC_TREE" +zelta backup --no-snapshot "$SRC_TREE" "$TGT_TREE" +# TO-DO: Sync with exclude pattern + + +# Riddle source with special cases + +# A child with no snapshot on the source +zfs create "$SRC_TREE"/sub1/child +# A child with no snapshot on the target +zfs create -u "$TGT_TREE"/sub1/kid + +# A written target +#zfs set readonly=off "$TGT_TREE"/sub1 +#zfs mount "$TGT_TREE" +#zfs mount "$TGT_TREE"/sub1 +#touch /"$TGT_TREE"/sub1/data.file + +# An orphan +zfs destroy "$SRC_TREE"/sub2@one + +# A diverged target +zfs snapshot "$TGT_TREE/sub3/space name@blocker" + +# An unsyncable dataset +zfs destroy "$TGT_TREE"/vol1@go + +set +x + +#dd if=/dev/urandom of=/tmp/zelta-test-key bs=1m count=512 + +#zfs create -vp $SRC_TREE/'minus/two/one/0/lift off' +#zfs create -vp $SRC_TREE/'minus/two/one/0/lift off' +#for num in `jot 2`; do +# etch $num +#done +#etch 1; etch 2; etch 3 + +#etch 8 +#zelta sync "$SRC_TREE" "$TGT_TREE" +#zelta match "$SRC_TREE" "$TGT_TREE" \ No newline at end of file diff --git a/spec/bin/divergent_test/divergent_test_env.sh b/spec/bin/divergent_test/divergent_test_env.sh new file mode 100644 index 0000000..66af680 --- /dev/null +++ b/spec/bin/divergent_test/divergent_test_env.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +. spec/bin/all_tests_setup/common_test_env.sh +. spec/lib/common.sh + +export TGT_SETUP="$TGT_POOL/temp" diff --git a/spec/bin/divergent_test/divergent_test_spec.sh b/spec/bin/divergent_test/divergent_test_spec.sh new file mode 100644 index 0000000..929cf66 --- /dev/null +++ b/spec/bin/divergent_test/divergent_test_spec.sh @@ -0,0 +1,327 @@ +. spec/bin/divergent_test/divergent_test_env.sh + +# Custom validation functions +zelta_match_after_backup_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub2 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub2/orphan @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub3 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub3/space name @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/vol1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "8 up-to-date") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +match_after_rotate_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub2 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub2/orphan @two @zelta_"*" @two syncable (incremental)"|\ + "/sub3 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub3/space name @two @zelta_"*" @two syncable (incremental)"|\ + "/vol1 @go @zelta_"*" @go syncable (incremental)"|\ + "3 up-to-date, 5 syncable"|\ + "8 total datasets compared") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + + +match_rotate_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "source is written; snapshotting: @zelta_"*|\ + "renaming '"*"/treetop' to '"*"/treetop_go'"|\ + "warning: insufficient snapshots; performing full backup for 2 datasets"|\ + "to ensure target is up-to-date, run: zelta backup "*" "*"/treetop"|\ + "no source: "*"/treetop/sub1/kid"|\ + *"K sent, 8 streams received in "*" seconds") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +match_after_first_backup_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') + + case "${normalized}" in + "source is written; snapshotting: @zelta_"*) + # New snapshot created on source + ;; + "syncing 9 datasets") + # Starting sync operation + ;; + "no source: $TGT_TREE/sub1/kid") + # Dataset exists on target but not on source + ;; + "target snapshots beyond the source match: $TGT_TREE/sub2") + # Target has snapshots newer than source's latest matching snapshot + ;; + "target snapshots beyond the source match: $TGT_TREE/sub2/orphan") + # Target has snapshots newer than source's latest matching snapshot + ;; + "target snapshots beyond the source match: $TGT_TREE/sub3/space name") + # Target has snapshots newer than source's latest matching snapshot + ;; + "no snapshot; target diverged: $TGT_TREE/vol1") + # No common snapshot found; target has diverged from source + ;; + "15K sent, 5 streams received in 0.09 seconds") + # Summary statistics + ;; + *) + echo "Unexpected line format: $line" >&2 + return 1 + ;; + esac + done +} + + +match_after_divergent_snapshots_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') + + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @go @go @go up-to-date"|\ + "/sub1 @go @go @go up-to-date"|\ + "/sub1/child - - - syncable (full)"|\ + "/sub1/kid - - - no source (target only)"|\ + "/sub2 @go @two @two blocked sync: target diverged"|\ + "/sub2/orphan @go @two @two blocked sync: target diverged"|\ + "/sub3 @go @two @go syncable (incremental)"|\ + "/sub3/space name @go @two @blocker blocked sync: target diverged"|\ + "/vol1 - @go - blocked sync: no target snapshots"|\ + "2 up-to-date, 2 syncable, 5 blocked"|\ + "9 total datasets compared") + # Pattern matches + ;; + *) + echo "Unexpected line format: $line" >&2 + return 1 + ;; + esac + done +} + + + +divergent_initial_match_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') + + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @go @go @go up-to-date"|\ + "/sub1 @go @go @go up-to-date"|\ + "/sub1/child - - - syncable (full)"|\ + "/sub1/kid - - - no source (target only)"|\ + "/sub2 @go @go @go up-to-date"|\ + "/sub2/orphan @go @go @go up-to-date"|\ + "/sub3 @go @go @go up-to-date"|\ + "/sub3/space name @go @go @blocker blocked sync: target diverged"|\ + "/vol1 - @go - blocked sync: no target snapshots"|\ + "5 up-to-date, 1 syncable, 3 blocked"|\ + "9 total datasets compared") + # Pattern matches + ;; + *) + echo "Unexpected line format: $line" >&2 + return 1 + ;; + esac + done +} + + + +validate_divergent_snap_tree_zfs_output() { + while IFS= read -r line; do + # Skip header line + #[ "$line" = "NAME USED AVAIL REFER MOUNTPOINT" ] && continue + #[[ "$line" = "NAME"*"USED"*"AVAIL"*"REFER"*"MOUNTPOINT" ]] && continue + + # Pattern: NAME * * * MOUNTPOINT + case "$line" in + "NAME"*"USED"*"AVAIL"*"REFER"*"MOUNTPOINT") continue ;; + apool*"/apool"|\ + $SRC_TREE*"/$SRC_TREE"|\ + $SRC_TREE/sub1*"/$SRC_TREE/sub1"|\ + $SRC_TREE/sub1/child*"/$SRC_TREE/sub1/child"|\ + $SRC_TREE/sub2*"/$SRC_TREE/sub2"|\ + $SRC_TREE/sub2/orphan*"/$SRC_TREE/sub2/orphan"|\ + $SRC_TREE/sub3*"/$SRC_TREE/sub3"|\ + $SRC_TREE/sub3/space\ name*"/$SRC_TREE/sub3/space name"|\ + $SRC_TREE/vol1*"-"|\ + bpool*"/bpool"|\ + bpool/backups*"/bpool/backups"|\ + $TGT_TREE*"/$TGT_TREE"|\ + $TGT_TREE/sub1*"/$TGT_TREE/sub1"|\ + $TGT_TREE/sub1/kid*"/$TGT_TREE/sub1/kid"|\ + $TGT_TREE/sub2*"/$TGT_TREE/sub2"|\ + $TGT_TREE/sub2/orphan*"/$TGT_TREE/sub2/orphan"|\ + $TGT_TREE/sub3*"/$TGT_TREE/sub3"|\ + $TGT_TREE/sub3/space\ name*"/$TGT_TREE/sub3/space name"|\ + $TGT_TREE/vol1*"-"|\ + $TGT_SETUP*"/$TGT_SETUP"|\ + $TGT_SETUP/sub1*"/$TGT_SETUP/sub1"|\ + $TGT_SETUP/sub2*"/$TGT_SETUP/sub2"|\ + $TGT_SETUP/sub2/orphan*"/$TGT_SETUP/sub2/orphan"|\ + $TGT_SETUP/sub3*"/$TGT_SETUP/sub3"|\ + $TGT_SETUP/sub3/space\ name*"/$TGT_SETUP/sub3/space name"|\ + $TGT_SETUP/vol1*"-"|\ + ${TGT_SETUP}_set*"/${TGT_SETUP}_set"|\ + ${TGT_SETUP}_set/sub1*"/${TGT_SETUP}_set/sub1"|\ + ${TGT_SETUP}_set/sub2*"/${TGT_SETUP}_set/sub2"|\ + ${TGT_SETUP}_set/sub2/orphan*"/${TGT_SETUP}_set/sub2/orphan"|\ + ${TGT_SETUP}_set/sub3*"/${TGT_SETUP}_set/sub3"|\ + ${TGT_SETUP}_set/sub3/space\ name*"${TGT_SETUP}_set/sub3/space name"|\ + ${TGT_SETUP}_set/vol1*"-") + # Pattern matches + ;; + *) + echo "Unexpected line format: $line" >&2 + return 1 + ;; + esac + done +} + +add_divergent_snapshots() { + zelta snapshot "$SOURCE"/sub2@two + zelta snapshot "$TARGET"/sub2@two +} + +Describe 'confirm zfs setup' + before_all() { + %logger "-- before_all: confirm zfs setup" + echo + } + + after_all() { + %logger "-- after_all: confirm zfs setup" + } + + + #BeforeAll before_all + #AfterAll after_all + + Describe 'zfs list output validation' + It 'matches expected pattern for each line' + When call exec_on "$TGT_SVR" zfs list -r -H $SRC_POOL $TGT_POOL + + The output should satisfy validate_divergent_snap_tree_zfs_output + End + End + + Describe 'check initial zelta match state' + It "initial match has 5 up-to-date, 1 syncable, 3 blocked, with 9 total datasets compared" + When call zelta match $SOURCE $TARGET + The output should satisfy divergent_initial_match_output + End + End + + Describe 'add incremental source snapshot' + It "adds $SOURCE/sub3@two snapshot" + When call zelta snapshot "$SOURCE"/sub3@two + The output should equal "snapshot created '$SRC_TREE/sub3@two'" + The stderr should be blank + The status should eq 0 + End + End + + Describe 'add divergent snapshots of same name' + It "adds divergent snapshots for $SOURCE/sub2@two and $TARGET/sub2@two" + When call add_divergent_snapshots + The line 1 of output should equal "snapshot created '$SRC_TREE/sub2@two'" + The line 2 of output should equal "snapshot created '$TGT_TREE/sub2@two'" + The stderr should be blank + The status should eq 0 + End + End + + Describe 'check zelta match after divergent snapshots' + It "after divergent snapshot match has 2 up-to-date, 2 syncable, 5 blocked, with 9 total datasets compared" + When call zelta match $SOURCE $TARGET + The output should satisfy match_after_divergent_snapshots_output + End + End +End + + +Describe 'Divergent match, rotate, match' + It "shows current match for divergent $SOURCE and $TARGET" + When call zelta match $SOURCE $TARGET + The output should satisfy match_after_divergent_snapshots_output + End + + It "rotate divergent $SOURCE and $TARGET" + When call zelta rotate $SOURCE $TARGET + The output should satisfy match_rotate_output + The stderr should equal "warning: insufficient snapshots; performing full backup for 2 datasets" + The status should equal 0 + End + + It "match $SOURCE and $TARGET after divergent rotate" + When call zelta match $SOURCE $TARGET + The output should satisfy match_after_rotate_output + The status should equal 0 + End +End + + +Describe 'Divergent backup, then match' + It "backup divergent $SOURCE to $TARGET" + When call zelta backup $SOURCE $TARGET + The output line 1 should equal "syncing 8 datasets" + The output line 2 should equal "8 datasets up-to-date" + The output line 3 should match pattern "* sent, 5 streams received in * seconds" + The status should equal 0 + End + + It "match after backup" + When call zelta backup $SOURCE $TARGET + The output should satisfy zelta_match_after_backup_output + The status should equal 0 + End +End + + diff --git a/spec/bin/hello_example.sh b/spec/bin/hello_example.sh new file mode 100644 index 0000000..4b39f7e --- /dev/null +++ b/spec/bin/hello_example.sh @@ -0,0 +1,23 @@ +# simple test showing helping function inclusion, logging and output matching +# simple example spec + +Describe 'hello shellspec' + Include spec/lib/hello.sh + setup() { + %logger "-- hello spec setup" + %logger "-- reference this example to help you get started with writing tests" + #spec/initialize/initialize_testing_setup.sh + } + + cleanup() { + %logger "-- hello spec cleanup " + } + + BeforeAll 'setup' + AfterAll 'cleanup' + It 'says hello' + When call hello ShellSpec + %logger "Your temp dir is {$SHELLSPEC_TMPBASE}" + The output should match pattern "What's up? Hello ShellSpec! TMPDIR: *" + End +End diff --git a/spec/bin/one_time_setup/setup_sudoers.sh b/spec/bin/one_time_setup/setup_sudoers.sh new file mode 100755 index 0000000..33f2716 --- /dev/null +++ b/spec/bin/one_time_setup/setup_sudoers.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +# This scripts runs as ssh on the designated remote host +# and there is no environment set. We make the current directory +# the location of the git clone for zelta +cd_to_git_clone_dir() { + script_dir=$(cd "$(dirname "$0")" && pwd) + parent_dir=$(dirname "$script_dir") + cd "$parent_dir/../.." || exit 1 + cur_dir=$(pwd) + echo "git zelta clone directory is: {$cur_dir}" +} + +cd_to_git_clone_dir + +. spec/bin/all_tests_setup/common_test_env.sh + +echo "Setting sudo for backup user {$BACKUP_USER}" +echo "For zelta-dev root {$ZELTA_DEV_PATH}" + +# Detect OS and set paths +if [ -f /etc/os-release ]; then + . /etc/os-release + os_name="$ID" +else + os_name=$(uname -s | tr '[:upper:]' '[:lower:]') +fi + +# Set ZFS/zpool paths based on OS +case "$os_name" in + ubuntu|debian|linux) + zfs_path="/usr/sbin/zfs" + zpool_path="/usr/sbin/zpool" + mount_path="/usr/bin/mount" + mkdir_path="/usr/bin/mkdir" + sudoers_dir="/etc/sudoers.d" + ;; + freebsd) + zfs_path="/sbin/zfs" + zpool_path="/sbin/zpool" + mount_path="/sbin/mount" + mkdir_path="/bin/mkdir" + sudoers_dir="/usr/local/etc/sudoers.d" + ;; + *) + echo "Unsupported OS: $os_name" >&2 + exit 1 + ;; +esac + +# Sudoers entry +setup_script="${ZELTA_DEV_PATH}/spec/bin/ssh_tests_setup/setup_zfs_pools_on_remote.sh" +sudoers_entry="${BACKUP_USER} ALL=(ALL) NOPASSWD: ${zpool_path}, ${zfs_path}, ${mount_path}, ${mkdir_path}, ${setup_script}" + +# Sudoers file location +sudoers_file="$sudoers_dir/zelta-${BACKUP_USER}" + +# Check if running as root +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root" >&2 + exit 1 +fi + +# Check if user exists +if ! id "$BACKUP_USER" >/dev/null 2>&1; then + echo "User '$BACKUP_USER' does not exist" >&2 + exit 1 +fi + +# Create sudoers entry +cat > "$sudoers_file" << EOF +# Allow $BACKUP_USER to run ZFS commands without password for zelta testing +# NOTE: This is for test environments only - DO NOT use in production +$sudoers_entry +EOF + +# Set correct permissions +chmod 0440 "$sudoers_file" + +# Validate the sudoers file +if visudo -c -f "$sudoers_file" >/dev/null 2>&1; then + echo "Successfully created sudoers entry at: $sudoers_file" + echo "Entry: $sudoers_entry" +else + echo "ERROR: Invalid sudoers syntax, removing file" >&2 + rm -f "$sudoers_file" + exit 1 +fi diff --git a/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh b/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh new file mode 100755 index 0000000..a749077 --- /dev/null +++ b/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +. spec/bin/all_tests_setup/common_test_env.sh +. spec/lib/script_util.sh + +if ! validate_tree_name "$@"; then + return 1 +fi + +echo "TREE_NAME is {$TREE_NAME}" + +# Use a string for the following remote setup so sudo password only has to be entered once. +# pull down zelta from github into a clean dir +# checkout the test branch +# update sudoers +# setup the test env, install zelta, create pools +# create the requested snap tree + + +printf "\n*** Enter sudo password for remote setup:\n" +ssh -t ${BACKUP_USER}@${REMOTE_TEST_HOST} " + set -e && + set -x && + + sudo rm -fr ${ZELTA_GIT_CLONE_DIR} && + + mkdir -p ${ZELTA_GIT_CLONE_DIR} && + chmod 777 ${ZELTA_GIT_CLONE_DIR} && + git clone https://github.com/bellhyve/zelta.git ${ZELTA_GIT_CLONE_DIR} && + cd ${ZELTA_GIT_CLONE_DIR} && + git checkout ${GIT_TEST_BRANCH} && + + sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/one_time_setup/setup_sudoers.sh && + sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/all_tests_setup/all_tests_setup.sh && + sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/${TREE_NAME}_test/${TREE_NAME}_snap_tree.sh && + + echo 'Remote setup complete' +" diff --git a/spec/bin/standard_test/standard_snap_tree.sh b/spec/bin/standard_test/standard_snap_tree.sh new file mode 100755 index 0000000..5821f18 --- /dev/null +++ b/spec/bin/standard_test/standard_snap_tree.sh @@ -0,0 +1,137 @@ +#!/bin/sh + +. spec/bin/standard_test/standard_test_env.sh + +DATASETS="${SRC_TREE} ${TGT_TREE}" + +dataset_exists() { + exec_cmd zfs list "$1" &>/dev/null + return $? +} + +#create_tree_via_zfs() { +# exec_cmd sudo zfs create -vp "$SRC_TREE" +# exec_cmd sudo zfs create -vp "$SRC_TREE/$ALL_DATASETS" +# exec_cmd sudo zfs create -vp "$TGT_POOL/$BACKUPS_DSN" +# #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' +#} + +#export SRC_TREE="$SRC_POOL/$TREETOP_DSN" +#export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" + +#try1_create_test_tree() { +# local pool="$1" +# local root="$2" +# local mount_base="$3" +# +# mkdir -p "${mount_base}/${root}/one/two/three" +# zfs create -o mountpoint="${mount_base}/${root}" "${pool}/${root}" +# zfs create -o mountpoint="${mount_base}/${root}/one" "${pool}/${root}/one" +# zfs create -o mountpoint="${mount_base}/${root}/one/two" "${pool}/${root}/one/two" +# zfs create -o mountpoint="${mount_base}/${root}/one/two/three" "${pool}/${root}/one/two/three" +# +# echo "Test tree created successfully" +# echo "Mounted at: $mount_base" +#} + + +new_create_tree_via_zfs() { + #exec_cmd zfs create -o mountpoint="${mount_base}/$TREETOP_DSN} -vp "$SRC_TREE" + mkdir -p "${ZFS_MOUNT_BASE}/${TREETOP_DSN}" + mkdir -p "${ZFS_MOUNT_BASE}/${BACKUPS_DSN}" + exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${TREETOP_DSN}" -vp "$SRC_POOL/$TREETOP_DSN" + #exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${TREETOP_DSN}" -vp "$SRC_POOL/$TREETOP_DSN" + #$SRC_TREE/$ALL_DATASETS" + exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${BACKUPS_DSN}" -vp "$TGT_POOL/$BACKUPS_DSN" + #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' +} + +old_create_tree_via_zfs() { + exec_cmd sudo zfs create -vp "$SRC_TREE" + exec_cmd sudo zfs create -vp "$SRC_TREE/$ALL_DATASETS" + exec_cmd sudo zfs create -vp "$TGT_POOL/$BACKUPS_DSN" + #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' +} + + +create_tree_via_zelta() { + exec_cmd zelta backup "$SRC_POOL" "$TGT_POOL/$TREETOP_DSN/$ALL_DATASETS" + exec_cmd zelta revert "$TGT_POOL/$TREETOP_DSN" + exec_cmd zelta backup "$TGT_POOL/$TREETOP_DSN" "$SRC_POOL/$TREETOP_DSN" +} + +#rm_test_datasets() { +# for dataset in "${DATASETS[@]}"; do +# if zfs list "$dataset" &>/dev/null; then +# echo "Destroying $dataset..." +# exec_cmd sudo zfs destroy -vR "$dataset" +# else +# echo "Skipping $dataset (does not exist)" +# fi +# done +#} + +rm_all_datasets_for_pool() { + poolname=$1 + + # shellcheck disable=SC2120 + reverse_lines() { + awk '{lines[NR]=$0} END {for(i=NR;i>0;i--) print lines[i]}' "$@" + } + + echo "removing all datasets for pool {$poolname}" + dataset_list=$(zfs list -H -o name -t filesystem,volume -r $poolname | grep -v "^${poolname}$" | reverse_lines) + echo $dataset_list + + zfs list -H -o name -t filesystem,volume -r $poolname | grep -v "^${poolname}\$" | reverse_lines | while IFS= read -r dataset; do + exec_cmd sudo zfs destroy -r "$dataset" + done + +} + +x_rm_test_datasets() { + for dataset in $DATASETS; do + #for dataset in 'apool'; do + if dataset_exists "$dataset"; then + echo "found dataset, please delete it" + else + echo "there is no dataset to remove " + fi + + if exec_cmd sudo zfs list "$dataset" &>/dev/null; then + #echo "Destroying $dataset..." + echo "need to destroy dataset $dataset" + #exec_cmd zfs destroy -vR "$dataset" + else + echo "Skipping $dataset (does not exist)" + fi + done +} + +setup_simple_snap_tree() { + #set -x + echo "Make a fresh test tree" + #rm_test_datasets + rm_all_datasets_for_pool $SRC_POOL + rm_all_datasets_for_pool $TGT_POOL + #new_create_tree_via_zfs + #try1_create_test_tree "$SRC_POOL" "$TREETOP_DSN" "$ZFS_MOUNT_BASE" + old_create_tree_via_zfs + + # TODO: create via zelta + #create_tree_via_zelta + #setup_zfs_allow + + TREE_STATUS=$? + #set +x + #true + return $TREE_STATUS +} + + +#set -x +setup_simple_snap_tree +#set +x + +#STATUS=$? +#echo "status: $STATUS" diff --git a/spec/bin/standard_test/standard_test_env.sh b/spec/bin/standard_test/standard_test_env.sh new file mode 100755 index 0000000..9ffbdc8 --- /dev/null +++ b/spec/bin/standard_test/standard_test_env.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +. spec/bin/all_tests_setup/common_test_env.sh +. spec/lib/common.sh + +export ALL_DATASETS="one/two/three" diff --git a/spec/bin/standard_test/standard_test_spec.sh b/spec/bin/standard_test/standard_test_spec.sh new file mode 100644 index 0000000..a24aeb4 --- /dev/null +++ b/spec/bin/standard_test/standard_test_spec.sh @@ -0,0 +1,153 @@ +. spec/bin/standard_test/standard_test_env.sh +. spec/lib/common.sh + +# valid xtrace usage if found +validate_options() { + # Direct test - executes if function returns 0 (success) + if ! check_if_xtrace_usage_valid; then + echo "xtrace options are not correct" >&2 + echo "to show expectations use --shell bash and bash version >= 4" >&2 + echo "NOTE Use: --xtrace --shell bash" >&2 + return 1 + fi + return 0 +} + + +# allow for 2 vaild matches for current shellspec line/subject being considered +match_either() { + case $SHELLSPEC_SUBJECT in + "$1"|"$2") + return 0 + ;; + *) + return 1 + ;; + esac +} + +# TODO: is it possible to setup a snap tree on FreeBSD as the backup user? +# TODO: when should this code be removed: left over from an attempt to setup a snap tree prior to start of every test +#global_setup_function() { +# %putsn "global_setup_function" +# %putsn "before: SRC_POOL=$SRC_POOL, TGT_POOL=$TGT_POOL" +# #export SRC_SVR=dever@fzfsdev: +# #export TGT_SVR=dever@fzfsdev: +# #SRC_POOL='apool' +# #TGT_POOL='bpool' +# export TREETOP_DSN='treetop' +# export BACKUPS_DSN='backups' +# export SOURCE=${SRC_SVR}${SRC_POOL}/${TREETOP_DSN} +# export TARGET=${TGT_SVR}${TGT_POOL}/${BACKUPS_DSN} +# export SRC_TREE="$SRC_POOL/$TREETOP_DSN" +# export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" +# export ALL_DATASETS="one/two/three" +# %putsn "after: SRC_POOL=$SRC_POOL, TGT_POOL=$TGT" +# CWD=$(pwd) +# #sudo /home/dever/src/repos/zelta/spec/initialize/setup_simple_snap_tree.sh +# #./spec/initializize/setup_simple_snap_tree.sh +# %putsn "current dir {$CWD}" +# %putsn "current dir {$CWD}" +# %putsn "current dir {$CWD}" +# %putsn "current dir {$CWD}" +# %putsn "current dir {$CWD}" +#} + + +BeforeAll validate_options + +Describe 'confirm zfs setup' + before_all() { + %logger "-- before_all: confirm zfs setup" + echo + } + + after_all() { + %logger "-- after_all: confirm zfs setup" + } + + #BeforeAll before_all + #AfterAll after_all + + It "has good initial SRC_POOL:{$SRC_POOL} simple snap tree" + When call exec_on "$SRC_SVR" zfs list -r "$SRC_POOL" + The line 2 of output should match pattern "* /$SRC_POOL" + The line 3 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN" + The line 4 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one" + The line 5 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one/two" + The line 6 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one/two/three" + End + + It "has good initial TGT_POOL:{$TGT_POOL} simple snap tree" + When call exec_on "$TGT_SVR" zfs list -r "$TGT_POOL" + The line 2 of output should match pattern "* /$TGT_POOL" + End +End + +Describe 'try backup' + + It 'backs up the initial tree' + When call zelta backup $SOURCE $TARGET + + The line 1 of output should match pattern "source is written; snapshotting: @zelta_*" + The line 2 of output should equal "syncing 4 datasets" + The line 3 of output should match pattern "* sent, 4 streams received in * seconds" + The status should eq 0 + End + + It 'has valid backup' + When call exec_on "$TGT_SVR" zfs list -r "$TGT_POOL" + The line 2 of output should match pattern "$TGT_POOL * /$TGT_POOL" + The line 3 of output should match pattern "$TGT_POOL/$BACKUPS_DSN * /$TGT_POOL/$BACKUPS_DSN" + The line 4 of output should match pattern "$TGT_TREE * /$TGT_TREE" + The line 5 of output should match pattern "$TGT_TREE/one * /$TGT_TREE/one" + The line 6 of output should match pattern "$TGT_TREE/one/two * /$TGT_TREE/one/two" + The line 7 of output should match pattern "$TGT_TREE/one/two/three * /$TGT_TREE/one/two/three" + + The stderr should be blank + The status should eq 0 + End + + Parameters + 8 '^(apool|bpool)' $TGT_SVR + 4 apool/treetop $TGT_SVR + 4 bpool/backups/treetop $TGT_SVR + End + + It "has $1 snapshots on ${3:-localhost} ${2:+matching pattern '$2'}" + When call snapshot_count $1 $2 $3 + The stderr should be blank + The status should eq 0 + End +End + +Describe 'zelta rotate' + It 'rotates the backed up tree' + # force snapshot timestamp to be at 1 second in future to prevent backup snapshot conflict + sleep 1 + + When call zelta rotate $SOURCE $TARGET + + # TODO: verify that '$SRC_TREE and TGT_TREE' will work for remotes, or if i need to use $SOURCE and $TARGET instead + The line 1 of output should match pattern "action requires a snapshot delta; snapshotting: @zelta_*" + The line 2 of output should match pattern "rotating from source: ${SRC_TREE}@zelta_*" + The line 3 of output should match pattern "renaming '${TGT_TREE}' to '${TGT_TREE}_zelta_*'" + The line 4 of output should match pattern "to ensure target is up-to-date, run: zelta backup ${SOURCE} ${TARGET}" + The line 5 of output should match pattern "* datasets up-to-date" + The line 6 of output should match pattern "* sent, * streams received in * seconds" + The stderr should be blank + The status should eq 0 + End + + Parameters + 16 '^(apool|bpool)' $TGT_SVR + 8 apool/treetop $TGT_SVR + 8 bpool/backups/treetop $TGT_SVR + End + + It "has $1 snapshots on ${3:-localhost} ${2:+matching pattern '$2'}" + When call snapshot_count $1 $2 $3 + The stderr should be blank + The status should eq 0 + End +End diff --git a/spec/doc/vm/README.md b/spec/doc/vm/README.md new file mode 100644 index 0000000..60bdae8 --- /dev/null +++ b/spec/doc/vm/README.md @@ -0,0 +1,5 @@ +# Setting up a Ubuntu VM + +- [Installing kvm/qemu/virt-manager](./installing-kvm.md) +- [Creating a Ubuntu VM](./creation.md) +- [ZFS Configuration](./zfs-configuration.md) diff --git a/spec/doc/vm/creation.md b/spec/doc/vm/creation.md new file mode 100644 index 0000000..ef71fca --- /dev/null +++ b/spec/doc/vm/creation.md @@ -0,0 +1,41 @@ +# VM Creation Guide + + +## setting up an Ubuntu VM on Ubuntu + +```shell +#!/bin/sh +# Download Ubuntu ISO +#wget https://releases.ubuntu.com/noble/ubuntu-24.04.3-desktop-amd64.iso + +set -x + +UBUNTU_ISO_SRC_DIR="/home/dever/Downloads" +UBUNTU_ISO_VIRT_DIR="/var/lib/libvirt/boot" +UBUNTU_ISO_NAME="ubuntu-24.04.3-live-server-amd64.iso" + +UBUNTU_SRC_ISO="$UBUNTU_ISO_SRC_DIR/$UBUNTU_ISO_NAME" +UBUNTU_TGT_ISO="$UBUNTU_ISO_VIRT_DIR/$UBUNTU_ISO_NAME" + +sudo mkdir -p "${UBUNTU_ISO_VIRT_DIR}" +sudo cp -f "${UBUNTU_SRC_ISO}" "$UBUNTU_TGT_ISO" + +# Optional: lock down permissions +sudo chmod 644 "$UBUNTU_TGT_ISO" +sudo chown root:root "$UBUNTU_TGT_ISO" + + +sudo virt-install \ + --name zfs-dev \ + --ram 8192 \ + --disk path=/var/lib/libvirt/images/zfs-dev.qcow2,size=40,format=qcow2 \ + --vcpus 4 \ + --os-variant ubuntu24.04 \ + --network bridge=virbr0 \ + --graphics spice \ + --cdrom "${UBUNTU_TGT_ISO}" + + +set +x + +``` diff --git a/spec/doc/vm/installing-kvm.md b/spec/doc/vm/installing-kvm.md new file mode 100644 index 0000000..143d132 --- /dev/null +++ b/spec/doc/vm/installing-kvm.md @@ -0,0 +1,55 @@ +``` +#!/bin/sh + +set -x + +# 1. Check if your CPU supports virtualization +# If this returns > 0, you're good +if egrep -c '(vmx|svm)' /proc/cpuinfo; then + echo "CPU supports virtualizaton, install KVM/QEMU and virt-manager" +else + echo "your cpu does not support virualization, cannot install KVM!" + return 1 +fi + + +# 2. Install KVM and tools +sudo apt update +sudo apt install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils virt-manager + +# 3. Add your user to libvirt groups +sudo usermod -aG libvirt dever +sudo usermod -aG kvm dever + +# 4. Start the libvirt service +sudo systemctl enable --now libvirtd + +# 5. Verify installation +sudo virsh list --all + +# 6. Log out and back in (for group membership to take effect) + +set +x +``` + + +``` +#!/bin/sh + +set -x +sudo apt install -y virt-viewer libvirt-daemon-system libvirt-clients qemu-kvm +sudo systemctl enable --now libvirtd +sudo systemctl status libvirtd + +sudo usermod -aG libvirt,kvm $USER +id $USER +virsh -c qemu:///system list +virt-viewer --connect qemu:///system zfs-dev + +set +x + + +# ToDo +echo "on the VM run:" +echo "sudo systemctl enable --now serial-getty@ttyS0.service" +``` diff --git a/spec/doc/vm/running.md b/spec/doc/vm/running.md new file mode 100644 index 0000000..e69de29 diff --git a/spec/doc/vm/zfs-configuration.md b/spec/doc/vm/zfs-configuration.md new file mode 100644 index 0000000..2fcf972 --- /dev/null +++ b/spec/doc/vm/zfs-configuration.md @@ -0,0 +1,13 @@ +# ZFS setup for zelta + + +## Install ZFS + 1. Update package lists `sudo apt update` + 2. Install ZFS userspace tools `sudo apt install zfsutils-linux` + +## Verify ZFS version +- _Verify ZFS is installed and kernel and user tools versions match_ + ``` + zfs version + cat /sys/module/zfs/version + ``` diff --git a/spec/lib/common.sh b/spec/lib/common.sh new file mode 100644 index 0000000..3fdba82 --- /dev/null +++ b/spec/lib/common.sh @@ -0,0 +1,206 @@ +# shellcheck shell=sh + +GREEN=$(printf '\033[32m') +RED=$(printf '\033[31m') +NC=$(printf '\033[0m') + +#printf "%sThis is red%s\n" "$RED" "$NC" + +check_zfs_installed() { + # Check if zfs is already on PATH + if ! command -v zfs >/dev/null 2>&1; then + # Allow user to override ZFS_BIN location, default to /usr/local/sbin + ZFS_BIN="${ZFS_BIN:-/usr/local/sbin}" + + # Add ZFS_BIN to PATH if not already present + case ":$PATH:" in + *":$ZFS_BIN:"*) ;; + *) PATH="$ZFS_BIN:$PATH" ;; + esac + export PATH + + # Verify zfs command is now available + if ! command -v zfs >/dev/null 2>&1; then + echo "Error: zfs command not found. Please set ZFS_BIN to the correct location." >&2 + return 1 + fi + fi +} + + +echo_alert() { + msg=$1 + printf "${RED}[** alert **] %s${NC}\n", "$msg" +} + +exec_cmd() { + CMD=$(printf "%s " "$@") + CMD=${CMD% } # trim trailing space + #CMD="$@" + if "$@"; then + [ "${EXEC_CMD_QUIET:-}" != "1" ] && printf "${GREEN}[success] %s${NC}\n" "${CMD}" + return 0 + else + _exit_code=$? + [ "${EXEC_CMD_QUIET:-}" != "1" ] && printf "${RED}[failed] %s returned %d${NC}\n" "${CMD}" "$_exit_code" + return "$_exit_code" + fi +} + +exec_on() { + local server="$1" + shift + + if [ -n "$server" ]; then + ssh "$server" "$@" + else + "$@" + fi +} + +snapshot_count() { + expected_count=$1 + pattern=$2 # Optional regex pattern + svr=$3 # Optional server name + + # Validate arguments + if [ -z "$expected_count" ]; then + echo "Error: snapshot_count requires expected_count argument" >&2 + return 1 + fi + + # Validate expected_count is a number + case "$expected_count" in + ''|*[!0-9]*) + echo "Error: expected_count must be a number" >&2 + return 1 + ;; + esac + + # Get snapshot list + snapshot_list=$(exec_on "$svr" zfs list -t snapshot -H -o name) + + # Count snapshots, optionally filtering by pattern + if [ -n "$pattern" ]; then + # Count only snapshots matching the pattern + snapshot_count=$(echo "$snapshot_list" | grep -E "$pattern" | wc -l) + else + # Count all snapshots + snapshot_count=$(echo "$snapshot_list" | wc -l) + fi + + # Test the count + if [ "$expected_count" -eq "$snapshot_count" ]; then + return 0 + else + if [ -n "$pattern" ]; then + echo "Expected $expected_count snapshots matching pattern '$pattern', found $snapshot_count" >&2 + else + echo "Expected $expected_count snapshots, found $snapshot_count" >&2 + fi + return 1 + fi +} + +# Shellspec has a nice tracing feature when you specify --xtrace, but it doesn't execute +# expectations unless you use --shell bash, and the bash shell has to be >= version 4. +# Using --xtrace without --shell is an easy mistake to make and it looks like tests are +# passing when they are not, as no expectations are run. Therefore, we use this function +# to check if --xtrace has been specifie +# d, we assert that --shell bash is also present. +# see https://deepwiki.com/shellspec/shellspec/5.1-command-line-options#tracing-and-profiling +#check_if_xtrace_expectations_supported() { +check_if_xtrace_usage_valid() { + # use --shell bash --xtrace to see trace of execution and evaluates expectations + # bash version must be >= 4 + + # Return error if SHELLSPEC_XTRACE is defined, SHELLSPEC_SHELL contains bash, + # and bash version is less than 4 + if [ -n "$SHELLSPEC_XTRACE" ]; then + #echo "*** checking SHELLSPEC_SHELL: {$SHELLSPEC_SHELL}" + if echo "$SHELLSPEC_SHELL" | grep -q bash; then + #echo "*** found bash: {$SHELLSPEC_SHELL}" + if [ -n "$BASH_VERSION" ]; then + # Extract major version (first element of BASH_VERSINFO) + if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Error: xtrace with bash requires version 4 or higher (current: $BASH_VERSION)" >&2 + return 1 + fi + else + # SHELLSPEC_SHELL contains bash but we're not running in bash + # Try to check the version of the specified bash + bash_version=$("$SHELLSPEC_SHELL" --version 2>/dev/null | head -n1) + if echo "$bash_version" | grep -q "version [0-3]\."; then + echo "Error: xtrace with bash requires version 4 or higher (detected: $bash_version)" >&2 + return 1 + fi + fi + else + echo "Error: --xtrace requires bash shell, please add the option --shell bash" >&2 + return 1 + fi + fi +} + +setup_linux_zfs_allow() { + export SRC_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" + export TGT_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" +} + +setup_freebsd_zfs_allow() { + export SRC_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" + export TGT_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" +} + +setup_linux_env() { + setup_linux_zfs_allow + export POOL_TYPE="$LOOP_DEV_POOL" + export ZELTA_AWK=mawk +} + +setup_freebsd_env() { + setup_freebsd_zfs_allow + export POOL_TYPE="$MEMORY_DISK_POOL" +} + +setup_zfs_allow() { + exec_cmd sudo zfs allow -u "$BACKUP_USER" "$SRC_ZFS_CMDS" "$SRC_POOL" + exec_cmd sudo zfs allow -u "$BACKUP_USER" "$TGT_ZFS_CMDS" "$TGT_POOL" +} + +setup_os_specific_env() { + # uname is the most reliable cross‑platform starting point + OS_TYPE=$(uname -s) + echo "Settings OS specific environment for {$OS_TYPE}" + + # Check for Ubuntu specifically + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" = "ubuntu" ]; then + # NOTE: This isn't being used currently + export LINUX_DISTRO_IS_UBUNTU=1 + fi + fi + + case "$OS_TYPE" in + Linux) + # Linux distros + setup_linux_env + ;; + FreeBSD|Darwin) + setup_freebsd_env + ;; + *) + echo "$OS_TYPE: Unsupported OS_TYPE: {$OS_TYPE}" >&2 + return 1 + ;; + esac + + echo "OS_TYPE: $OS_TYPE: set POOL_TYPE={$POOL_TYPE}" +} + + + +setup_zelta_env() { + :; +} \ No newline at end of file diff --git a/spec/lib/hello.sh b/spec/lib/hello.sh new file mode 100644 index 0000000..9c51246 --- /dev/null +++ b/spec/lib/hello.sh @@ -0,0 +1,4 @@ +# simple helper function example +hello() { + echo "What's up? Hello ${1}! TMPDIR: $SHELLSPEC_TMPDIR $SHELLSPEC_TMPBASE" +} diff --git a/spec/lib/script_util.sh b/spec/lib/script_util.sh new file mode 100644 index 0000000..b7c505e --- /dev/null +++ b/spec/lib/script_util.sh @@ -0,0 +1,41 @@ +. spec/bin/all_tests_setup/env_constants.sh + +validate_target() { + if [ $# -ne 1 ]; then + echo "Error: validate_target requires exactly 1 argument" >&2 + return 1 + fi + + case "$1" in + "$RUN_LOCALLY"|"$RUN_REMOTELY") + export RUNNING_MODE="$1" + ;; + *) + echo "Error: Invalid target '$1'" >&2 + echo "Must be one of: ${RUN_LOCALLY}, ${RUN_REMOTELY}" >&2 + return 1 + ;; + esac +} + + +validate_tree_name() { + if [ $# -ne 1 ]; then + echo "Error: Expected exactly 1 argument, got $#" >&2 + echo "Usage: $0 " >&2 + echo " tree_name must be one of: ${STANDARD_TREE}, ${DIVERGENT_TREE}, ${ENCRYPTED_TREE}" + return 1 + fi + + case "$1" in + "$STANDARD_TREE"|"$DIVERGENT_TREE"|"$ENCRYPTED_TREE") + export TREE_NAME=$1 + # Valid value + ;; + *) + echo "Error: Invalid tree_name '$1'" >&2 + echo "Must be one of: ${STANDARD_TREE}, ${DIVERGENT_TREE}, ${ENCRYPTED_TREE}" + return 1 + ;; + esac +} diff --git a/spec/spec_helper.sh b/spec/spec_helper.sh new file mode 100644 index 0000000..184067e --- /dev/null +++ b/spec/spec_helper.sh @@ -0,0 +1,111 @@ +# shellcheck shell=sh + +#. spec/initialize/test_env.sh +# Defining variables and functions here will affect all specfiles. +# Change shell options inside a function may cause different behavior, +# so it is better to set them here. +# set -eu + + +#case_insensitive_equals() { +# $str=$1 +# $str=$2 +# if [ "$(echo "$str1" | tr '[:upper:]' '[:lower:]')" = "$(echo "$str2" | tr '[:upper:]' '[:lower:]')" ]; then +# return 0 +# fi +# return 1 +#} + +# This callback function will be invoked only once before loading specfiles. +spec_helper_precheck() { + # Available functions: info, warn, error, abort, setenv, unsetenv + # Available variables: VERSION, SHELL_TYPE, SHELL_VERSION + : minimum_version "0.28.1" + info "specshell precheck: version:$VERSION shell: $SHELL_TYPE $SHELL_VERSION" + info "*** TREE_NAME is {$TREE_NAME}" + info "*** RUNNING_MODE is {$RUNNING_MODE}" + if [ "$RUNNING_MODE" = "$RUN_REMOTELY" ]; then + info "***" + info "*** Running Remotely" + info "*** Source Server is SRC_SVR:{$SRC_SVR}" + info "*** Target Server is TGT_SVR:{$TGT_SVR}" + info "***" + else + info "***" + info "*** Running Locally" + info "***" + fi + + # Convert both to lowercase for comparison + #if case_insensitive_equals $RUNNING_MODE "remote" +} + +# This callback function will be invoked after a specfile has been loaded. +spec_helper_loaded() { + : + #echo "spec_helper.sh loaded from $SHELLSPEC_HELPERDIR" +} + +start_spec() { + : + # echo "starting {$SHELLSPEC_SPECFILE}" +} + +end_spec() { + : + # echo "ending {$SHELLSPEC_SPECFILE}" +} + +start_all() { + : + #exec_cmd "echo 'hello there'" + #ls -l "./spec/lib/create_file_backed_zfs_test_pools.sh" + #. "./spec/lib/create_file_backed_zfs_test_pools.sh" + #curdir=$(pwd) + #echo "spec_helper.sh start_all curdir:$curdir" + #./spec/initialize/create_file_backed_zfs_test_pools.sh + #./spec/initialize/initialize_testing_setup.sh + + #echo "staring all" + #. ./spec/initialize/initialize_testing_setup +} + +end_all() { + : + #echo "after all" +} + +# This callback function will be invoked after core modules has been loaded. +spec_helper_configure() { + # Available functions: import, before_each, after_each, before_all, after_all + : import 'support/custom_matcher' + before_each start_spec + after_each end_spec + before_all start_all + after_all end_all +} + +# Define helper functions AFTER spec_helper_configure +# These will be available in all spec files and in before_all/after_all blocks +exec_cmd() { + printf '%s' "$*" >&2 + if "$@"; then + printf ' :* succeeded\n' >&2 + return 0 + else + _exit_code=$? + printf ' :! failed (exit code: %d)\n' "$_exit_code" >&2 + return "$_exit_code" + fi +} + + +# In spec_helper.sh +capture_stderr() { + RESULT=$({ "$@" 2>&1 1>/dev/null; } 2>&1) || true + #RESULT="hello" +} + +#spec/initialize/test_env.sh + +#exec_cmd printf "hello there\n" diff --git a/spec/util/README.md b/spec/util/README.md new file mode 100644 index 0000000..a486fc4 --- /dev/null +++ b/spec/util/README.md @@ -0,0 +1,47 @@ +## Utilities + +`match_function_generator.sh` - create case statement for shellspec + +## Examples + +### Shellspec matcher example +- ### Generating a matcher function or shellspec from zelta match output +```shell +$ ./matcher_func_generator.sh test_data/zelta_match_output.txt match_after_rotate_output +match_after_rotate_output() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub2 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub2/orphan @two @zelta_"*" @two syncable (incremental)"|\ + "/sub3 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub3/space name @two @zelta_"*" @two syncable (incremental)"|\ + "/vol1 @go @zelta_"*" @go syncable (incremental)"|\ + "3 up-to-date, 5 syncable"|\ + "8 total datasets compared") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} +``` + +- ### Shellspec example using the generated matcher +```shell +Describe 'test zelta match output example' + It "match $SOURCE and $TARGET" + When call zelta match $SOURCE $TARGET + The output should satisfy match_after_rotate_output + The status should equal 0 + End +Enc +``` \ No newline at end of file diff --git a/spec/util/generate_case_stmt_func.awk b/spec/util/generate_case_stmt_func.awk new file mode 100644 index 0000000..3340ab2 --- /dev/null +++ b/spec/util/generate_case_stmt_func.awk @@ -0,0 +1,40 @@ +#!/usr/bin/awk -f + +# ignore blank lines +/^$/ { next } + +# ignore comment lines starting with # +/^[[:space:]]*#/ { next } + +# Process data lines +{ + gsub(/[[:space:]]+/, " ", $0) + gsub(/@zelta_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, "@zelta_\"*\"",$0) + lines[count++] = $0 +} + +END { + print func_name "() {" + print " while IFS= read -r line; do" + print " # normalize whitespace, remove leading/trailing spaces" + print " normalized=$(echo \"$line\" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" + print " case \"$normalized\" in" + + line_continue = "\"|\\" + case_end = "\")" + + for (i = 0; i < count; i++) { + line_end = (i + 1 == count) ? case_end : line_continue + print " \"" lines[i] line_end + } + + print " ;;" + print " *)" + print " printf \"Unexpected line format: %s\\n\" \"$line\" >&2" + print " return 1" + print " ;;" + print " esac" + print " done" + print " return 0" + print "}" +} diff --git a/spec/util/matcher_func_generator.sh b/spec/util/matcher_func_generator.sh new file mode 100755 index 0000000..83b11b6 --- /dev/null +++ b/spec/util/matcher_func_generator.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Check for required arguments +if [ $# -ne 2 ]; then + printf "Usage: %s \n" "$0" >&2 + exit 1 +fi + +input_file="$1" +func_name="$2" + +# Check if input file exists +if [ ! -f "$input_file" ]; then + printf "Error: File '%s' not found\n" "$input_file" >&2 + exit 1 +fi + +# Pass to AWKt +awk -v func_name="$func_name" -f generate_case_stmt_func.awk "$input_file" + diff --git a/spec/util/test_data/zelta_backup_after_rotate_output.txt b/spec/util/test_data/zelta_backup_after_rotate_output.txt new file mode 100644 index 0000000..4f6af25 --- /dev/null +++ b/spec/util/test_data/zelta_backup_after_rotate_output.txt @@ -0,0 +1,3 @@ +syncing 8 datasets +8 datasets up-to-date +3K sent, 5 streams received in 0.11 seconds \ No newline at end of file diff --git a/spec/util/test_data/zelta_match_after_backup_output.txt b/spec/util/test_data/zelta_match_after_backup_output.txt new file mode 100644 index 0000000..5af95aa --- /dev/null +++ b/spec/util/test_data/zelta_match_after_backup_output.txt @@ -0,0 +1,10 @@ +DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO +[treetop] @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub1/child @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub2 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub2/orphan @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub3 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub3/space name @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/vol1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +8 up-to-date \ No newline at end of file diff --git a/spec/util/test_data/zelta_match_output.txt b/spec/util/test_data/zelta_match_output.txt new file mode 100644 index 0000000..7610a58 --- /dev/null +++ b/spec/util/test_data/zelta_match_output.txt @@ -0,0 +1,11 @@ +DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO +[treetop] @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub1/child @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date +/sub2 @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) +/sub2/orphan @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) +/sub3 @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) +/sub3/space name @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) +/vol1 @go @zelta_2026-01-11_21.50.46 @go syncable (incremental) +3 up-to-date, 5 syncable +8 total datasets compared diff --git a/test/test_runner.sh b/test/test_runner.sh new file mode 100755 index 0000000..b7838f1 --- /dev/null +++ b/test/test_runner.sh @@ -0,0 +1,87 @@ +#!/bin/sh + +set -e + +. spec/lib/script_util.sh + + + +test_setup() { + if [ $# -ne 2 ]; then + echo "Error: Expected 2 arguments: " >&2 + echo "Usage: $0 <${RUN_LOCALLY}|${RUN_REMOTELY}> <${STANDARD_TREE}|${DIVERGENT_TREE}|${ENCRYPTED_TREE}>" >&2 + return 1 + fi + + if ! validate_target "$1"; then + return 1 + fi + + if ! validate_tree_name "$2"; then + return 1 + fi + + case "$RUNNING_MODE" in + "$RUN_LOCALLY") + unset TGT_SVR + unset SRV_SVR + exec_local_setup + ;; + "$RUN_REMOTELY") + export SRC_SVR="${SRC_SVR:-dever@fzfsdev}" + # TODO: sort out 2nd server send/receive with host alias fzfsdev2 + # export TGT_SVR="${TGT_SVR:-dever@fzfsdev2}" + export TGT_SVR="${TGT_SVR:-dever@fzfsdev}" + exec_remote_setup + ;; + esac + +} + +exec_local_setup() { + printf "\n***\n*** Running Locally\n***\n" + + echo "Step 1/3: Initializing local test environment..." + spec/bin/all_tests_setup/all_tests_setup.sh + + echo "Step 2/3: Creating test dataset tree..." + sudo spec/bin/${TREE_NAME}_test/${TREE_NAME}_snap_tree.sh +} + +exec_remote_setup() { + printf "\n***\n*** Running Remotely: SRC_SVR:{$SRC_SVR} TGT_SVR:{$TGT_SVR}\n***\n" + + echo "Steps 1 and 2, Initializing remote setup, create pools setup snap tree" + spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh $TREE_NAME +} + +run_tests() { + echo "Step 3/3: Running zelta tests..." + + # shellspec options to include + #SHELLSPEC_TESTOPT="${SHELLSPEC_TESTOPT:-}" + + # this options will show a trace with expectation evaluation + SHELLSPEC_TESTOPT="--xtrace --shell bash" + + # comment out this line to show the trace + unset SHELLSPEC_TESTOPT + + shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh + + # examples of selective tests runs + # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@1 + # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2 + # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2-1 + # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2-2 + + echo "" + echo "✓ Tests complete" +} + +if test_setup "$@"; then + # NOTE: update the environment after SRC_SVR and TGT_SVR are set!! + . spec/bin/all_tests_setup/common_test_env.sh + run_tests + #printf "***\n*** check tree run tests manually\n***\n" +fi From 2f603f16b526cd42accc15c710e931b56a1220ee Mon Sep 17 00:00:00 2001 From: Richard Logwood Date: Wed, 14 Jan 2026 22:02:36 -0500 Subject: [PATCH 03/47] update the README TOC --- spec/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/README.md b/spec/README.md index af49785..102cf97 100644 --- a/spec/README.md +++ b/spec/README.md @@ -7,6 +7,8 @@ * [🔰 Making your first test - a simple Example](#-making-your-first-test---a-simple-example) * [Setting up a local development environment](#setting-up-a-local-development-environment) * [Testing `zelta`](#testing-zelta) + * [:star: Using the test helper](#star-using-the-test-helper) + * [Example test_runner.sh output](#example-test_runnersh-output) * [:zap: To run the standard Zelta test](#zap-to-run-the-standard-zelta-test-) * [:zap: To run all tests](#zap-to-run-all-tests) * [🔰 shellspec examples](#-shellspec-examples) From b6365a5e72cd7b7fec9ee620983bd48024a21a52 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 15 Jan 2026 07:53:09 -0500 Subject: [PATCH 04/47] shellspec options experiment feat: make ssh_opts_test spec configurable with environment variables feat: add independent target endpoint and clone validation tests fix: missing override options and rename dryrun test: shellspec opts test stuff test: clean up option parsing tests to eliminate warnings and validate outputs test: rename opts test --- share/zelta/zelta-backup.awk | 5 + share/zelta/zelta-opts.tsv | 4 +- spec/bin/opts_test/opts_spec.sh | 223 ++++++++++++++++++++++++++++++++ spec/bin/opts_test/test_env.sh | 36 ++++++ 4 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 spec/bin/opts_test/opts_spec.sh create mode 100644 spec/bin/opts_test/test_env.sh diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 9b2a70f..1ddc043 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -900,6 +900,11 @@ function rename_dataset(endpoint, _old_ds, _new_ds, _remote, _snap, _cmd_arr["old_ds"] = rq(_remote, _old_ds) _cmd_arr["new_ds"] = rq(_remote, _new_ds) _cmd = build_command("RENAME", _cmd_arr) + if (Opt["DRYRUN"]) { + report(LOG_NOTICE, "would rename '" _old_ds "' to '" _new_ds"'") + report(LOG_NOTICE, "+ "_cmd) + return _new_ds + } report(LOG_NOTICE, "renaming '" _old_ds "' to '" _new_ds"'") report(LOG_DEBUG, "`"_cmd"`") _cmd = _cmd CAPTURE_OUTPUT diff --git a/share/zelta/zelta-opts.tsv b/share/zelta/zelta-opts.tsv index 5097404..babdca0 100644 --- a/share/zelta/zelta-opts.tsv +++ b/share/zelta/zelta-opts.tsv @@ -26,10 +26,12 @@ all --yule YULE true # Options for 'zelta backup' and other 'zfs send/recv/clone' verbs policy,backup,replicate,sync,rotate --json,-j LOG_MODE set json json output -policy,backup,replicate,sync,rotate -b,--backup,-c,--compressed,-D,--dedup,-e,--embed,--holds,-L,--largeblock,-p,--parsable,--proctitle,--props,--raw,--skipmissing,-V,-w SEND_OVERRIDE arglist policy,backup,replicate,sync,rotate --send-check SEND_CHECK true +policy,backup,replicate,sync,rotate -b,--backup,-c,--compressed,-D,--dedup,-e,--embed,--holds,-L,--largeblock,-p,--parsable,--proctitle,--props,--raw,--skipmissing,-V,-w SEND_OVERRIDE arglist policy,backup,replicate,sync,rotate -F SEND_OVERRIDE arglist destructive option 'F' used; consider 'zelta rotate' when possible +policy,backup,replicate,sync,rotate --send-override SEND_OVERRIDE set policy,backup,replicate,sync,rotate -e,-M,-u RECV_OVERRIDE arglist +policy,backup,replicate,sync,rotate --recv-override RECV_OVERRIDE set policy,backup,replicate,sync,rotate --rotate VERB set rotate policy,backup,replicate,sync,rotate --replicate,-R VERB set replicate policy,backup,replicate,sync,rotate --resume RESUME true save partial transfers (default) diff --git a/spec/bin/opts_test/opts_spec.sh b/spec/bin/opts_test/opts_spec.sh new file mode 100644 index 0000000..fb8ae9a --- /dev/null +++ b/spec/bin/opts_test/opts_spec.sh @@ -0,0 +1,223 @@ +# Option parsing validation tests for Zelta +# Configurable endpoints - override via environment or test_env.sh +# +# Usage: +# # With defaults (local testpool) +# shellspec spec/bin/opts_test/opts_spec.sh +# +# # With remote endpoints +# SRC_ENDPOINT=user@host:pool/src TGT_ENDPOINT=backupuser@backuphost:backuppool/tgt shellspec spec/bin/opts_test/opts_spec.sh + +# Load configurable environment +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [ -f "${SCRIPT_DIR}/test_env.sh" ]; then + . "${SCRIPT_DIR}/test_env.sh" +fi + +# Fallback defaults if test_env.sh not loaded or vars not set +: "${SRC_ENDPOINT:=testpool/source}" +: "${TGT_ENDPOINT:=testpool/target}" +: "${CLONE_ENDPOINT:=${SRC_ENDPOINT}_clone}" +: "${CLONE_ENDPOINT_INVALID:=otherhost:differentpool/clone}" +: "${TEST_SNAP_NAME:=test_snapshot}" +: "${TEST_DEPTH:=2}" +: "${TEST_EXCLUDE:=*/swap,@*_hourly}" + +Describe "Option Parsing Validation" + + Describe "zelta backup with comprehensive options" + It "parses all major options and produces valid JSON output" + When run zelta backup \ + --dryrun \ + --depth "$TEST_DEPTH" \ + --exclude "$TEST_EXCLUDE" \ + --snap-name "$TEST_SNAP_NAME" \ + --snapshot \ + --intermediate \ + --resume \ + --push \ + --send-default '-Lce' \ + --send-raw '-Lw' \ + --send-new '-p' \ + --recv-default '' \ + --recv-top '-o readonly=on' \ + --recv-fs '-u -x mountpoint -o canmount=noauto' \ + --recv-vol '-o volmode=none' \ + -o 'compression=lz4' \ + -x 'mountpoint' \ + --json \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + The output should include '"sourceEndpoint":' + The output should include '"targetEndpoint":' + The output should include '"replicationStreamsSent":' + The stderr should not include "error" + The stderr should not include "invalid" + End + End + + Describe "zelta backup with alternative option forms" + It "parses short options and alternative flags" + When run zelta backup \ + -n \ + -qq \ + -d 1 \ + -X '/tmp,*/cache' \ + -i \ + --no-resume \ + --pull \ + --no-snapshot \ + -L \ + --largeblock \ + --compressed \ + --embed \ + --props \ + --raw \ + -u \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + The stderr should equal "" + End + End + + Describe "zelta backup with override options" + It "parses send and recv override options" + When run zelta backup \ + --dryrun \ + -qq \ + --send-override '-Lce' \ + --recv-override '-o readonly=on' \ + --recv-pipe 'cat' \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + The stderr should equal "" + End + End + + Describe "zelta match options" + It "parses match-specific options and produces output" + When run zelta match \ + -H \ + -p \ + -o 'ds_suffix,match,xfer_size' \ + --written \ + --time \ + -d "$TEST_DEPTH" \ + -X '*/swap' \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + The output should be defined + End + End + + Describe "zelta snapshot options" + It "parses snapshot options in dryrun mode" + When run zelta snapshot \ + --dryrun \ + -qq \ + --snap-name 'manual_test' \ + -d 1 \ + "$SRC_ENDPOINT" + The status should equal 0 + End + End + + Describe "zelta clone options" + It "parses clone options in dryrun mode" + When run zelta clone \ + --dryrun \ + -qq \ + --snapshot \ + --snap-name 'clone_snap' \ + -d "$TEST_DEPTH" \ + "$SRC_ENDPOINT" "$CLONE_ENDPOINT" + The status should equal 0 + End + End + + Describe "zelta clone endpoint validation" + It "rejects clone to mismatched pool/host" + When run zelta clone \ + --dryrun \ + -qq \ + "$SRC_ENDPOINT" "$CLONE_ENDPOINT_INVALID" + The status should not equal 0 + The stderr should include "cannot clone" + End + End + + Describe "zelta rotate options" + It "parses rotate options in dryrun mode" + When run zelta rotate \ + --dryrun \ + -qq \ + --no-snapshot \ + --push \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + End + End + + Describe "zelta revert options" + It "parses revert options in dryrun mode" + When run zelta revert \ + --dryrun \ + -qq \ + "$SRC_ENDPOINT" + The status should equal 0 + End + End + + Describe "zelta prune options" + It "parses prune options" + When run zelta prune \ + --dryrun \ + -qq \ + --keep-snap-days 90 \ + --keep-snap-num 100 \ + --no-ranges \ + -d "$TEST_DEPTH" \ + -X '*/tmp' \ + "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should equal 0 + End + End + + Describe "deprecated option warnings" + It "warns about deprecated -s option" + When run zelta backup --dryrun -qq -s "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The stderr should include "deprecated" + End + + It "warns about deprecated -t option" + When run zelta backup --dryrun -qq -t "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The stderr should include "deprecated" + End + + It "warns about deprecated -T option" + When run zelta backup --dryrun -qq -T "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The stderr should include "deprecated" + End + End + + Describe "invalid option handling" + It "rejects invalid options gracefully" + When run zelta backup --dryrun -qq --invalid-option-xyz "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should not equal 0 + The stderr should include "invalid" + End + + It "rejects deprecated --initiator option" + When run zelta backup --dryrun -qq --initiator PULL "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should not equal 0 + The stderr should include "deprecated" + End + + It "rejects deprecated --progress option" + When run zelta backup --dryrun -qq --progress "$SRC_ENDPOINT" "$TGT_ENDPOINT" + The status should not equal 0 + The stderr should include "deprecated" + End + End + +End diff --git a/spec/bin/opts_test/test_env.sh b/spec/bin/opts_test/test_env.sh new file mode 100644 index 0000000..6025c66 --- /dev/null +++ b/spec/bin/opts_test/test_env.sh @@ -0,0 +1,36 @@ +# Configurable test environment for opts_test +# Override these via environment variables before running tests +# +# Examples: +# export SRC_ENDPOINT="user@host:testpool/source" +# export TGT_ENDPOINT="otheruser@backuphost:backuppool/target" +# +# Or for local testing: +# export SRC_ENDPOINT="testpool/source" +# export TGT_ENDPOINT="backuppool/target" +# +# Run tests: +# shellspec spec/bin/opts_test/opts_spec.sh + +# Source endpoint - the dataset tree to back up / match / snapshot +: "${SRC_ENDPOINT:=testpool/source}" + +# Target endpoint - where backups go (can be completely different host/pool) +: "${TGT_ENDPOINT:=testpool/target}" + +# Clone endpoint - for zelta clone tests +# Must be on same pool as source (user@host:pool must match exactly) +# Default: derive from SRC_ENDPOINT by appending _clone to the dataset path +: "${CLONE_ENDPOINT:=${SRC_ENDPOINT}_clone}" + +# Invalid clone endpoint - for testing clone failure on mismatched pool/host +: "${CLONE_ENDPOINT_INVALID:=otherhost:differentpool/clone}" + +# Snapshot name for tests that create snapshots +: "${TEST_SNAP_NAME:=test_snapshot}" + +# Depth limit for recursive operations (0 = unlimited) +: "${TEST_DEPTH:=2}" + +# Exclusion patterns for testing +: "${TEST_EXCLUDE:=*/swap,@*_hourly}" From 3bddf6d837e90d3fc8f4586e296f320b8992605a Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 15 Jan 2026 09:39:17 -0500 Subject: [PATCH 05/47] feat: support multiple operand endpoints and fix top-level dataset reporting --- share/zelta/zelta-report.awk | 58 ++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/share/zelta/zelta-report.awk b/share/zelta/zelta-report.awk index bdb4280..e59bbb2 100644 --- a/share/zelta/zelta-report.awk +++ b/share/zelta/zelta-report.awk @@ -8,17 +8,7 @@ ## Initialization ################# -function init_report( _endpoint_str) { - # Get backup root from options or first operand - _endpoint_str = Opt["BACKUP_ROOT"] - if (!_endpoint_str && NumOperands >= 1) - _endpoint_str = Operands[1] - if (!_endpoint_str) - stop(1, "BACKUP_ROOT not set") - - # Parse the endpoint to handle remote targets - load_endpoint(_endpoint_str, BackupRoot) - +function init_report() { # Get Slack hook from options SlackHook = Opt["SLACK_HOOK"] if (!SlackHook) @@ -28,11 +18,24 @@ function init_report( _endpoint_str) { TooOld = sys_time() - 86400 } +# Initialize state for processing a single endpoint +function init_endpoint(_endpoint_str) { + # Clear previous endpoint state + delete BackupRoot + delete SeenDS + delete OldList + OutOfDateCount = 0 + UpToDateCount = 0 + + # Parse the endpoint to handle remote targets + load_endpoint(_endpoint_str, BackupRoot) +} + ## ZFS List ########### # Build and run the zfs list command for the backup root -function get_snapshot_ages( _cmd_arr, _cmd, _remote) { +function get_snapshot_ages( _cmd, _remote) { _remote = get_remote_cmd(BackupRoot) _cmd = "zfs list -t filesystem,volume -Hpr -o name,snapshots_changed -S snapshots_changed" _cmd = str_add(_cmd, qq(BackupRoot["DS"])) @@ -50,12 +53,14 @@ function parse_snapshot_list( _cmd, _ds, _changed, _rel_name) { # Skip if we've already seen this dataset if (_ds in SeenDS) continue SeenDS[_ds] = 1 - # Skip datasets without snapshot info - if (_changed == "-") continue # Get relative name by removing backup root prefix _rel_name = _ds sub("^" BackupRoot["DS"] "/?", "", _rel_name) if (!_rel_name) _rel_name = BackupRoot["LEAF"] + # Skip top-level dataset if it has no snapshots (that's OK) + if (_rel_name == BackupRoot["LEAF"] && _changed == "-") continue + # Skip datasets without snapshot info + if (_changed == "-") continue # Categorize by age if (_changed < TooOld) { OldList[++OutOfDateCount] = _rel_name @@ -92,18 +97,33 @@ function send_slack_message(message, _curl) { _curl = "curl -s -X POST -H 'Content-type: application/json; charset=utf-8' " \ "--data '{ \"username\": \"zeport\", \"icon_emoji\": \":camera_with_flash:\", \"text\": \"" \ message "\" }' " SlackHook -print _curl _curl | getline close(_curl) } +# Process a single endpoint +function process_endpoint(_endpoint_str, _msg) { + init_endpoint(_endpoint_str) + parse_snapshot_list() + _msg = build_slack_message() + report(LOG_NOTICE, _msg) + send_slack_message(_msg) +} + ## Main ####### BEGIN { init_report() - parse_snapshot_list() - SlackMessage = build_slack_message() - report(LOG_NOTICE, SlackMessage) - send_slack_message(SlackMessage) + + # Process BACKUP_ROOT if set, otherwise use operands + if (Opt["BACKUP_ROOT"]) { + process_endpoint(Opt["BACKUP_ROOT"]) + } else if (NumOperands >= 1) { + for (_i = 1; _i <= NumOperands; _i++) { + process_endpoint(Operands[_i]) + } + } else { + stop(1, "BACKUP_ROOT not set and no endpoints provided") + } } From 5d942b65512238f52efe6f3cb12349eb34f7184c Mon Sep 17 00:00:00 2001 From: Richard Logwood Date: Thu, 15 Jan 2026 17:39:42 +0000 Subject: [PATCH 06/47] generalize backup validation test --- spec/bin/standard_test/standard_test_spec.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/bin/standard_test/standard_test_spec.sh b/spec/bin/standard_test/standard_test_spec.sh index a24aeb4..612674b 100644 --- a/spec/bin/standard_test/standard_test_spec.sh +++ b/spec/bin/standard_test/standard_test_spec.sh @@ -109,9 +109,9 @@ Describe 'try backup' End Parameters - 8 '^(apool|bpool)' $TGT_SVR - 4 apool/treetop $TGT_SVR - 4 bpool/backups/treetop $TGT_SVR + 8 "^($SRC_POOL|$TGT_TREE)" $TGT_SVR + 4 $SRC_TREE $TGT_SVR + 4 $TGT_TREE $TGT_SVR End It "has $1 snapshots on ${3:-localhost} ${2:+matching pattern '$2'}" From 0ac745e5e5142be53b4f1d0e8a909864e515c761 Mon Sep 17 00:00:00 2001 From: Richard Logwood Date: Thu, 15 Jan 2026 17:42:52 +0000 Subject: [PATCH 07/47] fix typo in backup verification --- spec/bin/standard_test/standard_test_spec.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/bin/standard_test/standard_test_spec.sh b/spec/bin/standard_test/standard_test_spec.sh index 612674b..1928923 100644 --- a/spec/bin/standard_test/standard_test_spec.sh +++ b/spec/bin/standard_test/standard_test_spec.sh @@ -109,7 +109,7 @@ Describe 'try backup' End Parameters - 8 "^($SRC_POOL|$TGT_TREE)" $TGT_SVR + 8 "^($SRC_POOL|$TGT_POOL)" $TGT_SVR 4 $SRC_TREE $TGT_SVR 4 $TGT_TREE $TGT_SVR End From f12fa712ea6ce5f64da34c03a4f11092a71accde Mon Sep 17 00:00:00 2001 From: Richard Logwood Date: Thu, 15 Jan 2026 23:02:23 +0000 Subject: [PATCH 08/47] run snapshot count on the associated server --- spec/bin/standard_test/standard_test_spec.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/bin/standard_test/standard_test_spec.sh b/spec/bin/standard_test/standard_test_spec.sh index 1928923..0b4a093 100644 --- a/spec/bin/standard_test/standard_test_spec.sh +++ b/spec/bin/standard_test/standard_test_spec.sh @@ -109,8 +109,7 @@ Describe 'try backup' End Parameters - 8 "^($SRC_POOL|$TGT_POOL)" $TGT_SVR - 4 $SRC_TREE $TGT_SVR + 4 $SRC_TREE $SRC_SVR 4 $TGT_TREE $TGT_SVR End From 43925fc20fe8f8765e72a4481fcb4ccbc2e25e79 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 15 Jan 2026 19:43:21 -0500 Subject: [PATCH 09/47] drop some unused lines, clean explain_sync_status, better diverged message --- share/zelta/zelta-backup.awk | 56 +++++------------------------------- 1 file changed, 7 insertions(+), 49 deletions(-) diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 1ddc043..ef2cba3 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -247,50 +247,12 @@ function compute_send_range(ds_suffix, _ds_suffix, _src_idx, _final_ds_snap) { DSPair[_ds_suffix, "source_end"] = _final_ds_snap } -# Describe possible actions -function explain_sync_status(ds_suffix, _src_idx, _tgt_idx, _src_ds, _tgt_ds) { - _src_idx = "SRC" SUBSEP ds_suffix - _tgt_idx = "TGT" SUBSEP ds_suffix - _src_ds = Opt["SRC_DS"] ds_suffix - _tgt_ds = Opt["TGT_DS"] ds_suffix - +# Report block reasons for datasets that couldn't be synced +function explain_sync_status(ds_suffix, _tgt_ds) { + _tgt_ds = Opt["TGT_DS"] ds_suffix + # Only report if there's a block reason to explain if (Action[ds_suffix, "block_reason"]) - report(LOG_NOTICE, Action[ds_suffix, "block_reason"]": " _tgt_ds) - return - - # TO-DO: Review this - if (!DSPair[ds_suffix, "sync_blocked"]) { - if (!Dataset[_tgt_idx, "exists"]) - report(LOG_NOTICE, "full backup pending or incomplete: " _tgt_ds) - else - report(LOG_NOTICE, "incremental/intermediate backup pending or incomplete: " _tgt_ds) - return 1 - } - # States that can't be resolved with a sync or rotate - else if (!Dataset[_src_idx, "exists"]) - report(LOG_NOTICE, "missing source, cannot sync: " _tgt_ds) - else if (!Dataset[_src_idx, "latest_snapshot"]) - report(LOG_NOTICE, "missing source snapshot, cannot sync: " _tgt_ds) - else if (Dataset[_src_idx, "latest_snapshot"] == Dataset[_tgt_idx, "latest_snapshot"]) - report(LOG_INFO, "up-to-date: " _tgt_ds) - # States that require 'zelta rotate', 'zfs rollback', or 'zfs rename' - else if (Dataset[_tgt_idx, "exists"] && !Dataset[_tgt_idx, "latest_snapshot"]) { - report(LOG_NOTICE, "sync blocked; target exists with no snapsots: " _tgt_ds) - report(LOG_NOTICE, "- full backup is required; try 'zelta rotate' or 'zfs rename'") - } - else if (Dataset[_tgt_idx, "exists"] && !DSPair[ds_suffix, "match"]) { - report(LOG_NOTICE, "sync blocked; target has no matching snapshots: " _tgt_ds) - if (Dataset[_src_idx, "origin"]) - report(LOG_NOTICE, "- source is a clone; try 'zelta rotate' to recover or") - report(LOG_NOTICE, "- create a new full backup using 'zelta rotate' or 'zfs rename'") - } - else if ((Dataset[_tgt_idx, "latest_snapshot"] != DSPair[ds_suffix, "match"]) || \ - (DSPair[ds_suffix, "target_is_written"] && DSPair[ds_suffix, "match"])) { - report(LOG_NOTICE, "sync blocked; target diverged: " _tgt_ds) - report(LOG_NOTICE, "- backup history can be retained with 'zelta rotate'") - report(LOG_NOTICE, "- or destroy divergent dataset with: zfs rollback " _tgt_ds DSPair[ds_suffix, "match"]) - } - else report(LOG_WARNING, "unknown sync state for " _tgt_ds) + report(LOG_NOTICE, Action[ds_suffix, "block_reason"]": " _tgt_ds) } # Ensure source snapshots are avialable and load snapshot relationship data @@ -410,9 +372,9 @@ function compute_eligibility( _i, _ds_suffix, _src_idx, _tgt_idx, continue } - # Target is ahead + # Target is ahead or has diverged otherwise if (_match != _tgt_latest) { - Action[_ds_suffix, "block_reason"] = "target snapshots beyond the source match" + Action[_ds_suffix, "block_reason"] = "target has divereged" Action[_ds_suffix, "can_rotate"] = 1 DSTree["rotatable"]++ continue @@ -764,7 +726,6 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, # TO-DO: Dryrun mode probably goes here if (Opt["VERB"] == "rotate" && !Action[ds_suffix, "can_rotate"]) return if (Opt["VERB"] != "rotate" && !Action[ds_suffix, "can_sync"]) return - #IGNORE_ZFS_SEND_OUTPUT = "^(incremental|full)| records (in|out)$|bytes.*transferred|(create|receive) mountpoint|ignoring$" IGNORE_ZFS_SEND_OUTPUT = "^(incremental|full)" IGNORE_RESUME_OUTPUT = "^nvlist version|^\t(fromguid|object|offset|bytes|toguid|toname|embedok|compressok)" WARN_ZFS_RECV_PROPS = "cannot receive .* property" @@ -840,7 +801,6 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, Action[ds_suffix, "can_sync"] = 0 } } - #jlist("errorMessages", error_list) ## Construct replication commands function get_sync_command(ds_suffix, _src_idx, _tgt_idx, _cmd, _zfs_send, _zfs_recv, @@ -1082,8 +1042,6 @@ function print_summary( _status, _i, _ds_suffix, _num_streams) { _ds_suffix = DSList[_i] explain_sync_status(_ds_suffix) } -# if (!Summary["replicationStreamsSent"]) -# report(LOG_NOTICE, "nothing to sync") } _bytes_sent = h_num(Summary["replicationSize"]) _num_streams = Summary["replicationStreamsReceived"] From 4cd49a2249e072ec2529e20bb9ad0578915d9b25 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 15 Jan 2026 21:36:14 -0500 Subject: [PATCH 10/47] doc bugs, increment version to rc1 --- CHANGELOG.md | 2 +- README.md | 6 +++--- bin/zelta | 2 +- doc/zelta-backup.md | 4 ++-- doc/zelta-revert.md | 2 +- doc/zelta-rotate.md | 2 +- doc/zelta.md | 4 ++-- zelta.conf | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dad7c0c..2f0d248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to the Zelta will be documented in this file. -## [1.1.beta4] - 2026-01-12 +## [1.1.rc1] - 2026-01-15 This section will be modified until v1.1 is officially released. ### Added diff --git a/README.md b/README.md index 1863764..80ae512 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ ![Zelta Logo](https://zelta.space/index/zelta-banner.svg) # The Zelta Backup and Recovery Suite -*Version v1.1-beta4, 2026-01-13* +*Version v1.1-rc1, 2026-01-15* --- -> **âš ī¸ BETA NOTICE** -> This is a beta release with significant improvements over the stable 1.0 branch. Documentation and QA are ongoing, but the codebase is solid and is actively being used in production for thousands of backup workflows. +> **âš ī¸ RELEASE CANDIDATE** +> This is a release has significant improvements over the stable 1.0 branch. Documentation and QA are near completion, and the codebase is solid and is actively being used in production for thousands of backup workflows. > > - **Stable Release:** For the most tried-and-true version of Zelta see the [March 2024 release (v1.0)](https://github.com/bellhyve/zelta/tree/release/1.0) > - **What's New:** Check [CHANGELOG.md](CHANGELOG.md) for the latest changes diff --git a/bin/zelta b/bin/zelta index 5ec6b96..a297f48 100755 --- a/bin/zelta +++ b/bin/zelta @@ -4,7 +4,7 @@ # # Initialize the environment for Zelta subcommands -ZELTA_VERSION="Zelta Replication Suite v1.1-beta4" +ZELTA_VERSION="Zelta Replication Suite v1.1-rc1" zelta_usage() { exec >&2 diff --git a/doc/zelta-backup.md b/doc/zelta-backup.md index 9d9283b..3082943 100644 --- a/doc/zelta-backup.md +++ b/doc/zelta-backup.md @@ -76,7 +76,7 @@ _target_ : Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. **-j, \--json** -: Output results in JSON format. See **zelta-options(8)** for details. +: Output results in JSON format. See **zelta-options(7)** for details. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them. @@ -273,7 +273,7 @@ Returns 0 on success, non-zero on error. # NOTES -See **zelta-options(8)** for environment variables, `zelta.env` configuration, and `zelta policy` integration. +See **zelta-options(7)** for environment variables, `zelta.env` configuration, and `zelta policy` integration. The `zelta sync` command is a convenience alias for `zelta backup -i` and may be extended in future versions with additional optimizations for continuous replication workflows. diff --git a/doc/zelta-revert.md b/doc/zelta-revert.md index d795d33..7517d4f 100644 --- a/doc/zelta-revert.md +++ b/doc/zelta-revert.md @@ -69,7 +69,7 @@ The original diverged datasets remain accessible as `sink/source/dataset_ diff --git a/zelta.conf b/zelta.conf index 5cb42b6..fc47d4a 100644 --- a/zelta.conf +++ b/zelta.conf @@ -1,4 +1,4 @@ -# Zelta Policy Backup Configuration v1.1-beta4 +# Zelta Policy Backup Configuration v1.1-rc1 # # This file configures "zelta policy" backup jobs. For global defaults that apply to all Zelta # commands (zelta backup, zelta match, etc.), see zelta.env. For complete option documentation From fc3ecd445747b05ee0a63b633d0436cb106e966a Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 15 Jan 2026 21:43:59 -0500 Subject: [PATCH 11/47] docs: tighten prose and fix errors in STYLE.md drop beta warning --- README.md | 5 +- STYLE.md | 220 +++++++++++++++++++----------------------------------- 2 files changed, 77 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index 80ae512..3b69844 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,9 @@ --- -> **âš ī¸ RELEASE CANDIDATE** -> This is a release has significant improvements over the stable 1.0 branch. Documentation and QA are near completion, and the codebase is solid and is actively being used in production for thousands of backup workflows. -> -> - **Stable Release:** For the most tried-and-true version of Zelta see the [March 2024 release (v1.0)](https://github.com/bellhyve/zelta/tree/release/1.0) > - **What's New:** Check [CHANGELOG.md](CHANGELOG.md) for the latest changes > - **Found a Bug?** Please [open an issue](https://github.com/bellhyve/zelta/issues) +> - **Previous Release:** [March 2024, Zelta v1.0](https://github.com/bellhyve/zelta/tree/release/1.0) --- diff --git a/STYLE.md b/STYLE.md index 1003c31..40f7f0f 100644 --- a/STYLE.md +++ b/STYLE.md @@ -2,36 +2,36 @@ This document defines the coding standards for the Zelta Backup and Recovery Suite. Zelta is designed for **portability** and **safety** and is written in portable Bourne Shell and Awk. -A good rule of thumb is to lean towards POSIX and "Original Awk" standards. However, to be portable with most systems that have ever run OpenZFS, we can't adhere obsessively to not to a specific standard or ideal. For example, some POSIX standards are absent from some operating system defaults and everything needs to be tested. +Lean towards POSIX and "Original Awk" standards, but don't adhere obsessively to any single standard. Some POSIX features are absent from certain OS defaults, so everything needs testing on real systems. --- ## 1. Naming Conventions -We use specific casing to denote variable scope and type. +Casing denotes variable scope and type. | Type | Case Style | Example | Context | | :--- | :--- | :--- | :--- | -| **Constants** | CAPS | `SNAP_NAME` | Values that do not change during runtime. | -| **Globals** | CamelCase | `Dataset`, `NumDS` | Global state variables, available throughout execution. | -| **Locals** | _snake_case | `_idx`, `_temp_val` | specific variables internal to a function or loop. | -| **Arguments** | lowercase | `target_ds`, `flag` | Variables passed into a function. | -| **Array Keys (Const)** | "CAPS" | `Opt["VERBOSE"]` | Settings or fixed keys within associative arrays. | -| **Array Keys (Local)** | "lowercase" | `Dataset[ds_suffix, "match"]` | Keys that are applied during script runtime. | +| **Constants** | CAPS | `SNAP_NAME` | Values unchanged during runtime | +| **Globals** | CamelCase | `Dataset`, `NumDS` | Global state variables | +| **Locals** | _snake_case | `_idx`, `_temp_val` | Variables internal to a function or loop | +| **Arguments** | lowercase | `target_ds`, `flag` | Variables passed into a function | +| **Array Keys (Const)** | "CAPS" | `Opt["VERBOSE"]` | Settings or fixed keys | +| **Array Keys (Local)** | "lowercase" | `Dataset[ds_suffix, "match"]` | Keys applied during runtime | --- -## 2. Core Concepts & Vocabulary +## 2. Core Vocabulary -Though we follow OpenZFS's language concepts when possible, some terms aren't clearly defined in the OpenZFS project documentation. +We follow OpenZFS language where possible, with these clarifications: -* **endpoint (ep):** The location and name of a ZFS object. +* **endpoint (ep):** Location and name of a ZFS object. * **dataset (ds):** A specific individual ZFS dataset. * **tree:** A dataset and its recursive children. -* **dataset tree:** The preferred term when describing recursive operations on a dataset and all its descendants. +* **dataset tree:** Preferred term for recursive operations on a dataset and all descendants. * **ds_snap:** A specific snapshot instance (e.g., `pool/data@snap1`). -* **ds_suffix:** The relative path of a child element within a tree with a leading `/` (formerly referred to as `rel_name`). - * *Example:* If root is `zroot/usr`, and we process `zroot/usr/local`, the `ds_suffix` is `/local`. +* **ds_suffix:** Relative path of a child within a tree, with leading `/`. + * *Example:* If root is `zroot/usr` and we process `zroot/usr/local`, the `ds_suffix` is `/local`. --- @@ -40,39 +40,36 @@ Though we follow OpenZFS's language concepts when possible, some terms aren't cl Global state is managed via multidimensional associative arrays in AWK. ### `Opt[]` -**User Settings.** -Defined definitions in `zelta` sh script and parsing rules in `zelta-opts.tsv`. +**User Settings.** Defined in `zelta` shell script; parsing rules in `zelta-opts.tsv`. * Index: `Opt["VARIABLE_NAME"]` ### `Dataset[]` -**Properties of each dataset.** -Indexed by Endpoint, the Suffix, and the Property Name. +**Properties of each dataset.** Indexed by endpoint, suffix, and property name. * **Index:** `(endpoint, ds_suffix, property)` * **Standard Properties:** - * `"exists"`: (Boolean) Does it exist? - * `"earliest_snapshot"`: The oldest snapshot on the system. - * `"latest_snapshot"`: The newest snapshot. - * `[zfs_property]`: Any native ZFS property (e.g., `"compression"`, `"origin"`), sourced via `zelta-match`. + * `"exists"`: Boolean + * `"earliest_snapshot"`: Oldest snapshot + * `"latest_snapshot"`: Newest snapshot + * `[zfs_property]`: Any native ZFS property (e.g., `"compression"`, `"origin"`) ### `DSPair[]` -**Derived Replication State.** -Compares a dataset and its replica counterpart. +**Derived Replication State.** Compares a dataset with its replica. * **Index:** `(ds_suffix, property)` * **Standard Properties:** - * `"match"`: The common snapshot or bookmark shared between Source and Target. - * `"source_start"`: The incremental source snapshot/bookmark used as the send basis. - * `"source_end"`: The target snapshot intended to be synced. + * `"match"`: Common snapshot or bookmark between source and target + * `"source_start"`: Incremental source snapshot/bookmark for send basis + * `"source_end"`: Target snapshot to sync ### `DSTree[]` **Global Tree Metadata.** * **Index:** `(property)` or `(endpoint, property)` * **Standard Properties:** - * `"SRC", "count"`: Number of datasets on source. - * `"TGT", "count"`: Number of datasets on target. + * `"SRC", "count"`: Number of datasets on source + * `"TGT", "count"`: Number of datasets on target ### Global Scalars -* `NumDS`: Integer count of datasets in the current tree. -* `DSList`: An ordered list (often space-separated or indexed array) of `ds_suffix` elements in replication order. +* `NumDS`: Dataset count in current tree +* `DSList`: Ordered list of `ds_suffix` elements in replication order --- @@ -82,67 +79,63 @@ Compares a dataset and its replica counterpart. * **Shebang:** `#!/bin/sh` * **No Bashisms:** No arrays (`arr=(...)`), no `[[ ]]`, no `function name() {`. Use `name() {`. * **Variables:** Quote all variables unless tokenization is explicitly desired. -* **Indentation:** Use **Tabs** for block indentation. Use **Spaces** for inline alignment of comments or assignments. +* **Indentation:** Tabs for blocks, spaces for inline alignment. ### AWK (`awk`) -* **Dialect:** Original-Awk (bwk/nawk) styled, code must run on stock FreeBSD, Illumos, and Debian `awk`. -* **Indentation:** Same as Shell (Tabs for structure, Spaces for alignment). +* **Dialect:** Original-Awk (bwk/nawk) style; must run on stock FreeBSD, Illumos, and Debian `awk`. +* **Indentation:** Same as shell. ### Portability * Assume the environment is hostile. -* Assume `grep` logic varies. -* Do not assume GNU or BSD extensions are present. +* Assume `grep` behavior varies. +* Do not assume GNU or BSD extensions. --- -## 5. Comments & Documentation +## 5. Comments -Good comments explain **why** and **what**, not just **how**. They should help future maintainers understand the intent and context. +Good comments explain **why** and **what**, not just **how**. ### Comment Styles | Style | Usage | Example | | :--- | :--- | :--- | -| `##` | **Section Headers** | `## Loading and setting properties` | -| `#` | **Function Headers** | `# Evaluate properties needed for snapshot decision` | -| `#` | **Inline Explanations** | `_idx = endpoint SUBSEP ds_suffix # Build array key` | -| `# TO-DO:` | **Future Work** | `# TO-DO: This should be its own function` | +| `##` | Section headers | `## Loading and setting properties` | +| `#` | Function headers, inline explanations | `# Build array key` | +| `# TO-DO:` | Future work | `# TO-DO: Extract to function` | ### What to Comment -* **Complex Array Indexing:** Explain multi-dimensional array structures. +* **Complex Array Indexing:** ```awk # Dataset properties indexed by: (endpoint, ds_suffix, property) Dataset[_idx, "latest_snapshot"] = snap_name ``` -* **Business Logic:** Explain the reasoning behind complex conditionals. +* **Business Logic:** ```awk - # If this is the first snapshot for the source, update the snap counter + # If first snapshot for source, update snap counter if (!_src_latest) Dataset[_idx, "earliest_snapshot"] = snap_name ``` -* **State Changes:** Document when and why global state is modified. +* **State Changes:** ```awk - # The target is updated via a sync and becomes our new match + # Target updated via sync; becomes new match DSPair[ds_suffix, "match"] = snap_name ``` -* **External Dependencies:** Explain interactions with external commands. +* **External Dependencies:** ```awk # Run 'zfs match' and pass to parser _cmd = build_command("MATCH", _cmd_arr) ``` ### What NOT to Comment - -* **Obvious Operations:** Don't comment simple assignments or standard patterns. -* **Redundant Descriptions:** Avoid comments that just restate the code. +* Obvious operations +* Redundant descriptions that restate the code ### Section Organization - -Use `##` headers to create logical code sections: ```awk ## Usage ######## @@ -150,39 +143,32 @@ Use `##` headers to create logical code sections: ## Loading and setting properties ################################# -## Compute derived data from properties and snapshots -##################################################### +## Compute derived data +####################### ``` --- ## 6. Documentation Standards -Zelta documentation serves different audiences with different needs. Maintain appropriate tone and style for each context. - ### Documentation Hierarchy | Document Type | Audience | Tone | Purpose | | :--- | :--- | :--- | :--- | -| **Man Pages** | Sysadmins at 4am | Strictly technical, no personality | Complete reference, troubleshooting | -| **README.md** | Evaluators, new users | Professional but approachable | "What" and "why", getting started | -| **Wiki Pages** | Community, learners | Conversational, tutorial-focused | How-to guides, examples, discussion | -| **Code Comments** | Developers, maintainers | Technical, explanatory | Intent and context | +| **Man Pages** | Sysadmins at 4am | Strictly technical | Complete reference | +| **README.md** | Evaluators, new users | Professional, approachable | "What" and "why" | +| **Wiki Pages** | Community, learners | Conversational | How-to guides | +| **Code Comments** | Developers | Technical | Intent and context | ### Man Page Standards -Man pages are reference documentation. They must be: - -* **Complete:** Cover all options, arguments, and behaviors -* **Precise:** Use exact terminology consistently -* **Scannable:** Use clear headers, tables, and formatting -* **Example-driven:** Show common use cases with realistic examples +Man pages must be **complete**, **precise**, **scannable**, and **example-driven**. -**Man Page Structure:** +**Structure:** ``` NAME - Brief description SYNOPSIS - Command syntax -DESCRIPTION - What it does and how +DESCRIPTION - What it does OPTIONS - All flags and arguments EXAMPLES - Common use cases EXIT STATUS - Return codes @@ -192,99 +178,45 @@ AUTHORS - Credit WWW - Project URL ``` -**Man Page Formatting:** -* Use `**bold**` for commands, options, and user input -* Use `*italic*` for arguments and placeholders -* Use `:` for definition lists +**Formatting:** +* `**bold**` for commands, options, user input +* `*italic*` for arguments and placeholders * Escape dashes in options: `**\--verbose**` -* Use tables for complex option lists ### README.md Standards -The README serves as the project's front door. It should be: +The README is the project's front door: **welcoming**, **focused**, **honest**, **actionable**. -* **Welcoming:** Professional tone, avoid jargon where possible -* **Focused:** Lead with value proposition and quickstart -* **Honest:** State limitations and beta status clearly -* **Actionable:** Clear next steps for different user types +**Avoid:** Marketing hyperbole, antagonistic comparisons, excessive casualness, unsubstantiated claims. -**Avoid in README.md:** -* Marketing hyperbole ("revolutionary", "game-changing") -* Antagonistic comparisons ("unlike X which is terrible") -* Excessive casualness ("no bull", "ridiculously simple") -* Unsubstantiated claims +**Prefer:** Concrete examples, factual statements, direct language, specific technical advantages. -**Prefer in README.md:** -* Concrete examples with realistic scenarios -* Factual statements about capabilities -* Clear, direct language ("simple", "straightforward") -* Specific technical advantages - -### Terminology Consistency - -Use these terms consistently across all documentation: +### Terminology | Preferred | Avoid | Context | | :--- | :--- | :--- | -| dataset tree | recursive datasets | When describing parent + children | -| endpoint | location, target system | For `user@host:pool/dataset` | -| backup | replication, sync | User-facing docs; matches command name and ZFS "backup stream" | -| replicate | sync, copy | Technical docs when describing ZFS `--replicate` behavior specifically | +| dataset tree | recursive datasets | Parent + children | +| endpoint | location, target system | `user@host:pool/dataset` | +| backup | replication, sync | User-facing docs | +| replicate | sync, copy | Technical docs for ZFS `-R` behavior | | snapshot | snap | Except in code/options | -**Note on "backup" vs "replicate":** User documentation prefers "backup" because it describes intent and matches the command name. Reserve "replicate" for contexts where the ZFS `--replicate` (`-R`) flag behavior is specifically relevant, as this term has a precise technical meaning in ZFS that differs from common English usage. - -### Example Formatting - -**Endpoint Examples:** -Always use realistic, complete examples: -* Good: `user@backup.example.com:tank/backups/dataset` -* Avoid: `host:pool/ds`, `remote:tank/backup` - -**Command Examples:** -Show complete, working commands: -```sh -# Good: Complete with context -zelta backup rpool/data backup@storage.example.com:tank/backups/data - -# Avoid: Incomplete or unclear -zelta backup source target -``` - -### Writing Style Guidelines +### Writing Style **Do:** -* Use active voice ("Zelta creates snapshots" not "Snapshots are created") +* Use active voice * Start with the most common use case -* Explain *why* before *how* when introducing concepts +* Explain *why* before *how* * Use parallel structure in lists -* Define acronyms on first use **Don't:** -* Use exclamation points in technical documentation +* Use exclamation points in technical docs * Make unverifiable claims -* Use marketing language in man pages -* Assume prior knowledge of ZFS internals -* Mix casual and formal tone in the same document - -### Cross-Reference Standards - -When referencing other documentation: - -* Man pages: Use standard notation `**command(section)**` (e.g., `**zfs(8)**`) -* Internal docs: Use relative links in markdown -* External docs: Use full URLs with descriptive text -* Be specific: "See the EXCLUSION PATTERNS section in **zelta-options(7)**" not "See zelta-options" - -### Consistency Checklist +* Mix casual and formal tone -Before committing documentation changes, verify: +### Cross-References -- [ ] Terminology matches STYLE.md vocabulary -- [ ] Tone appropriate for document type -- [ ] Examples use realistic hostnames (*.example.com) -- [ ] Command formatting consistent (escaped dashes in man pages) -- [ ] Cross-references use correct notation -- [ ] No marketing language in technical docs -- [ ] Tables formatted consistently -- [ ] Code blocks use appropriate syntax highlighting +* Man pages: `**command(section)**` (e.g., `**zfs(8)**`) +* Internal docs: Relative links +* External docs: Full URLs +* Be specific: "See EXCLUSION PATTERNS in **zelta-options(7)**" From f94000de499d7ad05046f2fab0c40b84abc1e32c Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Sat, 17 Jan 2026 20:01:23 -0500 Subject: [PATCH 12/47] test/sanity checks experiments test-what-you-can shellspec experiment fix basic args example test --- .shellspec | 20 ++----------- bin/zelta | 2 +- test/sanity_spec.sh | 40 +++++++++++++++++++++++++ test/test_helper.sh | 71 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 test/sanity_spec.sh create mode 100644 test/test_helper.sh diff --git a/.shellspec b/.shellspec index d0d9e43..2ca0a5b 100644 --- a/.shellspec +++ b/.shellspec @@ -1,17 +1,3 @@ ---require spec_helper - -# Set test environment variables -#--env-from spec/initialize/test_env.sh - -## Default kcov (coverage) options -# --kcov-options "--include-path=. --path-strip-level=1" -# --kcov-options "--include-pattern=.sh" -# --kcov-options "--exclude-pattern=/.shellspec,/spec/,/coverage/,/report/" - -## Example: Include script "myprog" with no extension -# --kcov-options "--include-pattern=.sh,myprog" - -## Example: Only specified files/directories -# --kcov-options "--include-pattern=myprog,/lib/" - - +--helperdir test +--require test_helper +--default-path test diff --git a/bin/zelta b/bin/zelta index a297f48..f2f0d8a 100755 --- a/bin/zelta +++ b/bin/zelta @@ -4,7 +4,7 @@ # # Initialize the environment for Zelta subcommands -ZELTA_VERSION="Zelta Replication Suite v1.1-rc1" +ZELTA_VERSION="Zelta 1.1-rc1" zelta_usage() { exec >&2 diff --git a/test/sanity_spec.sh b/test/sanity_spec.sh new file mode 100644 index 0000000..a009afd --- /dev/null +++ b/test/sanity_spec.sh @@ -0,0 +1,40 @@ +# shellcheck shell=sh + +Describe 'Zelta sanity checks' + Describe 'zelta command' + It 'is executable' + When run command command -v zelta + The status should be success + The output should include 'zelta' + End + It 'shows help with no arguments' + When run command zelta + The status should be failure + The error should include 'usage' + End + + It 'shows version' + When run command zelta version + The status should be success + The output should include 'Zelta' + End + End + Describe 'zelta match' + It 'shows zfs list commands for one operand' + When run command zelta match --dryrun "$ZELTA_TEST_SRC_DS" + The status should be success + The output should include '+ zfs list' + End + It 'shows zfs list commands for two operands' + When run command zelta match --dryrun zelta-nonexistent-pool/nonexistent + The status should be success + The output should include '+ zfs list' + End + + It 'respects depth parameter' + When run command zelta match --dryrun --depth 69 zelta-nonexistent-pool/nonexistent + The status should be success + The output should include '69' + End + End +End diff --git a/test/test_helper.sh b/test/test_helper.sh new file mode 100644 index 0000000..1c50096 --- /dev/null +++ b/test/test_helper.sh @@ -0,0 +1,71 @@ +# shellcheck shell=sh + +# Zelta Test Helper +# +# Environment variables: +# ZELTA_TEST_SRC_POOL - Source pool (default: zelta-source) +# ZELTA_TEST_TGT_POOL - Target pool (default: zelta-target) +# ZELTA_TEST_SRC_HOST - Source host for remote tests (optional) +# ZELTA_TEST_TGT_HOST - Target host for remote tests (optional) +# +# If pools/hosts are not configured, only basic sanity tests run. + +## Test namespace defaults (designed to never collide) +: "${ZELTA_TEST_SRC_POOL:=zelta-source}" +: "${ZELTA_TEST_TGT_POOL:=zelta-target}" +: "${ZELTA_TEST_SRC_DS:=zelta-source/test-data}" +: "${ZELTA_TEST_TGT_DS:=zelta-target/test-backups}" + +## Use repo's zelta by default +REPO_ROOT="$SHELLSPEC_PROJECT_ROOT" +export PATH="$REPO_ROOT/bin:$PATH" +export ZELTA_SHARE="$REPO_ROOT/share/zelta" + +## Build endpoints +if [ -n "$ZELTA_TEST_SRC_HOST" ]; then + ZELTA_TEST_SRC_EP="${ZELTA_TEST_SRC_HOST}:${ZELTA_TEST_SRC_DS}" +else + ZELTA_TEST_SRC_EP="$ZELTA_TEST_SRC_DS" +fi + +if [ -n "$ZELTA_TEST_TGT_HOST" ]; then + ZELTA_TEST_TGT_EP="${ZELTA_TEST_TGT_HOST}:${ZELTA_TEST_TGT_DS}" +else + ZELTA_TEST_TGT_EP="$ZELTA_TEST_TGT_DS" +fi + +export ZELTA_TEST_SRC_POOL ZELTA_TEST_TGT_POOL +export ZELTA_TEST_SRC_DS ZELTA_TEST_TGT_DS +export ZELTA_TEST_SRC_EP ZELTA_TEST_TGT_EP + +## Helper: check if test pools exist +zelta_test_pools_exist() { + zfs list -H -o name "$ZELTA_TEST_SRC_POOL" >/dev/null 2>&1 && + zfs list -H -o name "$ZELTA_TEST_TGT_POOL" >/dev/null 2>&1 +} + +## Helper: check if remote hosts are reachable +zelta_test_remote_available() { + [ -n "$ZELTA_TEST_SRC_HOST" ] && [ -n "$ZELTA_TEST_TGT_HOST" ] +} + +spec_helper_precheck() { + : minimum_version "0.28.1" + info "Zelta test environment:" + info " ZELTA_SHARE: $ZELTA_SHARE" + info " Source: $ZELTA_TEST_SRC_EP" + info " Target: $ZELTA_TEST_TGT_EP" + if zelta_test_remote_available; then + info " Remote tests: enabled" + else + info " Remote tests: disabled (set ZELTA_TEST_SRC_HOST and ZELTA_TEST_TGT_HOST)" + fi +} + +spec_helper_loaded() { + : +} + +spec_helper_configure() { + : import 'support/custom_matcher' +} From 5e431967302f4c9d3e42b02e1774009cb656c8d2 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Mon, 19 Jan 2026 12:41:19 -0500 Subject: [PATCH 13/47] shellspec experiments --- share/zelta/zelta-cmds.tsv | 2 +- test/01_no_op_spec.sh | 48 ++++++++++++++++++++++++ test/02_pool_spec.sh | 3 ++ test/03_no_op_ds_spec.sh | 3 ++ test/sanity_spec.sh | 40 -------------------- test/test_helper.sh | 76 +++++++++++++++----------------------- 6 files changed, 85 insertions(+), 87 deletions(-) create mode 100644 test/01_no_op_spec.sh create mode 100644 test/02_pool_spec.sh create mode 100644 test/03_no_op_ds_spec.sh delete mode 100644 test/sanity_spec.sh diff --git a/share/zelta/zelta-cmds.tsv b/share/zelta/zelta-cmds.tsv index fdc500f..3f69e4d 100644 --- a/share/zelta/zelta-cmds.tsv +++ b/share/zelta/zelta-cmds.tsv @@ -8,7 +8,7 @@ # VARS: List of options provided from a referenced associative array. # CHECK DEFAULT zfs list -Ho name ds -LIST DEFAULT zfs list -Hpr -t all -Screatetxg -o props flags ds +LIST DEFAULT zfs list -Hpr -t all -Screatetxg -o props flags ds PROPS DEFAULT zfs get -Hpr -s local,none -t filesystem,volume -o name,property,value all flags ds CREATE DEFAULT zfs create -vupo canmount=noauto ds SNAP DEFAULT zfs snapshot -r flags ds_snap diff --git a/test/01_no_op_spec.sh b/test/01_no_op_spec.sh new file mode 100644 index 0000000..7c660a5 --- /dev/null +++ b/test/01_no_op_spec.sh @@ -0,0 +1,48 @@ +# shellcheck shell=sh + +Describe 'Zelta no-op command checks' + Describe 'zelta command' + It 'is executable' + When run command command -v zelta + The status should be success + The output should include 'zelta' + End + It 'shows usage with no arguments' + When run command zelta + The status should be failure + The error should include 'usage' + End + It 'shows version' + When run command zelta version + The status should be success + The output should include 'Zelta' + End + It 'shows help' + When run command zelta help + The status should be success + The output should include 'zelta(8)' + End + End + Describe 'zelta match' + It 'shows zfs list commands for one operand' + When run command zelta match --dryrun zelta-test-pool/test-source + The status should be success + The output should include '+ zfs list' + End + It 'shows zfs list commands for two operands' + When run command zelta match --dryrun zelta-test-pool/test-source zelta-test-pool/test-target + The status should be success + The output should include '+ zfs list' + End + It 'respects single-dash parameters' + When run command zelta match -Hpvqn -X '*/swap' -d42 -o ds_suffix zelta-test-pool/test + The status should be success + The output should include '42' + End + It 'respects all parameters' + When run command zelta match -Hpvqn -X '*/swap' -d42 -o ds_suffix --verbose --quiet --log-level 2 --log-mode=text --text --dryrun --depth 42 --exclude="@one,/two" zelta-test-pool/test zelta-test-pool/test-target + The status should be success + The output should include '42' + End + End +End diff --git a/test/02_pool_spec.sh b/test/02_pool_spec.sh new file mode 100644 index 0000000..5de4572 --- /dev/null +++ b/test/02_pool_spec.sh @@ -0,0 +1,3 @@ +Describe 'Pool setup' + Skip if ! pools_defined +End diff --git a/test/03_no_op_ds_spec.sh b/test/03_no_op_ds_spec.sh new file mode 100644 index 0000000..91bb809 --- /dev/null +++ b/test/03_no_op_ds_spec.sh @@ -0,0 +1,3 @@ +Describe 'Dataset' + Skip if ! pools_defined +End diff --git a/test/sanity_spec.sh b/test/sanity_spec.sh deleted file mode 100644 index a009afd..0000000 --- a/test/sanity_spec.sh +++ /dev/null @@ -1,40 +0,0 @@ -# shellcheck shell=sh - -Describe 'Zelta sanity checks' - Describe 'zelta command' - It 'is executable' - When run command command -v zelta - The status should be success - The output should include 'zelta' - End - It 'shows help with no arguments' - When run command zelta - The status should be failure - The error should include 'usage' - End - - It 'shows version' - When run command zelta version - The status should be success - The output should include 'Zelta' - End - End - Describe 'zelta match' - It 'shows zfs list commands for one operand' - When run command zelta match --dryrun "$ZELTA_TEST_SRC_DS" - The status should be success - The output should include '+ zfs list' - End - It 'shows zfs list commands for two operands' - When run command zelta match --dryrun zelta-nonexistent-pool/nonexistent - The status should be success - The output should include '+ zfs list' - End - - It 'respects depth parameter' - When run command zelta match --dryrun --depth 69 zelta-nonexistent-pool/nonexistent - The status should be success - The output should include '69' - End - End -End diff --git a/test/test_helper.sh b/test/test_helper.sh index 1c50096..35cd1ff 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -3,69 +3,53 @@ # Zelta Test Helper # # Environment variables: -# ZELTA_TEST_SRC_POOL - Source pool (default: zelta-source) -# ZELTA_TEST_TGT_POOL - Target pool (default: zelta-target) -# ZELTA_TEST_SRC_HOST - Source host for remote tests (optional) -# ZELTA_TEST_TGT_HOST - Target host for remote tests (optional) +# SANDBOX_ZELTA_SRC_POOL - Source pool +# SANDBOX_ZELTA_TGT_POOL - Target pool +# SANDBOX_ZELTA_SRC_REMOTE - Source [user@]host for remote tests +# SANDBOX_ZELTA_TGT_REMOTE - Target [user@]host for remote tests +# SANDBOX_ZELTA_SRC_DS - Source dataset +# SANDBOX_ZELTA_TGT_DS - Target dataset # # If pools/hosts are not configured, only basic sanity tests run. -## Test namespace defaults (designed to never collide) -: "${ZELTA_TEST_SRC_POOL:=zelta-source}" -: "${ZELTA_TEST_TGT_POOL:=zelta-target}" -: "${ZELTA_TEST_SRC_DS:=zelta-source/test-data}" -: "${ZELTA_TEST_TGT_DS:=zelta-target/test-backups}" - -## Use repo's zelta by default +## Use repo's zelta REPO_ROOT="$SHELLSPEC_PROJECT_ROOT" export PATH="$REPO_ROOT/bin:$PATH" export ZELTA_SHARE="$REPO_ROOT/share/zelta" ## Build endpoints -if [ -n "$ZELTA_TEST_SRC_HOST" ]; then - ZELTA_TEST_SRC_EP="${ZELTA_TEST_SRC_HOST}:${ZELTA_TEST_SRC_DS}" +if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then + SANDBOX_ZELTA_SRC_EP="${SANDBOX_ZELTA_SRC_REMOTE}:${SANDBOX_ZELTA_SRC_DS}" + SANDBOX_ZELTA_SRC_CMD="ssh ${SANDBOX_ZELTA_SRC_REMOTE} " else - ZELTA_TEST_SRC_EP="$ZELTA_TEST_SRC_DS" + SANDBOX_ZELTA_SRC_EP="$SANDBOX_ZELTA_SRC_DS" fi -if [ -n "$ZELTA_TEST_TGT_HOST" ]; then - ZELTA_TEST_TGT_EP="${ZELTA_TEST_TGT_HOST}:${ZELTA_TEST_TGT_DS}" +if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then + SANDBOX_ZELTA_TGT_EP="${SANDBOX_ZELTA_TGT_REMOTE}:${SANDBOX_ZELTA_TGT_DS}" + SANDBOX_ZELTA_SRC_CMD="ssh ${SANDBOX_ZELTA_SRC_REMOTE} " else - ZELTA_TEST_TGT_EP="$ZELTA_TEST_TGT_DS" + SANDBOX_ZELTA_TGT_EP="$SANDBOX_ZELTA_TGT_DS" fi -export ZELTA_TEST_SRC_POOL ZELTA_TEST_TGT_POOL -export ZELTA_TEST_SRC_DS ZELTA_TEST_TGT_DS -export ZELTA_TEST_SRC_EP ZELTA_TEST_TGT_EP +# We should determine if sudo is actually needed +SANDBOX_ZELTA_SRC_CMD="${SANDBOX_ZELTA_SRC_CMD}sudo " +SANDBOX_ZELTA_TGT_CMD="${SANDBOX_ZELTA_TGT_CMD}sudo " -## Helper: check if test pools exist -zelta_test_pools_exist() { - zfs list -H -o name "$ZELTA_TEST_SRC_POOL" >/dev/null 2>&1 && - zfs list -H -o name "$ZELTA_TEST_TGT_POOL" >/dev/null 2>&1 -} +# Consider using unique tests +# SANDBOX_ZELTA_SRC_DS="${SANDBOX_ZELTA_SRC_DS}/zelta_test_src_$$" +# SANDBOX_ZELTA_TGT_DS="${SANDBOX_ZELTA_TGT_DS}/zelta_test_tgt_$$" -## Helper: check if remote hosts are reachable -zelta_test_remote_available() { - [ -n "$ZELTA_TEST_SRC_HOST" ] && [ -n "$ZELTA_TEST_TGT_HOST" ] -} +export SANDBOX_ZELTA_SRC_POOL SANDBOX_ZELTA_TGT_POOL +export SANDBOX_ZELTA_SRC_DS SANDBOX_ZELTA_TGT_DS +export SANDBOX_ZELTA_SRC_EP SANDBOX_ZELTA_TGT_EP +export SANDBOX_ZELTA_SRC_CMD SANDBOX_ZELTA_TGT_CMD -spec_helper_precheck() { - : minimum_version "0.28.1" - info "Zelta test environment:" - info " ZELTA_SHARE: $ZELTA_SHARE" - info " Source: $ZELTA_TEST_SRC_EP" - info " Target: $ZELTA_TEST_TGT_EP" - if zelta_test_remote_available; then - info " Remote tests: enabled" - else - info " Remote tests: disabled (set ZELTA_TEST_SRC_HOST and ZELTA_TEST_TGT_HOST)" - fi -} -spec_helper_loaded() { - : -} +## Helpers +########## -spec_helper_configure() { - : import 'support/custom_matcher' +pools_defined() { + [ -z "$SANDBOX_ZELTA_SRC_POOL$SANDBOX_ZELTA_TGT_POOL" ] + } From 6b4d5f5da5c398288b1e99d18f1528869c08ed2d Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Mon, 19 Jan 2026 17:59:51 -0500 Subject: [PATCH 14/47] docs cleanup manppages update and dropped unnecessary -L for --raw sends docs: standardize SYNOPSIS sections in man pages dropped unnecessary -L for --raw sends docs: add missing EXIT STATUS and NOTES sections to man pages tweak prune to use 'tee' instead of a blind redirect docs: standardize code blocks, fix typos, and update references move freebsd ports template increment to Zelta version 1.1 docs: add zelta-prune(8) to new man pages list --- CHANGELOG.md | 50 +++---- README.md | 2 +- bin/zelta | 4 +- doc/zelta-backup.8 | 8 +- doc/zelta-backup.md | 38 +++-- doc/zelta-clone.8 | 17 ++- doc/zelta-clone.md | 84 ++++++----- doc/zelta-match.8 | 27 ++-- doc/zelta-match.md | 34 ++--- doc/zelta-policy.8 | 10 +- doc/zelta-policy.md | 14 +- doc/zelta-prune.8 | 157 +++++++++++++++++++++ doc/zelta-prune.md | 16 ++- doc/zelta-revert.8 | 2 +- doc/zelta-revert.md | 4 + doc/zelta-rotate.8 | 2 +- doc/zelta-rotate.md | 4 + doc/zelta-snapshot.8 | 110 +++++++++++++++ doc/zelta.8 | 4 +- doc/zelta.md | 9 +- {port-files => packaging/freebsd}/Makefile | 14 +- packaging/freebsd/distinfo | 3 + packaging/freebsd/files/pkg-message.in | 21 +++ packaging/freebsd/pkg-descr | 8 ++ packaging/freebsd/pkg-plist | 22 +++ port-files/distinfo | 3 - port-files/files/pkg-message.in | 17 --- port-files/pkg-descr | 1 - port-files/pkg-plist | 15 -- 29 files changed, 512 insertions(+), 188 deletions(-) create mode 100644 doc/zelta-prune.8 create mode 100644 doc/zelta-snapshot.8 rename {port-files => packaging/freebsd}/Makefile (71%) create mode 100644 packaging/freebsd/distinfo create mode 100644 packaging/freebsd/files/pkg-message.in create mode 100644 packaging/freebsd/pkg-descr create mode 100644 packaging/freebsd/pkg-plist delete mode 100644 port-files/distinfo delete mode 100644 port-files/files/pkg-message.in delete mode 100644 port-files/pkg-descr delete mode 100644 port-files/pkg-plist diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0d248..fa0e7bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,49 +1,51 @@ # Changelog -All notable changes to the Zelta will be documented in this file. +All notable changes to Zelta will be documented in this file. -## [1.1.rc1] - 2026-01-15 -This section will be modified until v1.1 is officially released. +## [1.1.0] - 2026-01-20 ### Added -- **Commands**: `zelta revert` for in-place rollbacks via rename/clone. -- **Commands**: `zelta rotate` for divergent version handling, improved from original `--rotate`. -- **Commands**: `zelta prune` identifies snapshots in `zfs destroy` range syntax based on replication state and a sliding window for exclusions. +- **Commands**: `zelta revert` for in-place rollbacks via rename and clone. +- **Commands**: `zelta rotate` for divergent version handling, evolved from original `--rotate` flag. +- **Commands**: (Experimental) `zelta prune` identifies snapshots in `zfs destroy` range syntax based on replication state and a sliding window for exclusions. - **Uninstaller**: Added `uninstall.sh` for clean removal of Zelta installations, including legacy paths from earlier betas. -- **Core**: `zelta-args.awk` added as a separate argument preprocessor. -- **Core**: `zelta-common.awk` library for centralized string/logging functions. -- **Config**: Data-driven TSV configuration (`zelta-opts.tsv`, `zelta-cmds.tsv`). -- **`zelta.env` documentation**: Expanded environment file with comprehensive inline documentation and examples for all major configuration categories. -- **Docs**: New `zelta-options(7)`, zelta-revert(8), zelta-rollback(8) manpages. -- **Docs**: Added tool to sync manpages with the zelta.space wiki. +- **Core**: `zelta-args.awk` added as a separate data-driven argument preprocessor. +- **Core**: `zelta-common.awk` library for centralized string and logging functions. +- **Config**: Data-driven TSV configuration (`zelta-cmds.tsv`, `zelta-cols.tsv`, `zelta-json.tsv`, `zelta-opts.tsv`). +- **Docs**: `zelta.env` expanded with comprehensive inline documentation and examples for all major configuration categories. +- **Docs**: New man pages: `zelta-options(7)`, `zelta-revert(8)`, `zelta-rotate(8)`, `zelta-prune(8)`. +- **Docs**: Added tool to sync man pages with the zelta.space wiki. ### Changed -- **Architecture**: Refactor of all core scripts for maintainability and simpler logic. -- **Core**: `bin/zelta` controller improved with centralized logging and better option handling. +- **Architecture**: Refactored all core scripts for maintainability and simpler logic. +- **Core**: Improved `bin/zelta` controller with centralized logging and better option handling. - **Core**: More centralized error handling. - **Backup**: Rewritten `zelta backup` engine with improved state tracking and resume support. - **Backup**: Core script renamed from `zelta-replicate.awk` to `zelta-backup.awk`. -- **Backup**: Granular options overrides, `zfs recv -o/-x`. +- **Backup**: Added granular option overrides via `zfs recv -o` and `-x`. - **Match**: `zelta match` now calls itself rather than a redundant script. -- **Match**: Output columns are now data driven with a simpler and clearer 'info' column. -- **Match**: Added exclusion patterns. +- **Match**: Output columns are now data-driven with a simpler and clearer 'info' column. +- **Match**: Added exclusion patterns (`-X`, `--exclude`). +- **Policy**: Improved hierarchical scoping and refactored internal job handling with clearer variable naming and function documentation. - **Rotate**: Better handling of naming. -- **Policy**: Better hierarchical scoping. -- **Policy**: Refactored internal job handling with clearer variable naming and improved function documentation. +- **Snapshot**: Operates independently and works with Zelta arguments or an OpenZFS operand. - **Orchestration**: Zelta is no longer required to be installed on endpoints. -- **Logging**: Better alerts, deprecation system, legacy option system, and warning messages. +- **Logging**: Better alerts, deprecation system, legacy option handling, and warning messages. +- **Experimental**: Refactored `zelta report` to use newer ZFS features and multiple endpoints. ### Fixed -- Option regressions including legacy overrides, and backup depth -- Better handling of dataset names with spaces/special characters. +- Option regressions including legacy overrides and backup depth. +- Better handling of dataset names with spaces and special characters. - Dataset type detection with environment variables for each (TOP, NEW, FS, VOL, RAW, etc.). - Improved option hierarchy for `zelta policy`. -- Fixed namespace configuration repeated targets in `zelta policy`. +- Fixed namespace configuration and repeated targets in `zelta policy`. +- Workaround for GNU Awk 5.2.1 bug. +- Resume token handling and other context-aware ZFS option handling. ### Deprecated - `zelta endpoint` and other functions have been merged into the core library. - Dropped unneeded interprocess communication features such as `sync_code` and `-z`. -- Removed "initiator" context which has been replaced by a simple `-pull` (default) and `--push` mechanic. +- Removed "initiator" context, replaced by simple `--pull` (default) and `--push` mechanic. - Progress pipes (`RECEIVE_PREFIX`) now only work if the local host is involved in replication. ## [1.0.0] - 2024-03-31 diff --git a/README.md b/README.md index 3b69844..e30bb51 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![Zelta Logo](https://zelta.space/index/zelta-banner.svg) # The Zelta Backup and Recovery Suite -*Version v1.1-rc1, 2026-01-15* +*Version v1.1, 2026-01-20* --- diff --git a/bin/zelta b/bin/zelta index f2f0d8a..1c4fca8 100755 --- a/bin/zelta +++ b/bin/zelta @@ -159,10 +159,10 @@ zelta_init() { # `zfs send` defaults : ${ZELTA_SEND_DEFAULT:="-L -c -e"} - : ${ZELTA_SEND_RAW:="-L -w"} + : ${ZELTA_SEND_RAW:="--raw"} : ${ZELTA_SEND_NEW:="-p"} : ${ZELTA_SEND_INTR:="1"} - : ${ZELTA_SEND_REPLICATE:="-L -s -R -w"} + : ${ZELTA_SEND_REPLICATE:="--raw -s -R"} # `zfs recv` defaults : ${ZELTA_RECV_DEFAULT:=""} diff --git a/doc/zelta-backup.8 b/doc/zelta-backup.8 index 2e55cd4..99aebd7 100644 --- a/doc/zelta-backup.8 +++ b/doc/zelta-backup.8 @@ -117,7 +117,7 @@ errors. .TP \f[B]\-j, \-\-json\f[R] Output results in JSON format. -See \f[B]zelta\-options(8)\f[R] for details. +See \f[B]zelta\-options(7)\f[R] for details. .TP \f[B]\-n, \-\-dryrun, \-\-dry\-run\f[R] Display \f[CR]zfs\f[R] commands without executing them. @@ -311,7 +311,7 @@ flags are passed through directly: \f[B]zfs send:\f[R] \f[CR]\-b, \-\-backup, \-\-embed, \-\-holds, \-L, \-\-largeblock, \-\-proctitle, \-\-props, \-\-raw, \-\-skipmissing, \-V, \-w\f[R] .PP -\f[B]zfs recv:\f[R] \f[CR]\-F, \-M, \-u\f[R] +\f[B]zfs recv:\f[R] \f[CR]\-F, \-M, \-u, \-o, \-x\f[R] .PP \f[B]Ambiguous or unsupported flags:\f[R] \- Single\-dash options with multiple meanings are \f[B]not supported\f[R]: @@ -323,7 +323,7 @@ handlers (see above): \f[CR]\-I, \-i, \-R, \-X, \-n\f[R] .EX # Recompress at target with zstd\-5 # WARNING: This disables encrypted sends! -zelta backup \-L \-\-recv\-override \(dq\-o compression=zstd\-5\(dq source target +zelta backup \-L \-o compression=zstd\-5 source target .EE .PP \f[B]When in doubt, use the granular override options instead.\f[R] @@ -369,7 +369,7 @@ zelta backup \-d 2 sink/source tank/target .SH EXIT STATUS Returns 0 on success, non\-zero on error. .SH NOTES -See \f[B]zelta\-options(8)\f[R] for environment variables, +See \f[B]zelta\-options(7)\f[R] for environment variables, \f[CR]zelta.env\f[R] configuration, and \f[CR]zelta policy\f[R] integration. .PP diff --git a/doc/zelta-backup.md b/doc/zelta-backup.md index 3082943..86dc06c 100644 --- a/doc/zelta-backup.md +++ b/doc/zelta-backup.md @@ -138,9 +138,8 @@ Zelta automatically applies non-destructive and efficient `zfs send` and `zfs re **Most Common Use Case: Recompress Backups** Typically, users adjust Zelta options because they would like to aggressively compress data on their backup endpoints. This is best done with the `--send-default` and `--recv-default` flags, which will not prevent Zelta from sending encrypted backups in raw (encrypted) format: -``` -zelta backup --send-default -Le --recv-default '-o compression=zstd-5' source backup -``` + + zelta backup --send-default -Le --recv-default '-o compression=zstd-5' source backup All overrides can also be configured globally in `zelta.env` or per-job via `zelta.conf`. @@ -188,16 +187,15 @@ For precise control in a dataset tree with mixed types, override specific contex : Additional `zfs recv` options for volume datasets (default: `-o volmode=none`) **Examples:** -``` -# Allow target to recompress unencrypted data -zelta backup --send-default "-Le" source target -# Change volume mode on target -zelta backup --recv-vol "-o volmode=dev" source target + # Allow target to recompress unencrypted data + zelta backup --send-default "-Le" source target + + # Change volume mode on target + zelta backup --recv-vol "-o volmode=dev" source target -# Avoid mountpoint permission issues on some systems -zelta backup --recv-fs "-o mountpoint=none" source target -``` + # Avoid mountpoint permission issues on some systems + zelta backup --recv-fs "-o mountpoint=none" source target ### Dataset Tree Override Options @@ -210,10 +208,9 @@ These options **replace all context-specific defaults** for an entire backup job : Override all default `zfs recv` options **Example:** -``` -# Use minimal `zfs send` to send uncompressed (**and decrypted!**) streams and recompress aggressively on the target -zelta backup --send-override "-L" --recv-override "-o compression=zstd-5" source target -``` + + # Use minimal `zfs send` to send uncompressed (**and decrypted!**) streams and recompress aggressively on the target + zelta backup --send-override "-L" --recv-override "-o compression=zstd-5" source target ### Pass-Through Override Flags @@ -230,11 +227,10 @@ The following unambiguous `zfs send` and `zfs recv` flags are passed through dir - Options with Zelta-specific handlers (see above): `-I, -i, -R, -X, -n` **Example:** -``` -# Recompress at target with zstd-5 -# WARNING: This disables encrypted sends! -zelta backup -L -o compression=zstd-5 source target -``` + + # Recompress at target with zstd-5 + # WARNING: This disables encrypted sends! + zelta backup -L -o compression=zstd-5 source target **When in doubt, use the granular override options instead.** @@ -279,7 +275,7 @@ The `zelta sync` command is a convenience alias for `zelta backup -i` and may be # SEE ALSO -zelta(8), zelta-options(7), zelta-match(8), zelta-policy(8), zelta-clone(8), zelta-revert(8), zelta-rotate(8), ssh(1), zfs(8), zfs-send(8), zfs-receive(8) +zelta(8), zelta-options(7), zelta-match(8), zelta-policy(8), zelta-clone(8), zelta-revert(8), zelta-rotate(8), zelta-snapshot(8), ssh(1), zfs(8), zfs-send(8), zfs-receive(8) # AUTHORS diff --git a/doc/zelta-clone.8 b/doc/zelta-clone.8 index f9c9906..9078f00 100644 --- a/doc/zelta-clone.8 +++ b/doc/zelta-clone.8 @@ -67,30 +67,29 @@ See \f[CR]zelta.env.example\f[R] to adjust the default naming scheme. \f[B]\-d \f[BI]depth\f[B], \-\-depth \f[BI]depth\f[B]\f[R] Limits the depth of all Zelta operations. .SH EXAMPLES -\f[B]Clone a dataset tree:\f[R] +Clone a dataset tree: .IP .EX -zelta clone tank/vm/myos tank/temp/myos\-202404 +zelta clone tank/vm/myos tank/temp/myos\-202404 .EE .PP -\f[B]Recover a dataset tree, in place, to a previous snapshot\(cqs -state:\f[R] +Recover a dataset tree, in place, to a previous snapshot\(cqs state: .IP .EX zfs rename tank/vm/myos tank/Archives/myos\-202404 zelta clone tank/Archives/myos\-202404\(atgoodsnapshot tank/vm/myos .EE .PP -\f[B]Dry Run:\f[R] Display the \f[CR]zfs clone\f[R] commands without -executing them. +Dry Run: Display the \f[CR]zfs clone\f[R] commands without executing +them. .IP .EX zelta clone \-n tank/source/dataset tank/target/dataset .EE .SH SEE ALSO -zelta(8), zelta\-backup(8), zelta\-match(8), zelta\-options(8), -zelta\-policy(8), zelta\-revert(8), zelta\-rotate(8), zelta\-sync(8), -cron(8), ssh(1), zfs(8), zfs\-clone(8), zfs\-promote(8) +zelta(8), zelta\-backup(8), zelta\-match(8), zelta\-options(7), +zelta\-policy(8), zelta\-revert(8), zelta\-rotate(8), cron(8), ssh(1), +zfs(8), zfs\-clone(8), zfs\-promote(8) .SH AUTHORS Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> .SH WWW diff --git a/doc/zelta-clone.md b/doc/zelta-clone.md index 3f4f204..85c6bc6 100644 --- a/doc/zelta-clone.md +++ b/doc/zelta-clone.md @@ -2,81 +2,89 @@ # NAME -**zelta clone** - Perform a recursive clone operation. - +**zelta clone** - perform a recursive clone operation # SYNOPSIS -**zelta clone** [**-d** _depth_] _source/dataset[@snap]_ _target/dataset_ - +**zelta clone** [_OPTIONS_] _source_[@_snapshot_] _target_ # DESCRIPTION -`zelta clone` performs a recursive **zfs clone** operation on a dataset. This is useful for recursive duplication of dataset trees and backup inspection and recovery of a files replicated with `zelta backup`. The clones will reference the latest or indicated snapshot, and consume practically no additional space. Clones can be modified and destroyed without affecting their origin datasets. -The _source_ and _target_ must be on the same host and pool. The mountpoint will be inherited below the target parent (as provided by `zfs clone`). The _target_ dataset must not exist. To create a clone on a remote host ensure the _source_ and _target_ are identical including the username and hostname used: +**zelta clone** performs a recursive **zfs clone** operation on a dataset. This is useful for recursive duplication of dataset trees and backup inspection and recovery of files replicated with **zelta backup**. The clones will reference the latest or indicated snapshot, and consume practically no additional space. Clones can be modified and destroyed without affecting their origin datasets. + +The _source_ and _target_ must be on the same host and pool. The mountpoint will be inherited below the target parent (as provided by **zfs clone**). The _target_ dataset must not exist. To create a clone on a remote host, ensure the _source_ and _target_ are identical including the username and hostname used. + +Remote endpoint names follow **scp(1)** conventions. Dataset names follow **zfs(8)** naming conventions. Example remote operation: - zelta clone backup@host1.com:tank/zones/data host1.com:tank/clones/data + zelta clone backup@host1.com:tank/zones/data backup@host1.com:tank/clones/data # OPTIONS -**Required Options** +**Endpoint Arguments (Required)** _source_ -: A dataset, in the form **pool[/dataset][@snapshot]**, which will be cloned along with all of its descendents. If a snapshot is not given, the most recent snapshot will be used. +: A dataset, in the form **pool[/dataset][@snapshot]**, which will be cloned along with all of its descendants. If a snapshot is not given, the most recent snapshot will be used. _target_ -: A dataset on the same pool as the **source/dataset**, where the clones will be created. This dataset must not exist. +: A dataset on the same pool as the _source_, where the clones will be created. This dataset must not exist. + +**Output Options** -**Logging Options** +**-v, \--verbose** +: Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. -**-n, \--dryrun** -: Don't clone, but show the `zfs clone` commands that would be executed. +**-q, \--quiet** +: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. -**-q** -: Reduce verbosity. +**-n, \--dryrun, \--dry-run** +: Display `zfs` commands without executing them. -**-v** -: Increase verbosity. +**Snapshot Options** -**Dataset and Snapshot Options** +**\--snapshot, \--snapshot-always** +: Ensure a snapshot before cloning. -**\--snapshot-always** -: Ensure a snapshot before cloning. +**\--snap-name** _NAME_ +: Specify snapshot name. Use `$(command)` for dynamic generation. Default: `$(date -u +zelta_%Y-%m-%d_%H.%M.%S)`. -**\--snapshot-name** -: Specify a snapshot name. See `zelta.env.example` to adjust the default naming scheme. +**Dataset Options** -**-d _depth_, \--depth _depth_** -: Limits the depth of all Zelta operations. +**-d, \--depth** _LEVELS_ +: Limit recursion depth. For example, a depth of 1 includes only the specified dataset. # EXAMPLES -**Clone a dataset tree:** +Clone a dataset tree: + + zelta clone tank/vm/myos tank/temp/myos-202404 -```sh -zelta clone tank/vm/myos tank/temp/myos-202404 -``` +Recover a dataset tree, in place, to a previous snapshot's state: -**Recover a dataset tree, in place, to a previous snapshot's state:** + zfs rename tank/vm/myos tank/Archives/myos-202404 + zelta clone tank/Archives/myos-202404@goodsnapshot tank/vm/myos -```sh -zfs rename tank/vm/myos tank/Archives/myos-202404 -zelta clone tank/Archives/myos-202404@goodsnapshot tank/vm/myos -``` +Dry run to display the `zfs clone` commands without executing them: -**Dry Run:** Display the `zfs clone` commands without executing them. + zelta clone -n tank/source/dataset tank/target/dataset -```sh -zelta clone -n tank/source/dataset tank/target/dataset -``` +# EXIT STATUS + +Returns 0 on success, non-zero on error. + +# NOTES + +See **zelta-options(7)** for environment variables and `zelta.env` configuration. # SEE ALSO -zelta(8), zelta-backup(8), zelta-match(8), zelta-options(7), zelta-policy(8), zelta-revert(8), zelta-rotate(8), cron(8), ssh(1), zfs(8), zfs-clone(8), zfs-promote(8) + +zelta(8), zelta-options(7), zelta-backup(8), zelta-match(8), zelta-policy(8), zelta-revert(8), zelta-rotate(8), ssh(1), zfs(8), zfs-clone(8), zfs-promote(8) # AUTHORS + Daniel J. Bell <_bellhyve@zelta.space_> # WWW + https://zelta.space diff --git a/doc/zelta-match.8 b/doc/zelta-match.8 index 2642ded..9c8f5ee 100644 --- a/doc/zelta-match.8 +++ b/doc/zelta-match.8 @@ -20,11 +20,11 @@ backups. .PP \f[B]Logging Options\f[R] .TP -\f[B]\-v,\-\-verbose\f[R] +\f[B]\-v, \-\-verbose\f[R] Increase verbosity. Specify once for operational detail and twice (\-vv) for debug output. .TP -\f[B]\-q,\-\-quiet\f[R] +\f[B]\-q, \-\-quiet\f[R] Quiet output. Specify once to suppress warnings and twice (\-qq) to suppress errors. .TP @@ -39,22 +39,22 @@ supported. \f[B]\-\-text\f[R] Forces default output (notices) to print as plain text standard output. .TP -\f[B]\-n,\-\-dryrun\f[R] +\f[B]\-n, \-\-dryrun\f[R] Display `zfs' commands related to the action rather than running them. .PP -** Dataset and Snapshot Options** +\f[B]Dataset and Snapshot Options\f[R] .TP -\f[B]\-d,\-\-depth\f[R] +\f[B]\-d, \-\-depth\f[R] Limit the recursion depth of operations to the number of levels indicated. For example, a depth of 1 will only include the indicated dataset. .TP -\f[B]\-\-exclude,\-X\f[R] +\f[B]\-\-exclude, \-X\f[R] Exclude datasets or source snapshots matching the specified exclusion pattern. -This option can be include multiple patters separated by commas and can -be specified multiple times. -See \f[I]EXCLUSION PATTERNS\f[R] in \f[B]zelta help options\f[R] for +This option can include multiple patterns separated by commas and can be +specified multiple times. +See \f[I]EXCLUSION PATTERNS\f[R] in \f[B]zelta\-options(7)\f[R] for details. .PP \f[B]Columns and Summary Behavior\f[R] @@ -63,7 +63,8 @@ details. Suppress column headers and separate columns with a single tab. .TP \f[B]\-p\f[R] -Out sizes in exact numbers instead of human\-readable values like `1M'. +Output sizes in exact numbers instead of human\-readable values like +`1M'. .TP \f[B]\-o\f[R] Specify a list of `zelta match' columns. @@ -200,15 +201,15 @@ of between each host. zelta match \-d2 backuphost:rust101/Backups rust000/Backups .EE .PP -\f[B]Dry Run:\f[R] Display the \f[CR]zfs list\f[R] commands that woud be -used without executing them. +\f[B]Dry Run:\f[R] Display the \f[CR]zfs list\f[R] commands that would +be used without executing them. .IP .EX zelta match \-n tank/source/dataset tank/target/dataset .EE .SH SEE ALSO zelta(8), zelta\-backup(8), zelta\-policy(8), zelta\-clone(8), -zelta\-options(8), zelta\-revert(8), zelta\-rotate(8), cron(8), ssh(1), +zelta\-options(7), zelta\-revert(8), zelta\-rotate(8), cron(8), ssh(1), zfs(8) .SH AUTHORS Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> diff --git a/doc/zelta-match.md b/doc/zelta-match.md index 2dafb2a..77766d6 100644 --- a/doc/zelta-match.md +++ b/doc/zelta-match.md @@ -82,32 +82,32 @@ # EXAMPLES -**Basic Comparison:** Compare snapshots between local source and target datasets. +Basic comparison between local source and target datasets: -```sh -zelta match tank/source/dataset tank/target/dataset -``` + zelta match tank/source/dataset tank/target/dataset -**Remote Comparison:** Compare snapshots and show the size in bytes of missing snapshots on the second system. +Remote comparison showing the size in bytes of missing snapshots on the second system: -```sh -zelta match user@remote.host1:tank/source/dataset user2@remote.host2:tank/target/dataset -``` + zelta match user@remote.host1:tank/source/dataset user2@remote.host2:tank/target/dataset -**Quick backup integrity check:** Compare the top two levels of similar backup repositories to see which backups might be missing or out of between each host. +Quick backup integrity check—compare the top two levels of similar backup repositories to see which backups might be missing or out of sync between each host: -```sh -zelta match -d2 backuphost:rust101/Backups rust000/Backups -``` + zelta match -d2 backuphost:rust101/Backups rust000/Backups -**Dry Run:** Display the `zfs list` commands that would be used without executing them. +Dry run to display the `zfs list` commands that would be used without executing them: -```sh -zelta match -n tank/source/dataset tank/target/dataset -``` + zelta match -n tank/source/dataset tank/target/dataset + +# EXIT STATUS + +Returns 0 on success, non-zero on error. + +# NOTES + +See **zelta-options(7)** for environment variables and `zelta.env` configuration. # SEE ALSO -zelta(8), zelta-backup(8), zelta-policy(8), zelta-clone(8), zelta-options(7), zelta-revert(8), zelta-rotate(8), cron(8), ssh(1), zfs(8) +zelta(8), zelta-backup(8), zelta-policy(8), zelta-clone(8), zelta-options(7), zelta-revert(8), zelta-rotate(8), zelta-snapshot(8), cron(8), ssh(1), zfs(8) # AUTHORS Daniel J. Bell <_bellhyve@zelta.space_> diff --git a/doc/zelta-policy.8 b/doc/zelta-policy.8 index 11dd843..919ce4c 100644 --- a/doc/zelta-policy.8 +++ b/doc/zelta-policy.8 @@ -72,7 +72,7 @@ the replication for any matching dataset on any host. \f[I]backup\f[R] dataset endpoint name, equivalent to the paramemters of \f[B]zelta backup\f[R]. .PP -\f[B]\f[BI]dataset_pattern\f[B]\f[R] Speicfy the final source or target +\f[B]\f[BI]dataset_pattern\f[B]\f[R] Specify the final source or target dataset label. For example, \f[CR]vm\f[R] would run all backup jobs with datasets ending in \f[CR]/vm\f[R]. @@ -81,14 +81,14 @@ For detailed documentation of the \f[B]zelta policy\f[R] configuration see \f[CR]zelta.conf.example\f[R]. .TP \f[B]/usr/local/etc/zelta/zelta.conf\f[R] -The default configuration file locaiton. +The default configuration file location. .SH ENVIRONMENT For detailed documentation of the \f[B]zelta\f[R] environment variables see \f[CR]zelta help options\f[R]. .SH SEE ALSO -zelta(8), zelta\-clone(8), zelta\-backup(8), zelta\-options(8), -zelta\-match(8), zelta\-revert(8), zelta\-rotate(8), zelta\-sync(8), -ssh(1), zfs(8), zfs\-list(8) +zelta(8), zelta\-clone(8), zelta\-backup(8), zelta\-options(7), +zelta\-match(8), zelta\-revert(8), zelta\-rotate(8), ssh(1), zfs(8), +zfs\-list(8) .SH AUTHORS Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> .SH WWW diff --git a/doc/zelta-policy.md b/doc/zelta-policy.md index 499baa6..04ec882 100644 --- a/doc/zelta-policy.md +++ b/doc/zelta-policy.md @@ -36,7 +36,7 @@ Without additional parameters, **zelta policy** will run a **zelta backup** job **_dataset_** Run a backup job only for the _dataset_ listed. Note this parameter can match the _source_ **or** _target_ dataset, e.g., requesting `zroot` would run the replication for any matching dataset on any host. -**_host:dataset_** Specify a _source_ or _backup_ dataset endpoint name, equivalent to the paramemters of **zelta backup**. +**_host:dataset_** Specify a _source_ or _backup_ dataset endpoint name, equivalent to the parameters of **zelta backup**. **_dataset_pattern_** Specify the final source or target dataset label. For example, `vm` would run all backup jobs with datasets ending in `/vm`. @@ -44,13 +44,21 @@ Without additional parameters, **zelta policy** will run a **zelta backup** job For detailed documentation of the **zelta policy** configuration see `zelta.conf.example`. **/usr/local/etc/zelta/zelta.conf** -: The default configuration file locaiton. +: The default configuration file location. # ENVIRONMENT For detailed documentation of the **zelta** environment variables see `zelta help options`. +# EXIT STATUS + +Returns 0 on success, non-zero on error. + +# NOTES + +See **zelta-options(7)** for environment variables and `zelta.env` configuration. + # SEE ALSO -zelta(8), zelta-clone(8), zelta-backup(8), zelta-options(7), zelta-match(8), zelta-revert(8), zelta-rotate(8), ssh(1), zfs(8), zfs-list(8) +zelta(8), zelta-clone(8), zelta-backup(8), zelta-options(7), zelta-match(8), zelta-revert(8), zelta-rotate(8), zelta-snapshot(8), ssh(1), zfs(8), zfs-list(8) # AUTHORS Daniel J. Bell <_bellhyve@zelta.space_> diff --git a/doc/zelta-prune.8 b/doc/zelta-prune.8 new file mode 100644 index 0000000..559b6b2 --- /dev/null +++ b/doc/zelta-prune.8 @@ -0,0 +1,157 @@ +.\" Automatically generated by Pandoc 3.8.3 +.\" +.TH "zelta\-prune" "8" "" "" "System Manager\(cqs Manual" +.SH NAME +\f[B]zelta prune\f[R] \- identify snapshots safe for deletion based on +backup state +.SH SYNOPSIS +\f[B]zelta prune\f[R] [\f[I]OPTIONS\f[R]] \f[I]source\f[R] +\f[I]target\f[R] +.SH DESCRIPTION +\f[B]zelta prune\f[R] identifies snapshots on a source dataset that have +been safely replicated to a target and are eligible for deletion based +on retention policies. +This command is useful for managing snapshot accumulation on production +systems while ensuring backup integrity. +.PP +\f[B]zelta prune\f[R] only suggests snapshots for deletion\(emit does +not delete them. +Output is provided in a format suitable for review before execution. +.PP +As with other Zelta commands, \f[B]zelta prune\f[R] works recursively on +dataset trees. +Both source and target may be local or remote via \f[B]ssh(1)\f[R]. +.SS Safety Criteria +\f[B]WARNING:\f[R] \f[CR]zelta prune\f[R] is a new feature currently in +production testing and will be formally released in Zelta 1.2 (May +2026). +.PP +A snapshot is considered safe to prune only if: +.IP "1." 3 +The snapshot exists on the target (has been replicated) +.IP "2." 3 +The snapshot is older than the most recent common match point +.IP "3." 3 +The snapshot meets the minimum retention requirements +.PP +Snapshots newer than the common match point are never suggested for +deletion, as they may be needed for future incremental replication. +.PP +Remote endpoint names follow \f[B]scp(1)\f[R] conventions. +Dataset names follow \f[B]zfs(8)\f[R] naming conventions. +.SH OPTIONS +\f[B]Endpoint Arguments (Required)\f[R] +.TP +\f[I]source\f[R] +The dataset tree containing snapshots to evaluate for pruning. +.TP +\f[I]target\f[R] +The backup dataset tree used to verify replication status. +.PP +\f[B]Output Options\f[R] +.TP +\f[B]\-v, \-\-verbose\f[R] +Increase verbosity. +Specify once for operational detail, twice (\f[CR]\-vv\f[R]) for debug +output. +.TP +\f[B]\-q, \-\-quiet\f[R] +Quiet output. +Specify once to suppress warnings, twice (\f[CR]\-qq\f[R]) to suppress +errors. +.TP +\f[B]\-n, \-\-dryrun, \-\-dry\-run\f[R] +Display \f[CR]zfs\f[R] commands without executing them. +.PP +\f[B]Retention Options\f[R] +.TP +\f[B]\-\-keep\-snap\-num\f[R] \f[I]N\f[R] +Minimum number of snapshots to keep after the match point. +Default: 100. +.TP +\f[B]\-\-keep\-snap\-days\f[R] \f[I]N\f[R] +Minimum age in days before a snapshot is eligible for deletion. +Default: 90. +.TP +\f[B]\-\-no\-ranges\f[R] +Disable range compression in output. +By default, consecutive snapshots are displayed as ranges (e.g., +\f[CR]snap1%snap5\f[R]). +This option outputs individual snapshot names, one per line. +.PP +\f[B]Dataset Options\f[R] +.TP +\f[B]\-d, \-\-depth\f[R] \f[I]LEVELS\f[R] +Limit recursion depth. +For example, a depth of 1 includes only the specified dataset. +.TP +\f[B]\-X, \-\-exclude\f[R] \f[I]PATTERN\f[R] +Exclude datasets matching the specified pattern. +See \f[B]zelta\-options(7)\f[R] for pattern syntax. +.SH OUTPUT FORMAT +By default, \f[B]zelta prune\f[R] outputs snapshot ranges using ZFS +range syntax: +.IP +.EX +pool/dataset\(atoldest_snap%newest_snap +.EE +.PP +This format is compatible with \f[CR]zfs destroy\f[R] for batch +deletion. +With \f[CR]\-\-no\-ranges\f[R], individual snapshot names are output one +per line. +.SH EXAMPLES +Identify prunable snapshots with default retention (100 snapshots, 90 +days): +.IP +.EX +zelta prune tank/data backup\-host.example:tank/backups/data +.EE +.PP +Use stricter retention (keep 200 snapshots, 180 days minimum age): +.IP +.EX +zelta prune \-\-keep\-snap\-num=200 \-\-keep\-snap\-days=180 \(rs + tank/data backup\-host.example:tank/backups/data +.EE +.PP +Output individual snapshots instead of ranges: +.IP +.EX +zelta prune \-\-no\-ranges tank/data backup\-host.example:tank/backups/data +.EE +.PP +Review and then delete prunable snapshots: +.IP +.EX +# First, review what would be deleted +zelta prune tank/data backup:tank/backups/data + +# If satisfied, pipe to xargs for deletion (use with caution) +zelta prune tank/data backup:tank/backups/data | xargs \-n1 zfs destroy +.EE +.PP +Exclude temporary datasets from consideration: +.IP +.EX +zelta prune \-X \(aq*/tmp\(aq tank/data backup:tank/backups/data +.EE +.SH EXIT STATUS +Returns 0 on success, non\-zero on error. +.SH NOTES +\f[B]zelta prune\f[R] is experimental. +Always review output before executing deletions. +.PP +This command is driven by the \f[B]zelta match\f[R] comparison engine. +See \f[B]zelta\-match(8)\f[R] for details on how source and target +snapshots are compared. +.PP +See \f[B]zelta\-options(7)\f[R] for environment variables and +\f[CR]zelta.env\f[R] configuration. +.SH SEE ALSO +zelta(8), zelta\-options(7), zelta\-match(8), zelta\-backup(8), zfs(8), +zfs\-destroy(8) +.SH AUTHORS +Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> +.SH WWW +https://zelta.space diff --git a/doc/zelta-prune.md b/doc/zelta-prune.md index 4bc732a..d85baf1 100644 --- a/doc/zelta-prune.md +++ b/doc/zelta-prune.md @@ -18,6 +18,8 @@ As with other Zelta commands, **zelta prune** works recursively on dataset trees ## Safety Criteria +**WARNING:** `zelta prune` is a new feature currently in production testing and will be formally released in Zelta 1.2 (May 2026). + A snapshot is considered safe to prune only if: 1. The snapshot exists on the target (has been replicated) @@ -91,13 +93,19 @@ Output individual snapshots instead of ranges: zelta prune --no-ranges tank/data backup-host.example:tank/backups/data -Review and then delete prunable snapshots: +Review prunable snapshots, then delete after confirmation: # First, review what would be deleted zelta prune tank/data backup:tank/backups/data - # If satisfied, pipe to xargs for deletion (use with caution) - zelta prune tank/data backup:tank/backups/data | xargs -n1 zfs destroy + # Save to a file for review + zelta prune tank/data backup:tank/backups/data | tee /tmp/prune-list.txt + + # Review the list carefully + # Use `zelta prune -v` to see matching snapshots being kept + + # If satisfied, delete the replicated source snapshots + cat /tmp/prune-list.txt | xargs -n1 zfs destroy Exclude temporary datasets from consideration: @@ -109,7 +117,7 @@ Returns 0 on success, non-zero on error. # NOTES -**zelta prune** is experimental. Always review output before executing deletions. +**zelta prune** is experimental. Always review output before executing deletions. Deleting snapshots is irreversible. This command is driven by the **zelta match** comparison engine. See **zelta-match(8)** for details on how source and target snapshots are compared. diff --git a/doc/zelta-revert.8 b/doc/zelta-revert.8 index 2d05e6b..a19cbb7 100644 --- a/doc/zelta-revert.8 +++ b/doc/zelta-revert.8 @@ -99,7 +99,7 @@ The original diverged datasets remain accessible as .SH EXIT STATUS Returns 0 on success, non\-zero on error. .SH NOTES -See \f[B]zelta\-options(8)\f[R] for environment variables, +See \f[B]zelta\-options(7)\f[R] for environment variables, \f[CR]zelta.env\f[R] configuration, and \f[CR]zelta policy\f[R] integration. .SH SEE ALSO diff --git a/doc/zelta-revert.md b/doc/zelta-revert.md index 7517d4f..00a614f 100644 --- a/doc/zelta-revert.md +++ b/doc/zelta-revert.md @@ -4,6 +4,10 @@ **zelta revert** - rewind a ZFS dataset tree in place by renaming and cloning +# SYNOPSIS + +**zelta revert** [_OPTIONS_] _endpoint_[@_snapshot_] + # DESCRIPTION **zelta revert** rewinds a dataset to a previous snapshot state by renaming the current dataset and creating a clone from the specified snapshot. This technique is a non-destructive alternative to `zfs rollback`, preserving the current state for forensic analysis, testing, or recovery scenarios. diff --git a/doc/zelta-rotate.8 b/doc/zelta-rotate.8 index b95cd9b..dee36e8 100644 --- a/doc/zelta-rotate.8 +++ b/doc/zelta-rotate.8 @@ -145,7 +145,7 @@ The original diverged datasets remain accessible as .SH EXIT STATUS Returns 0 on success, non\-zero on error. .SH NOTES -See \f[B]zelta\-options(8)\f[R] for environment variables, +See \f[B]zelta\-options(7)\f[R] for environment variables, \f[CR]zelta.env\f[R] configuration, and \f[CR]zelta policy\f[R] integration. .SH SEE ALSO diff --git a/doc/zelta-rotate.md b/doc/zelta-rotate.md index 76aa719..e27607b 100644 --- a/doc/zelta-rotate.md +++ b/doc/zelta-rotate.md @@ -4,6 +4,10 @@ **zelta rotate** - recover sync continuity by renaming, cloning, and incrementally syncing a ZFS replica +# SYNOPSIS + +**zelta rotate** [_OPTIONS_] _source_ _target_ + # DESCRIPTION **zelta rotate** renames a target replica and performs a multi-way clone and sync operation to restore sync continuity when a source and target have diverged. The operation considers up to four dataset states: the current source, the current target, the source's origin (if cloned), and the target's origin, finding the optimal sync path between them. diff --git a/doc/zelta-snapshot.8 b/doc/zelta-snapshot.8 new file mode 100644 index 0000000..a198c37 --- /dev/null +++ b/doc/zelta-snapshot.8 @@ -0,0 +1,110 @@ +.\" Automatically generated by Pandoc 3.8.3 +.\" +.TH "zelta\-snapshot" "8" "" "" "System Manager\(cqs Manual" +.SH NAME +\f[B]zelta snapshot\f[R] \- create recursive ZFS snapshots locally or +remotely +.SH SYNOPSIS +\f[B]zelta snapshot\f[R] [\f[I]OPTIONS\f[R]] +\f[I]endpoint\f[R][\(at_snapshot_] +.SH DESCRIPTION +\f[B]zelta snapshot\f[R] creates a recursive snapshot on a dataset tree. +The endpoint may be local or remote via \f[B]ssh(1)\f[R]. +.PP +As with other Zelta commands, \f[B]zelta snapshot\f[R] works recursively +on a dataset tree. +This provides a simple way to create consistent, atomic snapshots across +an entire dataset hierarchy without requiring Zelta to be installed on +remote systems. +.PP +Remote endpoint names follow \f[B]scp(1)\f[R] conventions. +Dataset names follow \f[B]zfs(8)\f[R] naming conventions. +.PP +Examples: +.IP +.EX +Local: pool/dataset +Local: pool/dataset\(atmy\-snapshot +Remote: user\(atexample.com:pool/dataset +Remote: user\(atexample.com:pool/dataset\(atbackup\-2025\-01\-15 +.EE +.SH OPTIONS +\f[B]Endpoint Argument (Required)\f[R] +.TP +\f[I]endpoint\f[R] +The dataset to snapshot. +If a snapshot name is specified with \f[CR]\(atsnapshot\f[R], that name +is used. +Otherwise, the name is determined by the \f[CR]\-\-snap\-name\f[R] +option. +.PP +\f[B]Output Options\f[R] +.TP +\f[B]\-v, \-\-verbose\f[R] +Increase verbosity. +Specify once for operational detail, twice (\f[CR]\-vv\f[R]) for debug +output. +.TP +\f[B]\-q, \-\-quiet\f[R] +Quiet output. +Specify once to suppress warnings, twice (\f[CR]\-qq\f[R]) to suppress +errors. +.TP +\f[B]\-n, \-\-dryrun, \-\-dry\-run\f[R] +Display \f[CR]zfs\f[R] commands without executing them. +.PP +\f[B]Snapshot Options\f[R] +.TP +\f[B]\-\-snap\-name\f[R] \f[I]NAME\f[R] +Specify snapshot name. +Use \f[CR]$(command)\f[R] for dynamic generation. +Default: \f[CR]$(date \-u +zelta_%Y\-%m\-%d_%H.%M.%S)\f[R]. +This option is ignored if a snapshot name is provided in the endpoint +argument. +.PP +\f[B]Dataset Options\f[R] +.TP +\f[B]\-d, \-\-depth\f[R] \f[I]LEVELS\f[R] +Limit recursion depth. +For example, a depth of 1 includes only the specified dataset. +.SH EXAMPLES +Create a snapshot with the default naming scheme: +.IP +.EX +zelta snapshot tank/data +.EE +.PP +Create a snapshot with a specific name: +.IP +.EX +zelta snapshot tank/data\(atbefore\-upgrade +.EE +.PP +Create a snapshot on a remote host: +.IP +.EX +zelta snapshot backup\(atstorage.example.com:tank/backups +.EE +.PP +Create a snapshot with a custom naming scheme: +.IP +.EX +zelta snapshot \-\-snap\-name \(dqmanual_$(date +%Y%m%d)\(dq tank/data +.EE +.PP +Dry run to preview the command: +.IP +.EX +zelta snapshot \-n tank/data +.EE +.SH EXIT STATUS +Returns 0 on success, non\-zero on error. +.SH NOTES +See \f[B]zelta\-options(7)\f[R] for environment variables and +\f[CR]zelta.env\f[R] configuration. +.SH SEE ALSO +zelta(8), zelta\-options(7), zelta\-backup(8), zfs(8), zfs\-snapshot(8) +.SH AUTHORS +Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> +.SH WWW +https://zelta.space diff --git a/doc/zelta.8 b/doc/zelta.8 index 9f7151e..8f7e302 100644 --- a/doc/zelta.8 +++ b/doc/zelta.8 @@ -80,7 +80,7 @@ Configuration follows a hierarchy from lowest to highest precedence: 5. Command\-line arguments .EE .PP -See \f[B]zelta\-options(8)\f[R] for details. +See \f[B]zelta\-options(7)\f[R] for details. .SH FILES .TP \f[B]/usr/local/etc/zelta/zelta.conf\f[R] @@ -125,7 +125,7 @@ zelta policy Returns 0 on success, non\-zero on error. .SH SEE ALSO zelta\-match(8), zelta\-backup(8), zelta\-policy(8), zelta\-clone(8), -zelta\-options(8), zelta\-revert(8), zelta\-rotate(8), cron(8), ssh(1), +zelta\-options(7), zelta\-revert(8), zelta\-rotate(8), cron(8), ssh(1), zfs(8) .SH AUTHORS Daniel J. Bell <\f[I]bellhyve\(atzelta.space\f[R]> diff --git a/doc/zelta.md b/doc/zelta.md index dad89c7..bc10981 100644 --- a/doc/zelta.md +++ b/doc/zelta.md @@ -38,6 +38,9 @@ For detailed usage of each subcommand, run **zelta help ** or see th **zelta backup** _source_ _target_ : Replicate a dataset tree. Creates snapshots if needed, detects optimal send options, and replicates intermediate snapshots. See **zelta-backup(8)**. +**zelta snapshot** _endpoint_ +: Create recursive snapshots on a local or remote endpoint. See **zelta-snapshot(8)**. + ## Recovery **zelta clone** _dataset_ _target_ @@ -98,8 +101,12 @@ Back up everything defined in policy settings (**zelta.conf**). # EXIT STATUS Returns 0 on success, non-zero on error. +# NOTES + +See **zelta-options(7)** for environment variables and `zelta.env` configuration. + # SEE ALSO -zelta-match(8), zelta-backup(8), zelta-policy(8), zelta-clone(8), zelta-options(7), zelta-revert(8), zelta-rotate(8), cron(8), ssh(1), zfs(8) +zelta-match(8), zelta-backup(8), zelta-policy(8), zelta-clone(8), zelta-options(7), zelta-revert(8), zelta-rotate(8), zelta-snapshot(8), cron(8), ssh(1), zfs(8) # AUTHORS Daniel J. Bell <_bellhyve@zelta.space_> diff --git a/port-files/Makefile b/packaging/freebsd/Makefile similarity index 71% rename from port-files/Makefile rename to packaging/freebsd/Makefile index 8cc8cc6..4e35366 100644 --- a/port-files/Makefile +++ b/packaging/freebsd/Makefile @@ -1,17 +1,17 @@ PORTNAME= zelta DISTVERSIONPREFIX= v -DISTVERSION= 0.5 -DISTVERSIONSUFFIX= -beta +DISTVERSION= 1.1.0 +PORTREVISION= 1 CATEGORIES= sysutils -MAINTAINER= jt@obs-sec.com -COMMENT= Zelta Replication Suite +MAINTAINER= daniel@belltech.it +COMMENT= ZFS tools used for data migration and backup management WWW= https://github.com/bellhyve/zelta LICENSE= BSD2CLAUSE USE_GITHUB= yes -GH_ACCOUNT= q5sys +GH_ACCOUNT= bellhyve NO_ARCH= yes NO_BUILD= yes @@ -32,6 +32,8 @@ do-install: ${INSTALL_DATA} ${WRKSRC}/${_ZELTA_ENV} ${STAGEDIR}${ETCDIR}/${_ZELTA_ENV}.sample ${INSTALL_DATA} ${WRKSRC}/${_ZELTA_CONF} ${STAGEDIR}${ETCDIR}/${_ZELTA_CONF}.sample ${MKDIR} ${STAGEDIR}${_ZELTA_SHARE} - ${INSTALL_DATA} ${WRKSRC}/share/zelta/* ${STAGEDIR}${_ZELTA_SHARE} + ${INSTALL_SCRIPT} ${WRKSRC}/share/zelta/* ${STAGEDIR}${_ZELTA_SHARE} + ${INSTALL_MAN} ${WRKSRC}/doc/*.8 ${STAGEDIR}${PREFIX}/share/man/man8 + ${INSTALL_MAN} ${WRKSRC}/doc/*.7 ${STAGEDIR}${PREFIX}/share/man/man7 .include diff --git a/packaging/freebsd/distinfo b/packaging/freebsd/distinfo new file mode 100644 index 0000000..58efd4e --- /dev/null +++ b/packaging/freebsd/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1717424774 +SHA256 (bellhyve-zelta-v1.0.1_GH0.tar.gz) = e6bff745d3125bd0b435097c282769b4e175ef85ea386500bd456d91346af9ab +SIZE (bellhyve-zelta-v1.0.1_GH0.tar.gz) = 34979 diff --git a/packaging/freebsd/files/pkg-message.in b/packaging/freebsd/files/pkg-message.in new file mode 100644 index 0000000..7f8e093 --- /dev/null +++ b/packaging/freebsd/files/pkg-message.in @@ -0,0 +1,21 @@ +[ +{ type: install + message: <]' or the wiki: + + https://zelta.space/home + +If you find any bugs please file them +at https://github.com/bellhyve/zelta/issues. +EOM +} +] diff --git a/packaging/freebsd/pkg-descr b/packaging/freebsd/pkg-descr new file mode 100644 index 0000000..22760c1 --- /dev/null +++ b/packaging/freebsd/pkg-descr @@ -0,0 +1,8 @@ +Zelta is a suite of tools offering a streamlined approach to managing +ZFS snapshot replication across systems. It's built with the intention +of simplifying complex ZFS functions into safe and user-friendly +commands while also being the foundation for large-scale backup +and failover environments. It's easy and accessible while working +with most UNIX and UNIX-like base systems without additional packages. +It's optimized for environments with strict permission separation, +and integrates well into many types of existing ZFS workflows. diff --git a/packaging/freebsd/pkg-plist b/packaging/freebsd/pkg-plist new file mode 100644 index 0000000..80b5ba7 --- /dev/null +++ b/packaging/freebsd/pkg-plist @@ -0,0 +1,22 @@ +bin/zelta +bin/zeport +bin/zmatch +bin/zpull +bin/zsync +@sample %%ETCDIR%%/zelta.conf.sample +@sample %%ETCDIR%%/zelta.env.sample +%%DATADIR%%/zelta-backup.awk +%%DATADIR%%/zelta-match-pipe.awk +%%DATADIR%%/zelta-match.awk +%%DATADIR%%/zelta-policy.awk +%%DATADIR%%/zelta-report.awk +%%DATADIR%%/zelta-sendopts.awk +%%DATADIR%%/zelta-snapshot.awk +%%DATADIR%%/zelta-usage.sh +share/man/man7/zelta-options.7.gz +share/man/man8/zelta-backup.8.gz +share/man/man8/zelta-clone.8.gz +share/man/man8/zelta-match.8.gz +share/man/man8/zelta-policy.8.gz +share/man/man8/zelta-sync.8.gz +share/man/man8/zelta.8.gz diff --git a/port-files/distinfo b/port-files/distinfo deleted file mode 100644 index 1fc8762..0000000 --- a/port-files/distinfo +++ /dev/null @@ -1,3 +0,0 @@ -TIMESTAMP = 1710344098 -SHA256 (q5sys-zelta-v0.5-beta_GH0.tar.gz) = 9c195649e4e47b9ab27eaad64b1fe4971085269169c36f1399e4c520829cc6bd -SIZE (q5sys-zelta-v0.5-beta_GH0.tar.gz) = 25357 diff --git a/port-files/files/pkg-message.in b/port-files/files/pkg-message.in deleted file mode 100644 index b8dd235..0000000 --- a/port-files/files/pkg-message.in +++ /dev/null @@ -1,17 +0,0 @@ -[ -{ type: install - message: < Date: Tue, 20 Jan 2026 11:58:06 -0500 Subject: [PATCH 15/47] add testing examples local review of shellspec tester fix: replace command string concatenation with functions for shell compatibility fix: ensure clean exit statuses in test helper functions fix: remove external commands from skip functions to prevent Shellspec errors fix: add -n flag to ssh commands in test helpers to prevent stdin reading fix missing 'ssh -n' input capture bug with shellspec test: add divergent tree setup spec and helpers test: add divergent tree match test with complex scenarios divergent test example fix divergent test testing: add canmount permission for target creation test: fix skip condition for divergent spec tests testing: switch to cleaner testing output mode test: add helper functions for pool and command validation --- .shellspec | 2 + README.md | 6 +- bin/zelta | 2 +- test/01_no_op_spec.sh | 2 +- test/02_pool_spec.sh | 3 - test/02_setup_spec.sh | 12 +++ test/03_divergent_spec.sh | 24 +++++ test/03_no_op_ds_spec.sh | 3 - test/99_cleanup_spec.sh | 12 +++ test/test_helper.sh | 202 +++++++++++++++++++++++++++++++++++--- zelta.conf | 2 +- 11 files changed, 244 insertions(+), 26 deletions(-) delete mode 100644 test/02_pool_spec.sh create mode 100644 test/02_setup_spec.sh create mode 100644 test/03_divergent_spec.sh delete mode 100644 test/03_no_op_ds_spec.sh create mode 100644 test/99_cleanup_spec.sh diff --git a/.shellspec b/.shellspec index 2ca0a5b..ad4947d 100644 --- a/.shellspec +++ b/.shellspec @@ -1,3 +1,5 @@ --helperdir test --require test_helper --default-path test +--shell sh +--format tap diff --git a/README.md b/README.md index e30bb51..3d0346e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![Zelta Logo](https://zelta.space/index/zelta-banner.svg) # The Zelta Backup and Recovery Suite -*Version v1.1, 2026-01-20* +*Version 1.1, 2026-01-20* --- @@ -39,7 +39,7 @@ Written in portable Bourne shell and AWK, Zelta runs anywhere ZFS runs. No packa ## Installation -### From Source (Recommended for v1.1) +### From Source (Recommended for Zelta 1.1) ```sh git clone https://github.com/bellhyve/zelta.git cd zelta @@ -49,7 +49,7 @@ sudo ./install.sh ``` ### FreeBSD Ports -Zelta 1.0 (March 2024) is available in the FreeBSD Ports Collection. For the latest features, install from GitHub. +Zelta 1.0.1_1 (March 2024) is available in the FreeBSD Ports Collection. For the latest features, install from GitHub. ```sh pkg install zelta ``` diff --git a/bin/zelta b/bin/zelta index 1c4fca8..7d8b0a7 100755 --- a/bin/zelta +++ b/bin/zelta @@ -4,7 +4,7 @@ # # Initialize the environment for Zelta subcommands -ZELTA_VERSION="Zelta 1.1-rc1" +ZELTA_VERSION="Zelta 1.1.0" zelta_usage() { exec >&2 diff --git a/test/01_no_op_spec.sh b/test/01_no_op_spec.sh index 7c660a5..64ba65c 100644 --- a/test/01_no_op_spec.sh +++ b/test/01_no_op_spec.sh @@ -17,7 +17,7 @@ Describe 'Zelta no-op command checks' The status should be success The output should include 'Zelta' End - It 'shows help' + It 'shows man page' When run command zelta help The status should be success The output should include 'zelta(8)' diff --git a/test/02_pool_spec.sh b/test/02_pool_spec.sh deleted file mode 100644 index 5de4572..0000000 --- a/test/02_pool_spec.sh +++ /dev/null @@ -1,3 +0,0 @@ -Describe 'Pool setup' - Skip if ! pools_defined -End diff --git a/test/02_setup_spec.sh b/test/02_setup_spec.sh new file mode 100644 index 0000000..b800a04 --- /dev/null +++ b/test/02_setup_spec.sh @@ -0,0 +1,12 @@ +Describe 'Pool setup' + It 'create' + Skip if 'source not defined' skip_src_pool + When call make_src_pool + The status should be success + End + It 'create' + Skip if 'target not defined' skip_tgt_pool + When call make_tgt_pool + The status should be success + End +End diff --git a/test/03_divergent_spec.sh b/test/03_divergent_spec.sh new file mode 100644 index 0000000..5d780c7 --- /dev/null +++ b/test/03_divergent_spec.sh @@ -0,0 +1,24 @@ +Describe 'Divergent tree setup' + It 'creates divergent tree on source' + Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" + When call make_divergent_tree + The status should be success + The output should include 'snapshotting' + The output should include 'syncing 9 datasets' + The error should not include 'error:' + End +End + +Describe 'Divergent tree match' + It 'shows expected divergence types' + Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" + When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The status should be success + The output should include 'up-to-date' + The output should include 'syncable (full)' + The output should include 'syncable (incremental)' + The output should include 'blocked sync: target diverged' + The output should include 'blocked sync: no target snapshots' + The output should include '11 total datasets compared' + End +End diff --git a/test/03_no_op_ds_spec.sh b/test/03_no_op_ds_spec.sh deleted file mode 100644 index 91bb809..0000000 --- a/test/03_no_op_ds_spec.sh +++ /dev/null @@ -1,3 +0,0 @@ -Describe 'Dataset' - Skip if ! pools_defined -End diff --git a/test/99_cleanup_spec.sh b/test/99_cleanup_spec.sh new file mode 100644 index 0000000..12c0f4c --- /dev/null +++ b/test/99_cleanup_spec.sh @@ -0,0 +1,12 @@ +Describe 'Pool cleanup' + It 'destroy source' + Skip if 'no pools defined' skip_pools + When call nuke_src_pool + The status should be success + End + It 'destroy target' + Skip if 'no pools defined' skip_pools + When call nuke_tgt_pool + The status should be success + End +End diff --git a/test/test_helper.sh b/test/test_helper.sh index 35cd1ff..95d4a27 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -20,36 +20,210 @@ export ZELTA_SHARE="$REPO_ROOT/share/zelta" ## Build endpoints if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then SANDBOX_ZELTA_SRC_EP="${SANDBOX_ZELTA_SRC_REMOTE}:${SANDBOX_ZELTA_SRC_DS}" - SANDBOX_ZELTA_SRC_CMD="ssh ${SANDBOX_ZELTA_SRC_REMOTE} " else SANDBOX_ZELTA_SRC_EP="$SANDBOX_ZELTA_SRC_DS" fi if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then SANDBOX_ZELTA_TGT_EP="${SANDBOX_ZELTA_TGT_REMOTE}:${SANDBOX_ZELTA_TGT_DS}" - SANDBOX_ZELTA_SRC_CMD="ssh ${SANDBOX_ZELTA_SRC_REMOTE} " else SANDBOX_ZELTA_TGT_EP="$SANDBOX_ZELTA_TGT_DS" fi -# We should determine if sudo is actually needed -SANDBOX_ZELTA_SRC_CMD="${SANDBOX_ZELTA_SRC_CMD}sudo " -SANDBOX_ZELTA_TGT_CMD="${SANDBOX_ZELTA_TGT_CMD}sudo " - -# Consider using unique tests -# SANDBOX_ZELTA_SRC_DS="${SANDBOX_ZELTA_SRC_DS}/zelta_test_src_$$" -# SANDBOX_ZELTA_TGT_DS="${SANDBOX_ZELTA_TGT_DS}/zelta_test_tgt_$$" - export SANDBOX_ZELTA_SRC_POOL SANDBOX_ZELTA_TGT_POOL export SANDBOX_ZELTA_SRC_DS SANDBOX_ZELTA_TGT_DS export SANDBOX_ZELTA_SRC_EP SANDBOX_ZELTA_TGT_EP -export SANDBOX_ZELTA_SRC_CMD SANDBOX_ZELTA_TGT_CMD + + +## Command execution wrappers +############################## + +# Execute command on source (remote or local with sudo) +src_exec() { + if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then + ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" sudo "$@" + else + sudo "$@" + fi +} + +# Execute command on target (remote or local with sudo) +tgt_exec() { + if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then + ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" sudo "$@" + else + sudo "$@" + fi +} ## Helpers ########## -pools_defined() { - [ -z "$SANDBOX_ZELTA_SRC_POOL$SANDBOX_ZELTA_TGT_POOL" ] - +# Check if source pool is a prefix of source dataset +src_pool_matches_ds() { + case "$SANDBOX_ZELTA_SRC_DS" in + "$SANDBOX_ZELTA_SRC_POOL"/*) return 0 ;; + *) return 1 ;; + esac +} + +# Check if target pool is a prefix of target dataset +tgt_pool_matches_ds() { + case "$SANDBOX_ZELTA_TGT_DS" in + "$SANDBOX_ZELTA_TGT_POOL"/*) return 0 ;; + *) return 1 ;; + esac +} + +# Check if source command with sudo works +src_cmd_works() { + src_exec true +} + +# Check if target command with sudo works +tgt_cmd_works() { + tgt_exec true +} + +# Skip if pools not configured (pure shell check, no external commands) +skip_pools() { + [ -z "$SANDBOX_ZELTA_SRC_POOL" ] || [ -z "$SANDBOX_ZELTA_TGT_POOL" ] +} + +skip_src_pool() { + if [ -n "$SANDBOX_ZELTA_SRC_POOL" ] && src_pool_matches_ds; then + return 1 + fi + return 0 +} + +skip_tgt_pool() { + if [ -n "$SANDBOX_ZELTA_TGT_POOL" ] && tgt_pool_matches_ds; then + return 1 + fi + return 0 +} + +nuke_pool() { + _pool_name="$1" + _exec_func="$2" + _pool_file=$($_exec_func pwd)/$_pool_name.img + $_exec_func zpool destroy -f "$_pool_name" >/dev/null 2>&1 + $_exec_func rm -f "$_pool_file" + return 0 +} + +make_pool() { + _pool_name="$1" + _exec_func="$2" + _pool_file=$($_exec_func pwd)/$_pool_name.img + nuke_pool "$_pool_name" "$_exec_func" + $_exec_func truncate -s 1G "$_pool_file" + $_exec_func zpool create -f "$_pool_name" "$_pool_file" + return $? +} + +nuke_src_pool() { + nuke_pool "$SANDBOX_ZELTA_SRC_POOL" src_exec + return $? +} + +nuke_tgt_pool() { + nuke_pool "$SANDBOX_ZELTA_TGT_POOL" tgt_exec + return $? +} + +make_src_pool() { + make_pool "$SANDBOX_ZELTA_SRC_POOL" src_exec || return 1 + + # Grant ZFS permissions for source pool + if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then + #ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" + src_exec "zfs allow -u \$USER snapshot,bookmark,send,hold $SANDBOX_ZELTA_SRC_POOL" + else + zfs allow -u "$USER" snapshot,bookmark,send,hold "$SANDBOX_ZELTA_SRC_POOL" + fi + return $? +} + +make_tgt_pool() { + make_pool "$SANDBOX_ZELTA_TGT_POOL" tgt_exec || return 1 + + # Grant ZFS permissions for target pool + if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then + #ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" "zfs allow -u \$USER mount,create,rename $SANDBOX_ZELTA_TGT_POOL" + tgt_exec "zfs allow -u \$USER receive,mount,create,canmount,rename $SANDBOX_ZELTA_TGT_POOL" + else + zfs allow -u "$USER" receive:append,mount,create,canmount,rename "$SANDBOX_ZELTA_TGT_POOL" + fi + return $? +} + +# Check if source dataset exists +src_ds_exists() { + src_exec zfs list -H -o name "$SANDBOX_ZELTA_SRC_DS" >/dev/null 2>&1 + return $? +} + +# Check if target dataset exists +tgt_ds_exists() { + tgt_exec zfs list -H -o name "$SANDBOX_ZELTA_TGT_DS" >/dev/null 2>&1 + return $? +} + +# Clean source dataset if it exists +clean_src_ds() { + if src_ds_exists; then + src_exec zfs destroy -r "$SANDBOX_ZELTA_SRC_DS" + return $? + fi + return 0 +} + +# Clean target dataset if it exists +clean_tgt_ds() { + if tgt_ds_exists; then + tgt_exec zfs destroy -r "$SANDBOX_ZELTA_TGT_DS" + return $? + fi + return 0 +} + +# Create divergent tree structure on source +# Creates a dataset tree with snapshots that will diverge from target +make_divergent_tree() { + clean_src_ds || return 1 + clean_tgt_ds || return 1 + + # Create encryption key + src_exec dd if=/dev/urandom bs=32 count=1 of=/tmp/zfs_test_enc_key >/dev/null 2>&1 || return 1 + + # Create root dataset + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS" || return 1 + + # Create child datasets + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub1" || return 1 + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub2" || return 1 + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub2/orphan" || return 1 + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3" || return 1 + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 + src_exec zfs create -sV 100M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 + src_exec zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///tmp/zfs_test_enc_key "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 + + # Replicate to target with @start snapshot + zelta backup --snap-name @start "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" || return 1 + + # Generate divergence + src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub1/child" || return 1 + tgt_exec zfs create -u "$SANDBOX_ZELTA_TGT_DS/sub1/kid" || return 1 + src_exec zfs destroy "$SANDBOX_ZELTA_SRC_DS/sub2@start" || return 1 + tgt_exec zfs snapshot "$SANDBOX_ZELTA_TGT_DS/sub3/space\ name@blocker" || return 1 + tgt_exec zfs destroy "$SANDBOX_ZELTA_TGT_DS/sub4/zvol@start" || return 1 + src_exec zfs snapshot "$SANDBOX_ZELTA_SRC_DS/sub3@two" || return 1 + src_exec zfs snapshot "$SANDBOX_ZELTA_SRC_DS/sub2@two" || return 1 + tgt_exec zfs snapshot "$SANDBOX_ZELTA_TGT_DS/sub2@two" || return 1 + + return 0 } diff --git a/zelta.conf b/zelta.conf index fc47d4a..6f7a508 100644 --- a/zelta.conf +++ b/zelta.conf @@ -1,4 +1,4 @@ -# Zelta Policy Backup Configuration v1.1-rc1 +# Zelta 1.1 Policy Backup Configuration # # This file configures "zelta policy" backup jobs. For global defaults that apply to all Zelta # commands (zelta backup, zelta match, etc.), see zelta.env. For complete option documentation From e9d52f394da65fcb8597303e216d08cb658bd3cf Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Tue, 20 Jan 2026 22:35:06 -0500 Subject: [PATCH 16/47] fix local execution --- test/test_helper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index 95d4a27..c4c3ee2 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -43,7 +43,7 @@ src_exec() { if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" sudo "$@" else - sudo "$@" + sudo sh -c "$*" fi } @@ -52,7 +52,7 @@ tgt_exec() { if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" sudo "$@" else - sudo "$@" + sudo sh -c "$*" fi } From 69989a782bb00c3b4cc609931208452d06fd0e5f Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Tue, 20 Jan 2026 22:40:55 -0500 Subject: [PATCH 17/47] fix local execution --- test/test_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index c4c3ee2..01f59d1 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -155,7 +155,7 @@ make_tgt_pool() { #ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" "zfs allow -u \$USER mount,create,rename $SANDBOX_ZELTA_TGT_POOL" tgt_exec "zfs allow -u \$USER receive,mount,create,canmount,rename $SANDBOX_ZELTA_TGT_POOL" else - zfs allow -u "$USER" receive:append,mount,create,canmount,rename "$SANDBOX_ZELTA_TGT_POOL" + zfs allow -u "$USER" receive,mount,create,canmount,rename "$SANDBOX_ZELTA_TGT_POOL" fi return $? } From 64964b91170194a2365a2e351c0815da941d4547 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 21 Jan 2026 11:32:46 -0500 Subject: [PATCH 18/47] fix: add missing field mappings to zelta-json.tsv for complete JSON output --- share/zelta/zelta-json.tsv | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/share/zelta/zelta-json.tsv b/share/zelta/zelta-json.tsv index 0392975..2c3daa7 100644 --- a/share/zelta/zelta-json.tsv +++ b/share/zelta/zelta-json.tsv @@ -1,12 +1,24 @@ # MAP_KEY # VAR_SOURCE # VAR_KEY # MAP_KEY # NULL_MODE # Field name # Awk array # Array key +startTime Summary startTime +endTime Summary endTime +runTime Summary runTime sourceUser Opt SRC_USER sourceHost Opt SRC_HOST sourceDataset Opt SRC_DS sourceSnapshot Opt SRC_SNAP sourceEndpoint Opt SRC_ID +sourceListTime Summary sourceListTime +sourceWritten Summary sourceWritten targetUser Opt TGT_USER targetHost Opt TGT_HOST targetDataset Opt TGT_DS targetSnapshot Opt TGT_SNAP targetEndpoint Opt TGT_ID +targetListTime Summary targetListTime +targetsCloned Summary targetsCloned +targetsResumed Summary targetsResumed +replicationSize Summary replicationSize +replicationStreamsSent Summary replicationStreamsSent +replicationStreamsReceived Summary replicationStreamsReceived +replicationTime Summary replicationTime From dc67d49c798fc8241a7236c1d5256892baa16a3e Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 21 Jan 2026 11:41:28 -0500 Subject: [PATCH 19/47] fix: add error message collection and JSON output for replication errors --- share/zelta/zelta-backup.awk | 25 +++++++++++++++++++------ share/zelta/zelta-common.awk | 2 +- share/zelta/zelta-json.tsv | 1 + 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index ef2cba3..9c71306 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -721,7 +721,7 @@ function create_recv_command(ds_suffix, src_idx, remote_ep, _cmd_arr, _cmd, _t # Runs a sync, collecting "zfs send" output function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, - _size, _time, _streams, _sync_msg) { + _size, _time, _streams, _sync_msg, _error_msg) { # TO-DO: Make 'rotate' logic more explicit # TO-DO: Dryrun mode probably goes here if (Opt["VERB"] == "rotate" && !Action[ds_suffix, "can_rotate"]) return @@ -772,7 +772,10 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, report(LOG_INFO, "to abort a failed resume, run: 'zfs receive -A " Opt["SRC_DS"] ds_suffix"'") } else if ($0 ~ FAIL_ZFS_SEND_RECV_OUTPUT) { - report(LOG_ERROR, $0 ": " Opt["TGT_DS"] ds_suffix) + _error_msg = $0 ": " Opt["TGT_DS"] ds_suffix + report(LOG_ERROR, _error_msg) + ErrorMessagesList[++NumErrorMessages] = _error_msg + Summary["replicationErrorCode"] = 2 break } else if ($0 ~ WARN_ZFS_RECV_PROPS) { @@ -785,8 +788,12 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, report(LOG_INFO, $0) else if ($0 ~ IGNORE_ZFS_SEND_OUTPUT) {} else if ($0 ~ IGNORE_RESUME_OUTPUT) {} - else if (log_common_command_feedback() == LOG_ERROR) + else if (log_common_command_feedback() == LOG_ERROR) { + _error_msg = $0 + ErrorMessagesList[++NumErrorMessages] = _error_msg + Summary["replicationErrorCode"] = 2 break + } } close(_cmd) if (_streams) { @@ -1052,6 +1059,11 @@ function print_summary( _status, _i, _ds_suffix, _num_streams) { for (_i = 1; _i <= NumStreamsSent; _i++) json_element(SentStreamsList[_i]) json_close_array() } + if (NumErrorMessages && (Opt["LOG_MODE"] == "json")) { + json_new_array("errorMessages") + for (_i = 1; _i <= NumErrorMessages; _i++) json_element(ErrorMessagesList[_i]) + json_close_array() + } } # Main planning function @@ -1068,9 +1080,10 @@ BEGIN { SNAP_LATEST = 4 # Telemetry - DSTree["vers_major"] = 1 - DSTree["vers_minor"] = 1 + GlobalState["vers_major"] = 1 + GlobalState["vers_minor"] = 1 Summary["startTime"] = sys_time() + Summary["replicationErrorCode"] = 0 # Misc variables DSTree["final_snapshot"] = Opt["SRC_SNAP"] @@ -1110,5 +1123,5 @@ BEGIN { load_summary_vars() print_summary() - stop() + stop(Summary["replicationErrorCode"]) } diff --git a/share/zelta/zelta-common.awk b/share/zelta/zelta-common.awk index 0ef64a8..6cb55b9 100644 --- a/share/zelta/zelta-common.awk +++ b/share/zelta/zelta-common.awk @@ -87,7 +87,6 @@ function json_write(_j, _depth, _fs, _rs, _val, _next_val) { _fs = " " _rs = "\n" _depth = 0 - if (LoadSummaryVars) json_close_object() for (_j = 1; _j <= JsonNum; _j++) { _val = JsonOutput[_j] _next_val = JsonOutput[_j+1] @@ -127,6 +126,7 @@ function load_summary_data( _tsv, _key, _val) { while (getline < _tsv) { _key = $1 if ($2 == "Opt") _val = Opt[$3] + else if ($2 == "Summary") _val = Summary[$3] else continue if (!_val) { if (!$4) continue diff --git a/share/zelta/zelta-json.tsv b/share/zelta/zelta-json.tsv index 2c3daa7..4f6ae02 100644 --- a/share/zelta/zelta-json.tsv +++ b/share/zelta/zelta-json.tsv @@ -21,4 +21,5 @@ targetsResumed Summary targetsResumed replicationSize Summary replicationSize replicationStreamsSent Summary replicationStreamsSent replicationStreamsReceived Summary replicationStreamsReceived +replicationErrorCode Summary replicationErrorCode replicationTime Summary replicationTime From cf0aa30fc8e3c3658582d0884c180d70e6aeb188 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 21 Jan 2026 11:48:33 -0500 Subject: [PATCH 20/47] fix: properly close JSON object in stop function to fix broken output --- share/zelta/zelta-common.awk | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/share/zelta/zelta-common.awk b/share/zelta/zelta-common.awk index 6cb55b9..926235a 100644 --- a/share/zelta/zelta-common.awk +++ b/share/zelta/zelta-common.awk @@ -152,10 +152,17 @@ function load_summary_vars( _j) { } } +function close_json_output() { + if (Opt["LOG_MODE"] == "json" && LoadSummaryVars) { + json_close_object() + } +} + # Flush buffers and quit function stop(_error_code, _error_msg) { if (_error_msg) report(LOG_ERROR, _error_msg) if (log_output_count) close(Opt["LOG_COMMAND"]) + close_json_output() if (JsonNum) json_write() if (Opt["JSON_FILE"]) close(Opt["JSON_FILE"]) exit _error_code From bcfa950dc4182220549d153af2f0bea6800f4528 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 21 Jan 2026 13:42:02 -0500 Subject: [PATCH 21/47] json and output improvements fix: output source and target list times in zelta match summary when --time is used command build order fix: make snapshot name parsing portable across awk variants fix LOG_MODE processing especially for 'zelta match' improve json stream list output fix: add SYSTIME option for mawk JSON timestamp compatibility ensure replicationStreams* json doesn't report 'null' --- CHANGELOG.md | 5 ++++ doc/zelta-options.7.md | 7 ++++++ share/zelta/zelta-args.awk | 1 + share/zelta/zelta-backup.awk | 25 +++++++------------ share/zelta/zelta-common.awk | 48 +++++++++++++++++++++++++++++------- share/zelta/zelta-match.awk | 24 +++++++++++------- share/zelta/zelta-opts.tsv | 1 - 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0e7bf..b70ca5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Zelta will be documented in this file. ## [1.1.0] - 2026-01-20 +### Known Issues +- **JSON field naming**: Uses mixed camelCase/snake_case conventions. Will align with OpenZFS `zfs list -j` standards in 1.2 or 1.3 after upstream coordination. +- **mawk timestamps**: JSON timestamps require `ZELTA_SYSTIME='date +%s'` when using mawk. gawk and original-awk work without this setting. + ### Added - **Commands**: `zelta revert` for in-place rollbacks via rename and clone. - **Commands**: `zelta rotate` for divergent version handling, evolved from original `--rotate` flag. @@ -41,6 +45,7 @@ All notable changes to Zelta will be documented in this file. - Fixed namespace configuration and repeated targets in `zelta policy`. - Workaround for GNU Awk 5.2.1 bug. - Resume token handling and other context-aware ZFS option handling. +- Added `SYSTIME` option for mawk compatibility with JSON timestamps. ### Deprecated - `zelta endpoint` and other functions have been merged into the core library. diff --git a/doc/zelta-options.7.md b/doc/zelta-options.7.md index 9dce37c..7839de0 100644 --- a/doc/zelta-options.7.md +++ b/doc/zelta-options.7.md @@ -59,6 +59,13 @@ The following options should be modified in the environment to ensure proper ins **LOG_MODE** : Enable the specified log modes. Currently supported: 'text' (default) and 'json' (`zelta backup` related verbs only). +# JSON OUTPUT OPTIONS + +**SYSTIME** +: Override the system time function for JSON timestamps. Required for mawk users, as mawk's `srand()` does not reliably return epoch time. Set to `date +%s` for compatibility. Not needed with gawk or original-awk (nawk/bwk). + + Example: `ZELTA_SYSTIME='date +%s'` + # SSH OPTIONS **REMOTE_COMMAND** diff --git a/share/zelta/zelta-args.awk b/share/zelta/zelta-args.awk index cee126b..1fe9476 100644 --- a/share/zelta/zelta-args.awk +++ b/share/zelta/zelta-args.awk @@ -181,6 +181,7 @@ function override_options( _e) { } BEGIN { + Opt["LOG_MODE"] = "" SUBOPT_TYPES["list"] = 1 SUBOPT_TYPES["set"] = 1 load_option_list() diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 9c71306..b9c74f6 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -730,8 +730,13 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, IGNORE_RESUME_OUTPUT = "^nvlist version|^\t(fromguid|object|offset|bytes|toguid|toname|embedok|compressok)" WARN_ZFS_RECV_PROPS = "cannot receive .* property" FAIL_ZFS_SEND_RECV_OUTPUT = "^(cannot receive .* stream|cannot send|missing.*argument)" - _message = DSPair[ds_suffix, "source_start"] ? DSPair[ds_suffix, "source_start"]"::" : "" - _message = _message DSPair[ds_suffix, "source_end"] + _message = Source["DS"] ds_suffix + if (DSPair[ds_suffix, "source_start"]) + _message = _message DSPair[ds_suffix, "source_start"] "%" substr(DSPair[ds_suffix, "source_end"], 2) + else + _message = _message DSPair[ds_suffix, "source_end"] + #_message = _message (DSPair[ds_suffix, "source_start"] ? DSPair[ds_suffix, "source_start"]"%" : "") + #_message = _message (DSPair[ds_suffix, "source_end"] _ds_snap = Opt["SRC_DS"] ds_suffix DSPair[ds_suffix, "source_end"] _sync_msg = "synced" SentStreamsList[++NumStreamsSent] = _message @@ -774,7 +779,6 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, else if ($0 ~ FAIL_ZFS_SEND_RECV_OUTPUT) { _error_msg = $0 ": " Opt["TGT_DS"] ds_suffix report(LOG_ERROR, _error_msg) - ErrorMessagesList[++NumErrorMessages] = _error_msg Summary["replicationErrorCode"] = 2 break } @@ -790,7 +794,6 @@ function run_zfs_sync(ds_suffix, _cmd, _stream_info, _message, _ds_snap, else if ($0 ~ IGNORE_RESUME_OUTPUT) {} else if (log_common_command_feedback() == LOG_ERROR) { _error_msg = $0 - ErrorMessagesList[++NumErrorMessages] = _error_msg Summary["replicationErrorCode"] = 2 break } @@ -1054,16 +1057,6 @@ function print_summary( _status, _i, _ds_suffix, _num_streams) { _num_streams = Summary["replicationStreamsReceived"] _seconds = Summary["replicationTime"] if (_num_streams) report(LOG_NOTICE, _bytes_sent " sent, "_num_streams" streams received in "_seconds" seconds") - if (_num_streams && (Opt["LOG_MODE"] == "json")) { - json_new_array("sentStreams") - for (_i = 1; _i <= NumStreamsSent; _i++) json_element(SentStreamsList[_i]) - json_close_array() - } - if (NumErrorMessages && (Opt["LOG_MODE"] == "json")) { - json_new_array("errorMessages") - for (_i = 1; _i <= NumErrorMessages; _i++) json_element(ErrorMessagesList[_i]) - json_close_array() - } } # Main planning function @@ -1084,6 +1077,8 @@ BEGIN { GlobalState["vers_minor"] = 1 Summary["startTime"] = sys_time() Summary["replicationErrorCode"] = 0 + Summary["replicationStreamsReceived"] = 0 + Summary["replicationStreamsSent"] = 0 # Misc variables DSTree["final_snapshot"] = Opt["SRC_SNAP"] @@ -1119,8 +1114,6 @@ BEGIN { Summary["runTime"] = Summary["endTime"] - Summary["startTime"] compute_eligibility() - load_summary_data() - load_summary_vars() print_summary() stop(Summary["replicationErrorCode"]) diff --git a/share/zelta/zelta-common.awk b/share/zelta/zelta-common.awk index 926235a..16d2ab2 100644 --- a/share/zelta/zelta-common.awk +++ b/share/zelta/zelta-common.awk @@ -81,6 +81,8 @@ function report(mode, message, _mode_message) { _mode_message = mode SUBSEP message print _mode_message | Opt["LOG_COMMAND"] log_output_count++ + if (mode <= 1) + ErrorMessagesList[++NumErrorMessages] = message } function json_write(_j, _depth, _fs, _rs, _val, _next_val) { @@ -93,7 +95,7 @@ function json_write(_j, _depth, _fs, _rs, _val, _next_val) { if (_val ~ /^[\]\}]/) _depth-- # Enable JSON_PRETTY or provide 'jq'-like output if (Opt["JSON_PRETTY"]) printf str_rep(_fs, _depth) - printf(_val) + printf("%s", _val) if (_next_val && _val !~ /[{\[]$/ && _next_val !~ /^[\]\}]/) { printf(",") } @@ -152,6 +154,13 @@ function load_summary_vars( _j) { } } +function json_write_array( _name, _arr, _num, _i) { + if (!_num) return + json_new_array(_name) + for (_i = 1; _i <= _num; _i++) json_element(_arr[_i]) + json_close_array() +} + function close_json_output() { if (Opt["LOG_MODE"] == "json" && LoadSummaryVars) { json_close_object() @@ -162,9 +171,16 @@ function close_json_output() { function stop(_error_code, _error_msg) { if (_error_msg) report(LOG_ERROR, _error_msg) if (log_output_count) close(Opt["LOG_COMMAND"]) - close_json_output() - if (JsonNum) json_write() - if (Opt["JSON_FILE"]) close(Opt["JSON_FILE"]) + if (Opt["LOG_MODE"] == "json") { + #if (JsonNum) json_write() + load_summary_data() + load_summary_vars() + json_write_array("sentStreams", SentStreamsList, NumStreamsSent) + json_write_array("errorMessages", ErrorMessagesList, NumErrorMessages) + close_json_output() + json_write() + if (Opt["JSON_FILE"]) close(Opt["JSON_FILE"]) + } exit _error_code } @@ -254,9 +270,14 @@ function glob_to_regex(r, s) { # systime() doesn't work on a lot of systems despite being in the POSIX spec. # This workaround might not be entirely portable either; needs QA or replacement -function sys_time() { - srand(); - return srand(); +function sys_time( _time) { + if (Opt["SYSTIME"]) { + Opt["SYSTIME"] | getline _time + close(Opt["SYSTIME"]) + return _time + } + srand() + return srand() } # Convert to a human-readable number @@ -335,8 +356,17 @@ function build_command(action, vars, _remote_prefix, _cmd, _num_vars, _var_lis function get_snap_name( _snap_name, _snap_cmd) { _snap_name = Opt["SNAP_NAME"] - if (sub(/^['\"]*[$][(]/, "", _snap_name)) { - sub(/[])]['\"]*$/, "", _snap_name) + # Strip leading quote or double-quote if present + if (substr(_snap_name, 1, 1) == "'") { + _snap_name = substr(_snap_name, 2) + sub(/'$/, "", _snap_name) + } else if (substr(_snap_name, 1, 1) == "\"") { + _snap_name = substr(_snap_name, 2) + sub(/"$/, "", _snap_name) + } + # Check if it's a command substitution pattern + if (sub(/^\$\(/, "", _snap_name)) { + sub(/\)$/, "", _snap_name) _snap_cmd = _snap_name _snap_cmd | getline _snap_name close(_snap_cmd) diff --git a/share/zelta/zelta-match.awk b/share/zelta/zelta-match.awk index 82dc402..098acbe 100644 --- a/share/zelta/zelta-match.awk +++ b/share/zelta/zelta-match.awk @@ -71,12 +71,12 @@ function add_written() { } # TO-DO: Add this feature to build_command() -function wrap_time_cmd(cmd, _cmd_part, _p) { - cmd_part[_p++] = Opt["SH_COMMAND_PREFIX"] - cmd_part[_p++] = Opt["TIME_COMMAND"] - cmd_part[_p++] = cmd - cmd_part[_p++] = Opt["SH_COMMAND_SUFFIX"] - cmd = arr_join(_cmd_part) +function wrap_time_cmd(cmd, _cmd_part, _p) { + _cmd_part[++_p] = Opt["SH_COMMAND_PREFIX"] + _cmd_part[++_p] = Opt["TIME_COMMAND"] + _cmd_part[++_p] = cmd + _cmd_part[++_p] = Opt["SH_COMMAND_SUFFIX"] + cmd = arr_join(_cmd_part) return cmd } @@ -93,7 +93,7 @@ function zfs_list_cmd(endpoint, _ep, _ds, _remote, _cmd) { _cmd_arr["flags"] = "-d" Opt["DEPTH"] _cmd = build_command("LIST", _cmd_arr) if (Opt["DRYRUN"]) _cmd = report(LOG_NOTICE, "+ " _cmd) - if (Opt["TIME"]) _cmd = wrap_time_cmd(_cmd) + if (Opt["CHECK_TIME"]) _cmd = wrap_time_cmd(_cmd) _cmd = str_add(_cmd, CAPTURE_OUTPUT) return _cmd } @@ -105,7 +105,7 @@ function pipe_zfs_list_source( _match_cmd, _src_list_cmd) { _src_list_cmd = zfs_list_cmd(Source) if (Opt["DRYRUN"]) { zfs_list_cmd(Target) - stop() + stop(0) } report(LOG_DEBUG, "`"_match_cmd"`") report(LOG_INFO, "listing source: " Source["ID"]) @@ -782,6 +782,12 @@ function summary( _r, _line, _ds_suffix, _c, _key, _val, _cell) { if ((NumDSPair > 1) && (Global["summary"] ~ /,/)) report(LOG_NOTICE, NumDSPair " total datasets compared") } + if (Opt["CHECK_TIME"]) { + if (Source["list_time"]) + report(LOG_NOTICE, "SOURCE_LIST_TIME:\t" Source["list_time"]) + if (Target["list_time"]) + report(LOG_NOTICE, "TARGET_LIST_TIME:\t" Target["list_time"]) + } } ## Main Workflow Rules @@ -821,7 +827,7 @@ BEGIN { Target["ds_length"] = length(Target["DS"]) + 1 Source["ds_length"] = length(Source["DS"]) + 1 Source["list_time"] = 0 - Target["list_name"] = 0 + Target["list_time"] = 0 load_exclude_patterns() run_zfs_list_target() # Continues to process the incoming pipes 'pipe_zfs_list_source()' diff --git a/share/zelta/zelta-opts.tsv b/share/zelta/zelta-opts.tsv index babdca0..70517fb 100644 --- a/share/zelta/zelta-opts.tsv +++ b/share/zelta/zelta-opts.tsv @@ -22,7 +22,6 @@ all --help,-h,-? USAGE true all --dryrun,-n DRYRUN true show commands but do not run them all --depth,-d DEPTH set limit the recursion depth all --exclude,-X EXCLUDE list exclude source dataset or snapshot patterns -all --yule YULE true # Options for 'zelta backup' and other 'zfs send/recv/clone' verbs policy,backup,replicate,sync,rotate --json,-j LOG_MODE set json json output From cc8d5757a80594925c4e9d8788148d9a6fc91689 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 22 Jan 2026 21:00:22 -0500 Subject: [PATCH 22/47] test: add installation tests and helpers add install/uninstall to test spec --- test/00_install_spec.sh | 29 +++++++++++++++++++++++++++++ test/99_cleanup_spec.sh | 34 +++++++++++++++++++++++++--------- test/test_helper.sh | 37 +++++++++++++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 test/00_install_spec.sh diff --git a/test/00_install_spec.sh b/test/00_install_spec.sh new file mode 100644 index 0000000..256ae09 --- /dev/null +++ b/test/00_install_spec.sh @@ -0,0 +1,29 @@ +# shellcheck shell=sh + +Describe 'Zelta installation' + It 'runs installer without errors' + When run ./install.sh + The status should be success + The output should include 'installing' + End + + It 'installs zelta binary' + When run test -x "$ZELTA_BIN/zelta" + The status should be success + End + + It 'installs share files' + When run test -f "$ZELTA_SHARE/zelta-common.awk" + The status should be success + End + + It 'installs man pages' + When run test -f "$ZELTA_DOC/man8/zelta.8" + The status should be success + End + + It 'creates config examples' + When run test -f "$ZELTA_ETC/zelta.conf.example" + The status should be success + End +End diff --git a/test/99_cleanup_spec.sh b/test/99_cleanup_spec.sh index 12c0f4c..fc0a739 100644 --- a/test/99_cleanup_spec.sh +++ b/test/99_cleanup_spec.sh @@ -1,12 +1,28 @@ -Describe 'Pool cleanup' - It 'destroy source' - Skip if 'no pools defined' skip_pools - When call nuke_src_pool - The status should be success +Describe 'Cleanup' + Describe 'Pool cleanup' + It 'destroy source' + Skip if 'no pools defined' skip_pools + When call nuke_src_pool + The status should be success + End + It 'destroy target' + Skip if 'no pools defined' skip_pools + When call nuke_tgt_pool + The status should be success + End End - It 'destroy target' - Skip if 'no pools defined' skip_pools - When call nuke_tgt_pool - The status should be success + + Describe 'Installation cleanup' + It 'uninstall script' + Skip if 'avoid systemwide uninstall' skip_if_root + When run sh uninstall.sh + The status should be success + The output should include 'removing' + End + It 'remove temporary installation' + When call cleanup_temp_install + The status should be success + The output should include '2' + End End End diff --git a/test/test_helper.sh b/test/test_helper.sh index 01f59d1..1aed1f4 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -12,10 +12,23 @@ # # If pools/hosts are not configured, only basic sanity tests run. -## Use repo's zelta -REPO_ROOT="$SHELLSPEC_PROJECT_ROOT" -export PATH="$REPO_ROOT/bin:$PATH" -export ZELTA_SHARE="$REPO_ROOT/share/zelta" +## Setup temporary installation for testing +############################################# + +export SANDBOX_ZELTA_TMP_DIR="/tmp/zelta$$" +export ZELTA_BIN="$SANDBOX_ZELTA_TMP_DIR/bin" +export ZELTA_SHARE="$SANDBOX_ZELTA_TMP_DIR/share" +export ZELTA_ETC="$SANDBOX_ZELTA_TMP_DIR/etc" +export ZELTA_DOC="$SANDBOX_ZELTA_TMP_DIR/man" +export PATH="$ZELTA_BIN:$PATH" + + +# We could use the repo dirs, but better to test installation +# use_repo_zelta() { +# REPO_ROOT="$SHELLSPEC_PROJECT_ROOT" +# export PATH="$REPO_ROOT/bin:$PATH" +# export ZELTA_SHARE="$REPO_ROOT/share/zelta" +# } ## Build endpoints if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then @@ -60,6 +73,22 @@ tgt_exec() { ## Helpers ########## +# Make sure the installer worked and clean up carefully +cleanup_temp_install() { + find "$SANDBOX_ZELTA_TMP_DIR" -type f | wc -w + if [ -d "$SANDBOX_ZELTA_TMP_DIR" ]; then + rm "$ZELTA_ETC"/zelta.* + rmdir "$SANDBOX_ZELTA_TMP_DIR"/* + rmdir "$SANDBOX_ZELTA_TMP_DIR" + [ ! -e "SANDBOX_ZELTA_TMP_DIR" ] && return 0 + fi + return 1 +} + +skip_if_root() { + [ "$(id -u)" -eq 0 ] +} + # Check if source pool is a prefix of source dataset src_pool_matches_ds() { case "$SANDBOX_ZELTA_SRC_DS" in From 8c6b17eb42b56e038dfe216dd3d438a3913b51e3 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 22 Jan 2026 21:50:12 -0500 Subject: [PATCH 23/47] feat: add phase parameter to uninstall.sh for selective removal --- uninstall.sh | 122 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/uninstall.sh b/uninstall.sh index 59d1acc..0f74775 100644 --- a/uninstall.sh +++ b/uninstall.sh @@ -5,6 +5,27 @@ # Removes Zelta installation files, including legacy paths. # Checks both system-wide and user installation locations. +usage() { + cat <<-EOF + Usage: uninstall.sh [PHASE...] + + Remove Zelta installation files from specified locations. + + PHASES: + system Remove system-wide installation (/usr/local/*) + home Remove default user installation (~/.local/*, ~/bin) + env Remove custom installation from ZELTA_* environment variables + all Remove from all locations (default if no phase specified) + + EXAMPLES: + uninstall.sh # Remove from all locations + uninstall.sh system # Remove only system-wide installation + uninstall.sh home env # Remove user and custom installations + + EOF + exit 0 +} + remove_if_exists() { if [ -L "$1" ] || [ -f "$1" ] && [ -w "$1" ]; then echo "- removing: $1" @@ -136,43 +157,88 @@ zelta_tidy() { fi } +# Parse arguments +run_system=0 +run_home=0 +run_env=0 + +if [ $# -eq 0 ]; then + # No arguments: run all phases (default behavior) + run_system=1 + run_home=1 + run_env=1 +else + # Parse phase arguments + for arg in "$@"; do + case "$arg" in + system) + run_system=1 + ;; + home) + run_home=1 + ;; + env) + run_env=1 + ;; + all) + run_system=1 + run_home=1 + run_env=1 + ;; + -h|--help|help) + usage + ;; + *) + echo "Error: Unknown phase '$arg'" + echo "Run 'uninstall.sh --help' for usage information." + exit 1 + ;; + esac + done +fi + echo "Uninstalling Zelta" echo "==================" echo # Phase 1: System-wide installation (if root or accessible) -zelta_tidy "System-wide" \ - "/usr/local/bin" \ - "/usr/local/sbin" \ - "/usr/local/share/zelta" \ - "/usr/local/etc/zelta" \ - "/usr/local/man" \ - "/usr/local/share/man" - +if [ $run_system -eq 1 ]; then + zelta_tidy "System-wide" \ + "/usr/local/bin" \ + "/usr/local/sbin" \ + "/usr/local/share/zelta" \ + "/usr/local/etc/zelta" \ + "/usr/local/man" \ + "/usr/local/share/man" +fi # Phase 2: Default user installation -zelta_tidy "User default" \ - "$HOME/bin" \ - "" \ - "$HOME/.local/share/zelta" \ - "$HOME/.config/zelta" \ - "$HOME/.local/share/zelta/doc" \ - "$HOME/.local/share/man" +if [ $run_home -eq 1 ]; then + zelta_tidy "User default" \ + "$HOME/bin" \ + "" \ + "$HOME/.local/share/zelta" \ + "$HOME/.config/zelta" \ + "$HOME/.local/share/zelta/doc" \ + "$HOME/.local/share/man" +fi # Phase 3: Custom paths from environment (if set and different) -if [ -n "$ZELTA_BIN" ] || [ -n "$ZELTA_SHARE" ] || [ -n "$ZELTA_ETC" ]; then - custom_bin="${ZELTA_BIN:-$HOME/bin}" - custom_share="${ZELTA_SHARE:-$HOME/.local/share/zelta}" - custom_etc="${ZELTA_ETC:-$HOME/.config/zelta}" - custom_doc="${ZELTA_DOC:-$custom_share/doc}" - -zelta_tidy "Custom environment" \ - "$custom_bin" \ - "" \ - "$custom_share" \ - "$custom_etc" \ - "$custom_doc" \ - "" +if [ $run_env -eq 1 ]; then + if [ -n "$ZELTA_BIN" ] || [ -n "$ZELTA_SHARE" ] || [ -n "$ZELTA_ETC" ]; then + custom_bin="${ZELTA_BIN:-$HOME/bin}" + custom_share="${ZELTA_SHARE:-$HOME/.local/share/zelta}" + custom_etc="${ZELTA_ETC:-$HOME/.config/zelta}" + custom_doc="${ZELTA_DOC:-$custom_share/doc}" + + zelta_tidy "Custom environment" \ + "$custom_bin" \ + "" \ + "$custom_share" \ + "$custom_etc" \ + "$custom_doc" \ + "" + fi fi echo From c2ac79b09577c7bf34e90ed7af926863a2461876 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 22 Jan 2026 22:01:02 -0500 Subject: [PATCH 24/47] make uninstaller require explicit 'phase' option --- uninstall.sh | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/uninstall.sh b/uninstall.sh index 0f74775..3d23390 100644 --- a/uninstall.sh +++ b/uninstall.sh @@ -7,21 +7,21 @@ usage() { cat <<-EOF - Usage: uninstall.sh [PHASE...] - + usage: uninstall.sh [PHASE...] + Remove Zelta installation files from specified locations. - + PHASES: system Remove system-wide installation (/usr/local/*) home Remove default user installation (~/.local/*, ~/bin) env Remove custom installation from ZELTA_* environment variables - all Remove from all locations (default if no phase specified) - + all Remove from all locations + EXAMPLES: - uninstall.sh # Remove from all locations + uninstall.sh all # Remove from all locations uninstall.sh system # Remove only system-wide installation uninstall.sh home env # Remove user and custom installations - + EOF exit 0 } @@ -48,7 +48,7 @@ find_zelta_symlinks() { if [ ! -d "$search_dir" ]; then return fi - + for link in "$search_dir"/*; do if [ -L "$link" ] && [ -e "$link" ]; then target=$(readlink "$link") @@ -66,7 +66,7 @@ is_in_git_repo() { if [ ! -e "$check_path" ]; then return 1 fi - + # Check if path or any parent is a git repo current="$check_path" while [ "$current" != "/" ] && [ "$current" != "." ]; do @@ -86,8 +86,8 @@ zelta_tidy() { etc_path="$5" doc_path="$6" legacy_doc="$7" - - + + # Check write permissions for at least one location can_write=0 for check_path in "$bin_path" "$share_path" "$etc_path"; do @@ -96,26 +96,26 @@ zelta_tidy() { break fi done - + if [ $can_write -eq 0 ] && [ -d "$bin_path" -o -d "$share_path" -o -d "$etc_path" ]; then return fi - + # Protect git repositories if is_in_git_repo "$share_path"; then return fi - + echo "$phase_name:" # Remove zelta binaries remove_if_exists "$bin_path/zelta" [ -n "$sbin_path" ] && remove_if_exists "$sbin_path/zelta" - + # Remove symlinks find_zelta_symlinks "$bin_path" find_zelta_symlinks "$sbin_path" - + # Remove man pages (current location) for manpath in "$doc_path" "$legacy_doc"; do for file in "$manpath"/zelta*; do @@ -130,7 +130,7 @@ zelta_tidy() { done remove_dir_if_exists "$manpath" done - + # Remove share directory if [ -d "$share_path" ] && ! is_in_git_repo "$share_path"; then for share_file in "$share_path"/zelta-*; do @@ -138,12 +138,12 @@ zelta_tidy() { done remove_dir_if_exists "$share_path" fi - + # Remove sample configs (but preserve user configs) if [ -d "$etc_path" ]; then remove_if_exists "$etc_path/zelta.conf.example" remove_if_exists "$etc_path/zelta.env.example" - + # Check for user configs if [ -w "$etc_path/zelta.conf" ] || [ -w "$etc_path/zelta.env" ]; then echo "- User configuration files preserved in $etc_path" @@ -164,9 +164,8 @@ run_env=0 if [ $# -eq 0 ]; then # No arguments: run all phases (default behavior) - run_system=1 - run_home=1 - run_env=1 + usage + exit 1 else # Parse phase arguments for arg in "$@"; do @@ -189,8 +188,8 @@ else usage ;; *) - echo "Error: Unknown phase '$arg'" - echo "Run 'uninstall.sh --help' for usage information." + echo "error: unknown phase '$arg'" + usage exit 1 ;; esac @@ -230,7 +229,7 @@ if [ $run_env -eq 1 ]; then custom_share="${ZELTA_SHARE:-$HOME/.local/share/zelta}" custom_etc="${ZELTA_ETC:-$HOME/.config/zelta}" custom_doc="${ZELTA_DOC:-$custom_share/doc}" - + zelta_tidy "Custom environment" \ "$custom_bin" \ "" \ From 1117307c281b4d4f5d0c42717a5d07d957035362 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Thu, 22 Jan 2026 22:04:01 -0500 Subject: [PATCH 25/47] make uninstaller require explicit 'phase' option --- test/99_cleanup_spec.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/99_cleanup_spec.sh b/test/99_cleanup_spec.sh index fc0a739..edc6c37 100644 --- a/test/99_cleanup_spec.sh +++ b/test/99_cleanup_spec.sh @@ -11,11 +11,9 @@ Describe 'Cleanup' The status should be success End End - Describe 'Installation cleanup' It 'uninstall script' - Skip if 'avoid systemwide uninstall' skip_if_root - When run sh uninstall.sh + When run sh uninstall.sh env The status should be success The output should include 'removing' End From e207a52169028485dc2f06616edfcc32cfd429c5 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 09:59:11 -0500 Subject: [PATCH 26/47] spec test improvements --- share/zelta/zelta-backup.awk | 5 ++-- test/00_install_spec.sh | 22 +++------------- test/01_no_op_spec.sh | 2 +- test/02_setup_spec.sh | 50 +++++++++++++++++++++++++++++++++--- test/03_backup_spec.sh | 21 +++++++++++++++ test/03_divergent_spec.sh | 24 ----------------- test/99_cleanup_spec.sh | 4 +-- test/test_helper.sh | 32 +++++++++++++++++++++++ 8 files changed, 108 insertions(+), 52 deletions(-) create mode 100644 test/03_backup_spec.sh delete mode 100644 test/03_divergent_spec.sh diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index b9c74f6..6534de8 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -1028,9 +1028,10 @@ function run_revert( _ds) { } # 'zelta backup' and 'zelta sync' orchestration -function run_backup( _i, _ds_suffix, _syncable) { +function run_backup( _i, _ds_suffix, _syncing, _syncable) { + _syncing = Opt["DRYRUN"] ? "would sync " : "syncing " if (DSTree["syncable"]) - report(LOG_NOTICE, "syncing " NumDS " datasets") + report(LOG_NOTICE, _syncing NumDS " datasets") for (_i = 1; _i <= NumDS; _i++) { _ds_suffix = DSList[_i] # Run first pass sync diff --git a/test/00_install_spec.sh b/test/00_install_spec.sh index 256ae09..3f73acf 100644 --- a/test/00_install_spec.sh +++ b/test/00_install_spec.sh @@ -1,4 +1,4 @@ -# shellcheck shell=sh +# Install Zelta Describe 'Zelta installation' It 'runs installer without errors' @@ -6,24 +6,8 @@ Describe 'Zelta installation' The status should be success The output should include 'installing' End - - It 'installs zelta binary' - When run test -x "$ZELTA_BIN/zelta" - The status should be success - End - - It 'installs share files' - When run test -f "$ZELTA_SHARE/zelta-common.awk" - The status should be success - End - - It 'installs man pages' - When run test -f "$ZELTA_DOC/man8/zelta.8" - The status should be success - End - - It 'creates config examples' - When run test -f "$ZELTA_ETC/zelta.conf.example" + It 'check installed files' + When call check_install The status should be success End End diff --git a/test/01_no_op_spec.sh b/test/01_no_op_spec.sh index 64ba65c..701e2a8 100644 --- a/test/01_no_op_spec.sh +++ b/test/01_no_op_spec.sh @@ -1,4 +1,4 @@ -# shellcheck shell=sh +# Check Zelta usage, help, version, and `zelta match` option processing Describe 'Zelta no-op command checks' Describe 'zelta command' diff --git a/test/02_setup_spec.sh b/test/02_setup_spec.sh index b800a04..5b1b09c 100644 --- a/test/02_setup_spec.sh +++ b/test/02_setup_spec.sh @@ -1,12 +1,54 @@ +# Check remotes and create pools and datasets + +Describe 'Remote check' + It 'source accessible' + Skip if 'SANDBOX_ZELTA_SRC_REMOTE undefined' [ -z "$SANDBOX_ZELTA_SRC_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" true + The status should be success + The error should not include 'Warning' + End + It 'target accessible' + Skip if 'SANDBOX_ZELTA_TGT_REMOTE undefined' [ -z "$SANDBOX_ZELTA_TGT_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" true + The status should be success + The error should not include 'Warning' + End +End + Describe 'Pool setup' - It 'create' - Skip if 'source not defined' skip_src_pool + It 'create source' + Skip if 'SANDBOX_ZELTA_SRC_POOL undefined' [ -z "$SANDBOX_ZELTA_SRC_POOL" ] When call make_src_pool The status should be success End - It 'create' - Skip if 'target not defined' skip_tgt_pool + It 'create target' + Skip if 'SANDBOX_ZELTA_TGT_POOL undefined' [ -z "$SANDBOX_ZELTA_TGT_POOL" ] When call make_tgt_pool The status should be success End End + +Describe 'Divergent tree setup' + It 'creates divergent tree on source' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + When call make_divergent_tree + The status should be success + The output should include 'snapshotting' + The output should include 'syncing 9 datasets' + The error should not include 'error:' + End +End + +Describe 'Divergent tree match' + It 'shows expected divergence types' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The status should be success + The output should include 'up-to-date' + The output should include 'syncable (full)' + The output should include 'syncable (incremental)' + The output should include 'blocked sync: target diverged' + The output should include 'blocked sync: no target snapshots' + The output should include '11 total datasets compared' + End +End diff --git a/test/03_backup_spec.sh b/test/03_backup_spec.sh new file mode 100644 index 0000000..8f1ed3b --- /dev/null +++ b/test/03_backup_spec.sh @@ -0,0 +1,21 @@ +Describe 'Backup tests' + It 'no-op all options' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + When call backup_no_op_check + The status should be success + # In json mode, all unsuppressed notices will be stderr + The error should include '+ ' + The error should include 'sub4@start' + The error should not include 'sub3' + The error should include '@test' + # Check json + The output should include 'output_version' + End + It 'valid json' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + Skip if 'jq required' test -z "$(command -v jq)" + When call backup_check_json + The status should be success + The output should include 'zelta backup' + End +End diff --git a/test/03_divergent_spec.sh b/test/03_divergent_spec.sh deleted file mode 100644 index 5d780c7..0000000 --- a/test/03_divergent_spec.sh +++ /dev/null @@ -1,24 +0,0 @@ -Describe 'Divergent tree setup' - It 'creates divergent tree on source' - Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" - When call make_divergent_tree - The status should be success - The output should include 'snapshotting' - The output should include 'syncing 9 datasets' - The error should not include 'error:' - End -End - -Describe 'Divergent tree match' - It 'shows expected divergence types' - Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" - When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" - The status should be success - The output should include 'up-to-date' - The output should include 'syncable (full)' - The output should include 'syncable (incremental)' - The output should include 'blocked sync: target diverged' - The output should include 'blocked sync: no target snapshots' - The output should include '11 total datasets compared' - End -End diff --git a/test/99_cleanup_spec.sh b/test/99_cleanup_spec.sh index edc6c37..86867eb 100644 --- a/test/99_cleanup_spec.sh +++ b/test/99_cleanup_spec.sh @@ -1,12 +1,12 @@ Describe 'Cleanup' Describe 'Pool cleanup' It 'destroy source' - Skip if 'no pools defined' skip_pools + Skip if 'SANDBOX_ZELTA_SRC_POOL undefined' [ -z "$SANDBOX_ZELTA_SRC_POOL" ] When call nuke_src_pool The status should be success End It 'destroy target' - Skip if 'no pools defined' skip_pools + Skip if 'SANDBOX_ZELTA_TGT_POOL undefined' [ -z "$SANDBOX_ZELTA_TGT_POOL" ] When call nuke_tgt_pool The status should be success End diff --git a/test/test_helper.sh b/test/test_helper.sh index 1aed1f4..aba349a 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -73,6 +73,28 @@ tgt_exec() { ## Helpers ########## +check_install() { + _installed=1 + if [ ! -x "$ZELTA_BIN/zelta" ]; then + echo missing: "$ZELTA_BIN/zelta" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_SHARE/zelta-common.awk" ]; then + echo missing: "$ZELTA_SHARE/zelta-common.awk" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_DOC/man8/zelta.8" ]; then + echo missing: "$ZELTA_DOC/man8/zelta.8" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_ETC/zelta.conf.example" ]; then + echo missing: "$ZELTA_ETC/zelta.conf.example" >/dev/stderr + _installed=0 + fi + [ $_installed = 1 ] && return 0 + return 1 +} + # Make sure the installer worked and clean up carefully cleanup_temp_install() { find "$SANDBOX_ZELTA_TMP_DIR" -type f | wc -w @@ -89,6 +111,16 @@ skip_if_root() { [ "$(id -u)" -eq 0 ] } +backup_no_op_check() { + _options_all="-v --verbose -q --quiet --log-level=2 --log-mode=json --dryrun -n --depth 2 -d2 -X/sub3" + _options_backup="--json -j --resume --snap-name=@test --snapshot --pull -i -o compression=zstd -x mountpoint" + zelta backup $_options_all $_options_backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" +} + +backup_check_json() { + backup_no_op_check 2>/dev/null | jq -re .output_version.command +} + # Check if source pool is a prefix of source dataset src_pool_matches_ds() { case "$SANDBOX_ZELTA_SRC_DS" in From aecb42a0c24aa25f0399d1e7b8989d2ddf425c1a Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 09:59:11 -0500 Subject: [PATCH 27/47] spec test improvements test: add marker files to track pool creation in tests test: add process ID to marker files and dataset cleanup spec test improvement to only remove pool and trees if we created them --- share/zelta/zelta-backup.awk | 5 +-- test/00_install_spec.sh | 22 ++---------- test/01_no_op_spec.sh | 2 +- test/02_setup_spec.sh | 47 ++++++++++++++++++++++--- test/03_backup_spec.sh | 21 +++++++++++ test/03_divergent_spec.sh | 24 ------------- test/99_cleanup_spec.sh | 16 +++++++-- test/test_helper.sh | 68 ++++++++++++++++++++++++++++++++---- 8 files changed, 146 insertions(+), 59 deletions(-) create mode 100644 test/03_backup_spec.sh delete mode 100644 test/03_divergent_spec.sh diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index b9c74f6..6534de8 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -1028,9 +1028,10 @@ function run_revert( _ds) { } # 'zelta backup' and 'zelta sync' orchestration -function run_backup( _i, _ds_suffix, _syncable) { +function run_backup( _i, _ds_suffix, _syncing, _syncable) { + _syncing = Opt["DRYRUN"] ? "would sync " : "syncing " if (DSTree["syncable"]) - report(LOG_NOTICE, "syncing " NumDS " datasets") + report(LOG_NOTICE, _syncing NumDS " datasets") for (_i = 1; _i <= NumDS; _i++) { _ds_suffix = DSList[_i] # Run first pass sync diff --git a/test/00_install_spec.sh b/test/00_install_spec.sh index 256ae09..3f73acf 100644 --- a/test/00_install_spec.sh +++ b/test/00_install_spec.sh @@ -1,4 +1,4 @@ -# shellcheck shell=sh +# Install Zelta Describe 'Zelta installation' It 'runs installer without errors' @@ -6,24 +6,8 @@ Describe 'Zelta installation' The status should be success The output should include 'installing' End - - It 'installs zelta binary' - When run test -x "$ZELTA_BIN/zelta" - The status should be success - End - - It 'installs share files' - When run test -f "$ZELTA_SHARE/zelta-common.awk" - The status should be success - End - - It 'installs man pages' - When run test -f "$ZELTA_DOC/man8/zelta.8" - The status should be success - End - - It 'creates config examples' - When run test -f "$ZELTA_ETC/zelta.conf.example" + It 'check installed files' + When call check_install The status should be success End End diff --git a/test/01_no_op_spec.sh b/test/01_no_op_spec.sh index 64ba65c..701e2a8 100644 --- a/test/01_no_op_spec.sh +++ b/test/01_no_op_spec.sh @@ -1,4 +1,4 @@ -# shellcheck shell=sh +# Check Zelta usage, help, version, and `zelta match` option processing Describe 'Zelta no-op command checks' Describe 'zelta command' diff --git a/test/02_setup_spec.sh b/test/02_setup_spec.sh index b800a04..612ebb9 100644 --- a/test/02_setup_spec.sh +++ b/test/02_setup_spec.sh @@ -1,12 +1,51 @@ +# Check remotes and create pools and datasets + +Describe 'Remote check' + It 'source accessible' + Skip if 'SANDBOX_ZELTA_SRC_REMOTE undefined' [ -z "$SANDBOX_ZELTA_SRC_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" true + The status should be success + End + It 'target accessible' + Skip if 'SANDBOX_ZELTA_TGT_REMOTE undefined' [ -z "$SANDBOX_ZELTA_TGT_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" true + The status should be success + End +End + Describe 'Pool setup' - It 'create' - Skip if 'source not defined' skip_src_pool + It 'create source' + Skip if 'SANDBOX_ZELTA_SRC_POOL undefined' [ -z "$SANDBOX_ZELTA_SRC_POOL" ] When call make_src_pool The status should be success End - It 'create' - Skip if 'target not defined' skip_tgt_pool + It 'create target' + Skip if 'SANDBOX_ZELTA_TGT_POOL undefined' [ -z "$SANDBOX_ZELTA_TGT_POOL" ] When call make_tgt_pool The status should be success End End + +Describe 'Divergent tree setup' + It 'creates divergent tree on source' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" -a -z "$SANDBOX_ZELTA_TGT_DS" + When call make_divergent_tree + The status should be success + The output should include 'syncing 9 datasets' + The error should not include 'error:' + End +End + +Describe 'Divergent tree match' + It 'shows expected divergence types' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The status should be success + The output should include 'up-to-date' + The output should include 'syncable (full)' + The output should include 'syncable (incremental)' + The output should include 'blocked sync: target diverged' + The output should include 'blocked sync: no target snapshots' + The output should include '11 total datasets compared' + End +End diff --git a/test/03_backup_spec.sh b/test/03_backup_spec.sh new file mode 100644 index 0000000..8f1ed3b --- /dev/null +++ b/test/03_backup_spec.sh @@ -0,0 +1,21 @@ +Describe 'Backup tests' + It 'no-op all options' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + When call backup_no_op_check + The status should be success + # In json mode, all unsuppressed notices will be stderr + The error should include '+ ' + The error should include 'sub4@start' + The error should not include 'sub3' + The error should include '@test' + # Check json + The output should include 'output_version' + End + It 'valid json' + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + Skip if 'jq required' test -z "$(command -v jq)" + When call backup_check_json + The status should be success + The output should include 'zelta backup' + End +End diff --git a/test/03_divergent_spec.sh b/test/03_divergent_spec.sh deleted file mode 100644 index 5d780c7..0000000 --- a/test/03_divergent_spec.sh +++ /dev/null @@ -1,24 +0,0 @@ -Describe 'Divergent tree setup' - It 'creates divergent tree on source' - Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" - When call make_divergent_tree - The status should be success - The output should include 'snapshotting' - The output should include 'syncing 9 datasets' - The error should not include 'error:' - End -End - -Describe 'Divergent tree match' - It 'shows expected divergence types' - Skip if 'source dataset not configured' test -z "$SANDBOX_ZELTA_SRC_DS" - When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" - The status should be success - The output should include 'up-to-date' - The output should include 'syncable (full)' - The output should include 'syncable (incremental)' - The output should include 'blocked sync: target diverged' - The output should include 'blocked sync: no target snapshots' - The output should include '11 total datasets compared' - End -End diff --git a/test/99_cleanup_spec.sh b/test/99_cleanup_spec.sh index edc6c37..9314336 100644 --- a/test/99_cleanup_spec.sh +++ b/test/99_cleanup_spec.sh @@ -1,12 +1,24 @@ Describe 'Cleanup' + Describe 'Dataset cleanup' + It 'destroy source dataset' + Skip if 'dataset not created in this run' tmpfile_check divergent_tree_created + When call clean_src_ds + The status should be success + End + It 'destroy target dataset' + Skip if 'dataset not created in this run' tmpfile_check divergent_tree_created + When call clean_tgt_ds + The status should be success + End + End Describe 'Pool cleanup' It 'destroy source' - Skip if 'no pools defined' skip_pools + Skip if 'pool not created in this run' tmpfile_check src_pool_created When call nuke_src_pool The status should be success End It 'destroy target' - Skip if 'no pools defined' skip_pools + Skip if 'pool not created in this run' tmpfile_check tgt_pool_created When call nuke_tgt_pool The status should be success End diff --git a/test/test_helper.sh b/test/test_helper.sh index 1aed1f4..3c2ba52 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -16,6 +16,7 @@ ############################################# export SANDBOX_ZELTA_TMP_DIR="/tmp/zelta$$" +export SANDBOX_ZELTA_PROCNUM="$$" export ZELTA_BIN="$SANDBOX_ZELTA_TMP_DIR/bin" export ZELTA_SHARE="$SANDBOX_ZELTA_TMP_DIR/share" export ZELTA_ETC="$SANDBOX_ZELTA_TMP_DIR/etc" @@ -73,6 +74,28 @@ tgt_exec() { ## Helpers ########## +check_install() { + _installed=1 + if [ ! -x "$ZELTA_BIN/zelta" ]; then + echo missing: "$ZELTA_BIN/zelta" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_SHARE/zelta-common.awk" ]; then + echo missing: "$ZELTA_SHARE/zelta-common.awk" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_DOC/man8/zelta.8" ]; then + echo missing: "$ZELTA_DOC/man8/zelta.8" >/dev/stderr + _installed=0 + fi + if [ ! -f "$ZELTA_ETC/zelta.conf.example" ]; then + echo missing: "$ZELTA_ETC/zelta.conf.example" >/dev/stderr + _installed=0 + fi + [ $_installed = 1 ] && return 0 + return 1 +} + # Make sure the installer worked and clean up carefully cleanup_temp_install() { find "$SANDBOX_ZELTA_TMP_DIR" -type f | wc -w @@ -85,10 +108,28 @@ cleanup_temp_install() { return 1 } +tmpfile_touch() { + touch "${SHELLSPEC_TMPBASE}/${1}_${SANDBOX_ZELTA_PROCNUM}" +} + +tmpfile_check() { + [ ! -f "${SHELLSPEC_TMPBASE}/${1}_${SANDBOX_ZELTA_PROCNUM}" ] +} + skip_if_root() { [ "$(id -u)" -eq 0 ] } +backup_no_op_check() { + _options_all="-v --verbose -q --quiet --log-level=2 --log-mode=json --dryrun -n --depth 2 -d2 -X/sub3" + _options_backup="--json -j --resume --snap-name=@test --snapshot --pull -i -o compression=zstd -x mountpoint" + zelta backup $_options_all $_options_backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" +} + +backup_check_json() { + backup_no_op_check 2>/dev/null | jq -re .output_version.command +} + # Check if source pool is a prefix of source dataset src_pool_matches_ds() { case "$SANDBOX_ZELTA_SRC_DS" in @@ -147,7 +188,7 @@ make_pool() { _pool_name="$1" _exec_func="$2" _pool_file=$($_exec_func pwd)/$_pool_name.img - nuke_pool "$_pool_name" "$_exec_func" + #nuke_pool "$_pool_name" "$_exec_func" $_exec_func truncate -s 1G "$_pool_file" $_exec_func zpool create -f "$_pool_name" "$_pool_file" return $? @@ -165,7 +206,8 @@ nuke_tgt_pool() { make_src_pool() { make_pool "$SANDBOX_ZELTA_SRC_POOL" src_exec || return 1 - + tmpfile_touch "src_pool_created" + # Grant ZFS permissions for source pool if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then #ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" @@ -178,6 +220,7 @@ make_src_pool() { make_tgt_pool() { make_pool "$SANDBOX_ZELTA_TGT_POOL" tgt_exec || return 1 + tmpfile_touch "tgt_pool_created" # Grant ZFS permissions for target pool if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then @@ -204,6 +247,7 @@ tgt_ds_exists() { # Clean source dataset if it exists clean_src_ds() { if src_ds_exists; then + src_exec rm -f /tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM} src_exec zfs destroy -r "$SANDBOX_ZELTA_SRC_DS" return $? fi @@ -222,12 +266,22 @@ clean_tgt_ds() { # Create divergent tree structure on source # Creates a dataset tree with snapshots that will diverge from target make_divergent_tree() { - clean_src_ds || return 1 - clean_tgt_ds || return 1 + if src_ds_exists; then + echo "$SANDBOX_ZELTA_TGT_DS" already exists >/dev/stderr + return 1 + fi + if tgt_ds_exists; then + echo "$SANDBOX_ZELTA_SRC_DS" already exists >/dev/stderr + return 1 + fi + # If we get this far, it will be safe to attempt to clean it up + tmpfile_touch "divergent_tree_created" + # Create encryption key - src_exec dd if=/dev/urandom bs=32 count=1 of=/tmp/zfs_test_enc_key >/dev/null 2>&1 || return 1 - + src_exec dd if=/dev/urandom bs=32 count=1 of="/tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" >/dev/null 2>&1 || return 1 + + # Create root dataset src_exec zfs create "$SANDBOX_ZELTA_SRC_DS" || return 1 @@ -239,7 +293,7 @@ make_divergent_tree() { src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 src_exec zfs create -sV 100M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 - src_exec zfs create -o encryption=on -o keyformat=raw -o keylocation=file:///tmp/zfs_test_enc_key "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 + src_exec zfs create -o encryption=on -o keyformat=raw -o "keylocation=file:///tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 # Replicate to target with @start snapshot zelta backup --snap-name @start "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" || return 1 From ae64432093d5c767245a06e68f7e0e8b4e1f8ee0 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 12:11:48 -0500 Subject: [PATCH 28/47] set dryrun snapshot message to 'would snapshot' --- share/zelta/zelta-backup.awk | 14 +++++++++----- test/test_helper.sh | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 6534de8..cf724ee 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -391,20 +391,24 @@ function compute_eligibility( _i, _ds_suffix, _src_idx, _tgt_idx, ###################################### # Decide whether or not to take a snapshot; if so, returns a reason -function should_snapshot() { +function should_snapshot( _snapshotting) { # Only attempt a snapshot once if (DSTree["snapshot_attempted"]) return + if (Opt["DRYRUN"]) + _snapshotting = "would snapshot: " + else + _snapshotting = "snapshotting: " # Snapshot mode is "ALWAYS" or provide a reason else if (Opt["SNAP_MODE"] == "ALWAYS") - return "snapshotting: " + return _snapshotting else if (Opt["SNAP_MODE"] != "IF_NEEDED") return 0 else if (DSTree["snapshot_needed"] == SNAP_WRITTEN) - return "source is written; snapshotting: " + return "source is written; " _snapshotting else if (DSTree["snapshot_needed"] == SNAP_MISSING) - return "missing source snapshot; snapshotting: " + return "missing source snapshot; " _snapshotting else if (DSTree["snapshot_needed"] == SNAP_LATEST) - return "action requires a snapshot delta; snapshotting: " + return "action requires a snapshot delta; " _snapshotting else return 0 } diff --git a/test/test_helper.sh b/test/test_helper.sh index 3c2ba52..5b3f2d6 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -103,7 +103,7 @@ cleanup_temp_install() { rm "$ZELTA_ETC"/zelta.* rmdir "$SANDBOX_ZELTA_TMP_DIR"/* rmdir "$SANDBOX_ZELTA_TMP_DIR" - [ ! -e "SANDBOX_ZELTA_TMP_DIR" ] && return 0 + [ ! -e "$SANDBOX_ZELTA_TMP_DIR" ] && return 0 fi return 1 } From 8f990a5881fefb893a281fc85d3d7bfde72af862 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 13:49:53 -0500 Subject: [PATCH 29/47] extra else in should_snapshot --- share/zelta/zelta-backup.awk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index cf724ee..db1c2c8 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -398,8 +398,9 @@ function should_snapshot( _snapshotting) { _snapshotting = "would snapshot: " else _snapshotting = "snapshotting: " + # Snapshot mode is "ALWAYS" or provide a reason - else if (Opt["SNAP_MODE"] == "ALWAYS") + if (Opt["SNAP_MODE"] == "ALWAYS") return _snapshotting else if (Opt["SNAP_MODE"] != "IF_NEEDED") return 0 From 7777a0cffdb2dcf2ad3eb9f7adf3daf7385ad502 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 15:00:24 -0500 Subject: [PATCH 30/47] fix local spec test run --- test/test_helper.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index 5b3f2d6..2ced963 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -178,7 +178,7 @@ skip_tgt_pool() { nuke_pool() { _pool_name="$1" _exec_func="$2" - _pool_file=$($_exec_func pwd)/$_pool_name.img + _pool_file=/tmp/$_pool_name.img $_exec_func zpool destroy -f "$_pool_name" >/dev/null 2>&1 $_exec_func rm -f "$_pool_file" return 0 @@ -187,9 +187,8 @@ nuke_pool() { make_pool() { _pool_name="$1" _exec_func="$2" - _pool_file=$($_exec_func pwd)/$_pool_name.img - #nuke_pool "$_pool_name" "$_exec_func" - $_exec_func truncate -s 1G "$_pool_file" + _pool_file=/tmp/$_pool_name.img + $_exec_func truncate -s 265m "$_pool_file" $_exec_func zpool create -f "$_pool_name" "$_pool_file" return $? } @@ -292,7 +291,7 @@ make_divergent_tree() { src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3" || return 1 src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 - src_exec zfs create -sV 100M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 + src_exec zfs create -sV 32M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 src_exec zfs create -o encryption=on -o keyformat=raw -o "keylocation=file:///tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 # Replicate to target with @start snapshot From 24289a6d9d4d0834f6d5b7b4ab84d26c927d6606 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 16:48:46 -0500 Subject: [PATCH 31/47] fix zfs get depth command order --- share/zelta/zelta-backup.awk | 9 +++++---- share/zelta/zelta-cmds.tsv | 2 +- test/test_helper.sh | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index db1c2c8..477f9da 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -136,10 +136,11 @@ function check_snapshot_needed(endpoint, ds_suffix, prop_key, prop_val) { # Load zfs properties for an endpoint function load_properties(ep, _ds, _cmd_arr, _cmd, _cmd_id, _ds_suffix, _idx, _seen, _log_level) { - _cmd_id = "zfs get" - _ds = Opt[ep "_DS"] - _cmd_arr["endpoint"] = ep - _cmd_arr["ds"] = rq(Opt[ep"_REMOTE"],_ds) + _cmd_id = "zfs get" + _ds = Opt[ep "_DS"] + _cmd_arr["endpoint"] = ep + _cmd_arr["ds"] = rq(Opt[ep"_REMOTE"],_ds) + _cmd_arr["props"] = "all" if (Opt["DEPTH"]) _cmd_arr["flags"] = "-d" (Opt["DEPTH"]-1) _cmd = build_command("PROPS", _cmd_arr) report(LOG_INFO, "checking properties for " Opt[ep"_ID"]) diff --git a/share/zelta/zelta-cmds.tsv b/share/zelta/zelta-cmds.tsv index 3f69e4d..9089921 100644 --- a/share/zelta/zelta-cmds.tsv +++ b/share/zelta/zelta-cmds.tsv @@ -9,7 +9,7 @@ # CHECK DEFAULT zfs list -Ho name ds LIST DEFAULT zfs list -Hpr -t all -Screatetxg -o props flags ds -PROPS DEFAULT zfs get -Hpr -s local,none -t filesystem,volume -o name,property,value all flags ds +PROPS DEFAULT zfs get -Hpr -s local,none -t filesystem,volume -o name,property,value flags props ds CREATE DEFAULT zfs create -vupo canmount=noauto ds SNAP DEFAULT zfs snapshot -r flags ds_snap MATCH zelta ipc-run match --time --log-mode=text --log-level=2 -Hpo relname,match,srcnext,srclast,tgtlast flags diff --git a/test/test_helper.sh b/test/test_helper.sh index 2ced963..ba16460 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -57,7 +57,7 @@ src_exec() { if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" sudo "$@" else - sudo sh -c "$*" + eval sudo "$@" fi } @@ -66,7 +66,7 @@ tgt_exec() { if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" sudo "$@" else - sudo sh -c "$*" + eval sudo "$@" fi } @@ -188,7 +188,7 @@ make_pool() { _pool_name="$1" _exec_func="$2" _pool_file=/tmp/$_pool_name.img - $_exec_func truncate -s 265m "$_pool_file" + $_exec_func truncate -s 64m "$_pool_file" $_exec_func zpool create -f "$_pool_name" "$_pool_file" return $? } @@ -273,7 +273,7 @@ make_divergent_tree() { echo "$SANDBOX_ZELTA_SRC_DS" already exists >/dev/stderr return 1 fi - + # If we get this far, it will be safe to attempt to clean it up tmpfile_touch "divergent_tree_created" @@ -291,7 +291,7 @@ make_divergent_tree() { src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3" || return 1 src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 - src_exec zfs create -sV 32M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 + src_exec zfs create -sV 8M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 src_exec zfs create -o encryption=on -o keyformat=raw -o "keylocation=file:///tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 # Replicate to target with @start snapshot From 5bb8f1f92feb8a13d54741d0f2247f9b4d3c1d87 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 17:29:49 -0500 Subject: [PATCH 32/47] style: fix typos and grammar in documentation and code --- README.md | 2 +- STYLE.md | 2 +- bin/zelta | 2 +- share/zelta/zelta-backup.awk | 8 ++++---- share/zelta/zelta-match.awk | 2 +- share/zelta/zelta-policy.awk | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3d0346e..82ee653 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ For commercial support, custom feature development, and consulting on secure, hi ## Roadmap -Zelta 1.1 represents a major refactor improving POSIX compliance, portability, and code maintainability. The following features are already used internally or by Bell Tower clients and will be upstreamed by Q2 2026. +Zelta 1.1 represents a major refactor that improves POSIX compliance, portability, and code maintainability. The following features are already used internally or by Bell Tower clients and will be upstreamed by Q2 2026. ### Features In Development diff --git a/STYLE.md b/STYLE.md index 40f7f0f..58c176b 100644 --- a/STYLE.md +++ b/STYLE.md @@ -2,7 +2,7 @@ This document defines the coding standards for the Zelta Backup and Recovery Suite. Zelta is designed for **portability** and **safety** and is written in portable Bourne Shell and Awk. -Lean towards POSIX and "Original Awk" standards, but don't adhere obsessively to any single standard. Some POSIX features are absent from certain OS defaults, so everything needs testing on real systems. +Lean toward POSIX and "Original Awk" standards, but don't adhere obsessively to any single standard. Some POSIX features are absent from certain OS defaults, so everything needs testing on real systems. --- diff --git a/bin/zelta b/bin/zelta index 7d8b0a7..fb12a1c 100755 --- a/bin/zelta +++ b/bin/zelta @@ -9,7 +9,7 @@ ZELTA_VERSION="Zelta 1.1.0" zelta_usage() { exec >&2 if [ -n "$1" ] ; then - echo unrecognized command \'"$1"\' + echo "unrecognized command: '$1'" case "$1" in list) echo did you mean "'zelta match'?" ;; send) echo did you mean "'zelta backup'?" ;; diff --git a/share/zelta/zelta-backup.awk b/share/zelta/zelta-backup.awk index 477f9da..404b048 100644 --- a/share/zelta/zelta-backup.awk +++ b/share/zelta/zelta-backup.awk @@ -256,7 +256,7 @@ function explain_sync_status(ds_suffix, _tgt_ds) { report(LOG_NOTICE, Action[ds_suffix, "block_reason"]": " _tgt_ds) } -# Ensure source snapshots are avialable and load snapshot relationship data +# Ensure source snapshots are available and load snapshot relationship data function validate_snapshots( _i, _ds_suffix, _src_idx, _match, _src_exists, _src_latest) { create_source_snapshot() load_snapshot_deltas() @@ -367,7 +367,7 @@ function compute_eligibility( _i, _ds_suffix, _src_idx, _tgt_idx, Action[_ds_suffix, "block_reason"] = "target has local writes" continue } - # TO-DO: Imrpove verbose output + # TO-DO: Improve verbose output #Action[_ds_suffix, "block_reason"] = "up-to-date" DSTree["up_to_date"]++ continue @@ -375,7 +375,7 @@ function compute_eligibility( _i, _ds_suffix, _src_idx, _tgt_idx, # Target is ahead or has diverged otherwise if (_match != _tgt_latest) { - Action[_ds_suffix, "block_reason"] = "target has divereged" + Action[_ds_suffix, "block_reason"] = "target has diverged" Action[_ds_suffix, "can_rotate"] = 1 DSTree["rotatable"]++ continue @@ -986,7 +986,7 @@ function run_rotate( _src_ds_snap, _up_to_date, _src_origin_ds, _origin_arr, _n load_snapshot_deltas() if (DSTree["snapshots_diverged"]) - report(LOG_NOTICE, "ensure preservation of diverged replica with: zelta backup " _src_origin_ds " " DSTree["target_origin"]) + report(LOG_NOTICE, "ensure the preservation of diverged replica with: zelta backup " _src_origin_ds " " DSTree["target_origin"]) report(LOG_NOTICE, "to ensure target is up-to-date, run: zelta backup " Source["ID"] " " Target["ID"]) } diff --git a/share/zelta/zelta-match.awk b/share/zelta/zelta-match.awk index 098acbe..cda7636 100644 --- a/share/zelta/zelta-match.awk +++ b/share/zelta/zelta-match.awk @@ -44,7 +44,7 @@ function usage_prune(message) { STDERR = "/dev/stderr" printf (message ? message "\n" : "") "usage:" > STDERR print "\tprune [--keep-snap-num=N] [--keep-snap-days=N] [-X pattern] SOURCE TARGET\n" > STDERR - print "Identifies snapshots on SOURCE that exist on TARGET\n" > STDERR + print "Identifies snapshots on SOURCE that exist on TARGET.\n" > STDERR print "Options:" > STDERR print "\t--keep-snap-num=N Minimum number of snapshots to keep after match (default: 10)" > STDERR print "\t--keep-snap-days=N Minimum age in days for snapshot deletion (default: 90)" > STDERR diff --git a/share/zelta/zelta-policy.awk b/share/zelta/zelta-policy.awk index 59bd35a..81ef1e5 100644 --- a/share/zelta/zelta-policy.awk +++ b/share/zelta/zelta-policy.awk @@ -83,7 +83,7 @@ function set_var(option_list, var, val) { var = toupper(var) if (var in PolicyLegacy) { # TO-DO: Legacy warning - report(LOG_DEBUG, "reassigning option '"var"' to '"PolicyLegacy[var]"'") + report(LOG_DEBUG, "reassigning the option '"var"' to '"PolicyLegacy[var]"'") var = PolicyLegacy[var] } if (!(var in PolicyOpt)) usage("unknown option: "var) @@ -165,7 +165,7 @@ function load_config( _conf_error, _arr, _context, _job, _line_num, if (split($0, _arr, "#")) { $0 = _arr[1] } - gsub(/[ \t]+$/, "", $0) + gsub(/[ ]+$/, "", $0) if (! $0) { continue } # Global options From 8a065bc8a9271dd074fd4e353f309e63fb02bc52 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 18:16:14 -0500 Subject: [PATCH 33/47] regex typo --- share/zelta/zelta-policy.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/zelta/zelta-policy.awk b/share/zelta/zelta-policy.awk index 81ef1e5..ff065bc 100644 --- a/share/zelta/zelta-policy.awk +++ b/share/zelta/zelta-policy.awk @@ -165,7 +165,7 @@ function load_config( _conf_error, _arr, _context, _job, _line_num, if (split($0, _arr, "#")) { $0 = _arr[1] } - gsub(/[ ]+$/, "", $0) + sub(/[ \t]+$/, "") if (! $0) { continue } # Global options From a0baef5fda4dead7720278f89cd0ea359714da88 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 23 Jan 2026 18:26:45 -0500 Subject: [PATCH 34/47] fix spec test output --- test/03_backup_spec.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/03_backup_spec.sh b/test/03_backup_spec.sh index 8f1ed3b..a377682 100644 --- a/test/03_backup_spec.sh +++ b/test/03_backup_spec.sh @@ -4,10 +4,11 @@ Describe 'Backup tests' When call backup_no_op_check The status should be success # In json mode, all unsuppressed notices will be stderr - The error should include '+ ' - The error should include 'sub4@start' - The error should not include 'sub3' - The error should include '@test' + The error should include 'would snapshot' + The error should include 'zfs snapshot' + The error should include 'diverged' + The error should not include 'snapshotting' + The error should not include 'error:' # Check json The output should include 'output_version' End From d6e13edceb3c3ded6ac8b3d8141b92d34d1f0b80 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Tue, 27 Jan 2026 20:42:09 -0500 Subject: [PATCH 35/47] experiment --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f3aefbe..1a4c208 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ hide.* *~ \#*# TAGS +A*d .#* retired/ From aa75e81a7424d03a2dd76e12695420365f5dcc05 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 28 Jan 2026 13:57:46 -0500 Subject: [PATCH 36/47] fix zelta prune usage -x should be -X --- share/zelta/zelta-match.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/zelta/zelta-match.awk b/share/zelta/zelta-match.awk index cda7636..eea4074 100644 --- a/share/zelta/zelta-match.awk +++ b/share/zelta/zelta-match.awk @@ -49,7 +49,7 @@ function usage_prune(message) { print "\t--keep-snap-num=N Minimum number of snapshots to keep after match (default: 10)" > STDERR print "\t--keep-snap-days=N Minimum age in days for snapshot deletion (default: 90)" > STDERR print "\t--no-ranges Disable range compression (output individual snapshots)" > STDERR - print "\t-x pattern Exclude datasets matching pattern\n" > STDERR + print "\t-X pattern Exclude datasets matching pattern\n" > STDERR print "Only snapshots older than the common match point and replicated to TARGET are considered." > STDERR print "Output shows snapshot names (one per line) that are safe to prune.\n" > STDERR print "For complete documentation: zelta help prune" > STDERR From fdbb61a417426bfe81702a2467e196581e2a6656 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Tue, 10 Feb 2026 19:40:49 -0500 Subject: [PATCH 37/47] drop packaging (to be moved to multi-os package repo) --- packaging/freebsd/Makefile | 39 -------------------------- packaging/freebsd/distinfo | 3 -- packaging/freebsd/files/pkg-message.in | 21 -------------- packaging/freebsd/pkg-descr | 8 ------ packaging/freebsd/pkg-plist | 22 --------------- 5 files changed, 93 deletions(-) delete mode 100644 packaging/freebsd/Makefile delete mode 100644 packaging/freebsd/distinfo delete mode 100644 packaging/freebsd/files/pkg-message.in delete mode 100644 packaging/freebsd/pkg-descr delete mode 100644 packaging/freebsd/pkg-plist diff --git a/packaging/freebsd/Makefile b/packaging/freebsd/Makefile deleted file mode 100644 index 4e35366..0000000 --- a/packaging/freebsd/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -PORTNAME= zelta -DISTVERSIONPREFIX= v -DISTVERSION= 1.1.0 -PORTREVISION= 1 -CATEGORIES= sysutils - -MAINTAINER= daniel@belltech.it -COMMENT= ZFS tools used for data migration and backup management -WWW= https://github.com/bellhyve/zelta - -LICENSE= BSD2CLAUSE - -USE_GITHUB= yes -GH_ACCOUNT= bellhyve - -NO_ARCH= yes -NO_BUILD= yes - -SUB_FILES= pkg-message - -_ZELTA_SHARE= ${PREFIX}/share/zelta -_ZELTA_CONF= zelta.conf -_ZELTA_ENV= zelta.env - -do-install: - ${INSTALL_SCRIPT} ${WRKSRC}/bin/zelta ${STAGEDIR}${PREFIX}/bin - ${RLN} ${STAGEDIR}${PREFIX}/bin/zelta ${STAGEDIR}${PREFIX}/bin/zeport - ${RLN} ${STAGEDIR}${PREFIX}/bin/zelta ${STAGEDIR}${PREFIX}/bin/zmatch - ${RLN} ${STAGEDIR}${PREFIX}/bin/zelta ${STAGEDIR}${PREFIX}/bin/zpull - ${RLN} ${STAGEDIR}${PREFIX}/bin/zelta ${STAGEDIR}${PREFIX}/bin/zsync - ${MKDIR} ${STAGEDIR}${ETCDIR} - ${INSTALL_DATA} ${WRKSRC}/${_ZELTA_ENV} ${STAGEDIR}${ETCDIR}/${_ZELTA_ENV}.sample - ${INSTALL_DATA} ${WRKSRC}/${_ZELTA_CONF} ${STAGEDIR}${ETCDIR}/${_ZELTA_CONF}.sample - ${MKDIR} ${STAGEDIR}${_ZELTA_SHARE} - ${INSTALL_SCRIPT} ${WRKSRC}/share/zelta/* ${STAGEDIR}${_ZELTA_SHARE} - ${INSTALL_MAN} ${WRKSRC}/doc/*.8 ${STAGEDIR}${PREFIX}/share/man/man8 - ${INSTALL_MAN} ${WRKSRC}/doc/*.7 ${STAGEDIR}${PREFIX}/share/man/man7 - -.include diff --git a/packaging/freebsd/distinfo b/packaging/freebsd/distinfo deleted file mode 100644 index 58efd4e..0000000 --- a/packaging/freebsd/distinfo +++ /dev/null @@ -1,3 +0,0 @@ -TIMESTAMP = 1717424774 -SHA256 (bellhyve-zelta-v1.0.1_GH0.tar.gz) = e6bff745d3125bd0b435097c282769b4e175ef85ea386500bd456d91346af9ab -SIZE (bellhyve-zelta-v1.0.1_GH0.tar.gz) = 34979 diff --git a/packaging/freebsd/files/pkg-message.in b/packaging/freebsd/files/pkg-message.in deleted file mode 100644 index 7f8e093..0000000 --- a/packaging/freebsd/files/pkg-message.in +++ /dev/null @@ -1,21 +0,0 @@ -[ -{ type: install - message: <]' or the wiki: - - https://zelta.space/home - -If you find any bugs please file them -at https://github.com/bellhyve/zelta/issues. -EOM -} -] diff --git a/packaging/freebsd/pkg-descr b/packaging/freebsd/pkg-descr deleted file mode 100644 index 22760c1..0000000 --- a/packaging/freebsd/pkg-descr +++ /dev/null @@ -1,8 +0,0 @@ -Zelta is a suite of tools offering a streamlined approach to managing -ZFS snapshot replication across systems. It's built with the intention -of simplifying complex ZFS functions into safe and user-friendly -commands while also being the foundation for large-scale backup -and failover environments. It's easy and accessible while working -with most UNIX and UNIX-like base systems without additional packages. -It's optimized for environments with strict permission separation, -and integrates well into many types of existing ZFS workflows. diff --git a/packaging/freebsd/pkg-plist b/packaging/freebsd/pkg-plist deleted file mode 100644 index 80b5ba7..0000000 --- a/packaging/freebsd/pkg-plist +++ /dev/null @@ -1,22 +0,0 @@ -bin/zelta -bin/zeport -bin/zmatch -bin/zpull -bin/zsync -@sample %%ETCDIR%%/zelta.conf.sample -@sample %%ETCDIR%%/zelta.env.sample -%%DATADIR%%/zelta-backup.awk -%%DATADIR%%/zelta-match-pipe.awk -%%DATADIR%%/zelta-match.awk -%%DATADIR%%/zelta-policy.awk -%%DATADIR%%/zelta-report.awk -%%DATADIR%%/zelta-sendopts.awk -%%DATADIR%%/zelta-snapshot.awk -%%DATADIR%%/zelta-usage.sh -share/man/man7/zelta-options.7.gz -share/man/man8/zelta-backup.8.gz -share/man/man8/zelta-clone.8.gz -share/man/man8/zelta-match.8.gz -share/man/man8/zelta-policy.8.gz -share/man/man8/zelta-sync.8.gz -share/man/man8/zelta.8.gz From 93c3708a1c839130928b5f204ea767078145be2a Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 11 Feb 2026 15:32:38 -0500 Subject: [PATCH 38/47] feat: add one-shot installer script and documentation --- README.md | 7 ++++++ contrib/install-from-git.sh | 50 +++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 contrib/install-from-git.sh diff --git a/README.md b/README.md index 82ee653..a986cd2 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,13 @@ Zelta 1.0.1_1 (March 2024) is available in the FreeBSD Ports Collection. For the pkg install zelta ``` +### Experimental: One-Shot Install +```sh +curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sudo sh +``` + +**Note:** This is experimental. For production use, we recommend cloning the repository and reviewing `install.sh` before running it. + --- ## Quickstart: Developer Workflow diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh new file mode 100644 index 0000000..220e192 --- /dev/null +++ b/contrib/install-from-git.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Zelta One-Shot Installer +# Downloads latest Zelta from GitHub and runs install.sh +# +# Usage: curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sh +# Or specify branch: curl ... | sh -s -- --branch=develop + +set -e + +REPO="https://github.com/bellhyve/zelta.git" +BRANCH="${1:-main}" +TMPDIR="${TMPDIR:-/tmp}" +WORKDIR="$TMPDIR/zelta-install-$$" + +# Detect git +if ! command -v git >/dev/null 2>&1; then + echo "Error: git is required but not found" + exit 1 +fi + +# Clone to temp location +echo "Downloading Zelta from GitHub..." +git clone --depth=1 --branch="$BRANCH" "$REPO" "$WORKDIR" || { + echo "Error: Failed to clone repository" + exit 1 +} + +cd "$WORKDIR" + +# Verify we got a real repo +if [ ! -f "install.sh" ] || [ ! -d ".git" ]; then + echo "Error: Downloaded files appear incomplete" + rm -rf "$WORKDIR" + exit 1 +fi + +# Show what we're installing +echo +echo "Installing Zelta from commit: $(git rev-parse --short HEAD)" +echo + +# Run the real installer +sh install.sh "$@" +_exit=$? + +# Cleanup +cd / +rm -rf "$WORKDIR" + +exit $_exit From 0d35501528b764f26fa975fe96896cdece01f56f Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 11 Feb 2026 15:56:43 -0500 Subject: [PATCH 39/47] feat: Add timestamp preservation to prevent unnecessary reinstallation --- contrib/install-from-git.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh index 220e192..94cfa20 100644 --- a/contrib/install-from-git.sh +++ b/contrib/install-from-git.sh @@ -34,6 +34,19 @@ if [ ! -f "install.sh" ] || [ ! -d ".git" ]; then exit 1 fi +# Preserve commit timestamps to avoid unnecessary reinstallation +_commit_ts=$(git log -1 --format=%ct 2>/dev/null) || _commit_ts="" +if [ -n "$_commit_ts" ]; then + # Convert Unix timestamp to touch -t format (YYYYMMDDHHMM.SS) + # Try BSD date (-r seconds) first, then GNU date (-d @seconds) + _touch_ts=$(date -u -r "$_commit_ts" "+%Y%m%d%H%M.%S" 2>/dev/null || \ + date -u -d "@$_commit_ts" "+%Y%m%d%H%M.%S" 2>/dev/null || \ + echo "") + if [ -n "$_touch_ts" ]; then + find . -type f -exec touch -t "$_touch_ts" {} + + fi +fi + # Show what we're installing echo echo "Installing Zelta from commit: $(git rev-parse --short HEAD)" From 26c5c98c72f4d63b88b971e268bf2fe384cc9b26 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 11 Feb 2026 16:17:20 -0500 Subject: [PATCH 40/47] feat: add non-root user install support with PATH warning refactor: improve install-from-git.sh branch parsing and PATH warnings --- contrib/install-from-git.sh | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh index 94cfa20..86fa0a5 100644 --- a/contrib/install-from-git.sh +++ b/contrib/install-from-git.sh @@ -8,7 +8,19 @@ set -e REPO="https://github.com/bellhyve/zelta.git" -BRANCH="${1:-main}" +# Parse branch argument: supports 'main', '--branch=main', or '-b=main' +BRANCH="main" +while [ $# -gt 0 ]; do + case "$1" in + --branch=*|-b=*) + BRANCH="${1#*=}" + shift + ;; + *) + break + ;; + esac +done TMPDIR="${TMPDIR:-/tmp}" WORKDIR="$TMPDIR/zelta-install-$$" @@ -56,6 +68,28 @@ echo sh install.sh "$@" _exit=$? +# Determine expected installation location (matching install.sh logic) +if [ -n "${ZELTA_BIN:-}" ]; then + _expected_bin="$ZELTA_BIN" +elif [ "$(id -u)" -eq 0 ]; then + _expected_bin="/usr/local/bin" +else + _expected_bin="$HOME/bin" +fi + +# Verify the installed zelta is first in PATH +_installed_zelta="$_expected_bin/zelta" +_current_zelta=$(command -v zelta 2>/dev/null || echo "") + +if [ "$_exit" -eq 0 ] && [ -n "$_current_zelta" ] && [ "$_current_zelta" != "$_installed_zelta" ]; then + echo + echo "Warning: A different 'zelta' appears first in PATH." + echo "Installed: $_installed_zelta" + echo "Found: $_current_zelta" + echo "To use the newly installed version, ensure $_expected_bin precedes other locations in PATH." + echo +fi + # Cleanup cd / rm -rf "$WORKDIR" From a2c1d0cd6d9c856e48352eb6ece8d7c72e3c688b Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 11 Feb 2026 16:53:36 -0500 Subject: [PATCH 41/47] fix: wrap user-install prompt with TTY check and remove duplicate PATH warning logic feat: enhance PATH warning to detect conflicting zelta binaries in install.sh --- contrib/install-from-git.sh | 22 ---------------- install.sh | 50 ++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh index 86fa0a5..afd196e 100644 --- a/contrib/install-from-git.sh +++ b/contrib/install-from-git.sh @@ -68,28 +68,6 @@ echo sh install.sh "$@" _exit=$? -# Determine expected installation location (matching install.sh logic) -if [ -n "${ZELTA_BIN:-}" ]; then - _expected_bin="$ZELTA_BIN" -elif [ "$(id -u)" -eq 0 ]; then - _expected_bin="/usr/local/bin" -else - _expected_bin="$HOME/bin" -fi - -# Verify the installed zelta is first in PATH -_installed_zelta="$_expected_bin/zelta" -_current_zelta=$(command -v zelta 2>/dev/null || echo "") - -if [ "$_exit" -eq 0 ] && [ -n "$_current_zelta" ] && [ "$_current_zelta" != "$_installed_zelta" ]; then - echo - echo "Warning: A different 'zelta' appears first in PATH." - echo "Installed: $_installed_zelta" - echo "Found: $_current_zelta" - echo "To use the newly installed version, ensure $_expected_bin precedes other locations in PATH." - echo -fi - # Cleanup cd / rm -rf "$WORKDIR" diff --git a/install.sh b/install.sh index ef1195e..89482a7 100755 --- a/install.sh +++ b/install.sh @@ -15,25 +15,27 @@ elif [ -z "$ZELTA_BIN" ] || [ -z "$ZELTA_SHARE" ] || [ -z "$ZELTA_ETC" ] || [ -z : ${ZELTA_SHARE:="$HOME/.local/share/zelta"} : ${ZELTA_ETC:="$HOME/.config/zelta"} : ${ZELTA_DOC:="$ZELTA_SHARE/doc"} - echo To install Zelta for this user account: - echo - echo 1. Set the following environment variables in your startup script - echo or export them with your desired values: - echo - echo export ZELTA_BIN=\"$ZELTA_BIN\" - echo export ZELTA_SHARE=\"$ZELTA_SHARE\" - echo export ZELTA_ETC=\"$ZELTA_ETC\" - echo export ZELTA_DOC=\"$ZELTA_DOC\" - echo - echo 2. Ensure that \"$ZELTA_BIN\" is in PATH environment variable. - echo - echo Note: If you prefer a global installation, cancel this installation - echo and rerun this command as root, e.g. \`sudo install.sh\`. - echo - echo Proceed with installation? - echo - echo Press Control-C to stop or Return to install using the above paths. - read _wait + if [ -t 0 ]; then + echo To install Zelta for this user account: + echo + echo 1. Set the following environment variables in your startup script + echo or export them with your desired values: + echo + echo export ZELTA_BIN=\"$ZELTA_BIN\" + echo export ZELTA_SHARE=\"$ZELTA_SHARE\" + echo export ZELTA_ETC=\"$ZELTA_ETC\" + echo export ZELTA_DOC=\"$ZELTA_DOC\" + echo + echo 2. Ensure that \"$ZELTA_BIN\" is in PATH environment variable. + echo + echo Note: If you prefer a global installation, cancel this installation + echo and rerun this command as root, e.g. \`sudo install.sh\`. + echo + echo Proceed with installation? + echo + echo Press Control-C to stop or Return to install using the above paths. + read _wait + fi fi : ${ZELTA_CONF:="$ZELTA_ETC/zelta.conf"} @@ -85,9 +87,17 @@ if [ ! -s "$ZELTA_CONF" ]; then copy_file zelta.conf "$ZELTA_CONF" fi -if ! command -v zelta >/dev/null 2>&1; then +# Check if installed zelta will be used +_existing_zelta=$(command -v zelta 2>/dev/null || echo "") +if [ -z "$_existing_zelta" ]; then echo echo "Warning: 'zelta' not found in PATH." echo "Add this to your shell startup file (~/.zshrc, ~/.profile, etc.):" echo " export PATH=\"\$PATH:$ZELTA_BIN\"" +elif [ "$_existing_zelta" != "$ZELTA" ]; then + echo + echo "Warning: A different 'zelta' appears first in PATH." + echo "Installed: $ZELTA" + echo "Found: $_existing_zelta" + echo "To use the newly installed version, ensure $ZELTA_BIN precedes '$(dirname "$_existing_zelta")' in PATH." fi From 3a8686b962d0b1181cc35e36854118f70cce25c8 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Wed, 11 Feb 2026 17:28:24 -0500 Subject: [PATCH 42/47] feat: add auto-detection and guidance for non-root installations refactor: remove interactive prompts from install.sh and add ZELTA_QUIET support fix: suppress install.sh guidance in curl installer fix: update repo URL and simplify PATH guidance in install scripts --- contrib/install-from-git.sh | 31 ++++++++++++++++++++++++--- install.sh | 42 +++++++++++++------------------------ 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh index afd196e..1d9d8da 100644 --- a/contrib/install-from-git.sh +++ b/contrib/install-from-git.sh @@ -7,7 +7,7 @@ set -e -REPO="https://github.com/bellhyve/zelta.git" +REPO="https://github.com/bell-tower/zelta.git" # Parse branch argument: supports 'main', '--branch=main', or '-b=main' BRANCH="main" while [ $# -gt 0 ]; do @@ -64,10 +64,35 @@ echo echo "Installing Zelta from commit: $(git rev-parse --short HEAD)" echo -# Run the real installer -sh install.sh "$@" +# Run the real installer (suppress its user guidance) +ZELTA_QUIET=1 sh install.sh "$@" _exit=$? +# Post-installation guidance for non-root users +if [ "$(id -u)" -ne 0 ] && [ -z "$ZELTA_QUIET" ]; then + echo + echo "==========================================================" + echo "INSTALLATION COMPLETE - ACTION REQUIRED" + echo "==========================================================" + echo + echo "Zelta has been installed to user directories." + echo "To make zelta available in this and future shell sessions," + echo "add the following to your shell startup file" + echo "(~/.bashrc, ~/.zshrc, ~/.profile, etc.):" + echo + echo " # Zelta configuration" + echo " export ZELTA_BIN=\"${ZELTA_BIN:-$HOME/bin}\"" + echo " export ZELTA_SHARE=\"${ZELTA_SHARE:-$HOME/.local/share/zelta}\"" + echo " export ZELTA_ETC=\"${ZELTA_ETC:-$HOME/.config/zelta}\"" + echo " export ZELTA_DOC=\"${ZELTA_DOC:-$HOME/.local/share/zelta/doc}\"" + echo " export PATH=\"\$ZELTA_BIN:\$PATH\"" + echo + echo "Then reload your configuration: source ~/.bashrc" + echo "(or the appropriate file for your shell)" + echo "==========================================================" + echo +fi + # Cleanup cd / rm -rf "$WORKDIR" diff --git a/install.sh b/install.sh index 89482a7..41a53ef 100755 --- a/install.sh +++ b/install.sh @@ -15,27 +15,6 @@ elif [ -z "$ZELTA_BIN" ] || [ -z "$ZELTA_SHARE" ] || [ -z "$ZELTA_ETC" ] || [ -z : ${ZELTA_SHARE:="$HOME/.local/share/zelta"} : ${ZELTA_ETC:="$HOME/.config/zelta"} : ${ZELTA_DOC:="$ZELTA_SHARE/doc"} - if [ -t 0 ]; then - echo To install Zelta for this user account: - echo - echo 1. Set the following environment variables in your startup script - echo or export them with your desired values: - echo - echo export ZELTA_BIN=\"$ZELTA_BIN\" - echo export ZELTA_SHARE=\"$ZELTA_SHARE\" - echo export ZELTA_ETC=\"$ZELTA_ETC\" - echo export ZELTA_DOC=\"$ZELTA_DOC\" - echo - echo 2. Ensure that \"$ZELTA_BIN\" is in PATH environment variable. - echo - echo Note: If you prefer a global installation, cancel this installation - echo and rerun this command as root, e.g. \`sudo install.sh\`. - echo - echo Proceed with installation? - echo - echo Press Control-C to stop or Return to install using the above paths. - read _wait - fi fi : ${ZELTA_CONF:="$ZELTA_ETC/zelta.conf"} @@ -89,15 +68,24 @@ fi # Check if installed zelta will be used _existing_zelta=$(command -v zelta 2>/dev/null || echo "") -if [ -z "$_existing_zelta" ]; then - echo - echo "Warning: 'zelta' not found in PATH." - echo "Add this to your shell startup file (~/.zshrc, ~/.profile, etc.):" - echo " export PATH=\"\$PATH:$ZELTA_BIN\"" -elif [ "$_existing_zelta" != "$ZELTA" ]; then +if [ -n "$_existing_zelta" ] && [ "$_existing_zelta" != "$ZELTA" ]; then echo echo "Warning: A different 'zelta' appears first in PATH." echo "Installed: $ZELTA" echo "Found: $_existing_zelta" echo "To use the newly installed version, ensure $ZELTA_BIN precedes '$(dirname "$_existing_zelta")' in PATH." fi + +# Post-installation summary for non-root users +if [ "$(id -u)" -ne 0 ] && [ -z "$ZELTA_QUIET" ]; then + echo + echo "Zelta installed to user directories." + echo "To use zelta, ensure these are set in your shell startup file:" + echo + echo " export ZELTA_BIN=\"$ZELTA_BIN\"" + echo " export ZELTA_SHARE=\"$ZELTA_SHARE\"" + echo " export ZELTA_ETC=\"$ZELTA_ETC\"" + echo " export ZELTA_DOC=\"$ZELTA_DOC\"" + echo " export PATH=\"\$ZELTA_BIN:\$PATH\"" + echo +fi From be87d82332bad881a363fc2f7d72f6f88f33bddb Mon Sep 17 00:00:00 2001 From: rlogwood Date: Thu, 12 Feb 2026 13:55:13 -0500 Subject: [PATCH 43/47] Feature/test refactor (#61) Refactor ShellSpec test infrastructure and generation system Restructure test directory with version 1 archive of prior work and a new modular architecture under test/runners. Test generation: Add Ruby-based ShellSpec test generator with YAML schema support Modularize placeholder substitution and use when_command in schema Make shellspec_name the single source of truth (extracted from YAML) Add YAML validation script (validate_yaml.rb) Handle command timeouts, failures, and empty output gracefully Fix env var substitution to match longest token first Escape backticks in output to prevent shell evaluation Force UTF-8 encoding on output handling during SysExec calls Exit matcher generation on awk errors Test coverage: Add generated tests for divergent tree, zelta revert, and zelta clone Break divergent tree setup into discrete phases (initial, backup, divergence) Wrap divergent tree tests with Describe blocks for conditional Skip If Environment and portability: Add helpers to configure env vars from the last tmp-installed zelta Allow bypass of process-based env vars for pre-configured environments Use which instead of command -v for Ubuntu compatibility Use ls -1d for portable directory listing Use sudo for local zfs allow; update error checks after permission changes Project structure: Reorganize test/runners into env/ and test_generation/ directories Implement absolute path resolution (SCRIPT_DIR pattern) across all scripts Rename manual_cleanup.sh to test_generator_cleanup.sh Move .rubocop.yml to lib/ruby/, fix RuboCop warnings Documentation: Rewrite test_generation and test/runners READMEs Add debugging workflow docs and file hierarchy restructuring plan --- .gitignore | 3 +- spec/.gitignore | 1 - spec/README.md | 230 ----------- spec/banner | 6 - spec/bin/all_tests_setup/all_tests_setup.sh | 66 --- spec/bin/all_tests_setup/common_test_env.sh | 79 ---- .../create_file_backed_zfs_test_pools.sh | 272 ------------- spec/bin/all_tests_setup/env_constants.sh | 33 -- .../all_tests_setup/install_local_zelta.sh | 8 - .../bin/divergent_test/divergent_snap_tree.sh | 69 ---- spec/bin/divergent_test/divergent_test_env.sh | 6 - .../bin/divergent_test/divergent_test_spec.sh | 327 --------------- spec/bin/hello_example.sh | 23 -- spec/bin/one_time_setup/setup_sudoers.sh | 88 ---- spec/bin/opts_test/opts_spec.sh | 223 ---------- spec/bin/opts_test/test_env.sh | 36 -- .../setup_remote_host_test_env.sh | 38 -- spec/bin/standard_test/standard_snap_tree.sh | 137 ------- spec/bin/standard_test/standard_test_env.sh | 6 - spec/bin/standard_test/standard_test_spec.sh | 152 ------- spec/doc/vm/README.md | 5 - spec/doc/vm/creation.md | 41 -- spec/doc/vm/installing-kvm.md | 55 --- spec/doc/vm/running.md | 0 spec/doc/vm/zfs-configuration.md | 13 - spec/lib/common.sh | 206 ---------- spec/lib/hello.sh | 4 - spec/lib/script_util.sh | 41 -- spec/spec_helper.sh | 111 ----- spec/util/README.md | 47 --- .../zelta_backup_after_rotate_output.txt | 3 - .../zelta_match_after_backup_output.txt | 10 - spec/util/test_data/zelta_match_output.txt | 11 - test/01_no_op_spec.sh | 2 +- test/021_setup_pools_spec.sh | 25 ++ test/022_setup_tree_spec.sh | 39 ++ test/02_setup_spec.sh | 51 --- test/040_zelta_tests_spec.sh | 130 ++++++ test/050_zelta_revert_spec.sh | 144 +++++++ test/060_zelta_clone_spec.sh | 57 +++ test/README.md | 26 ++ test/runners/README.md | 379 +++++++++++++++++ test/runners/doc/README_AliasHelpers.md | 51 +++ test/runners/env/helpers.sh | 40 ++ test/runners/env/reset_env.sh | 7 + test/runners/env/set_reuse_tmp_env.sh | 46 +++ test/runners/env/setup_debug_env.sh | 12 + test/runners/env/test_env.sh | 15 + test/runners/env/test_generator_cleanup.sh | 7 + test/runners/test_generation/Gemfile | 5 + test/runners/test_generation/Gemfile.lock | 18 + test/runners/test_generation/README.md | 358 ++++++++++++++++ test/runners/test_generation/bin/debug_gen.sh | 51 +++ .../test_generation/bin/generate_new_tests.sh | 34 ++ .../test_generation/bin/validate_yaml.rb | 56 +++ .../config/test_config_schema.yml | 51 +++ .../config/test_defs/040_zelta_tests.yml | 23 ++ .../test_defs/050_zelta_revert_test.yml | 33 ++ .../config/test_defs/060_zelta_clone_test.yml | 16 + .../config/test_defs/dbg_test.yml | 18 + .../config/test_defs/example_test.yml | 10 + .../debug/run_test_gen_dbg_env.rb | 35 ++ .../lib/orchestration/generate_test.sh | 101 +++++ .../lib/orchestration/setup_tree.sh | 24 ++ .../test_generation/lib/ruby/placeholders.rb | 68 ++++ .../test_generation/lib/ruby/sys_exec.rb | 106 +++++ .../lib/ruby/test_generator.rb | 382 ++++++++++++++++++ .../scripts/awk}/generate_case_stmt_func.awk | 16 + .../scripts/sh/generate_matcher.sh | 80 ++++ .../scripts/sh}/matcher_func_generator.sh | 8 +- test/test.sh | 33 -- test/test_helper.sh | 124 +++--- test/test_runner.sh | 87 ---- 73 files changed, 2550 insertions(+), 2568 deletions(-) delete mode 100644 spec/.gitignore delete mode 100644 spec/README.md delete mode 100644 spec/banner delete mode 100755 spec/bin/all_tests_setup/all_tests_setup.sh delete mode 100755 spec/bin/all_tests_setup/common_test_env.sh delete mode 100755 spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh delete mode 100644 spec/bin/all_tests_setup/env_constants.sh delete mode 100755 spec/bin/all_tests_setup/install_local_zelta.sh delete mode 100755 spec/bin/divergent_test/divergent_snap_tree.sh delete mode 100644 spec/bin/divergent_test/divergent_test_env.sh delete mode 100644 spec/bin/divergent_test/divergent_test_spec.sh delete mode 100644 spec/bin/hello_example.sh delete mode 100755 spec/bin/one_time_setup/setup_sudoers.sh delete mode 100644 spec/bin/opts_test/opts_spec.sh delete mode 100644 spec/bin/opts_test/test_env.sh delete mode 100755 spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh delete mode 100755 spec/bin/standard_test/standard_snap_tree.sh delete mode 100755 spec/bin/standard_test/standard_test_env.sh delete mode 100644 spec/bin/standard_test/standard_test_spec.sh delete mode 100644 spec/doc/vm/README.md delete mode 100644 spec/doc/vm/creation.md delete mode 100644 spec/doc/vm/installing-kvm.md delete mode 100644 spec/doc/vm/running.md delete mode 100644 spec/doc/vm/zfs-configuration.md delete mode 100644 spec/lib/common.sh delete mode 100644 spec/lib/hello.sh delete mode 100644 spec/lib/script_util.sh delete mode 100644 spec/spec_helper.sh delete mode 100644 spec/util/README.md delete mode 100644 spec/util/test_data/zelta_backup_after_rotate_output.txt delete mode 100644 spec/util/test_data/zelta_match_after_backup_output.txt delete mode 100644 spec/util/test_data/zelta_match_output.txt create mode 100644 test/021_setup_pools_spec.sh create mode 100644 test/022_setup_tree_spec.sh delete mode 100644 test/02_setup_spec.sh create mode 100644 test/040_zelta_tests_spec.sh create mode 100644 test/050_zelta_revert_spec.sh create mode 100644 test/060_zelta_clone_spec.sh create mode 100644 test/README.md create mode 100644 test/runners/README.md create mode 100644 test/runners/doc/README_AliasHelpers.md create mode 100644 test/runners/env/helpers.sh create mode 100755 test/runners/env/reset_env.sh create mode 100755 test/runners/env/set_reuse_tmp_env.sh create mode 100644 test/runners/env/setup_debug_env.sh create mode 100644 test/runners/env/test_env.sh create mode 100755 test/runners/env/test_generator_cleanup.sh create mode 100644 test/runners/test_generation/Gemfile create mode 100644 test/runners/test_generation/Gemfile.lock create mode 100644 test/runners/test_generation/README.md create mode 100755 test/runners/test_generation/bin/debug_gen.sh create mode 100755 test/runners/test_generation/bin/generate_new_tests.sh create mode 100755 test/runners/test_generation/bin/validate_yaml.rb create mode 100644 test/runners/test_generation/config/test_config_schema.yml create mode 100644 test/runners/test_generation/config/test_defs/040_zelta_tests.yml create mode 100644 test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml create mode 100644 test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml create mode 100644 test/runners/test_generation/config/test_defs/dbg_test.yml create mode 100644 test/runners/test_generation/config/test_defs/example_test.yml create mode 100755 test/runners/test_generation/debug/run_test_gen_dbg_env.rb create mode 100755 test/runners/test_generation/lib/orchestration/generate_test.sh create mode 100644 test/runners/test_generation/lib/orchestration/setup_tree.sh create mode 100644 test/runners/test_generation/lib/ruby/placeholders.rb create mode 100644 test/runners/test_generation/lib/ruby/sys_exec.rb create mode 100755 test/runners/test_generation/lib/ruby/test_generator.rb rename {spec/util => test/runners/test_generation/scripts/awk}/generate_case_stmt_func.awk (60%) create mode 100755 test/runners/test_generation/scripts/sh/generate_matcher.sh rename {spec/util => test/runners/test_generation/scripts/sh}/matcher_func_generator.sh (62%) delete mode 100755 test/test.sh delete mode 100755 test/test_runner.sh diff --git a/.gitignore b/.gitignore index 1a4c208..db73fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ .* doc/man? tmp -spec/tmp hide.* *~ \#*# @@ -13,3 +12,5 @@ TAGS A*d .#* retired/ +logs +log diff --git a/spec/.gitignore b/spec/.gitignore deleted file mode 100644 index a9a5aec..0000000 --- a/spec/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp diff --git a/spec/README.md b/spec/README.md deleted file mode 100644 index 102cf97..0000000 --- a/spec/README.md +++ /dev/null @@ -1,230 +0,0 @@ -# Shellspec Testing -* * * - -* [Shellspec Testing](#shellspec-testing) - * [Overview](#overview) - * [Installing ShellSpec](#installing-shellspec) - * [🔰 Making your first test - a simple Example](#-making-your-first-test---a-simple-example) - * [Setting up a local development environment](#setting-up-a-local-development-environment) - * [Testing `zelta`](#testing-zelta) - * [:star: Using the test helper](#star-using-the-test-helper) - * [Example test_runner.sh output](#example-test_runnersh-output) - * [:zap: To run the standard Zelta test](#zap-to-run-the-standard-zelta-test-) - * [:zap: To run all tests](#zap-to-run-all-tests) - * [🔰 shellspec examples](#-shellspec-examples) - -* * * -## Overview - -[zelta](https://zelta.space/) uses [ShellSpec](https://github.com/shellspec/shellspec) for testing. If you're new to ShellSpec the -the following links are helpful: -- https://github.com/shellspec/shellspec -- https://shellspec.info/ -- :star: https://deepwiki.com/shellspec/shellspec :heart: - -### Installing ShellSpec - -See the [ShellSpec installation guide](https://github.com/shellspec/shellspec#installation) for instructions. -- The following works on FreeBSD and Ubuntu -- `curl -fsSL https://git.io/shellspec | sh` -- Add `$HOME/.local/bin` to your `PATH` - -#### 🔰 Making your first test - a simple Example -Use the hello_spec.sh file as a template for your first test. -- [hello_example.sh](./bin/hello_example.sh) -```shell -shellspec -f d spec/bin/hello_example.sh -``` - -### Setting up a local development environment -- [Ubuntu VM](./doc/vm/README.md) - - -## Testing `zelta` -> 🔑 zfs must be installed on your system. -> -> â„šī¸ sudo is required because root privilege is needed for zfs and zelta commands -> -> â›‘ī¸ Only temporary file backed zfs pools are used during testing -> -> đŸĻē Installs are local to a temporary directory -* * * -### :star: Using the test helper -```shell -% ~/src/repos/zelta$ test/test_runner.sh -Error: Expected 2 arguments: -Usage: test/test_runner.sh -``` -- For remote tests setup your server and backup user -- Export the following env vars before running -```shell -# for example -# TODO: use the same server, multiple servers is a WIP -# NOTE: different servers for SRC and TGT is a WIP -export SRC_SVR="backupuser@server" -export TGT_SVR="backupuser@server" -``` - -- Typical test secnarios for local testing - - TODO: encrypted trees aren't implemented yet -``` -test/test_runner.sh local standard -test/test_runner.sh local divergent -test/test_runner.sh remote standard -test/test_runner.sh remote divergent -``` - -### Example test_runner.sh output -- Recommendation: test locally first, before trying remote - -
- -remote standard run - -```shell -% test/test_runner.sh remote standard - -# setup output omitted -# respond to install prompt and sudo password for pool setup - _____ _ _ _____ _ -|__ /___| | |_ __ _ |_ _|__ ___| |_ - / // _ \ | __/ _` | | |/ _ \/ __| __| - / /| __/ | || (_| | | | __/\__ \ |_ -/____\___|_|\__\__,_| |_|\___||___/\__| - -[info] specshell precheck: version:0.28.1 shell: sh -[info] *** TREE_NAME is {standard} -[info] *** RUNNING_MODE is {remote} -[info] *** -[info] *** Running Remotely -[info] *** Source Server is SRC_SVR:{dever@fzfsdev} -[info] *** Target Server is TGT_SVR:{dever@fzfsdev} -[info] *** -Settings OS specific environment for {Linux} -OS_TYPE: Linux: set POOL_TYPE={2} -Running: /bin/sh [sh] - -confirm zfs setup - has good initial SRC_POOL:{apool} simple snap tree - has good initial TGT_POOL:{bpool} simple snap tree -try backup - backs up the initial tree - has valid backup - has 8 snapshots on dever@fzfsdev matching pattern '^(apool|bpool)' - has 4 snapshots on dever@fzfsdev matching pattern 'apool/treetop' - has 4 snapshots on dever@fzfsdev matching pattern 'bpool/backups/treetop' -zelta rotate - rotates the backed up tree - has 16 snapshots on dever@fzfsdev matching pattern '^(apool|bpool)' - has 8 snapshots on dever@fzfsdev matching pattern 'apool/treetop' - has 8 snapshots on dever@fzfsdev matching pattern 'bpool/backups/treetop' - -Finished in 7.30 seconds (user 1.60 seconds, sys 0.12 seconds) -11 examples, 0 failures - - -✓ Tests complete - -``` - -
- - -
- -remote divergent run - -```shell -% test/test_runner.sh remote divergent - -# setup output omitted -# respond to install prompt and sudo password for pool setup - - _____ _ _ _____ _ -|__ /___| | |_ __ _ |_ _|__ ___| |_ - / // _ \ | __/ _` | | |/ _ \/ __| __| - / /| __/ | || (_| | | | __/\__ \ |_ -/____\___|_|\__\__,_| |_|\___||___/\__| - -[info] specshell precheck: version:0.28.1 shell: sh -[info] *** TREE_NAME is {divergent} -[info] *** RUNNING_MODE is {remote} -[info] *** -[info] *** Running Remotely -[info] *** Source Server is SRC_SVR:{dever@fzfsdev} -[info] *** Target Server is TGT_SVR:{dever@fzfsdev} -[info] *** -Settings OS specific environment for {Linux} -OS_TYPE: Linux: set POOL_TYPE={2} -Running: /bin/sh [sh] - -confirm zfs setup - zfs list output validation - matches expected pattern for each line - check initial zelta match state - initial match has 5 up-to-date, 1 syncable, 3 blocked, with 9 total datasets compared - add incremental source snapshot - adds dever@fzfsdev:apool/treetop/sub3@two snapshot - add divergent snapshots of same name - adds divergent snapshots for dever@fzfsdev:apool/treetop/sub2@two and dever@fzfsdev:bpool/backups/treetop/sub2@two - check zelta match after divergent snapshots - after divergent snapshot match has 2 up-to-date, 2 syncable, 5 blocked, with 9 total datasets compared -Divergent match, rotate, match - shows current match for divergent dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop - rotate divergent dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop - match dever@fzfsdev:apool/treetop and dever@fzfsdev:bpool/backups/treetop after divergent rotate -Divergent backup, then match - backup divergent dever@fzfsdev:apool/treetop to dever@fzfsdev:bpool/backups/treetop - match after backup - -Finished in 7.95 seconds (user 2.09 seconds, sys 0.22 seconds) -10 examples, 0 failures - - -✓ Tests complete -``` - -
- -* * * -### :zap: To run the standard Zelta test -[zelta_standard_test_spec.sh](./bin/zelta_standard_test_spec.sh) - - ``` - sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh - ``` - 🔎 This test will create a standard setup via [initialize_testing_setup.sh](initialize/initialize_testing_setup.sh) -* * * -### :zap: To run all tests -> â„šī¸ Tests will run in the order they are listed in the spec directory -> use `-P, --pattern PATTERN` to filter tests by pattern -> the default pattern is `"*_spec.sh"` -```shell -sudo -E env "PATH=$PATH" shellspec -f d -``` - -* * * -### 🔰 shellspec examples -- Run all files matching a pattern [default: "*_spec.sh"] -`sudo -E env "PATH=$PATH" shellspec -f d -P "*_setup_*"` -- List all Groups (`Describe`) and Examples (`It`) - ```shell - # shellspec --list examples (directory/file) - $ shellspec --list examples spec/bin - spec/bin/zelta_standard_test_spec.sh:@1-1 - spec/bin/zelta_standard_test_spec.sh:@1-2 - spec/bin/zelta_standard_test_spec.sh:@2-1 - spec/bin/zelta_standard_test_spec.sh:@2-2 - ``` -- `:@1` 🟰 Run all examples in group @1 - ```shell - sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh:@1 - ``` -- `:@-1` 🟰 Run only example #1 in group @1 - ```shell - sudo -E env "PATH=$PATH" shellspec -f d spec/bin/zelta_standard_test_spec.sh:@1-1 - ``` -- use options `--xtrace --shell bash` to show a trace with expectation evaluation - ```shell - shellspec -f d --xtrace --shell bash spec/bin/standard_test/standard_test_spec.sh:@2-2 - ``` diff --git a/spec/banner b/spec/banner deleted file mode 100644 index 42edab5..0000000 --- a/spec/banner +++ /dev/null @@ -1,6 +0,0 @@ - _____ _ _ _____ _ -|__ /___| | |_ __ _ |_ _|__ ___| |_ - / // _ \ | __/ _` | | |/ _ \/ __| __| - / /| __/ | || (_| | | | __/\__ \ |_ -/____\___|_|\__\__,_| |_|\___||___/\__| - diff --git a/spec/bin/all_tests_setup/all_tests_setup.sh b/spec/bin/all_tests_setup/all_tests_setup.sh deleted file mode 100755 index f79559f..0000000 --- a/spec/bin/all_tests_setup/all_tests_setup.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -echo "check SRC_SVR:{$SRC_SVR}" -echo "check TGT_SVR:{$SRC_SVR}" - -set -x -. spec/bin/all_tests_setup/common_test_env.sh -set +x - -. spec/lib/common.sh - - -verify_root() { - # Check if running as root - if [ "$(id -u)" -ne 0 ]; then - echo "Error: You must run as root or with sudo" >&2 - return 1 - fi -} - -initialize_zelta_test() { - echo "-- BeforeAll setup" - - echo "-- installing zelta" - "${ALL_TESTS_SETUP_DIR}"/install_local_zelta.sh - INSTALL_STATUS=$? - if [ $INSTALL_STATUS -ne 0 ]; then - echo "** Error: zelta install failed" - fi - - #"${INITIALIZE_DIR}"/create_device_backed_zfs_test_pools.sh - #"${INITIALIZE_DIR}"/create_file_backed_zfs_test_pools.sh - #TREE_STATUS=$? - -# echo "-- creating test pools" - if "${ALL_TESTS_SETUP_DIR}"/create_file_backed_zfs_test_pools.sh; then - #if "${INITIALIZE_DIR}"/create_device_backed_zfs_test_pools.sh; then - #echo "-- setting up snap tree" - #"${INITIALIZE_DIR}"/setup_simple_snap_tree.sh - TREE_STATUS=$? - # NOTE: moving create snap tree into test specs, we'll have different kinds of trees - #TREE_STATUS=0 - else - echo "** Error: failed to setup zfs pool" >&2 - TREE_STATUS=1 - fi - - #CREATE_STATUS=$? - - #echo "-- Create pool status: {$CREATE_STATUS}" - echo "-- Install Zelta status: {$INSTALL_STATUS}" - echo "-- Make snap tree status: {$TREE_STATUS}" - - #SETUP_STATUS=$((CREATE_STATUS || INSTALL_STATUS || TREE_STATUS)) - SETUP_STATUS=$((INSTALL_STATUS || TREE_STATUS)) - echo "-- returning SETUP_STATUS:{$SETUP_STATUS}" - - if [ $SETUP_STATUS -ne 0 ]; then - echo "** Error: zfs pool and/or zelta install failed!" >&2 - fi - - return $SETUP_STATUS -} - -# NOTE: root is no longer required, unless there is a sloppy state left by improper sudo use -initialize_zelta_test diff --git a/spec/bin/all_tests_setup/common_test_env.sh b/spec/bin/all_tests_setup/common_test_env.sh deleted file mode 100755 index 8ceedc7..0000000 --- a/spec/bin/all_tests_setup/common_test_env.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh - -. spec/lib/common.sh - -CUR_DIR=$(pwd) - -. spec/bin/all_tests_setup/env_constants.sh - -# backup username, should be configured for ssh access on remotes -export BACKUP_USER="${BACKUP_USER:-dever}" -#export BACKUP_USER="${SUDO_USER:-$(whoami)}" - -# location for git pulls of source for testing -export ZELTA_GIT_CLONE_DIR="${ZELTA_GIT_CLONE_DIR:-/tmp/zelta-dev}" -export GIT_TEST_BRANCH=${GIT_TEST_BRANCH:-feature/zelta-test} - -# remote test host is the machine we'll setup for remote testing -# when a target server is specified, it should be the REMOTE_TEST_HOST - -# TODO: consider eliminating REMOTE_TEST_HOST, it looks redundant, consider TGT_SVR, updated dependencies, review, test - -export REMOTE_TEST_HOST=${REMOTE_TEST_HOST:-fzfsdev} - -# Zelta supports remote commands, by default SRC and TGT servers are the current host -export SRC_SVR="${SRC_SVR:-}" -export TGT_SVR="${TGT_SVR:-}" - - -if [ -z "$SRC_SVR" ]; then - export ZELTA_SRC_POOL="${SRC_POOL}" -else - export ZELTA_SRC_POOL="${SRC_SVR}:${SRC_POOL}" -fi - -if [ -z "$TGT_SVR" ]; then - export ZELTA_TGT_POOL="${TGT_POOL}" -else - export ZELTA_TGT_POOL="${TGT_SVR}:${TGT_POOL}" -fi - -ALL_TESTS_SETUP_DIR=${CUR_DIR}/spec/bin/all_tests_setup - -export LOCAL_TMP="${CUR_DIR}/spec/tmp" -export TEST_INSTALL="${LOCAL_TMP}/test_install" -export ZELTA_BIN="$TEST_INSTALL/bin" -export ZELTA_SHARE="$TEST_INSTALL/share/zelta" -export ZELTA_ETC="$TEST_INSTALL/zelta" -export ZELTA_MAN8="$TEST_INSTALL/share/man/man8" - -# TODO: remove device support completely or clean it up, currently using image files for pools -# TODO: if keeping it, clean up the tested code for this and support the POOL_TYPE FLAG that selects it -# Default devices if not set -: ${SRC_POOL_DEVICES:="/dev/nvme1n1"} -: ${TGT_POOL_DEVICES:="/dev/nvme2n1"} - -export SRC_POOL_DEVICES -export TGT_POOL_DEVICES - -export ZFS_MOUNT_BASE="${LOCAL_TMP}/zfs-test-mounts" -export ZELTA_ZFS_STORE_TEST_DIR="${LOCAL_TMP}/zelta-zfs-store-test" -export ZELTA_ZFS_TEST_POOL_SIZE="20G" - - -# set default pool type -export POOL_TYPE=$FILE_IMG_POOL - -setup_os_specific_env -#echo "Using POOL_TYPE: {$POOL_TYPE}" - -check_zfs_installed - -# If you need to modify the version of awk used -#export ZELTA_AWK=mawk - -export PATH="${ZELTA_BIN}:$PATH" - -# make exec_cmd silent -# export EXEC_CMD_QUIET=1 - diff --git a/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh b/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh deleted file mode 100755 index 39033f1..0000000 --- a/spec/bin/all_tests_setup/create_file_backed_zfs_test_pools.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/bin/sh - -. spec/lib/common.sh - -pool_image_file() { - pool_name=$1 - echo "${ZELTA_ZFS_STORE_TEST_DIR}/${pool_name}.img" # return value is output to stdout -} - - -md_unit_file() { - pool_name=$1 - echo "${ZELTA_ZFS_STORE_TEST_DIR}/${pool_name}.md" # return value is output to stdout -} - - -create_freebsd_mem_disk_pool() { - pool_name=$1 - img_file=$(pool_image_file $pool_name) - create_file_img $img_file - - # Attach as memory disk - md_unit=$(mdconfig -a -t vnode -f "$img_file") - echo "Created $md_unit for $pool_name" - - # Create pool on the device - exec_cmd sudo zpool create "$pool_name" "/dev/$md_unit" - - # Store md unit for cleanup - md_file=$(md_unit_file "$pool_name") - echo "$md_unit" > "$md_file" -} - -## Destroy image-backed pool on FreeBSD -# NOTE: the pool has already been destroyed -destroy_freebsd_mem_disk_md_device_and_img() { - pool_name=$1 - img_file=$(pool_image_file "$pool_name") - - md_file=$(md_unit_file "$pool_name") - - # Detach md device - if [ -f "$md_file" ]; then - md_unit=$(cat "$md_file") - mdconfig -d -u "${md_unit#md}" - rm "$md_file" - fi - - # Remove image file - rm -f "$img_file" -} - -create_freebsd_test_pool() { - pool_name=$1 - echo_alert "running create_freebsd_test_pool - pool_name {$pool_name}" - destroy_freebsd_mem_disk_md_device_and_img $pool_name - create_freebsd_mem_disk_pool $pool_name -} - - - -check_pool_exists() { - pool_name="$1" - if [ -z "$pool_name" ]; then - echo "** Error: Pool name required" >&2 - return 1 - fi - exec_cmd sudo zpool list "$pool_name" >/dev/null 2>&1 -} - - - -destroy_pool() { - pool_name=$1 - echo "Destroying pool '$pool_name'..." - if exec_cmd sudo zpool export -f "$pool_name"; then - # TODO: the export seems to remove the pool and then zpool destory fails - # TODO: research this - exec_cmd sudo zpool destroy -f "$pool_name" - - fi - - # since the above isn't working as expected, we check if the pool - # still exists and return an error if it does - if check_pool_exists $pool_name; then - return 1 - fi - - # forcing this to return 0 because of the above - #return 0 - -# if ! exec_cmd sudo zpool destroy -f "$pool_name"; then -# echo "Destroy for pool '$pool_name' failed, trying export then destroy" -# # Export is only needed when the pool is busy/imported but destroy can't complete -# exec_cmd sudo zpool export -f "$pool_name" && exec_cmd sudo zpool destroy -f "$pool_name" -# fi -} - -## 1. Export (destroy) the pool -#zfs destroy -r poolname # destroys all datasets (optional, if you want to be thorough) -#zpool destroy poolname # destroys the pool itself -# -## 2. Detach the loop device -#sudo losetup -d /dev/loop0 # replace loop0 with your actual loop device - - - - - -destroy_pool_if_exists() { - pool_name="$1" - if check_pool_exists "$pool_name"; then - destroy_pool "$pool_name" - else - echo "Pool '$pool_name' does not exist, no need to destroy" - fi -} - -rm_img_and_its_loop_devices() { - img=$1 - echo "removing loop devices associated with image file: {$img}" - - # Remove all loop devices at once - sudo losetup -j "$img" | cut -d: -f1 | xargs -r -n1 sudo losetup -d - - echo "removing image file: {$img}" - exec_cmd sudo rm -f "$img" -} - -#x2rm_img_and_its_loop_devices() { -# img=$1 -# echo "removing loop devices associated with image file: {$img}" -# -# # Get all loop devices for this image -# while IFS= read -r loop_device; do -# if [[ -n "$loop_device" ]]; then -# echo "removing loop device {$loop_device}" -# exec_cmd sudo losetup -d "$loop_device" -# fi -# done < <(sudo losetup -j "$img" | cut -d: -f1) -# -# echo "removing image file: {$img}" -# exec_cmd sudo rm -f "$img" -#} - -#xxrm_img_and_its_loop_devices() { -# img=$1 -# echo "removing loop devices associated with image file: {$img}" -# #exec_cmd sudo losetup -j "$img" | cut -d: -f1 | xargs -r exec_cmd sudo losetup -d -# -# loop_device=$(sudo losetup -j "$img" | cut -d: -f1) -# echo "removing loop device {$loop_device}" -# exec_cmd sudo losetup -d $loop_device -# echo "removing image file: {$img}" -# exec_cmd sudo rm -f "$img" -#} - -create_file_img() { - img_file=$1 - - # NOTE: important to remove the image file first so that all zfs metadata is removed. - # truncate on an existing file will resize it and leave the metadata in place leading to crashes - rm -f "${img_file}" - - echo "Creating ${ZELTA_ZFS_TEST_POOL_SIZE}" "${img_file}" - truncate -s "${ZELTA_ZFS_TEST_POOL_SIZE}" "${img_file}" - - echo "showing created file image:" - ls -lh "${img_file}" -} -# old name -#create_pool_from_loop_device() - -create_linux_loop_device_pool() { - pool_name=$1 - - echo_alert "running create_linux_loop_device_pool - pool {$pool_name}" - - img_file=$(pool_image_file $pool_name) - - rm_img_and_its_loop_devices "$img_file" - - create_file_img "$img_file" - - echo "create loop device for file image: {$img_file}" - exec_cmd sudo losetup -f "$img_file" - - loop_device=$(losetup --list --noheadings --output NAME --associated "$img_file") - echo "created loop_device:{$loop_device} for image file:{$img_file}" - - echo "create pool {$pool_name} for loop device {$loop_device}" - exec_cmd sudo zpool create -f -m "/${pool_name}" "${pool_name}" "${loop_device}" -} - -create_pool_from_image_file() { - pool_name=$1 - img_file=$(pool_image_file $pool_name) - - create_file_img "$img_file" - echo "Creating zfs pool {$pool_name} from image file {$img_file}" - exec_cmd sudo zpool create -f -m "/${pool_name}" "${pool_name}" "${img_file}" -} - -create_test_pool() { - pool_name="$1" - if ! destroy_pool_if_exists "${pool_name}"; then - echo "** Error: Can't delete pool {$pool_name}" >&2 - return 1 - fi - - if [ "$POOL_TYPE" = "$MEMORY_DISK_POOL" ]; then - create_freebsd_test_pool $pool_name - elif [ "$POOL_TYPE" = "$LOOP_DEV_POOL" ]; then - create_linux_loop_device_pool "$pool_name" - elif [ "$POOL_TYPE" = "$FILE_IMG_POOL" ]; then - create_pool_from_image_file $pool_name - else - echo "Can't create pools for unsupported POOL_TYPE: {$POOL_TYPE}" >&2 - return 1 - fi - - echo "Created ${pool_name}" - exec_cmd sudo zpool list -v "${pool_name}" -} - - -verify_pool_creation() { - pool_name="$1" - expected_size="$2" - - if check_pool_exists "$pool_name"; then - actual_size=$(exec_cmd sudo zpool list -H -o size "$pool_name") - echo "Success: Pool '$pool_name' created successfully. Size: $actual_size (Expected: $expected_size)" - else - echo "Error: Pool '$pool_name' was NOT created." - return 1 - fi -} - -create_pools() { - echo "" - echo "=== create pool ${SRC_POOL} ===" - create_test_pool "${SRC_POOL}" - SRC_STATUS=$? - echo "" - echo "=== create pool ${TGT_POOL} ===" - create_test_pool "${TGT_POOL}" - TGT_STATUS=$? - - echo "SRC_STATUS:{$SRC_STATUS}" - echo "TGT_STATUS:{$TGT_STATUS}" - - return $((SRC_STATUS || TGT_STATUS)) -} -#set -x -rm -fR "${ZFS_MOUNT_BASE}" -mkdir -p "${ZFS_MOUNT_BASE}" -chmod 777 "${ZFS_MOUNT_BASE}" -chown ${BACKUP_USER} "${ZFS_MOUNT_BASE}" -chgrp ${BACKUP_USER} "${ZFS_MOUNT_BASE}" - -ls -ld "${ZFS_MOUNT_BASE}" -mkdir -p "${ZELTA_ZFS_STORE_TEST_DIR}" - -create_pools -setup_zfs_allow - -#set +x -#setup_loop_img "${SRC_POOL}" -#setup_loop_img "${TGT_POOL}" - - diff --git a/spec/bin/all_tests_setup/env_constants.sh b/spec/bin/all_tests_setup/env_constants.sh deleted file mode 100644 index f871cee..0000000 --- a/spec/bin/all_tests_setup/env_constants.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Snap tree types, these correspond to different types of tests -export STANDARD_TREE=standard -export DIVERGENT_TREE=divergent -export ENCRYPTED_TREE=encrypted - -# Running modes -export RUN_REMOTELY=remote -export RUN_LOCALLY=local - -# we use a a/b pool naming convention, were a is the starting point -# and b is used for backups or perturbations to a -export SRC_POOL="apool" -export TGT_POOL="bpool" - -# zfs pool creation strategy types -export FILE_IMG_POOL=1 - -# On Ubuntu we use a loop device backed by a file -export LOOP_DEV_POOL=2 - -# On FreeBSD we use a memory disk backed by a file -export MEMORY_DISK_POOL=3 - -export TREETOP_DSN='treetop' -export BACKUPS_DSN='backups' - -# zelta version for pool names will include the remote -export SOURCE="${ZELTA_SRC_POOL}/${TREETOP_DSN}" -export TARGET="${ZELTA_TGT_POOL}/${BACKUPS_DSN}/${TREETOP_DSN}" - -# zfs versions for pool names do not include th remote -export SRC_TREE="$SRC_POOL/$TREETOP_DSN" -export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" diff --git a/spec/bin/all_tests_setup/install_local_zelta.sh b/spec/bin/all_tests_setup/install_local_zelta.sh deleted file mode 100755 index 4c68a95..0000000 --- a/spec/bin/all_tests_setup/install_local_zelta.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -rm -fr "${TEST_INSTALL}" -mkdir -p "${TEST_INSTALL}" -mkdir -p "$ZELTA_MAN8" -# After setting the needed environment variables we -# can use the standard install script -. ./install.sh diff --git a/spec/bin/divergent_test/divergent_snap_tree.sh b/spec/bin/divergent_test/divergent_snap_tree.sh deleted file mode 100755 index aca61a2..0000000 --- a/spec/bin/divergent_test/divergent_snap_tree.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh - -. spec/bin/divergent_test/divergent_test_env.sh - -# Add deterministic data based on snapshot name -etch () { - zfs list -Hro name -t filesystem $SRC_TREE | tr '\n' '\0' | xargs -0 -I% -n1 \ - dd if=/dev/random of='/%/file' bs=64k count=1 > /dev/null 2>&1 - zfs list -Hro name -t volume $SRC_TREE | tr '\n' '\0' | xargs -0 -I% -n1 \ - dd if=/dev/random of='/dev/zvol/%' bs=64k count=1 > /dev/null 2>&1 - zfs snapshot -r "$SRC_TREE"@snap$1 -} - -set -x -# Clean house -zfs destroy -vR "$SRC_POOL" -zfs destroy -vR "$TGT_POOL" - -# Create the setup tree -zelta backup "$SRC_POOL" "$TGT_SETUP"/sub1 -zelta backup "$SRC_POOL" "$TGT_SETUP"/sub2/orphan -zelta backup "$SRC_POOL" "${TGT_SETUP}/sub3/space name" -zfs create -vsV 16G -o volmode=dev $TGT_SETUP'/vol1' -# TO-DO: Add encrypted dataset - -# Sync the temp tree to $SRC_TREE -zelta snapshot "$TGT_SETUP"@set -zelta revert --snap-name "go" "$TGT_SETUP" -zelta backup --snap-name "one" "$TGT_SETUP" "$SRC_TREE" -zelta backup --no-snapshot "$SRC_TREE" "$TGT_TREE" -# TO-DO: Sync with exclude pattern - - -# Riddle source with special cases - -# A child with no snapshot on the source -zfs create "$SRC_TREE"/sub1/child -# A child with no snapshot on the target -zfs create -u "$TGT_TREE"/sub1/kid - -# A written target -#zfs set readonly=off "$TGT_TREE"/sub1 -#zfs mount "$TGT_TREE" -#zfs mount "$TGT_TREE"/sub1 -#touch /"$TGT_TREE"/sub1/data.file - -# An orphan -zfs destroy "$SRC_TREE"/sub2@one - -# A diverged target -zfs snapshot "$TGT_TREE/sub3/space name@blocker" - -# An unsyncable dataset -zfs destroy "$TGT_TREE"/vol1@go - -set +x - -#dd if=/dev/urandom of=/tmp/zelta-test-key bs=1m count=512 - -#zfs create -vp $SRC_TREE/'minus/two/one/0/lift off' -#zfs create -vp $SRC_TREE/'minus/two/one/0/lift off' -#for num in `jot 2`; do -# etch $num -#done -#etch 1; etch 2; etch 3 - -#etch 8 -#zelta sync "$SRC_TREE" "$TGT_TREE" -#zelta match "$SRC_TREE" "$TGT_TREE" \ No newline at end of file diff --git a/spec/bin/divergent_test/divergent_test_env.sh b/spec/bin/divergent_test/divergent_test_env.sh deleted file mode 100644 index 66af680..0000000 --- a/spec/bin/divergent_test/divergent_test_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -. spec/bin/all_tests_setup/common_test_env.sh -. spec/lib/common.sh - -export TGT_SETUP="$TGT_POOL/temp" diff --git a/spec/bin/divergent_test/divergent_test_spec.sh b/spec/bin/divergent_test/divergent_test_spec.sh deleted file mode 100644 index 929cf66..0000000 --- a/spec/bin/divergent_test/divergent_test_spec.sh +++ /dev/null @@ -1,327 +0,0 @@ -. spec/bin/divergent_test/divergent_test_env.sh - -# Custom validation functions -zelta_match_after_backup_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ - "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub2 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub2/orphan @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub3 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub3/space name @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/vol1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "8 up-to-date") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} - -match_after_rotate_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ - "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub2 @two @zelta_"*" @two syncable (incremental)"|\ - "/sub2/orphan @two @zelta_"*" @two syncable (incremental)"|\ - "/sub3 @two @zelta_"*" @two syncable (incremental)"|\ - "/sub3/space name @two @zelta_"*" @two syncable (incremental)"|\ - "/vol1 @go @zelta_"*" @go syncable (incremental)"|\ - "3 up-to-date, 5 syncable"|\ - "8 total datasets compared") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} - - -match_rotate_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "source is written; snapshotting: @zelta_"*|\ - "renaming '"*"/treetop' to '"*"/treetop_go'"|\ - "warning: insufficient snapshots; performing full backup for 2 datasets"|\ - "to ensure target is up-to-date, run: zelta backup "*" "*"/treetop"|\ - "no source: "*"/treetop/sub1/kid"|\ - *"K sent, 8 streams received in "*" seconds") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} - -match_after_first_backup_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') - - case "${normalized}" in - "source is written; snapshotting: @zelta_"*) - # New snapshot created on source - ;; - "syncing 9 datasets") - # Starting sync operation - ;; - "no source: $TGT_TREE/sub1/kid") - # Dataset exists on target but not on source - ;; - "target snapshots beyond the source match: $TGT_TREE/sub2") - # Target has snapshots newer than source's latest matching snapshot - ;; - "target snapshots beyond the source match: $TGT_TREE/sub2/orphan") - # Target has snapshots newer than source's latest matching snapshot - ;; - "target snapshots beyond the source match: $TGT_TREE/sub3/space name") - # Target has snapshots newer than source's latest matching snapshot - ;; - "no snapshot; target diverged: $TGT_TREE/vol1") - # No common snapshot found; target has diverged from source - ;; - "15K sent, 5 streams received in 0.09 seconds") - # Summary statistics - ;; - *) - echo "Unexpected line format: $line" >&2 - return 1 - ;; - esac - done -} - - -match_after_divergent_snapshots_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') - - case "$normalized" in - "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ - "[treetop] @go @go @go up-to-date"|\ - "/sub1 @go @go @go up-to-date"|\ - "/sub1/child - - - syncable (full)"|\ - "/sub1/kid - - - no source (target only)"|\ - "/sub2 @go @two @two blocked sync: target diverged"|\ - "/sub2/orphan @go @two @two blocked sync: target diverged"|\ - "/sub3 @go @two @go syncable (incremental)"|\ - "/sub3/space name @go @two @blocker blocked sync: target diverged"|\ - "/vol1 - @go - blocked sync: no target snapshots"|\ - "2 up-to-date, 2 syncable, 5 blocked"|\ - "9 total datasets compared") - # Pattern matches - ;; - *) - echo "Unexpected line format: $line" >&2 - return 1 - ;; - esac - done -} - - - -divergent_initial_match_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') - - case "$normalized" in - "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ - "[treetop] @go @go @go up-to-date"|\ - "/sub1 @go @go @go up-to-date"|\ - "/sub1/child - - - syncable (full)"|\ - "/sub1/kid - - - no source (target only)"|\ - "/sub2 @go @go @go up-to-date"|\ - "/sub2/orphan @go @go @go up-to-date"|\ - "/sub3 @go @go @go up-to-date"|\ - "/sub3/space name @go @go @blocker blocked sync: target diverged"|\ - "/vol1 - @go - blocked sync: no target snapshots"|\ - "5 up-to-date, 1 syncable, 3 blocked"|\ - "9 total datasets compared") - # Pattern matches - ;; - *) - echo "Unexpected line format: $line" >&2 - return 1 - ;; - esac - done -} - - - -validate_divergent_snap_tree_zfs_output() { - while IFS= read -r line; do - # Skip header line - #[ "$line" = "NAME USED AVAIL REFER MOUNTPOINT" ] && continue - #[[ "$line" = "NAME"*"USED"*"AVAIL"*"REFER"*"MOUNTPOINT" ]] && continue - - # Pattern: NAME * * * MOUNTPOINT - case "$line" in - "NAME"*"USED"*"AVAIL"*"REFER"*"MOUNTPOINT") continue ;; - apool*"/apool"|\ - $SRC_TREE*"/$SRC_TREE"|\ - $SRC_TREE/sub1*"/$SRC_TREE/sub1"|\ - $SRC_TREE/sub1/child*"/$SRC_TREE/sub1/child"|\ - $SRC_TREE/sub2*"/$SRC_TREE/sub2"|\ - $SRC_TREE/sub2/orphan*"/$SRC_TREE/sub2/orphan"|\ - $SRC_TREE/sub3*"/$SRC_TREE/sub3"|\ - $SRC_TREE/sub3/space\ name*"/$SRC_TREE/sub3/space name"|\ - $SRC_TREE/vol1*"-"|\ - bpool*"/bpool"|\ - bpool/backups*"/bpool/backups"|\ - $TGT_TREE*"/$TGT_TREE"|\ - $TGT_TREE/sub1*"/$TGT_TREE/sub1"|\ - $TGT_TREE/sub1/kid*"/$TGT_TREE/sub1/kid"|\ - $TGT_TREE/sub2*"/$TGT_TREE/sub2"|\ - $TGT_TREE/sub2/orphan*"/$TGT_TREE/sub2/orphan"|\ - $TGT_TREE/sub3*"/$TGT_TREE/sub3"|\ - $TGT_TREE/sub3/space\ name*"/$TGT_TREE/sub3/space name"|\ - $TGT_TREE/vol1*"-"|\ - $TGT_SETUP*"/$TGT_SETUP"|\ - $TGT_SETUP/sub1*"/$TGT_SETUP/sub1"|\ - $TGT_SETUP/sub2*"/$TGT_SETUP/sub2"|\ - $TGT_SETUP/sub2/orphan*"/$TGT_SETUP/sub2/orphan"|\ - $TGT_SETUP/sub3*"/$TGT_SETUP/sub3"|\ - $TGT_SETUP/sub3/space\ name*"/$TGT_SETUP/sub3/space name"|\ - $TGT_SETUP/vol1*"-"|\ - ${TGT_SETUP}_set*"/${TGT_SETUP}_set"|\ - ${TGT_SETUP}_set/sub1*"/${TGT_SETUP}_set/sub1"|\ - ${TGT_SETUP}_set/sub2*"/${TGT_SETUP}_set/sub2"|\ - ${TGT_SETUP}_set/sub2/orphan*"/${TGT_SETUP}_set/sub2/orphan"|\ - ${TGT_SETUP}_set/sub3*"/${TGT_SETUP}_set/sub3"|\ - ${TGT_SETUP}_set/sub3/space\ name*"${TGT_SETUP}_set/sub3/space name"|\ - ${TGT_SETUP}_set/vol1*"-") - # Pattern matches - ;; - *) - echo "Unexpected line format: $line" >&2 - return 1 - ;; - esac - done -} - -add_divergent_snapshots() { - zelta snapshot "$SOURCE"/sub2@two - zelta snapshot "$TARGET"/sub2@two -} - -Describe 'confirm zfs setup' - before_all() { - %logger "-- before_all: confirm zfs setup" - echo - } - - after_all() { - %logger "-- after_all: confirm zfs setup" - } - - - #BeforeAll before_all - #AfterAll after_all - - Describe 'zfs list output validation' - It 'matches expected pattern for each line' - When call exec_on "$TGT_SVR" zfs list -r -H $SRC_POOL $TGT_POOL - - The output should satisfy validate_divergent_snap_tree_zfs_output - End - End - - Describe 'check initial zelta match state' - It "initial match has 5 up-to-date, 1 syncable, 3 blocked, with 9 total datasets compared" - When call zelta match $SOURCE $TARGET - The output should satisfy divergent_initial_match_output - End - End - - Describe 'add incremental source snapshot' - It "adds $SOURCE/sub3@two snapshot" - When call zelta snapshot "$SOURCE"/sub3@two - The output should equal "snapshot created '$SRC_TREE/sub3@two'" - The stderr should be blank - The status should eq 0 - End - End - - Describe 'add divergent snapshots of same name' - It "adds divergent snapshots for $SOURCE/sub2@two and $TARGET/sub2@two" - When call add_divergent_snapshots - The line 1 of output should equal "snapshot created '$SRC_TREE/sub2@two'" - The line 2 of output should equal "snapshot created '$TGT_TREE/sub2@two'" - The stderr should be blank - The status should eq 0 - End - End - - Describe 'check zelta match after divergent snapshots' - It "after divergent snapshot match has 2 up-to-date, 2 syncable, 5 blocked, with 9 total datasets compared" - When call zelta match $SOURCE $TARGET - The output should satisfy match_after_divergent_snapshots_output - End - End -End - - -Describe 'Divergent match, rotate, match' - It "shows current match for divergent $SOURCE and $TARGET" - When call zelta match $SOURCE $TARGET - The output should satisfy match_after_divergent_snapshots_output - End - - It "rotate divergent $SOURCE and $TARGET" - When call zelta rotate $SOURCE $TARGET - The output should satisfy match_rotate_output - The stderr should equal "warning: insufficient snapshots; performing full backup for 2 datasets" - The status should equal 0 - End - - It "match $SOURCE and $TARGET after divergent rotate" - When call zelta match $SOURCE $TARGET - The output should satisfy match_after_rotate_output - The status should equal 0 - End -End - - -Describe 'Divergent backup, then match' - It "backup divergent $SOURCE to $TARGET" - When call zelta backup $SOURCE $TARGET - The output line 1 should equal "syncing 8 datasets" - The output line 2 should equal "8 datasets up-to-date" - The output line 3 should match pattern "* sent, 5 streams received in * seconds" - The status should equal 0 - End - - It "match after backup" - When call zelta backup $SOURCE $TARGET - The output should satisfy zelta_match_after_backup_output - The status should equal 0 - End -End - - diff --git a/spec/bin/hello_example.sh b/spec/bin/hello_example.sh deleted file mode 100644 index 4b39f7e..0000000 --- a/spec/bin/hello_example.sh +++ /dev/null @@ -1,23 +0,0 @@ -# simple test showing helping function inclusion, logging and output matching -# simple example spec - -Describe 'hello shellspec' - Include spec/lib/hello.sh - setup() { - %logger "-- hello spec setup" - %logger "-- reference this example to help you get started with writing tests" - #spec/initialize/initialize_testing_setup.sh - } - - cleanup() { - %logger "-- hello spec cleanup " - } - - BeforeAll 'setup' - AfterAll 'cleanup' - It 'says hello' - When call hello ShellSpec - %logger "Your temp dir is {$SHELLSPEC_TMPBASE}" - The output should match pattern "What's up? Hello ShellSpec! TMPDIR: *" - End -End diff --git a/spec/bin/one_time_setup/setup_sudoers.sh b/spec/bin/one_time_setup/setup_sudoers.sh deleted file mode 100755 index 33f2716..0000000 --- a/spec/bin/one_time_setup/setup_sudoers.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/sh - -# This scripts runs as ssh on the designated remote host -# and there is no environment set. We make the current directory -# the location of the git clone for zelta -cd_to_git_clone_dir() { - script_dir=$(cd "$(dirname "$0")" && pwd) - parent_dir=$(dirname "$script_dir") - cd "$parent_dir/../.." || exit 1 - cur_dir=$(pwd) - echo "git zelta clone directory is: {$cur_dir}" -} - -cd_to_git_clone_dir - -. spec/bin/all_tests_setup/common_test_env.sh - -echo "Setting sudo for backup user {$BACKUP_USER}" -echo "For zelta-dev root {$ZELTA_DEV_PATH}" - -# Detect OS and set paths -if [ -f /etc/os-release ]; then - . /etc/os-release - os_name="$ID" -else - os_name=$(uname -s | tr '[:upper:]' '[:lower:]') -fi - -# Set ZFS/zpool paths based on OS -case "$os_name" in - ubuntu|debian|linux) - zfs_path="/usr/sbin/zfs" - zpool_path="/usr/sbin/zpool" - mount_path="/usr/bin/mount" - mkdir_path="/usr/bin/mkdir" - sudoers_dir="/etc/sudoers.d" - ;; - freebsd) - zfs_path="/sbin/zfs" - zpool_path="/sbin/zpool" - mount_path="/sbin/mount" - mkdir_path="/bin/mkdir" - sudoers_dir="/usr/local/etc/sudoers.d" - ;; - *) - echo "Unsupported OS: $os_name" >&2 - exit 1 - ;; -esac - -# Sudoers entry -setup_script="${ZELTA_DEV_PATH}/spec/bin/ssh_tests_setup/setup_zfs_pools_on_remote.sh" -sudoers_entry="${BACKUP_USER} ALL=(ALL) NOPASSWD: ${zpool_path}, ${zfs_path}, ${mount_path}, ${mkdir_path}, ${setup_script}" - -# Sudoers file location -sudoers_file="$sudoers_dir/zelta-${BACKUP_USER}" - -# Check if running as root -if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run as root" >&2 - exit 1 -fi - -# Check if user exists -if ! id "$BACKUP_USER" >/dev/null 2>&1; then - echo "User '$BACKUP_USER' does not exist" >&2 - exit 1 -fi - -# Create sudoers entry -cat > "$sudoers_file" << EOF -# Allow $BACKUP_USER to run ZFS commands without password for zelta testing -# NOTE: This is for test environments only - DO NOT use in production -$sudoers_entry -EOF - -# Set correct permissions -chmod 0440 "$sudoers_file" - -# Validate the sudoers file -if visudo -c -f "$sudoers_file" >/dev/null 2>&1; then - echo "Successfully created sudoers entry at: $sudoers_file" - echo "Entry: $sudoers_entry" -else - echo "ERROR: Invalid sudoers syntax, removing file" >&2 - rm -f "$sudoers_file" - exit 1 -fi diff --git a/spec/bin/opts_test/opts_spec.sh b/spec/bin/opts_test/opts_spec.sh deleted file mode 100644 index fb8ae9a..0000000 --- a/spec/bin/opts_test/opts_spec.sh +++ /dev/null @@ -1,223 +0,0 @@ -# Option parsing validation tests for Zelta -# Configurable endpoints - override via environment or test_env.sh -# -# Usage: -# # With defaults (local testpool) -# shellspec spec/bin/opts_test/opts_spec.sh -# -# # With remote endpoints -# SRC_ENDPOINT=user@host:pool/src TGT_ENDPOINT=backupuser@backuphost:backuppool/tgt shellspec spec/bin/opts_test/opts_spec.sh - -# Load configurable environment -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -if [ -f "${SCRIPT_DIR}/test_env.sh" ]; then - . "${SCRIPT_DIR}/test_env.sh" -fi - -# Fallback defaults if test_env.sh not loaded or vars not set -: "${SRC_ENDPOINT:=testpool/source}" -: "${TGT_ENDPOINT:=testpool/target}" -: "${CLONE_ENDPOINT:=${SRC_ENDPOINT}_clone}" -: "${CLONE_ENDPOINT_INVALID:=otherhost:differentpool/clone}" -: "${TEST_SNAP_NAME:=test_snapshot}" -: "${TEST_DEPTH:=2}" -: "${TEST_EXCLUDE:=*/swap,@*_hourly}" - -Describe "Option Parsing Validation" - - Describe "zelta backup with comprehensive options" - It "parses all major options and produces valid JSON output" - When run zelta backup \ - --dryrun \ - --depth "$TEST_DEPTH" \ - --exclude "$TEST_EXCLUDE" \ - --snap-name "$TEST_SNAP_NAME" \ - --snapshot \ - --intermediate \ - --resume \ - --push \ - --send-default '-Lce' \ - --send-raw '-Lw' \ - --send-new '-p' \ - --recv-default '' \ - --recv-top '-o readonly=on' \ - --recv-fs '-u -x mountpoint -o canmount=noauto' \ - --recv-vol '-o volmode=none' \ - -o 'compression=lz4' \ - -x 'mountpoint' \ - --json \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - The output should include '"sourceEndpoint":' - The output should include '"targetEndpoint":' - The output should include '"replicationStreamsSent":' - The stderr should not include "error" - The stderr should not include "invalid" - End - End - - Describe "zelta backup with alternative option forms" - It "parses short options and alternative flags" - When run zelta backup \ - -n \ - -qq \ - -d 1 \ - -X '/tmp,*/cache' \ - -i \ - --no-resume \ - --pull \ - --no-snapshot \ - -L \ - --largeblock \ - --compressed \ - --embed \ - --props \ - --raw \ - -u \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - The stderr should equal "" - End - End - - Describe "zelta backup with override options" - It "parses send and recv override options" - When run zelta backup \ - --dryrun \ - -qq \ - --send-override '-Lce' \ - --recv-override '-o readonly=on' \ - --recv-pipe 'cat' \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - The stderr should equal "" - End - End - - Describe "zelta match options" - It "parses match-specific options and produces output" - When run zelta match \ - -H \ - -p \ - -o 'ds_suffix,match,xfer_size' \ - --written \ - --time \ - -d "$TEST_DEPTH" \ - -X '*/swap' \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - The output should be defined - End - End - - Describe "zelta snapshot options" - It "parses snapshot options in dryrun mode" - When run zelta snapshot \ - --dryrun \ - -qq \ - --snap-name 'manual_test' \ - -d 1 \ - "$SRC_ENDPOINT" - The status should equal 0 - End - End - - Describe "zelta clone options" - It "parses clone options in dryrun mode" - When run zelta clone \ - --dryrun \ - -qq \ - --snapshot \ - --snap-name 'clone_snap' \ - -d "$TEST_DEPTH" \ - "$SRC_ENDPOINT" "$CLONE_ENDPOINT" - The status should equal 0 - End - End - - Describe "zelta clone endpoint validation" - It "rejects clone to mismatched pool/host" - When run zelta clone \ - --dryrun \ - -qq \ - "$SRC_ENDPOINT" "$CLONE_ENDPOINT_INVALID" - The status should not equal 0 - The stderr should include "cannot clone" - End - End - - Describe "zelta rotate options" - It "parses rotate options in dryrun mode" - When run zelta rotate \ - --dryrun \ - -qq \ - --no-snapshot \ - --push \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - End - End - - Describe "zelta revert options" - It "parses revert options in dryrun mode" - When run zelta revert \ - --dryrun \ - -qq \ - "$SRC_ENDPOINT" - The status should equal 0 - End - End - - Describe "zelta prune options" - It "parses prune options" - When run zelta prune \ - --dryrun \ - -qq \ - --keep-snap-days 90 \ - --keep-snap-num 100 \ - --no-ranges \ - -d "$TEST_DEPTH" \ - -X '*/tmp' \ - "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should equal 0 - End - End - - Describe "deprecated option warnings" - It "warns about deprecated -s option" - When run zelta backup --dryrun -qq -s "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The stderr should include "deprecated" - End - - It "warns about deprecated -t option" - When run zelta backup --dryrun -qq -t "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The stderr should include "deprecated" - End - - It "warns about deprecated -T option" - When run zelta backup --dryrun -qq -T "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The stderr should include "deprecated" - End - End - - Describe "invalid option handling" - It "rejects invalid options gracefully" - When run zelta backup --dryrun -qq --invalid-option-xyz "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should not equal 0 - The stderr should include "invalid" - End - - It "rejects deprecated --initiator option" - When run zelta backup --dryrun -qq --initiator PULL "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should not equal 0 - The stderr should include "deprecated" - End - - It "rejects deprecated --progress option" - When run zelta backup --dryrun -qq --progress "$SRC_ENDPOINT" "$TGT_ENDPOINT" - The status should not equal 0 - The stderr should include "deprecated" - End - End - -End diff --git a/spec/bin/opts_test/test_env.sh b/spec/bin/opts_test/test_env.sh deleted file mode 100644 index 6025c66..0000000 --- a/spec/bin/opts_test/test_env.sh +++ /dev/null @@ -1,36 +0,0 @@ -# Configurable test environment for opts_test -# Override these via environment variables before running tests -# -# Examples: -# export SRC_ENDPOINT="user@host:testpool/source" -# export TGT_ENDPOINT="otheruser@backuphost:backuppool/target" -# -# Or for local testing: -# export SRC_ENDPOINT="testpool/source" -# export TGT_ENDPOINT="backuppool/target" -# -# Run tests: -# shellspec spec/bin/opts_test/opts_spec.sh - -# Source endpoint - the dataset tree to back up / match / snapshot -: "${SRC_ENDPOINT:=testpool/source}" - -# Target endpoint - where backups go (can be completely different host/pool) -: "${TGT_ENDPOINT:=testpool/target}" - -# Clone endpoint - for zelta clone tests -# Must be on same pool as source (user@host:pool must match exactly) -# Default: derive from SRC_ENDPOINT by appending _clone to the dataset path -: "${CLONE_ENDPOINT:=${SRC_ENDPOINT}_clone}" - -# Invalid clone endpoint - for testing clone failure on mismatched pool/host -: "${CLONE_ENDPOINT_INVALID:=otherhost:differentpool/clone}" - -# Snapshot name for tests that create snapshots -: "${TEST_SNAP_NAME:=test_snapshot}" - -# Depth limit for recursive operations (0 = unlimited) -: "${TEST_DEPTH:=2}" - -# Exclusion patterns for testing -: "${TEST_EXCLUDE:=*/swap,@*_hourly}" diff --git a/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh b/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh deleted file mode 100755 index a749077..0000000 --- a/spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -. spec/bin/all_tests_setup/common_test_env.sh -. spec/lib/script_util.sh - -if ! validate_tree_name "$@"; then - return 1 -fi - -echo "TREE_NAME is {$TREE_NAME}" - -# Use a string for the following remote setup so sudo password only has to be entered once. -# pull down zelta from github into a clean dir -# checkout the test branch -# update sudoers -# setup the test env, install zelta, create pools -# create the requested snap tree - - -printf "\n*** Enter sudo password for remote setup:\n" -ssh -t ${BACKUP_USER}@${REMOTE_TEST_HOST} " - set -e && - set -x && - - sudo rm -fr ${ZELTA_GIT_CLONE_DIR} && - - mkdir -p ${ZELTA_GIT_CLONE_DIR} && - chmod 777 ${ZELTA_GIT_CLONE_DIR} && - git clone https://github.com/bellhyve/zelta.git ${ZELTA_GIT_CLONE_DIR} && - cd ${ZELTA_GIT_CLONE_DIR} && - git checkout ${GIT_TEST_BRANCH} && - - sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/one_time_setup/setup_sudoers.sh && - sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/all_tests_setup/all_tests_setup.sh && - sudo ${ZELTA_GIT_CLONE_DIR}/spec/bin/${TREE_NAME}_test/${TREE_NAME}_snap_tree.sh && - - echo 'Remote setup complete' -" diff --git a/spec/bin/standard_test/standard_snap_tree.sh b/spec/bin/standard_test/standard_snap_tree.sh deleted file mode 100755 index 5821f18..0000000 --- a/spec/bin/standard_test/standard_snap_tree.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/sh - -. spec/bin/standard_test/standard_test_env.sh - -DATASETS="${SRC_TREE} ${TGT_TREE}" - -dataset_exists() { - exec_cmd zfs list "$1" &>/dev/null - return $? -} - -#create_tree_via_zfs() { -# exec_cmd sudo zfs create -vp "$SRC_TREE" -# exec_cmd sudo zfs create -vp "$SRC_TREE/$ALL_DATASETS" -# exec_cmd sudo zfs create -vp "$TGT_POOL/$BACKUPS_DSN" -# #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' -#} - -#export SRC_TREE="$SRC_POOL/$TREETOP_DSN" -#export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" - -#try1_create_test_tree() { -# local pool="$1" -# local root="$2" -# local mount_base="$3" -# -# mkdir -p "${mount_base}/${root}/one/two/three" -# zfs create -o mountpoint="${mount_base}/${root}" "${pool}/${root}" -# zfs create -o mountpoint="${mount_base}/${root}/one" "${pool}/${root}/one" -# zfs create -o mountpoint="${mount_base}/${root}/one/two" "${pool}/${root}/one/two" -# zfs create -o mountpoint="${mount_base}/${root}/one/two/three" "${pool}/${root}/one/two/three" -# -# echo "Test tree created successfully" -# echo "Mounted at: $mount_base" -#} - - -new_create_tree_via_zfs() { - #exec_cmd zfs create -o mountpoint="${mount_base}/$TREETOP_DSN} -vp "$SRC_TREE" - mkdir -p "${ZFS_MOUNT_BASE}/${TREETOP_DSN}" - mkdir -p "${ZFS_MOUNT_BASE}/${BACKUPS_DSN}" - exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${TREETOP_DSN}" -vp "$SRC_POOL/$TREETOP_DSN" - #exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${TREETOP_DSN}" -vp "$SRC_POOL/$TREETOP_DSN" - #$SRC_TREE/$ALL_DATASETS" - exec_cmd echo zfs create -o mountpoint="${ZFS_MOUNT_BASE}/${BACKUPS_DSN}" -vp "$TGT_POOL/$BACKUPS_DSN" - #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' -} - -old_create_tree_via_zfs() { - exec_cmd sudo zfs create -vp "$SRC_TREE" - exec_cmd sudo zfs create -vp "$SRC_TREE/$ALL_DATASETS" - exec_cmd sudo zfs create -vp "$TGT_POOL/$BACKUPS_DSN" - #sudo zfs create -vsV 16G -o volmode=dev $SRCTREE'/vol1' -} - - -create_tree_via_zelta() { - exec_cmd zelta backup "$SRC_POOL" "$TGT_POOL/$TREETOP_DSN/$ALL_DATASETS" - exec_cmd zelta revert "$TGT_POOL/$TREETOP_DSN" - exec_cmd zelta backup "$TGT_POOL/$TREETOP_DSN" "$SRC_POOL/$TREETOP_DSN" -} - -#rm_test_datasets() { -# for dataset in "${DATASETS[@]}"; do -# if zfs list "$dataset" &>/dev/null; then -# echo "Destroying $dataset..." -# exec_cmd sudo zfs destroy -vR "$dataset" -# else -# echo "Skipping $dataset (does not exist)" -# fi -# done -#} - -rm_all_datasets_for_pool() { - poolname=$1 - - # shellcheck disable=SC2120 - reverse_lines() { - awk '{lines[NR]=$0} END {for(i=NR;i>0;i--) print lines[i]}' "$@" - } - - echo "removing all datasets for pool {$poolname}" - dataset_list=$(zfs list -H -o name -t filesystem,volume -r $poolname | grep -v "^${poolname}$" | reverse_lines) - echo $dataset_list - - zfs list -H -o name -t filesystem,volume -r $poolname | grep -v "^${poolname}\$" | reverse_lines | while IFS= read -r dataset; do - exec_cmd sudo zfs destroy -r "$dataset" - done - -} - -x_rm_test_datasets() { - for dataset in $DATASETS; do - #for dataset in 'apool'; do - if dataset_exists "$dataset"; then - echo "found dataset, please delete it" - else - echo "there is no dataset to remove " - fi - - if exec_cmd sudo zfs list "$dataset" &>/dev/null; then - #echo "Destroying $dataset..." - echo "need to destroy dataset $dataset" - #exec_cmd zfs destroy -vR "$dataset" - else - echo "Skipping $dataset (does not exist)" - fi - done -} - -setup_simple_snap_tree() { - #set -x - echo "Make a fresh test tree" - #rm_test_datasets - rm_all_datasets_for_pool $SRC_POOL - rm_all_datasets_for_pool $TGT_POOL - #new_create_tree_via_zfs - #try1_create_test_tree "$SRC_POOL" "$TREETOP_DSN" "$ZFS_MOUNT_BASE" - old_create_tree_via_zfs - - # TODO: create via zelta - #create_tree_via_zelta - #setup_zfs_allow - - TREE_STATUS=$? - #set +x - #true - return $TREE_STATUS -} - - -#set -x -setup_simple_snap_tree -#set +x - -#STATUS=$? -#echo "status: $STATUS" diff --git a/spec/bin/standard_test/standard_test_env.sh b/spec/bin/standard_test/standard_test_env.sh deleted file mode 100755 index 9ffbdc8..0000000 --- a/spec/bin/standard_test/standard_test_env.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -. spec/bin/all_tests_setup/common_test_env.sh -. spec/lib/common.sh - -export ALL_DATASETS="one/two/three" diff --git a/spec/bin/standard_test/standard_test_spec.sh b/spec/bin/standard_test/standard_test_spec.sh deleted file mode 100644 index 0b4a093..0000000 --- a/spec/bin/standard_test/standard_test_spec.sh +++ /dev/null @@ -1,152 +0,0 @@ -. spec/bin/standard_test/standard_test_env.sh -. spec/lib/common.sh - -# valid xtrace usage if found -validate_options() { - # Direct test - executes if function returns 0 (success) - if ! check_if_xtrace_usage_valid; then - echo "xtrace options are not correct" >&2 - echo "to show expectations use --shell bash and bash version >= 4" >&2 - echo "NOTE Use: --xtrace --shell bash" >&2 - return 1 - fi - return 0 -} - - -# allow for 2 vaild matches for current shellspec line/subject being considered -match_either() { - case $SHELLSPEC_SUBJECT in - "$1"|"$2") - return 0 - ;; - *) - return 1 - ;; - esac -} - -# TODO: is it possible to setup a snap tree on FreeBSD as the backup user? -# TODO: when should this code be removed: left over from an attempt to setup a snap tree prior to start of every test -#global_setup_function() { -# %putsn "global_setup_function" -# %putsn "before: SRC_POOL=$SRC_POOL, TGT_POOL=$TGT_POOL" -# #export SRC_SVR=dever@fzfsdev: -# #export TGT_SVR=dever@fzfsdev: -# #SRC_POOL='apool' -# #TGT_POOL='bpool' -# export TREETOP_DSN='treetop' -# export BACKUPS_DSN='backups' -# export SOURCE=${SRC_SVR}${SRC_POOL}/${TREETOP_DSN} -# export TARGET=${TGT_SVR}${TGT_POOL}/${BACKUPS_DSN} -# export SRC_TREE="$SRC_POOL/$TREETOP_DSN" -# export TGT_TREE="$TGT_POOL/$BACKUPS_DSN/$TREETOP_DSN" -# export ALL_DATASETS="one/two/three" -# %putsn "after: SRC_POOL=$SRC_POOL, TGT_POOL=$TGT" -# CWD=$(pwd) -# #sudo /home/dever/src/repos/zelta/spec/initialize/setup_simple_snap_tree.sh -# #./spec/initializize/setup_simple_snap_tree.sh -# %putsn "current dir {$CWD}" -# %putsn "current dir {$CWD}" -# %putsn "current dir {$CWD}" -# %putsn "current dir {$CWD}" -# %putsn "current dir {$CWD}" -#} - - -BeforeAll validate_options - -Describe 'confirm zfs setup' - before_all() { - %logger "-- before_all: confirm zfs setup" - echo - } - - after_all() { - %logger "-- after_all: confirm zfs setup" - } - - #BeforeAll before_all - #AfterAll after_all - - It "has good initial SRC_POOL:{$SRC_POOL} simple snap tree" - When call exec_on "$SRC_SVR" zfs list -r "$SRC_POOL" - The line 2 of output should match pattern "* /$SRC_POOL" - The line 3 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN" - The line 4 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one" - The line 5 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one/two" - The line 6 of output should match pattern "* /$SRC_POOL/$TREETOP_DSN/one/two/three" - End - - It "has good initial TGT_POOL:{$TGT_POOL} simple snap tree" - When call exec_on "$TGT_SVR" zfs list -r "$TGT_POOL" - The line 2 of output should match pattern "* /$TGT_POOL" - End -End - -Describe 'try backup' - - It 'backs up the initial tree' - When call zelta backup $SOURCE $TARGET - - The line 1 of output should match pattern "source is written; snapshotting: @zelta_*" - The line 2 of output should equal "syncing 4 datasets" - The line 3 of output should match pattern "* sent, 4 streams received in * seconds" - The status should eq 0 - End - - It 'has valid backup' - When call exec_on "$TGT_SVR" zfs list -r "$TGT_POOL" - The line 2 of output should match pattern "$TGT_POOL * /$TGT_POOL" - The line 3 of output should match pattern "$TGT_POOL/$BACKUPS_DSN * /$TGT_POOL/$BACKUPS_DSN" - The line 4 of output should match pattern "$TGT_TREE * /$TGT_TREE" - The line 5 of output should match pattern "$TGT_TREE/one * /$TGT_TREE/one" - The line 6 of output should match pattern "$TGT_TREE/one/two * /$TGT_TREE/one/two" - The line 7 of output should match pattern "$TGT_TREE/one/two/three * /$TGT_TREE/one/two/three" - - The stderr should be blank - The status should eq 0 - End - - Parameters - 4 $SRC_TREE $SRC_SVR - 4 $TGT_TREE $TGT_SVR - End - - It "has $1 snapshots on ${3:-localhost} ${2:+matching pattern '$2'}" - When call snapshot_count $1 $2 $3 - The stderr should be blank - The status should eq 0 - End -End - -Describe 'zelta rotate' - It 'rotates the backed up tree' - # force snapshot timestamp to be at 1 second in future to prevent backup snapshot conflict - sleep 1 - - When call zelta rotate $SOURCE $TARGET - - # TODO: verify that '$SRC_TREE and TGT_TREE' will work for remotes, or if i need to use $SOURCE and $TARGET instead - The line 1 of output should match pattern "action requires a snapshot delta; snapshotting: @zelta_*" - The line 2 of output should match pattern "rotating from source: ${SRC_TREE}@zelta_*" - The line 3 of output should match pattern "renaming '${TGT_TREE}' to '${TGT_TREE}_zelta_*'" - The line 4 of output should match pattern "to ensure target is up-to-date, run: zelta backup ${SOURCE} ${TARGET}" - The line 5 of output should match pattern "* datasets up-to-date" - The line 6 of output should match pattern "* sent, * streams received in * seconds" - The stderr should be blank - The status should eq 0 - End - - Parameters - 16 '^(apool|bpool)' $TGT_SVR - 8 apool/treetop $TGT_SVR - 8 bpool/backups/treetop $TGT_SVR - End - - It "has $1 snapshots on ${3:-localhost} ${2:+matching pattern '$2'}" - When call snapshot_count $1 $2 $3 - The stderr should be blank - The status should eq 0 - End -End diff --git a/spec/doc/vm/README.md b/spec/doc/vm/README.md deleted file mode 100644 index 60bdae8..0000000 --- a/spec/doc/vm/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Setting up a Ubuntu VM - -- [Installing kvm/qemu/virt-manager](./installing-kvm.md) -- [Creating a Ubuntu VM](./creation.md) -- [ZFS Configuration](./zfs-configuration.md) diff --git a/spec/doc/vm/creation.md b/spec/doc/vm/creation.md deleted file mode 100644 index ef71fca..0000000 --- a/spec/doc/vm/creation.md +++ /dev/null @@ -1,41 +0,0 @@ -# VM Creation Guide - - -## setting up an Ubuntu VM on Ubuntu - -```shell -#!/bin/sh -# Download Ubuntu ISO -#wget https://releases.ubuntu.com/noble/ubuntu-24.04.3-desktop-amd64.iso - -set -x - -UBUNTU_ISO_SRC_DIR="/home/dever/Downloads" -UBUNTU_ISO_VIRT_DIR="/var/lib/libvirt/boot" -UBUNTU_ISO_NAME="ubuntu-24.04.3-live-server-amd64.iso" - -UBUNTU_SRC_ISO="$UBUNTU_ISO_SRC_DIR/$UBUNTU_ISO_NAME" -UBUNTU_TGT_ISO="$UBUNTU_ISO_VIRT_DIR/$UBUNTU_ISO_NAME" - -sudo mkdir -p "${UBUNTU_ISO_VIRT_DIR}" -sudo cp -f "${UBUNTU_SRC_ISO}" "$UBUNTU_TGT_ISO" - -# Optional: lock down permissions -sudo chmod 644 "$UBUNTU_TGT_ISO" -sudo chown root:root "$UBUNTU_TGT_ISO" - - -sudo virt-install \ - --name zfs-dev \ - --ram 8192 \ - --disk path=/var/lib/libvirt/images/zfs-dev.qcow2,size=40,format=qcow2 \ - --vcpus 4 \ - --os-variant ubuntu24.04 \ - --network bridge=virbr0 \ - --graphics spice \ - --cdrom "${UBUNTU_TGT_ISO}" - - -set +x - -``` diff --git a/spec/doc/vm/installing-kvm.md b/spec/doc/vm/installing-kvm.md deleted file mode 100644 index 143d132..0000000 --- a/spec/doc/vm/installing-kvm.md +++ /dev/null @@ -1,55 +0,0 @@ -``` -#!/bin/sh - -set -x - -# 1. Check if your CPU supports virtualization -# If this returns > 0, you're good -if egrep -c '(vmx|svm)' /proc/cpuinfo; then - echo "CPU supports virtualizaton, install KVM/QEMU and virt-manager" -else - echo "your cpu does not support virualization, cannot install KVM!" - return 1 -fi - - -# 2. Install KVM and tools -sudo apt update -sudo apt install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils virt-manager - -# 3. Add your user to libvirt groups -sudo usermod -aG libvirt dever -sudo usermod -aG kvm dever - -# 4. Start the libvirt service -sudo systemctl enable --now libvirtd - -# 5. Verify installation -sudo virsh list --all - -# 6. Log out and back in (for group membership to take effect) - -set +x -``` - - -``` -#!/bin/sh - -set -x -sudo apt install -y virt-viewer libvirt-daemon-system libvirt-clients qemu-kvm -sudo systemctl enable --now libvirtd -sudo systemctl status libvirtd - -sudo usermod -aG libvirt,kvm $USER -id $USER -virsh -c qemu:///system list -virt-viewer --connect qemu:///system zfs-dev - -set +x - - -# ToDo -echo "on the VM run:" -echo "sudo systemctl enable --now serial-getty@ttyS0.service" -``` diff --git a/spec/doc/vm/running.md b/spec/doc/vm/running.md deleted file mode 100644 index e69de29..0000000 diff --git a/spec/doc/vm/zfs-configuration.md b/spec/doc/vm/zfs-configuration.md deleted file mode 100644 index 2fcf972..0000000 --- a/spec/doc/vm/zfs-configuration.md +++ /dev/null @@ -1,13 +0,0 @@ -# ZFS setup for zelta - - -## Install ZFS - 1. Update package lists `sudo apt update` - 2. Install ZFS userspace tools `sudo apt install zfsutils-linux` - -## Verify ZFS version -- _Verify ZFS is installed and kernel and user tools versions match_ - ``` - zfs version - cat /sys/module/zfs/version - ``` diff --git a/spec/lib/common.sh b/spec/lib/common.sh deleted file mode 100644 index 3fdba82..0000000 --- a/spec/lib/common.sh +++ /dev/null @@ -1,206 +0,0 @@ -# shellcheck shell=sh - -GREEN=$(printf '\033[32m') -RED=$(printf '\033[31m') -NC=$(printf '\033[0m') - -#printf "%sThis is red%s\n" "$RED" "$NC" - -check_zfs_installed() { - # Check if zfs is already on PATH - if ! command -v zfs >/dev/null 2>&1; then - # Allow user to override ZFS_BIN location, default to /usr/local/sbin - ZFS_BIN="${ZFS_BIN:-/usr/local/sbin}" - - # Add ZFS_BIN to PATH if not already present - case ":$PATH:" in - *":$ZFS_BIN:"*) ;; - *) PATH="$ZFS_BIN:$PATH" ;; - esac - export PATH - - # Verify zfs command is now available - if ! command -v zfs >/dev/null 2>&1; then - echo "Error: zfs command not found. Please set ZFS_BIN to the correct location." >&2 - return 1 - fi - fi -} - - -echo_alert() { - msg=$1 - printf "${RED}[** alert **] %s${NC}\n", "$msg" -} - -exec_cmd() { - CMD=$(printf "%s " "$@") - CMD=${CMD% } # trim trailing space - #CMD="$@" - if "$@"; then - [ "${EXEC_CMD_QUIET:-}" != "1" ] && printf "${GREEN}[success] %s${NC}\n" "${CMD}" - return 0 - else - _exit_code=$? - [ "${EXEC_CMD_QUIET:-}" != "1" ] && printf "${RED}[failed] %s returned %d${NC}\n" "${CMD}" "$_exit_code" - return "$_exit_code" - fi -} - -exec_on() { - local server="$1" - shift - - if [ -n "$server" ]; then - ssh "$server" "$@" - else - "$@" - fi -} - -snapshot_count() { - expected_count=$1 - pattern=$2 # Optional regex pattern - svr=$3 # Optional server name - - # Validate arguments - if [ -z "$expected_count" ]; then - echo "Error: snapshot_count requires expected_count argument" >&2 - return 1 - fi - - # Validate expected_count is a number - case "$expected_count" in - ''|*[!0-9]*) - echo "Error: expected_count must be a number" >&2 - return 1 - ;; - esac - - # Get snapshot list - snapshot_list=$(exec_on "$svr" zfs list -t snapshot -H -o name) - - # Count snapshots, optionally filtering by pattern - if [ -n "$pattern" ]; then - # Count only snapshots matching the pattern - snapshot_count=$(echo "$snapshot_list" | grep -E "$pattern" | wc -l) - else - # Count all snapshots - snapshot_count=$(echo "$snapshot_list" | wc -l) - fi - - # Test the count - if [ "$expected_count" -eq "$snapshot_count" ]; then - return 0 - else - if [ -n "$pattern" ]; then - echo "Expected $expected_count snapshots matching pattern '$pattern', found $snapshot_count" >&2 - else - echo "Expected $expected_count snapshots, found $snapshot_count" >&2 - fi - return 1 - fi -} - -# Shellspec has a nice tracing feature when you specify --xtrace, but it doesn't execute -# expectations unless you use --shell bash, and the bash shell has to be >= version 4. -# Using --xtrace without --shell is an easy mistake to make and it looks like tests are -# passing when they are not, as no expectations are run. Therefore, we use this function -# to check if --xtrace has been specifie -# d, we assert that --shell bash is also present. -# see https://deepwiki.com/shellspec/shellspec/5.1-command-line-options#tracing-and-profiling -#check_if_xtrace_expectations_supported() { -check_if_xtrace_usage_valid() { - # use --shell bash --xtrace to see trace of execution and evaluates expectations - # bash version must be >= 4 - - # Return error if SHELLSPEC_XTRACE is defined, SHELLSPEC_SHELL contains bash, - # and bash version is less than 4 - if [ -n "$SHELLSPEC_XTRACE" ]; then - #echo "*** checking SHELLSPEC_SHELL: {$SHELLSPEC_SHELL}" - if echo "$SHELLSPEC_SHELL" | grep -q bash; then - #echo "*** found bash: {$SHELLSPEC_SHELL}" - if [ -n "$BASH_VERSION" ]; then - # Extract major version (first element of BASH_VERSINFO) - if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then - echo "Error: xtrace with bash requires version 4 or higher (current: $BASH_VERSION)" >&2 - return 1 - fi - else - # SHELLSPEC_SHELL contains bash but we're not running in bash - # Try to check the version of the specified bash - bash_version=$("$SHELLSPEC_SHELL" --version 2>/dev/null | head -n1) - if echo "$bash_version" | grep -q "version [0-3]\."; then - echo "Error: xtrace with bash requires version 4 or higher (detected: $bash_version)" >&2 - return 1 - fi - fi - else - echo "Error: --xtrace requires bash shell, please add the option --shell bash" >&2 - return 1 - fi - fi -} - -setup_linux_zfs_allow() { - export SRC_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" - export TGT_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" -} - -setup_freebsd_zfs_allow() { - export SRC_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" - export TGT_ZFS_CMDS="send,snapshot,hold,bookmark,create,readonly,receive,volmode,mount,mountpoint,canmount,rename" -} - -setup_linux_env() { - setup_linux_zfs_allow - export POOL_TYPE="$LOOP_DEV_POOL" - export ZELTA_AWK=mawk -} - -setup_freebsd_env() { - setup_freebsd_zfs_allow - export POOL_TYPE="$MEMORY_DISK_POOL" -} - -setup_zfs_allow() { - exec_cmd sudo zfs allow -u "$BACKUP_USER" "$SRC_ZFS_CMDS" "$SRC_POOL" - exec_cmd sudo zfs allow -u "$BACKUP_USER" "$TGT_ZFS_CMDS" "$TGT_POOL" -} - -setup_os_specific_env() { - # uname is the most reliable cross‑platform starting point - OS_TYPE=$(uname -s) - echo "Settings OS specific environment for {$OS_TYPE}" - - # Check for Ubuntu specifically - if [ -f /etc/os-release ]; then - . /etc/os-release - if [ "$ID" = "ubuntu" ]; then - # NOTE: This isn't being used currently - export LINUX_DISTRO_IS_UBUNTU=1 - fi - fi - - case "$OS_TYPE" in - Linux) - # Linux distros - setup_linux_env - ;; - FreeBSD|Darwin) - setup_freebsd_env - ;; - *) - echo "$OS_TYPE: Unsupported OS_TYPE: {$OS_TYPE}" >&2 - return 1 - ;; - esac - - echo "OS_TYPE: $OS_TYPE: set POOL_TYPE={$POOL_TYPE}" -} - - - -setup_zelta_env() { - :; -} \ No newline at end of file diff --git a/spec/lib/hello.sh b/spec/lib/hello.sh deleted file mode 100644 index 9c51246..0000000 --- a/spec/lib/hello.sh +++ /dev/null @@ -1,4 +0,0 @@ -# simple helper function example -hello() { - echo "What's up? Hello ${1}! TMPDIR: $SHELLSPEC_TMPDIR $SHELLSPEC_TMPBASE" -} diff --git a/spec/lib/script_util.sh b/spec/lib/script_util.sh deleted file mode 100644 index b7c505e..0000000 --- a/spec/lib/script_util.sh +++ /dev/null @@ -1,41 +0,0 @@ -. spec/bin/all_tests_setup/env_constants.sh - -validate_target() { - if [ $# -ne 1 ]; then - echo "Error: validate_target requires exactly 1 argument" >&2 - return 1 - fi - - case "$1" in - "$RUN_LOCALLY"|"$RUN_REMOTELY") - export RUNNING_MODE="$1" - ;; - *) - echo "Error: Invalid target '$1'" >&2 - echo "Must be one of: ${RUN_LOCALLY}, ${RUN_REMOTELY}" >&2 - return 1 - ;; - esac -} - - -validate_tree_name() { - if [ $# -ne 1 ]; then - echo "Error: Expected exactly 1 argument, got $#" >&2 - echo "Usage: $0 " >&2 - echo " tree_name must be one of: ${STANDARD_TREE}, ${DIVERGENT_TREE}, ${ENCRYPTED_TREE}" - return 1 - fi - - case "$1" in - "$STANDARD_TREE"|"$DIVERGENT_TREE"|"$ENCRYPTED_TREE") - export TREE_NAME=$1 - # Valid value - ;; - *) - echo "Error: Invalid tree_name '$1'" >&2 - echo "Must be one of: ${STANDARD_TREE}, ${DIVERGENT_TREE}, ${ENCRYPTED_TREE}" - return 1 - ;; - esac -} diff --git a/spec/spec_helper.sh b/spec/spec_helper.sh deleted file mode 100644 index 184067e..0000000 --- a/spec/spec_helper.sh +++ /dev/null @@ -1,111 +0,0 @@ -# shellcheck shell=sh - -#. spec/initialize/test_env.sh -# Defining variables and functions here will affect all specfiles. -# Change shell options inside a function may cause different behavior, -# so it is better to set them here. -# set -eu - - -#case_insensitive_equals() { -# $str=$1 -# $str=$2 -# if [ "$(echo "$str1" | tr '[:upper:]' '[:lower:]')" = "$(echo "$str2" | tr '[:upper:]' '[:lower:]')" ]; then -# return 0 -# fi -# return 1 -#} - -# This callback function will be invoked only once before loading specfiles. -spec_helper_precheck() { - # Available functions: info, warn, error, abort, setenv, unsetenv - # Available variables: VERSION, SHELL_TYPE, SHELL_VERSION - : minimum_version "0.28.1" - info "specshell precheck: version:$VERSION shell: $SHELL_TYPE $SHELL_VERSION" - info "*** TREE_NAME is {$TREE_NAME}" - info "*** RUNNING_MODE is {$RUNNING_MODE}" - if [ "$RUNNING_MODE" = "$RUN_REMOTELY" ]; then - info "***" - info "*** Running Remotely" - info "*** Source Server is SRC_SVR:{$SRC_SVR}" - info "*** Target Server is TGT_SVR:{$TGT_SVR}" - info "***" - else - info "***" - info "*** Running Locally" - info "***" - fi - - # Convert both to lowercase for comparison - #if case_insensitive_equals $RUNNING_MODE "remote" -} - -# This callback function will be invoked after a specfile has been loaded. -spec_helper_loaded() { - : - #echo "spec_helper.sh loaded from $SHELLSPEC_HELPERDIR" -} - -start_spec() { - : - # echo "starting {$SHELLSPEC_SPECFILE}" -} - -end_spec() { - : - # echo "ending {$SHELLSPEC_SPECFILE}" -} - -start_all() { - : - #exec_cmd "echo 'hello there'" - #ls -l "./spec/lib/create_file_backed_zfs_test_pools.sh" - #. "./spec/lib/create_file_backed_zfs_test_pools.sh" - #curdir=$(pwd) - #echo "spec_helper.sh start_all curdir:$curdir" - #./spec/initialize/create_file_backed_zfs_test_pools.sh - #./spec/initialize/initialize_testing_setup.sh - - #echo "staring all" - #. ./spec/initialize/initialize_testing_setup -} - -end_all() { - : - #echo "after all" -} - -# This callback function will be invoked after core modules has been loaded. -spec_helper_configure() { - # Available functions: import, before_each, after_each, before_all, after_all - : import 'support/custom_matcher' - before_each start_spec - after_each end_spec - before_all start_all - after_all end_all -} - -# Define helper functions AFTER spec_helper_configure -# These will be available in all spec files and in before_all/after_all blocks -exec_cmd() { - printf '%s' "$*" >&2 - if "$@"; then - printf ' :* succeeded\n' >&2 - return 0 - else - _exit_code=$? - printf ' :! failed (exit code: %d)\n' "$_exit_code" >&2 - return "$_exit_code" - fi -} - - -# In spec_helper.sh -capture_stderr() { - RESULT=$({ "$@" 2>&1 1>/dev/null; } 2>&1) || true - #RESULT="hello" -} - -#spec/initialize/test_env.sh - -#exec_cmd printf "hello there\n" diff --git a/spec/util/README.md b/spec/util/README.md deleted file mode 100644 index a486fc4..0000000 --- a/spec/util/README.md +++ /dev/null @@ -1,47 +0,0 @@ -## Utilities - -`match_function_generator.sh` - create case statement for shellspec - -## Examples - -### Shellspec matcher example -- ### Generating a matcher function or shellspec from zelta match output -```shell -$ ./matcher_func_generator.sh test_data/zelta_match_output.txt match_after_rotate_output -match_after_rotate_output() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ - "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ - "/sub2 @two @zelta_"*" @two syncable (incremental)"|\ - "/sub2/orphan @two @zelta_"*" @two syncable (incremental)"|\ - "/sub3 @two @zelta_"*" @two syncable (incremental)"|\ - "/sub3/space name @two @zelta_"*" @two syncable (incremental)"|\ - "/vol1 @go @zelta_"*" @go syncable (incremental)"|\ - "3 up-to-date, 5 syncable"|\ - "8 total datasets compared") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} -``` - -- ### Shellspec example using the generated matcher -```shell -Describe 'test zelta match output example' - It "match $SOURCE and $TARGET" - When call zelta match $SOURCE $TARGET - The output should satisfy match_after_rotate_output - The status should equal 0 - End -Enc -``` \ No newline at end of file diff --git a/spec/util/test_data/zelta_backup_after_rotate_output.txt b/spec/util/test_data/zelta_backup_after_rotate_output.txt deleted file mode 100644 index 4f6af25..0000000 --- a/spec/util/test_data/zelta_backup_after_rotate_output.txt +++ /dev/null @@ -1,3 +0,0 @@ -syncing 8 datasets -8 datasets up-to-date -3K sent, 5 streams received in 0.11 seconds \ No newline at end of file diff --git a/spec/util/test_data/zelta_match_after_backup_output.txt b/spec/util/test_data/zelta_match_after_backup_output.txt deleted file mode 100644 index 5af95aa..0000000 --- a/spec/util/test_data/zelta_match_after_backup_output.txt +++ /dev/null @@ -1,10 +0,0 @@ -DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO -[treetop] @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub1/child @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub2 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub2/orphan @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub3 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub3/space name @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/vol1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -8 up-to-date \ No newline at end of file diff --git a/spec/util/test_data/zelta_match_output.txt b/spec/util/test_data/zelta_match_output.txt deleted file mode 100644 index 7610a58..0000000 --- a/spec/util/test_data/zelta_match_output.txt +++ /dev/null @@ -1,11 +0,0 @@ -DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO -[treetop] @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub1 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub1/child @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 @zelta_2026-01-11_21.50.46 up-to-date -/sub2 @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) -/sub2/orphan @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) -/sub3 @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) -/sub3/space name @two @zelta_2026-01-11_21.50.46 @two syncable (incremental) -/vol1 @go @zelta_2026-01-11_21.50.46 @go syncable (incremental) -3 up-to-date, 5 syncable -8 total datasets compared diff --git a/test/01_no_op_spec.sh b/test/01_no_op_spec.sh index 701e2a8..81b8be6 100644 --- a/test/01_no_op_spec.sh +++ b/test/01_no_op_spec.sh @@ -3,7 +3,7 @@ Describe 'Zelta no-op command checks' Describe 'zelta command' It 'is executable' - When run command command -v zelta + When run command which zelta The status should be success The output should include 'zelta' End diff --git a/test/021_setup_pools_spec.sh b/test/021_setup_pools_spec.sh new file mode 100644 index 0000000..870e052 --- /dev/null +++ b/test/021_setup_pools_spec.sh @@ -0,0 +1,25 @@ +Describe 'Remote check' + It 'source accessible' + Skip if 'SANDBOX_ZELTA_SRC_REMOTE undefined' [ -z "$SANDBOX_ZELTA_SRC_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" true + The status should be success + End + It 'target accessible' + Skip if 'SANDBOX_ZELTA_TGT_REMOTE undefined' [ -z "$SANDBOX_ZELTA_TGT_REMOTE" ] + When run ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" true + The status should be success + End +End + +Describe 'Pool setup' + It 'create source' + Skip if 'SANDBOX_ZELTA_SRC_POOL undefined' [ -z "$SANDBOX_ZELTA_SRC_POOL" ] + When call make_src_pool + The status should be success + End + It 'create target' + Skip if 'SANDBOX_ZELTA_TGT_POOL undefined' [ -z "$SANDBOX_ZELTA_TGT_POOL" ] + When call make_tgt_pool + The status should be success + End +End \ No newline at end of file diff --git a/test/022_setup_tree_spec.sh b/test/022_setup_tree_spec.sh new file mode 100644 index 0000000..76e0de8 --- /dev/null +++ b/test/022_setup_tree_spec.sh @@ -0,0 +1,39 @@ +# Check remotes and create pools and datasets + +Describe 'Divergent tree tests' + # TODO: Check with DB what env vars he wants to use for Skip if + Skip if 'SANDBOX_ZELTA_SRC_DS and SANDBOX_ZELTA_TGT_DS are undefined' test -z "$SANDBOX_ZELTA_SRC_DS" -a -z "$SANDBOX_ZELTA_TGT_DS" + + Describe 'setup' + It 'creates initial tree on source' + When call make_initial_tree + The status should be success + End + + It 'can zelta backup initial tree' + When call zelta backup --snap-name @start "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The status should be success + The output should include 'syncing 9 datasets' + The error should not include 'error:' + End + + It 'can create tree divergence' + When call make_tree_divergence + The status should be success + The error should not include 'error:' + End + End + + Describe 'zelta match' + It 'shows expected divergence types' + When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The status should be success + The output should include 'up-to-date' + The output should include 'syncable (full)' + The output should include 'syncable (incremental)' + The output should include 'blocked sync: target diverged' + The output should include 'blocked sync: no target snapshots' + The output should include '11 total datasets compared' + End + End +End diff --git a/test/02_setup_spec.sh b/test/02_setup_spec.sh deleted file mode 100644 index 612ebb9..0000000 --- a/test/02_setup_spec.sh +++ /dev/null @@ -1,51 +0,0 @@ -# Check remotes and create pools and datasets - -Describe 'Remote check' - It 'source accessible' - Skip if 'SANDBOX_ZELTA_SRC_REMOTE undefined' [ -z "$SANDBOX_ZELTA_SRC_REMOTE" ] - When run ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" true - The status should be success - End - It 'target accessible' - Skip if 'SANDBOX_ZELTA_TGT_REMOTE undefined' [ -z "$SANDBOX_ZELTA_TGT_REMOTE" ] - When run ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" true - The status should be success - End -End - -Describe 'Pool setup' - It 'create source' - Skip if 'SANDBOX_ZELTA_SRC_POOL undefined' [ -z "$SANDBOX_ZELTA_SRC_POOL" ] - When call make_src_pool - The status should be success - End - It 'create target' - Skip if 'SANDBOX_ZELTA_TGT_POOL undefined' [ -z "$SANDBOX_ZELTA_TGT_POOL" ] - When call make_tgt_pool - The status should be success - End -End - -Describe 'Divergent tree setup' - It 'creates divergent tree on source' - Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" -a -z "$SANDBOX_ZELTA_TGT_DS" - When call make_divergent_tree - The status should be success - The output should include 'syncing 9 datasets' - The error should not include 'error:' - End -End - -Describe 'Divergent tree match' - It 'shows expected divergence types' - Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" - When run zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" - The status should be success - The output should include 'up-to-date' - The output should include 'syncable (full)' - The output should include 'syncable (incremental)' - The output should include 'blocked sync: target diverged' - The output should include 'blocked sync: no target snapshots' - The output should include '11 total datasets compared' - End -End diff --git a/test/040_zelta_tests_spec.sh b/test/040_zelta_tests_spec.sh new file mode 100644 index 0000000..930ef66 --- /dev/null +++ b/test/040_zelta_tests_spec.sh @@ -0,0 +1,130 @@ +# Auto-generated ShellSpec test file +# Generated at: 2026-02-12 13:28:34 -0500 +# Source: 040_zelta_tests_spec +# WARNING: This file was automatically generated. Manual edits may be lost. + +output_for_match_after_divergence() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @start @start @start up-to-date"|\ + "/sub1 @start @start @start up-to-date"|\ + "/sub1/child - - - syncable (full)"|\ + "/sub1/kid - - - no source (target only)"|\ + "/sub2 - @two @two blocked sync: target diverged"|\ + "/sub2/orphan @start @start @start up-to-date"|\ + "/sub3 @start @two @start syncable (incremental)"|\ + "/sub3/space name @start @start @blocker blocked sync: target diverged"|\ + "/sub4 @start @start @start up-to-date"|\ + "/sub4/encrypted @start @start @start up-to-date"|\ + "/sub4/zvol - @start - blocked sync: no target snapshots"|\ + "5 up-to-date, 2 syncable, 4 blocked"|\ + "11 total datasets compared") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_rotate_after_divergence() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "source is written; snapshotting: @zelta_"*""|\ + "renaming '${SANDBOX_ZELTA_TGT_DS}' to '${SANDBOX_ZELTA_TGT_DS}_start'"|\ + "to ensure target is up-to-date, run: zelta backup ${SANDBOX_ZELTA_SRC_EP} ${SANDBOX_ZELTA_TGT_EP}"|\ + "no source: ${SANDBOX_ZELTA_TGT_DS}/sub1/kid"|\ + "* sent, 10 streams received in * seconds") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_match_after_rotate() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ + "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub1/child @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub2 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub2/orphan @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub3 @two @zelta_"*" @two syncable (incremental)"|\ + "/sub3/space name @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub4 @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub4/encrypted @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ + "/sub4/zvol @start @zelta_"*" @start syncable (incremental)"|\ + "7 up-to-date, 3 syncable"|\ + "10 total datasets compared") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_backup_after_rotate() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "syncing 10 datasets"|\ + "10 datasets up-to-date"|\ + "* sent, 3 streams received in * seconds") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +Describe 'Run zelta commands on divergent tree' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + + It "show divergence - zelta match \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_match_after_divergence + The status should be success + End + + It "rotates after divergence - zelta rotate \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta rotate "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_rotate_after_divergence + The error should equal "warning: insufficient snapshots; performing full backup for 3 datasets" + The status should be success + End + + It "match after rotate - zelta match \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_match_after_rotate + The status should be success + End + + It "backup after rotate - zelta backup \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_backup_after_rotate + The status should be success + End + +End diff --git a/test/050_zelta_revert_spec.sh b/test/050_zelta_revert_spec.sh new file mode 100644 index 0000000..99d0fac --- /dev/null +++ b/test/050_zelta_revert_spec.sh @@ -0,0 +1,144 @@ +# Auto-generated ShellSpec test file +# Generated at: 2026-02-12 13:29:24 -0500 +# Source: 050_zelta_revert_spec +# WARNING: This file was automatically generated. Manual edits may be lost. + +output_for_snapshot() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "snapshot created '${SANDBOX_ZELTA_SRC_DS}@manual_test'") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_backup_after_delta() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "source is written; snapshotting: @zelta_"*""|\ + "syncing 12 datasets"|\ + "* sent, 22 streams received in * seconds") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_snapshot_again() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "snapshot created '${SANDBOX_ZELTA_SRC_DS}@another_test'") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_revert() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "renaming '${SANDBOX_ZELTA_SRC_DS}' to '${SANDBOX_ZELTA_SRC_DS}_manual_test'"|\ + "cloned 12/12 datasets to ${SANDBOX_ZELTA_SRC_DS}"|\ + "snapshotting: @zelta_"*""|\ + "to retain replica history, run: zelta rotate '${SANDBOX_ZELTA_SRC_DS}' 'TARGET'") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_rotate_after_revert() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "renaming '${SANDBOX_ZELTA_TGT_DS}' to '${SANDBOX_ZELTA_TGT_DS}_zelta_"*"'"|\ + "to ensure target is up-to-date, run: zelta backup ${SANDBOX_ZELTA_SRC_EP} ${SANDBOX_ZELTA_TGT_EP}"|\ + "no source: ${SANDBOX_ZELTA_TGT_DS}/sub5"|\ + "no source: ${SANDBOX_ZELTA_TGT_DS}/sub5/child1"|\ + "* sent, 10 streams received in * seconds") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +Describe 'Test revert' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + + It "take a snapshot of tree before changes - zelta snapshot --snap-name \"manual_test\" \"$SANDBOX_ZELTA_SRC_EP\"" + When call zelta snapshot --snap-name "manual_test" "$SANDBOX_ZELTA_SRC_EP" + The output should satisfy output_for_snapshot + The status should be success + End + + It "add and remove src datasets - add_tree_delta" + When call add_tree_delta + The status should be success + End + + It "backup after deltas - zelta backup \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_backup_after_delta + The status should be success + End + + It "take a snapshot of tree after changes - zelta snapshot --snap-name \"another_test\" \"$SANDBOX_ZELTA_SRC_EP\"" + When call zelta snapshot --snap-name "another_test" "$SANDBOX_ZELTA_SRC_EP" + The output should satisfy output_for_snapshot_again + The status should be success + End + + It "revert to last snapshot - zelta revert \"$SANDBOX_ZELTA_SRC_EP\"@manual_test" + When call zelta revert "$SANDBOX_ZELTA_SRC_EP"@manual_test + The output should satisfy output_for_revert + The error should equal "warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: cannot open '${SANDBOX_ZELTA_SRC_DS}_manual_test/sub5@manual_test': dataset does not exist +warning: unexpected 'zfs clone' output: cannot open '${SANDBOX_ZELTA_SRC_DS}_manual_test/sub5/child1@manual_test': dataset does not exist" + The status should be success + End + + It "rotates after divergence - zelta rotate \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta rotate "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_rotate_after_revert + The status should be success + End + +End diff --git a/test/060_zelta_clone_spec.sh b/test/060_zelta_clone_spec.sh new file mode 100644 index 0000000..7b4d46c --- /dev/null +++ b/test/060_zelta_clone_spec.sh @@ -0,0 +1,57 @@ +# Auto-generated ShellSpec test file +# Generated at: 2026-02-12 13:30:29 -0500 +# Source: 060_zelta_clone_spec +# WARNING: This file was automatically generated. Manual edits may be lost. + +output_for_clone_sub2() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "cloned 2/2 datasets to ${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_zfs_list_for_clone() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "NAME ORIGIN"|\ + "${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2 ${SANDBOX_ZELTA_SRC_DS}/sub2@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2/orphan ${SANDBOX_ZELTA_SRC_DS}/sub2/orphan@zelta_"*"") + ;; + *) + printf "Unexpected line format: %s\n" "$line" >&2 + return 1 + ;; + esac + done + return 0 +} + +Describe 'Test clone' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + + It "zelta clone sub2 - zelta clone \"$SANDBOX_ZELTA_SRC_EP/sub2\" \"$SANDBOX_ZELTA_SRC_EP/copy_of_sub2\"" + When call zelta clone "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" + The output should satisfy output_for_clone_sub2 + The error should equal "warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root +warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root" + The status should be success + End + + It "verifies the clone - src_exec zfs list -ro name,origin $SANDBOX_ZELTA_SRC_DS/copy_of_sub2" + When call src_exec zfs list -ro name,origin $SANDBOX_ZELTA_SRC_DS/copy_of_sub2 + The output should satisfy output_for_zfs_list_for_clone + The status should be success + End + +End diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..de8fca2 --- /dev/null +++ b/test/README.md @@ -0,0 +1,26 @@ +# Running tests with shellspec + +- after installing `shellspec` + +- define your sandbox environment variables + - source an environment setup script + - modify or create a new env setup script, see [test/runners/env/test_env.sh](runners/env/test_env.sh) + - or set the environment variables directly in your shell session + - >NOTE: you can run a basic smoke test without setting any environment variables +- run `shellspec` + - cd to the repo root for `zelta` + - `shellspec` + +### If testing remotely: +- Setup your test user on your source and target machines + - update sudoers, for example on Linux + - create /etc/sudoers.d/zelta-tester + ``` + # Allow (mytestuser) to run ZFS commands without password for zelta testing + # NOTE: This is for test environments only - DO NOT use in production + # CAUTION: The wildcards show intent only, with globbing other commands may be allowed as well + (mytestuser) ALL=(ALL) NOPASSWD: /usr/bin/dd *, /usr/bin/rm -f /tmp/*, /usr/bin/truncate *, /usr/sbin/zpool *, /usr/sbin/zfs * + ``` + - TODO: confirm if usr/bin/mount *, /usr/bin/mkdir * are needed + + - setup zfs allow on your source and target machines will be set up automatically for your test pools diff --git a/test/runners/README.md b/test/runners/README.md new file mode 100644 index 0000000..79587a0 --- /dev/null +++ b/test/runners/README.md @@ -0,0 +1,379 @@ +# Test Runners + +Test infrastructure for the Zelta ZFS backup tool, including environment setup and automated test generation. + +## Overview + +This directory provides: +- **Environment Management**: Scripts to configure and manage test ZFS pools and datasets +- **Test Generation**: Automated creation of ShellSpec tests from YAML definitions +- **Development Tools**: Helpers for debugging and iterating on tests + +## Directory Structure + +``` +test/runners/ +├── README.md # This file +├── doc/ # Documentation +│ └── README_AliasHelpers.md # Shell aliases for test workflows +│ +├── env/ # Environment setup +│ ├── test_env.sh # Configure pools, datasets, remotes +│ ├── helpers.sh # Common functions (setup_env, run_it, etc.) +│ ├── reset_env.sh # Reset environment variables +│ ├── setup_debug_env.sh # Debug environment setup +│ ├── set_reuse_tmp_env.sh # Reuse existing /tmp/zelta* install +│ └── test_generator_cleanup.sh # Clean up after test generation +│ +└── test_generation/ # Automated test generation (see below) + ├── README.md # Detailed test generation docs + ├── bin/ # Entry point scripts + ├── config/ # YAML test definitions + ├── lib/ # Core implementation + ├── scripts/ # Utilities (shell, AWK) + └── tmp/ # Generated output +``` + +## Quick Start + +### 1. Configure Test Environment + +Edit your test pools and datasets: + +```bash +vi env/test_env.sh +``` + +**Important**: The pools and datasets you configure will be destroyed and recreated by tests. + +Example configuration: + +```bash +export SANDBOX_ZELTA_SRC_POOL=apool +export SANDBOX_ZELTA_TGT_POOL=bpool +export SANDBOX_ZELTA_SRC_DS=apool/treetop +export SANDBOX_ZELTA_TGT_DS=bpool/backups + +# Remote datasets and pools are optional and not needed for local tests +export SANDBOX_ZELTA_SRC_REMOTE=dever@zfsdev +export SANDBOX_ZELTA_TGT_REMOTE=dever@zfsdev +``` + +### 2. Run Tests + +```bash +# From repo root +. test/runners/env/test_env.sh +shellspec +``` + +Tests use SANDBOX variables form `test_env.sh` to set up the required environment. + +### 3. Generate New Tests + +```bash +cd test/runners/test_generation +./bin/generate_new_tests.sh +``` + +See [test_generation/README.md](test_generation/README.md) for detailed documentation. + +## Environment Scripts + +### test_env.sh + +Core configuration for test pools, datasets, and remotes. Source this file to set up environment variables: + +```bash +. test/runners/env/test_env.sh +``` + +### helpers.sh + +Common functions for environment management: + +- `setup_env(DEBUG_MODE)` - Setup debug or standard environment +- `run_it(function_name)` - Run a function and report success/failure +- `clean_ds_and_pools()` - Destroy all test datasets and pools + +### reset_env.sh + +Resets environment variables by unsetting `SANDBOX_ZELTA_TMP_DIR`: + +```bash +. test/runners/env/reset_env.sh +``` + +**Purpose**: Forces `test/test_helper.sh` to properly initialize all variables in the current ShellSpec process context when running `shellspec` from repo root. + +**When to use**: Rarely needed manually - automatically called by `test_generator_cleanup.sh`. You might need it if you've been running test generation and want to ensure a clean state before running `shellspec`. + +### setup_debug_env.sh + +Configure environment for debugging without running full ShellSpec suite. Useful when manually testing commands or developing new tests. + +### test_generator_cleanup.sh + +Cleans up ZFS pools and datasets after test generation: + +```bash +./env/test_generator_cleanup.sh +``` + +**When to use**: After completing test generation and copying all newly generated tests to `test/`. This script: +1. Calls `reset_env.sh` to reset environment variables +2. Destroys test pools and datasets +3. Prepares environment for normal test runs + +**Important**: Run this after test generation is complete and before running the standard test suite with `shellspec`. + +## Test Generation System + +The `test_generation/` directory contains a complete system for automatically generating ShellSpec tests from YAML configurations. + +### Key Features + +- **YAML-driven**: Define tests declaratively +- **Automatic matchers**: Generates output validation functions +- **Smart substitution**: Handles environment variables, timestamps, dynamic values +- **End-to-end validation**: Tests are validated before deployment + +### Basic Workflow + +1. **Create YAML definition** with test cases +2. **Run generator** which executes commands and captures output +3. **Validate generated test** automatically +4. **Deploy to production** if validation passes + +Example YAML: + +```yaml +output_dir: tmp +shellspec_name: "040_zelta_tests_spec" +describe_desc: "Test zelta commands" + +test_list: + - test_name: backup_operation + it_desc: performs backup - %{when_command} + when_command: zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" +``` + +For complete documentation, see [test_generation/README.md](test_generation/README.md). + +## Development Workflow + +### Running Existing Tests + +```bash +# Run all tests +shellspec + +# Run specific test +shellspec test/040_zelta_tests_spec.sh +``` + +### Creating New Tests + +#### Option 1: Write Manually + +Create a new ShellSpec file in `test/`: + +```bash +vi test/070_my_new_test_spec.sh +``` + +Follow existing test patterns. + +#### Option 2: Generate from YAML + +1. Create YAML definition: + ```bash + vi test/runners/test_generation/config/test_defs/070_my_test.yml + ``` + +2. Add to generation script: + ```bash + vi test/runners/test_generation/bin/generate_new_tests.sh + ``` + +3. Generate: + ```bash + cd test/runners/test_generation + ./bin/generate_new_tests.sh + ``` + +### Debugging Tests + +#### Use Debug Environment + +```bash +# Setup debug environment +. test/runners/env/setup_debug_env.sh + +# Manually run zelta commands +zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + +# Reset when done +. test/runners/env/reset_env.sh +``` + +#### Run Single Test + +```bash +# Run just one test file +shellspec test/040_zelta_tests_spec.sh + +# Run with detailed output +shellspec --format documentation test/040_zelta_tests_spec.sh +``` + +#### Examine Generated Tests + +```bash +# View generated test +cat test/runners/test_generation/tmp/040_zelta_tests_spec.sh + +# View generated matchers +cat test/runners/test_generation/tmp/output_for_*/output_for_*.sh +``` + +## Shell Aliases + +For productivity, source the alias helpers: + +```bash +. test/runners/doc/alias_setup.sh # if this exists +``` + +Or add to your shell rc file. See [doc/README_AliasHelpers.md](doc/README_AliasHelpers.md) for available aliases. + +Common aliases: +- `ztenv` - source sandbox env from `test/runners/env/test_env.sh` +- `zdbgenv` - Setup debug environment +- `zcd` - Change to repo root +- `zspect` - Run shellspec tests with tracing +- `zspecd` - Run shellspec tests with more detailed output +- `zclean` - Remove test pools and datasets, clean slate + - use `zclean` to reset env after test generation + - use `ztenv` to source sandbox env vars + - now you can run `shellspec` normally + +## Architecture + +### Test Execution Flow + +``` +Manually setup up your SANDBOX env vars +use . env/test_env.sh or equivalent + ↓ +shellspec + ↓ +test/*_spec.sh (ShellSpec tests) + ↓ +test/test_helper.sh (setup) + ↓ +Creates/configures test pools + ↓ +Runs test commands +``` + +### Test Generation Flow + +``` +bin/generate_new_tests.sh + ↓ +lib/orchestration/generate_test.sh + ↓ +1. Setup ZFS tree +2. Run lib/ruby/test_generator.rb + - Parse YAML + - Execute commands + - Generate matchers + - Assemble test file +3. Validate generated test +4. Copy to production +``` + +### Path Resolution + +- **Environment scripts**: Use relative paths (sourced from repo root) +- **Test generation scripts**: Use absolute paths (work from any directory) +- **Generated tests**: Use environment variables for portability + +## Troubleshooting + +### Tests Fail Immediately + +Check that `env/test_env.sh` is configured correctly: + +```bash +. test/runners/env/test_env.sh +echo $SANDBOX_ZELTA_SRC_POOL +echo $SANDBOX_ZELTA_TGT_POOL +``` + +### Pools Already Exist + +If you've been running test generation, clean up: + +```bash +./test/runners/env/test_generator_cleanup.sh +``` + +Then run tests normally: + +```bash +shellspec # Will recreate pools with fresh state +``` + +### Test Generation Fails + +1. Verify YAML against schema: + ```bash + cd test/runners/test_generation + ./bin/validate_yaml.rb config/test_defs/040_zelta_tests.yml + ``` + +2. Check Ruby dependencies: + ```bash + cd test/runners/test_generation + bundle install + ``` + +3. Run with debug: + ```bash + ./bin/debug_gen.sh config/test_defs/040_zelta_tests.yml + ``` + +### Generated Test Doesn't Match Output + +The matcher may be too strict or environment variables not substituted correctly. + +See [test_generation/README.md - Troubleshooting](test_generation/README.md#troubleshooting) for details. + +## Related Documentation + +- **[Test Generation Details](test_generation/README.md)** - Complete guide to automated test generation +- **[Alias Helpers](doc/README_AliasHelpers.md)** - Shell aliases for common workflows +- **[Main Test Directory](../)** - ShellSpec test files and test_helper.sh + +## Design Principles + +1. **Separation of Concerns** + - `env/` - Environment configuration (sourced, repo-root relative) + - `test_generation/` - Test generation (executable, path-independent) + +2. **Single Source of Truth** + - Test names come from YAML `shellspec_name` + - Paths resolve from known anchors (repo root, script location) + - Environment variables defined once in `env/test_env.sh` + +3. **Reproducibility** + - Environment can be reset to known state + - Tests generate identically from same YAML + - All paths absolute or explicitly resolved + +4. **Developer Friendly** + - Clear directory structure + - Comprehensive documentation + - Debug modes and helpers diff --git a/test/runners/doc/README_AliasHelpers.md b/test/runners/doc/README_AliasHelpers.md new file mode 100644 index 0000000..0421576 --- /dev/null +++ b/test/runners/doc/README_AliasHelpers.md @@ -0,0 +1,51 @@ +### Suggested alias helpers: +> NOTE: these aliases simplify the iterative process of setting up +> test environments for manual checks of zelta commands, and +> for resetting the environment to run from a clean slate. + +```shell +# shellspec test development aliases + + +# set to your repo location for zelta +ZELTA_REPO_ROOT=~/src/repos/bt/zelta # local repo location + +# env helpers for zelta testing +ZELTA_ENV="$ZELTA_REPO_ROOT/test/runners/env" + +# zelta test generation helper directory +ZELTA_TEST_GEN="$ZELTA_REPO_ROOT/test/runners/test_generation" + + +# show all the aliases we've setup for zelta testing +alias zhlp="alias | grep 'z'" +alias zcd="cd $ZELTA_REPO_ROOT" +alias ecd="cd $ZELTA_ENV" +alias gcd="cd $ZELTA_TEST_GEN" + + +# NOTE: the aliases work from the context of the zelta repo, use zcd before action + +# run shellspec with trace and evaluation +# note: macOS requires homebrew bash, use bash shell for your env +BASH_SH=/opt/homebrew/bin/bash +alias zspect="zcd && shellspec --xtrace --shell $BASH_SH" + +# runs shellspec with document format output +alias zspecd="zcd && shellspec --format document" + +# update current environment to allow running commands from the command line +# in the context of the current debug zfs tree state +alias zdbgenv="zcd && . $ZELTA_ENV/helpers.sh && setup_env 1" + +# force a clean up of pools, datasets and remotes +alias zclean="zcd && $ZELTA_ENV/test_generator_cleanup.sh" + +# force next evaluation of test/test_helpers.sh to initialize env fully +alias zrenv="zcd && . $ZELTA_ENV/reset_env.sh" + + +# setup env vars for your test environment +# setup pools, datasets and remotes env vars +alias ztenv="zcd && . $ZELTA_ENV/test_env.sh" +``` diff --git a/test/runners/env/helpers.sh b/test/runners/env/helpers.sh new file mode 100644 index 0000000..8359df3 --- /dev/null +++ b/test/runners/env/helpers.sh @@ -0,0 +1,40 @@ +# sourcing this file to setup some helpers +# for debug environment management + + +# create run environment +# DEBUG_MODE - not empty = setup the debug environment +# empty = setup the standard test environment +setup_env() { + DEBUG_MODE=$1 + + if [ -n "$DEBUG_MODE" ]; then + . test/runners/env/setup_debug_env.sh + else + printf '%s\n' "--> Normal shellspec Run" + . test/runners/env/reset_env.sh # reset the env, use test_helper.sh version + . test/runners/env/test_env.sh # set dataset, pools and remote env vars + # on normal run shellspec will automatically run test/test_helper.sh + fi +} + +# run a function, show it's status +run_it() { + _func=$1 + #if (eval set -x; "$_func";); then # if you want to see trace + if (eval "$_func";); then + printf " ✅ %s\n\n" "$_func" + else + printf " ❌ %s\n\n" "$_func" + exit 1 + fi +} + +# remove the zfs pools and datasets, clean slate for running shellspecs +clean_ds_and_pools() { + echo "cleaning up, datasets and pools" + run_it clean_src_ds + run_it clean_tgt_ds + run_it nuke_tgt_pool + run_it nuke_src_pool +} diff --git a/test/runners/env/reset_env.sh b/test/runners/env/reset_env.sh new file mode 100755 index 0000000..52dfaec --- /dev/null +++ b/test/runners/env/reset_env.sh @@ -0,0 +1,7 @@ +# source this file to ensure that the next run of shellspec will pick up +# the standard test environment for zelta that gets created by test/test_helper.sh + +echo "unsetting SANDBOX_ZELTA_TMP_DIR to forces test_helper.sh to re-evaluate ZELTA setup" +unset SANDBOX_ZELTA_TMP_DIR # forces test_helper.sh to re-evaluate ZELTA setup + +. ./test/runners/env/test_env.sh diff --git a/test/runners/env/set_reuse_tmp_env.sh b/test/runners/env/set_reuse_tmp_env.sh new file mode 100755 index 0000000..131ff90 --- /dev/null +++ b/test/runners/env/set_reuse_tmp_env.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Debug environment setup +# To facilitate creating and manually shellspec tests, and debugging existing tests +# use the last spec installed zelta version +# if no previous zelta install is found in /tmp, show user how to create one + +# find the last installed version of zelta installed by shellspec +last_tmp_installed_zelta_ver=$(ls -1d /tmp/zelta* | tail -1) + +# exit if no previous install found +if [ -z "$last_tmp_installed_zelta_ver" ]; then + printf " ❌ %s\n" "No previous zelta installs found in /tmp/zelta* " + printf " ***\n *** %s\n ***\n" "run shellspec test/00_install_spec.sh" + return 1 +fi + +# extract the process number used when zelta install wsa created +last_tmp_process_number=$(echo "$last_tmp_installed_zelta_ver" | grep -o '[0-9]\+$') + +#set -x +# use discovered zelta dir +export SANDBOX_ZELTA_TMP_DIR="$last_tmp_installed_zelta_ver" + +# use discovered process number +export SANDBOX_ZELTA_PROCNUM="$last_tmp_process_number" + +# standardized shellspec run environment depends on these env vars +# we use the last installed zelta and dbg area for shellspec +export ZELTA_BIN="$SANDBOX_ZELTA_TMP_DIR/bin" +export ZELTA_SHARE="$SANDBOX_ZELTA_TMP_DIR/share" +export ZELTA_ETC="$SANDBOX_ZELTA_TMP_DIR/etc" +export ZELTA_DOC="$SANDBOX_ZELTA_TMP_DIR/man" +export SHELLSPEC_TMPBASE=~/tmp/dbg_shellspecs +mkdir -p $SHELLSPEC_TMPBASE + +# add the tmp zelta bin if not already on path +if ! echo ":$PATH:" | grep -q ":$ZELTA_BIN:"; then + export PATH="$ZELTA_BIN:$PATH" +fi + +# SHELLSPEC_PROJECT_ROOT is not currently being used in the zelta test env setup +# if that changes we'll need to address it here when we are creating a custom debug environment +echo "*** NOTE: SHELLSPEC_PROJECT_ROOT is not set, make sure it's not used!" + +printf " ✅ %s\n\n" "using zelta $last_tmp_installed_zelta_ver with process number $last_tmp_process_number" diff --git a/test/runners/env/setup_debug_env.sh b/test/runners/env/setup_debug_env.sh new file mode 100644 index 0000000..8293103 --- /dev/null +++ b/test/runners/env/setup_debug_env.sh @@ -0,0 +1,12 @@ +# source this file to setup the zelta debug environment +# returns 0 on failure + +printf "\n*\n* Running in DEBUG MODE, sourcing setup files\n*\n" +# use debug env, the last version of zelta installed" + +if . test/runners/env/set_reuse_tmp_env.sh; then + . test/runners/env/test_env.sh # set dataset, pools and remote env vars + . test/test_helper.sh # make all the helper functions available +else + return 1 +fi diff --git a/test/runners/env/test_env.sh b/test/runners/env/test_env.sh new file mode 100644 index 0000000..6fb6d8a --- /dev/null +++ b/test/runners/env/test_env.sh @@ -0,0 +1,15 @@ +# Modify this file to configure your test pools, datasets and endpoints + +# pools +export SANDBOX_ZELTA_SRC_POOL=apool +export SANDBOX_ZELTA_TGT_POOL=bpool + +# datasets +export SANDBOX_ZELTA_SRC_DS=apool/treetop +export SANDBOX_ZELTA_TGT_DS=bpool/backups + +# remotes setup +# * leave these undefined if you're running locally +# * the endpoints are defined automatically and are REMOTE + DS +export SANDBOX_ZELTA_SRC_REMOTE=dever@zfsdev # Ubuntu source +export SANDBOX_ZELTA_TGT_REMOTE=dever@zfsdev # Ubuntu remote diff --git a/test/runners/env/test_generator_cleanup.sh b/test/runners/env/test_generator_cleanup.sh new file mode 100755 index 0000000..7a47faf --- /dev/null +++ b/test/runners/env/test_generator_cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +. ./test/runners/env/helpers.sh + +setup_env "" +. test/test_helper.sh +clean_ds_and_pools diff --git a/test/runners/test_generation/Gemfile b/test/runners/test_generation/Gemfile new file mode 100644 index 0000000..27aee2b --- /dev/null +++ b/test/runners/test_generation/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'json-schema', '~> 4.0' diff --git a/test/runners/test_generation/Gemfile.lock b/test/runners/test_generation/Gemfile.lock new file mode 100644 index 0000000..8d8f3f6 --- /dev/null +++ b/test/runners/test_generation/Gemfile.lock @@ -0,0 +1,18 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.8) + public_suffix (>= 2.0.2, < 8.0) + json-schema (4.3.1) + addressable (>= 2.8) + public_suffix (7.0.2) + +PLATFORMS + arm64-darwin-23 + ruby + +DEPENDENCIES + json-schema (~> 4.0) + +BUNDLED WITH + 2.6.4 diff --git a/test/runners/test_generation/README.md b/test/runners/test_generation/README.md new file mode 100644 index 0000000..9002057 --- /dev/null +++ b/test/runners/test_generation/README.md @@ -0,0 +1,358 @@ +# Test Generation System + +Automated generation of ShellSpec test files from YAML configurations for the Zelta ZFS backup tool. + +## Overview + +This system generates complete ShellSpec test files by: +1. Reading test definitions from YAML configuration files +2. Executing commands and capturing their output +3. Generating matcher functions that validate command output +4. Assembling complete ShellSpec test files with proper structure + +## Directory Structure + +``` +test_generation/ +├── bin/ # Entry point scripts +│ ├── generate_new_tests.sh # Generate multiple tests (040, 050, 060) +│ └── debug_gen.sh # Debug test generation +├── config/ +│ ├── test_defs/ # YAML test definitions +│ │ ├── 040_zelta_tests.yml +│ │ ├── 050_zelta_revert_test.yml +│ │ └── 060_zelta_clone_test.yml +│ └── test_config_schema.yml # YAML validation schema +├── lib/ +│ ├── ruby/ # Core Ruby implementation +│ │ ├── test_generator.rb # Main test generator class +│ │ ├── sys_exec.rb # Command execution with timeout +│ │ ├── placeholders.rb # Variable substitution +│ │ └── .rubocop.yml # Ruby style config +│ └── orchestration/ # Shell orchestration scripts +│ ├── generate_test.sh # Test generation workflow +│ └── setup_tree.sh # ZFS tree setup +├── scripts/ +│ ├── sh/ # Shell utilities +│ │ ├── generate_matcher.sh # Generate matcher functions +│ │ └── matcher_func_generator.sh +│ └── awk/ # AWK text processing +│ └── generate_case_stmt_func.awk +├── tmp/ # Generated test output +└── Gemfile # Ruby dependencies + +``` + +## Quick Start + +### Prerequisites + +```bash +# Install Ruby dependencies +cd test/runners/test_generation +bundle install + +# Configure your test environment +vi ../env/test_env.sh # Set pools, datasets, remotes +``` + +### Generate Tests + +```bash +# Generate all configured tests +./bin/generate_new_tests.sh +``` + +## YAML Test Configuration + +### Basic Structure + +```yaml +output_dir: tmp +shellspec_name: "040_zelta_tests_spec" +describe_desc: "Run zelta commands on divergent tree" + +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + +test_list: + - test_name: match_after_divergence + it_desc: show divergence - %{when_command} + when_command: zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" +``` + +### Advanced Features + +#### Setup Scripts + +Source helper scripts before running commands: + +```yaml +test_list: + - test_name: add_and_remove_src_datasets + setup_scripts: + - "test/test_helper.sh" # Paths relative to repo root + when_command: add_tree_delta +``` + +#### Commands with No Output + +For commands that shouldn't produce output: + +```yaml +test_list: + - test_name: cleanup_operation + allow_no_output: true + when_command: zelta cleanup "$SANDBOX_ZELTA_SRC_EP" +``` + +#### Variable Substitution + +Use `%{variable}` syntax in descriptions: + +```yaml +it_desc: "backup after rotate - %{when_command}" +``` + +## How It Works + +### 1. Test Generation Workflow + +``` +generate_test.sh + ↓ +1. Setup ZFS tree (via setup_tree.sh) + ↓ +2. Run test_generator.rb + ↓ + - Parse YAML config + - For each test: + a. Execute command + b. Capture stdout/stderr + c. Generate matcher function + d. Apply env var substitutions + e. Create ShellSpec test clause + ↓ +3. Setup ZFS tree again + ↓ +4. Run generated test with shellspec + ↓ +5. If passes, copy to production (test/) +``` + +### 2. Matcher Function Generation + +The system automatically generates matcher functions that: +- Normalize whitespace in output +- Replace environment variable values with variable references +- Replace timestamps with wildcards +- Create case statements for output validation + +Example generated matcher: + +```bash +output_for_match_after_divergence() { + while IFS= read -r line; do + normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "backing up from ${SANDBOX_ZELTA_SRC_EP}"|\ + "target ${SANDBOX_ZELTA_TGT_EP}"|\ + "* sent, * streams received in * seconds") + ;; + *) + printf "Unexpected line: %s\n" "$line" >&2 + return 1 + ;; + esac + done +} +``` + +### 3. Output Processing + +The generator automatically handles: +- **Environment variables**: Replaces actual values with `${VAR_NAME}` references +- **Timestamps**: Converts `@zelta_2024-01-15_10.30.45` to `@zelta_"*"` +- **Dynamic values**: Wildcards for transfer speeds, stream counts, etc. +- **Backticks**: Proper escaping for shell commands + +## Creating New Tests + +### Step 1: Create YAML Definition + +```bash +vi config/test_defs/070_my_new_test.yml +``` + +### Step 2: Define Test Structure + +```yaml +output_dir: tmp +shellspec_name: "070_my_new_test_spec" +describe_desc: "Test my new feature" + +skip_if_list: + - condition: if 'required var' test -z "$REQUIRED_VAR" + +test_list: + - test_name: first_operation + it_desc: performs operation - %{when_command} + when_command: zelta mycommand "$SANDBOX_ZELTA_SRC_EP" +``` + +### Step 3: Add to Generation Script + +Edit `bin/generate_new_tests.sh`: + +```bash +if ! "$GENERATE_TEST" \ + "$CONFIG_DIR/070_my_new_test.yml" \ + "test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh"; then + + printf "\n ❌ Failed to generate 070 test\n" + exit 1 +fi +``` + +### Step 4: Generate and Test + +```bash +# Generate the test +./bin/generate_new_tests.sh + +# Generated test is now in test/070_my_new_test_spec.sh +``` + +## Environment Variables + +The generator recognizes these environment variables for substitution: +- `SANDBOX_ZELTA_SRC_DS` - Source ZFS dataset +- `SANDBOX_ZELTA_TGT_DS` - Target ZFS dataset +- `SANDBOX_ZELTA_SRC_EP` - Source endpoint (remote:dataset) +- `SANDBOX_ZELTA_TGT_EP` - Target endpoint (remote:dataset) + +Add more in `test_generator.rb`: + +```ruby +DEFAULT_ENV_VAR_NAMES = 'VAR1:VAR2:VAR3' +``` + +## Debugging Generated Tests + +The `bin/debug_gen.sh` script helps you iteratively debug and verify generated tests. This is especially useful when test generation succeeds but the generated test fails validation. + +### Using debug_gen.sh + +1. **Edit the script** to configure your debug session: + ```bash + vi bin/debug_gen.sh + ``` + +2. **Set the required variables**: + ```bash + # Tests to run for tree setup (pipe-separated globs) + SPECS="test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh" + + # The generated test you're debugging + NEW_SPEC="$TEST_GEN_DIR/tmp/050_zelta_revert_spec.sh" + ``` + +3. **Optionally configure trace mode**: + ```bash + # Show detailed trace (helpful for debugging) + TRACE_OPTIONS="--xtrace --shell /opt/homebrew/bin/bash" + + # Or disable trace for cleaner output + #unset TRACE_OPTIONS + ``` + +4. **Run the debug script**: + ```bash + ./bin/debug_gen.sh + ``` + +### What debug_gen.sh Does + +1. Sets up ZFS tree by running the specified setup tests (`SPECS`) +2. Configures debug environment (sources `env/setup_debug_env.sh`) +3. Runs your generated test (`NEW_SPEC`) with optional trace +4. Reports success or failure + +### Why Edit Instead of Arguments? + +Debugging is iterative - you'll likely run the script multiple times while tweaking: +- YAML test definitions +- Tree setup steps +- Environment configuration + +Editing the script ensures you don't accidentally use wrong values and makes it easy to quickly re-run after changes. + +## Troubleshooting + +### Test Generation Fails + +```bash +# Check Ruby syntax +ruby -c lib/ruby/test_generator.rb + +# Validate YAML against schema +./bin/validate_yaml.rb config/test_defs/040_zelta_tests.yml +``` + +### Generated Test Fails During Generation + +Use `debug_gen.sh` to investigate: + +1. Generate the test (it will fail validation) +2. Edit `bin/debug_gen.sh` with appropriate SPECS and NEW_SPEC +3. Run `./bin/debug_gen.sh` to see detailed failure +4. Fix YAML definition or matcher expectations +5. Regenerate and test again + +### Generated Test Fails When Run Manually + +1. Check the generated test: `cat tmp/040_zelta_tests_spec.sh` +2. Check matcher functions: `cat tmp/output_for_*/output_for_*.sh` +3. Run the test manually: `shellspec tmp/040_zelta_tests_spec.sh` +4. Verify ZFS tree state matches expectations + +### Matcher Doesn't Match Output + +The matcher is too strict. Common issues: +- Output has extra whitespace (matcher normalizes this) +- Environment variable not in substitution list +- Timestamp format changed +- Dynamic values not wildcarded + +Regenerate with updated env var list or edit matcher manually. + +## Architecture + +### Path Resolution + +All paths use absolute resolution anchored to: +- **Repo root**: `git rev-parse --show-toplevel` +- **Test generation dir**: `File.expand_path(File.join(__dir__, '..', '..'))` + +This ensures scripts work regardless of current directory. + +### Single Source of Truth + +- **Test name**: Defined in YAML `shellspec_name` field +- **Output directory**: Resolved relative to test_generation dir +- **Setup scripts**: Resolved relative to repo root +- **Config paths**: Resolved relative to test_generator.rb location + +### Code Organization + +- **bin/**: User-facing entry points +- **lib/ruby/**: Core logic (Ruby) +- **lib/orchestration/**: Workflow management (Shell) +- **scripts/**: Utilities (Shell, AWK) +- **config/**: Test definitions and schema + +## See Also + +- [Test Runners Overview](../README.md) +- [Environment Setup](../env/) +- [Main Test Directory](../../) diff --git a/test/runners/test_generation/bin/debug_gen.sh b/test/runners/test_generation/bin/debug_gen.sh new file mode 100755 index 0000000..12cf650 --- /dev/null +++ b/test/runners/test_generation/bin/debug_gen.sh @@ -0,0 +1,51 @@ +# This is a helper for debugging and testing your generated tests +# This can be helpful if you use the generate_new_tests.sh approach +# and your test confirmation fails. +# +# Why? Because you can fine-tune the tree setup and run your test commands +# by hand to determine the cause of problems. Your test yml definition may +# need additional setup or a modified command. So the ability to iteratively +# test out the new spec can help you resolve problems. +# +# SPECS - the shellspec examples you run to setup the tree before running your test +# NEW_SPEC - the generated test you are debugging or verifying + +REPO_ROOT=$(git rev-parse --show-toplevel) + +# standard locations under test +RUNNERS_DIR="$REPO_ROOT/test/runners" +TEST_GEN_DIR="$REPO_ROOT/test/runners/test_generation" + +# tree setup utility +. "$TEST_GEN_DIR/lib/orchestration/setup_tree.sh" + +# prepare the zfs tree with the state represented by running the following examples/tests +SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh" + +# use the directory where your generated test is created +# by default we're using a temp directory off of test/runners/test_generation +NEW_SPEC="$TEST_GEN_DIR/tmp/050_zelta_revert_spec.sh" +echo "confirming new spec: {$NEW_SPEC}" + +if setup_tree "$SPECS"; then + printf "\n ✅ initial tree setup succeeded for specs: %s\n" "$SPECS" +else + printf "\n ❌ Goodbye, initial tree setup failed for specs: %s\n" "$SPECS" + exit 1 +fi + +# +. "$RUNNERS_DIR/env/setup_debug_env.sh" + +# show a detailed trace of the commands you are executing in your new test +TRACE_OPTIONS="--xtrace --shell /opt/homebrew/bin/bash" +# if you don't want/need a detailed trace unset the options var +#unset TRACE_OPTIONS + +# run the new test, show the outcome +if shellspec $TRACE_OPTIONS "$NEW_SPEC"; then + printf "\n ✅ confirmation test succeeded for new spec: %s\n" "$NEW_SPEC" +else + printf "\n ❌ confirmation test failed for new spec: %s\n" "$NEW_SPEC" + exit 1 +fi diff --git a/test/runners/test_generation/bin/generate_new_tests.sh b/test/runners/test_generation/bin/generate_new_tests.sh new file mode 100755 index 0000000..31a0efd --- /dev/null +++ b/test/runners/test_generation/bin/generate_new_tests.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TEST_GEN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG_DIR="$TEST_GEN_DIR/config/test_defs" +GENERATE_TEST="$TEST_GEN_DIR/lib/orchestration/generate_test.sh" + +# Generate tests for 40,50,60 examples + +if ! "$GENERATE_TEST" \ + "$CONFIG_DIR/040_zelta_tests.yml" \ + "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh"; then + + printf "\n ❌ Failed to generate 040 test\n" + exit 1 +fi + +if ! "$GENERATE_TEST" \ + "$CONFIG_DIR/050_zelta_revert_test.yml" \ + "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh"; then + + printf "\n ❌ Failed to generate 050 test\n" + exit 1 +fi + + +if ! "$GENERATE_TEST" \ + "$CONFIG_DIR/060_zelta_clone_test.yml" \ + "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh"; then + + printf "\n ❌ Failed to generate 060 test\n" + exit 1 +fi diff --git a/test/runners/test_generation/bin/validate_yaml.rb b/test/runners/test_generation/bin/validate_yaml.rb new file mode 100755 index 0000000..eae54b6 --- /dev/null +++ b/test/runners/test_generation/bin/validate_yaml.rb @@ -0,0 +1,56 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Validate a YAML test configuration against the schema + +require 'yaml' +require 'json-schema' + +if ARGV.empty? + puts "Usage: #{$PROGRAM_NAME} " + puts "\nExample:" + puts " #{$PROGRAM_NAME} config/test_defs/040_zelta_tests.yml" + exit 1 +end + +yaml_file = ARGV[0] +script_dir = File.dirname(__FILE__) +test_gen_dir = File.expand_path(File.join(script_dir, '..')) +schema_file = File.join(test_gen_dir, 'config', 'test_config_schema.yml') + +# Resolve yaml_file relative to test_generation directory if not absolute +yaml_file = File.join(test_gen_dir, yaml_file) unless yaml_file.start_with?('/') + +unless File.exist?(yaml_file) + puts "❌ YAML file not found: #{yaml_file}" + exit 1 +end + +unless File.exist?(schema_file) + puts "❌ Schema file not found: #{schema_file}" + exit 1 +end + +begin + config = YAML.load_file(yaml_file) + schema = YAML.load_file(schema_file) + + JSON::Validator.validate!(schema, config) + + puts "✅ Valid: #{yaml_file}" + puts "\nConfiguration:" + puts " shellspec_name: #{config['shellspec_name']}" + puts " describe_desc: #{config['describe_desc']}" + puts " output_dir: #{config['output_dir']}" + puts " tests: #{config['test_list']&.length || 0}" + puts " skip conditions: #{config['skip_if_list']&.length || 0}" + + exit 0 +rescue JSON::Schema::ValidationError => e + puts "❌ Validation failed: #{yaml_file}" + puts "\nError: #{e.message}" + exit 1 +rescue StandardError => e + puts "❌ Error: #{e.message}" + exit 1 +end diff --git a/test/runners/test_generation/config/test_config_schema.yml b/test/runners/test_generation/config/test_config_schema.yml new file mode 100644 index 0000000..78234e8 --- /dev/null +++ b/test/runners/test_generation/config/test_config_schema.yml @@ -0,0 +1,51 @@ +type: object +required: + - shellspec_name + - describe_desc + - output_dir + - test_list +properties: + shellspec_name: + type: string + description: Name for the generated ShellSpec file + describe_desc: + type: string + description: Description for the Describe block + output_dir: + type: string + description: Directory for generated test files + skip_if_list: + type: array + description: Condition to skip test generation + items: + type: object + required: + - condition + properties: + condition: + type: string + test_list: + type: array + items: + type: object + required: + - test_name + - it_desc + - when_command + properties: + setup_scripts: + type: array + description: Scripts to source/run before executing when_command to set up environment + items: + type: string + description: Path to script file (relative to test directory or absolute) + test_name: + type: string + it_desc: + type: string + when_command: + type: string + allow_no_output: + type: boolean + description: If true, skip validation that command produces output + default: false diff --git a/test/runners/test_generation/config/test_defs/040_zelta_tests.yml b/test/runners/test_generation/config/test_defs/040_zelta_tests.yml new file mode 100644 index 0000000..1794477 --- /dev/null +++ b/test/runners/test_generation/config/test_defs/040_zelta_tests.yml @@ -0,0 +1,23 @@ +output_dir: tmp +shellspec_name: "040_zelta_tests_spec" +describe_desc: "Run zelta commands on divergent tree" +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + - condition: if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + +test_list: + - test_name: match_after_divergence + it_desc: show divergence - %{when_command} + when_command: zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + + - test_name: rotate_after_divergence + it_desc: rotates after divergence - %{when_command} + when_command: zelta rotate "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + + - test_name: match_after_rotate + it_desc: match after rotate - %{when_command} + when_command: zelta match "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + + - test_name: backup_after_rotate + it_desc: backup after rotate - %{when_command} + when_command: zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" diff --git a/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml b/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml new file mode 100644 index 0000000..5423773 --- /dev/null +++ b/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml @@ -0,0 +1,33 @@ +output_dir: tmp +shellspec_name: "050_zelta_revert_spec" +describe_desc: "Test revert" +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + +test_list: + - test_name: snapshot + it_desc: take a snapshot of tree before changes - %{when_command} + when_command: zelta snapshot --snap-name "manual_test" "$SANDBOX_ZELTA_SRC_EP" + + - test_name: add_and_remove_src_datasets + setup_scripts: + - "test/test_helper.sh" + allow_no_output: true + it_desc: add and remove src datasets - %{when_command} + when_command: add_tree_delta + + - test_name: backup_after_delta + it_desc: backup after deltas - %{when_command} + when_command: zelta backup "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + + - test_name: snapshot_again + it_desc: take a snapshot of tree after changes - %{when_command} + when_command: zelta snapshot --snap-name "another_test" "$SANDBOX_ZELTA_SRC_EP" + + - test_name: revert + it_desc: revert to last snapshot - %{when_command} + when_command: zelta revert "$SANDBOX_ZELTA_SRC_EP"@manual_test + + - test_name: rotate_after_revert + it_desc: rotates after divergence - %{when_command} + when_command: zelta rotate "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" diff --git a/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml b/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml new file mode 100644 index 0000000..d8101ce --- /dev/null +++ b/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml @@ -0,0 +1,16 @@ +output_dir: tmp +shellspec_name: "060_zelta_clone_spec" +describe_desc: "Test clone" +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + +test_list: + - test_name: clone_sub2 + it_desc: zelta clone sub2 - %{when_command} + when_command: zelta clone "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" + + - test_name: zfs_list_for_clone + setup_scripts: + - "test/test_helper.sh" + it_desc: verifies the clone - %{when_command} + when_command: src_exec zfs list -ro name,origin $SANDBOX_ZELTA_SRC_DS/copy_of_sub2 diff --git a/test/runners/test_generation/config/test_defs/dbg_test.yml b/test/runners/test_generation/config/test_defs/dbg_test.yml new file mode 100644 index 0000000..e565b30 --- /dev/null +++ b/test/runners/test_generation/config/test_defs/dbg_test.yml @@ -0,0 +1,18 @@ +output_dir: tmp +shellspec_name: "050_zelta_revert_spec" +describe_desc: "Test revert" +test_list: + - test_name: snapshot + it_desc: take a snapshot of tree before changes - %{when_command} + when_command: zelta snapshot --snap-name "manual_test" "$SANDBOX_ZELTA_SRC_EP" + - test_name: add_and_remove_src_datasets + setup_scripts: + - "../../test_helper.sh" + allow_no_output: true + it_desc: add and remove src datasets - %{when_command} + when_command: add_tree_delta + - test_name: revert + it_desc: revert to last snapshot - %{when_command} + when_command: zelta revert "$SANDBOX_ZELTA_SRC_EP"@manual_test + +#zelta snapshot --snap-name "manual_$(date +%Y%m%d)" "$SANDBOX_ZELTA_SRC_EP" diff --git a/test/runners/test_generation/config/test_defs/example_test.yml b/test/runners/test_generation/config/test_defs/example_test.yml new file mode 100644 index 0000000..bedb48e --- /dev/null +++ b/test/runners/test_generation/config/test_defs/example_test.yml @@ -0,0 +1,10 @@ +output_dir: tmp +shellspec_name: "example_spec" +describe_desc: "Example test" +test_list: + - test_name: zelta_version + it_desc: "should display version information: %{when_command}" + when_command: zelta --version + - test_name: zelta_help + it_desc: "should display help message: %{when_command}" + when_command: zelta --help diff --git a/test/runners/test_generation/debug/run_test_gen_dbg_env.rb b/test/runners/test_generation/debug/run_test_gen_dbg_env.rb new file mode 100755 index 0000000..c751878 --- /dev/null +++ b/test/runners/test_generation/debug/run_test_gen_dbg_env.rb @@ -0,0 +1,35 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# This script is used to debug the test generation environment by sourcing the debug environment setup script +# and then executing the test_generator.rb script with the provided arguments. It's useful ruby debugging +# in RubyMine or other IDEs. By debugging ths script, you can step through the test generation process. +# in a ruby debugger. + +# Find repo root (assumes we're in a git repo) +repo_root = `git rev-parse --show-toplevel`.chomp + +# Source shell environment setup from repo root +debug_env_script = File.join(repo_root, 'test/runners/env/setup_debug_env.sh') + +# Source shell environment setup +if File.exist?(debug_env_script) + output = `bash -c 'cd #{repo_root} && source #{debug_env_script} && env' 2>&1` + + if $?.exitstatus != 0 + $stderr.puts "Failed to source #{debug_env_script}:" + $stderr.puts output + exit $?.exitstatus + end + + ENV.update( + output + .split("\n") + .map { |line| line.split('=', 2) } + .select { |pair| pair.size == 2 } + .to_h + ) +end + +# Replace Ruby process with test_generator.rb +exec('ruby', '../lib/ruby/test_generator.rb', *ARGV) diff --git a/test/runners/test_generation/lib/orchestration/generate_test.sh b/test/runners/test_generation/lib/orchestration/generate_test.sh new file mode 100755 index 0000000..36ef186 --- /dev/null +++ b/test/runners/test_generation/lib/orchestration/generate_test.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +# Check for required arguments +if [ $# -lt 2 ]; then + printf "Usage: %s \n" "$0" >&2 + printf "\t-> * setup zfs tree with state represented by \n" + printf "\t-> * use the to generate a test\n" + printf "\t-> * setup zfs tree again\n" + printf "\t-> * test the generated test from \n" + printf "\t-> * if test passes, move it to production\n" + printf "example:\n" + printf "%s \ \n" "$0" + printf " test_defs/050_zelta_revert_test.yml \ \n" + printf " test/00*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_zelta_tests_spec.sh\n" + exit 1 +fi + +TEST_CONFIG=$1 +SETUP_TREE_SPECS=$2 +REPO_ROOT=$(git rev-parse --show-toplevel) + +PROD_TEST_DIR="$REPO_ROOT/test" +TEST_GEN_DIR="$REPO_ROOT/test/runners/test_generation" +GENERATED_TEST_NAME="" +GENERATED_TEST_PATH="" +PROD_TEST_PATH="" + +echo "REPO_ROOT is: {$REPO_ROOT}" + +. "$TEST_GEN_DIR/lib/orchestration/setup_tree.sh" + +generate_test() { + printf "\n======================\n" + printf "*\n* generating test from %s\n*\n" "$TEST_CONFIG" + printf "======================\n" + # Capture output and extract shellspec_name + gen_output=$("$TEST_GEN_DIR/lib/ruby/test_generator.rb" "$TEST_CONFIG") + gen_result=$? + + echo "$gen_output" + + if [ $gen_result -ne 0 ]; then + return 1 + fi + + # Extract shellspec_name from output + GENERATED_TEST_NAME=$(echo "$gen_output" | grep "^__SHELLSPEC_NAME__:" | cut -d: -f2) + + if [ -z "$GENERATED_TEST_NAME" ]; then + printf "\n ❌ Failed to extract shellspec_name from generator output\n" + return 1 + fi + + # Set paths now that we have the test name + GENERATED_TEST_PATH="$TEST_GEN_DIR/tmp/$GENERATED_TEST_NAME.sh" + PROD_TEST_PATH="$PROD_TEST_DIR/$GENERATED_TEST_NAME.sh" + + printf "\n*\n* Generated test: %s\n*\n" "$GENERATED_TEST_NAME" +} + +confirm_generated_test_works() { + printf "\n*\n* confirming test works %s.sh\n*\n" "$GENERATED_TEST_NAME" + + if shellspec "$GENERATED_TEST_PATH"; then + echo "test confirmed, copy to production" + rm -f "$PROD_TEST_PATH" + mv "$GENERATED_TEST_PATH" "$PROD_TEST_DIR" + else + return 1 + fi +} + +## generate and confirm test + +# setup zfs pools to desired state before running test + if ! setup_tree "$SETUP_TREE_SPECS"; then + printf "\n ❌ Failed to setup ZFS tree with specs %s\n!" "$SETUP_TREE_SPECS" + exit 1 + fi + +# generate the test +if ! generate_test; then + printf "\n ❌ Test generation failed for %s!\n" "$TEST_CONFIG" + exit 1 +fi + +# setup zfs pools to desired state again before running generated test + if ! setup_tree "$SETUP_TREE_SPECS"; then + printf "\n ❌ Failed to setup ZFS tree for testing generated tree with specs %s\n!" "$SETUP_TREE_SPECS" + exit 1 + fi + +# confirm generated test works +if ! confirm_generated_test_works; then + printf "\n ❌ Generated test failed %s\n!" "$SETUP_TREE_SPECS" + exit 1 +fi + +# good test generated and copied to prod +printf "\n ✅ Success, Generated test copied to production %s\n\n" "$PROD_TEST_PATH" +set +x diff --git a/test/runners/test_generation/lib/orchestration/setup_tree.sh b/test/runners/test_generation/lib/orchestration/setup_tree.sh new file mode 100644 index 0000000..cb1dbce --- /dev/null +++ b/test/runners/test_generation/lib/orchestration/setup_tree.sh @@ -0,0 +1,24 @@ +# helper to setup the test zfs pools and datasets in the +# state needed for creating/generating a new test and +# for testing the same. + +REPO_ROOT=${REPO_ROOT:=$(git rev-parse --show-toplevel)} +echo "REPO ROOT: $REPO_ROOT" + +setup_tree() { + setup_specs=$1 + trace_options=$2 + + cd "$REPO_ROOT" || exit 1 + . ./test/test_helper.sh + . ./test/runners/env/helpers.sh + setup_env "1" # setup debug environment + clean_ds_and_pools # reset tree + + if shellspec $trace_options --pattern "$setup_specs"; then + printf "\n ✅ setup succeeded for specs: %s\n" "$setup_specs" + else + printf "\n ❌ setup failed for specs: %s\n" "$setup_specs" + exit 1 + fi +} diff --git a/test/runners/test_generation/lib/ruby/placeholders.rb b/test/runners/test_generation/lib/ruby/placeholders.rb new file mode 100644 index 0000000..24b4898 --- /dev/null +++ b/test/runners/test_generation/lib/ruby/placeholders.rb @@ -0,0 +1,68 @@ +# Performs variable substitution in a string using the values from an object's instance variables. +# The substitution is performed using the %{variable_name} syntax. +# +# Usage examples +# Placeholders.substitute("run %{when_command}", my_obj) +# Placeholders.substitute("run %{when_command}", my_obj, exclusions: [:internal_state]) +# Placeholders.substitute("run %{when_command}", my_obj, inclusions: [:when_command, :runner]) + +module Placeholders + def self.substitute(string, source, inclusions: nil, exclusions: nil) + raise ArgumentError, 'Cannot specify both inclusions and exclusions' if inclusions && exclusions + + vars = if source.is_a?(Hash) + filter_hash(source, inclusions, exclusions) + else + extract_vars_from_object(source, inclusions, exclusions) + end + + print "Substituting variables in string: #{string}\n" + print "Using variables: #{vars.inspect}\n" + string.gsub(/%\{(\w+)\}/) { vars[$1] || vars[$1.to_sym] } + end + + class << self + def filter_hash(hash, inclusions, exclusions) + return hash if inclusions.nil? && exclusions.nil? + + hash.select do |key, _| + key_matches = key_matches_filter?(key, inclusions, exclusions) + puts "Filtering key: #{key}" if key_matches + key_matches + end + end + + def key_matches_filter?(key, inclusions, exclusions) + key_variants = [key, key.to_s, key.to_s.to_sym] + + if inclusions + key_variants.any? { |k| inclusions.include?(k) } + elsif exclusions + key_variants.none? { |k| exclusions.include?(k) } + else + true + end + end + + def extract_vars_from_object(obj, inclusions, exclusions) + obj.instance_variables.each_with_object({}) do |var, hash| + var_name = var.to_s.delete('@') + next unless var_matches_filter?(var_name, inclusions, exclusions) + + hash[var_name] = obj.instance_variable_get(var) + end + end + + def var_matches_filter?(var_name, inclusions, exclusions) + var_variants = [var_name, var_name.to_sym] + + if inclusions + var_variants.any? { |v| inclusions.include?(v) } + elsif exclusions + var_variants.none? { |v| exclusions.include?(v) } + else + true + end + end + end +end diff --git a/test/runners/test_generation/lib/ruby/sys_exec.rb b/test/runners/test_generation/lib/ruby/sys_exec.rb new file mode 100644 index 0000000..bed99cb --- /dev/null +++ b/test/runners/test_generation/lib/ruby/sys_exec.rb @@ -0,0 +1,106 @@ +# lib/sys_exec.rb +require 'open3' +require 'timeout' + +module SysExec + class ExecutionTimeout < StandardError; end + class SysExecFailed < StandardError; end + + def self.run(cmd, timeout: 30, raise_on_failure: true, debug: true) + puts "Executing: #{cmd}" if debug + + stdout = '' + stderr = '' + status = nil + pid = nil + timed_out = false + + # handle timeouts, always raise an exception if the command times out + begin + Open3.popen3(cmd) do |stdin, out, err, wait_thr| + pid = wait_thr.pid + stdin.close + + timed_out = read_streams_with_timeout(out, err, stdout, stderr, wait_thr, timeout) + + if timed_out + Process.kill('TERM', pid) rescue nil + sleep 0.1 + Process.kill('KILL', pid) rescue nil + else + status = wait_thr.value + end + end + rescue => e + timed_out = true + end + + if timed_out + raise ExecutionTimeout, error_msg(reason: "Command timed out after #{timeout} seconds", + cmd: cmd, stdout: stdout, stderr: stderr) + end + + if debug + puts "STDOUT: #{stdout.encode('UTF-8', invalid: :replace, undef: :replace)}" unless stdout.empty? + puts "STDERR: #{stderr.encode('UTF-8', invalid: :replace, undef: :replace)}" unless stderr.empty? + puts "Exit status: #{status&.exitstatus}" + end + + # if command failed and raise_on_failure is true, raise an exception + if status && status.exitstatus != 0 && raise_on_failure + raise SysExecFailed,error_msg(reason: "Command failed with exit status #{status.exitstatus}", + cmd: cmd, stdout: stdout, stderr: stderr) + end + + { stdout: stdout, stderr: stderr, exit_status: status&.exitstatus } + end + def self.read_streams_with_timeout(out, err, stdout, stderr, wait_thr, timeout) + start_time = Time.now + + loop do + if Time.now - start_time > timeout + return true + end + + # Use select to check if data is available + ready = IO.select([out, err], nil, nil, 0.1) + if ready + ready[0].each do |io| + begin + data = io.read_nonblock(1024) + data = data.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, undef: :replace) + stdout << data if io == out + stderr << data if io == err + rescue IO::WaitReadable + # Nothing available right now + rescue EOFError + # Stream closed + end + end + end + + # Check if process finished + break unless wait_thr.alive? + end + + false + end + + def self.error_msg(reason:, cmd:, stdout:, stderr:) + env_cmd = cmd.gsub(/\$\{?(\w+)\}?/) { ENV[$1] || "#{$&}:undefined" } + + msg = <<~MSG + \nERROR: #{reason} + Command: #{cmd} + Command with env substitution: #{env_cmd} + STDOUT so far: + #{stdout.lines.map { |line| " : #{line}" }.join} + STDERR so far: + #{stderr.lines.map { |line| " : #{line}" }.join} + MSG + + msg.lines.map { |line| "*** #{line}" }.join + end + + private_class_method :error_msg, :read_streams_with_timeout +end diff --git a/test/runners/test_generation/lib/ruby/test_generator.rb b/test/runners/test_generation/lib/ruby/test_generator.rb new file mode 100755 index 0000000..1acf825 --- /dev/null +++ b/test/runners/test_generation/lib/ruby/test_generator.rb @@ -0,0 +1,382 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Manage generation of ShellSpec test files from YAML configuration + +require 'English' +require 'json-schema' +require 'yaml' +require 'fileutils' +require 'time' +require_relative 'placeholders' +require_relative 'sys_exec' + +# TestGenerator - Generates ShellSpec test files from YAML configuration +class TestGenerator + REPO_ROOT = `git rev-parse --show-toplevel`.strip + TEST_GEN_DIR = File.expand_path(File.join(__dir__, '..', '..')) + GENERATE_MATCHER_SH_SCRIPT = File.join(TEST_GEN_DIR, 'scripts', 'sh', 'generate_matcher.sh') + + # TODO: determine if additional defaults are needed, as more tests are added, we may need to update this + DEFAULT_ENV_VAR_NAMES = 'SANDBOX_ZELTA_TGT_DS:SANDBOX_ZELTA_SRC_DS:SANDBOX_ZELTA_TGT_EP:SANDBOX_ZELTA_SRC_EP' + + private_constant :REPO_ROOT, :TEST_GEN_DIR, :GENERATE_MATCHER_SH_SCRIPT + + attr_reader :config, :output_dir, :shellspec_name, :describe_desc, :test_list, :skip_if_list, + :matcher_files, :wip_file_path, :final_file_path, :env_var_names, :sorted_env_map + + def initialize(yaml_file_path, env_var_names = DEFAULT_ENV_VAR_NAMES) + # Resolve path relative to this file's directory if it's a relative path + yaml_file_path = File.expand_path(yaml_file_path, __dir__) unless yaml_file_path.start_with?('/') + + raise "YAML file not found: #{yaml_file_path}" unless File.exist?(yaml_file_path) + + @config = YAML.load_file(yaml_file_path) + validate_config! + + @shellspec_name = @config['shellspec_name'] + @describe_desc = @config['describe_desc'] + + # Resolve output_dir relative to test_generation directory + output_dir = @config['output_dir'] + @output_dir = output_dir.start_with?('/') ? output_dir : File.join(TEST_GEN_DIR, output_dir) + + @test_list = @config['test_list'] || [] + @skip_if_list = @config['skip_if_list'] || [] + @matcher_files = [] + @wip_file_path = File.join(@output_dir, "#{@shellspec_name}_wip.sh") + # remove _spec to prevent shellspec from finding the WIP file + @wip_file_path.sub!('_spec', '') + @final_file_path = File.join(@output_dir, "#{@shellspec_name}.sh") + @env_var_names = env_var_names + @sorted_env_map = build_sorted_env_map + puts "Loading configuration from: #{@config.inspect}\n" + puts '=' * 60 + end + + def generate + create_output_directory + create_wip_file + process_tests + assemble_final_file + report_summary + end + + private + + def build_sorted_env_map + # Parse and sort env vars by value length (descending) + env_map = @env_var_names.split(':').each_with_object({}) do |name, hash| + hash[name] = ENV[name] if ENV[name] + end + + # Sort by value length descending to replace longest matches first + env_map.sort_by { |_name, value| -value.length } + end + + def matcher_func_name(test_name) + "output_for_#{test_name}" + end + + def validate_config!(schema_path = File.join(TEST_GEN_DIR, 'config', 'test_config_schema.yml')) + schema = YAML.load_file(schema_path) + JSON::Validator.validate!(schema, @config) + end + + def create_output_directory + FileUtils.mkdir_p(@output_dir) + puts "Created output directory: #{@output_dir}" + end + + def create_wip_file + File.open(@wip_file_path, 'w') do |file| + file.puts "Describe '#{@describe_desc}'" + + # Add Skip If statements for each condition + @skip_if_list.each do |skip_item| + file.puts " Skip #{skip_item['condition']}" + end + file.puts '' unless @skip_if_list.empty? + end + puts "Created WIP file: #{@wip_file_path}" + end + + def process_tests + @test_list.each do |test| + test_name = test['test_name'] + # allow var substitution in test description + it_desc = Placeholders.substitute(test['it_desc'], test, inclusions: [:when_command]) + + when_command = test['when_command'] + setup_scripts = test['setup_scripts'] || [] + allow_no_output = test['allow_no_output'] || false + + puts "Processing test: #{test_name}" + + # Generate matcher files + generate_matcher_files(test_name, when_command, setup_scripts, allow_no_output) + + # Append It clause to WIP file + append_it_clause(test_name, it_desc, when_command, allow_no_output) + end + + # Close Describe block + File.open(@wip_file_path, 'a') do |file| + file.puts 'End' + end + end + + def generate_matcher_files(test_name, when_command, setup_scripts, allow_no_output) + matcher_script = GENERATE_MATCHER_SH_SCRIPT + matcher_function_name = matcher_func_name(test_name) + + unless File.exist?(matcher_script) + puts "Warning: Matcher generator script not found: #{matcher_script}" + return + end + + # Build command with optional setup scripts + full_command = build_command_with_setup(when_command, setup_scripts) + + # Add allow_no_output flag + allow_no_output_flag = allow_no_output ? "true" : "false" + + cmd = "#{matcher_script} \"#{full_command}\" #{matcher_function_name} #{@output_dir} #{allow_no_output_flag}" + SysExec.run(cmd, timeout: 10) + + unless allow_no_output + # Track the generated matcher file + func_name = matcher_func_name(test_name) + matcher_file = File.join(@output_dir, func_name, "#{func_name}.sh") + + # Post-process the matcher file to apply env substitutions + if File.exist?(matcher_file) + post_process_matcher_file(matcher_file) + puts "Generated matcher file: #{matcher_file}" + @matcher_files << matcher_file + end + end + end + + def post_process_matcher_file(matcher_file) + # Read the matcher file and apply env substitutions to case statement patterns + content = File.read(matcher_file) + lines = content.lines + + # Process each line + processed_lines = lines.map do |line| + # Only process lines that look like case patterns (contain quoted strings) + if line =~ /^\s*".*"(?:\)|\|\\)$/ + # Extract the quoted content, normalize it, and reconstruct the line + if line =~ /^(\s*)"(.*)"(\)|\|\\)$/ + indent = $1 + pattern = $2 + suffix = $3 + normalized = normalize_output_line(pattern) + "#{indent}\"#{normalized}\"#{suffix}\n" + else + line + end + else + line + end + end + + # Write back the processed content + File.write(matcher_file, processed_lines.join) + end + + def build_command_with_setup(when_command, setup_scripts) + return when_command if setup_scripts.empty? + + # Resolve relative script paths to absolute paths relative to repo root + resolved_scripts = setup_scripts.map do |script| + if script.start_with?('/') + script + else + File.join(REPO_ROOT, script) + end + end + + # Build source commands for each setup script (using . for POSIX compatibility) + source_cmds = resolved_scripts.map { |script| ". #{script}" } + + # Combine all source commands with the actual command + "#{source_cmds.join(' && ')} && #{when_command}" + end + + def append_it_clause(test_name, it_desc, when_command, allow_no_output) + File.open(@wip_file_path, 'a') do |file| + file.puts " It \"#{it_desc.gsub('"', '\\"')}\"" + + func_name = matcher_func_name(test_name) + + # TODO: clean up all the trial and error with shellspec error output, documented approaches don't work! + # Check for stderr output + stderr_file = File.join(@output_dir, func_name, "#{func_name}_stderr.out") + expected_error = nil + if File.exist?(stderr_file) && !File.zero?(stderr_file) + expected_error = format_expected_error(stderr_file) + #file.puts expected_error + #status_line = ' The status should be failure' + else + #status_line = ' The status should equal 0' + end + + # TODO: zelta exits with 0 even when there is error output + #status_line = ' The status should equal 0' + status_line = ' The status should be success' + + file.puts " When call #{when_command}" + + file.puts " The output should satisfy #{matcher_func_name(test_name)}" unless allow_no_output + + file.puts " The error should equal \"#{expected_error}\"\n" if expected_error + file.puts status_line + + file.puts ' End' + file.puts '' + end + end + + def v1_format_expected_error(stderr_file) + lines = read_stderr_file(stderr_file) + result = " expected_error=%text\n" + lines.each do |line| + result += " #|#{line}\n" + end + "#{result} End\n" + end + # expected_error() { %text + # #|warning: insufficient snapshots; performing full backup for 3 datasets + # #|warning: missing `zfs allow` permissions: readonly,mountpoint + # } + def v2_format_expected_error(stderr_file) + lines = read_stderr_file(stderr_file) + result = " expected_error() { %text\n" + lines.each do |line| + result += " #|#{line}\n" + end + "#{result} }\n" + end + + def format_expected_error(stderr_file) + lines = read_stderr_file(stderr_file) + lines.map! { |line| normalize_output_line(line) } + lines.join("\n") + end + + def normalize_output_line(line) + # Normalize whitespace + normalized = line.gsub(/\s+/, ' ').strip + + # Replace timestamp patterns (both @zelta_ and _zelta_ prefixes) + normalized.gsub!(/@zelta_\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}/, '@zelta_"*"') + normalized.gsub!(/_zelta_\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}/, '_zelta_"*"') + + # Escape backticks + normalized.gsub!('`', '\\\`') + + # Wildcard time and quantity sent + if normalized =~ /(\d+[KMGT]? sent, )(\d+ streams)( received in \d+\.\d+ seconds)/ + stream_count = $2 + normalized.gsub!(/\d+[KMGT]? sent, \d+ streams received in \d+\.\d+ seconds/, + "* sent, #{stream_count} received in * seconds") + end + + # Substitute env var names for values (longest first) + # Use a placeholder to prevent already-substituted values from being re-matched + placeholder_map = {} + @sorted_env_map.each_with_index do |(name, value), idx| + placeholder = "__ENV_PLACEHOLDER_#{idx}__" + normalized.gsub!(value, placeholder) + placeholder_map[placeholder] = "${#{name}}" + end + + # Replace placeholders with actual env var references + placeholder_map.each do |placeholder, replacement| + normalized.gsub!(placeholder, replacement) + end + + normalized + end + + def read_stderr_file(stderr_file) + File.readlines(stderr_file).map(&:chomp) + rescue StandardError => e + puts "Warning: Could not read stderr file #{stderr_file}: #{e.message}" + [] + end + + def assemble_final_file + File.open(@final_file_path, 'w') do |final| + final.puts '# Auto-generated ShellSpec test file' + final.puts "# Generated at: #{Time.now}" + final.puts "# Source: #{@shellspec_name}" + final.puts '# WARNING: This file was automatically generated. Manual edits may be lost.' + final.puts '' + + # Copy all matcher function files + @matcher_files.each do |matcher_file| + if File.exist?(matcher_file) + final.puts File.read(matcher_file) + final.puts '' + end + end + + # Copy the WIP file content + final.puts File.read(@wip_file_path) if File.exist?(@wip_file_path) + end + puts "Assembled final test file: #{@final_file_path}" + end + + def report_summary + puts "\n#{'=' * 60}" + puts 'Test Generation Summary' + puts '=' * 60 + puts "YAML Configuration: #{@config.inspect}" + puts "ShellSpec Name: #{@shellspec_name}" + puts "Description: #{@describe_desc}" + puts "Output Directory: #{@output_dir}" + puts "Tests Processed: #{@test_list.length}" + puts "Matcher Files Generated: #{@matcher_files.length}" + puts "\nGenerated Files:" + puts " - WIP File: #{@wip_file_path}" + @matcher_files.each do |file| + puts " - Matcher: #{file}" + end + puts "\nFinal ShellSpec Test File:" + puts " Location: #{@final_file_path}" + puts '=' * 60 + puts "__SHELLSPEC_NAME__:#{@shellspec_name}" + end +end + +def run_generator + if ARGV.empty? + puts "Usage: #{$PROGRAM_NAME} " + puts "\nExample YAML format:" + puts <<~YAML + shellspec_name: example_tests + describe_desc: Example Zelta Command Tests + output_dir: test/output + test_list: + - test_name: test_version + it_desc: should display version information + when_command: zelta --version + - test_name: test_help + it_desc: should display help message + when_command: zelta --help + YAML + return 1 + end + + yaml_file = ARGV[0] + generator = TestGenerator.new(yaml_file) + generator.generate + 0 +end + + +# Script execution +run_generator if __FILE__ == $PROGRAM_NAME diff --git a/spec/util/generate_case_stmt_func.awk b/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk similarity index 60% rename from spec/util/generate_case_stmt_func.awk rename to test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk index 3340ab2..ba51731 100644 --- a/spec/util/generate_case_stmt_func.awk +++ b/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk @@ -10,6 +10,22 @@ { gsub(/[[:space:]]+/, " ", $0) gsub(/@zelta_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, "@zelta_\"*\"",$0) + gsub(/_zelta_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, "_zelta_\"*\"",$0) + gsub(/`/, "\\`", $0) + + # wildcard time and quantity sent + if (match($0, /[0-9]+[KMGT]? sent, [0-9]+ streams/)) { + # Extract the part with streams + streams_part = substr($0, RSTART, RLENGTH) + # Extract just the number before " streams" + match(streams_part, /[0-9]+ streams/) + stream_count = substr(streams_part, RSTART, RLENGTH) + gsub(/[0-9]+[KMGT]? sent, [0-9]+ streams received in [0-9]+\.[0-9]+ seconds/, "* sent, " stream_count " received in * seconds", $0) + } + + # remove trailing spaces + sub(/[[:space:]]+$/, "", $0) + lines[count++] = $0 } diff --git a/test/runners/test_generation/scripts/sh/generate_matcher.sh b/test/runners/test_generation/scripts/sh/generate_matcher.sh new file mode 100755 index 0000000..0aecbc6 --- /dev/null +++ b/test/runners/test_generation/scripts/sh/generate_matcher.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Check for required arguments +if [ $# -lt 3 ] || [ $# -gt 4 ]; then + printf "Usage: %s [allow_no_output]\n" "$0" >&2 + printf "\t-> * put your zfs datasets in the desired state before running\n" + printf "\t-> * to capture the zelta output that would be captured by a test\n" + printf "\t-> * allow_no_output: 'true' to skip output validation (optional)\n" + exit 1 +fi + +zelta_cmd=$1 +func_name=$2 +output_dir=$3 + +if [ $# -eq 4 ]; then + allow_no_output=$4 +else + allow_no_output="false" +fi + +OUT_DIR=${output_dir}/${func_name} +OUT_FL=${OUT_DIR}/${func_name}_stdout.out +ERR_FL=${OUT_DIR}/${func_name}_stderr.out +MATCHER_FL=${OUT_DIR}/${func_name}.sh +MATCHER_FUNC_GEN="$SCRIPT_DIR/matcher_func_generator.sh" + +echo "" +echo "====================================" +echo "Creating matcher file with settings:" +echo "====================================" +echo "zelta_cmd={$zelta_cmd}" +echo "func_name={$func_name}" +echo "output dir={$output_dir}" +echo "matcher output dir={$OUT_DIR}" +echo "allow_no_output={$allow_no_output}" +echo "" +echo "running from dir:{$(pwd)}" +echo "running generator:{$MATCHER_FUNC_GEN}" +echo "====================================" + +mkdir -p "$OUT_DIR" + +if ! sh -c "$zelta_cmd" > "$OUT_FL" 2> "$ERR_FL"; then + # zelta exiting with error is allowed, then stderr should have output that will be put into + # a test + # TODO: test this case where zelta exits with non-zero code, does the test generator work correctly? + printf " ❌ Zelta command failed: %s\n\n" "$zelta_cmd" + cat "$ERR_FL" +fi + +if [ ! -s "$OUT_FL" ]; then + if [ "$allow_no_output" != "true" ]; then + printf "\n ❌ Error: zelta produced no output\n" + printf "****-> review and update zelta cmd: \"%s\"\n" "$zelta_cmd" + exit 1 + else + printf "\n âš ī¸ Command produced no output (skipping matcher generation)\n" + exit 0 + fi +fi + +# Skip matcher generation if allow_no_output is true +if [ "$allow_no_output" = "true" ]; then + printf "\n â„šī¸ Skipping matcher generation (allow_no_output=true)\n" + exit 0 +fi + +printf "Generating matcher function...\n" +$MATCHER_FUNC_GEN "$OUT_FL" "$func_name" > "$MATCHER_FL" + +if [ $? -eq 0 ] && [ -s "$MATCHER_FL" ]; then + printf " ✅ Success, matcher generated to file %s\n\n" "$MATCHER_FL" +else + printf "\n ❌ Matcher generation failed!\n" + exit 1 +fi diff --git a/spec/util/matcher_func_generator.sh b/test/runners/test_generation/scripts/sh/matcher_func_generator.sh similarity index 62% rename from spec/util/matcher_func_generator.sh rename to test/runners/test_generation/scripts/sh/matcher_func_generator.sh index 83b11b6..6f7852f 100755 --- a/spec/util/matcher_func_generator.sh +++ b/test/runners/test_generation/scripts/sh/matcher_func_generator.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + # Check for required arguments if [ $# -ne 2 ]; then printf "Usage: %s \n" "$0" >&2 @@ -15,6 +18,5 @@ if [ ! -f "$input_file" ]; then exit 1 fi -# Pass to AWKt -awk -v func_name="$func_name" -f generate_case_stmt_func.awk "$input_file" - +awk -v func_name="$func_name" \ + -f "$SCRIPT_DIR/../awk/generate_case_stmt_func.awk" "$input_file" diff --git a/test/test.sh b/test/test.sh deleted file mode 100755 index d57c056..0000000 --- a/test/test.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -apool=apool -bpool=bpool -src_ep='root@host07.bts' -tgt_ep='root@host07.bts' -src="$tgt_ep:$apool/treetop" -tgt="$tgt_ep:$bpool/bleetop" -which zelta -echo $ZELTA_SHARE -export ZELTA_LOG_LEVEL=4 -export AWK="gawk" -clear - -{ - -set -x -sleep 1 - -zelta backup "$src" "$tgt" -zelta rotate "$src" "$tgt" -zelta revert "$src" -zelta rotate "$src" "$tgt" - -#zelta backup --snapshot-always "$src" "$tgt" -#zelta backup --snapshot-always "$src" "$tgt" -#zelta match "$src" "$tgt" -#zelta rotate "$src" "$tgt" -#zelta match "$src" "$tgt" -##zelta revert "$src" -#zelta rotate "$src" "$tgt" -#zelta match "$src" "$tgt" - -} diff --git a/test/test_helper.sh b/test/test_helper.sh index ba16460..625581f 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -15,15 +15,44 @@ ## Setup temporary installation for testing ############################################# -export SANDBOX_ZELTA_TMP_DIR="/tmp/zelta$$" -export SANDBOX_ZELTA_PROCNUM="$$" -export ZELTA_BIN="$SANDBOX_ZELTA_TMP_DIR/bin" -export ZELTA_SHARE="$SANDBOX_ZELTA_TMP_DIR/share" -export ZELTA_ETC="$SANDBOX_ZELTA_TMP_DIR/etc" -export ZELTA_DOC="$SANDBOX_ZELTA_TMP_DIR/man" -export PATH="$ZELTA_BIN:$PATH" +setup_env() { + export SANDBOX_ZELTA_TMP_DIR="/tmp/zelta$$" + export SANDBOX_ZELTA_PROCNUM="$$" + export ZELTA_BIN="$SANDBOX_ZELTA_TMP_DIR/bin" + export ZELTA_SHARE="$SANDBOX_ZELTA_TMP_DIR/share" + export ZELTA_ETC="$SANDBOX_ZELTA_TMP_DIR/etc" + export ZELTA_DOC="$SANDBOX_ZELTA_TMP_DIR/man" + export PATH="$ZELTA_BIN:$PATH" +} + +build_endpoints() { + ## Build endpoints + if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then + SANDBOX_ZELTA_SRC_EP="${SANDBOX_ZELTA_SRC_REMOTE}:${SANDBOX_ZELTA_SRC_DS}" + else + SANDBOX_ZELTA_SRC_EP="$SANDBOX_ZELTA_SRC_DS" + fi + + if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then + SANDBOX_ZELTA_TGT_EP="${SANDBOX_ZELTA_TGT_REMOTE}:${SANDBOX_ZELTA_TGT_DS}" + else + SANDBOX_ZELTA_TGT_EP="$SANDBOX_ZELTA_TGT_DS" + fi + + export SANDBOX_ZELTA_SRC_POOL SANDBOX_ZELTA_TGT_POOL + export SANDBOX_ZELTA_SRC_DS SANDBOX_ZELTA_TGT_DS + export SANDBOX_ZELTA_SRC_EP SANDBOX_ZELTA_TGT_EP +} + +# bypass using $$ if we've manually set these vars +if [ -z "$SANDBOX_ZELTA_TMP_DIR" ]; then + setup_env +fi + +build_endpoints + # We could use the repo dirs, but better to test installation # use_repo_zelta() { # REPO_ROOT="$SHELLSPEC_PROJECT_ROOT" @@ -31,22 +60,6 @@ export PATH="$ZELTA_BIN:$PATH" # export ZELTA_SHARE="$REPO_ROOT/share/zelta" # } -## Build endpoints -if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then - SANDBOX_ZELTA_SRC_EP="${SANDBOX_ZELTA_SRC_REMOTE}:${SANDBOX_ZELTA_SRC_DS}" -else - SANDBOX_ZELTA_SRC_EP="$SANDBOX_ZELTA_SRC_DS" -fi - -if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then - SANDBOX_ZELTA_TGT_EP="${SANDBOX_ZELTA_TGT_REMOTE}:${SANDBOX_ZELTA_TGT_DS}" -else - SANDBOX_ZELTA_TGT_EP="$SANDBOX_ZELTA_TGT_DS" -fi - -export SANDBOX_ZELTA_SRC_POOL SANDBOX_ZELTA_TGT_POOL -export SANDBOX_ZELTA_SRC_DS SANDBOX_ZELTA_TGT_DS -export SANDBOX_ZELTA_SRC_EP SANDBOX_ZELTA_TGT_EP ## Command execution wrappers @@ -208,11 +221,13 @@ make_src_pool() { tmpfile_touch "src_pool_created" # Grant ZFS permissions for source pool + # original list snapshot,bookmark,send,hold + ZFS_SRC_PERMS=snapshot,bookmark,send,hold,clone,create,mount,canmount,mountpoint,rename,readonly if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then #ssh -n "$SANDBOX_ZELTA_SRC_REMOTE" - src_exec "zfs allow -u \$USER snapshot,bookmark,send,hold $SANDBOX_ZELTA_SRC_POOL" + src_exec "zfs allow -u \$USER $ZFS_SRC_PERMS $SANDBOX_ZELTA_SRC_POOL" else - zfs allow -u "$USER" snapshot,bookmark,send,hold "$SANDBOX_ZELTA_SRC_POOL" + sudo zfs allow -u "$USER" $ZFS_SRC_PERMS "$SANDBOX_ZELTA_SRC_POOL" fi return $? } @@ -222,11 +237,13 @@ make_tgt_pool() { tmpfile_touch "tgt_pool_created" # Grant ZFS permissions for target pool + # original list receive,mount,create,canmount,rename + ZFS_TGT_PERMS=receive,snapshot,bookmark,send,hold,clone,create,mount,canmount,mountpoint,rename,readonly if [ -n "$SANDBOX_ZELTA_TGT_REMOTE" ]; then #ssh -n "$SANDBOX_ZELTA_TGT_REMOTE" "zfs allow -u \$USER mount,create,rename $SANDBOX_ZELTA_TGT_POOL" - tgt_exec "zfs allow -u \$USER receive,mount,create,canmount,rename $SANDBOX_ZELTA_TGT_POOL" + tgt_exec "zfs allow -u \$USER $ZFS_TGT_PERMS $SANDBOX_ZELTA_TGT_POOL" else - zfs allow -u "$USER" receive,mount,create,canmount,rename "$SANDBOX_ZELTA_TGT_POOL" + sudo zfs allow -u "$USER" $ZFS_TGT_PERMS "$SANDBOX_ZELTA_TGT_POOL" fi return $? } @@ -264,13 +281,16 @@ clean_tgt_ds() { # Create divergent tree structure on source # Creates a dataset tree with snapshots that will diverge from target -make_divergent_tree() { + +# Create divergent tree structure on source +# Creates a dataset tree with snapshots that will diverge from target +make_initial_tree() { if src_ds_exists; then - echo "$SANDBOX_ZELTA_TGT_DS" already exists >/dev/stderr + echo "$SANDBOX_ZELTA_SRC_DS" already exists >/dev/stderr return 1 fi if tgt_ds_exists; then - echo "$SANDBOX_ZELTA_SRC_DS" already exists >/dev/stderr + echo "$SANDBOX_ZELTA_TGT_DS" already exists >/dev/stderr return 1 fi @@ -282,23 +302,29 @@ make_divergent_tree() { # Create root dataset - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS" || return 1 - + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS" || return 1 + # Create child datasets - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub1" || return 1 - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub2" || return 1 - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub2/orphan" || return 1 - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3" || return 1 - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub1" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub2" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub2/orphan" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub3" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub3/space\ name" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub4" || return 1 src_exec zfs create -sV 8M "$SANDBOX_ZELTA_SRC_DS/sub4/zvol" || return 1 - src_exec zfs create -o encryption=on -o keyformat=raw -o "keylocation=file:///tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 - - # Replicate to target with @start snapshot - zelta backup --snap-name @start "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" || return 1 - + src_exec zfs create -u -o encryption=on -o keyformat=raw -o "keylocation=file:///tmp/zfs_test_enc_key_${SANDBOX_ZELTA_PROCNUM}" "$SANDBOX_ZELTA_SRC_DS/sub4/encrypted" || return 1 + + return 0 +} + +# zelta backup moved to 022_setup_tree_spec as: +# Divergent ree tests -> setup -> can zelta backup initial tree +# Replicate to target with @start snapshot +# zelta backup --snap-name @start "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" || return 1 + +make_tree_divergence() { # Generate divergence - src_exec zfs create "$SANDBOX_ZELTA_SRC_DS/sub1/child" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub1/child" || return 1 tgt_exec zfs create -u "$SANDBOX_ZELTA_TGT_DS/sub1/kid" || return 1 src_exec zfs destroy "$SANDBOX_ZELTA_SRC_DS/sub2@start" || return 1 tgt_exec zfs snapshot "$SANDBOX_ZELTA_TGT_DS/sub3/space\ name@blocker" || return 1 @@ -306,6 +332,14 @@ make_divergent_tree() { src_exec zfs snapshot "$SANDBOX_ZELTA_SRC_DS/sub3@two" || return 1 src_exec zfs snapshot "$SANDBOX_ZELTA_SRC_DS/sub2@two" || return 1 tgt_exec zfs snapshot "$SANDBOX_ZELTA_TGT_DS/sub2@two" || return 1 - + + return 0 +} + +add_tree_delta() { + # make changes, we'll call this after snapshotting + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub5" || return 1 + src_exec zfs create -u "$SANDBOX_ZELTA_SRC_DS/sub5/child1" || return 1 + return 0 } diff --git a/test/test_runner.sh b/test/test_runner.sh deleted file mode 100755 index b7838f1..0000000 --- a/test/test_runner.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh - -set -e - -. spec/lib/script_util.sh - - - -test_setup() { - if [ $# -ne 2 ]; then - echo "Error: Expected 2 arguments: " >&2 - echo "Usage: $0 <${RUN_LOCALLY}|${RUN_REMOTELY}> <${STANDARD_TREE}|${DIVERGENT_TREE}|${ENCRYPTED_TREE}>" >&2 - return 1 - fi - - if ! validate_target "$1"; then - return 1 - fi - - if ! validate_tree_name "$2"; then - return 1 - fi - - case "$RUNNING_MODE" in - "$RUN_LOCALLY") - unset TGT_SVR - unset SRV_SVR - exec_local_setup - ;; - "$RUN_REMOTELY") - export SRC_SVR="${SRC_SVR:-dever@fzfsdev}" - # TODO: sort out 2nd server send/receive with host alias fzfsdev2 - # export TGT_SVR="${TGT_SVR:-dever@fzfsdev2}" - export TGT_SVR="${TGT_SVR:-dever@fzfsdev}" - exec_remote_setup - ;; - esac - -} - -exec_local_setup() { - printf "\n***\n*** Running Locally\n***\n" - - echo "Step 1/3: Initializing local test environment..." - spec/bin/all_tests_setup/all_tests_setup.sh - - echo "Step 2/3: Creating test dataset tree..." - sudo spec/bin/${TREE_NAME}_test/${TREE_NAME}_snap_tree.sh -} - -exec_remote_setup() { - printf "\n***\n*** Running Remotely: SRC_SVR:{$SRC_SVR} TGT_SVR:{$TGT_SVR}\n***\n" - - echo "Steps 1 and 2, Initializing remote setup, create pools setup snap tree" - spec/bin/ssh_tests_setup/setup_remote_host_test_env.sh $TREE_NAME -} - -run_tests() { - echo "Step 3/3: Running zelta tests..." - - # shellspec options to include - #SHELLSPEC_TESTOPT="${SHELLSPEC_TESTOPT:-}" - - # this options will show a trace with expectation evaluation - SHELLSPEC_TESTOPT="--xtrace --shell bash" - - # comment out this line to show the trace - unset SHELLSPEC_TESTOPT - - shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh - - # examples of selective tests runs - # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@1 - # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2 - # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2-1 - # shellspec -f d $SHELLSPEC_TESTOPT spec/bin/${TREE_NAME}_test/${TREE_NAME}_test_spec.sh:@2-2 - - echo "" - echo "✓ Tests complete" -} - -if test_setup "$@"; then - # NOTE: update the environment after SRC_SVR and TGT_SVR are set!! - . spec/bin/all_tests_setup/common_test_env.sh - run_tests - #printf "***\n*** check tree run tests manually\n***\n" -fi From 6fc00fc7ed3d39b5ef309c0d0fc5b9e086d52d9e Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 6 Mar 2026 11:51:27 -0500 Subject: [PATCH 44/47] reamde and changelog tweak --- CHANGELOG.md | 2 ++ README.md | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b70ca5c..2cab86f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to Zelta will be documented in this file. - **Commands**: `zelta revert` for in-place rollbacks via rename and clone. - **Commands**: `zelta rotate` for divergent version handling, evolved from original `--rotate` flag. - **Commands**: (Experimental) `zelta prune` identifies snapshots in `zfs destroy` range syntax based on replication state and a sliding window for exclusions. +- **Installer**: (Experimental) Added a one-liner/pipe to shell installer option. - **Uninstaller**: Added `uninstall.sh` for clean removal of Zelta installations, including legacy paths from earlier betas. - **Core**: `zelta-args.awk` added as a separate data-driven argument preprocessor. - **Core**: `zelta-common.awk` library for centralized string and logging functions. @@ -19,6 +20,7 @@ All notable changes to Zelta will be documented in this file. - **Docs**: `zelta.env` expanded with comprehensive inline documentation and examples for all major configuration categories. - **Docs**: New man pages: `zelta-options(7)`, `zelta-revert(8)`, `zelta-rotate(8)`, `zelta-prune(8)`. - **Docs**: Added tool to sync man pages with the zelta.space wiki. +- **Testing**: Added new advanced Shellspec-based testing suite. ### Changed - **Architecture**: Refactored all core scripts for maintainability and simpler logic. diff --git a/README.md b/README.md index a986cd2..0864695 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,11 @@ pkg install zelta ``` ### Experimental: One-Shot Install +The following script will `git clone` Zelta from the main branch and run the installer. For non-root installations, follow the instructions to add the required environment variables. ```sh -curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sudo sh +curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sh ``` -**Note:** This is experimental. For production use, we recommend cloning the repository and reviewing `install.sh` before running it. - --- ## Quickstart: Developer Workflow From fa49aab764e5b2d229ddccc97c505dd12fc5873a Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Fri, 6 Mar 2026 12:20:09 -0500 Subject: [PATCH 45/47] doc: clearer note about the one-shot installer --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0864695..efcd9b5 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,17 @@ pkg install zelta ``` ### Experimental: One-Shot Install -The following script will `git clone` Zelta from the main branch and run the installer. For non-root installations, follow the instructions to add the required environment variables. + +The following command clones Zelta from the `main` branch and launches the installer. Run this as a personal or backup user for a local, non-root installation. If you prefer a system-wide installation, run the command as root or via `sudo/doas`. + ```sh curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sh ``` +The installer will detect your privileges and guide you through adding the necessary environment variables to your shell profile. + +*Security Note: As with any script piped from the internet, we encourage you to [inspect the installer source](https://github.com/bellhyve/zelta/blob/main/contrib/install-from-git.sh) before execution.* + --- ## Quickstart: Developer Workflow @@ -187,11 +193,11 @@ For other inquiries including business questions, you can reach the Zelta team a ### Conference Talks -**BSDCan 2024: Zelta: A Safe and Powerful Approach to ZFS Replication** -By Daniel J. Bell +**BSDCan 2024: Zelta: A Safe and Powerful Approach to ZFS Replication** +By Daniel J. Bell [Watch on YouTube](https://www.youtube.com/watch?v=_nmgQTs8wgE) -**OpenZFS Summit 2025: Responsible Replication with Zelta** +**OpenZFS Summit 2025: Responsible Replication with Zelta** [Watch on YouTube](https://www.youtube.com/watch?v=G3weooQqcXw) ### Bell Tower Services From 6273f213516382364883fc765688838518d3b655 Mon Sep 17 00:00:00 2001 From: rlogwood Date: Sun, 15 Mar 2026 23:01:11 -0400 Subject: [PATCH 46/47] Add new ShellSpec tests, CI workflow, test generation improvements - Added new ShellSpec tests for policy and prune - Fixed automatic test generation output comparison across different OSes - Regenerated all YAML-defined tests and verified with mixed remote testing on Ubuntu and FreeBSD - Added GitHub Actions workflow (.github/workflows/shellspec.yml) to run ShellSpec tests on push to main/dev/feature branches and PRs to main/dev - Added ShellSpec Tests status badge to README - Added -qq flag to zelta clone and zelta revert to accommodate OS-specific output differences - Enabled combined use of pattern and selector specs for debugging tests (test/runners/test_generation/lib/orchestration/setup_tree) - Centralized environment variable substitution cleanup in test_generator.rb, removed from awk matcher generation - Improved test generator error reporting - Added VM setup helpers to create a local Ubuntu VM that mirrors the GitHub Actions testing environment - Improved test README --- .github/workflows/shellspec.yml | 46 +++++++ .gitignore | 5 +- README.md | 19 ++- contrib/install-from-git.sh | 2 +- test/040_zelta_tests_spec.sh | 26 ++-- test/050_zelta_revert_spec.sh | 61 +++------ test/060_zelta_clone_spec.sh | 30 +---- test/070_zelta_prune_spec.sh | 67 ++++++++++ test/080_zelta_policy_spec.sh | 38 ++++++ test/README.md | 25 +++- test/runners/doc/README_AliasHelpers.md | 4 + test/runners/env/setup_debug_env.sh | 1 + test/runners/env/test_env.sh | 16 ++- test/runners/test_generation/Gemfile | 1 + test/runners/test_generation/Gemfile.lock | 34 +++++ test/runners/test_generation/bin/debug_gen.sh | 49 +++---- .../bin/generate_40_divergent_test.sh | 6 + .../bin/generate_50_revert_test.sh | 6 + .../bin/generate_60_clone_test.sh | 6 + .../bin/generate_70_prune_test.sh | 6 + .../bin/generate_80_policy_test.sh | 6 + .../test_generation/bin/generate_all_tests.sh | 11 ++ .../test_generation/bin/generate_new_tests.sh | 34 ----- .../test_generation/bin/generate_test.sh | 25 ++++ .../bin/generate_zelta_policy_config.sh | 39 ++++++ .../bin/setup_debug_state.bash | 59 +++++++++ .../test_defs/050_zelta_revert_test.yml | 5 +- .../config/test_defs/060_zelta_clone_test.yml | 5 +- .../config/test_defs/070_zelta_prune_test.yml | 15 +++ .../test_defs/080_zelta_policy_test.yml | 17 +++ .../config/zelta_test_policy.conf.example | 12 ++ .../lib/orchestration/generate_test.sh | 2 +- .../lib/orchestration/setup_tree.sh | 80 ++++++++++-- .../lib/ruby/env_substitutor.rb | 40 ++++++ .../test_generation/lib/ruby/path_config.rb | 14 ++ .../lib/ruby/test_generator.rb | 123 ++++++------------ .../scripts/awk/generate_case_stmt_func.awk | 24 ++-- .../scripts/sh/generate_matcher.sh | 1 + test/runners/vm/VM-README.md | 111 ++++++++++++++++ test/runners/vm/vm-setup.sh | 71 ++++++++++ 40 files changed, 857 insertions(+), 285 deletions(-) create mode 100644 .github/workflows/shellspec.yml create mode 100644 test/070_zelta_prune_spec.sh create mode 100644 test/080_zelta_policy_spec.sh create mode 100755 test/runners/test_generation/bin/generate_40_divergent_test.sh create mode 100755 test/runners/test_generation/bin/generate_50_revert_test.sh create mode 100755 test/runners/test_generation/bin/generate_60_clone_test.sh create mode 100755 test/runners/test_generation/bin/generate_70_prune_test.sh create mode 100755 test/runners/test_generation/bin/generate_80_policy_test.sh create mode 100755 test/runners/test_generation/bin/generate_all_tests.sh delete mode 100755 test/runners/test_generation/bin/generate_new_tests.sh create mode 100755 test/runners/test_generation/bin/generate_test.sh create mode 100755 test/runners/test_generation/bin/generate_zelta_policy_config.sh create mode 100644 test/runners/test_generation/bin/setup_debug_state.bash create mode 100644 test/runners/test_generation/config/test_defs/070_zelta_prune_test.yml create mode 100644 test/runners/test_generation/config/test_defs/080_zelta_policy_test.yml create mode 100644 test/runners/test_generation/config/zelta_test_policy.conf.example create mode 100644 test/runners/test_generation/lib/ruby/env_substitutor.rb create mode 100644 test/runners/test_generation/lib/ruby/path_config.rb create mode 100644 test/runners/vm/VM-README.md create mode 100644 test/runners/vm/vm-setup.sh diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml new file mode 100644 index 0000000..0c91e40 --- /dev/null +++ b/.github/workflows/shellspec.yml @@ -0,0 +1,46 @@ +name: ShellSpec Tests + +on: + push: + branches: [main, dev, "feature/**"] + pull_request: + branches: [main, dev] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y zfsutils-linux man-db + + - name: Install ShellSpec + run: curl -fsSL https://git.io/shellspec | sudo sh -s -- --yes --prefix /usr/local + + - name: Create test user + run: | + sudo useradd -m -s /bin/bash testuser + echo 'testuser ALL=(ALL) NOPASSWD: /usr/bin/dd *, /usr/bin/rm -f /tmp/*, /usr/bin/truncate *, /usr/sbin/zpool *, /usr/sbin/zfs *' \ + | sudo tee /etc/sudoers.d/testuser + sudo chown root:root /etc/sudoers.d/testuser + sudo chmod 0440 /etc/sudoers.d/testuser + sudo visudo -cf /etc/sudoers.d/testuser + + - name: Copy repo for test user + run: | + sudo cp -r "$GITHUB_WORKSPACE" /home/testuser/zelta + sudo chown -R testuser:testuser /home/testuser/zelta + + - name: Run ShellSpec tests + run: | + sudo -u testuser env \ + SANDBOX_ZELTA_SRC_POOL=apool \ + SANDBOX_ZELTA_TGT_POOL=bpool \ + SANDBOX_ZELTA_SRC_DS=apool/treetop \ + SANDBOX_ZELTA_TGT_DS=bpool/backups \ + bash -c 'cd /home/testuser/zelta && shellspec' diff --git a/.gitignore b/.gitignore index db73fa3..5aede50 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ +.* !.gitignore !.shellspec +!.github/ +test/runners/test_generation/config/zelta_test_policy.conf *.swp *.swo -.* doc/man? tmp hide.* @@ -14,3 +16,4 @@ A*d retired/ logs log +ci/* \ No newline at end of file diff --git a/README.md b/README.md index efcd9b5..d408d8b 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ ![Zelta Logo](https://zelta.space/index/zelta-banner.svg) - # The Zelta Backup and Recovery Suite *Version 1.1, 2026-01-20* --- - > - **What's New:** Check [CHANGELOG.md](CHANGELOG.md) for the latest changes -> - **Found a Bug?** Please [open an issue](https://github.com/bellhyve/zelta/issues) -> - **Previous Release:** [March 2024, Zelta v1.0](https://github.com/bellhyve/zelta/tree/release/1.0) - +> - **Found a Bug?** Please [open an issue](https://github.com/bell-tower/zelta/issues) +> - **Previous Release:** [March 2024, Zelta v1.0](https://github.com/bell-tower/zelta/tree/release/1.0) +> +>      ![ShellSpec Tests](https://github.com/bell-tower/zelta/actions/workflows/shellspec.yml/badge.svg) --- -[zelta.space](https://zelta.space) | [Documentation](https://zelta.space/en/home) | [GitHub](https://github.com/bellhyve/zelta) +[zelta.space](https://zelta.space) | [Documentation](https://zelta.space/en/home) | [GitHub](https://github.com/bell-tower/zelta) **Zelta** provides bulletproof backups that meet strict compliance requirements while remaining straightforward to deploy and operate. It transforms complex backup and recovery operations into safe, auditable commands—protecting your data without requiring specialized expertise. @@ -41,7 +40,7 @@ Written in portable Bourne shell and AWK, Zelta runs anywhere ZFS runs. No packa ### From Source (Recommended for Zelta 1.1) ```sh -git clone https://github.com/bellhyve/zelta.git +git clone https://github.com/bell-tower/zelta.git cd zelta sudo ./install.sh # The installer will guide you through setup. @@ -59,12 +58,12 @@ pkg install zelta The following command clones Zelta from the `main` branch and launches the installer. Run this as a personal or backup user for a local, non-root installation. If you prefer a system-wide installation, run the command as root or via `sudo/doas`. ```sh -curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sh +curl -fsSL https://raw.githubusercontent.com/bell-tower/zelta/main/contrib/install-from-git.sh | sh ``` The installer will detect your privileges and guide you through adding the necessary environment variables to your shell profile. -*Security Note: As with any script piped from the internet, we encourage you to [inspect the installer source](https://github.com/bellhyve/zelta/blob/main/contrib/install-from-git.sh) before execution.* +*Security Note: As with any script piped from the internet, we encourage you to [inspect the installer source](https://github.com/bell-tower/zelta/blob/main/contrib/install-from-git.sh) before execution.* --- @@ -187,7 +186,7 @@ We welcome contributors who are passionate about data protection and recovery. B ### Contact -We welcome questions, bug reports, and feature requests at [GitHub Issues](https://github.com/bellhyve/zelta/issues). +We welcome questions, bug reports, and feature requests at [GitHub Issues](https://github.com/bell-tower/zelta/issues). For other inquiries including business questions, you can reach the Zelta team at Bell Tower via our [contact form](https://belltower.it/contact/). diff --git a/contrib/install-from-git.sh b/contrib/install-from-git.sh index 1d9d8da..6d3a25d 100644 --- a/contrib/install-from-git.sh +++ b/contrib/install-from-git.sh @@ -2,7 +2,7 @@ # Zelta One-Shot Installer # Downloads latest Zelta from GitHub and runs install.sh # -# Usage: curl -fsSL https://raw.githubusercontent.com/bellhyve/zelta/main/contrib/install-from-git.sh | sh +# Usage: curl -fsSL https://raw.githubusercontent.com/bell-tower/zelta/main/contrib/install-from-git.sh | sh # Or specify branch: curl ... | sh -s -- --branch=develop set -e diff --git a/test/040_zelta_tests_spec.sh b/test/040_zelta_tests_spec.sh index 930ef66..983bd97 100644 --- a/test/040_zelta_tests_spec.sh +++ b/test/040_zelta_tests_spec.sh @@ -1,12 +1,12 @@ # Auto-generated ShellSpec test file -# Generated at: 2026-02-12 13:28:34 -0500 +# Generated at: 2026-03-15 02:59:07 -0400 # Source: 040_zelta_tests_spec # WARNING: This file was automatically generated. Manual edits may be lost. output_for_match_after_divergence() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ "[treetop] @start @start @start up-to-date"|\ @@ -24,7 +24,8 @@ output_for_match_after_divergence() { "11 total datasets compared") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -35,16 +36,17 @@ output_for_match_after_divergence() { output_for_rotate_after_divergence() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "source is written; snapshotting: @zelta_"*""|\ "renaming '${SANDBOX_ZELTA_TGT_DS}' to '${SANDBOX_ZELTA_TGT_DS}_start'"|\ "to ensure target is up-to-date, run: zelta backup ${SANDBOX_ZELTA_SRC_EP} ${SANDBOX_ZELTA_TGT_EP}"|\ "no source: ${SANDBOX_ZELTA_TGT_DS}/sub1/kid"|\ - "* sent, 10 streams received in * seconds") + ""*" sent, 10 streams received in "*" seconds") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -55,7 +57,7 @@ output_for_rotate_after_divergence() { output_for_match_after_rotate() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "DS_SUFFIX MATCH SRC_LAST TGT_LAST INFO"|\ "[treetop] @zelta_"*" @zelta_"*" @zelta_"*" up-to-date"|\ @@ -72,7 +74,8 @@ output_for_match_after_rotate() { "10 total datasets compared") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -83,14 +86,15 @@ output_for_match_after_rotate() { output_for_backup_after_rotate() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "syncing 10 datasets"|\ "10 datasets up-to-date"|\ - "* sent, 3 streams received in * seconds") + ""*" sent, 3 streams received in "*" seconds") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac diff --git a/test/050_zelta_revert_spec.sh b/test/050_zelta_revert_spec.sh index 99d0fac..6d32785 100644 --- a/test/050_zelta_revert_spec.sh +++ b/test/050_zelta_revert_spec.sh @@ -1,17 +1,18 @@ # Auto-generated ShellSpec test file -# Generated at: 2026-02-12 13:29:24 -0500 +# Generated at: 2026-03-15 02:59:54 -0400 # Source: 050_zelta_revert_spec # WARNING: This file was automatically generated. Manual edits may be lost. output_for_snapshot() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "snapshot created '${SANDBOX_ZELTA_SRC_DS}@manual_test'") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -22,14 +23,15 @@ output_for_snapshot() { output_for_backup_after_delta() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "source is written; snapshotting: @zelta_"*""|\ "syncing 12 datasets"|\ - "* sent, 22 streams received in * seconds") + ""*" sent, 22 streams received in "*" seconds") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -40,31 +42,13 @@ output_for_backup_after_delta() { output_for_snapshot_again() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "snapshot created '${SANDBOX_ZELTA_SRC_DS}@another_test'") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} - -output_for_revert() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "renaming '${SANDBOX_ZELTA_SRC_DS}' to '${SANDBOX_ZELTA_SRC_DS}_manual_test'"|\ - "cloned 12/12 datasets to ${SANDBOX_ZELTA_SRC_DS}"|\ - "snapshotting: @zelta_"*""|\ - "to retain replica history, run: zelta rotate '${SANDBOX_ZELTA_SRC_DS}' 'TARGET'") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -75,16 +59,17 @@ output_for_revert() { output_for_rotate_after_revert() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "renaming '${SANDBOX_ZELTA_TGT_DS}' to '${SANDBOX_ZELTA_TGT_DS}_zelta_"*"'"|\ "to ensure target is up-to-date, run: zelta backup ${SANDBOX_ZELTA_SRC_EP} ${SANDBOX_ZELTA_TGT_EP}"|\ "no source: ${SANDBOX_ZELTA_TGT_DS}/sub5"|\ "no source: ${SANDBOX_ZELTA_TGT_DS}/sub5/child1"|\ - "* sent, 10 streams received in * seconds") + ""*" sent, 10 streams received in "*" seconds") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -118,20 +103,8 @@ Describe 'Test revert' The status should be success End - It "revert to last snapshot - zelta revert \"$SANDBOX_ZELTA_SRC_EP\"@manual_test" - When call zelta revert "$SANDBOX_ZELTA_SRC_EP"@manual_test - The output should satisfy output_for_revert - The error should equal "warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: cannot open '${SANDBOX_ZELTA_SRC_DS}_manual_test/sub5@manual_test': dataset does not exist -warning: unexpected 'zfs clone' output: cannot open '${SANDBOX_ZELTA_SRC_DS}_manual_test/sub5/child1@manual_test': dataset does not exist" + It "revert to last snapshot (ignore warnings) - zelta revert -qq \"$SANDBOX_ZELTA_SRC_EP\"@manual_test" + When call zelta revert -qq "$SANDBOX_ZELTA_SRC_EP"@manual_test The status should be success End diff --git a/test/060_zelta_clone_spec.sh b/test/060_zelta_clone_spec.sh index 7b4d46c..fb661b2 100644 --- a/test/060_zelta_clone_spec.sh +++ b/test/060_zelta_clone_spec.sh @@ -1,35 +1,20 @@ # Auto-generated ShellSpec test file -# Generated at: 2026-02-12 13:30:29 -0500 +# Generated at: 2026-03-15 03:00:54 -0400 # Source: 060_zelta_clone_spec # WARNING: This file was automatically generated. Manual edits may be lost. -output_for_clone_sub2() { - while IFS= read -r line; do - # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - case "$normalized" in - "cloned 2/2 datasets to ${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2") - ;; - *) - printf "Unexpected line format: %s\n" "$line" >&2 - return 1 - ;; - esac - done - return 0 -} - output_for_zfs_list_for_clone() { while IFS= read -r line; do # normalize whitespace, remove leading/trailing spaces - normalized=$(echo "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') case "$normalized" in "NAME ORIGIN"|\ "${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2 ${SANDBOX_ZELTA_SRC_DS}/sub2@zelta_"*""|\ "${SANDBOX_ZELTA_SRC_DS}/copy_of_sub2/orphan ${SANDBOX_ZELTA_SRC_DS}/sub2/orphan@zelta_"*"") ;; *) - printf "Unexpected line format: %s\n" "$line" >&2 + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 return 1 ;; esac @@ -40,11 +25,8 @@ output_for_zfs_list_for_clone() { Describe 'Test clone' Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" - It "zelta clone sub2 - zelta clone \"$SANDBOX_ZELTA_SRC_EP/sub2\" \"$SANDBOX_ZELTA_SRC_EP/copy_of_sub2\"" - When call zelta clone "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" - The output should satisfy output_for_clone_sub2 - The error should equal "warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root -warning: unexpected 'zfs clone' output: filesystem successfully created, but it may only be mounted by root" + It "zelta clone sub2 (ignore warnings) - zelta clone -qq \"$SANDBOX_ZELTA_SRC_EP/sub2\" \"$SANDBOX_ZELTA_SRC_EP/copy_of_sub2\"" + When call zelta clone -qq "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" The status should be success End diff --git a/test/070_zelta_prune_spec.sh b/test/070_zelta_prune_spec.sh new file mode 100644 index 0000000..04d9564 --- /dev/null +++ b/test/070_zelta_prune_spec.sh @@ -0,0 +1,67 @@ +# Auto-generated ShellSpec test file +# Generated at: 2026-03-15 03:01:59 -0400 +# Source: 070_zelta_prune_spec +# WARNING: This file was automatically generated. Manual edits may be lost. + +output_for_backup_with_snapshot() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "snapshotting: @zelta_"*""|\ + "syncing 12 datasets"|\ + ""*" sent, 12 streams received in "*" seconds") + ;; + *) + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 + return 1 + ;; + esac + done + return 0 +} + +output_for_prune_check() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "${SANDBOX_ZELTA_SRC_DS}@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub1@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub1/child@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub2@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub2/orphan@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub3@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub3/space name@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub4@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub4/encrypted@zelta_"*""|\ + "${SANDBOX_ZELTA_SRC_DS}/sub4/zvol@zelta_"*"") + ;; + *) + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 + return 1 + ;; + esac + done + return 0 +} + +Describe 'Test prune' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + + It "backup with snapshot - zelta backup --snapshot \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta backup --snapshot "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_backup_with_snapshot + The status should be success + End + + It "only suggest snapshots existing on target - zelta prune --keep-snap-num=0 --keep-snap-days=0 \"$SANDBOX_ZELTA_SRC_EP\" \"$SANDBOX_ZELTA_TGT_EP\"" + When call zelta prune --keep-snap-num=0 --keep-snap-days=0 "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + The output should satisfy output_for_prune_check + The status should be success + End + +End diff --git a/test/080_zelta_policy_spec.sh b/test/080_zelta_policy_spec.sh new file mode 100644 index 0000000..5db0fdc --- /dev/null +++ b/test/080_zelta_policy_spec.sh @@ -0,0 +1,38 @@ +# Auto-generated ShellSpec test file +# Generated at: 2026-03-15 03:03:10 -0400 +# Source: 080_zelta_policy_spec +# WARNING: This file was automatically generated. Manual edits may be lost. + +output_for_policy_check() { + while IFS= read -r line; do + # normalize whitespace, remove leading/trailing spaces + normalized=$(printf '%s' "$line" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') + case "$normalized" in + "[BACKUP_SITE: ${SANDBOX_ZELTA_TGT_EP}] ${SANDBOX_ZELTA_SRC_EP}: 12 datasets up-to-date") + ;; + *) + printf "Unexpected line format : %s\n" "$line" >&2 + printf "Comparing to normalized: %s\n" "$normalized" >&2 + return 1 + ;; + esac + done + return 0 +} + +Describe 'Test zelta policy' + Skip if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + Skip if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + + It "generate zelta policy - ./test/runners/test_generation/bin/generate_zelta_policy_config.sh" + When call ./test/runners/test_generation/bin/generate_zelta_policy_config.sh + The status should be success + End + + It "test zelta policy - zelta policy -C ./test/runners/test_generation/config/zelta_test_policy.conf" + When call zelta policy -C ./test/runners/test_generation/config/zelta_test_policy.conf + The output should satisfy output_for_policy_check + The status should be success + End + +End diff --git a/test/README.md b/test/README.md index de8fca2..96e31b3 100644 --- a/test/README.md +++ b/test/README.md @@ -13,14 +13,25 @@ ### If testing remotely: - Setup your test user on your source and target machines - - update sudoers, for example on Linux + - update sudoers - create /etc/sudoers.d/zelta-tester - ``` - # Allow (mytestuser) to run ZFS commands without password for zelta testing - # NOTE: This is for test environments only - DO NOT use in production - # CAUTION: The wildcards show intent only, with globbing other commands may be allowed as well - (mytestuser) ALL=(ALL) NOPASSWD: /usr/bin/dd *, /usr/bin/rm -f /tmp/*, /usr/bin/truncate *, /usr/sbin/zpool *, /usr/sbin/zfs * - ``` + - add this comment to the file + ``` + # Allow (mytestuser) to run ZFS commands without password for zelta testing + # NOTE: This is for test environments only - DO NOT use in production + # CAUTION: The wildcards show intent only, with globbing other commands may be allowed as well + ``` + + - Ubuntu entry + ``` + (mytestuser) ALL=(ALL) NOPASSWD: /usr/bin/dd *, /usr/bin/rm -f /tmp/*, /usr/bin/truncate *, /usr/sbin/zpool *, /usr/sbin/zfs * + ``` + + - FreeBSD entry + ``` + (mytestuser) ALL=(ALL) NOPASSWD: /bin/dd *, /bin/rm -f /tmp/*, /usr/bin/truncate *, /sbin/zpool *, /sbin/zfs * + ``` + - TODO: confirm if usr/bin/mount *, /usr/bin/mkdir * are needed - setup zfs allow on your source and target machines will be set up automatically for your test pools diff --git a/test/runners/doc/README_AliasHelpers.md b/test/runners/doc/README_AliasHelpers.md index 0421576..5e379fe 100644 --- a/test/runners/doc/README_AliasHelpers.md +++ b/test/runners/doc/README_AliasHelpers.md @@ -48,4 +48,8 @@ alias zrenv="zcd && . $ZELTA_ENV/reset_env.sh" # setup env vars for your test environment # setup pools, datasets and remotes env vars alias ztenv="zcd && . $ZELTA_ENV/test_env.sh" + +# access zelta man pages, relies on prior setting of ZELTA_DOC env var +# which is the location of the ZELTA man pages +alias zman='man -M "$ZELTA_DOC"' ``` diff --git a/test/runners/env/setup_debug_env.sh b/test/runners/env/setup_debug_env.sh index 8293103..a7c5b9a 100644 --- a/test/runners/env/setup_debug_env.sh +++ b/test/runners/env/setup_debug_env.sh @@ -7,6 +7,7 @@ printf "\n*\n* Running in DEBUG MODE, sourcing setup files\n*\n" if . test/runners/env/set_reuse_tmp_env.sh; then . test/runners/env/test_env.sh # set dataset, pools and remote env vars . test/test_helper.sh # make all the helper functions available + env | grep -i sandbox else return 1 fi diff --git a/test/runners/env/test_env.sh b/test/runners/env/test_env.sh index 6fb6d8a..bbed31a 100644 --- a/test/runners/env/test_env.sh +++ b/test/runners/env/test_env.sh @@ -9,7 +9,15 @@ export SANDBOX_ZELTA_SRC_DS=apool/treetop export SANDBOX_ZELTA_TGT_DS=bpool/backups # remotes setup -# * leave these undefined if you're running locally -# * the endpoints are defined automatically and are REMOTE + DS -export SANDBOX_ZELTA_SRC_REMOTE=dever@zfsdev # Ubuntu source -export SANDBOX_ZELTA_TGT_REMOTE=dever@zfsdev # Ubuntu remote + +# always unset the current remotes first, then override as desired below +unset SANDBOX_ZELTA_SRC_REMOTE +unset SANDBOX_ZELTA_TGT_REMOTE + +# * leave these undefined if you're running locally +# * the endpoints are defined automatically and are REMOTE + DS +# Examples: uncomment and customize these if you want to run against remotes. +#export SANDBOX_ZELTA_SRC_REMOTE=user@example-host # e.g. Ubuntu source +#export SANDBOX_ZELTA_TGT_REMOTE=user@example-host # e.g. Ubuntu remote +#export SANDBOX_ZELTA_SRC_REMOTE=user@freebsd-host # e.g. FreeBSD source +#export SANDBOX_ZELTA_TGT_REMOTE=user@freebsd-host # e.g. FreeBSD remote diff --git a/test/runners/test_generation/Gemfile b/test/runners/test_generation/Gemfile index 27aee2b..f7a03d7 100644 --- a/test/runners/test_generation/Gemfile +++ b/test/runners/test_generation/Gemfile @@ -3,3 +3,4 @@ source 'https://rubygems.org' gem 'json-schema', '~> 4.0' +gem 'rubocop', '~> 1.60' diff --git a/test/runners/test_generation/Gemfile.lock b/test/runners/test_generation/Gemfile.lock index 8d8f3f6..5132158 100644 --- a/test/runners/test_generation/Gemfile.lock +++ b/test/runners/test_generation/Gemfile.lock @@ -3,9 +3,42 @@ GEM specs: addressable (2.8.8) public_suffix (>= 2.0.2, < 8.0) + ast (2.4.3) + json (2.19.1) json-schema (4.3.1) addressable (>= 2.8) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + mcp (0.8.0) + json-schema (>= 4.1) + parallel (1.27.0) + parser (3.3.10.2) + ast (~> 2.4.1) + racc + prism (1.9.0) public_suffix (7.0.2) + racc (1.8.1) + rainbow (3.1.1) + regexp_parser (2.11.3) + rubocop (1.85.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + mcp (~> 0.6) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS arm64-darwin-23 @@ -13,6 +46,7 @@ PLATFORMS DEPENDENCIES json-schema (~> 4.0) + rubocop (~> 1.60) BUNDLED WITH 2.6.4 diff --git a/test/runners/test_generation/bin/debug_gen.sh b/test/runners/test_generation/bin/debug_gen.sh index 12cf650..c733cf6 100755 --- a/test/runners/test_generation/bin/debug_gen.sh +++ b/test/runners/test_generation/bin/debug_gen.sh @@ -1,44 +1,29 @@ -# This is a helper for debugging and testing your generated tests -# This can be helpful if you use the generate_new_tests.sh approach -# and your test confirmation fails. -# -# Why? Because you can fine-tune the tree setup and run your test commands -# by hand to determine the cause of problems. Your test yml definition may -# need additional setup or a modified command. So the ability to iteratively -# test out the new spec can help you resolve problems. -# -# SPECS - the shellspec examples you run to setup the tree before running your test -# NEW_SPEC - the generated test you are debugging or verifying - -REPO_ROOT=$(git rev-parse --show-toplevel) - -# standard locations under test -RUNNERS_DIR="$REPO_ROOT/test/runners" -TEST_GEN_DIR="$REPO_ROOT/test/runners/test_generation" - -# tree setup utility -. "$TEST_GEN_DIR/lib/orchestration/setup_tree.sh" +#!/usr/bin/env bash +# Get the directory where this script is located +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# SPECS - the shellspec examples you run to setup the tree before running your test # prepare the zfs tree with the state represented by running the following examples/tests -SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh" +# TODO: change the SPECS variable to include all tests you need to run before the new test +SPECS="test/01*_spec.sh|test/02*_spec.sh|test/03_*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh|test/060_*_spec.sh" + +if ! . "$SCRIPT_DIR/setup_debug_state.bash" "$SPECS"; then + printf "\n ❌ ERROR: debug state setup failed\n" + exit 1 +fi # use the directory where your generated test is created # by default we're using a temp directory off of test/runners/test_generation -NEW_SPEC="$TEST_GEN_DIR/tmp/050_zelta_revert_spec.sh" +# TODO: change the NEWS_SPEC variable to be spec of the new test you debugging +NEW_SPEC="$TEST_GEN_DIR/tmp/070_zelta_prune_spec.sh" echo "confirming new spec: {$NEW_SPEC}" -if setup_tree "$SPECS"; then - printf "\n ✅ initial tree setup succeeded for specs: %s\n" "$SPECS" -else - printf "\n ❌ Goodbye, initial tree setup failed for specs: %s\n" "$SPECS" - exit 1 -fi - -# -. "$RUNNERS_DIR/env/setup_debug_env.sh" # show a detailed trace of the commands you are executing in your new test -TRACE_OPTIONS="--xtrace --shell /opt/homebrew/bin/bash" +# macOS BASH_SH=/opt/homebrew/bin/bash +# Arch BASH_SH=/usr/bin/bash +BASH_SH=/usr/bin/bash +TRACE_OPTIONS="--xtrace --shell $BASH_SH" # if you don't want/need a detailed trace unset the options var #unset TRACE_OPTIONS diff --git a/test/runners/test_generation/bin/generate_40_divergent_test.sh b/test/runners/test_generation/bin/generate_40_divergent_test.sh new file mode 100755 index 0000000..0de67d5 --- /dev/null +++ b/test/runners/test_generation/bin/generate_40_divergent_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +TEST_DEF=040_zelta_tests.yml +SETUP_SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh" + +./generate_test.sh $TEST_DEF "$SETUP_SPECS" \ No newline at end of file diff --git a/test/runners/test_generation/bin/generate_50_revert_test.sh b/test/runners/test_generation/bin/generate_50_revert_test.sh new file mode 100755 index 0000000..a104238 --- /dev/null +++ b/test/runners/test_generation/bin/generate_50_revert_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +TEST_DEF=050_zelta_revert_test.yml +SETUP_SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh" + +./generate_test.sh $TEST_DEF "$SETUP_SPECS" diff --git a/test/runners/test_generation/bin/generate_60_clone_test.sh b/test/runners/test_generation/bin/generate_60_clone_test.sh new file mode 100755 index 0000000..ba30dea --- /dev/null +++ b/test/runners/test_generation/bin/generate_60_clone_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +TEST_DEF=060_zelta_clone_test.yml +SETUP_SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh" + +./generate_test.sh $TEST_DEF "$SETUP_SPECS" diff --git a/test/runners/test_generation/bin/generate_70_prune_test.sh b/test/runners/test_generation/bin/generate_70_prune_test.sh new file mode 100755 index 0000000..ccf4baf --- /dev/null +++ b/test/runners/test_generation/bin/generate_70_prune_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +TEST_DEF=070_zelta_prune_test.yml +SETUP_SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh|test/060_*_spec.sh" + +./generate_test.sh $TEST_DEF "$SETUP_SPECS" diff --git a/test/runners/test_generation/bin/generate_80_policy_test.sh b/test/runners/test_generation/bin/generate_80_policy_test.sh new file mode 100755 index 0000000..725b69a --- /dev/null +++ b/test/runners/test_generation/bin/generate_80_policy_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +TEST_DEF=080_zelta_policy_test.yml +SETUP_SPECS="test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh|test/060_*_spec.sh|test/070_*_spec.sh" + +./generate_test.sh $TEST_DEF "$SETUP_SPECS" diff --git a/test/runners/test_generation/bin/generate_all_tests.sh b/test/runners/test_generation/bin/generate_all_tests.sh new file mode 100755 index 0000000..eb412eb --- /dev/null +++ b/test/runners/test_generation/bin/generate_all_tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Get the directory where this script is located +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +set -e + +"$SCRIPT_DIR/generate_40_divergent_test.sh" +"$SCRIPT_DIR/generate_50_revert_test.sh" +"$SCRIPT_DIR/generate_60_clone_test.sh" +"$SCRIPT_DIR/generate_70_prune_test.sh" +"$SCRIPT_DIR/generate_80_policy_test.sh" diff --git a/test/runners/test_generation/bin/generate_new_tests.sh b/test/runners/test_generation/bin/generate_new_tests.sh deleted file mode 100755 index 31a0efd..0000000 --- a/test/runners/test_generation/bin/generate_new_tests.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -# Get the directory where this script is located -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -TEST_GEN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -CONFIG_DIR="$TEST_GEN_DIR/config/test_defs" -GENERATE_TEST="$TEST_GEN_DIR/lib/orchestration/generate_test.sh" - -# Generate tests for 40,50,60 examples - -if ! "$GENERATE_TEST" \ - "$CONFIG_DIR/040_zelta_tests.yml" \ - "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh"; then - - printf "\n ❌ Failed to generate 040 test\n" - exit 1 -fi - -if ! "$GENERATE_TEST" \ - "$CONFIG_DIR/050_zelta_revert_test.yml" \ - "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh"; then - - printf "\n ❌ Failed to generate 050 test\n" - exit 1 -fi - - -if ! "$GENERATE_TEST" \ - "$CONFIG_DIR/060_zelta_clone_test.yml" \ - "test/01*_spec.sh|test/01*_spec.sh|test/02*_spec.sh|test/040_*_spec.sh|test/050_*_spec.sh"; then - - printf "\n ❌ Failed to generate 060 test\n" - exit 1 -fi diff --git a/test/runners/test_generation/bin/generate_test.sh b/test/runners/test_generation/bin/generate_test.sh new file mode 100755 index 0000000..0d7e9bf --- /dev/null +++ b/test/runners/test_generation/bin/generate_test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Check if arguments were provided +if [ $# -ne 2 ]; then + echo "Invalid number of arguments!" + echo "Usage: $0 (test yml file) (setup specs)" + exit 1 +fi + +TEST_DEF=$1 +SETUP_SPECS=$2 + +# Get the directory where this script is located +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TEST_GEN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG_DIR="$TEST_GEN_DIR/config/test_defs" +GENERATE_TEST="$TEST_GEN_DIR/lib/orchestration/generate_test.sh" + + +printf "\n***\n*** Generating test for %s\n***\n" "$CONFIG_DIR/$TEST_DEF" +if ! "$GENERATE_TEST" "$CONFIG_DIR/$TEST_DEF" "$SETUP_SPECS"; then + printf "\n ❌ Failed to generate test for %s\n" "$TEST_DEF" + exit 1 +fi + diff --git a/test/runners/test_generation/bin/generate_zelta_policy_config.sh b/test/runners/test_generation/bin/generate_zelta_policy_config.sh new file mode 100755 index 0000000..1c60cc8 --- /dev/null +++ b/test/runners/test_generation/bin/generate_zelta_policy_config.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# called by shellspec test for policy, this script +# will generate a basic zelta policy file for the +# current SANDBOX environment variables + +# Get the directory where this script is located +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TEST_GEN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG_DIR="$TEST_GEN_DIR/config" +ZELTA_TEST_POLICY_CONFIG_FILE="$CONFIG_DIR/zelta_test_policy.conf" + + +if [ -n "$SANDBOX_ZELTA_SRC_REMOTE" ]; then + src_host="$SANDBOX_ZELTA_SRC_REMOTE" +else + src_host="localhost" +fi + +tgt_host="" +if [ -n "${SANDBOX_ZELTA_TGT_REMOTE}" ]; then + tgt_host="${SANDBOX_ZELTA_TGT_REMOTE}:" +fi + +# remove any existing policy file +rm -f "$ZELTA_TEST_POLICY_CONFIG_FILE" + +CUR_TIME_STAMP=$(date -u +%Y-%m-%d_%H.%M.%S) +BACKUP_NAME="zelta_policy_backup_${CUR_TIME_STAMP}" + +# generate new policy file +cat < $ZELTA_TEST_POLICY_CONFIG_FILE +# shellspec auto generated test zelta policy file at: ($CUR_TIME_STAMP) +# NOTE: any modification will be lost +BACKUP_SITE: + ${src_host}: + datasets: + - ${SANDBOX_ZELTA_SRC_DS}: ${SANDBOX_ZELTA_TGT_EP} +EOF diff --git a/test/runners/test_generation/bin/setup_debug_state.bash b/test/runners/test_generation/bin/setup_debug_state.bash new file mode 100644 index 0000000..04ed7b9 --- /dev/null +++ b/test/runners/test_generation/bin/setup_debug_state.bash @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# *** only source this script in bash, do not execute it + +# This is a helper for setting up a test environment with the datasets +# as they would exist after running all the specs defined by SPECS +# +# This can be helpful if you use the generate_new_tests.sh approach +# and your test confirmation fails. +# +# Why? Because you can fine-tune the tree setup and run your test commands +# by hand to determine the cause of problems. Your test yml definition may +# need additional setup or a modified command. So the ability to iteratively +# test out the new spec can help you resolve problems. + +REPO_ROOT=$(git rev-parse --show-toplevel) + +# standard locations under test +RUNNERS_DIR="$REPO_ROOT/test/runners" +TEST_GEN_DIR="$REPO_ROOT/test/runners/test_generation" + +# Requires Bash — will produce syntax errors in fish, csh, etc. +# Usage: source setup.bash + +# Guard: reject non-Bash POSIX shells (sh, dash, zsh) +if [ -z "$BASH_VERSION" ] || case "$SHELLOPTS" in *posix*) true;; *) false;; esac; then + echo "Error: This script requires Bash (not sh)." >&2 + return 1 2>/dev/null || exit 1 +fi + +# Guard: reject direct execution +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + echo "Error: This script must be sourced, not executed." >&2 + echo "Usage: source ${BASH_SOURCE[0]}" >&2 + return 1 +fi + +# Check if arguments were provided +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + echo "Invalid number of arguments!" + echo "Usage: $0 'pattern specs list' 'selector specs list" + return 1 +fi + +# tree setup utility +. "$TEST_GEN_DIR/lib/orchestration/setup_tree.sh" + +if setup_tree "$@"; then + printf "\n ✅ initial tree setup succeeded for specs: %s\n" "$@" +else + printf "\n ❌ Goodbye, initial tree setup failed for specs: %s\n" "$@" + return 1 +fi + +# setup the debugging environment +. "$RUNNERS_DIR/env/setup_debug_env.sh" + +# now you can execute commands as if they were in a test +# that runs after all the SPECS hae completed diff --git a/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml b/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml index 5423773..31fb8a1 100644 --- a/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml +++ b/test/runners/test_generation/config/test_defs/050_zelta_revert_test.yml @@ -25,8 +25,9 @@ test_list: when_command: zelta snapshot --snap-name "another_test" "$SANDBOX_ZELTA_SRC_EP" - test_name: revert - it_desc: revert to last snapshot - %{when_command} - when_command: zelta revert "$SANDBOX_ZELTA_SRC_EP"@manual_test + it_desc: revert to last snapshot (ignore warnings) - %{when_command} + allow_no_output: true + when_command: zelta revert -qq "$SANDBOX_ZELTA_SRC_EP"@manual_test - test_name: rotate_after_revert it_desc: rotates after divergence - %{when_command} diff --git a/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml b/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml index d8101ce..fac22d7 100644 --- a/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml +++ b/test/runners/test_generation/config/test_defs/060_zelta_clone_test.yml @@ -6,8 +6,9 @@ skip_if_list: test_list: - test_name: clone_sub2 - it_desc: zelta clone sub2 - %{when_command} - when_command: zelta clone "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" + it_desc: zelta clone sub2 (ignore warnings) - %{when_command} + allow_no_output: true + when_command: zelta clone -qq "$SANDBOX_ZELTA_SRC_EP/sub2" "$SANDBOX_ZELTA_SRC_EP/copy_of_sub2" - test_name: zfs_list_for_clone setup_scripts: diff --git a/test/runners/test_generation/config/test_defs/070_zelta_prune_test.yml b/test/runners/test_generation/config/test_defs/070_zelta_prune_test.yml new file mode 100644 index 0000000..c63a5c1 --- /dev/null +++ b/test/runners/test_generation/config/test_defs/070_zelta_prune_test.yml @@ -0,0 +1,15 @@ +output_dir: tmp +shellspec_name: "070_zelta_prune_spec" +describe_desc: "Test prune" +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + - condition: if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + +test_list: + - test_name: backup_with_snapshot + it_desc: backup with snapshot - %{when_command} + when_command: zelta backup --snapshot "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" + + - test_name: prune_check + it_desc: only suggest snapshots existing on target - %{when_command} + when_command: zelta prune --keep-snap-num=0 --keep-snap-days=0 "$SANDBOX_ZELTA_SRC_EP" "$SANDBOX_ZELTA_TGT_EP" diff --git a/test/runners/test_generation/config/test_defs/080_zelta_policy_test.yml b/test/runners/test_generation/config/test_defs/080_zelta_policy_test.yml new file mode 100644 index 0000000..5ad2eff --- /dev/null +++ b/test/runners/test_generation/config/test_defs/080_zelta_policy_test.yml @@ -0,0 +1,17 @@ +output_dir: tmp +shellspec_name: "080_zelta_policy_spec" +describe_desc: "Test zelta policy" +skip_if_list: + - condition: if 'SANDBOX_ZELTA_SRC_DS undefined' test -z "$SANDBOX_ZELTA_SRC_DS" + - condition: if 'SANDBOX_ZELTA_TGT_DS undefined' test -z "$SANDBOX_ZELTA_TGT_DS" + +test_list: + - test_name: generate_policy_config + it_desc: generate zelta policy - %{when_command} + allow_no_output: true + # generate zelta policy config file from env variables + when_command: ./test/runners/test_generation/bin/generate_zelta_policy_config.sh + + - test_name: policy_check + it_desc: test zelta policy - %{when_command} + when_command: zelta policy -C ./test/runners/test_generation/config/zelta_test_policy.conf diff --git a/test/runners/test_generation/config/zelta_test_policy.conf.example b/test/runners/test_generation/config/zelta_test_policy.conf.example new file mode 100644 index 0000000..bc1b810 --- /dev/null +++ b/test/runners/test_generation/config/zelta_test_policy.conf.example @@ -0,0 +1,12 @@ +# This is an example file to show the format of the automatically generated +# shellspec test policy file. The generated file will be +# written to test/runners/test_generation/config/zelta_test_policy.conf +# and is ignored by git. + +# shellspec auto generated test zelta policy file at: (0000-00-00_00.00.00) +# NOTE: this file is dynamically generated during shellspec runs, do not modify. +# NOTE: shellspec will create a local version of this file from your environment. +BACKUP_SITE: + user@source-host: + datasets: + - srcpool/data: user@backup-host:dstpool/backups diff --git a/test/runners/test_generation/lib/orchestration/generate_test.sh b/test/runners/test_generation/lib/orchestration/generate_test.sh index 36ef186..9c3072a 100755 --- a/test/runners/test_generation/lib/orchestration/generate_test.sh +++ b/test/runners/test_generation/lib/orchestration/generate_test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Check for required arguments if [ $# -lt 2 ]; then diff --git a/test/runners/test_generation/lib/orchestration/setup_tree.sh b/test/runners/test_generation/lib/orchestration/setup_tree.sh index cb1dbce..cbec50f 100644 --- a/test/runners/test_generation/lib/orchestration/setup_tree.sh +++ b/test/runners/test_generation/lib/orchestration/setup_tree.sh @@ -6,19 +6,75 @@ REPO_ROOT=${REPO_ROOT:=$(git rev-parse --show-toplevel)} echo "REPO ROOT: $REPO_ROOT" setup_tree() { - setup_specs=$1 - trace_options=$2 + pattern_specs=$1 + selector_specs=$2 + trace_options=$3 - cd "$REPO_ROOT" || exit 1 - . ./test/test_helper.sh - . ./test/runners/env/helpers.sh - setup_env "1" # setup debug environment - clean_ds_and_pools # reset tree + cd "$REPO_ROOT" || return 1 - if shellspec $trace_options --pattern "$setup_specs"; then - printf "\n ✅ setup succeeded for specs: %s\n" "$setup_specs" - else - printf "\n ❌ setup failed for specs: %s\n" "$setup_specs" - exit 1 + if ! . ./test/test_helper.sh; then + echo "source ./test/test_helper.sh failed" + return 1 fi + + if ! . ./test/runners/env/helpers.sh; then + echo "source ./test/runners/env/helpers.sh failed" + return 1 + fi + + if ! setup_env "1"; then + echo "setup_env failed" + return 1 + fi + + if ! clean_ds_and_pools; then + echo "clean_ds_and_pools failed" + return 1 + fi + + # Split trace_options into array (if not empty) + trace_opts=() + if [ -n "$trace_options" ]; then + read -ra trace_opts <<< "$trace_options" + fi + + # Split selector_specs into array (if not empty) + selector_opts=() + if [ -n "$selector_specs" ]; then + read -ra selector_opts <<< "$selector_specs" + fi + + cmd1=() + if [ -n "$pattern_specs" ]; then + cmd1=(shellspec) + if [ ${#trace_opts[@]} -gt 0 ]; then + cmd1+=("${trace_opts[@]}") + fi + cmd1+=(--pattern "$pattern_specs") + fi + + cmd2=() + if [ ${#selector_opts[@]} -gt 0 ]; then + cmd2=(shellspec) + if [ ${#trace_opts[@]} -gt 0 ]; then + cmd2+=("${trace_opts[@]}") + fi + cmd2+=("${selector_opts[@]}") + fi + + set -x + if [ ${#cmd1[@]} -gt 0 ] && ! "${cmd1[@]}"; then + printf "\n ❌ setup failed for command: %s\n" "${cmd1[*]}" + set +x + return 1 + fi + + if [ ${#cmd2[@]} -gt 0 ] && ! "${cmd2[@]}"; then + printf "\n ❌ setup failed for command: %s\n" "${cmd2[*]}" + set +x + return 1 + fi + set +x + + printf "\n ✅ setup succeeded\n" } diff --git a/test/runners/test_generation/lib/ruby/env_substitutor.rb b/test/runners/test_generation/lib/ruby/env_substitutor.rb new file mode 100644 index 0000000..986a8e9 --- /dev/null +++ b/test/runners/test_generation/lib/ruby/env_substitutor.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# EnvSubstitutor - Handles environment variable substitution in test output +class EnvSubstitutor + attr_reader :sorted_env_map + + def initialize(env_var_names) + @sorted_env_map = build_sorted_env_map(env_var_names) + end + + def substitute(line) + # Substitute env var names for values (longest first) + # Use a placeholder to prevent already-substituted values from being re-matched + replaced = line + placeholder_map = {} + @sorted_env_map.each_with_index do |(name, value), idx| + placeholder = "__ENV_PLACEHOLDER_#{idx}__" + replaced.gsub!(value, placeholder) + placeholder_map[placeholder] = "${#{name}}" + end + + # Replace placeholders with actual env var references + placeholder_map.each do |placeholder, replacement| + replaced.gsub!(placeholder, replacement) + end + replaced + end + + private + + def build_sorted_env_map(env_var_names) + # Parse and sort env vars by value length (descending) + env_map = env_var_names.split(':').each_with_object({}) do |name, hash| + hash[name] = ENV[name] if ENV[name]&.length&.positive? + end + + # Sort by value length descending to replace longest matches first + env_map.sort_by { |_name, value| -value.length } + end +end diff --git a/test/runners/test_generation/lib/ruby/path_config.rb b/test/runners/test_generation/lib/ruby/path_config.rb new file mode 100644 index 0000000..e47ab74 --- /dev/null +++ b/test/runners/test_generation/lib/ruby/path_config.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# PathConfig - Manages file paths for test generation +class PathConfig + attr_reader :output_dir, :wip_file_path, :final_file_path + + def initialize(output_dir, shellspec_name, test_gen_dir) + @output_dir = output_dir.start_with?('/') ? output_dir : File.join(test_gen_dir, output_dir) + @wip_file_path = File.join(@output_dir, "#{shellspec_name}_wip.sh") + # remove _spec to prevent shellspec from finding the WIP file + @wip_file_path.sub!('_spec', '') + @final_file_path = File.join(@output_dir, "#{shellspec_name}.sh") + end +end diff --git a/test/runners/test_generation/lib/ruby/test_generator.rb b/test/runners/test_generation/lib/ruby/test_generator.rb index 1acf825..f43dfc6 100755 --- a/test/runners/test_generation/lib/ruby/test_generator.rb +++ b/test/runners/test_generation/lib/ruby/test_generator.rb @@ -10,6 +10,8 @@ require 'time' require_relative 'placeholders' require_relative 'sys_exec' +require_relative 'env_substitutor' +require_relative 'path_config' # TestGenerator - Generates ShellSpec test files from YAML configuration class TestGenerator @@ -22,8 +24,8 @@ class TestGenerator private_constant :REPO_ROOT, :TEST_GEN_DIR, :GENERATE_MATCHER_SH_SCRIPT - attr_reader :config, :output_dir, :shellspec_name, :describe_desc, :test_list, :skip_if_list, - :matcher_files, :wip_file_path, :final_file_path, :env_var_names, :sorted_env_map + attr_reader :config, :shellspec_name, :describe_desc, :test_list, :skip_if_list, + :matcher_files, :paths def initialize(yaml_file_path, env_var_names = DEFAULT_ENV_VAR_NAMES) # Resolve path relative to this file's directory if it's a relative path @@ -36,20 +38,14 @@ def initialize(yaml_file_path, env_var_names = DEFAULT_ENV_VAR_NAMES) @shellspec_name = @config['shellspec_name'] @describe_desc = @config['describe_desc'] - - # Resolve output_dir relative to test_generation directory - output_dir = @config['output_dir'] - @output_dir = output_dir.start_with?('/') ? output_dir : File.join(TEST_GEN_DIR, output_dir) - @test_list = @config['test_list'] || [] @skip_if_list = @config['skip_if_list'] || [] @matcher_files = [] - @wip_file_path = File.join(@output_dir, "#{@shellspec_name}_wip.sh") - # remove _spec to prevent shellspec from finding the WIP file - @wip_file_path.sub!('_spec', '') - @final_file_path = File.join(@output_dir, "#{@shellspec_name}.sh") - @env_var_names = env_var_names - @sorted_env_map = build_sorted_env_map + + # Initialize helper objects + @paths = PathConfig.new(@config['output_dir'], @shellspec_name, TEST_GEN_DIR) + @env_substitutor = EnvSubstitutor.new(env_var_names) + puts "Loading configuration from: #{@config.inspect}\n" puts '=' * 60 end @@ -64,16 +60,6 @@ def generate private - def build_sorted_env_map - # Parse and sort env vars by value length (descending) - env_map = @env_var_names.split(':').each_with_object({}) do |name, hash| - hash[name] = ENV[name] if ENV[name] - end - - # Sort by value length descending to replace longest matches first - env_map.sort_by { |_name, value| -value.length } - end - def matcher_func_name(test_name) "output_for_#{test_name}" end @@ -84,12 +70,12 @@ def validate_config!(schema_path = File.join(TEST_GEN_DIR, 'config', 'test_confi end def create_output_directory - FileUtils.mkdir_p(@output_dir) - puts "Created output directory: #{@output_dir}" + FileUtils.mkdir_p(@paths.output_dir) + puts "Created output directory: #{@paths.output_dir}" end def create_wip_file - File.open(@wip_file_path, 'w') do |file| + File.open(@paths.wip_file_path, 'w') do |file| file.puts "Describe '#{@describe_desc}'" # Add Skip If statements for each condition @@ -98,7 +84,7 @@ def create_wip_file end file.puts '' unless @skip_if_list.empty? end - puts "Created WIP file: #{@wip_file_path}" + puts "Created WIP file: #{@paths.wip_file_path}" end def process_tests @@ -121,7 +107,7 @@ def process_tests end # Close Describe block - File.open(@wip_file_path, 'a') do |file| + File.open(@paths.wip_file_path, 'a') do |file| file.puts 'End' end end @@ -141,13 +127,13 @@ def generate_matcher_files(test_name, when_command, setup_scripts, allow_no_outp # Add allow_no_output flag allow_no_output_flag = allow_no_output ? "true" : "false" - cmd = "#{matcher_script} \"#{full_command}\" #{matcher_function_name} #{@output_dir} #{allow_no_output_flag}" + cmd = "#{matcher_script} \"#{full_command}\" #{matcher_function_name} #{@paths.output_dir} #{allow_no_output_flag}" SysExec.run(cmd, timeout: 10) unless allow_no_output # Track the generated matcher file func_name = matcher_func_name(test_name) - matcher_file = File.join(@output_dir, func_name, "#{func_name}.sh") + matcher_file = File.join(@paths.output_dir, func_name, "#{func_name}.sh") # Post-process the matcher file to apply env substitutions if File.exist?(matcher_file) @@ -206,67 +192,39 @@ def build_command_with_setup(when_command, setup_scripts) end def append_it_clause(test_name, it_desc, when_command, allow_no_output) - File.open(@wip_file_path, 'a') do |file| + File.open(@paths.wip_file_path, 'a') do |file| file.puts " It \"#{it_desc.gsub('"', '\\"')}\"" func_name = matcher_func_name(test_name) - # TODO: clean up all the trial and error with shellspec error output, documented approaches don't work! # Check for stderr output - stderr_file = File.join(@output_dir, func_name, "#{func_name}_stderr.out") + stderr_file = File.join(@paths.output_dir, func_name, "#{func_name}_stderr.out") expected_error = nil if File.exist?(stderr_file) && !File.zero?(stderr_file) expected_error = format_expected_error(stderr_file) - #file.puts expected_error - #status_line = ' The status should be failure' - else - #status_line = ' The status should equal 0' end - # TODO: zelta exits with 0 even when there is error output - #status_line = ' The status should equal 0' + # TODO: double check if zelta exits with 0 even when there is error output status_line = ' The status should be success' file.puts " When call #{when_command}" - file.puts " The output should satisfy #{matcher_func_name(test_name)}" unless allow_no_output + # NOTE: this style of checking error output was the only one that worked for me, inline equal file.puts " The error should equal \"#{expected_error}\"\n" if expected_error file.puts status_line - file.puts ' End' file.puts '' end end - def v1_format_expected_error(stderr_file) - lines = read_stderr_file(stderr_file) - result = " expected_error=%text\n" - lines.each do |line| - result += " #|#{line}\n" - end - "#{result} End\n" - end - # expected_error() { %text - # #|warning: insufficient snapshots; performing full backup for 3 datasets - # #|warning: missing `zfs allow` permissions: readonly,mountpoint - # } - def v2_format_expected_error(stderr_file) - lines = read_stderr_file(stderr_file) - result = " expected_error() { %text\n" - lines.each do |line| - result += " #|#{line}\n" - end - "#{result} }\n" - end - def format_expected_error(stderr_file) lines = read_stderr_file(stderr_file) lines.map! { |line| normalize_output_line(line) } lines.join("\n") end - def normalize_output_line(line) + def clean_up_output_line(line) # Normalize whitespace normalized = line.gsub(/\s+/, ' ').strip @@ -274,6 +232,9 @@ def normalize_output_line(line) normalized.gsub!(/@zelta_\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}/, '@zelta_"*"') normalized.gsub!(/_zelta_\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}/, '_zelta_"*"') + # replace timestamp for generated zelta policy files is YYYY-MM-DD_HH.MM.SS with * + normalized.gsub!(/_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, '_"*"') + # Escape backticks normalized.gsub!('`', '\\\`') @@ -281,26 +242,17 @@ def normalize_output_line(line) if normalized =~ /(\d+[KMGT]? sent, )(\d+ streams)( received in \d+\.\d+ seconds)/ stream_count = $2 normalized.gsub!(/\d+[KMGT]? sent, \d+ streams received in \d+\.\d+ seconds/, - "* sent, #{stream_count} received in * seconds") - end - - # Substitute env var names for values (longest first) - # Use a placeholder to prevent already-substituted values from being re-matched - placeholder_map = {} - @sorted_env_map.each_with_index do |(name, value), idx| - placeholder = "__ENV_PLACEHOLDER_#{idx}__" - normalized.gsub!(value, placeholder) - placeholder_map[placeholder] = "${#{name}}" + "\"*\" sent, #{stream_count} received in \"*\" seconds") end - - # Replace placeholders with actual env var references - placeholder_map.each do |placeholder, replacement| - normalized.gsub!(placeholder, replacement) - end - normalized end + def normalize_output_line(line) + @env_substitutor.substitute( + clean_up_output_line(line) + ) + end + def read_stderr_file(stderr_file) File.readlines(stderr_file).map(&:chomp) rescue StandardError => e @@ -309,7 +261,7 @@ def read_stderr_file(stderr_file) end def assemble_final_file - File.open(@final_file_path, 'w') do |final| + File.open(@paths.final_file_path, 'w') do |final| final.puts '# Auto-generated ShellSpec test file' final.puts "# Generated at: #{Time.now}" final.puts "# Source: #{@shellspec_name}" @@ -325,9 +277,9 @@ def assemble_final_file end # Copy the WIP file content - final.puts File.read(@wip_file_path) if File.exist?(@wip_file_path) + final.puts File.read(@paths.wip_file_path) if File.exist?(@paths.wip_file_path) end - puts "Assembled final test file: #{@final_file_path}" + puts "Assembled final test file: #{@paths.final_file_path}" end def report_summary @@ -337,16 +289,16 @@ def report_summary puts "YAML Configuration: #{@config.inspect}" puts "ShellSpec Name: #{@shellspec_name}" puts "Description: #{@describe_desc}" - puts "Output Directory: #{@output_dir}" + puts "Output Directory: #{@paths.output_dir}" puts "Tests Processed: #{@test_list.length}" puts "Matcher Files Generated: #{@matcher_files.length}" puts "\nGenerated Files:" - puts " - WIP File: #{@wip_file_path}" + puts " - WIP File: #{@paths.wip_file_path}" @matcher_files.each do |file| puts " - Matcher: #{file}" end puts "\nFinal ShellSpec Test File:" - puts " Location: #{@final_file_path}" + puts " Location: #{@paths.final_file_path}" puts '=' * 60 puts "__SHELLSPEC_NAME__:#{@shellspec_name}" end @@ -380,3 +332,4 @@ def run_generator # Script execution run_generator if __FILE__ == $PROGRAM_NAME + diff --git a/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk b/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk index ba51731..2791479 100644 --- a/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk +++ b/test/runners/test_generation/scripts/awk/generate_case_stmt_func.awk @@ -8,20 +8,8 @@ # Process data lines { + # normalize whitespace to a single space gsub(/[[:space:]]+/, " ", $0) - gsub(/@zelta_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, "@zelta_\"*\"",$0) - gsub(/_zelta_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\.[0-9]{2}\.[0-9]{2}/, "_zelta_\"*\"",$0) - gsub(/`/, "\\`", $0) - - # wildcard time and quantity sent - if (match($0, /[0-9]+[KMGT]? sent, [0-9]+ streams/)) { - # Extract the part with streams - streams_part = substr($0, RSTART, RLENGTH) - # Extract just the number before " streams" - match(streams_part, /[0-9]+ streams/) - stream_count = substr(streams_part, RSTART, RLENGTH) - gsub(/[0-9]+[KMGT]? sent, [0-9]+ streams received in [0-9]+\.[0-9]+ seconds/, "* sent, " stream_count " received in * seconds", $0) - } # remove trailing spaces sub(/[[:space:]]+$/, "", $0) @@ -33,7 +21,12 @@ END { print func_name "() {" print " while IFS= read -r line; do" print " # normalize whitespace, remove leading/trailing spaces" - print " normalized=$(echo \"$line\" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" + + # Use printf with %s format specifier to avoid \$ escapes + dollar = "$" + printf " normalized=%s(printf '%%s' \"%sline\" | tr -s '[:space:]' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*%s//')\n", dollar, dollar, dollar + + print " # check line against expected output" print " case \"$normalized\" in" line_continue = "\"|\\" @@ -46,7 +39,8 @@ END { print " ;;" print " *)" - print " printf \"Unexpected line format: %s\\n\" \"$line\" >&2" + print " printf \"Unexpected line format : %s\\n\" \"$line\" >&2" + print " printf \"Comparing to normalized: %s\\n\" \"$normalized\" >&2" print " return 1" print " ;;" print " esac" diff --git a/test/runners/test_generation/scripts/sh/generate_matcher.sh b/test/runners/test_generation/scripts/sh/generate_matcher.sh index 0aecbc6..4aea15f 100755 --- a/test/runners/test_generation/scripts/sh/generate_matcher.sh +++ b/test/runners/test_generation/scripts/sh/generate_matcher.sh @@ -55,6 +55,7 @@ fi if [ ! -s "$OUT_FL" ]; then if [ "$allow_no_output" != "true" ]; then printf "\n ❌ Error: zelta produced no output\n" + printf " đŸ› ī¸âš™ī¸âžĄī¸ If that is expected, 💡 add option 'allow_no_output: true' to your test\n" printf "****-> review and update zelta cmd: \"%s\"\n" "$zelta_cmd" exit 1 else diff --git a/test/runners/vm/VM-README.md b/test/runners/vm/VM-README.md new file mode 100644 index 0000000..bee7584 --- /dev/null +++ b/test/runners/vm/VM-README.md @@ -0,0 +1,111 @@ +# Local CI VM for ShellSpec Testing + +A lightweight Ubuntu 24.04 VM for running ShellSpec tests locally, matching the GitHub Actions `ubuntu-latest` environment. + +## Prerequisites + +- QEMU/KVM with libvirt (`virt-install`, `virsh`) +- A working bridge network (e.g., `br0`) or the default NAT network + +## Creating the VM + +Download the Ubuntu Server 24.04 cloud image: + +```bash +cd /var/lib/libvirt/images +sudo wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img +``` + +Create the VM: + +```bash +sudo virt-install \ + --name zelta-ci \ + --ram 2048 \ + --vcpus 2 \ + --disk path=/var/lib/libvirt/images/zelta-ci.qcow2,backing_store=/var/lib/libvirt/images/noble-server-cloudimg-amd64.img,size=10 \ + --os-variant ubuntu24.04 \ + --network bridge=br0 \ + --cloud-init root-password-generate=on \ + --noautoconsole +``` + +Save the generated root password from the output. + +## Initial Setup + +Connect to the VM console to get networking up: + +```bash +sudo virsh console zelta-ci # Ctrl+] to disconnect +``` + +### Configure VM with setup script from GitHub +From the VM console as root, pull and run the setup script: +```bash +curl -fsSL https://raw.githubusercontent.com/bell-tower/zelta/main/test/runners/vm/vm-setup.sh | bash +``` + +### configure VM via scp +Find the VM's IP: + +```bash +sudo virsh domifaddr zelta-ci +``` + +SSH in and run the setup script: + +```bash +ssh root@ +# Copy vm-setup.sh to the VM, then: +bash vm-setup.sh +``` + +Copy the zelta repo to the VM: + +```bash +scp -r /path/to/zelta root@:/home/testuser/zelta +# Or from inside the VM: +su - testuser -c 'git clone ~/zelta' +``` + +Make sure ownership is correct: + +```bash +chown -R testuser:testuser /home/testuser/zelta +``` + +## Running Tests + +```bash +su - testuser -c ~testuser/run_test.sh +``` + +## VM Management + +Snapshot after setup (while running): + +```bash +sudo virsh snapshot-create-as zelta-ci clean-baseline --description "Fresh setup with ShellSpec and ZFS" +``` + +Revert to snapshot: + +```bash +sudo virsh snapshot-revert zelta-ci clean-baseline +``` + +Start / stop / check status: + +```bash +sudo virsh start zelta-ci +sudo virsh shutdown zelta-ci +sudo virsh dominfo zelta-ci +``` + +Delete the VM entirely: + +```bash +sudo virsh destroy zelta-ci +sudo virsh undefine zelta-ci --remove-all-storage +``` diff --git a/test/runners/vm/vm-setup.sh b/test/runners/vm/vm-setup.sh new file mode 100644 index 0000000..e70c4a2 --- /dev/null +++ b/test/runners/vm/vm-setup.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Local CI VM setup script for zelta ShellSpec tests +# Run as root on a fresh Ubuntu 24.04 VM + +set -euo pipefail + +echo "==> Installing system packages" +apt-get update && apt-get install -y \ + zfsutils-linux \ + sudo \ + curl \ + git \ + man-db \ + openssh-server + +echo "==> Enabling SSH" +sed -i 's/^#*PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config +sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config +# Remove cloud-init drop-in that may override password auth +rm -f /etc/ssh/sshd_config.d/60-cloudimg-settings.conf +systemctl enable --now ssh + +echo "==> Installing ShellSpec" +if command -v shellspec &>/dev/null; then + echo " ShellSpec already installed, skipping" +else + curl -fsSL https://git.io/shellspec | sh -s -- --yes --prefix /usr/local +fi + +echo "==> Creating test user" +if id testuser &>/dev/null; then + echo " testuser already exists, skipping" +else + useradd -m -s /bin/bash testuser +fi + +echo "==> Configuring sudoers" +echo 'testuser ALL=(ALL) NOPASSWD: /usr/bin/dd *, /usr/bin/rm -f /tmp/*, /usr/bin/truncate *, /usr/sbin/zpool *, /usr/sbin/zfs *' \ + > /etc/sudoers.d/testuser + +chown root:root /etc/sudoers.d/testuser +chmod 0440 /etc/sudoers.d/testuser +visudo -cf /etc/sudoers.d/testuser + +echo "==> Creating test runner script" +cat > /home/testuser/run_test.sh << 'EOF' +#!/bin/bash +cd ~/zelta +export SANDBOX_ZELTA_SRC_POOL=apool +export SANDBOX_ZELTA_TGT_POOL=bpool +export SANDBOX_ZELTA_SRC_DS=apool/treetop +export SANDBOX_ZELTA_TGT_DS=bpool/backups +shellspec +EOF + +chmod +x /home/testuser/run_test.sh +chown testuser:testuser /home/testuser/run_test.sh +echo "created test runner: /home/testuser/run_test.sh" + +git clone https://github.com/bell-tower/zelta.git /home/testuser/zelta +chown -R testuser:testuser /home/testuser/zelta +echo "cloned zelta to: /home/testuser/zelta" + +echo "" +echo "==> Setup complete!" +echo "" +echo "To run tests:" +echo " su - testuser" +echo " cd zelta" +echo " git checkout (branchname)" +echo " ~/run_test.sh" From d75a6f9af01b5db0b017e7bb0577dd90b849ac57 Mon Sep 17 00:00:00 2001 From: "Daniel J. Bell" Date: Mon, 16 Mar 2026 11:51:42 -0400 Subject: [PATCH 47/47] Multiple docs updated: fixed `--quiet` description across commands --- doc/zelta-backup.md | 2 +- doc/zelta-clone.md | 2 +- doc/zelta-match.md | 2 +- doc/zelta-prune.md | 2 +- doc/zelta-revert.md | 2 +- doc/zelta-rotate.md | 2 +- doc/zelta-snapshot.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/zelta-backup.md b/doc/zelta-backup.md index 86dc06c..754cfd6 100644 --- a/doc/zelta-backup.md +++ b/doc/zelta-backup.md @@ -73,7 +73,7 @@ _target_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-j, \--json** : Output results in JSON format. See **zelta-options(7)** for details. diff --git a/doc/zelta-clone.md b/doc/zelta-clone.md index 85c6bc6..3663495 100644 --- a/doc/zelta-clone.md +++ b/doc/zelta-clone.md @@ -36,7 +36,7 @@ _target_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them. diff --git a/doc/zelta-match.md b/doc/zelta-match.md index 77766d6..6542bd5 100644 --- a/doc/zelta-match.md +++ b/doc/zelta-match.md @@ -18,7 +18,7 @@ : Increase verbosity. Specify once for operational detail and twice (-vv) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings and twice (-qq) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (-qq) to suppress warnings. **\--log-level** : Specify a log level value 0-4: errors (0), warnings (1), notices (2, default), info (3, verbose), and debug (4). diff --git a/doc/zelta-prune.md b/doc/zelta-prune.md index d85baf1..9898b6a 100644 --- a/doc/zelta-prune.md +++ b/doc/zelta-prune.md @@ -46,7 +46,7 @@ _target_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them. diff --git a/doc/zelta-revert.md b/doc/zelta-revert.md index 00a614f..0743184 100644 --- a/doc/zelta-revert.md +++ b/doc/zelta-revert.md @@ -45,7 +45,7 @@ _endpoint_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them. diff --git a/doc/zelta-rotate.md b/doc/zelta-rotate.md index e27607b..fb0b828 100644 --- a/doc/zelta-rotate.md +++ b/doc/zelta-rotate.md @@ -56,7 +56,7 @@ _target_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them. diff --git a/doc/zelta-snapshot.md b/doc/zelta-snapshot.md index 1d6b24f..1537ee0 100644 --- a/doc/zelta-snapshot.md +++ b/doc/zelta-snapshot.md @@ -36,7 +36,7 @@ _endpoint_ : Increase verbosity. Specify once for operational detail, twice (`-vv`) for debug output. **-q, \--quiet** -: Quiet output. Specify once to suppress warnings, twice (`-qq`) to suppress errors. +: Decrease log level. Specify once to suppress notices, twice (`-qq`) to suppress warnings. **-n, \--dryrun, \--dry-run** : Display `zfs` commands without executing them.