diff --git a/docs/config/config-dotfiles.md b/docs/config/config-dotfiles.md index a8dec7c5..1db9e450 100644 --- a/docs/config/config-dotfiles.md +++ b/docs/config/config-dotfiles.md @@ -10,6 +10,7 @@ Entry | Description `actions` | List of action keys that need to be defined in the **actions** entry below (See [actions](config-actions.md)) `chmod` | Defines the file permissions in octal notation to apply during installation or the special keyword `preserve` (See [permissions](config-file.md#permissions)) `cmpignore` | List of patterns to ignore when comparing (enclose in quotes when using wildcards; see [ignore patterns](config-file.md#ignore-patterns)) +`dir_as_block` | List of patterns (globs/regex) to match directories that should be handled as a single block during install operations (see [ignore patterns](config-file.md#ignore-patterns)). `ignore_missing_in_dotdrop` | Ignore missing files in dotdrop when comparing and importing (see [Ignore missing](config-file.md#ignore-missing)) `ignoreempty` | If true, an empty template will not be deployed (defaults to the value of `ignoreempty`) `instignore` | List of patterns to ignore when installing (enclose in quotes when using wildcards; see [ignore patterns](config-file.md#ignore-patterns)) @@ -216,4 +217,26 @@ profiles: - f_test ``` -Make sure to quote the link value in the config file. \ No newline at end of file +Make sure to quote the link value in the config file. + +## Handle directories as blocks + +When managing dotfiles that are directories, dotdrop normally processes each file and subdirectory individually. This allows for precise control over the contents, showing individual file differences, and selectively updating files. +However, in some cases, you may prefer to treat an entire directory as a single unit. + +For these scenarios, you can use the `dir_as_block` option on specific dotfiles: + +```yaml +dotfiles: + d_config: + src: app + dst: ~/.config/app + dir_as_block: + - "*app" + - "*otherdir*" +``` + +Note: +- During **install** operations, any directory matching a pattern in `dir_as_block` will be replaced as a whole +- This option has no effect on **compare** operations, which will always show file-by-file differences +- This option has no effect on dotfiles that are regular files \ No newline at end of file diff --git a/dotdrop/dotdrop.py b/dotdrop/dotdrop.py index 98c3d009..037f64b1 100644 --- a/dotdrop/dotdrop.py +++ b/dotdrop/dotdrop.py @@ -232,12 +232,17 @@ def _dotfile_install(opts, dotfile, tmpdir=None): LinkTypes.RELATIVE, LinkTypes.ABSOLUTE ): # nolink|relative|absolute|link_children - ret, err = inst.install(templ, dotfile.src, dotfile.dst, - dotfile.link, - actionexec=pre_actions_exec, - is_template=is_template, - ignore=ignores, - chmod=dotfile.chmod) + ret, err = inst.install( + templ, + dotfile.src, + dotfile.dst, + dotfile.link, + actionexec=pre_actions_exec, + is_template=is_template, + ignore=ignores, + chmod=dotfile.chmod, + dir_as_block=dotfile.dir_as_block, + ) else: # nolink src = dotfile.src @@ -250,13 +255,18 @@ def _dotfile_install(opts, dotfile, tmpdir=None): src = tmp # make sure to re-evaluate if is template is_template = dotfile.template and Templategen.path_is_template(src) - ret, err = inst.install(templ, src, dotfile.dst, - LinkTypes.NOLINK, - actionexec=pre_actions_exec, - noempty=dotfile.noempty, - ignore=ignores, - is_template=is_template, - chmod=dotfile.chmod) + ret, err = inst.install( + templ, + src, + dotfile.dst, + LinkTypes.NOLINK, + actionexec=pre_actions_exec, + noempty=dotfile.noempty, + ignore=ignores, + is_template=is_template, + chmod=dotfile.chmod, + dir_as_block=dotfile.dir_as_block, + ) if tmp: tmp = os.path.join(opts.dotpath, tmp) if os.path.exists(tmp): diff --git a/dotdrop/dotfile.py b/dotdrop/dotfile.py index e5868091..9838a80e 100644 --- a/dotdrop/dotfile.py +++ b/dotdrop/dotfile.py @@ -17,13 +17,14 @@ class Dotfile(DictParser): key_trans_install = 'trans_install' key_trans_update = 'trans_update' key_template = 'template' + key_dir_as_block = 'dir_as_block' def __init__(self, key, dst, src, actions=None, trans_install=None, trans_update=None, link=LinkTypes.NOLINK, noempty=False, cmpignore=None, upignore=None, instignore=None, template=True, chmod=None, - ignore_missing_in_dotdrop=False): + ignore_missing_in_dotdrop=False, dir_as_block=None): """ constructor @key: dotfile key @@ -39,6 +40,7 @@ def __init__(self, key, dst, src, @instignore: patterns to ignore when installing @template: template this dotfile @chmod: file permission + @dir_as_block: handle directory matching patterns as a single block """ self.actions = actions or [] self.dst = dst @@ -54,6 +56,7 @@ def __init__(self, key, dst, src, self.template = template self.chmod = chmod self.ignore_missing_in_dotdrop = ignore_missing_in_dotdrop + self.dir_as_block = dir_as_block or [] if self.link != LinkTypes.NOLINK and \ ( @@ -96,6 +99,8 @@ def _adjust_yaml_keys(cls, value): """patch dict""" value['noempty'] = value.get(cls.key_noempty, False) value['template'] = value.get(cls.key_template, True) + value['dir_as_block'] = value.get( + cls.key_dir_as_block, []) # remove old entries value.pop(cls.key_noempty, None) return value @@ -121,6 +126,8 @@ def __str__(self): msg += f', chmod:{self.chmod:o}' else: msg += f', chmod:\"{self.chmod}\"' + if self.dir_as_block: + msg += f', dir_as_block:{self.dir_as_block}' return msg def prt(self): @@ -136,6 +143,9 @@ def prt(self): out += f'\n{indent}chmod: \"{self.chmod:o}\"' else: out += f'\n{indent}chmod: \"{self.chmod}\"' + if self.dir_as_block: + out += (f'\n{indent}dir_as_block: ' + f'"{self.dir_as_block}"') out += f'\n{indent}pre-action:' some = self.get_pre_actions() diff --git a/dotdrop/ftree.py b/dotdrop/ftree.py index 4d6c888b..31d7c126 100644 --- a/dotdrop/ftree.py +++ b/dotdrop/ftree.py @@ -18,7 +18,9 @@ class FTreeDir: directory tree for comparison """ - def __init__(self, path, ignores=None, debug=False): + def __init__(self, path, + ignores=None, + debug=False): self.path = path self.ignores = ignores self.debug = debug diff --git a/dotdrop/installer.py b/dotdrop/installer.py index d3233bf0..2c127c99 100644 --- a/dotdrop/installer.py +++ b/dotdrop/installer.py @@ -5,9 +5,12 @@ handle the installation of dotfiles """ +# pylint: disable=C0302 + import os import errno import shutil +import fnmatch # local imports from dotdrop.logger import Logger @@ -79,7 +82,7 @@ def __init__(self, base='.', create=True, backup=True, def install(self, templater, src, dst, linktype, actionexec=None, noempty=False, ignore=None, is_template=True, - chmod=None): + chmod=None, dir_as_block=None): """ install src to dst @@ -92,12 +95,14 @@ def install(self, templater, src, dst, linktype, @ignore: pattern to ignore when installing @is_template: this dotfile is a template @chmod: rights to apply if any + @dir_as_block: handle directories matching pattern as a single block return - True, None : success - False, error_msg : error - False, None : ignored """ + dir_as_block = dir_as_block or [] if not src or not dst: # fake dotfile self.log.dbg('fake dotfile installed') @@ -131,6 +136,32 @@ def install(self, templater, src, dst, linktype, isdir = os.path.isdir(src) self.log.dbg(f'install {src} to {dst}') self.log.dbg(f'\"{src}\" is a directory: {isdir}') + self.log.dbg(f'dir_as_block: {dir_as_block}') + + treat_as_block = any( + fnmatch.fnmatch(src, pattern) + for pattern in dir_as_block + ) + self.log.dbg( + f'dir_as_block patterns: {dir_as_block}, ' + f'treat_as_block: {treat_as_block}' + ) + if treat_as_block: + self.log.dbg( + f'handling directory {src} ' + 'as a block for installation' + ) + ret, err, ins = self._copy_dir( + templater, src, dst, + actionexec=actionexec, + noempty=noempty, ignore=ignore, + is_template=is_template, + chmod=chmod, + dir_as_block=True + ) + if self.remove_existing_in_dir and ins: + self._remove_existing_in_dir(dst, ins) + return self._log_install(ret, err) if linktype == LinkTypes.NOLINK: # normal file @@ -139,7 +170,8 @@ def install(self, templater, src, dst, linktype, actionexec=actionexec, noempty=noempty, ignore=ignore, is_template=is_template, - chmod=chmod) + chmod=chmod, + dir_as_block=dir_as_block) if self.remove_existing_in_dir and ins: self._remove_existing_in_dir(dst, ins) else: @@ -602,7 +634,7 @@ def _copy_file(self, templater, src, dst, def _copy_dir(self, templater, src, dst, actionexec=None, noempty=False, ignore=None, is_template=True, - chmod=None): + chmod=None, dir_as_block=False): """ install src to dst when is a directory @@ -617,6 +649,69 @@ def _copy_dir(self, templater, src, dst, fails """ self.log.dbg(f'deploy dir {src}') + self.log.dbg(f'dir_as_block: {dir_as_block}') + + # Handle directory as a block if option is enabled + if dir_as_block: + self.log.dbg( + f'handling directory {src} as a block for installation') + dst_dotfiles = [] + + # Ask user for confirmation if safe mode is on + if os.path.exists(dst): + msg = f'Overwrite entire directory "{dst}" with "{src}"?' + if self.safe and not self.log.ask(msg): + return False, 'aborted', [] + + # Remove existing directory completely + if self.dry: + self.log.dry(f'would rm -r {dst}') + else: + self.log.dbg(f'rm -r {dst}') + if not removepath(dst, logger=self.log): + msg = f'unable to remove {dst}, do manually' + self.log.warn(msg) + return False, msg, [] + + # Create parent directory if needed + parent_dir = os.path.dirname(dst) + if not os.path.exists(parent_dir): + if self.dry: + self.log.dry(f'would mkdir -p {parent_dir}') + else: + if not self._create_dirs(parent_dir): + err = f'error creating directory for {dst}' + return False, err, [] + + # Copy directory recursively + if self.dry: + self.log.dry(f'would cp -r {src} {dst}') + return True, None, [dst] + try: + # Execute pre actions + ret, err = self._exec_pre_actions(actionexec) + if not ret: + return False, err, [] + + # Copy the directory as a whole + shutil.copytree(src, dst) + + # Record all files that were installed + for root, _, files in os.walk(dst): + for file in files: + path = os.path.join(root, file) + dst_dotfiles.append(path) + + if not self.comparing: + self.log.sub( + f'installed directory {src} to {dst} as a block') + return True, None, dst_dotfiles + except (shutil.Error, OSError) as exc: + err = f'{src} installation failed: {exc}' + self.log.warn(err) + return False, err, [] + + # Regular directory installation (file by file) # default to nothing installed and no error ret = False dst_dotfiles = [] @@ -644,7 +739,6 @@ def _copy_dir(self, templater, src, dst, if res: # something got installed - ret = True else: # is directory @@ -655,7 +749,8 @@ def _copy_dir(self, templater, src, dst, actionexec=actionexec, noempty=noempty, ignore=ignore, - is_template=is_template) + is_template=is_template, + dir_as_block=dir_as_block) dst_dotfiles.extend(subs) if not res and err: # error occured @@ -804,7 +899,6 @@ def _write(self, src, dst, content=None, ######################################################## # helpers ######################################################## - @classmethod def _get_tmp_file_vars(cls, src, dst): tmp = {} diff --git a/dotdrop/updater.py b/dotdrop/updater.py index e482237c..e23ee128 100644 --- a/dotdrop/updater.py +++ b/dotdrop/updater.py @@ -39,6 +39,7 @@ def __init__(self, dotpath, variables, conf, @debug: enable debug @ignore: pattern to ignore when updating @showpatch: show patch if dotfile to update is a template + @ignore_missing_in_dotdrop: ignore missing files in dotdrop """ self.dotpath = dotpath self.variables = variables diff --git a/scripts/check-syntax.sh b/scripts/check-syntax.sh index e97be17c..b8b6cc79 100755 --- a/scripts/check-syntax.sh +++ b/scripts/check-syntax.sh @@ -117,7 +117,12 @@ done # check other python scripts echo "-----------------------------------------" echo "checking other python scripts with pylint" -find . -name "*.py" -not -path "./dotdrop/*" -not -regex "\./\.?v?env/.*" | while read -r script; do +find . -name "*.py" \ + -not -path "./dotdrop/*" \ + -not -path "./build/*" \ + -not -path "./dist/*" \ + -not -regex "\./\.?v?env/.*" \ + | while read -r script; do echo "checking ${script}" pylint -sn \ --disable=W0012 \ diff --git a/scripts/check_links.py b/scripts/check_links.py index 1278e6b1..42f7d868 100755 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -28,11 +28,13 @@ VALID_RET = [ 200, 302, + 429, ] IGNORES = [ 'badgen.net', 'coveralls.io', 'packages.ubuntu.com', + 'www.gnu.org', ] OK_WHEN_FORBIDDEN = [ 'linux.die.net', diff --git a/tests-ng/install-dir-as-block.sh b/tests-ng/install-dir-as-block.sh new file mode 100755 index 00000000..21e3ace0 --- /dev/null +++ b/tests-ng/install-dir-as-block.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2025, deadc0de6 +# +# test dir_as block +# + +## start-cookie +set -eu -o errtrace -o pipefail +cur=$(cd "$(dirname "${0}")" && pwd) +ddpath="${cur}/../" +PPATH="{PYTHONPATH:-}" +export PYTHONPATH="${ddpath}:${PPATH}" +altbin="python3 -m dotdrop.dotdrop" +if hash coverage 2>/dev/null; then + mkdir -p coverages/ + altbin="coverage run -p --data-file coverages/coverage --source=dotdrop -m dotdrop.dotdrop" +fi +bin="${DT_BIN:-${altbin}}" +# shellcheck source=tests-ng/helpers +source "${cur}"/helpers +echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" +## end-cookie + +################################################################ +# this is the test +################################################################ + +# Setup temp dirs +tmpd=$(mktemp -d --suffix='-dotdrop-tests' || mktemp -d) +dotpath="${tmpd}/dotfiles" +mkdir -p "${dotpath}" +instroot="${tmpd}/install" +mkdir -p "${instroot}" + +clear_on_exit "${tmpd}" + +# Create source directories and files +mkdir -p "${dotpath}/blockme1/subdir" +echo "file1" > "${dotpath}/blockme1/file1.txt" +echo "file2" > "${dotpath}/blockme1/subdir/file2.txt" + +mkdir -p "${dotpath}/blockme2" +echo "file3" > "${dotpath}/blockme2/file3.txt" + +mkdir -p "${dotpath}/noblock" +echo "file4" > "${dotpath}/noblock/file4.txt" + +# Add a subdirectory that matches the dir_as_block pattern +mkdir -p "${dotpath}/blockme1/matchsub" +echo "subfile1" > "${dotpath}/blockme1/matchsub/subfile1.txt" + +# Add a subdirectory that does NOT match the pattern +mkdir -p "${dotpath}/blockme1/nomatchsub" +echo "subfile2" > "${dotpath}/blockme1/nomatchsub/subfile2.txt" + +# Create config file with multiple dir_as_block patterns +cfg="${tmpd}/config.yaml" +cat > "${cfg}" << _EOF +config: + backup: false + create: true + dotpath: dotfiles +dotfiles: + d_blockme1: + src: blockme1 + dst: ${instroot}/blockme1 + dir_as_block: + - "*blockme1" + - "*blockme2" + d_blockme2: + src: blockme2 + dst: ${instroot}/blockme2 + dir_as_block: + - "*blockme1" + - "*blockme2" + d_noblock: + src: noblock + dst: ${instroot}/noblock + dir_as_block: + - "*blockme1" + - "*blockme2" +profiles: + p1: + dotfiles: + - d_blockme1 + - d_blockme2 + - d_noblock +_EOF + +# Install +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 + +# Check that blockme1 and blockme2 were installed as a block (directory replaced as a whole) +# Remove a file from blockme1, reinstall, and check it is restored (block behavior) +rm -f "${instroot}/blockme1/file1.txt" +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 +[ -f "${instroot}/blockme1/file1.txt" ] || (echo "blockme1 not restored" && exit 1) + +# Remove a file from noblock, reinstall, and check it is restored +rm -f "${instroot}/noblock/file4.txt" +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 +[ -f "${instroot}/noblock/file4.txt" ] || (echo "noblock not restored" && exit 1) + +# Check that blockme2 was installed as a block +rm -f "${instroot}/blockme2/file3.txt" +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 +[ -f "${instroot}/blockme2/file3.txt" ] || (echo "blockme2 not restored as block" && exit 1) + +# Check that subdir and its file are present in blockme1 +[ -d "${instroot}/blockme1/subdir" ] || (echo "blockme1/subdir missing" && exit 1) +[ -f "${instroot}/blockme1/subdir/file2.txt" ] || (echo "blockme1/subdir/file2.txt missing" && exit 1) + +# Reinstall to ensure both subdirs are installed +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 + +# Remove a file from the matching subdir, reinstall, and check it is restored (block behavior) +rm -f "${instroot}/blockme1/matchsub/subfile1.txt" +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 +[ -f "${instroot}/blockme1/matchsub/subfile1.txt" ] || (echo "blockme1/matchsub not restored as block" && exit 1) + +# Remove a file from the non-matching subdir, reinstall, and check it is NOT restored (not a block) +rm -f "${instroot}/blockme1/nomatchsub/subfile2.txt" +cd "${ddpath}" | ${bin} install -f -c "${cfg}" --verbose -p p1 +[ ! -f "${instroot}/blockme1/nomatchsub/subfile2.txt" ] || (echo "blockme1/nomatchsub should not be restored as block" && exit 1) + +# Check confirmation prompts for block and non-block directories +# Remove -f to enable confirmation prompts + +# Function to count confirmation prompts in output +count_prompts() { + grep -c "Do you want to continue" "$1" +} + +# Test: blockme1 (should prompt ONCE for the whole dir) +rm -rf "${instroot}/blockme1" +# Use script to simulate 'y' input and capture output +script -q -c "cd \"${ddpath}\" && ${bin} install -c \"${cfg}\" --verbose -p p1" blockme1.out <<<'y' +blockme1_prompts=$(count_prompts blockme1.out) +if [ "$blockme1_prompts" -ne 1 ]; then + echo "blockme1: expected 1 prompt, got $blockme1_prompts" && exit 1 +fi + +# Test: noblock (should prompt for each file) +rm -rf "${instroot}/noblock" +script -q -c "cd \"${ddpath}\" && ${bin} install -c \"${cfg}\" --verbose -p p1" noblock.out <<<'y +y' +noblock_prompts=$(count_prompts noblock.out) +# There is only one file, so expect 1 prompt +if [ "$noblock_prompts" -ne 1 ]; then + echo "noblock: expected 1 prompt, got $noblock_prompts" && exit 1 +fi + +# Test: blockme1 with multiple files (should still prompt ONCE) +rm -rf "${instroot}/blockme1" +# Add another file to blockme1 +echo "fileX" > "${dotpath}/blockme1/fileX.txt" +script -q -c "cd \"${ddpath}\" && ${bin} install -c \"${cfg}\" --verbose -p p1" blockme1_multi.out <<<'y' +blockme1_multi_prompts=$(count_prompts blockme1_multi.out) +if [ "$blockme1_multi_prompts" -ne 1 ]; then + echo "blockme1 (multi): expected 1 prompt, got $blockme1_multi_prompts" && exit 1 +fi + +# Test: blockme1/nomatchsub (should prompt for each file in non-block subdir) +rm -rf "${instroot}/blockme1/nomatchsub" +script -q -c "cd \"${ddpath}\" && ${bin} install -c \"${cfg}\" --verbose -p p1" nomatchsub.out <<<'y' +nomatchsub_prompts=$(count_prompts nomatchsub.out) +# Only one file, so expect 1 prompt +if [ "$nomatchsub_prompts" -ne 1 ]; then + echo "nomatchsub: expected 1 prompt, got $nomatchsub_prompts" && exit 1 +fi + +# Test: blockme1/matchsub (should prompt ONCE for the whole subdir) +rm -rf "${instroot}/blockme1/matchsub" +script -q -c "cd \"${ddpath}\" && ${bin} install -c \"${cfg}\" --verbose -p p1" matchsub.out <<<'y' +matchsub_prompts=$(count_prompts matchsub.out) +if [ "$matchsub_prompts" -ne 1 ]; then + echo "matchsub: expected 1 prompt, got $matchsub_prompts" && exit 1 +fi + +# Check confirmation prompt count for block directory +rm -f "${instroot}/blockme1/file1.txt" +# Run install interactively and capture output +prompt_output_block=$(cd "${ddpath}" && echo y | ${bin} install -c "${cfg}" --verbose -p p1 2>&1) +# Count confirmation prompts (look for 'Overwrite' or 'replace' or similar) +prompt_count_block=$(echo "$prompt_output_block" | grep -E -i 'overwrite|replace|confirm' | wc -l) +if [ "$prompt_count_block" -ne 1 ]; then + echo "Expected 1 confirmation prompt for block directory, got $prompt_count_block" && exit 1 +fi + +# Check confirmation prompt count for non-block directory +rm -f "${instroot}/noblock/file4.txt" +# Run install interactively and capture output +prompt_output_noblock=$(cd "${ddpath}" && echo y | ${bin} install -c "${cfg}" --verbose -p p1 2>&1) +# Count confirmation prompts (should be at least 1 for the file, could be more if more files) +prompt_count_noblock=$(echo "$prompt_output_noblock" | grep -E -i 'overwrite|replace|confirm' | wc -l) +if [ "$prompt_count_noblock" -lt 1 ]; then + echo "Expected at least 1 confirmation prompt for non-block directory, got $prompt_count_noblock" && exit 1 +fi + +echo "OK" +exit 0