From f31694e4a1946067ba2a9b4ca07d131ebdac7cf2 Mon Sep 17 00:00:00 2001 From: ngharo Date: Thu, 27 Feb 2020 23:06:51 -0600 Subject: [PATCH 1/5] Laying out snapshot pruning functionality --- README.md | 12 ++++ app/prune.go | 158 +++++++++++++++++++++++++++++++++++++++++++ cmd/prune.go | 31 +++++++++ config/job.go | 33 ++++++++- go.mod | 2 + testdata/globals.yml | 6 ++ 6 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 app/prune.go create mode 100644 cmd/prune.go diff --git a/README.md b/README.md index 3c16184..97c7877 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,13 @@ rsync: #override_global_excluded: true #override_global_args: true +# FIXME needs more details ( +retention: + daily: int # number of daily backups to keep + weekly: int # number of weekly backups to keep + monthly: int # number of monthly backups to keep + yearly: int # number of yearly backups to keep + # Inline scripts executed on the remote host before and after rsyncing, # and before any `pre.*.sh` and/or `post.*.sh` scripts for this host. pre_script: string @@ -222,6 +229,11 @@ rsync: - "--hard-links" - "--block-size=2048" - "--recursive" +retention: + daily: 14 + weekly: 4 + monthly: 6 + yearly: 5 ``` # Copyright diff --git a/app/prune.go b/app/prune.go new file mode 100644 index 0000000..ee15896 --- /dev/null +++ b/app/prune.go @@ -0,0 +1,158 @@ +package app + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/digineo/zackup/config" + "github.com/sirupsen/logrus" +) + +var ( + patterns = map[string]string{ + "daily": "2006-01-02", + "weekly": "", // See special case in keepers() + "monthly": "2006-01", + "yearly": "2006", + } +) + +type snapshot struct { + Ds string // Snapshot dataset name "backups/foo@RFC3339" + Time time.Time // Parsed timestamp from the dataset name +} + +func PruneSnapshots(job *config.JobConfig) { + var host = job.Host() + + // Set defaults if config is not set + if job.Retention == nil { + job.Retention = &config.RetentionConfig{ + Daily: 100000, + Weekly: 100000, + Monthly: 100000, + Yearly: 100000, + } + } + + // This catches any gaps in the config + if job.Retention.Daily == 0 { + job.Retention.Daily = 100000 + } + if job.Retention.Weekly == 0 { + job.Retention.Weekly = 100000 + } + if job.Retention.Monthly == 0 { + job.Retention.Monthly = 100000 + } + if job.Retention.Yearly == 0 { + job.Retention.Yearly = 100000 + } + + // FIXME probably should iterate over a list instead here + for _, snapshot := range listKeepers(host, "daily", job.Retention.Daily) { + log.WithFields(logrus.Fields{ + "snapshot": snapshot, + "period": "daily", + }).Debug("keeping snapshot") + } + for _, snapshot := range listKeepers(host, "weekly", job.Retention.Weekly) { + log.WithFields(logrus.Fields{ + "snapshot": snapshot, + "period": "weekly", + }).Debug("keeping snapshot") + } + for _, snapshot := range listKeepers(host, "monthly", job.Retention.Monthly) { + log.WithFields(logrus.Fields{ + "snapshot": snapshot, + "period": "monthly", + }).Debug("keeping snapshot") + } + for _, snapshot := range listKeepers(host, "yearly", job.Retention.Yearly) { + log.WithFields(logrus.Fields{ + "snapshot": snapshot, + "period": "yearly", + }).Debug("keeping snapshot") + } + + // TODO subtract keepers from the list of snapshots and rm -rf them +} + +// listKeepers returns a list of snapshot that are not subject to deletion +// for a given host, pattern, and keep_count. +func listKeepers(host string, pattern string, keep_count uint) []snapshot { + var keepers []snapshot + var last string + + for _, snapshot := range listSnapshots(host) { + var period string + + // Weekly is special because golang doesn't have support for "week number in year" + // in Time.Format strings. + if pattern == "weekly" { + year, week := snapshot.Time.Local().ISOWeek() + period = fmt.Sprintf("%d-%d", year, week) + } else { + period = snapshot.Time.Local().Format(patterns[pattern]) + } + + if period != last { + last = period + keepers = append(keepers, snapshot) + + if uint(len(keepers)) == keep_count { + break + } + } + } + + return keepers +} + +// listSnapshots calls out to ZFS for a list of snapshots for a given host. +// Returned data will be sorted by time, most recent first. +func listSnapshots(host string) []snapshot { + var snapshots []snapshot + + ds := newDataset(host) + + args := []string{ + "list", + "-H", // no field headers in output + "-o", "name", // only name field + "-t", "snapshot", // type snapshot + ds.Name, + } + o, e, err := execProgram("zfs", args...) + if err != nil { + f := appendStdlogs(logrus.Fields{ + logrus.ErrorKey: err, + "prefix": "zfs", + "command": append([]string{"zfs"}, args...), + }, o, e) + log.WithFields(f).Errorf("executing zfs list failed") + } + + for _, ss := range strings.Fields(o.String()) { + ts, err := time.Parse(time.RFC3339, strings.Split(ss, "@")[1]) + + if err != nil { + log.WithField("snapshot", ss).Error("Unable to parse timestamp from snapshot") + continue + } + + snapshots = append(snapshots, snapshot{ + Ds: ss, + Time: ts, + }) + } + + // ZFS list _should_ be in chronological order but just in case ... + sort.Slice(snapshots, func(i, j int) bool { + return snapshots[i].Time.After(snapshots[j].Time) + }) + + return snapshots +} diff --git a/cmd/prune.go b/cmd/prune.go new file mode 100644 index 0000000..11cdf6c --- /dev/null +++ b/cmd/prune.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "github.com/digineo/zackup/app" + "github.com/spf13/cobra" +) + +// pruneCmd represents the prune command +var pruneCmd = &cobra.Command{ + Use: "prune [host [...]]", + Short: "Prunes backups per-host ZFS dataset", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + args = tree.Hosts() + } + + for _, host := range args { + job := tree.Host(host) + if job == nil { + log.WithField("prune", host).Warn("unknown host, ignoring") + continue + } + + app.PruneSnapshots(job) + } + }, +} + +func init() { + rootCmd.AddCommand(pruneCmd) +} diff --git a/config/job.go b/config/job.go index 219d20d..32aea12 100644 --- a/config/job.go +++ b/config/job.go @@ -4,8 +4,9 @@ package config type JobConfig struct { host string - SSH *SSHConfig `yaml:"ssh"` - RSync *RsyncConfig `yaml:"rsync"` + SSH *SSHConfig `yaml:"ssh"` + RSync *RsyncConfig `yaml:"rsync"` + Retention *RetentionConfig `yaml:"retention"` PreScript Script `yaml:"pre_script"` // from yaml file PostScript Script `yaml:"post_script"` // from yaml file @@ -18,6 +19,14 @@ type SSHConfig struct { Timeout *uint `yaml:"timeout"` // number of seconds, defaults to 15 } +// RetentionConfig holds backup retention periods +type RetentionConfig struct { + Daily uint `yaml:"daily"` // defaults to 1000000 + Weekly uint `yaml:"weekly"` // defaults to 1000000 + Monthly uint `yaml:"monthly"` // defaults to 1000000 + Yearly uint `yaml:"yearly"` // defaults to 1000000 +} + // Host returns the hostname for this job. func (j *JobConfig) Host() string { return j.host @@ -59,6 +68,26 @@ func (j *JobConfig) mergeGlobals(globals *JobConfig) { } } + if globals.Retention != nil { + if j.Retention == nil { + dup := *globals.Retention + j.Retention = &dup + } else { + if j.Retention.Daily == 0 { + j.Retention.Daily = globals.Retention.Daily + } + if j.Retention.Weekly == 0 { + j.Retention.Weekly = globals.Retention.Weekly + } + if j.Retention.Monthly == 0 { + j.Retention.Monthly = globals.Retention.Monthly + } + if j.Retention.Yearly == 0 { + j.Retention.Yearly = globals.Retention.Yearly + } + } + } + // globals.PreScript j.PreScript.inline = append(globals.PreScript.inline, j.PreScript.inline...) j.PreScript.scripts = append(globals.PreScript.scripts, j.PreScript.scripts...) diff --git a/go.mod b/go.mod index 19d91bf..5843b26 100644 --- a/go.mod +++ b/go.mod @@ -35,3 +35,5 @@ require ( gopkg.in/gemnasium/logrus-graylog-hook.v2 v2.0.7 gopkg.in/yaml.v2 v2.2.2 ) + +go 1.13 diff --git a/testdata/globals.yml b/testdata/globals.yml index 317c712..114aa19 100644 --- a/testdata/globals.yml +++ b/testdata/globals.yml @@ -4,6 +4,12 @@ ssh: port: 22 identity_file: /etc/zackup/id_rsa.pub +retention: + daily: 14 + weekly: 4 + monthly: 6 + yearly: 5 + rsync: included: - /etc From 040e49b5f9752036c68f174b170ab991e284a4fa Mon Sep 17 00:00:00 2001 From: ngharo Date: Thu, 27 Feb 2020 23:49:12 -0600 Subject: [PATCH 2/5] Loop over each bucket when finding keepers --- app/prune.go | 50 +++++++++++++++++++++----------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/app/prune.go b/app/prune.go index ee15896..d161d5c 100644 --- a/app/prune.go +++ b/app/prune.go @@ -24,10 +24,12 @@ type snapshot struct { Time time.Time // Parsed timestamp from the dataset name } +// FIXME PruneSnapshots does not actually perform any destructive operations +// on your datasets at this time. func PruneSnapshots(job *config.JobConfig) { var host = job.Host() - // Set defaults if config is not set + // Defaults: if config is not set if job.Retention == nil { job.Retention = &config.RetentionConfig{ Daily: 100000, @@ -37,7 +39,7 @@ func PruneSnapshots(job *config.JobConfig) { } } - // This catches any gaps in the config + // Defaults: catch any gaps in the config if job.Retention.Daily == 0 { job.Retention.Daily = 100000 } @@ -51,30 +53,20 @@ func PruneSnapshots(job *config.JobConfig) { job.Retention.Yearly = 100000 } - // FIXME probably should iterate over a list instead here - for _, snapshot := range listKeepers(host, "daily", job.Retention.Daily) { - log.WithFields(logrus.Fields{ - "snapshot": snapshot, - "period": "daily", - }).Debug("keeping snapshot") + var keep_counts = map[string]uint{ + "daily": job.Retention.Daily, + "weekly": job.Retention.Weekly, + "monthly": job.Retention.Monthly, + "yearly": job.Retention.Yearly, } - for _, snapshot := range listKeepers(host, "weekly", job.Retention.Weekly) { - log.WithFields(logrus.Fields{ - "snapshot": snapshot, - "period": "weekly", - }).Debug("keeping snapshot") - } - for _, snapshot := range listKeepers(host, "monthly", job.Retention.Monthly) { - log.WithFields(logrus.Fields{ - "snapshot": snapshot, - "period": "monthly", - }).Debug("keeping snapshot") - } - for _, snapshot := range listKeepers(host, "yearly", job.Retention.Yearly) { - log.WithFields(logrus.Fields{ - "snapshot": snapshot, - "period": "yearly", - }).Debug("keeping snapshot") + + for bucket, keep_count := range keep_counts { + for _, snapshot := range listKeepers(host, bucket, keep_count) { + log.WithFields(logrus.Fields{ + "snapshot": snapshot, + "bucket": bucket, + }).Debug("keeping snapshot") + } } // TODO subtract keepers from the list of snapshots and rm -rf them @@ -82,7 +74,7 @@ func PruneSnapshots(job *config.JobConfig) { // listKeepers returns a list of snapshot that are not subject to deletion // for a given host, pattern, and keep_count. -func listKeepers(host string, pattern string, keep_count uint) []snapshot { +func listKeepers(host string, bucket string, keep_count uint) []snapshot { var keepers []snapshot var last string @@ -90,12 +82,12 @@ func listKeepers(host string, pattern string, keep_count uint) []snapshot { var period string // Weekly is special because golang doesn't have support for "week number in year" - // in Time.Format strings. - if pattern == "weekly" { + // as Time.Format string pattern. + if bucket == "weekly" { year, week := snapshot.Time.Local().ISOWeek() period = fmt.Sprintf("%d-%d", year, week) } else { - period = snapshot.Time.Local().Format(patterns[pattern]) + period = snapshot.Time.Local().Format(patterns[bucket]) } if period != last { From 49cbad046bef4a337a8d587d7c4e2bc8e0bc945c Mon Sep 17 00:00:00 2001 From: ngharo Date: Sun, 1 Mar 2020 11:09:40 -0600 Subject: [PATCH 3/5] Add -r to zfs list command --- app/prune.go | 1 + 1 file changed, 1 insertion(+) diff --git a/app/prune.go b/app/prune.go index d161d5c..2c199b2 100644 --- a/app/prune.go +++ b/app/prune.go @@ -112,6 +112,7 @@ func listSnapshots(host string) []snapshot { args := []string{ "list", + "-r", // recursive "-H", // no field headers in output "-o", "name", // only name field "-t", "snapshot", // type snapshot From c5d8f7dab647f2f179b1acddcf9705eac6fe89c0 Mon Sep 17 00:00:00 2001 From: ngharo Date: Sun, 1 Mar 2020 12:13:15 -0600 Subject: [PATCH 4/5] Optimize call to zfs list and make nil retention policy be infinite --- app/prune.go | 55 ++++++++++++++++++++++++++------------------------- config/job.go | 16 +++++++-------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/app/prune.go b/app/prune.go index 2c199b2..b9b8cb9 100644 --- a/app/prune.go +++ b/app/prune.go @@ -32,40 +32,36 @@ func PruneSnapshots(job *config.JobConfig) { // Defaults: if config is not set if job.Retention == nil { job.Retention = &config.RetentionConfig{ - Daily: 100000, - Weekly: 100000, - Monthly: 100000, - Yearly: 100000, + Daily: nil, + Weekly: nil, + Monthly: nil, + Yearly: nil, } } - // Defaults: catch any gaps in the config - if job.Retention.Daily == 0 { - job.Retention.Daily = 100000 - } - if job.Retention.Weekly == 0 { - job.Retention.Weekly = 100000 - } - if job.Retention.Monthly == 0 { - job.Retention.Monthly = 100000 - } - if job.Retention.Yearly == 0 { - job.Retention.Yearly = 100000 - } - - var keep_counts = map[string]uint{ + var policies = map[string]*int{ "daily": job.Retention.Daily, "weekly": job.Retention.Weekly, "monthly": job.Retention.Monthly, "yearly": job.Retention.Yearly, } - for bucket, keep_count := range keep_counts { - for _, snapshot := range listKeepers(host, bucket, keep_count) { - log.WithFields(logrus.Fields{ + snapshots := listSnapshots(host) + + for bucket, retention := range policies { + for _, snapshot := range listKeepers(snapshots, bucket, retention) { + l := log.WithFields(logrus.Fields{ "snapshot": snapshot, "bucket": bucket, - }).Debug("keeping snapshot") + }) + + if retention == nil { + l = l.WithField("retention", "infinite") + } else { + l = l.WithField("retention", *retention) + } + + l.Debug("keeping snapshot") } } @@ -73,12 +69,12 @@ func PruneSnapshots(job *config.JobConfig) { } // listKeepers returns a list of snapshot that are not subject to deletion -// for a given host, pattern, and keep_count. -func listKeepers(host string, bucket string, keep_count uint) []snapshot { +// for a given host, pattern, and retention. +func listKeepers(snapshots []snapshot, bucket string, retention *int) []snapshot { var keepers []snapshot var last string - for _, snapshot := range listSnapshots(host) { + for _, snapshot := range snapshots { var period string // Weekly is special because golang doesn't have support for "week number in year" @@ -94,7 +90,12 @@ func listKeepers(host string, bucket string, keep_count uint) []snapshot { last = period keepers = append(keepers, snapshot) - if uint(len(keepers)) == keep_count { + // nil will keep infinite snapshots + if retention == nil { + continue + } + + if len(keepers) == *retention { break } } diff --git a/config/job.go b/config/job.go index 32aea12..70ff42d 100644 --- a/config/job.go +++ b/config/job.go @@ -21,10 +21,10 @@ type SSHConfig struct { // RetentionConfig holds backup retention periods type RetentionConfig struct { - Daily uint `yaml:"daily"` // defaults to 1000000 - Weekly uint `yaml:"weekly"` // defaults to 1000000 - Monthly uint `yaml:"monthly"` // defaults to 1000000 - Yearly uint `yaml:"yearly"` // defaults to 1000000 + Daily *int `yaml:"daily"` // defaults to infinite + Weekly *int `yaml:"weekly"` // defaults to infinite + Monthly *int `yaml:"monthly"` // defaults to infinite + Yearly *int `yaml:"yearly"` // defaults to infinite } // Host returns the hostname for this job. @@ -73,16 +73,16 @@ func (j *JobConfig) mergeGlobals(globals *JobConfig) { dup := *globals.Retention j.Retention = &dup } else { - if j.Retention.Daily == 0 { + if j.Retention.Daily == nil { j.Retention.Daily = globals.Retention.Daily } - if j.Retention.Weekly == 0 { + if j.Retention.Weekly == nil { j.Retention.Weekly = globals.Retention.Weekly } - if j.Retention.Monthly == 0 { + if j.Retention.Monthly == nil { j.Retention.Monthly = globals.Retention.Monthly } - if j.Retention.Yearly == 0 { + if j.Retention.Yearly == nil { j.Retention.Yearly = globals.Retention.Yearly } } From ea5b6a4e5f5b118ebf1a3b082f3aafc73b7ad39c Mon Sep 17 00:00:00 2001 From: ngharo Date: Sun, 1 Mar 2020 12:18:28 -0600 Subject: [PATCH 5/5] Rename snapshot.Ds to snapshot.Name for clarity --- app/prune.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/prune.go b/app/prune.go index b9b8cb9..47eacf3 100644 --- a/app/prune.go +++ b/app/prune.go @@ -20,7 +20,7 @@ var ( ) type snapshot struct { - Ds string // Snapshot dataset name "backups/foo@RFC3339" + Name string // Snapshot dataset name "backups/foo@RFC3339" Time time.Time // Parsed timestamp from the dataset name } @@ -138,7 +138,7 @@ func listSnapshots(host string) []snapshot { } snapshots = append(snapshots, snapshot{ - Ds: ss, + Name: ss, Time: ts, }) }