From 37a01c9daaf4295c31b1166b3bf781c1bec5c54c Mon Sep 17 00:00:00 2001 From: Steve Atkins Date: Fri, 19 Aug 2022 11:52:52 +0100 Subject: [PATCH 1/2] First pass at helper tool. --- .gitignore | 1 + cmd/donut/cmd/fetch.go | 162 +++++++++++++++++++++++++++++++++++++ cmd/donut/cmd/root.go | 48 +++++++++++ cmd/donut/cmd/sources.go | 34 ++++++++ cmd/donut/cmd/sources.json | 40 +++++++++ cmd/donut/donut.go | 7 ++ go.mod | 3 + go.sum | 7 ++ 8 files changed, 302 insertions(+) create mode 100644 cmd/donut/cmd/fetch.go create mode 100644 cmd/donut/cmd/root.go create mode 100644 cmd/donut/cmd/sources.go create mode 100644 cmd/donut/cmd/sources.json create mode 100644 cmd/donut/donut.go diff --git a/.gitignore b/.gitignore index 1479445..1b120cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Binaries for programs and plugins donutdns +cmd/donut/donut *.exe *.exe~ *.dll diff --git a/cmd/donut/cmd/fetch.go b/cmd/donut/cmd/fetch.go new file mode 100644 index 0000000..aa84432 --- /dev/null +++ b/cmd/donut/cmd/fetch.go @@ -0,0 +1,162 @@ +package cmd + +import ( + "fmt" + "github.com/spf13/cobra" + "io" + "log" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" +) + +var fetchCmd = &cobra.Command{ + Use: "fetch", + Short: "fetch upstream blocklists", + RunE: fetchCommand, +} + +func init() { + rootCmd.AddCommand(fetchCmd) +} + +// filenamesFromUrls makes nice filenames to store fetched sources +// in the cache, so they're pleasant-ish to use by hand. +func filenamesFromUrls(sources map[string][]string) map[string]string { + urlRe := regexp.MustCompile(`^https?://([^/]+)/([^?]+)`) + ret := map[string]string{} + for cat, list := range sources { + seen := map[string]struct{}{} + var fname string + for _, url := range list { + matches := urlRe.FindStringSubmatch(url) + if matches == nil { + _, fname = path.Split(url) + continue + } + hostname := matches[1] + parts := strings.Split(matches[2], "/") + switch hostname { + case "raw.githubusercontent.com", "bitbucket.org", "s3.amazonaws.com": + hostname = parts[0] + parts = parts[1:] + default: + } + if len(parts) > 0 { + fname = hostname + "-" + parts[0] + } else { + fname = hostname + } + + if fname == "" { + fname = "list" + } + _, ok := seen[fname] + if ok { + i := 1 + for { + fn := fmt.Sprintf("%s.%d", fname, i) + _, ok = seen[fn] + if !ok { + fname = fn + break + } + i++ + } + } + ret[url] = cat + "-" + fname + } + } + return ret +} + +func fetchCommand(*cobra.Command, []string) error { + _, err := fetchSources(true, true) + return err +} + +func cacheDir() (string, error) { + cd := conf.CacheDir + if cd == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + cd = filepath.Join(home, ".donutcache") + } + err := os.MkdirAll(cd, 0755) + if err != nil { + log.Printf("Failed to create '%s': %v\n", cd, err) + cd, err = os.MkdirTemp(os.TempDir(), "donut-*") + if err != nil { + return "", err + } + log.Printf("Using '%s' instead\n", cd) + } + return cd, nil +} + +func fetchSources(force, verbose bool) (map[string][]byte, error) { + sourceList, err := sources() + if err != nil { + return nil, err + } + filenames := filenamesFromUrls(sourceList) + content := map[string][]byte{} + cache, err := cacheDir() + if err != nil { + return nil, err + } + for url, file := range filenames { + filename := filepath.Join(cache, file) + if !force { + fi, err := os.Stat(filename) + if err == nil && fi.ModTime().Add(conf.CacheLifetime).Before(time.Now()) { + cachedContent, err := os.ReadFile(filename) + if err == nil { + content[url] = cachedContent + if verbose { + log.Printf("%s (cached)", url) + } + continue + } + } + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("while creating request from %s: %w", url, err) + } + req.Header.Set("User-Agent", fmt.Sprintf("com.wordtothewise.donut (%s)", conf.Version)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("`%s (failed, %v)", url, err) + continue + } + if resp.StatusCode != http.StatusOK { + log.Printf("%s (failed, %s)", url, resp.Status) + _ = resp.Body.Close() + continue + } + newContent, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + log.Printf("%s (failed, %v)", url, err) + continue + } + content[filename] = newContent + err = os.WriteFile(filename, newContent, 0644) + if err != nil { + log.Printf("failed to write cache file: %v", err) + _ = os.Remove(filename) + continue + } + if verbose { + log.Printf("%s (OK)", url) + } + } + return content, nil +} diff --git a/cmd/donut/cmd/root.go b/cmd/donut/cmd/root.go new file mode 100644 index 0000000..4952047 --- /dev/null +++ b/cmd/donut/cmd/root.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "os" + "time" + + "github.com/spf13/cobra" +) + +type Config struct { + SourcesFile string + Sources map[string][]string + CacheDir string + CacheLifetime time.Duration + Version string +} + +var conf = Config{ + Sources: map[string][]string{}, + CacheDir: "", + CacheLifetime: 3600 * time.Second, + Version: "0.1", +} + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "donut", + Short: "Manage donutdns", + Long: `donut is a tool to manage donutdns block lists.`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + rootCmd.PersistentFlags().StringVar(&conf.CacheDir, "cache", "", "cache directory (default is $HOME/.donut)") + rootCmd.PersistentFlags().StringVar(&conf.SourcesFile, "sources", "", "json sources file (default is built-in defaults)") +} diff --git a/cmd/donut/cmd/sources.go b/cmd/donut/cmd/sources.go new file mode 100644 index 0000000..4a7f15b --- /dev/null +++ b/cmd/donut/cmd/sources.go @@ -0,0 +1,34 @@ +package cmd + +import ( + _ "embed" + "encoding/json" + "fmt" + "os" +) + +//go:embed sources.json +var defaults []byte + +func sources() (map[string][]string, error) { + if len(conf.Sources) != 0 { + return conf.Sources, nil + } + var sourceJson []byte + var err error + if conf.SourcesFile == "" { + sourceJson = defaults + } else { + sourceJson, err = os.ReadFile(conf.SourcesFile) + if err != nil { + return nil, fmt.Errorf("while reading sources file: %w", err) + } + } + s := map[string][]string{} + err = json.Unmarshal(sourceJson, &s) + if err != nil { + return nil, fmt.Errorf("while parsing sources file: %w", err) + } + conf.Sources = s + return conf.Sources, nil +} diff --git a/cmd/donut/cmd/sources.json b/cmd/donut/cmd/sources.json new file mode 100644 index 0000000..f39ef70 --- /dev/null +++ b/cmd/donut/cmd/sources.json @@ -0,0 +1,40 @@ +{ + "suspicious": [ + "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt", + "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts", + "https://v.firebog.net/hosts/static/w3kbl.txt" + ], + "advertising": [ + "https://adaway.org/hosts.txt", + "https://v.firebog.net/hosts/AdguardDNS.txt", + "https://v.firebog.net/hosts/Admiral.txt", + "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", + "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt", + "https://v.firebog.net/hosts/Easylist.txt", + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext", + "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts", + "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts" + ], + "tracking": [ + "https://v.firebog.net/hosts/Easyprivacy.txt", + "https://v.firebog.net/hosts/Prigent-Ads.txt", + "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts", + "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", + "https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt" + ], + "malicious": [ + "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt", + "https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", + "https://v.firebog.net/hosts/Prigent-Crypto.txt", + "https://bitbucket.org/ethanr/dns-blacklists/raw/8575c9f96e5b4a1308f2f12394abd86d0927a4a0/bad_lists/Mandiant_APT1_Report_Appendix_D.txt", + "https://phishing.army/download/phishing_army_blocklist_extended.txt", + "https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt", + "https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt", + "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts", + "https://urlhaus.abuse.ch/downloads/hostfile/" + ], + "miners": [ + "https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser" + ] +} \ No newline at end of file diff --git a/cmd/donut/donut.go b/cmd/donut/donut.go new file mode 100644 index 0000000..ee13ef2 --- /dev/null +++ b/cmd/donut/donut.go @@ -0,0 +1,7 @@ +package main + +import "gophers.dev/cmds/donutdns/cmd/donut/cmd" + +func main() { + cmd.Execute() +} diff --git a/go.mod b/go.mod index 7ecd8bb..f20b978 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -32,6 +33,8 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect + github.com/spf13/cobra v1.5.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365 // indirect golang.org/x/text v0.3.6 // indirect diff --git a/go.sum b/go.sum index 737ed30..8fafdd2 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,7 @@ github.com/coredns/coredns v1.8.5 h1:+xc+6c7O330zB6OF+TxfzI4SWss2kl9CUddQgKgai2Y github.com/coredns/coredns v1.8.5/go.mod h1:JSMrIFILtjdV5bnwtwEciNrsvDwnGapCnE+MEIkB/Ho= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -261,6 +262,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9/go.mod h1:BaIJzjD2ZnHmx2acPF6XfGLPzNCMiBbMRqJr+8/8uRI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= @@ -355,6 +358,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shoenig/test v0.2.4 h1:TM/otFRxMLAU/QWe0Xrhj/mb/oLdR+9tYwxoiPdedmk= github.com/shoenig/test v0.2.4/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -363,7 +367,10 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= From a1e2013995c4312d8c6062bdf88a37a94946a718 Mon Sep 17 00:00:00 2001 From: Steve Atkins Date: Fri, 19 Aug 2022 14:19:05 +0100 Subject: [PATCH 2/2] Add why and flatten commands. --- cmd/donut/cmd/allowdeny.go | 40 ++++++++++++++++++++ cmd/donut/cmd/fetch.go | 34 ++++++++--------- cmd/donut/cmd/flatten.go | 68 ++++++++++++++++++++++++++++++++++ cmd/donut/cmd/root.go | 6 ++- cmd/donut/cmd/sources.go | 16 ++++++++ cmd/donut/cmd/why.go | 75 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 cmd/donut/cmd/allowdeny.go create mode 100644 cmd/donut/cmd/flatten.go create mode 100644 cmd/donut/cmd/why.go diff --git a/cmd/donut/cmd/allowdeny.go b/cmd/donut/cmd/allowdeny.go new file mode 100644 index 0000000..6e0412f --- /dev/null +++ b/cmd/donut/cmd/allowdeny.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" +) + +func Allow() (map[string]int, error) { + return hostFile(conf.Allow) +} + +func Block() (map[string]int, error) { + return hostFile(conf.Block) +} + +func hostFile(filename string) (map[string]int, error) { + if filename == "" { + return map[string]int{}, nil + } + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("while reading hostname list file: %w", err) + } + entries := map[string]int{} + scanner := bufio.NewScanner(f) + lineNo := 0 + regexRe := regexp.MustCompile(Generic) + for scanner.Scan() { + line := scanner.Text() + lineNo++ + host := regexRe.FindString(line) + if host != "" { + entries[strings.ToLower(host)] = lineNo + } + } + return entries, nil +} diff --git a/cmd/donut/cmd/fetch.go b/cmd/donut/cmd/fetch.go index aa84432..8f45595 100644 --- a/cmd/donut/cmd/fetch.go +++ b/cmd/donut/cmd/fetch.go @@ -36,22 +36,21 @@ func filenamesFromUrls(sources map[string][]string) map[string]string { matches := urlRe.FindStringSubmatch(url) if matches == nil { _, fname = path.Split(url) - continue - } - hostname := matches[1] - parts := strings.Split(matches[2], "/") - switch hostname { - case "raw.githubusercontent.com", "bitbucket.org", "s3.amazonaws.com": - hostname = parts[0] - parts = parts[1:] - default: - } - if len(parts) > 0 { - fname = hostname + "-" + parts[0] } else { - fname = hostname + hostname := matches[1] + parts := strings.Split(matches[2], "/") + switch hostname { + case "raw.githubusercontent.com", "bitbucket.org", "s3.amazonaws.com": + hostname = parts[0] + parts = parts[1:] + default: + } + if len(parts) > 0 { + fname = hostname + "-" + parts[len(parts)-1] + } else { + fname = hostname + } } - if fname == "" { fname = "list" } @@ -68,6 +67,7 @@ func filenamesFromUrls(sources map[string][]string) map[string]string { i++ } } + seen[fname] = struct{}{} ret[url] = cat + "-" + fname } } @@ -75,7 +75,7 @@ func filenamesFromUrls(sources map[string][]string) map[string]string { } func fetchCommand(*cobra.Command, []string) error { - _, err := fetchSources(true, true) + _, err := fetchLists(true, true) return err } @@ -100,7 +100,7 @@ func cacheDir() (string, error) { return cd, nil } -func fetchSources(force, verbose bool) (map[string][]byte, error) { +func fetchLists(force, verbose bool) (map[string][]byte, error) { sourceList, err := sources() if err != nil { return nil, err @@ -115,7 +115,7 @@ func fetchSources(force, verbose bool) (map[string][]byte, error) { filename := filepath.Join(cache, file) if !force { fi, err := os.Stat(filename) - if err == nil && fi.ModTime().Add(conf.CacheLifetime).Before(time.Now()) { + if err == nil && fi.ModTime().Add(conf.CacheLifetime).After(time.Now()) { cachedContent, err := os.ReadFile(filename) if err == nil { content[url] = cachedContent diff --git a/cmd/donut/cmd/flatten.go b/cmd/donut/cmd/flatten.go new file mode 100644 index 0000000..5b4b797 --- /dev/null +++ b/cmd/donut/cmd/flatten.go @@ -0,0 +1,68 @@ +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "github.com/spf13/cobra" + "regexp" + "strings" +) + +var flattenCmd = &cobra.Command{ + Use: "flatten", + Short: "Combine block and allow lists into a single list", + RunE: flattenCommand, +} + +func init() { + rootCmd.AddCommand(flattenCmd) +} + +func flattenCommand(*cobra.Command, []string) error { + src, err := sources() + if err != nil { + return err + } + lists, err := fetchLists(false, false) + if err != nil { + return err + } + regexRe := regexp.MustCompile(Generic) + allow, err := Allow() + if err != nil { + return err + } + block, err := Block() + if err != nil { + return err + } + fmt.Printf("# Local block list\n") + for b := range block { + _, ok := allow[b] + if !ok { + fmt.Printf("%s\n", b) + } + } + fmt.Printf("\n\n") + for cat, urls := range src { + for _, url := range urls { + fmt.Printf("\n# %s: %s\n\n", cat, url) + list := lists[url] + scanner := bufio.NewScanner(bytes.NewReader(list)) + for scanner.Scan() { + line := scanner.Text() + if line == "" || line[0] == '#' { + continue + } + pattern := regexRe.FindString(line) + host := strings.ToLower(pattern) + _, ok := allow[host] + if !ok { + fmt.Printf("%s\n", host) + } + } + } + } + return nil +} diff --git a/cmd/donut/cmd/root.go b/cmd/donut/cmd/root.go index 4952047..715a094 100644 --- a/cmd/donut/cmd/root.go +++ b/cmd/donut/cmd/root.go @@ -13,6 +13,8 @@ type Config struct { CacheDir string CacheLifetime time.Duration Version string + Allow string + Block string } var conf = Config{ @@ -43,6 +45,8 @@ func init() { // Cobra supports persistent flags, which, if defined here, // will be global for your application. - rootCmd.PersistentFlags().StringVar(&conf.CacheDir, "cache", "", "cache directory (default is $HOME/.donut)") + rootCmd.PersistentFlags().StringVar(&conf.CacheDir, "cache", "", "cache directory (default is $HOME/.donutcache)") rootCmd.PersistentFlags().StringVar(&conf.SourcesFile, "sources", "", "json sources file (default is built-in defaults)") + rootCmd.PersistentFlags().StringVar(&conf.Block, "block", "", "file with list of hostnames to block") + rootCmd.PersistentFlags().StringVar(&conf.Allow, "allow", "", "file with list of hostnames to allow") } diff --git a/cmd/donut/cmd/sources.go b/cmd/donut/cmd/sources.go index 4a7f15b..b46a03e 100644 --- a/cmd/donut/cmd/sources.go +++ b/cmd/donut/cmd/sources.go @@ -4,12 +4,28 @@ import ( _ "embed" "encoding/json" "fmt" + "github.com/spf13/cobra" "os" ) //go:embed sources.json var defaults []byte +var defaultsCmd = &cobra.Command{ + Use: "defaults", + Short: "print the embedded list of sources", + RunE: defaultsCommand, +} + +func init() { + rootCmd.AddCommand(defaultsCmd) +} + +func defaultsCommand(*cobra.Command, []string) error { + _, err := os.Stdout.Write(defaults) + return err +} + func sources() (map[string][]string, error) { if len(conf.Sources) != 0 { return conf.Sources, nil diff --git a/cmd/donut/cmd/why.go b/cmd/donut/cmd/why.go new file mode 100644 index 0000000..30c06bc --- /dev/null +++ b/cmd/donut/cmd/why.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "github.com/spf13/cobra" + "regexp" + "strings" +) + +const Generic = `(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]` + +var whyCmd = &cobra.Command{ + Use: "why [hostname]", + Short: "Find out where a hostname is blocked", + RunE: whyCommand, +} + +func init() { + rootCmd.AddCommand(whyCmd) +} + +func whyCommand(_ *cobra.Command, args []string) error { + if len(args) == 0 { + return nil + } + src, err := sources() + if err != nil { + return err + } + lists, err := fetchLists(false, false) + if err != nil { + return err + } + regexRe := regexp.MustCompile(Generic) + allow, err := Allow() + if err != nil { + return err + } + block, err := Block() + if err != nil { + return err + } + for _, host := range args { + cleanHost := strings.ToLower(strings.TrimSuffix(host, ".")) + for cat, urls := range src { + for _, url := range urls { + list := lists[url] + scanner := bufio.NewScanner(bytes.NewReader(list)) + lineNo := 0 + for scanner.Scan() { + line := scanner.Text() + lineNo++ + if line == "" || line[0] == '#' { + continue + } + pattern := regexRe.FindString(line) + if strings.ToLower(pattern) == cleanHost { + fmt.Printf("%s listed as %s on line %d of %s\n", cleanHost, cat, lineNo, url) + } + } + } + } + lineNo, ok := block[cleanHost] + if ok { + fmt.Printf("%s listed on line %d of local block list\n", cleanHost, lineNo) + } + lineNo, ok = allow[cleanHost] + if ok { + fmt.Printf("%s will not be blocked, it is on line %d of local allow list\n", cleanHost, lineNo) + } + } + return nil +}