Adding a utility to help with managing lists

.gitignore

@@ -1,5 +1,6 @@
 # Binaries for programs and plugins
 donutdns
+cmd/donut/donut
 *.exe
 *.exe~
 *.dll

cmd/donut/cmd/allowdeny.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+)
+
+func Allow() (map[string]int, error) {
+	return hostFile(conf.Allow)
+}
+
+func Block() (map[string]int, error) {
+	return hostFile(conf.Block)
+}
+
+func hostFile(filename string) (map[string]int, error) {
+	if filename == "" {
+		return map[string]int{}, nil
+	}
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("while reading hostname list file: %w", err)
+	}
+	entries := map[string]int{}
+	scanner := bufio.NewScanner(f)
+	lineNo := 0
+	regexRe := regexp.MustCompile(Generic)
+	for scanner.Scan() {
+		line := scanner.Text()
+		lineNo++
+		host := regexRe.FindString(line)
+		if host != "" {
+			entries[strings.ToLower(host)] = lineNo
+		}
+	}
+	return entries, nil
+}

cmd/donut/cmd/fetch.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var fetchCmd = &cobra.Command{
+	Use:   "fetch",
+	Short: "fetch upstream blocklists",
+	RunE:  fetchCommand,
+}
+
+func init() {
+	rootCmd.AddCommand(fetchCmd)
+}
+
+// filenamesFromUrls makes nice filenames to store fetched sources
+// in the cache, so they're pleasant-ish to use by hand.
+func filenamesFromUrls(sources map[string][]string) map[string]string {
+	urlRe := regexp.MustCompile(`^https?://([^/]+)/([^?]+)`)
+	ret := map[string]string{}
+	for cat, list := range sources {
+		seen := map[string]struct{}{}
+		var fname string
+		for _, url := range list {
+			matches := urlRe.FindStringSubmatch(url)
+			if matches == nil {
+				_, fname = path.Split(url)
+			} else {
+				hostname := matches[1]
+				parts := strings.Split(matches[2], "/")
+				switch hostname {
+				case "raw.githubusercontent.com", "bitbucket.org", "s3.amazonaws.com":
+					hostname = parts[0]
+					parts = parts[1:]
+				default:
+				}
+				if len(parts) > 0 {
+					fname = hostname + "-" + parts[len(parts)-1]
+				} else {
+					fname = hostname
+				}
+			}
+			if fname == "" {
+				fname = "list"
+			}
+			_, ok := seen[fname]
+			if ok {
+				i := 1
+				for {
+					fn := fmt.Sprintf("%s.%d", fname, i)
+					_, ok = seen[fn]
+					if !ok {
+						fname = fn
+						break
+					}
+					i++
+				}
+			}
+			seen[fname] = struct{}{}
+			ret[url] = cat + "-" + fname
+		}
+	}
+	return ret
+}
+
+func fetchCommand(*cobra.Command, []string) error {
+	_, err := fetchLists(true, true)
+	return err
+}
+
+func cacheDir() (string, error) {
+	cd := conf.CacheDir
+	if cd == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", err
+		}
+		cd = filepath.Join(home, ".donutcache")
+	}
+	err := os.MkdirAll(cd, 0755)
+	if err != nil {
+		log.Printf("Failed to create '%s': %v\n", cd, err)
+		cd, err = os.MkdirTemp(os.TempDir(), "donut-*")
+		if err != nil {
+			return "", err
+		}
+		log.Printf("Using '%s' instead\n", cd)
+	}
+	return cd, nil
+}
+
+func fetchLists(force, verbose bool) (map[string][]byte, error) {
+	sourceList, err := sources()
+	if err != nil {
+		return nil, err
+	}
+	filenames := filenamesFromUrls(sourceList)
+	content := map[string][]byte{}
+	cache, err := cacheDir()
+	if err != nil {
+		return nil, err
+	}
+	for url, file := range filenames {
+		filename := filepath.Join(cache, file)
+		if !force {
+			fi, err := os.Stat(filename)
+			if err == nil && fi.ModTime().Add(conf.CacheLifetime).After(time.Now()) {
+				cachedContent, err := os.ReadFile(filename)
+				if err == nil {
+					content[url] = cachedContent
+					if verbose {
+						log.Printf("%s (cached)", url)
+					}
+					continue
+				}
+			}
+		}
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			return nil, fmt.Errorf("while creating request from %s: %w", url, err)
+		}
+		req.Header.Set("User-Agent", fmt.Sprintf("com.wordtothewise.donut (%s)", conf.Version))
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Printf("`%s (failed, %v)", url, err)
+			continue
+		}
+		if resp.StatusCode != http.StatusOK {
+			log.Printf("%s (failed, %s)", url, resp.Status)
+			_ = resp.Body.Close()
+			continue
+		}
+		newContent, err := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			log.Printf("%s (failed, %v)", url, err)
+			continue
+		}
+		content[filename] = newContent
+		err = os.WriteFile(filename, newContent, 0644)
+		if err != nil {
+			log.Printf("failed to write cache file: %v", err)
+			_ = os.Remove(filename)
+			continue
+		}
+		if verbose {
+			log.Printf("%s (OK)", url)
+		}
+	}
+	return content, nil
+}

cmd/donut/cmd/flatten.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"github.com/spf13/cobra"
+	"regexp"
+	"strings"
+)
+
+var flattenCmd = &cobra.Command{
+	Use:   "flatten",
+	Short: "Combine block and allow lists into a single list",
+	RunE:  flattenCommand,
+}
+
+func init() {
+	rootCmd.AddCommand(flattenCmd)
+}
+
+func flattenCommand(*cobra.Command, []string) error {
+	src, err := sources()
+	if err != nil {
+		return err
+	}
+	lists, err := fetchLists(false, false)
+	if err != nil {
+		return err
+	}
+	regexRe := regexp.MustCompile(Generic)
+	allow, err := Allow()
+	if err != nil {
+		return err
+	}
+	block, err := Block()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("# Local block list\n")
+	for b := range block {
+		_, ok := allow[b]
+		if !ok {
+			fmt.Printf("%s\n", b)
+		}
+	}
+	fmt.Printf("\n\n")
+	for cat, urls := range src {
+		for _, url := range urls {
+			fmt.Printf("\n# %s: %s\n\n", cat, url)
+			list := lists[url]
+			scanner := bufio.NewScanner(bytes.NewReader(list))
+			for scanner.Scan() {
+				line := scanner.Text()
+				if line == "" || line[0] == '#' {
+					continue
+				}
+				pattern := regexRe.FindString(line)
+				host := strings.ToLower(pattern)
+				_, ok := allow[host]
+				if !ok {
+					fmt.Printf("%s\n", host)
+				}
+			}
+		}
+	}
+	return nil
+}

cmd/donut/cmd/root.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+type Config struct {
+	SourcesFile   string
+	Sources       map[string][]string
+	CacheDir      string
+	CacheLifetime time.Duration
+	Version       string
+	Allow         string
+	Block         string
+}
+
+var conf = Config{
+	Sources:       map[string][]string{},
+	CacheDir:      "",
+	CacheLifetime: 3600 * time.Second,
+	Version:       "0.1",
+}
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "donut",
+	Short: "Manage donutdns",
+	Long:  `donut is a tool to manage donutdns block lists.`,
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	rootCmd.PersistentFlags().StringVar(&conf.CacheDir, "cache", "", "cache directory (default is $HOME/.donutcache)")
+	rootCmd.PersistentFlags().StringVar(&conf.SourcesFile, "sources", "", "json sources file (default is built-in defaults)")
+	rootCmd.PersistentFlags().StringVar(&conf.Block, "block", "", "file with list of hostnames to block")
+	rootCmd.PersistentFlags().StringVar(&conf.Allow, "allow", "", "file with list of hostnames to allow")
+}

cmd/donut/cmd/sources.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"github.com/spf13/cobra"
+	"os"
+)
+
+//go:embed sources.json
+var defaults []byte
+
+var defaultsCmd = &cobra.Command{
+	Use:   "defaults",
+	Short: "print the embedded list of sources",
+	RunE:  defaultsCommand,
+}
+
+func init() {
+	rootCmd.AddCommand(defaultsCmd)
+}
+
+func defaultsCommand(*cobra.Command, []string) error {
+	_, err := os.Stdout.Write(defaults)
+	return err
+}
+
+func sources() (map[string][]string, error) {
+	if len(conf.Sources) != 0 {
+		return conf.Sources, nil
+	}
+	var sourceJson []byte
+	var err error
+	if conf.SourcesFile == "" {
+		sourceJson = defaults
+	} else {
+		sourceJson, err = os.ReadFile(conf.SourcesFile)
+		if err != nil {
+			return nil, fmt.Errorf("while reading sources file: %w", err)
+		}
+	}
+	s := map[string][]string{}
+	err = json.Unmarshal(sourceJson, &s)
+	if err != nil {
+		return nil, fmt.Errorf("while parsing sources file: %w", err)
+	}
+	conf.Sources = s
+	return conf.Sources, nil
+}