diff --git a/README.md b/README.md index 2bbee33..2077fa9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Over time we've now gained a few more features: * See [SMTP-setup](#smtp-setup) for details. * The ability to include/exclude feed items from the emails. * For example receive emails only of feed items that contain the pattern "playstation". - +* A well-behaved HTTP-polling behaviour, using the appropriate cache-related HTTP-headers. @@ -53,7 +53,6 @@ To install from source simply clone the repository and build in the usual manner ```sh git clone https://github.com/skx/rss2email cd rss2email -go build . go install . ``` @@ -66,7 +65,7 @@ Finally you can find automatically generated docker images, these are built on a **Version NOTES**: * You'll need go version **1.21** or higher to build. - * We use `go embed` to embed our (default) email-template within the binary, this was introduced with golang **v1.17**. + * We use `go embed` to embed the (default) email-template within the binary, this was introduced with golang **v1.17**. * We use the [slog logging package](https://go.dev/blog/slog) introduced with golang **v1.21**. * We use the fuzzing support which was introduced with golang **v1.18** to test our configuration-file loading/parsing. * See [configfile/FUZZING.md](configfile/FUZZING.md) for details of using that. diff --git a/go.mod b/go.mod index 4225d5d..165e215 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/mmcdole/goxpp v1.1.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect ) diff --git a/go.sum b/go.sum index 9b95349..58b08a6 100644 --- a/go.sum +++ b/go.sum @@ -48,13 +48,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -62,8 +62,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -73,8 +73,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/httpfetch/httpfetch.go b/httpfetch/httpfetch.go index 5ed108f..c8c2fda 100644 --- a/httpfetch/httpfetch.go +++ b/httpfetch/httpfetch.go @@ -6,18 +6,21 @@ package httpfetch import ( "crypto/tls" + "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" "os" + "path/filepath" "strconv" "strings" "time" "github.com/mmcdole/gofeed" "github.com/skx/rss2email/configfile" + statePath "github.com/skx/rss2email/state" ) var ( @@ -95,6 +98,25 @@ func New(entry configfile.Feed, log *slog.Logger, version string) *HTTPFetch { userAgent: fmt.Sprintf("rss2email %s (https://github.com/skx/rss2email)", version), } + // Path to the cache file, which we read from-disk if we can. + fileName := filepath.Join(statePath.Directory(), "httpcache.json") + data, err := os.ReadFile(fileName) + + // If we got an error, and it wasn't a not-found log it + if err != nil && !os.IsNotExist(err) { + log.Debug("failed to read cache-values", + slog.String("path", fileName), + slog.String("error", err.Error())) + } else { + // We can't even log it usefully. + err = json.Unmarshal(data, &cache) + if err != nil { + log.Debug("failed to unmarshall cache-values", + slog.String("path", fileName), + slog.String("error", err.Error())) + } + } + // Get the user's sleep period - if overridden this will become the // default frequency for each feed item. sleep := os.Getenv("SLEEP") @@ -305,6 +327,18 @@ func (h *HTTPFetch) fetch() error { } cache[h.url] = x + // Save cache. + encoded, errEncoding := json.Marshal(cache) + if errEncoding == nil { + fileName := filepath.Join(statePath.Directory(), "httpcache.json") + errWrite := os.WriteFile(fileName, encoded, 0644) + if errWrite != nil { + h.logger.Debug("failed to write cache to json", + slog.String("path", fileName), + slog.String("error", errWrite.Error())) + } + } + // // Did the remote page not change? //