From 7033f242c0f8788341927fdfc88b5b9aa5fe5fc3 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Fri, 11 Aug 2023 00:51:34 +0200 Subject: [PATCH] Restore the cache update code from version 2.1.4 for now --- dnscrypt-proxy/config.go | 4 +- dnscrypt-proxy/sources.go | 114 +++++++++++++++++---------------- dnscrypt-proxy/sources_test.go | 13 ++-- 3 files changed, 67 insertions(+), 64 deletions(-) diff --git a/dnscrypt-proxy/config.go b/dnscrypt-proxy/config.go index 6d7aac8e..b141ee42 100644 --- a/dnscrypt-proxy/config.go +++ b/dnscrypt-proxy/config.go @@ -892,8 +892,8 @@ func (config *Config) loadSource(proxy *Proxy, cfgSourceName string, cfgSource * if cfgSource.FormatStr == "" { cfgSource.FormatStr = "v2" } - if cfgSource.RefreshDelay < 24 { - cfgSource.RefreshDelay = 24 + if cfgSource.RefreshDelay <= 0 { + cfgSource.RefreshDelay = 72 } else if cfgSource.RefreshDelay > 168 { cfgSource.RefreshDelay = 168 } diff --git a/dnscrypt-proxy/sources.go b/dnscrypt-proxy/sources.go index bc3eeb8e..1953dc35 100644 --- a/dnscrypt-proxy/sources.go +++ b/dnscrypt-proxy/sources.go @@ -3,10 +3,10 @@ package main import ( "bytes" "fmt" - "math" "math/rand" "net/url" "os" + "path/filepath" "strings" "time" @@ -23,18 +23,21 @@ const ( SourceFormatV2 = iota ) -const MinimumPrefetchInterval time.Duration = 10 * time.Minute +const ( + DefaultPrefetchDelay time.Duration = 24 * time.Hour + MinimumPrefetchInterval time.Duration = 10 * time.Minute +) type Source struct { - name string - urls []*url.URL - bin []byte // copy of the file content - there's something wrong in our logic, we shouldn't need to keep that in memory - minisignKey *minisign.PublicKey - cacheFile string - prefix string - cacheTTL time.Duration - refresh time.Time - format SourceFormat + name string + urls []*url.URL + format SourceFormat + bin []byte + minisignKey *minisign.PublicKey + cacheFile string + cacheTTL, prefetchDelay time.Duration + refresh time.Time + prefix string } func (source *Source) checkSignature(bin, sig []byte) (err error) { @@ -48,7 +51,7 @@ func (source *Source) checkSignature(bin, sig []byte) (err error) { // timeNow() can be replaced by tests to provide a static value var timeNow = time.Now -func (source *Source) fetchFromCache(now time.Time) (remaining time.Duration, err error) { +func (source *Source) fetchFromCache(now time.Time) (ttl time.Duration, err error) { var bin, sig []byte if bin, err = os.ReadFile(source.cacheFile); err != nil { return @@ -65,8 +68,8 @@ func (source *Source) fetchFromCache(now time.Time) (remaining time.Duration, er return } if elapsed := now.Sub(fi.ModTime()); elapsed < source.cacheTTL { - remaining = source.cacheTTL - elapsed - dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update in %v min", source.name, source.cacheFile, math.Round(remaining.Minutes())) + ttl = source.prefetchDelay - elapsed + dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update: %v", source.name, source.cacheFile, ttl) } else { dlog.Debugf("Source [%s] cache file [%s] needs to be refreshed", source.name, source.cacheFile) } @@ -95,24 +98,25 @@ func writeSource(f string, bin, sig []byte) (err error) { return fSig.Commit() } -func (source *Source) updateCache(bin, sig []byte, now time.Time) error { +func (source *Source) updateCache(bin, sig []byte, now time.Time) { f := source.cacheFile - // If the data is unchanged, update the files timestamps only - if bytes.Equal(source.bin, bin) { - _ = os.Chtimes(f, now, now) - _ = os.Chtimes(f+".minisig", now, now) - return nil + var writeErr error // an error writing cache isn't fatal + defer func() { + source.bin = bin + if writeErr == nil { + return + } + if absPath, absErr := filepath.Abs(f); absErr == nil { + f = absPath + } + dlog.Warnf("%s: %s", f, writeErr) + }() + if !bytes.Equal(source.bin, bin) { + if writeErr = writeSource(f, bin, sig); writeErr != nil { + return + } } - // Otherwise, write the new data and signature - if err := writeSource(f, bin, sig); err != nil { - dlog.Warnf("Source [%s] failed to update cache file [%s]: %v", source.name, f, err) - return err - } - source.bin = bin // In-memory copy of the cache file content - // The tests require the timestamps to be updated, no idea why - _ = os.Chtimes(f, now, now) - _ = os.Chtimes(f+".minisig", now, now) - return nil + writeErr = os.Chtimes(f, now, now) } func (source *Source) parseURLs(urls []string) { @@ -130,23 +134,23 @@ func fetchFromURL(xTransport *XTransport, u *url.URL) (bin []byte, err error) { return bin, err } -func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (time.Duration, error) { - remaining, err := source.fetchFromCache(now) - if err != nil { +func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (ttl time.Duration, err error) { + if ttl, err = source.fetchFromCache(now); err != nil { if len(source.urls) == 0 { - dlog.Fatalf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile) - return 0, err + dlog.Errorf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile) + return } dlog.Debugf("Source [%s] cache file [%s] not present", source.name, source.cacheFile) } - if len(source.urls) == 0 { - dlog.Debugf("No URL to update [%s]", source.name) - return 24 * time.Hour, nil + if len(source.urls) > 0 { + defer func() { + source.refresh = now.Add(ttl) + }() } - if remaining > 0 { - source.refresh = now.Add(remaining) - return remaining, nil + if len(source.urls) == 0 || ttl > 0 { + return } + ttl = MinimumPrefetchInterval var bin, sig []byte for _, srcURL := range source.urls { dlog.Infof("Source [%s] loading from URL [%s]", source.name, srcURL) @@ -167,13 +171,11 @@ func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (tim dlog.Debugf("Source [%s] failed signature check using URL [%s]", source.name, srcURL) } if err != nil { - source.refresh = now.Add(MinimumPrefetchInterval) - return MinimumPrefetchInterval, err + return } source.updateCache(bin, sig, now) - remaining = source.cacheTTL - source.refresh = now.Add(remaining) - return remaining, nil + ttl = source.prefetchDelay + return } // NewSource loads a new source using the given cacheFile and urls, ensuring it has a valid signature @@ -187,12 +189,16 @@ func NewSource( refreshDelay time.Duration, prefix string, ) (source *Source, err error) { + if refreshDelay < DefaultPrefetchDelay { + refreshDelay = DefaultPrefetchDelay + } source = &Source{ - name: name, - urls: []*url.URL{}, - cacheFile: cacheFile, - cacheTTL: refreshDelay, - prefix: prefix, + name: name, + urls: []*url.URL{}, + cacheFile: cacheFile, + cacheTTL: refreshDelay, + prefetchDelay: DefaultPrefetchDelay, + prefix: prefix, } if formatStr == "v2" { source.format = SourceFormatV2 @@ -223,7 +229,7 @@ func PrefetchSources(xTransport *XTransport, sources []*Source) time.Duration { if delay, err := source.fetchWithCache(xTransport, now); err != nil { dlog.Infof("Prefetching [%s] failed: %v, will retry in %v", source.name, err, interval) } else { - dlog.Debugf("Prefetching [%s] succeeded, next update: %v min", source.name, math.Round(delay.Minutes())) + dlog.Debugf("Prefetching [%s] succeeded, next update in %v min", source.name, delay) if delay >= MinimumPrefetchInterval && (interval == MinimumPrefetchInterval || interval > delay) { interval = delay } @@ -248,8 +254,8 @@ func (source *Source) parseV2() ([]RegisteredServer, error) { stampErrs = append(stampErrs, stampErr) dlog.Warn(stampErr) } - bin := string(source.bin) - parts := strings.Split(bin, "## ") + in := string(source.bin) + parts := strings.Split(in, "## ") if len(parts) < 2 { return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls) } diff --git a/dnscrypt-proxy/sources_test.go b/dnscrypt-proxy/sources_test.go index c87c415f..04a6eac7 100644 --- a/dnscrypt-proxy/sources_test.go +++ b/dnscrypt-proxy/sources_test.go @@ -42,8 +42,6 @@ const ( TestStatePathErr // unparseable path to files (download only) ) -const DefaultPrefetchDelay time.Duration = 24 * time.Hour - type SourceTestData struct { n int // subtest counter xTransport *XTransport @@ -352,7 +350,7 @@ func prepSourceTestDownload( } if e.success { e.err = "" - e.delay = e.Source.cacheTTL + e.delay = DefaultPrefetchDelay } else { e.delay = MinimumPrefetchInterval } @@ -373,7 +371,7 @@ func setupSourceTestCase(t *testing.T, d *SourceTestData, i int, } e.Source = &Source{ name: id, urls: []*url.URL{}, format: SourceFormatV2, minisignKey: d.key, - cacheFile: e.cachePath, cacheTTL: DefaultPrefetchDelay * 3, + cacheFile: e.cachePath, cacheTTL: DefaultPrefetchDelay * 3, prefetchDelay: DefaultPrefetchDelay, } if cacheTest != nil { prepSourceTestCache(t, d, e, d.sources[i], *cacheTest) @@ -407,9 +405,9 @@ func TestNewSource(t *testing.T) { refreshDelay time.Duration e *SourceTestExpect }{ - {"", "", 0, &SourceTestExpect{err: " ", Source: &Source{name: "short refresh delay", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay, prefix: ""}}}, - {"v1", d.keyStr, DefaultPrefetchDelay * 2, &SourceTestExpect{err: "Unsupported source format", Source: &Source{name: "old format", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 2}}}, - {"v2", "", DefaultPrefetchDelay * 3, &SourceTestExpect{err: "Invalid encoded public key", Source: &Source{name: "invalid public key", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 3}}}, + {"", "", 0, &SourceTestExpect{err: " ", Source: &Source{name: "short refresh delay", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay, prefetchDelay: DefaultPrefetchDelay, prefix: ""}}}, + {"v1", d.keyStr, DefaultPrefetchDelay * 2, &SourceTestExpect{err: "Unsupported source format", Source: &Source{name: "old format", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 2, prefetchDelay: DefaultPrefetchDelay}}}, + {"v2", "", DefaultPrefetchDelay * 3, &SourceTestExpect{err: "Invalid encoded public key", Source: &Source{name: "invalid public key", urls: []*url.URL{}, cacheTTL: DefaultPrefetchDelay * 3, prefetchDelay: DefaultPrefetchDelay}}}, } { t.Run(tt.e.Source.name, func(t *testing.T) { got, err := NewSource( @@ -480,7 +478,6 @@ func TestPrefetchSources(t *testing.T) { s := &Source{} *s = *e.Source s.bin = nil - s.refresh = d.timeNow sources = append(sources, s) expects = append(expects, e) }