miracle091/dnscrypt-proxy
miracle091
/
dnscrypt-proxy
mirror of https://github.com/DNSCrypt/dnscrypt-proxy.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Move prefetch URLs onto Source struct 

This is mostly in preparation for further refactoring, but does reduce the number of return values from `NewSource()` too.
This commit is contained in:
William Elwood 2019-10-31 01:04:08 +00:00 committed by Frank Denis
parent 4a792026eb
commit 78f2dead79
4 changed files with 31 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dnscrypt-proxy/config.go
View File

@ -613,12 +613,12 @@ func (config *Config) loadSource(proxy *Proxy, requiredProps stamps.ServerInform
if cfgSource.RefreshDelay <= 0 { if cfgSource.RefreshDelay <= 0 {
cfgSource.RefreshDelay = 72 cfgSource.RefreshDelay = 72
} }
source, sourceUrlsToPrefetch, err := NewSource(proxy.xTransport, cfgSource.URLs, cfgSource.MinisignKeyStr, cfgSource.CacheFile, cfgSource.FormatStr, time.Duration(cfgSource.RefreshDelay)*time.Hour) source, err := NewSource(proxy.xTransport, cfgSource.URLs, cfgSource.MinisignKeyStr, cfgSource.CacheFile, cfgSource.FormatStr, time.Duration(cfgSource.RefreshDelay)*time.Hour)
proxy.urlsToPrefetch = append(proxy.urlsToPrefetch, sourceUrlsToPrefetch...)
if err != nil { if err != nil {
dlog.Criticalf("Unable to retrieve source [%s]: [%s]", cfgSourceName, err) dlog.Criticalf("Unable to retrieve source [%s]: [%s]", cfgSourceName, err)
return err return err
} }
proxy.urlsToPrefetch = append(proxy.urlsToPrefetch, source.prefetch...)
registeredServers, err := source.Parse(cfgSource.Prefix) registeredServers, err := source.Parse(cfgSource.Prefix)
if err != nil { if err != nil {
if len(registeredServers) == 0 { if len(registeredServers) == 0 {

9
dnscrypt-proxy/proxy.go
View File

@ -60,7 +60,7 @@ type Proxy struct {
forwardFile string forwardFile string
cloakFile string cloakFile string
pluginsGlobals PluginsGlobals pluginsGlobals PluginsGlobals
urlsToPrefetch []URLToPrefetch urlsToPrefetch []*URLToPrefetch
clientsCount uint32 clientsCount uint32
maxClients uint32 maxClients uint32
xTransport *XTransport xTransport *XTransport
@ -177,7 +177,7 @@ func (proxy *Proxy) StartProxy() {
dlog.Error(err) dlog.Error(err)
dlog.Notice("dnscrypt-proxy is waiting for at least one server to be reachable") dlog.Notice("dnscrypt-proxy is waiting for at least one server to be reachable")
} }
proxy.prefetcher(&proxy.urlsToPrefetch) proxy.prefetcher(proxy.urlsToPrefetch)
if len(proxy.serversInfo.registeredServers) > 0 { if len(proxy.serversInfo.registeredServers) > 0 {
go func() { go func() {
for { for {
@ -195,12 +195,11 @@ func (proxy *Proxy) StartProxy() {
} }
} }
func (proxy *Proxy) prefetcher(urlsToPrefetch *[]URLToPrefetch) { func (proxy *Proxy) prefetcher(urlsToPrefetch []*URLToPrefetch) {
go func() { go func() {
for { for {
now := time.Now() now := time.Now()
for i := range *urlsToPrefetch { for _, urlToPrefetch := range urlsToPrefetch {
urlToPrefetch := &(*urlsToPrefetch)[i]
if now.After(urlToPrefetch.when) { if now.After(urlToPrefetch.when) {
dlog.Debugf("Prefetching [%s]", urlToPrefetch.url) dlog.Debugf("Prefetching [%s]", urlToPrefetch.url)
if err := PrefetchSourceURL(proxy.xTransport, urlToPrefetch); err != nil { if err := PrefetchSourceURL(proxy.xTransport, urlToPrefetch); err != nil {

13
dnscrypt-proxy/sources.go
View File

@ -31,6 +31,7 @@ const (
type Source struct { type Source struct {
urls []string urls []string
prefetch []*URLToPrefetch
format SourceFormat format SourceFormat
in []byte in []byte
minisignKey *minisign.PublicKey minisignKey *minisign.PublicKey
@ -118,21 +119,21 @@ type URLToPrefetch struct {
when time.Time when time.Time
} }
func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (source Source, urlsToPrefetch []URLToPrefetch, err error) { func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (source Source, err error) {
source = Source{urls: urls} source = Source{urls: urls}
urlsToPrefetch = []URLToPrefetch{}
if formatStr == "v2" { if formatStr == "v2" {
source.format = SourceFormatV2 source.format = SourceFormatV2
} else { } else {
return source, []URLToPrefetch{}, fmt.Errorf("Unsupported source format: [%s]", formatStr) return source, fmt.Errorf("Unsupported source format: [%s]", formatStr)
} }
if minisignKey, err := minisign.NewPublicKey(minisignKeyStr); err == nil { if minisignKey, err := minisign.NewPublicKey(minisignKeyStr); err == nil {
source.minisignKey = &minisignKey source.minisignKey = &minisignKey
} else { } else {
return source, urlsToPrefetch, err return source, err
} }
now := timeNow() now := timeNow()
sigCacheFile := cacheFile + ".minisig" sigCacheFile := cacheFile + ".minisig"
source.prefetch = []*URLToPrefetch{}
var bin, sig []byte var bin, sig []byte
var delayTillNextUpdate, sigDelayTillNextUpdate time.Duration var delayTillNextUpdate, sigDelayTillNextUpdate time.Duration
@ -157,8 +158,8 @@ func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cac
if len(preloadURL) > 0 { if len(preloadURL) > 0 {
url := preloadURL url := preloadURL
sigURL := url + ".minisig" sigURL := url + ".minisig"
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: url, cacheFile: cacheFile, when: now.Add(delayTillNextUpdate)}) source.prefetch = append(source.prefetch, &URLToPrefetch{url: url, cacheFile: cacheFile, when: now.Add(delayTillNextUpdate)})
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: sigURL, cacheFile: sigCacheFile, when: now.Add(sigDelayTillNextUpdate)}) source.prefetch = append(source.prefetch, &URLToPrefetch{url: sigURL, cacheFile: sigCacheFile, when: now.Add(sigDelayTillNextUpdate)})
} }
if sigErr != nil && err == nil { if sigErr != nil && err == nil {
err = sigErr err = sigErr

42
dnscrypt-proxy/sources_test.go
View File

@ -55,12 +55,9 @@ type SourceTestData struct {
type SourceTestExpect struct { type SourceTestExpect struct {
success, download bool success, download bool
in []byte
cachePath string cachePath string
cache []SourceFixture cache []SourceFixture
refresh time.Time refresh time.Time
urls []string
prefetchUrls []URLToPrefetch
Source *Source Source *Source
err string err string
} }
@ -249,9 +246,9 @@ func prepSourceTestCache(t *testing.T, d *SourceTestData, e *SourceTestExpect, s
e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]} e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]}
switch state { switch state {
case TestStateCorrect: case TestStateCorrect:
e.in, e.success, e.refresh = e.cache[0].content, true, d.timeUpd e.Source.in, e.success, e.refresh = e.cache[0].content, true, d.timeUpd
case TestStateExpired: case TestStateExpired:
e.in = e.cache[0].content e.Source.in = e.cache[0].content
case TestStatePartial, TestStatePartialSig: case TestStatePartial, TestStatePartialSig:
e.err = "signature" e.err = "signature"
case TestStateMissing, TestStateMissingSig: case TestStateMissing, TestStateMissingSig:
@ -270,7 +267,7 @@ func prepSourceTestDownload(t *testing.T, d *SourceTestData, e *SourceTestExpect
switch state { switch state {
case TestStateCorrect: case TestStateCorrect:
e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]} e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]}
e.in, e.success, e.refresh = e.cache[0].content, true, d.timeUpd e.Source.in, e.success, e.refresh = e.cache[0].content, true, d.timeUpd
fallthrough fallthrough
case TestStateMissingSig, TestStatePartial, TestStatePartialSig, TestStateReadSigErr: case TestStateMissingSig, TestStatePartial, TestStatePartialSig, TestStateReadSigErr:
d.reqExpect[path+".minisig"]++ d.reqExpect[path+".minisig"]++
@ -293,10 +290,10 @@ func prepSourceTestDownload(t *testing.T, d *SourceTestData, e *SourceTestExpect
path = "..." + path // non-numeric port fails URL parsing path = "..." + path // non-numeric port fails URL parsing
e.err = "parse" e.err = "parse"
} }
e.urls = append(e.urls, d.server.URL+path) e.Source.urls = append(e.Source.urls, d.server.URL+path)
if state != TestStatePathErr { if state != TestStatePathErr {
e.prefetchUrls = append(e.prefetchUrls, URLToPrefetch{d.server.URL + path, e.cachePath, e.refresh}) e.Source.prefetch = append(e.Source.prefetch, &URLToPrefetch{d.server.URL + path, e.cachePath, e.refresh})
e.prefetchUrls = append(e.prefetchUrls, URLToPrefetch{d.server.URL + path + ".minisig", e.cachePath + ".minisig", e.refresh}) e.Source.prefetch = append(e.Source.prefetch, &URLToPrefetch{d.server.URL + path + ".minisig", e.cachePath + ".minisig", e.refresh})
} }
} }
if e.success { if e.success {
@ -309,32 +306,29 @@ func setupSourceTestCase(t *testing.T, d *SourceTestData, i int,
cacheTest *SourceTestState, downloadTest []SourceTestState) (id string, e *SourceTestExpect) { cacheTest *SourceTestState, downloadTest []SourceTestState) (id string, e *SourceTestExpect) {
id = strconv.Itoa(d.n) + "-" + strconv.Itoa(i) id = strconv.Itoa(d.n) + "-" + strconv.Itoa(i)
e = &SourceTestExpect{ e = &SourceTestExpect{
cachePath: filepath.Join(d.tempDir, id), cachePath: filepath.Join(d.tempDir, id),
refresh: d.timeNow, refresh: d.timeNow,
urls: []string{}, Source: &Source{urls: []string{}, prefetch: []*URLToPrefetch{}, format: SourceFormatV2, minisignKey: d.key},
prefetchUrls: []URLToPrefetch{},
} }
if cacheTest != nil { if cacheTest != nil {
prepSourceTestCache(t, d, e, d.sources[i], *cacheTest) prepSourceTestCache(t, d, e, d.sources[i], *cacheTest)
i = (i + 1) % len(d.sources) // make the cached and downloaded fixtures different i = (i + 1) % len(d.sources) // make the cached and downloaded fixtures different
} }
prepSourceTestDownload(t, d, e, d.sources[i], downloadTest) prepSourceTestDownload(t, d, e, d.sources[i], downloadTest)
e.Source = &Source{e.urls, SourceFormatV2, e.in, d.key}
return return
} }
func TestNewSource(t *testing.T) { func TestNewSource(t *testing.T) {
teardown, d := setupSourceTest(t) teardown, d := setupSourceTest(t)
defer teardown() defer teardown()
doTest := func(t *testing.T, e *SourceTestExpect, got Source, urls []URLToPrefetch, err error) { doTest := func(t *testing.T, e *SourceTestExpect, got Source, err error) {
c := check.T(t) c := check.T(t)
if len(e.err) > 0 { if len(e.err) > 0 {
c.Match(err, e.err, "Unexpected error") c.Match(err, e.err, "Unexpected error")
} else { } else {
c.Nil(err, "Unexpected error") c.Nil(err, "Unexpected error")
} }
c.DeepEqual(got, *e.Source, "Unexpected return Source") c.DeepEqual(got, *e.Source, "Unexpected return")
c.DeepEqual(urls, e.prefetchUrls, "Unexpected return prefetch URLs")
checkTestServer(c, d) checkTestServer(c, d)
checkSourceCache(c, e.cachePath, e.cache) checkSourceCache(c, e.cachePath, e.cache)
} }
@ -345,13 +339,13 @@ func TestNewSource(t *testing.T) {
e *SourceTestExpect e *SourceTestExpect
}{ }{
{"old format", d.keyStr, "v1", MinSourcesUpdateDelay * 3, &SourceTestExpect{ {"old format", d.keyStr, "v1", MinSourcesUpdateDelay * 3, &SourceTestExpect{
Source: &Source{urls: nil}, prefetchUrls: []URLToPrefetch{}, err: "Unsupported source format"}}, Source: &Source{}, err: "Unsupported source format"}},
{"invalid public key", "", "v2", MinSourcesUpdateDelay * 3, &SourceTestExpect{ {"invalid public key", "", "v2", MinSourcesUpdateDelay * 3, &SourceTestExpect{
Source: &Source{urls: nil}, prefetchUrls: []URLToPrefetch{}, err: "Invalid encoded public key"}}, Source: &Source{}, err: "Invalid encoded public key"}},
} { } {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
got, urls, err := NewSource(d.xTransport, tt.e.urls, tt.key, tt.e.cachePath, tt.v, tt.refresh) got, err := NewSource(d.xTransport, tt.e.Source.urls, tt.key, tt.e.cachePath, tt.v, tt.refresh)
doTest(t, tt.e, got, urls, err) doTest(t, tt.e, got, err)
}) })
} }
for cacheTestName, cacheTest := range d.cacheTests { for cacheTestName, cacheTest := range d.cacheTests {
@ -360,8 +354,8 @@ func TestNewSource(t *testing.T) {
for i := range d.sources { for i := range d.sources {
id, e := setupSourceTestCase(t, d, i, &cacheTest, downloadTest) id, e := setupSourceTestCase(t, d, i, &cacheTest, downloadTest)
t.Run("cache "+cacheTestName+", download "+downloadTestName+"/"+id, func(t *testing.T) { t.Run("cache "+cacheTestName+", download "+downloadTestName+"/"+id, func(t *testing.T) {
got, urls, err := NewSource(d.xTransport, e.urls, d.keyStr, e.cachePath, "v2", MinSourcesUpdateDelay*3) got, err := NewSource(d.xTransport, e.Source.urls, d.keyStr, e.cachePath, "v2", MinSourcesUpdateDelay*3)
doTest(t, e, got, urls, err) doTest(t, e, got, err)
}) })
} }
} }
@ -374,7 +368,7 @@ func TestPrefetchSourceURL(t *testing.T) {
doTest := func(t *testing.T, expects []*SourceTestExpect) { doTest := func(t *testing.T, expects []*SourceTestExpect) {
c := check.T(t) c := check.T(t)
for _, e := range expects { for _, e := range expects {
for _, url := range e.urls { for _, url := range e.Source.urls {
for _, suffix := range []string{"", ".minisig"} { for _, suffix := range []string{"", ".minisig"} {
pf := &URLToPrefetch{url + suffix, e.cachePath + suffix, d.timeOld} pf := &URLToPrefetch{url + suffix, e.cachePath + suffix, d.timeOld}
PrefetchSourceURL(d.xTransport, pf) PrefetchSourceURL(d.xTransport, pf)