Move most of the prefetching code into sources.go

The proxy shouldn't need to know how prefetching works, just that it needs to do it occasionally. Now the prefetching algorithm can be refactored without having to touch the proxy code.

dnscrypt-proxy/config.go

@@ -618,7 +618,7 @@ func (config *Config) loadSource(proxy *Proxy, requiredProps stamps.ServerInform
 		dlog.Criticalf("Unable to retrieve source [%s]: [%s]", cfgSourceName, err)
 		return err
 	}
-	proxy.urlsToPrefetch = append(proxy.urlsToPrefetch, source.prefetch...)
+	proxy.sources = append(proxy.sources, source)
 	registeredServers, err := source.Parse(cfgSource.Prefix)
 	if err != nil {
 		if len(registeredServers) == 0 {

dnscrypt-proxy/proxy.go

@@ -60,7 +60,7 @@ type Proxy struct {
 	forwardFile                  string
 	cloakFile                    string
 	pluginsGlobals               PluginsGlobals
-	urlsToPrefetch               []*URLToPrefetch
+	sources                      []*Source
 	clientsCount                 uint32
 	maxClients                   uint32
 	xTransport                   *XTransport
@@ -177,7 +177,11 @@ func (proxy *Proxy) StartProxy() {
 		dlog.Error(err)
 		dlog.Notice("dnscrypt-proxy is waiting for at least one server to be reachable")
 	}
-	proxy.prefetcher(proxy.urlsToPrefetch)
+	go func() {
+		for {
+			clocksmith.Sleep(PrefetchSources(proxy.xTransport, proxy.sources))
+		}
+	}()
 	if len(proxy.serversInfo.registeredServers) > 0 {
 		go func() {
 			for {
@@ -195,25 +199,6 @@ func (proxy *Proxy) StartProxy() {
 	}
 }
 
-func (proxy *Proxy) prefetcher(urlsToPrefetch []*URLToPrefetch) {
-	go func() {
-		for {
-			now := time.Now()
-			for _, urlToPrefetch := range urlsToPrefetch {
-				if now.After(urlToPrefetch.when) {
-					dlog.Debugf("Prefetching [%s]", urlToPrefetch.url)
-					if err := PrefetchSourceURL(proxy.xTransport, urlToPrefetch); err != nil {
-						dlog.Debugf("Prefetching [%s] failed: %s", urlToPrefetch.url, err)
-					} else {
-						dlog.Debugf("Prefetching [%s] succeeded. Next refresh scheduled for %v", urlToPrefetch.url, urlToPrefetch.when)
-					}
-				}
-			}
-			clocksmith.Sleep(60 * time.Second)
-		}
-	}()
-}
-
 func (proxy *Proxy) udpListener(clientPc *net.UDPConn) {
 	defer clientPc.Close()
 	for {

dnscrypt-proxy/sources.go

@@ -119,8 +119,8 @@ type URLToPrefetch struct {
 	when      time.Time
 }
 
-func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (source Source, err error) {
-	source = Source{urls: urls}
+func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (source *Source, err error) {
+	source = &Source{urls: urls}
 	if formatStr == "v2" {
 		source.format = SourceFormatV2
 	} else {
@@ -176,6 +176,23 @@ func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cac
 	return
 }
 
+func PrefetchSources(xTransport *XTransport, sources []*Source) time.Duration {
+	now := timeNow()
+	for _, source := range sources {
+		for _, urlToPrefetch := range source.prefetch {
+			if now.After(urlToPrefetch.when) {
+				dlog.Debugf("Prefetching [%s]", urlToPrefetch.url)
+				if err := PrefetchSourceURL(xTransport, urlToPrefetch); err != nil {
+					dlog.Debugf("Prefetching [%s] failed: %s", urlToPrefetch.url, err)
+				} else {
+					dlog.Debugf("Prefetching [%s] succeeded. Next refresh scheduled for %v", urlToPrefetch.url, urlToPrefetch.when)
+				}
+			}
+		}
+	}
+	return 60 * time.Second
+}
+
 func (source *Source) Parse(prefix string) ([]RegisteredServer, error) {
 	if source.format == SourceFormatV2 {
 		return source.parseV2(prefix)

dnscrypt-proxy/sources_test.go

@@ -321,14 +321,14 @@ func setupSourceTestCase(t *testing.T, d *SourceTestData, i int,
 func TestNewSource(t *testing.T) {
 	teardown, d := setupSourceTest(t)
 	defer teardown()
-	doTest := func(t *testing.T, e *SourceTestExpect, got Source, err error) {
+	doTest := func(t *testing.T, e *SourceTestExpect, got *Source, err error) {
 		c := check.T(t)
 		if len(e.err) > 0 {
 			c.Match(err, e.err, "Unexpected error")
 		} else {
 			c.Nil(err, "Unexpected error")
 		}
-		c.DeepEqual(got, *e.Source, "Unexpected return")
+		c.DeepEqual(got, e.Source, "Unexpected return")
 		checkTestServer(c, d)
 		checkSourceCache(c, e.cachePath, e.cache)
 	}