miracle091/dnscrypt-proxy
miracle091
/
dnscrypt-proxy
mirror of https://github.com/DNSCrypt/dnscrypt-proxy.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Support multiple URLs for a given source 

Fixes #265
This commit is contained in:
Frank Denis 2018-03-28 13:36:19 +02:00
parent 7ed4ce17d7
commit ede564ccf7
3 changed files with 50 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dnscrypt-proxy/config.go
View File

@ -89,6 +89,7 @@ type StaticConfig struct {
type SourceConfig struct {
URL string
URLs []string
MinisignKeyStr string `toml:"minisign_key"`
CacheFile string `toml:"cache_file"`
FormatStr string `toml:"format"`
@ -387,8 +388,12 @@ func (config *Config) loadSources(proxy *Proxy) error {
}
func (config *Config) loadSource(proxy *Proxy, requiredProps ServerInformalProperties, cfgSourceName string, cfgSource *SourceConfig) error {
if cfgSource.URL == "" {
dlog.Debugf("Missing URL for source [%s]", cfgSourceName)
if len(cfgSource.URLs) == 0 {
if len(cfgSource.URL) == 0 {
dlog.Debugf("Missing URLs for source [%s]", cfgSourceName)
} else {
cfgSource.URLs = []string{cfgSource.URL}
}
}
if cfgSource.MinisignKeyStr == "" {
return fmt.Errorf("Missing Minisign key for source [%s]", cfgSourceName)
@ -402,7 +407,7 @@ func (config *Config) loadSource(proxy *Proxy, requiredProps ServerInformalPrope
if cfgSource.RefreshDelay <= 0 {
cfgSource.RefreshDelay = 72
}
source, sourceUrlsToPrefetch, err := NewSource(proxy.xTransport, cfgSource.URL, cfgSource.MinisignKeyStr, cfgSource.CacheFile, cfgSource.FormatStr, time.Duration(cfgSource.RefreshDelay)*time.Hour)
source, sourceUrlsToPrefetch, err := NewSource(proxy.xTransport, cfgSource.URLs, cfgSource.MinisignKeyStr, cfgSource.CacheFile, cfgSource.FormatStr, time.Duration(cfgSource.RefreshDelay)*time.Hour)
proxy.urlsToPrefetch = append(proxy.urlsToPrefetch, sourceUrlsToPrefetch...)
if err != nil {
dlog.Criticalf("Unable use source [%s]: [%s]", cfgSourceName, err)

6
dnscrypt-proxy/example-dnscrypt-proxy.toml
View File

@ -363,7 +363,7 @@ cache_neg_ttl = 60
## different servers. In that case, names listed in `server_names`
## must include the prefixes.
##
## If the `url` property is missing, cache files and valid signatures
## If the `urls` property is missing, cache files and valid signatures
## must be already present; This doesn't prevent these cache files from
## expiring after `refresh_delay` hours.
@ -372,7 +372,7 @@ cache_neg_ttl = 60
## An example of a remote source
[sources.'public-resolvers']
url = 'https://download.dnscrypt.info/resolvers-list/v2/public-resolvers.md'
urls = ['https://raw.githubusercontent.com/DNSCrypt/dnscrypt-resolvers/master/v2/public-resolvers.md', 'https://download.dnscrypt.info/resolvers-list/v2/public-resolvers.md']
cache_file = 'public-resolvers.md'
minisign_key = 'RWQf6LRCGA9i53mlYecO4IzT51TGPpvWucNSCh1CBM0QTaLn73Y7GFO3'
refresh_delay = 72
@ -382,7 +382,7 @@ cache_neg_ttl = 60
## This is a subset of the `public-resolvers` list, so enabling both is useless
# [sources.'parental-control']
# url = 'https://download.dnscrypt.info/resolvers-list/v2/parental-control.md'
# urls = ['https://raw.githubusercontent.com/DNSCrypt/dnscrypt-resolvers/master/v2/parental-control.md', 'https://download.dnscrypt.info/resolvers-list/v2/parental-control.md']
# cache_file = 'parental-control.md'
# minisign_key = 'RWQf6LRCGA9i53mlYecO4IzT51TGPpvWucNSCh1CBM0QTaLn73Y7GFO3'

58
dnscrypt-proxy/sources.go
View File

@ -32,7 +32,7 @@ const (
)
type Source struct {
url string
urls []string
format SourceFormat
in string
}
@ -125,9 +125,9 @@ type URLToPrefetch struct {
when time.Time
}
func NewSource(xTransport *XTransport, url string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (Source, []URLToPrefetch, error) {
func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (Source, []URLToPrefetch, error) {
_ = refreshDelay
source := Source{url: url}
source := Source{urls: urls}
if formatStr == "v1" {
source.format = SourceFormatV1
} else if formatStr == "v2" {
@ -141,19 +141,39 @@ func NewSource(xTransport *XTransport, url string, minisignKeyStr string, cacheF
}
now := time.Now()
urlsToPrefetch := []URLToPrefetch{}
in, cached, delayTillNextUpdate, err := fetchWithCache(xTransport, url, cacheFile)
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: url, cacheFile: cacheFile, when: now.Add(delayTillNextUpdate)})
sigURL := url + ".minisig"
sigCacheFile := cacheFile + ".minisig"
sigStr, sigCached, sigDelayTillNextUpdate, sigErr := fetchWithCache(xTransport, sigURL, sigCacheFile)
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: sigURL, cacheFile: sigCacheFile, when: now.Add(sigDelayTillNextUpdate)})
if err != nil || sigErr != nil {
if err == nil {
err = sigErr
var sigStr, in string
var cached, sigCached bool
var delayTillNextUpdate, sigDelayTillNextUpdate time.Duration
var sigErr error
var preloadURL string
if len(urls) <= 0 {
in, cached, delayTillNextUpdate, err = fetchWithCache(xTransport, "", cacheFile)
sigStr, sigCached, sigDelayTillNextUpdate, sigErr = fetchWithCache(xTransport, "", sigCacheFile)
} else {
preloadURL = urls[0]
for _, url := range urls {
sigURL := url + ".minisig"
in, cached, delayTillNextUpdate, err = fetchWithCache(xTransport, url, cacheFile)
sigStr, sigCached, sigDelayTillNextUpdate, sigErr = fetchWithCache(xTransport, sigURL, sigCacheFile)
if err == nil && sigErr == nil {
preloadURL = url
break
}
dlog.Infof("Loading from [%s] failed", url)
}
}
if len(preloadURL) > 0 {
url := preloadURL
sigURL := url + ".minisig"
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: url, cacheFile: cacheFile, when: now.Add(delayTillNextUpdate)})
urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: sigURL, cacheFile: sigCacheFile, when: now.Add(sigDelayTillNextUpdate)})
}
if sigErr != nil && err == nil {
err = sigErr
}
if err != nil {
return source, urlsToPrefetch, err
}
@ -183,7 +203,7 @@ func NewSource(xTransport *XTransport, url string, minisignKeyStr string, cacheF
}
}
}
dlog.Noticef("Source [%s] loaded", url)
dlog.Noticef("Source [%s] loaded", cacheFile)
source.in = in
return source, urlsToPrefetch, nil
}
@ -246,18 +266,18 @@ func (source *Source) parseV2(prefix string) ([]RegisteredServer, error) {
in := string(source.in)
parts := strings.Split(in, "## ")
if len(parts) < 2 {
return registeredServers, fmt.Errorf("Invalid format for source at [%s]", source.url)
return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
}
parts = parts[1:]
for _, part := range parts {
part = strings.TrimFunc(part, unicode.IsSpace)
subparts := strings.Split(part, "\n")
if len(subparts) < 2 {
return registeredServers, fmt.Errorf("Invalid format for source at [%s]", source.url)
return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
}
name := strings.TrimFunc(subparts[0], unicode.IsSpace)
if len(name) == 0 {
return registeredServers, fmt.Errorf("Invalid format for source at [%s]", source.url)
return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
}
subparts = subparts[1:]
name = prefix + name
@ -266,7 +286,7 @@ func (source *Source) parseV2(prefix string) ([]RegisteredServer, error) {
subpart = strings.TrimFunc(subpart, unicode.IsSpace)
if strings.HasPrefix(subpart, "sdns://") {
if len(stampStr) > 0 {
return registeredServers, fmt.Errorf("Multiple stamps for server [%s] in source from [%s]", name, source.url)
return registeredServers, fmt.Errorf("Multiple stamps for server [%s] in source from [%v]", name, source.urls)
}
stampStr = subpart
continue
@ -279,7 +299,7 @@ func (source *Source) parseV2(prefix string) ([]RegisteredServer, error) {
description += subpart
}
if len(stampStr) < 8 {
return registeredServers, fmt.Errorf("Missing stamp for server [%s] in source from [%s]", name, source.url)
return registeredServers, fmt.Errorf("Missing stamp for server [%s] in source from [%v]", name, source.urls)
}
stamp, err := NewServerStampFromString(stampStr)
if err != nil {