Refactor reading a URL's content to own function

No longer shadows `url` package with variable of the same name.
This commit is contained in:
William Elwood 2019-10-30 04:12:50 +00:00 committed by Frank Denis
parent d851c9eeb6
commit e818eeb800
1 changed files with 12 additions and 11 deletions

View File

@ -79,6 +79,15 @@ func fetchFromCache(cacheFile string, refreshDelay time.Duration) (in string, ex
return return
} }
func fetchFromURL(xTransport *XTransport, u *url.URL) (bin []byte, err error) {
var resp *http.Response
if resp, _, err = xTransport.Get(u, "", DefaultTimeout); err == nil {
bin, err = ioutil.ReadAll(io.LimitReader(resp.Body, MaxHTTPBodyLength))
resp.Body.Close()
}
return
}
func fetchWithCache(xTransport *XTransport, urlStr string, cacheFile string, refreshDelay time.Duration) (in string, cached bool, delayTillNextUpdate time.Duration, err error) { func fetchWithCache(xTransport *XTransport, urlStr string, cacheFile string, refreshDelay time.Duration) (in string, cached bool, delayTillNextUpdate time.Duration, err error) {
cached = false cached = false
expired := false expired := false
@ -98,24 +107,16 @@ func fetchWithCache(xTransport *XTransport, urlStr string, cacheFile string, ref
return return
} }
var resp *http.Response
dlog.Infof("Loading source information from URL [%s]", urlStr) dlog.Infof("Loading source information from URL [%s]", urlStr)
url, err := url.Parse(urlStr) var u *url.URL
if err != nil { if u, err = url.Parse(urlStr); err != nil {
return
}
resp, _, err = xTransport.Get(url, "", 30*time.Second)
if err != nil {
return return
} }
var bin []byte var bin []byte
bin, err = ioutil.ReadAll(io.LimitReader(resp.Body, MaxHTTPBodyLength)) if bin, err = fetchFromURL(xTransport, u); err != nil {
resp.Body.Close()
if err != nil {
return return
} }
err = nil
cached = false cached = false
in = string(bin) in = string(bin)
delayTillNextUpdate = MinSourcesUpdateDelay delayTillNextUpdate = MinSourcesUpdateDelay