|
|
|
@ -20,6 +20,7 @@ import (
|
|
|
|
"crypto/tls"
|
|
|
|
"crypto/tls"
|
|
|
|
"fmt"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io"
|
|
|
|
|
|
|
|
"log/slog"
|
|
|
|
"net/http"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
|
|
|
"net/url"
|
|
|
|
"sync"
|
|
|
|
"sync"
|
|
|
|
@ -87,11 +88,13 @@ func (g *HTTPGetter) get(href string, opts getterOptions) (*bytes.Buffer, error)
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
slog.Debug("fetching", "url", href)
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
|
|
|
slog.Debug("fetch complete", "url", href, "status", resp.Status, "content-length", resp.ContentLength)
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
return nil, fmt.Errorf("failed to fetch %s : %s", href, resp.Status)
|
|
|
|
return nil, fmt.Errorf("failed to fetch %s : %s", href, resp.Status)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|