mediocregopher/deadlinks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add option to set http user agent

Browse Source
This commit is contained in:
Brian Picciano 2024-01-04 20:51:36 +01:00
parent f5a91f918e
commit db3e6029b9
2 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
client.go
View File

@ -36,6 +36,11 @@ type ClientOpts struct {
Do(*http.Request) (*http.Response, error) Do(*http.Request) (*http.Response, error)
} }
// HTTPUserAgent overwrites the user agent used by the HTTPClient.
//
// Defaults to whatever http.Client uses by default.
HTTPUserAgent string
// MaxRedirects indicates the maximum number of redirects which will be // MaxRedirects indicates the maximum number of redirects which will be
// allowed when resolving a resource. A negative value indicates no // allowed when resolving a resource. A negative value indicates no
// redirects are allowed. // redirects are allowed.
@ -190,6 +195,10 @@ func (c *client) getHTTP(
return "", nil, fmt.Errorf("building request: %w", err) return "", nil, fmt.Errorf("building request: %w", err)
} }
if c.opts.HTTPUserAgent != "" {
req.Header.Set("User-Agent", c.opts.HTTPUserAgent)
}
res, err := c.opts.HTTPClient.Do(req) res, err := c.opts.HTTPClient.Do(req)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("performing request: %w", err) return "", nil, fmt.Errorf("performing request: %w", err)

15
cmd/deadlinks/main.go
View File

@ -31,11 +31,12 @@ func (c loggingClient) Get(
func main() { func main() {
var ( var (
storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used") storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used")
maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path") maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path")
urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`) urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`)
patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well") patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well")
concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time") concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
httpUserAgent = flag.String("http-user-agent", "", "User-agent to use for http requests")
) )
flag.Parse() flag.Parse()
@ -64,7 +65,9 @@ func main() {
patterns, patterns,
&deadlinks.Opts{ &deadlinks.Opts{
NewClient: func() deadlinks.Client { NewClient: func() deadlinks.Client {
return loggingClient{deadlinks.NewClient(nil)} return loggingClient{deadlinks.NewClient(&deadlinks.ClientOpts{
HTTPUserAgent: *httpUserAgent,
})}
}, },
Concurrency: *concurrency, Concurrency: *concurrency,
OnError: func(err error) { OnError: func(err error) {