mediocregopher/deadlinks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add option to set http user agent

Browse Source
This commit is contained in:
Brian Picciano 2024-01-04 20:51:36 +01:00
parent f5a91f918e
commit db3e6029b9
2 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
client.go
View File

@ -36,6 +36,11 @@ type ClientOpts struct {
Do(*http.Request) (*http.Response, error)
}
// HTTPUserAgent overwrites the user agent used by the HTTPClient.
//
// Defaults to whatever http.Client uses by default.
HTTPUserAgent string
// MaxRedirects indicates the maximum number of redirects which will be
// allowed when resolving a resource. A negative value indicates no
// redirects are allowed.
@ -190,6 +195,10 @@ func (c *client) getHTTP(
return "", nil, fmt.Errorf("building request: %w", err)
}
if c.opts.HTTPUserAgent != "" {
req.Header.Set("User-Agent", c.opts.HTTPUserAgent)
}
res, err := c.opts.HTTPClient.Do(req)
if err != nil {
return "", nil, fmt.Errorf("performing request: %w", err)

15
cmd/deadlinks/main.go
View File

@ -31,11 +31,12 @@ func (c loggingClient) Get(
func main() {
var (
storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used")
maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path")
urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`)
patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well")
concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used")
maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path")
urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`)
patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well")
concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
httpUserAgent = flag.String("http-user-agent", "", "User-agent to use for http requests")
)
flag.Parse()
@ -64,7 +65,9 @@ func main() {
patterns,
&deadlinks.Opts{
NewClient: func() deadlinks.Client {
return loggingClient{deadlinks.NewClient(nil)}
return loggingClient{deadlinks.NewClient(&deadlinks.ClientOpts{
HTTPUserAgent: *httpUserAgent,
})}
},
Concurrency: *concurrency,
OnError: func(err error) {