Add option to set http user agent

client.go

@@ -36,6 +36,11 @@ type ClientOpts struct {
 		Do(*http.Request) (*http.Response, error)
 	}
 
+	// HTTPUserAgent overwrites the user agent used by the HTTPClient.
+	//
+	// Defaults to whatever http.Client uses by default.
+	HTTPUserAgent string
+
 	// MaxRedirects indicates the maximum number of redirects which will be
 	// allowed when resolving a resource. A negative value indicates no
 	// redirects are allowed.
@@ -190,6 +195,10 @@ func (c *client) getHTTP(
 		return "", nil, fmt.Errorf("building request: %w", err)
 	}
 
+	if c.opts.HTTPUserAgent != "" {
+		req.Header.Set("User-Agent", c.opts.HTTPUserAgent)
+	}
+
 	res, err := c.opts.HTTPClient.Do(req)
 	if err != nil {
 		return "", nil, fmt.Errorf("performing request: %w", err)

cmd/deadlinks/main.go

@@ -36,6 +36,7 @@ func main() {
 		urls          = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`)
 		patternsStr   = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well")
 		concurrency   = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
+		httpUserAgent = flag.String("http-user-agent", "", "User-agent to use for http requests")
 	)
 
 	flag.Parse()
@@ -64,7 +65,9 @@ func main() {
 		patterns,
 		&deadlinks.Opts{
 			NewClient: func() deadlinks.Client {
-				return loggingClient{deadlinks.NewClient(nil)}
+				return loggingClient{deadlinks.NewClient(&deadlinks.ClientOpts{
+					HTTPUserAgent: *httpUserAgent,
+				})}
 			},
 			Concurrency: *concurrency,
 			OnError: func(err error) {