You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
94 lines
2.4 KiB
94 lines
2.4 KiB
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"runtime"
|
|
"time"
|
|
|
|
"code.betamike.com/mediocregopher/deadlinks"
|
|
"code.betamike.com/mediocregopher/mediocre-go-lib/miter"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type loggingClient struct {
|
|
inner deadlinks.Client
|
|
}
|
|
|
|
func (c loggingClient) Get(
|
|
ctx context.Context, url deadlinks.URL,
|
|
) (
|
|
string, io.ReadCloser, error,
|
|
) {
|
|
log.Printf("querying %q", url)
|
|
return c.inner.Get(ctx, url)
|
|
}
|
|
|
|
func main() {
|
|
var (
|
|
storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used")
|
|
maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path")
|
|
urls = flagStrings("url", "URL which is always checked. Must be given at least once")
|
|
follows = flagStrings("follow", "URLs matching this regex will have their links checked as well. Can be specified multiple times")
|
|
concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
|
|
httpUserAgent = flag.String("http-user-agent", "", "User-agent to use for http requests")
|
|
)
|
|
|
|
flag.Parse()
|
|
|
|
if len(*urls.strs) == 0 {
|
|
log.Fatal("at least one -url is required")
|
|
}
|
|
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
|
defer cancel()
|
|
|
|
store := deadlinks.NewSQLiteStore(&deadlinks.SQLiteStoreOpts{
|
|
Path: *storePath,
|
|
})
|
|
defer store.Close()
|
|
|
|
dl, err := deadlinks.New(
|
|
ctx,
|
|
store,
|
|
*urls.strs,
|
|
&deadlinks.Opts{
|
|
NewClient: func() deadlinks.Client {
|
|
return loggingClient{deadlinks.NewClient(&deadlinks.ClientOpts{
|
|
HTTPUserAgent: *httpUserAgent,
|
|
})}
|
|
},
|
|
FollowRegexps: *follows.strs,
|
|
Concurrency: *concurrency,
|
|
OnError: func(err error) {
|
|
log.Printf("runtime error: %v", err)
|
|
},
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatalf("initialization error: %v", err)
|
|
}
|
|
|
|
lastCheckedBefore := time.Now().Add(-*maxAge)
|
|
|
|
if err := dl.Update(ctx, lastCheckedBefore); err != nil {
|
|
log.Fatalf("update encountered error: %v", err)
|
|
}
|
|
|
|
enc := yaml.NewEncoder(os.Stdout)
|
|
defer os.Stdout.Sync()
|
|
|
|
iter := dl.GetByStatus(deadlinks.ResourceStatusError)
|
|
err = miter.ForEach(ctx, iter, func(r deadlinks.Resource) error {
|
|
return enc.Encode(r)
|
|
})
|
|
|
|
if err != nil {
|
|
log.Fatalf("iterating over errored resources failed: %v", err)
|
|
}
|
|
}
|
|
|