diff --git a/cmd/deadlinks/main.go b/cmd/deadlinks/main.go new file mode 100644 index 0000000..767c9ff --- /dev/null +++ b/cmd/deadlinks/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "flag" + "io" + "log" + "os" + "os/signal" + "runtime" + "strings" + "time" + + "code.betamike.com/mediocregopher/deadlinks" + "code.betamike.com/mediocregopher/mediocre-go-lib/miter" + "gopkg.in/yaml.v3" +) + +type loggingClient struct { + inner deadlinks.Client +} + +func (c loggingClient) Get( + ctx context.Context, url deadlinks.URL, +) ( + string, io.ReadCloser, error, +) { + log.Printf("querying %q", url) + return c.inner.Get(ctx, url) +} + +func main() { + var ( + storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used") + maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path") + urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`) + patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well") + concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time") + ) + + flag.Parse() + + if *urls == "" { + log.Fatal("-urls is required") + } + + var patterns []string + if *patternsStr != "" { + patterns = strings.Split(*patternsStr, ",") + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + store := deadlinks.NewSQLiteStore(&deadlinks.SQLiteStoreOpts{ + Path: *storePath, + }) + defer store.Close() + + dl, err := deadlinks.New( + ctx, + store, + strings.Split(*urls, ","), + patterns, + &deadlinks.Opts{ + NewClient: func() deadlinks.Client { + return loggingClient{deadlinks.NewClient(nil)} + }, + Concurrency: *concurrency, + OnError: func(err error) { + log.Printf("runtime error: %v", err) + }, + }, + ) + + if err != nil { + log.Fatalf("initialization error: %v", err) + } + + lastCheckedBefore := time.Now().Add(-*maxAge) + + if err := dl.Update(ctx, lastCheckedBefore); err != nil { + log.Fatalf("update encountered error: %v", err) + } + + enc := yaml.NewEncoder(os.Stdout) + defer os.Stdout.Sync() + + iter := dl.GetByStatus(deadlinks.ResourceStatusError) + err = miter.ForEach(ctx, iter, func(r deadlinks.Resource) error { + return enc.Encode(r) + }) + + if err != nil { + log.Fatalf("iterating over errored resources failed: %v", err) + } +}