Add deadlinks command

cmd/deadlinks/main.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"io"
+	"log"
+	"os"
+	"os/signal"
+	"runtime"
+	"strings"
+	"time"
+
+	"code.betamike.com/mediocregopher/deadlinks"
+	"code.betamike.com/mediocregopher/mediocre-go-lib/miter"
+	"gopkg.in/yaml.v3"
+)
+
+type loggingClient struct {
+	inner deadlinks.Client
+}
+
+func (c loggingClient) Get(
+	ctx context.Context, url deadlinks.URL,
+) (
+	string, io.ReadCloser, error,
+) {
+	log.Printf("querying %q", url)
+	return c.inner.Get(ctx, url)
+}
+
+func main() {
+	var (
+		storePath   = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used")
+		maxAge      = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path")
+		urls        = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`)
+		patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well")
+		concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time")
+	)
+
+	flag.Parse()
+
+	if *urls == "" {
+		log.Fatal("-urls is required")
+	}
+
+	var patterns []string
+	if *patternsStr != "" {
+		patterns = strings.Split(*patternsStr, ",")
+	}
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	store := deadlinks.NewSQLiteStore(&deadlinks.SQLiteStoreOpts{
+		Path: *storePath,
+	})
+	defer store.Close()
+
+	dl, err := deadlinks.New(
+		ctx,
+		store,
+		strings.Split(*urls, ","),
+		patterns,
+		&deadlinks.Opts{
+			NewClient: func() deadlinks.Client {
+				return loggingClient{deadlinks.NewClient(nil)}
+			},
+			Concurrency: *concurrency,
+			OnError: func(err error) {
+				log.Printf("runtime error: %v", err)
+			},
+		},
+	)
+
+	if err != nil {
+		log.Fatalf("initialization error: %v", err)
+	}
+
+	lastCheckedBefore := time.Now().Add(-*maxAge)
+
+	if err := dl.Update(ctx, lastCheckedBefore); err != nil {
+		log.Fatalf("update encountered error: %v", err)
+	}
+
+	enc := yaml.NewEncoder(os.Stdout)
+	defer os.Stdout.Sync()
+
+	iter := dl.GetByStatus(deadlinks.ResourceStatusError)
+	err = miter.ForEach(ctx, iter, func(r deadlinks.Resource) error {
+		return enc.Encode(r)
+	})
+
+	if err != nil {
+		log.Fatalf("iterating over errored resources failed: %v", err)
+	}
+}