parent
eaccb83a7b
commit
07a5acceaf
@ -0,0 +1,97 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"flag" |
||||
"io" |
||||
"log" |
||||
"os" |
||||
"os/signal" |
||||
"runtime" |
||||
"strings" |
||||
"time" |
||||
|
||||
"code.betamike.com/mediocregopher/deadlinks" |
||||
"code.betamike.com/mediocregopher/mediocre-go-lib/miter" |
||||
"gopkg.in/yaml.v3" |
||||
) |
||||
|
||||
type loggingClient struct { |
||||
inner deadlinks.Client |
||||
} |
||||
|
||||
func (c loggingClient) Get( |
||||
ctx context.Context, url deadlinks.URL, |
||||
) ( |
||||
string, io.ReadCloser, error, |
||||
) { |
||||
log.Printf("querying %q", url) |
||||
return c.inner.Get(ctx, url) |
||||
} |
||||
|
||||
func main() { |
||||
var ( |
||||
storePath = flag.String("store-path", "", "Path to sqlite storage file. If not given then a temporary in-memory storage is used") |
||||
maxAge = flag.Duration("max-age", 0, "Maximum duration since last check of a resource, before it must be checked again. Must be used with -store-path") |
||||
urls = flag.String("urls", "", `Comma-separated list of URLs which are always checked. At least one is required`) |
||||
patternsStr = flag.String("patterns", "", "Comma-separated list of regexps. All URLs which match one of these will have their links checked as well") |
||||
concurrency = flag.Int("concurrency", runtime.NumCPU()/2, "Number simultaneous requests to make at a time") |
||||
) |
||||
|
||||
flag.Parse() |
||||
|
||||
if *urls == "" { |
||||
log.Fatal("-urls is required") |
||||
} |
||||
|
||||
var patterns []string |
||||
if *patternsStr != "" { |
||||
patterns = strings.Split(*patternsStr, ",") |
||||
} |
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) |
||||
defer cancel() |
||||
|
||||
store := deadlinks.NewSQLiteStore(&deadlinks.SQLiteStoreOpts{ |
||||
Path: *storePath, |
||||
}) |
||||
defer store.Close() |
||||
|
||||
dl, err := deadlinks.New( |
||||
ctx, |
||||
store, |
||||
strings.Split(*urls, ","), |
||||
patterns, |
||||
&deadlinks.Opts{ |
||||
NewClient: func() deadlinks.Client { |
||||
return loggingClient{deadlinks.NewClient(nil)} |
||||
}, |
||||
Concurrency: *concurrency, |
||||
OnError: func(err error) { |
||||
log.Printf("runtime error: %v", err) |
||||
}, |
||||
}, |
||||
) |
||||
|
||||
if err != nil { |
||||
log.Fatalf("initialization error: %v", err) |
||||
} |
||||
|
||||
lastCheckedBefore := time.Now().Add(-*maxAge) |
||||
|
||||
if err := dl.Update(ctx, lastCheckedBefore); err != nil { |
||||
log.Fatalf("update encountered error: %v", err) |
||||
} |
||||
|
||||
enc := yaml.NewEncoder(os.Stdout) |
||||
defer os.Stdout.Sync() |
||||
|
||||
iter := dl.GetByStatus(deadlinks.ResourceStatusError) |
||||
err = miter.ForEach(ctx, iter, func(r deadlinks.Resource) error { |
||||
return enc.Encode(r) |
||||
}) |
||||
|
||||
if err != nil { |
||||
log.Fatalf("iterating over errored resources failed: %v", err) |
||||
} |
||||
} |
Loading…
Reference in new issue