|
|
|
@ -14,8 +14,9 @@ import ( |
|
|
|
|
migrate "github.com/rubenv/sql-migrate" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
// Store keeps track of the current status of all discovered Resources.
|
|
|
|
|
// Resources with no incoming links will be periodically cleaned out.
|
|
|
|
|
// Store keeps track of the current status of all discovered Resources, and
|
|
|
|
|
// links between them. A Resource which is neither pinned nor linked to from
|
|
|
|
|
// another Resource is considered to not exist.
|
|
|
|
|
//
|
|
|
|
|
// An implementation of Store must be thread-safe.
|
|
|
|
|
type Store interface { |
|
|
|
@ -30,8 +31,10 @@ type Store interface { |
|
|
|
|
SetPinned(context.Context, []URL) error |
|
|
|
|
|
|
|
|
|
// Update updates the Resource identified by the given URL with the given
|
|
|
|
|
// arguments. The Resource must have been Touch'd previously, or this
|
|
|
|
|
// returns an error.
|
|
|
|
|
// arguments.
|
|
|
|
|
//
|
|
|
|
|
// Update returns an error if the URL has not been pinned nor referenced as
|
|
|
|
|
// an outgoing URL of a different Resource.
|
|
|
|
|
Update( |
|
|
|
|
ctx context.Context, |
|
|
|
|
now time.Time, |
|
|
|
@ -40,6 +43,9 @@ type Store interface { |
|
|
|
|
errorString string, |
|
|
|
|
outgoing []URL, |
|
|
|
|
) error |
|
|
|
|
|
|
|
|
|
// GC will garbage collect the store, removing any orphaned Resources.
|
|
|
|
|
GC(context.Context) error |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
var migrations = &migrate.MemoryMigrationSource{Migrations: []*migrate.Migration{ |
|
|
|
@ -74,13 +80,26 @@ var migrations = &migrate.MemoryMigrationSource{Migrations: []*migrate.Migration |
|
|
|
|
}, |
|
|
|
|
}} |
|
|
|
|
|
|
|
|
|
/* |
|
|
|
|
TODO |
|
|
|
|
- initialization options |
|
|
|
|
- cleanup period |
|
|
|
|
- document SQLiteStore properly |
|
|
|
|
- teardown the cleanup goroutine |
|
|
|
|
*/ |
|
|
|
|
// SQLiteSQLiteStoreOpts are optional fields which can be provided to NewSQLiteStore.
|
|
|
|
|
// A nil SQLiteSQLiteStoreOpts is equivalent to an empty one.
|
|
|
|
|
type SQLiteStoreOpts struct { |
|
|
|
|
// Path to the database file to use.
|
|
|
|
|
//
|
|
|
|
|
// Defaults to ":memory:", indicating an in-memory database will be used.
|
|
|
|
|
Path string |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (o *SQLiteStoreOpts) withDefaults() *SQLiteStoreOpts { |
|
|
|
|
if o == nil { |
|
|
|
|
o = new(SQLiteStoreOpts) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if o.Path == "" { |
|
|
|
|
o.Path = ":memory:" |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return o |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
type SQLiteStore struct { |
|
|
|
|
db *sql.DB |
|
|
|
@ -88,10 +107,12 @@ type SQLiteStore struct { |
|
|
|
|
|
|
|
|
|
var _ Store = (*SQLiteStore)(nil) |
|
|
|
|
|
|
|
|
|
// NewInMemStore returns a Store implementation which uses an in-memory SQLite
|
|
|
|
|
// NewSQLiteStore returns a Store implementation which uses an in-memory SQLite
|
|
|
|
|
// db.
|
|
|
|
|
func NewInMemStore() *SQLiteStore { |
|
|
|
|
db, err := sql.Open("sqlite3", ":memory:?_foreign_keys=1") |
|
|
|
|
func NewSQLiteStore(o *SQLiteStoreOpts) *SQLiteStore { |
|
|
|
|
o = o.withDefaults() |
|
|
|
|
|
|
|
|
|
db, err := sql.Open("sqlite3", o.Path+"?_foreign_keys=1") |
|
|
|
|
if err != nil { |
|
|
|
|
panic(fmt.Errorf("opening sqlite in memory: %w", err)) |
|
|
|
|
} |
|
|
|
@ -141,7 +162,8 @@ func (s *SQLiteStore) GetByStatus(status ResourceStatus) miter.Iterator[Resource |
|
|
|
|
JOIN urls ON (urls.id = resources.url_id) |
|
|
|
|
LEFT JOIN incoming ON (incoming.url_id = resources.url_id) |
|
|
|
|
LEFT JOIN outgoing ON (outgoing.url_id = resources.url_id) |
|
|
|
|
WHERE status = ?` |
|
|
|
|
WHERE status = ? |
|
|
|
|
AND (pinned OR incoming.urls IS NOT NULL)` |
|
|
|
|
|
|
|
|
|
return miter.Lazily(func(ctx context.Context) (miter.Iterator[Resource], error) { |
|
|
|
|
rows, err := s.db.QueryContext(ctx, query, status) |
|
|
|
@ -208,10 +230,18 @@ func (s *SQLiteStore) GetURLsByLastChecked( |
|
|
|
|
olderThan time.Time, |
|
|
|
|
) miter.Iterator[URL] { |
|
|
|
|
const query = ` |
|
|
|
|
WITH |
|
|
|
|
incoming(url_id, urls) AS ( |
|
|
|
|
SELECT to_url_id, COUNT(1) |
|
|
|
|
FROM links |
|
|
|
|
GROUP BY to_url_id |
|
|
|
|
) |
|
|
|
|
SELECT url |
|
|
|
|
FROM resources |
|
|
|
|
JOIN urls ON (urls.id = resources.url_id) |
|
|
|
|
WHERE last_checked < ?` |
|
|
|
|
LEFT JOIN incoming ON (incoming.url_id = resources.url_id) |
|
|
|
|
WHERE last_checked < ? |
|
|
|
|
AND (pinned OR incoming.urls IS NOT NULL)` |
|
|
|
|
|
|
|
|
|
return miter.Lazily(func(ctx context.Context) (miter.Iterator[URL], error) { |
|
|
|
|
rows, err := s.db.QueryContext(ctx, query, olderThan.Unix()) |
|
|
|
@ -379,7 +409,8 @@ func (s *SQLiteStore) Update( |
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (s *SQLiteStore) deleteOrphans(ctx context.Context) error { |
|
|
|
|
// GC implements the method for the Store interface.
|
|
|
|
|
func (s *SQLiteStore) GC(ctx context.Context) error { |
|
|
|
|
const query = ` |
|
|
|
|
WITH orphans AS ( |
|
|
|
|
SELECT url_id FROM resources |
|
|
|
|