From bc97c948bf4ba171206ec26e9436ba77a6d7359f Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Sat, 21 Jul 2018 19:56:40 +0000 Subject: [PATCH] mbigtable: implement basic cfg and basic tests --- env.test | 6 ++ mdb/mbigtable/bigtable.go | 109 +++++++++++++++++++++++++++++++++ mdb/mbigtable/bigtable_test.go | 52 ++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 mdb/mbigtable/bigtable.go create mode 100644 mdb/mbigtable/bigtable_test.go diff --git a/env.test b/env.test index 992bf74..b639da3 100644 --- a/env.test +++ b/env.test @@ -11,3 +11,9 @@ if [ "$(ps aux | grep '[c]loud-datastore-emulator')" = "" ]; then yes | gcloud beta emulators datastore start >/dev/null 2>&1 & fi $(gcloud beta emulators datastore env-init) + +if [ "$(ps aux | grep '[b]igtable-emulator')" = "" ]; then + echo "starting bigtable emulator" + yes | gcloud beta emulators bigtable start --host-port 127.0.0.1:8086 >/dev/null 2>&1 & +fi +$(gcloud beta emulators bigtable env-init) diff --git a/mdb/mbigtable/bigtable.go b/mdb/mbigtable/bigtable.go new file mode 100644 index 0000000..e3da043 --- /dev/null +++ b/mdb/mbigtable/bigtable.go @@ -0,0 +1,109 @@ +// Package mbigtable implements connecting to Google's Bigtable service and +// simplifying a number of interactions with it. +package mbigtable + +import ( + "context" + "strings" + + "cloud.google.com/go/bigtable" + "github.com/mediocregopher/mediocre-go-lib/m" + "github.com/mediocregopher/mediocre-go-lib/mcfg" + "github.com/mediocregopher/mediocre-go-lib/mdb" + "github.com/mediocregopher/mediocre-go-lib/mlog" +) + +func isErrAlreadyExists(err error) bool { + if err == nil { + return false + } + return strings.HasSuffix(err.Error(), " already exists") +} + +// Bigtable is a wrapper around a bigtable client providing more functionality. +type Bigtable struct { + *bigtable.Client + Instance string + + gce *mdb.GCE + log *mlog.Logger +} + +// Cfg configurs and returns a Bigtable instance which will be usable once +// StartRun is called on the passed in Cfg instance. +// +// defaultInstance can be given as the instance name to use as the default +// parameter value. If empty the parameter will be required to be set. +func Cfg(cfg *mcfg.Cfg, defaultInstance string) *Bigtable { + cfg = cfg.Child("bigtable") + var bt Bigtable + bt.gce = mdb.CfgGCE(cfg) + bt.log = m.Log(cfg, &bt) + + var inst *string + { + name, descr := "instance", "name of the bigtable instance in the project to connect to" + if defaultInstance != "" { + inst = cfg.ParamString(name, defaultInstance, descr) + } else { + inst = cfg.ParamRequiredString(name, descr) + } + } + + cfg.Start.Then(func(ctx context.Context) error { + bt.Instance = *inst + bt.log.Info("connecting to bigtable") + var err error + bt.Client, err = bigtable.NewClient( + ctx, + bt.gce.Project, bt.Instance, + bt.gce.ClientOptions()..., + ) + return mlog.ErrWithKV(err, &bt) + }) + return &bt +} + +// KV implements the mlog.KVer interface. +func (bt *Bigtable) KV() mlog.KV { + return bt.gce.KV().Set("instance", bt.Instance) +} + +// EnsureTable ensures that the given table exists and has (at least) the given +// column families. +// +// This method requires admin privileges on the bigtable instance. +func (bt *Bigtable) EnsureTable(ctx context.Context, name string, colFams ...string) error { + kv := bt.KV().Set("table", name) + bt.log.Info("ensuring table", kv) + + bt.log.Debug("creating admin client", kv) + adminClient, err := bigtable.NewAdminClient(ctx, bt.gce.Project, bt.Instance) + if err != nil { + return mlog.ErrWithKV(err, kv) + } + defer adminClient.Close() + + bt.log.Debug("creating bigtable table (if needed)", kv) + err = adminClient.CreateTable(ctx, name) + if err != nil && !isErrAlreadyExists(err) { + return mlog.ErrWithKV(err, kv) + } + + for _, colFam := range colFams { + kv := kv.Set("family", colFam) + bt.log.Debug("creating bigtable column family (if needed)", kv) + err := adminClient.CreateColumnFamily(ctx, name, colFam) + if err != nil && !isErrAlreadyExists(err) { + return mlog.ErrWithKV(err, kv) + } + } + + return nil +} + +// Table returns the bigtable.Table instance which can be used to write/query +// the given table. +func (bt *Bigtable) Table(tableName string) *bigtable.Table { + return bt.Open(tableName) +} diff --git a/mdb/mbigtable/bigtable_test.go b/mdb/mbigtable/bigtable_test.go new file mode 100644 index 0000000..4e36317 --- /dev/null +++ b/mdb/mbigtable/bigtable_test.go @@ -0,0 +1,52 @@ +package mbigtable + +import ( + "context" + . "testing" + "time" + + "cloud.google.com/go/bigtable" + "github.com/mediocregopher/mediocre-go-lib/mcfg" + "github.com/mediocregopher/mediocre-go-lib/mdb" + "github.com/mediocregopher/mediocre-go-lib/mrand" + "github.com/mediocregopher/mediocre-go-lib/mtest/massert" +) + +var testBT *Bigtable + +func init() { + mdb.DefaultGCEProject = "test" + cfg := mcfg.New() + testBT = Cfg(cfg, "test") + cfg.StartTestRun() +} + +func TestBasic(t *T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + tableName := "test-" + mrand.Hex(8) + colFam := "colFam-" + mrand.Hex(8) + if err := testBT.EnsureTable(ctx, tableName, colFam); err != nil { + t.Fatal(err) + } + + table := testBT.Table(tableName) + row := "row-" + mrand.Hex(8) + mut := bigtable.NewMutation() + mut.Set(colFam, "col", bigtable.Time(time.Now()), []byte("bar")) + if err := table.Apply(ctx, row, mut); err != nil { + t.Fatal(err) + } + + readRow, err := table.ReadRow(ctx, row) + if err != nil { + t.Fatal(err) + } + readColFam := readRow[colFam] + massert.Fatal(t, massert.All( + massert.Len(readColFam, 1), + massert.Equal(colFam+":col", readColFam[0].Column), + massert.Equal([]byte("bar"), readColFam[0].Value), + )) +}