1
0
Fork 0

Refactor merr, remove almost everything else

main
Brian Picciano 3 years ago
parent 3e2713a850
commit 360d41e2b8
  1. 1
      .dockerignore
  2. 11
      Dockerfile
  3. 11
      README.md
  4. 2
      TODO
  5. 122
      cmd/totp-proxy/main.go
  6. 29
      env.test
  7. 14
      go.mod
  8. 176
      go.sum
  9. 55
      jstream/byte_blob_reader.go
  10. 186
      jstream/byte_blob_reader_test.go
  11. 410
      jstream/jstream.go
  12. 246
      jstream/jstream_test.go
  13. 152
      m/m.go
  14. 50
      m/m_test.go
  15. 29
      mcrypto/mcrypto.go
  16. 246
      mcrypto/pair.go
  17. 17
      mcrypto/pair_test.go
  18. 84
      mcrypto/secret.go
  19. 54
      mcrypto/secret_test.go
  20. 184
      mcrypto/sig.go
  21. 44
      mcrypto/sig_test.go
  22. 97
      mcrypto/uuid.go
  23. 39
      mcrypto/uuid_test.go
  24. 11
      mctx/annotate.go
  25. 2
      mctx/annotate_test.go
  26. 168
      mdb/mbigquery/bigquery.go
  27. 124
      mdb/mbigtable/bigtable.go
  28. 44
      mdb/mbigtable/bigtable_test.go
  29. 54
      mdb/mdatastore/datastore.go
  30. 40
      mdb/mdatastore/datastore_test.go
  31. 79
      mdb/mdb.go
  32. 373
      mdb/mpubsub/pubsub.go
  33. 174
      mdb/mpubsub/pubsub_test.go
  34. 75
      mdb/mredis/redis.go
  35. 22
      mdb/mredis/redis_test.go
  36. 247
      mdb/mredis/stream.go
  37. 158
      mdb/mredis/stream_test.go
  38. 69
      mdb/msql/sql.go
  39. 18
      mdb/msql/sql_test.go
  40. 212
      merr/merr.go
  41. 176
      merr/merr_test.go
  42. 41
      merr/stack.go
  43. 48
      merr/stack_test.go
  44. 107
      mhttp/mhttp.go
  45. 70
      mhttp/mhttp_test.go
  46. 49
      mlog/mlog.go
  47. 21
      mlog/mlog_test.go
  48. 211
      mnet/mnet.go
  49. 61
      mnet/mnet_test.go
  50. 67
      mrand/lockedSource.go
  51. 105
      mrand/mrand.go
  52. 59
      mrand/mrand_test.go
  53. 138
      mrpc/mrpc.go
  54. 67
      mrpc/mrpc_test.go
  55. 213
      mtest/mchk/mchk.go
  56. 49
      mtest/mchk/mchk_test.go
  57. 69
      mtest/mtest.go
  58. 19
      mtest/mtest_test.go
  59. 42
      mtime/dur.go
  60. 30
      mtime/dur_test.go
  61. 2
      mtime/mtime.go
  62. 118
      mtime/ts.go
  63. 118
      mtime/ts_test.go

@ -1 +0,0 @@
Dockerfile

@ -1,11 +0,0 @@
FROM golang:1.12 AS builder
WORKDIR /app
COPY . .
RUN GOBIN=$(pwd)/bin CGO_ENABLED=0 GOOS=linux go install -a -installsuffix cgo ./cmd/...
FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /app/bin
COPY --from=builder /app/bin /app/bin
ENV PATH="/app/bin:${PATH}"
CMD echo "Available commands:" && ls

@ -1,16 +1,7 @@
# mediocre-go-lib
This is a collection of packages which I use across many of my personal
projects. All packages intended to be used start with an `m`, packages not
starting with `m` are for internal use within this set of packages.
## Usage notes
* In general, all checking of equality of errors, e.g. `err == io.EOF`, done on
errors returned from the packages in this project should be done using
`merr.Equal`, e.g. `merr.Equal(err, io.EOF)`. The `merr` package is used to
wrap errors and embed further metadata in them, like stack traces and so
forth.
projects.
## Styleguide

@ -1,2 +0,0 @@
- read through all docs, especially package docs, make sure they make sense
- write examples

@ -1,122 +0,0 @@
package main
/*
totp-proxy is a reverse proxy which implements basic time-based one-time
password (totp) authentication for any website.
It takes in a JSON object which maps usernames to totp secrets (generated at
a site like https://freeotp.github.io/qrcode.html), as well as a url to
proxy requests to. Users are prompted with a basic-auth prompt, and if they
succeed their totp challenge a cookie is set and requests are proxied to the
destination.
*/
import (
"context"
"net/http"
"net/url"
"time"
"github.com/mediocregopher/mediocre-go-lib/m"
"github.com/mediocregopher/mediocre-go-lib/mcfg"
"github.com/mediocregopher/mediocre-go-lib/mcrypto"
"github.com/mediocregopher/mediocre-go-lib/mctx"
"github.com/mediocregopher/mediocre-go-lib/merr"
"github.com/mediocregopher/mediocre-go-lib/mhttp"
"github.com/mediocregopher/mediocre-go-lib/mlog"
"github.com/mediocregopher/mediocre-go-lib/mrand"
"github.com/mediocregopher/mediocre-go-lib/mrun"
"github.com/mediocregopher/mediocre-go-lib/mtime"
"github.com/pquerna/otp/totp"
)
func main() {
cmp := m.RootServiceComponent()
cookieName := mcfg.String(cmp, "cookie-name",
mcfg.ParamDefault("_totp_proxy"),
mcfg.ParamUsage("String to use as the name for cookies"))
cookieTimeout := mcfg.Duration(cmp, "cookie-timeout",
mcfg.ParamDefault(mtime.Duration{1 * time.Hour}),
mcfg.ParamUsage("Timeout for cookies"))
var userSecrets map[string]string
mcfg.JSON(cmp, "users", &userSecrets,
mcfg.ParamRequired(),
mcfg.ParamUsage("JSON object which maps usernames to their TOTP secret strings"))
var secret mcrypto.Secret
secretStr := mcfg.String(cmp, "secret",
mcfg.ParamUsage("String used to sign authentication tokens. If one isn't given a new one will be generated on each startup, invalidating all previous tokens."))
mrun.InitHook(cmp, func(context.Context) error {
if *secretStr == "" {
*secretStr = mrand.Hex(32)
}
mlog.From(cmp).Info("generating secret")
secret = mcrypto.NewSecret([]byte(*secretStr))
return nil
})
proxyHandler := new(struct{ http.Handler })
proxyURL := mcfg.String(cmp, "dst-url",
mcfg.ParamRequired(),
mcfg.ParamUsage("URL to proxy requests to. Only the scheme and host should be set."))
mrun.InitHook(cmp, func(context.Context) error {
u, err := url.Parse(*proxyURL)
if err != nil {
return merr.Wrap(err, cmp.Context())
}
proxyHandler.Handler = mhttp.ReverseProxy(u)
return nil
})
authHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// TODO mlog.FromHTTP?
ctx := r.Context()
unauthorized := func() {
mlog.From(cmp).Debug("connection is unauthorized")
w.Header().Add("WWW-Authenticate", "Basic")
w.WriteHeader(http.StatusUnauthorized)
}
authorized := func() {
mlog.From(cmp).Debug("connection is authorized, rewriting cookies")
sig := mcrypto.SignString(secret, "")
http.SetCookie(w, &http.Cookie{
Name: *cookieName,
Value: sig.String(),
MaxAge: int((*cookieTimeout).Seconds()),
})
proxyHandler.ServeHTTP(w, r)
}
if cookie, _ := r.Cookie(*cookieName); cookie != nil {
mlog.From(cmp).Debug("authenticating with cookie",
mctx.Annotate(ctx, "cookie", cookie.String()))
var sig mcrypto.Signature
if err := sig.UnmarshalText([]byte(cookie.Value)); err == nil {
err := mcrypto.VerifyString(secret, sig, "")
if err == nil && time.Since(sig.Time()) < (*cookieTimeout).Duration {
authorized()
return
}
}
}
if user, pass, ok := r.BasicAuth(); ok && pass != "" {
mlog.From(cmp).Debug("authenticating with user",
mctx.Annotate(ctx, "user", user))
if userSecret, ok := userSecrets[user]; ok {
if totp.Validate(pass, userSecret) {
authorized()
return
}
}
}
unauthorized()
})
mhttp.InstListeningServer(cmp, authHandler)
m.Exec(cmp)
}

@ -1,29 +0,0 @@
export CLOUDSDK_CORE_PROJECT="test"
if [ "$(ps aux | grep '[p]ubsub-emulator')" = "" ]; then
echo "starting pubsub emulator"
yes | gcloud beta emulators pubsub start >/dev/null 2>&1 &
fi
$(gcloud beta emulators pubsub env-init)
if [ "$(ps aux | grep '[c]loud-datastore-emulator')" = "" ]; then
echo "starting datastore emulator"
yes | gcloud beta emulators datastore start >/dev/null 2>&1 &
fi
$(gcloud beta emulators datastore env-init)
if [ "$(ps aux | grep '[b]igtable-emulator')" = "" ]; then
echo "starting bigtable emulator"
yes | gcloud beta emulators bigtable start --host-port 127.0.0.1:8086 >/dev/null 2>&1 &
fi
$(gcloud beta emulators bigtable env-init)
if ! (sudo systemctl status mysqld 1>/dev/null); then
echo "starting mysqld"
sudo systemctl start mysqld
fi
if ! (sudo systemctl status redis 1>/dev/null); then
echo "starting redis"
sudo systemctl start redis
fi

@ -1,17 +1,3 @@
module github.com/mediocregopher/mediocre-go-lib/v2
go 1.15
require (
cloud.google.com/go v0.36.0
github.com/boombuler/barcode v1.0.0 // indirect
github.com/go-sql-driver/mysql v1.4.0
github.com/jmoiron/sqlx v1.2.0
github.com/mediocregopher/radix/v3 v3.3.2
github.com/pquerna/otp v1.1.0
github.com/stretchr/testify v1.3.0
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd // indirect
google.golang.org/api v0.1.0
google.golang.org/grpc v1.18.0
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce
)

176
go.sum

@ -1,176 +0,0 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.36.0 h1:+aCSj7tOo2LODWVEuZDZeGCckdt6MlSF+X/rB3wUiS8=
cloud.google.com/go v0.36.0/go.mod h1:RUoy9p/M4ge0HzT8L+SDZ8jg+Q6fth0CiBuhFJpSV40=
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/boombuler/barcode v1.0.0 h1:s1TvRnXwL2xJRaccrdcBQMZxq6X7DvsMogtmJeHDdrc=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mediocregopher/radix/v3 v3.3.2 h1:2gAC5aDBWQr1LBgaNQiVLb2LGX4lvkARDkfjsuonKJE=
github.com/mediocregopher/radix/v3 v3.3.2/go.mod h1:RsC7cELtyL4TGkg0nwRPTa+J2TXZ0dh/ruohD3rnjMk=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/otp v1.1.0 h1:q2gMsMuMl3JzneUaAX1MRGxLvOG6bzXV51hivBaStf0=
github.com/pquerna/otp v1.1.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497 h1:GXMDsk4xWZCVzkAWCabrabzCCVmfiYSw72f1K/S9QIY=
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522 h1:bhOzK9QyoD0ogCnFro1m2mz41+Ib0oOhfJnBp5MR4K4=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 h1:mBVYJnbrXLA/ZCBTCe7PtEgAUP+1bg92qTaFoPHdz+8=
google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922/go.mod h1:L3J43x8/uS+qIUoksaLKe6OS3nUKxOKuIFz1sl2/jx4=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=
google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

@ -1,55 +0,0 @@
package jstream
import (
"bytes"
"encoding/base64"
"io"
)
type delimReader struct {
r io.Reader
delim byte
rest []byte
}
func (dr *delimReader) Read(b []byte) (int, error) {
if dr.delim != 0 {
return 0, io.EOF
}
n, err := dr.r.Read(b)
if i := bytes.IndexAny(b[:n], bbDelims); i >= 0 {
dr.delim = b[i]
dr.rest = append([]byte(nil), b[i+1:n]...)
return i, err
}
return n, err
}
type byteBlobReader struct {
dr *delimReader
dec io.Reader
}
func newByteBlobReader(r io.Reader) *byteBlobReader {
dr := &delimReader{r: r}
return &byteBlobReader{
dr: dr,
dec: base64.NewDecoder(base64.StdEncoding, dr),
}
}
func (bbr *byteBlobReader) Read(into []byte) (int, error) {
n, err := bbr.dec.Read(into)
if bbr.dr.delim == bbEnd {
return n, io.EOF
} else if bbr.dr.delim == bbCancel {
return n, ErrCanceled
}
return n, err
}
// returns the bytes which were read off the underlying io.Reader but which
// haven't been consumed yet.
func (bbr *byteBlobReader) buffered() io.Reader {
return bytes.NewBuffer(bbr.dr.rest)
}

@ -1,186 +0,0 @@
package jstream
import (
"bytes"
"encoding/base64"
"io"
"io/ioutil"
. "testing"
"github.com/mediocregopher/mediocre-go-lib/mrand"
"github.com/stretchr/testify/assert"
)
type bbrTest struct {
wsSuffix []byte // whitespace
body []byte
shouldCancel bool
intoSize int
}
func randBBRTest(minBodySize, maxBodySize int) bbrTest {
var whitespace = []byte{' ', '\n', '\t', '\r'}
genWhitespace := func(n int) []byte {
ws := make([]byte, n)
for i := range ws {
ws[i] = whitespace[mrand.Intn(len(whitespace))]
}
return ws
}
body := mrand.Bytes(minBodySize + mrand.Intn(maxBodySize-minBodySize))
return bbrTest{
wsSuffix: genWhitespace(mrand.Intn(10)),
body: body,
intoSize: 1 + mrand.Intn(len(body)+1),
}
}
func (bt bbrTest) msgAndArgs() []interface{} {
return []interface{}{"bt:%#v len(body):%d", bt, len(bt.body)}
}
func (bt bbrTest) mkBytes() []byte {
buf := new(bytes.Buffer)
enc := base64.NewEncoder(base64.StdEncoding, buf)
if bt.shouldCancel {
enc.Write(bt.body[:len(bt.body)/2])
enc.Close()
buf.WriteByte(bbCancel)
} else {
enc.Write(bt.body)
enc.Close()
buf.WriteByte(bbEnd)
}
buf.Write(bt.wsSuffix)
return buf.Bytes()
}
func (bt bbrTest) do(t *T) bool {
buf := bytes.NewBuffer(bt.mkBytes())
bbr := newByteBlobReader(buf)
into := make([]byte, bt.intoSize)
outBuf := new(bytes.Buffer)
_, err := io.CopyBuffer(outBuf, bbr, into)
if bt.shouldCancel {
return assert.Equal(t, ErrCanceled, err, bt.msgAndArgs()...)
}
if !assert.NoError(t, err, bt.msgAndArgs()...) {
return false
}
if !assert.Equal(t, bt.body, outBuf.Bytes(), bt.msgAndArgs()...) {
return false
}
fullRest := append(bbr.dr.rest, buf.Bytes()...)
if len(bt.wsSuffix) == 0 {
return assert.Empty(t, fullRest, bt.msgAndArgs()...)
}
return assert.Equal(t, bt.wsSuffix, fullRest, bt.msgAndArgs()...)
}
func TestByteBlobReader(t *T) {
// some sanity tests
bbrTest{
body: []byte{2, 3, 4, 5},
intoSize: 4,
}.do(t)
bbrTest{
body: []byte{2, 3, 4, 5},
intoSize: 3,
}.do(t)
bbrTest{
body: []byte{2, 3, 4, 5},
shouldCancel: true,
intoSize: 3,
}.do(t)
// fuzz this bitch
for i := 0; i < 50000; i++ {
bt := randBBRTest(0, 1000)
if !bt.do(t) {
return
}
bt.shouldCancel = true
if !bt.do(t) {
return
}
}
}
func BenchmarkByteBlobReader(b *B) {
type bench struct {
bt bbrTest
body []byte
buf *bytes.Reader
cpBuf []byte
}
mkTestSet := func(minBodySize, maxBodySize int) []bench {
n := 100
benches := make([]bench, n)
for i := range benches {
bt := randBBRTest(minBodySize, maxBodySize)
body := bt.mkBytes()
benches[i] = bench{
bt: bt,
body: body,
buf: bytes.NewReader(nil),
cpBuf: make([]byte, bt.intoSize),
}
}
return benches
}
testRaw := func(b *B, benches []bench) {
j := 0
for i := 0; i < b.N; i++ {
if j >= len(benches) {
j = 0
}
benches[j].buf.Reset(benches[j].body)
io.CopyBuffer(ioutil.Discard, benches[j].buf, benches[j].cpBuf)
j++
}
}
testBBR := func(b *B, benches []bench) {
j := 0
for i := 0; i < b.N; i++ {
if j >= len(benches) {
j = 0
}
benches[j].buf.Reset(benches[j].body)
bbr := newByteBlobReader(benches[j].buf)
io.CopyBuffer(ioutil.Discard, bbr, benches[j].cpBuf)
j++
}
}
benches := []struct {
name string
minBodySize, maxBodySize int
}{
{"small", 0, 1000},
{"medium", 1000, 10000},
{"large", 10000, 100000},
{"xlarge", 100000, 1000000},
}
b.StopTimer()
for i := range benches {
b.Run(benches[i].name, func(b *B) {
set := mkTestSet(benches[i].minBodySize, benches[i].maxBodySize)
b.StartTimer()
b.Run("raw", func(b *B) {
testRaw(b, set)
})
b.Run("bbr", func(b *B) {
testBBR(b, set)
})
b.StopTimer()
})
}
}

@ -1,410 +0,0 @@
// Package jstream defines and implements the JSON Stream protocol
//
// Purpose
//
// The purpose of the jstream protocol is to provide a very simple layer on top
// of an existing JSON implementation to allow for streaming arbitrary numbers
// of JSON objects and byte blobs of arbitrary size in a standard way, and to
// allow for embedding streams within each other.
//
// The order of priorities when designing jstream is as follows:
// 1) Protocol simplicity
// 2) Implementation simplicity
// 3) Efficiency, both in parsing speed and bandwidth
//
// The justification for this is that protocol simplicity generally spills into
// implementation simplicity anyway, and accounts for future languages which
// have different properties than current ones. Parsing speed isn't much of a
// concern when reading data off a network (the primary use-case here), as RTT
// is always going to be the main blocker. Bandwidth can be a concern, but it's
// one better solved by wrapping the byte stream with a compressor.
//
// jstream protocol
//
// The jstream protocol is carried over a byte stream (in go: an io.Reader). To
// read the protocol a JSON object is read off the byte stream and inspected to
// determine what kind of jstream element it is.
//
// Multiple jstream elements are sequentially read off the same byte stream.
// Each element may be separated from the other by any amount of whitespace,
// with whitespace being defined as spaces, tabs, carriage returns, and/or
// newlines.
//
// jstream elements
//
// There are three jstream element types:
//
// * JSON Value: Any JSON value
// * Byte Blob: A stream of bytes of unknown, and possibly infinite, size
// * Stream: A heterogenous sequence of jstream elements of unknown, and
// possibly infinite, size
//
// JSON Value elements are defined as being JSON objects with a `val` field. The
// value of that field is the JSON Value.
//
// { "val":{"foo":"bar"} }
//
// Byte Blob elements are defined as being a JSON object with a `bytesStart`
// field with a value of `true`. Immediately following the JSON object are the
// bytes which are the Byte Blob, encoded using standard base64. Immediately
// following the encoded bytes is the character `$`, to indicate the bytes have
// been completely written. Alternatively the character `!` may be written
// immediately after the bytes to indicate writing was canceled prematurely by
// the writer.
//
// { "bytesStart":true }wXnxQHgUO8g=$
// { "bytesStart":true }WGYcTI8=!
//
// The JSON object may also contain a `sizeHint` field, which gives the
// estimated number of bytes in the Byte Blob (excluding the trailing
// delimiter). The hint is neither required to exist or be accurate if it does.
// The trailing delimeter (`$` or `!`) is required to be sent even if the hint
// is sent.
//
// Stream elements are defined as being a JSON object with a `streamStart` field
// with a value of `true`. Immediately following the JSON object will be zero
// more jstream elements of any type, possibly separated by whitespace. Finally
// the Stream is ended with another JSON object with a `streamEnd` field with a
// value of `true`.
//
// { "streamStart":true }
// { "val":{"foo":"bar"} }
// { "bytesStart":true }7TdlDQOnA6isxD9C$
// { "streamEnd":true }
//
// A Stream may also be prematurely canceled by the sending of a JSON object
// with the `streamCancel` field set to `true` (in place of one with `streamEnd`
// set to `true`).
//
// The Stream's original JSON object (the "head") may also have a `sizeHint`
// field, which gives the estimated number of jstream elements in the Stream.
// The hint is neither required to exist or be accurate if it does. The tail
// JSON object (with the `streamEnd` field) is required even if `sizeHint` is
// given.
//
// One of the elements in a Stream may itself be a Stream. In this way Streams
// may be embedded within each other.
//
// Here's an example of a complex Stream, which carries within it two different
// streams and some other elements:
//
// { "streamStart":true }
// { "val":{"foo":"bar" }
// { "streamStart":true, "sizeHint":2 }
// { "val":{"foo":"baz"} }
// { "val":{"foo":"biz"} }
// { "streamEnd":true }
// { "bytesStart":true }X7KCpLIjqIBJt9vA$
// { "streamStart":true }
// { "bytesStart":true }0jT+kNCuxHywUYy0$
// { "bytesStart":true }LUqjR6OACB2p1BG4$
// { "streamEnd":true }
// { "streamEnd":true }
//
// Finally, the byte stream off of which the jstream is based (i.e. the
// io.Reader) is implicitly treated as a Stream, with the Stream ending when the
// byte stream is closed.
//
package jstream
// TODO figure out how to expose the json.Encoder/Decoders so that users can set
// custom options on them (like UseNumber and whatnot)
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
)
// byte blob constants
const (
bbEnd = '$'
bbCancel = '!'
bbDelims = string(bbEnd) + string(bbCancel)
)
// Type is used to enumerate the types of jstream elements
type Type string
// The jstream element types
const (
TypeJSONValue Type = "jsonValue"
TypeByteBlob Type = "byteBlob"
TypeStream Type = "stream"
)
// ErrWrongType is an error returned by the Decode* methods on Decoder when the
// wrong decoding method has been called for the element which was read. The
// error contains the actual type of the element.
type ErrWrongType struct {
Actual Type
}
func (err ErrWrongType) Error() string {
return fmt.Sprintf("wrong type, actual type is %q", err.Actual)
}
var (
// ErrCanceled is returned when reading either a Byte Blob or a Stream,
// indicating that the writer has prematurely canceled the element.
ErrCanceled = errors.New("canceled by writer")
// ErrStreamEnded is returned from Next when the Stream being read has been
// ended by the writer.
ErrStreamEnded = errors.New("stream ended")
)
type element struct {
Value json.RawMessage `json:"val,omitempty"`
BytesStart bool `json:"bytesStart,omitempty"`
StreamStart bool `json:"streamStart,omitempty"`
StreamEnd bool `json:"streamEnd,omitempty"`
StreamCancel bool `json:"streamCancel,omitempty"`
SizeHint uint `json:"sizeHint,omitempty"`
}
// Element is a single jstream element which is read off a StreamReader.
//
// If a method is called which expects a particular Element type (e.g.
// DecodeValue, which expects a JSONValue Element) but the Element is not of
// that type then an ErrWrongType will be returned.
//
// If there was an error reading the Element off the StreamReader that error is
// kept in the Element and returned from any method call.
type Element struct {
element
// Err will be set if the StreamReader encountered an error while reading
// the next Element. If set then the Element is otherwise unusable.
//
// Err may be ErrCanceled or ErrStreamEnded, which would indicate the end of
// the stream but would not indicate the StreamReader is no longer usable,
// depending on the behavior of the writer on the other end.
Err error
// needed for byte blobs and streams
sr *StreamReader
}
// Type returns the Element's Type, or an error
func (el Element) Type() (Type, error) {
if el.Err != nil {
return "", el.Err
} else if el.element.StreamStart {
return TypeStream, nil
} else if el.element.BytesStart {
return TypeByteBlob, nil
} else if len(el.element.Value) > 0 {
return TypeJSONValue, nil
}
return "", errors.New("malformed Element, can't determine type")
}
func (el Element) assertType(is Type) error {
typ, err := el.Type()
if err != nil {
return err
} else if typ != is {
return ErrWrongType{Actual: typ}
}
return nil
}
// Value attempts to unmarshal a JSON Value Element's value into the given
// receiver.
//
// This method should not be called more than once.
func (el Element) Value(i interface{}) error {
if err := el.assertType(TypeJSONValue); err != nil {
return err
}
return json.Unmarshal(el.element.Value, i)
}
// SizeHint returns the size hint which may have been optionally sent for
// ByteBlob and Stream elements, or zero. The hint is never required to be
// sent or to be accurate.
func (el Element) SizeHint() uint {
return el.element.SizeHint
}
// Bytes returns an io.Reader which will contain the contents of a ByteBlob
// element. The io.Reader _must_ be read till io.EOF or ErrCanceled before the
// StreamReader may be used again.
//
// This method should not be called more than once.
func (el Element) Bytes() (io.Reader, error) {
if err := el.assertType(TypeByteBlob); err != nil {
return nil, err
}
return el.sr.readBytes(), nil
}
// Stream returns the embedded stream represented by this Element as a
// StreamReader. The returned StreamReader _must_ be iterated (via the Next
// method) till ErrStreamEnded or ErrCanceled is returned before the original
// StreamReader may be used again.
//
// This method should not be called more than once.
func (el Element) Stream() (*StreamReader, error) {
if err := el.assertType(TypeStream); err != nil {
return nil, err
}
return el.sr, nil
}
// StreamReader represents a Stream from which Elements may be read using the
// Next method.
type StreamReader struct {
orig io.Reader
// only one of these can be set at a time
dec *json.Decoder
bbr *byteBlobReader
}
// NewStreamReader takes an io.Reader and interprets it as a jstream Stream.
func NewStreamReader(r io.Reader) *StreamReader {
return &StreamReader{orig: r}
}
// pulls buffered bytes out of either the json.Decoder or byteBlobReader, if
// possible, and returns an io.MultiReader of those and orig. Will also set the
// json.Decoder/byteBlobReader to nil if that's where the bytes came from.
func (sr *StreamReader) multiReader() io.Reader {
if sr.dec != nil {
buf := sr.dec.Buffered()
sr.dec = nil
return io.MultiReader(buf, sr.orig)
} else if sr.bbr != nil {
buf := sr.bbr.buffered()
sr.bbr = nil
return io.MultiReader(buf, sr.orig)
}
return sr.orig
}
// Next reads, decodes, and returns the next Element off the StreamReader. If
// the Element is a ByteBlob or embedded Stream then it _must_ be fully consumed
// before Next is called on this StreamReader again.
//
// The returned Element's Err field will be ErrStreamEnd if the Stream was
// ended, or ErrCanceled if it was canceled, and this StreamReader should not be
// used again in those cases.
//
// If the underlying io.Reader is closed the returned Err field will be io.EOF.
func (sr *StreamReader) Next() Element {
if sr.dec == nil {
sr.dec = json.NewDecoder(sr.multiReader())
}
var el element
var err error
if err = sr.dec.Decode(&el); err != nil {
// welp
} else if el.StreamEnd {
err = ErrStreamEnded
} else if el.StreamCancel {
err = ErrCanceled
}
if err != nil {
return Element{Err: err}
}
return Element{sr: sr, element: el}
}
func (sr *StreamReader) readBytes() *byteBlobReader {
sr.bbr = newByteBlobReader(sr.multiReader())
return sr.bbr
}
////////////////////////////////////////////////////////////////////////////////
// StreamWriter represents a Stream to which Elements may be written using any
// of the Encode methods.
type StreamWriter struct {
w io.Writer
enc *json.Encoder
}
// NewStreamWriter takes an io.Writer and returns a StreamWriter which will
// write to it.
func NewStreamWriter(w io.Writer) *StreamWriter {
return &StreamWriter{w: w, enc: json.NewEncoder(w)}
}
// EncodeValue marshals the given value and writes it to the Stream as a
// JSONValue element.
func (sw *StreamWriter) EncodeValue(i interface{}) error {
b, err := json.Marshal(i)
if err != nil {
return err
}
return sw.enc.Encode(element{Value: b})
}
// EncodeBytes copies the given io.Reader, until io.EOF, onto the Stream as a
// ByteBlob element. This method will block until copying is completed or an
// error is encountered.
//
// If the io.Reader returns any error which isn't io.EOF then the ByteBlob is
// canceled and that error is returned from this method. Otherwise nil is
// returned.
//
// sizeHint may be given if it's known or can be guessed how many bytes the
// io.Reader will read out.
func (sw *StreamWriter) EncodeBytes(sizeHint uint, r io.Reader) error {
if err := sw.enc.Encode(element{
BytesStart: true,
SizeHint: sizeHint,
}); err != nil {
return err
}
enc := base64.NewEncoder(base64.StdEncoding, sw.w)
if _, err := io.Copy(enc, r); err != nil {
// if canceling doesn't work then the whole connection is broken and
// it's not worth doing anything about. if nothing else the brokeness of
// it will be discovered the next time it is used.
sw.w.Write([]byte{bbCancel})
return err
} else if err := enc.Close(); err != nil {
return err
} else if _, err := sw.w.Write([]byte{bbEnd}); err != nil {
return err
}
return nil
}
// EncodeStream encodes an embedded Stream element onto the Stream. The callback
// is given a new StreamWriter which represents the embedded Stream and to which
// any elemens may be written. This methods blocks until the callback has
// returned.
//
// If the callback returns nil the Stream is ended normally. If it returns
// anything else the embedded Stream is canceled and that error is returned from
// this method.
//
// sizeHint may be given if it's known or can be guessed how many elements will
// be in the embedded Stream.
func (sw *StreamWriter) EncodeStream(sizeHint uint, fn func(*StreamWriter) error) error {
if err := sw.enc.Encode(element{
StreamStart: true,
SizeHint: sizeHint,
}); err != nil {
return err
} else if err := fn(sw); err != nil {
// as when canceling a byte blob, we don't really care if this errors
sw.enc.Encode(element{StreamCancel: true})
return err
}
return sw.enc.Encode(element{StreamEnd: true})
}

@ -1,246 +0,0 @@
package jstream
import (
"bytes"
"errors"
"io"
"io/ioutil"
"sync"
. "testing"
"github.com/mediocregopher/mediocre-go-lib/mrand"
"github.com/stretchr/testify/assert"
)
type cancelBuffer struct {
lr *io.LimitedReader
}
func newCancelBuffer(b []byte) io.Reader {
return &cancelBuffer{
lr: &io.LimitedReader{
R: bytes.NewBuffer(b),
N: int64(len(b) / 2),
},
}
}
func (cb *cancelBuffer) Read(p []byte) (int, error) {
if cb.lr.N == 0 {
return 0, ErrCanceled
}
return cb.lr.Read(p)
}
func TestEncoderDecoder(t *T) {
type testCase struct {
typ Type
val interface{}
bytes []byte
stream []testCase
cancel bool
}
var randTestCase func(Type, bool) testCase
randTestCase = func(typ Type, cancelable bool) testCase {
// if typ isn't given then use a random one
if typ == "" {
pick := mrand.Intn(5)
switch {
case pick == 0:
typ = TypeStream
case pick < 4:
typ = TypeJSONValue
default:
typ = TypeByteBlob
}
}
tc := testCase{
typ: typ,
cancel: cancelable && mrand.Intn(10) == 0,
}
switch typ {
case TypeJSONValue:
tc.val = map[string]interface{}{
mrand.Hex(8): mrand.Hex(8),
mrand.Hex(8): mrand.Hex(8),
mrand.Hex(8): mrand.Hex(8),
mrand.Hex(8): mrand.Hex(8),
mrand.Hex(8): mrand.Hex(8),
}
return tc
case TypeByteBlob:
tc.bytes = mrand.Bytes(mrand.Intn(256))
return tc
case TypeStream:
for i := mrand.Intn(10); i > 0; i-- {
tc.stream = append(tc.stream, randTestCase("", true))
}
return tc
}
panic("shouldn't get here")
}
tcLog := func(tcs ...testCase) []interface{} {
return []interface{}{"%#v", tcs}
}
var assertRead func(*StreamReader, Element, testCase) bool
assertRead = func(r *StreamReader, el Element, tc testCase) bool {
l, success := tcLog(tc), true
typ, err := el.Type()
success = success && assert.NoError(t, err, l...)
success = success && assert.Equal(t, tc.typ, typ, l...)
switch typ {
case TypeJSONValue:
var val interface{}
success = success && assert.NoError(t, el.Value(&val), l...)
success = success && assert.Equal(t, tc.val, val, l...)
case TypeByteBlob:
br, err := el.Bytes()
success = success && assert.NoError(t, err, l...)
success = success && assert.Equal(t, uint(len(tc.bytes)), el.SizeHint(), l...)
all, err := ioutil.ReadAll(br)
if tc.cancel {
success = success && assert.Equal(t, ErrCanceled, err, l...)
} else {
success = success && assert.NoError(t, err, l...)
success = success && assert.Equal(t, tc.bytes, all, l...)
}
case TypeStream:
innerR, err := el.Stream()
success = success && assert.NoError(t, err, l...)
success = success && assert.Equal(t, uint(len(tc.stream)), el.SizeHint(), l...)
n := 0
for {
el := innerR.Next()
if tc.cancel && el.Err == ErrCanceled {
break
} else if n == len(tc.stream) {
success = success && assert.Equal(t, ErrStreamEnded, el.Err, l...)
break
}
success = success && assertRead(innerR, el, tc.stream[n])
n++
}
}
return success
}
var assertWrite func(*StreamWriter, testCase) bool
assertWrite = func(w *StreamWriter, tc testCase) bool {
l, success := tcLog(tc), true
switch tc.typ {
case TypeJSONValue:
success = success && assert.NoError(t, w.EncodeValue(tc.val), l...)
case TypeByteBlob:
if tc.cancel {
r := newCancelBuffer(tc.bytes)
err := w.EncodeBytes(uint(len(tc.bytes)), r)
success = success && assert.Equal(t, ErrCanceled, err, l...)
} else {
r := bytes.NewBuffer(tc.bytes)
err := w.EncodeBytes(uint(len(tc.bytes)), r)
success = success && assert.NoError(t, err, l...)
}
case TypeStream:
err := w.EncodeStream(uint(len(tc.stream)), func(innerW *StreamWriter) error {
if len(tc.stream) == 0 && tc.cancel {
return ErrCanceled
}
for i := range tc.stream {
if tc.cancel && i == len(tc.stream)/2 {
return ErrCanceled
} else if !assertWrite(w, tc.stream[i]) {
return errors.New("we got problems")
}
}
return nil
})
if tc.cancel {
success = success && assert.Equal(t, ErrCanceled, err, l...)
} else {
success = success && assert.NoError(t, err, l...)
}
}
return success
}
do := func(tcs ...testCase) bool {
// we keep a copy of all read/written bytes for debugging, but generally
// don't actually log them
ioR, ioW := io.Pipe()
cpR, cpW := new(bytes.Buffer), new(bytes.Buffer)
r, w := NewStreamReader(io.TeeReader(ioR, cpR)), NewStreamWriter(io.MultiWriter(ioW, cpW))
readCh, writeCh := make(chan bool, 1), make(chan bool, 1)
wg := new(sync.WaitGroup)
wg.Add(2)
go func() {
success := true
for _, tc := range tcs {
success = success && assertRead(r, r.Next(), tc)
}
success = success && assert.Equal(t, io.EOF, r.Next().Err)
readCh <- success
ioR.Close()
wg.Done()
}()
go func() {
success := true
for _, tc := range tcs {
success = success && assertWrite(w, tc)
}
writeCh <- success
ioW.Close()
wg.Done()
}()
wg.Wait()
//log.Printf("data written:%q", cpW.Bytes())
//log.Printf("data read: %q", cpR.Bytes())
if !(<-readCh && <-writeCh) {
assert.FailNow(t, "test case failed", tcLog(tcs...)...)
return false
}
return true
}
// some basic test cases
do() // empty stream
do(randTestCase(TypeJSONValue, false))
do(randTestCase(TypeByteBlob, false))
do(
randTestCase(TypeJSONValue, false),
randTestCase(TypeJSONValue, false),
randTestCase(TypeJSONValue, false),
)
do(
randTestCase(TypeJSONValue, false),
randTestCase(TypeByteBlob, false),
randTestCase(TypeJSONValue, false),
)
do(
randTestCase(TypeByteBlob, false),
randTestCase(TypeByteBlob, false),
randTestCase(TypeByteBlob, false),
)
do(
randTestCase(TypeJSONValue, false),
randTestCase(TypeStream, false),
randTestCase(TypeJSONValue, false),
)
// some special cases, empty elements which are canceled
do(testCase{typ: TypeStream, cancel: true})
do(testCase{typ: TypeByteBlob, cancel: true})
for i := 0; i < 1000; i++ {
tc := randTestCase(TypeStream, false)
do(tc.stream...)
}
}

152
m/m.go

@ -1,152 +0,0 @@
// Package m implements functionality specific to how I like my programs to
// work. It acts as glue between many of the other packages in this framework,
// putting them together in the way I find most useful.
package m
import (
"context"
"os"
"os/signal"
"time"
"github.com/mediocregopher/mediocre-go-lib/mcfg"
"github.com/mediocregopher/mediocre-go-lib/mcmp"
"github.com/mediocregopher/mediocre-go-lib/mctx"
"github.com/mediocregopher/mediocre-go-lib/merr"
"github.com/mediocregopher/mediocre-go-lib/mlog"
"github.com/mediocregopher/mediocre-go-lib/mrun"
)
type cmpKey int
const (
cmpKeyCfgSrc cmpKey = iota
cmpKeyInfoLog
)
func debugLog(cmp *mcmp.Component, msg string, ctxs ...context.Context) {
level := mlog.DebugLevel
if len(ctxs) > 0 {
if ok, _ := ctxs[0].Value(cmpKeyInfoLog).(bool); ok {
level = mlog.InfoLevel
}
}
mlog.From(cmp).Log(mlog.Message{
Level: level,
Description: msg,
Contexts: ctxs,
})
}
// RootComponent returns a Component which should be used as the root Component
// when implementing most programs.
//
// The returned Component will automatically handle setting up global
// configuration parameters like "log-level", as well as parsing those
// and all other parameters when the Init even is triggered on it.
func RootComponent() *mcmp.Component {
cmp := new(mcmp.Component)
// embed confuration source which should be used into the context.
cmp.SetValue(cmpKeyCfgSrc, mcfg.Source(new(mcfg.SourceCLI)))
// set up log level handling
logger := mlog.NewLogger()
mlog.SetLogger(cmp, logger)
// set up parameter parsing
mrun.InitHook(cmp, func(context.Context) error {
src, _ := cmp.Value(cmpKeyCfgSrc).(mcfg.Source)
if src == nil {
return merr.New("Component not sourced from m package", cmp.Context())
} else if err := mcfg.Populate(cmp, src); err != nil {
return merr.Wrap(err, cmp.Context())
}
return nil
})
logLevelStr := mcfg.String(cmp, "log-level",
mcfg.ParamDefault("info"),
mcfg.ParamUsage("Maximum log level which will be printed."))
mrun.InitHook(cmp, func(context.Context) error {
logLevel := mlog.LevelFromString(*logLevelStr)
if logLevel == nil {
return merr.New("invalid log level", cmp.Context(),
mctx.Annotated("log-level", *logLevelStr))
}
logger.SetMaxLevel(logLevel)
mlog.SetLogger(cmp, logger)
return nil
})
return cmp
}
// RootServiceComponent extends RootComponent so that it better supports long
// running processes which are expected to handle requests from outside clients.
//
// Additional behavior it adds includes setting up an http endpoint where debug
// information about the running process can be accessed.
func RootServiceComponent() *mcmp.Component {
cmp := RootComponent()
// services expect to use many different configuration sources
cmp.SetValue(cmpKeyCfgSrc, mcfg.Source(mcfg.Sources{