mediocregopher/mediocre-go-lib
1
0
Fork 0
Code Releases Activity

mchk: move mtest.Checker to mtest/mchk, and refactor its types a little bit

Browse Source
This commit is contained in:
Brian Picciano 2018-08-13 15:03:30 -04:00
parent 1964add0ed
commit 715b6c9491
5 changed files with 325 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
mcfg/cli_test.go
View File

@ -4,14 +4,13 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"log"
"strings" "strings"
. "testing" . "testing"
"time" "time"
"github.com/mediocregopher/mediocre-go-lib/mrand" "github.com/mediocregopher/mediocre-go-lib/mrand"
"github.com/mediocregopher/mediocre-go-lib/mtest"
"github.com/mediocregopher/mediocre-go-lib/mtest/massert" "github.com/mediocregopher/mediocre-go-lib/mtest/massert"
"github.com/mediocregopher/mediocre-go-lib/mtest/mchk"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -45,92 +44,29 @@ func TestSourceCLIHelp(t *T) {
assert.Equal(t, exp, buf.String()) assert.Equal(t, exp, buf.String())
} }
type testCLIState struct {
cfg *Cfg
availCfgs []*Cfg
SourceCLI
expPVs []ParamValue
}
type testCLIApplyer struct {
name string
availCfgI int // not technically needed, but makes subsequent steps easier
path []string
isBool bool
nonBoolType string // "int", "str", "duration", "json"
unset bool
nonBoolWEq bool // use equal sign when setting value
nonBoolVal string
}
func (tca testCLIApplyer) Apply(ss mtest.State) (mtest.State, error) {
s := ss.(testCLIState)
// the tca needs to get added to its cfg as a Param
thisCfg := s.availCfgs[tca.availCfgI]
p := Param{
Name: tca.name,
IsString: tca.nonBoolType == "str" || tca.nonBoolType == "duration",
IsBool: tca.isBool,
// the cli parser doesn't actually care about the other fields of Param,
// those are only used by Cfg once it has all ParamValues together
}
thisCfg.ParamAdd(p)
// if the arg is set then add it to the cli args and the expected output pvs
if !tca.unset {
arg := cliKeyPrefix
if len(tca.path) > 0 {
arg += strings.Join(tca.path, cliKeyJoin) + cliKeyJoin
}
arg += tca.name
if !tca.isBool {
if tca.nonBoolWEq {
arg += "="
} else {
s.SourceCLI.Args = append(s.SourceCLI.Args, arg)
arg = ""
}
arg += tca.nonBoolVal
}
s.SourceCLI.Args = append(s.SourceCLI.Args, arg)
log.Print(strings.Join(s.SourceCLI.Args, " "))
pv := ParamValue{
Param: p,
Path: tca.path,
}
if tca.isBool {
pv.Value = json.RawMessage("true")
} else {
switch tca.nonBoolType {
case "str", "duration":
pv.Value = json.RawMessage(fmt.Sprintf("%q", tca.nonBoolVal))
case "int", "json":
pv.Value = json.RawMessage(tca.nonBoolVal)
default:
panic("shouldn't get here")
}
}
s.expPVs = append(s.expPVs, pv)
}
// and finally the state needs to be checked
gotPVs, err := s.SourceCLI.Parse(s.cfg)
if err != nil {
return nil, err
}
return s, massert.All(
massert.Len(gotPVs, len(s.expPVs)),
massert.Subset(s.expPVs, gotPVs),
).Assert()
}
func TestSourceCLI(t *T) { func TestSourceCLI(t *T) {
chk := mtest.Checker{ type state struct {
Init: func() mtest.State { cfg *Cfg
var s testCLIState availCfgs []*Cfg
SourceCLI
expPVs []ParamValue
}
type params struct {
name string
availCfgI int // not technically needed, but makes subsequent steps easier
path []string
isBool bool
nonBoolType string // "int", "str", "duration", "json"
unset bool
nonBoolWEq bool // use equal sign when setting value
nonBoolVal string
}
chk := mchk.Checker{
Init: func() mchk.State {
var s state
s.cfg = New() s.cfg = New()
{ {
a := s.cfg.Child("a") a := s.cfg.Child("a")
@ -144,58 +80,120 @@ func TestSourceCLI(t *T) {
s.SourceCLI.Args = make([]string, 0, 16) s.SourceCLI.Args = make([]string, 0, 16)
return s return s
}, },
Actions: func(ss mtest.State) []mtest.Action { Next: func(ss mchk.State) mchk.Action {
s := ss.(testCLIState) s := ss.(state)
var tca testCLIApplyer var p params
if i := mrand.Intn(8); i == 0 { if i := mrand.Intn(8); i == 0 {
tca.name = mrand.Hex(1) + "-" + mrand.Hex(8) p.name = mrand.Hex(1) + "-" + mrand.Hex(8)
} else if i == 1 { } else if i == 1 {
tca.name = mrand.Hex(1) + "=" + mrand.Hex(8) p.name = mrand.Hex(1) + "=" + mrand.Hex(8)
} else { } else {
tca.name = mrand.Hex(8) p.name = mrand.Hex(8)
} }
tca.availCfgI = mrand.Intn(len(s.availCfgs)) p.availCfgI = mrand.Intn(len(s.availCfgs))
thisCfg := s.availCfgs[tca.availCfgI] thisCfg := s.availCfgs[p.availCfgI]
tca.path = thisCfg.Path p.path = thisCfg.Path
tca.isBool = mrand.Intn(2) == 0 p.isBool = mrand.Intn(2) == 0
if !tca.isBool { if !p.isBool {
tca.nonBoolType = mrand.Element([]string{ p.nonBoolType = mrand.Element([]string{
"int", "int",
"str", "str",
"duration", "duration",
"json", "json",
}, nil).(string) }, nil).(string)
} }
tca.unset = mrand.Intn(10) == 0 p.unset = mrand.Intn(10) == 0
if tca.isBool || tca.unset { if p.isBool || p.unset {
return []mtest.Action{{Applyer: tca}} return mchk.Action{Params: p}
} }
tca.nonBoolWEq = mrand.Intn(2) == 0 p.nonBoolWEq = mrand.Intn(2) == 0
switch tca.nonBoolType { switch p.nonBoolType {
case "int": case "int":
tca.nonBoolVal = fmt.Sprint(mrand.Int()) p.nonBoolVal = fmt.Sprint(mrand.Int())
case "str": case "str":
tca.nonBoolVal = mrand.Hex(16) p.nonBoolVal = mrand.Hex(16)
case "duration": case "duration":
dur := time.Duration(mrand.Intn(86400)) * time.Second dur := time.Duration(mrand.Intn(86400)) * time.Second
tca.nonBoolVal = dur.String() p.nonBoolVal = dur.String()
case "json": case "json":
b, _ := json.Marshal(map[string]int{ b, _ := json.Marshal(map[string]int{
mrand.Hex(4): mrand.Int(), mrand.Hex(4): mrand.Int(),
mrand.Hex(4): mrand.Int(), mrand.Hex(4): mrand.Int(),
mrand.Hex(4): mrand.Int(), mrand.Hex(4): mrand.Int(),
}) })
tca.nonBoolVal = string(b) p.nonBoolVal = string(b)
} }
return []mtest.Action{{Applyer: tca}} return mchk.Action{Params: p}
},
Apply: func(ss mchk.State, a mchk.Action) (mchk.State, error) {
s := ss.(state)
p := a.Params.(params)
// the param needs to get added to its cfg as a Param
thisCfg := s.availCfgs[p.availCfgI]
cfgP := Param{
Name: p.name,
IsString: p.nonBoolType == "str" || p.nonBoolType == "duration",
IsBool: p.isBool,
// the cli parser doesn't actually care about the other fields of Param,
// those are only used by Cfg once it has all ParamValues together
}
thisCfg.ParamAdd(cfgP)
// if the arg is set then add it to the cli args and the expected output pvs
if !p.unset {
arg := cliKeyPrefix
if len(p.path) > 0 {
arg += strings.Join(p.path, cliKeyJoin) + cliKeyJoin
}
arg += p.name
if !p.isBool {
if p.nonBoolWEq {
arg += "="
} else {
s.SourceCLI.Args = append(s.SourceCLI.Args, arg)
arg = ""
}
arg += p.nonBoolVal
}
s.SourceCLI.Args = append(s.SourceCLI.Args, arg)
pv := ParamValue{
Param: cfgP,
Path: p.path,
}
if p.isBool {
pv.Value = json.RawMessage("true")
} else {
switch p.nonBoolType {
case "str", "duration":
pv.Value = json.RawMessage(fmt.Sprintf("%q", p.nonBoolVal))
case "int", "json":
pv.Value = json.RawMessage(p.nonBoolVal)
default:
panic("shouldn't get here")
}
}
s.expPVs = append(s.expPVs, pv)
}
// and finally the state needs to be checked
gotPVs, err := s.SourceCLI.Parse(s.cfg)
if err != nil {
return nil, err
}
return s, massert.All(
massert.Len(gotPVs, len(s.expPVs)),
massert.Subset(s.expPVs, gotPVs),
).Assert()
}, },
} }
if err := chk.RunUntil(5 * time.Second); err != nil { if err := chk.RunFor(5 * time.Second); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }

159
mtest/checker.go
View File

@ -1,159 +0,0 @@
package mtest
import (
"bytes"
"fmt"
"strings"
"time"
"github.com/mediocregopher/mediocre-go-lib/mrand"
)
// CheckerErr represents an test case error which was returned by a Checker Run.
//
// The string form of CheckerErr includes the sequence of Applyers which can be
// copy-pasted directly into Checker's RunCase method's arguments.
type CheckerErr struct {
// The sequence of applied actions which generated the error
Applied []Applyer
// The error returned by the final Action
Err error
}
func (ce CheckerErr) Error() string {
typeName := func(a Applyer) string {
t := fmt.Sprintf("%T", a)
return strings.SplitN(t, ".", 2)[1] // remove the package name
}
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "Test case: []mtest.Applyer{\n")
for _, a := range ce.Applied {
fmt.Fprintf(buf, "\t%s(%#v),\n", typeName(a), a)
}
fmt.Fprintf(buf, "}\n")
fmt.Fprintf(buf, "Generated error: %s\n", ce.Err)
return buf.String()
}
// State represents the current state of a Checker run. It can be represented by
// any value convenient and useful to the test.
type State interface{}
// Applyer performs the applies an Action's changes to a State, returning the
// new State. After modifying the State the Applyer should also assert that the
// new State is what it's expected to be, returning an error if it's not.
type Applyer interface {
Apply(State) (State, error)
}
// Action describes a change which can take place on a state. It must contain an
// Applyer which will peform the actual change.
type Action struct {
Applyer
// Weight is used when the Checker is choosing which Action to apply on
// every loop. If not set it is assumed to be 1, and can be increased
// further to increase its chances of being picked.
Weight uint64
// Incomplete can be set to true to indicate that this Action should never
// be the last Action applied, even if that means the maxDepth of the Run is
// gone over.
Incomplete bool
// Terminate can be set to true to indicate that this Action should always
// be the last Action applied, even if that means the maxDepth hasn't been
// reached yet.
Terminate bool
}
// Checker implements a very basic property checker. It generates random test
// cases, attempting to find and print out failing ones.
type Checker struct {
// Init returns the initial state of the test. It should always return the
// exact same value.
Init func() State
// Actions returns possible Actions which could be applied to the given
// State. This is called after Init and after every subsequent Action is
// applied.
Actions func(State) []Action
// MaxLength indicates the maximum number of Actions which can be strung
// together in a single Run. Defaults to 10 if not set.
MaxLength int
}
func (c Checker) withDefaults() Checker {
if c.MaxLength == 0 {
c.MaxLength = 10
}
return c
}
// RunUntil performs Runs in a loop until maxDuration has elapsed.
func (c Checker) RunUntil(maxDuration time.Duration) error {
doneTimer := time.After(maxDuration)
for {
select {
case <-doneTimer:
return nil
default:
}
if err := c.Run(); err != nil {
return err
}
}
}
// Run generates a single sequence of Actions and applies them in order,
// returning nil once the number of Actions performed has reached MaxLength or a
// CheckErr if an error is returned.
func (c Checker) Run() error {
c = c.withDefaults()
s := c.Init()
applied := make([]Applyer, 0, c.MaxLength)
for {
actions := c.Actions(s)
action := mrand.Element(actions, func(i int) uint64 {
if actions[i].Weight == 0 {
return 1
}
return actions[i].Weight
}).(Action)
var err error
s, err = action.Apply(s)
applied = append(applied, action.Applyer)
if err != nil {
return CheckerErr{
Applied: applied,
Err: err,
}
} else if action.Incomplete {
continue
} else if action.Terminate || len(applied) >= c.MaxLength {
return nil
}
}
}
// RunCase performs a single sequence of Applyers in order, returning a CheckErr
// if one is returned by one of the Applyers.
func (c Checker) RunCase(aa ...Applyer) error {
s := c.Init()
for i := range aa {
var err error
if s, err = aa[i].Apply(s); err != nil {
return CheckerErr{
Applied: aa[:i+1],
Err: err,
}
}
}
return nil
}

55
mtest/checker_test.go
View File

@ -1,55 +0,0 @@
package mtest
import (
"errors"
. "testing"
"time"
)
type testCheckerRunIncr int
func (i testCheckerRunIncr) Apply(s State) (State, error) {
si := s.(int) + int(i)
if si > 5 {
return nil, errors.New("went over 5")
}
return si, nil
}
func TestCheckerRun(t *T) {
c := Checker{
Init: func() State { return 0 },
Actions: func(State) []Action {
return []Action{
{
Applyer: testCheckerRunIncr(1),
Weight: 2,
},
{
Applyer: testCheckerRunIncr(-1),
},
}
},
MaxLength: 4,
}
// 4 Actions should never be able to go over 5
if err := c.RunUntil(time.Second); err != nil {
t.Fatal(err)
}
// 20 should always go over 5 eventually
c.MaxLength = 20
err := c.RunUntil(time.Second)
if err == nil {
t.Fatal("expected error when maxDepth is 20")
} else if len(err.(CheckerErr).Applied) < 6 {
t.Fatalf("strange CheckerErr when maxDepth is 20: %s", err)
}
t.Logf("got expected error with large maxDepth:\n%s", err)
caseErr := c.RunCase(err.(CheckerErr).Applied...)
if caseErr == nil || err.Error() != caseErr.Error() {
t.Fatalf("unexpected caseErr: %v", caseErr)
}
}

168
mtest/mchk/mchk.go Normal file
View File

@ -0,0 +1,168 @@
// Package mchk implements a framework for writing property checker tests, where
// test cases are generated randomly and performed, and failing test cases are
// output in a way so as to easily be able to rerun them.
//
// The central type of the package is Checker. For every Run call on Checker a
// new initial State is generated, and then an Action is generated off of that.
// The Action is applied to the State to obtain a new State, and a new Action is
// generated from there, and so on. If any Action fails it is output along with
// all of the Actions leading up to it.
package mchk
import (
"bytes"
"fmt"
"strings"
"time"
)
// RunErr represents an test case error which was returned by a Checker Run.
//
// The string form of RunErr includes the sequence of Params which can be
// copy-pasted directly into Checker's RunCase method's arguments.
type RunErr struct {
// The sequence of Action Params which generated the error
Params []Params
// The error returned by the final Action
Err error
}
func (ce RunErr) Error() string {
typeName := func(p Params) string {
t := fmt.Sprintf("%T", p)
tSplit := strings.SplitN(t, ".", 2)
if len(tSplit) > 1 {
return tSplit[1] // remove the package name
}
return tSplit[0]
}
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "Test case: []mtest.Params{\n")
for _, p := range ce.Params {
fmt.Fprintf(buf, "\t%s(%#v),\n", typeName(p), p)
}
fmt.Fprintf(buf, "}\n")
fmt.Fprintf(buf, "Generated error: %s\n", ce.Err)
return buf.String()
}
// State represents the current state of a Checker run. It can be any value
// convenient and useful to the test.
type State interface{}
// Params represent the parameters to an Action used during a Checker run. It
// should be a static value, meaning no pointers or channels.
type Params interface{}
// Action describes a change which can take place on a state.
type Action struct {
// Params are defined by the test and affect the behavior of the Action.
Params Params
// Incomplete can be set to true to indicate that this Action should never
// be the last Action applied, even if that means the length of the Run goes
// over MaxLength.
Incomplete bool
// Terminate can be set to true to indicate that this Action should always
// be the last Action applied, even if the Run's length hasn't reached
// MaxLength yet.
Terminate bool
}
// Checker implements a very basic property checker. It generates random test
// cases, attempting to find and print out failing ones.
type Checker struct {
// Init returns the initial state of the test. It should always return the
// exact same value.
Init func() State
// Next returns a new Action which can be Apply'd to the given State. This
// function should not modify the State in any way.
Next func(State) Action
// Apply performs the Action's changes to a State, returning the new State.
// After modifying the State this function should also assert that the new
// State is what it's expected to be, returning an error if it's not.
Apply func(State, Action) (State, error)
// Cleanup is an optional function which can perform any necessary cleanup
// operations on the State. This is called even on error.
Cleanup func(State)
// MaxLength indicates the maximum number of Actions which can be strung
// together in a single Run. Defaults to 10 if not set.
MaxLength int
}
func (c Checker) withDefaults() Checker {
if c.MaxLength == 0 {
c.MaxLength = 10
}
return c
}
// RunFor performs Runs in a loop until maxDuration has elapsed.
func (c Checker) RunFor(maxDuration time.Duration) error {
doneTimer := time.After(maxDuration)
for {
select {
case <-doneTimer:
return nil
default:
}
if err := c.Run(); err != nil {
return err
}
}
}
// Run generates a single sequence of Actions and applies them in order,
// returning nil once the number of Actions performed has reached MaxLength or a
// CheckErr if an error is returned.
func (c Checker) Run() error {
c = c.withDefaults()
s := c.Init()
params := make([]Params, 0, c.MaxLength)
for {
action := c.Next(s)
var err error
s, err = c.Apply(s, action)
params = append(params, action.Params)
if err != nil {
return RunErr{
Params: params,
Err: err,
}
} else if action.Incomplete {
continue
} else if action.Terminate || len(params) >= c.MaxLength {
return nil
}
}
}
// RunCase performs a single sequence of Actions with the given Params.
func (c Checker) RunCase(params ...Params) error {
s := c.Init()
if c.Cleanup != nil {
// wrap in a function so we don't capture the value of s right here
defer func() {
c.Cleanup(s)
}()
}
for i := range params {
var err error
if s, err = c.Apply(s, Action{Params: params[i]}); err != nil {
return RunErr{
Params: params[:i+1],
Err: err,
}
}
}
return nil
}

49
mtest/mchk/mchk_test.go Normal file
View File

@ -0,0 +1,49 @@
package mchk
import (
"errors"
. "testing"
"time"
"github.com/mediocregopher/mediocre-go-lib/mrand"
)
func TestCheckerRun(t *T) {
c := Checker{
Init: func() State { return 0 },
Next: func(State) Action {
if mrand.Intn(3) == 0 {
return Action{Params: -1}
}
return Action{Params: 1}
},
Apply: func(s State, a Action) (State, error) {
si := s.(int) + a.Params.(int)
if si > 5 {
return nil, errors.New("went over 5")
}
return si, nil
},
MaxLength: 4,
}
// 4 Actions should never be able to go over 5
if err := c.RunFor(time.Second); err != nil {
t.Fatal(err)
}
// 20 should always go over 5 eventually
c.MaxLength = 20
err := c.RunFor(time.Second)
if err == nil {
t.Fatal("expected error when maxDepth is 20")
} else if len(err.(RunErr).Params) < 6 {
t.Fatalf("strange RunErr when maxDepth is 20: %s", err)
}
t.Logf("got expected error with large maxDepth:\n%s", err)
caseErr := c.RunCase(err.(RunErr).Params...)
if caseErr == nil || err.Error() != caseErr.Error() {
t.Fatalf("unexpected caseErr: %v", caseErr)
}
}