From 715b6c94913b5396ac12b64ce47741c4d777bab2 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 13 Aug 2018 15:03:30 -0400 Subject: [PATCH] mchk: move mtest.Checker to mtest/mchk, and refactor its types a little bit --- mcfg/cli_test.go | 218 ++++++++++++++++++++-------------------- mtest/checker.go | 159 ----------------------------- mtest/checker_test.go | 55 ---------- mtest/mchk/mchk.go | 168 +++++++++++++++++++++++++++++++ mtest/mchk/mchk_test.go | 49 +++++++++ 5 files changed, 325 insertions(+), 324 deletions(-) delete mode 100644 mtest/checker.go delete mode 100644 mtest/checker_test.go create mode 100644 mtest/mchk/mchk.go create mode 100644 mtest/mchk/mchk_test.go diff --git a/mcfg/cli_test.go b/mcfg/cli_test.go index f4b88e5..3dd3b27 100644 --- a/mcfg/cli_test.go +++ b/mcfg/cli_test.go @@ -4,14 +4,13 @@ import ( "bytes" "encoding/json" "fmt" - "log" "strings" . "testing" "time" "github.com/mediocregopher/mediocre-go-lib/mrand" - "github.com/mediocregopher/mediocre-go-lib/mtest" "github.com/mediocregopher/mediocre-go-lib/mtest/massert" + "github.com/mediocregopher/mediocre-go-lib/mtest/mchk" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,92 +44,29 @@ func TestSourceCLIHelp(t *T) { assert.Equal(t, exp, buf.String()) } -type testCLIState struct { - cfg *Cfg - availCfgs []*Cfg - - SourceCLI - expPVs []ParamValue -} - -type testCLIApplyer struct { - name string - availCfgI int // not technically needed, but makes subsequent steps easier - path []string - isBool bool - nonBoolType string // "int", "str", "duration", "json" - unset bool - nonBoolWEq bool // use equal sign when setting value - nonBoolVal string -} - -func (tca testCLIApplyer) Apply(ss mtest.State) (mtest.State, error) { - s := ss.(testCLIState) - - // the tca needs to get added to its cfg as a Param - thisCfg := s.availCfgs[tca.availCfgI] - p := Param{ - Name: tca.name, - IsString: tca.nonBoolType == "str" || tca.nonBoolType == "duration", - IsBool: tca.isBool, - // the cli parser doesn't actually care about the other fields of Param, - // those are only used by Cfg once it has all ParamValues together - } - thisCfg.ParamAdd(p) - - // if the arg is set then add it to the cli args and the expected output pvs - if !tca.unset { - arg := cliKeyPrefix - if len(tca.path) > 0 { - arg += strings.Join(tca.path, cliKeyJoin) + cliKeyJoin - } - arg += tca.name - if !tca.isBool { - if tca.nonBoolWEq { - arg += "=" - } else { - s.SourceCLI.Args = append(s.SourceCLI.Args, arg) - arg = "" - } - arg += tca.nonBoolVal - } - s.SourceCLI.Args = append(s.SourceCLI.Args, arg) - log.Print(strings.Join(s.SourceCLI.Args, " ")) - - pv := ParamValue{ - Param: p, - Path: tca.path, - } - if tca.isBool { - pv.Value = json.RawMessage("true") - } else { - switch tca.nonBoolType { - case "str", "duration": - pv.Value = json.RawMessage(fmt.Sprintf("%q", tca.nonBoolVal)) - case "int", "json": - pv.Value = json.RawMessage(tca.nonBoolVal) - default: - panic("shouldn't get here") - } - } - s.expPVs = append(s.expPVs, pv) - } - - // and finally the state needs to be checked - gotPVs, err := s.SourceCLI.Parse(s.cfg) - if err != nil { - return nil, err - } - return s, massert.All( - massert.Len(gotPVs, len(s.expPVs)), - massert.Subset(s.expPVs, gotPVs), - ).Assert() -} - func TestSourceCLI(t *T) { - chk := mtest.Checker{ - Init: func() mtest.State { - var s testCLIState + type state struct { + cfg *Cfg + availCfgs []*Cfg + + SourceCLI + expPVs []ParamValue + } + + type params struct { + name string + availCfgI int // not technically needed, but makes subsequent steps easier + path []string + isBool bool + nonBoolType string // "int", "str", "duration", "json" + unset bool + nonBoolWEq bool // use equal sign when setting value + nonBoolVal string + } + + chk := mchk.Checker{ + Init: func() mchk.State { + var s state s.cfg = New() { a := s.cfg.Child("a") @@ -144,58 +80,120 @@ func TestSourceCLI(t *T) { s.SourceCLI.Args = make([]string, 0, 16) return s }, - Actions: func(ss mtest.State) []mtest.Action { - s := ss.(testCLIState) - var tca testCLIApplyer + Next: func(ss mchk.State) mchk.Action { + s := ss.(state) + var p params if i := mrand.Intn(8); i == 0 { - tca.name = mrand.Hex(1) + "-" + mrand.Hex(8) + p.name = mrand.Hex(1) + "-" + mrand.Hex(8) } else if i == 1 { - tca.name = mrand.Hex(1) + "=" + mrand.Hex(8) + p.name = mrand.Hex(1) + "=" + mrand.Hex(8) } else { - tca.name = mrand.Hex(8) + p.name = mrand.Hex(8) } - tca.availCfgI = mrand.Intn(len(s.availCfgs)) - thisCfg := s.availCfgs[tca.availCfgI] - tca.path = thisCfg.Path + p.availCfgI = mrand.Intn(len(s.availCfgs)) + thisCfg := s.availCfgs[p.availCfgI] + p.path = thisCfg.Path - tca.isBool = mrand.Intn(2) == 0 - if !tca.isBool { - tca.nonBoolType = mrand.Element([]string{ + p.isBool = mrand.Intn(2) == 0 + if !p.isBool { + p.nonBoolType = mrand.Element([]string{ "int", "str", "duration", "json", }, nil).(string) } - tca.unset = mrand.Intn(10) == 0 + p.unset = mrand.Intn(10) == 0 - if tca.isBool || tca.unset { - return []mtest.Action{{Applyer: tca}} + if p.isBool || p.unset { + return mchk.Action{Params: p} } - tca.nonBoolWEq = mrand.Intn(2) == 0 - switch tca.nonBoolType { + p.nonBoolWEq = mrand.Intn(2) == 0 + switch p.nonBoolType { case "int": - tca.nonBoolVal = fmt.Sprint(mrand.Int()) + p.nonBoolVal = fmt.Sprint(mrand.Int()) case "str": - tca.nonBoolVal = mrand.Hex(16) + p.nonBoolVal = mrand.Hex(16) case "duration": dur := time.Duration(mrand.Intn(86400)) * time.Second - tca.nonBoolVal = dur.String() + p.nonBoolVal = dur.String() case "json": b, _ := json.Marshal(map[string]int{ mrand.Hex(4): mrand.Int(), mrand.Hex(4): mrand.Int(), mrand.Hex(4): mrand.Int(), }) - tca.nonBoolVal = string(b) + p.nonBoolVal = string(b) } - return []mtest.Action{{Applyer: tca}} + return mchk.Action{Params: p} + }, + Apply: func(ss mchk.State, a mchk.Action) (mchk.State, error) { + s := ss.(state) + p := a.Params.(params) + + // the param needs to get added to its cfg as a Param + thisCfg := s.availCfgs[p.availCfgI] + cfgP := Param{ + Name: p.name, + IsString: p.nonBoolType == "str" || p.nonBoolType == "duration", + IsBool: p.isBool, + // the cli parser doesn't actually care about the other fields of Param, + // those are only used by Cfg once it has all ParamValues together + } + thisCfg.ParamAdd(cfgP) + + // if the arg is set then add it to the cli args and the expected output pvs + if !p.unset { + arg := cliKeyPrefix + if len(p.path) > 0 { + arg += strings.Join(p.path, cliKeyJoin) + cliKeyJoin + } + arg += p.name + if !p.isBool { + if p.nonBoolWEq { + arg += "=" + } else { + s.SourceCLI.Args = append(s.SourceCLI.Args, arg) + arg = "" + } + arg += p.nonBoolVal + } + s.SourceCLI.Args = append(s.SourceCLI.Args, arg) + + pv := ParamValue{ + Param: cfgP, + Path: p.path, + } + if p.isBool { + pv.Value = json.RawMessage("true") + } else { + switch p.nonBoolType { + case "str", "duration": + pv.Value = json.RawMessage(fmt.Sprintf("%q", p.nonBoolVal)) + case "int", "json": + pv.Value = json.RawMessage(p.nonBoolVal) + default: + panic("shouldn't get here") + } + } + s.expPVs = append(s.expPVs, pv) + } + + // and finally the state needs to be checked + gotPVs, err := s.SourceCLI.Parse(s.cfg) + if err != nil { + return nil, err + } + return s, massert.All( + massert.Len(gotPVs, len(s.expPVs)), + massert.Subset(s.expPVs, gotPVs), + ).Assert() }, } - if err := chk.RunUntil(5 * time.Second); err != nil { + if err := chk.RunFor(5 * time.Second); err != nil { t.Fatal(err) } } diff --git a/mtest/checker.go b/mtest/checker.go deleted file mode 100644 index 5fe672a..0000000 --- a/mtest/checker.go +++ /dev/null @@ -1,159 +0,0 @@ -package mtest - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/mediocregopher/mediocre-go-lib/mrand" -) - -// CheckerErr represents an test case error which was returned by a Checker Run. -// -// The string form of CheckerErr includes the sequence of Applyers which can be -// copy-pasted directly into Checker's RunCase method's arguments. -type CheckerErr struct { - // The sequence of applied actions which generated the error - Applied []Applyer - - // The error returned by the final Action - Err error -} - -func (ce CheckerErr) Error() string { - typeName := func(a Applyer) string { - t := fmt.Sprintf("%T", a) - return strings.SplitN(t, ".", 2)[1] // remove the package name - } - - buf := new(bytes.Buffer) - fmt.Fprintf(buf, "Test case: []mtest.Applyer{\n") - for _, a := range ce.Applied { - fmt.Fprintf(buf, "\t%s(%#v),\n", typeName(a), a) - } - fmt.Fprintf(buf, "}\n") - fmt.Fprintf(buf, "Generated error: %s\n", ce.Err) - return buf.String() -} - -// State represents the current state of a Checker run. It can be represented by -// any value convenient and useful to the test. -type State interface{} - -// Applyer performs the applies an Action's changes to a State, returning the -// new State. After modifying the State the Applyer should also assert that the -// new State is what it's expected to be, returning an error if it's not. -type Applyer interface { - Apply(State) (State, error) -} - -// Action describes a change which can take place on a state. It must contain an -// Applyer which will peform the actual change. -type Action struct { - Applyer - - // Weight is used when the Checker is choosing which Action to apply on - // every loop. If not set it is assumed to be 1, and can be increased - // further to increase its chances of being picked. - Weight uint64 - - // Incomplete can be set to true to indicate that this Action should never - // be the last Action applied, even if that means the maxDepth of the Run is - // gone over. - Incomplete bool - - // Terminate can be set to true to indicate that this Action should always - // be the last Action applied, even if that means the maxDepth hasn't been - // reached yet. - Terminate bool -} - -// Checker implements a very basic property checker. It generates random test -// cases, attempting to find and print out failing ones. -type Checker struct { - // Init returns the initial state of the test. It should always return the - // exact same value. - Init func() State - - // Actions returns possible Actions which could be applied to the given - // State. This is called after Init and after every subsequent Action is - // applied. - Actions func(State) []Action - - // MaxLength indicates the maximum number of Actions which can be strung - // together in a single Run. Defaults to 10 if not set. - MaxLength int -} - -func (c Checker) withDefaults() Checker { - if c.MaxLength == 0 { - c.MaxLength = 10 - } - return c -} - -// RunUntil performs Runs in a loop until maxDuration has elapsed. -func (c Checker) RunUntil(maxDuration time.Duration) error { - doneTimer := time.After(maxDuration) - for { - select { - case <-doneTimer: - return nil - default: - } - - if err := c.Run(); err != nil { - return err - } - } -} - -// Run generates a single sequence of Actions and applies them in order, -// returning nil once the number of Actions performed has reached MaxLength or a -// CheckErr if an error is returned. -func (c Checker) Run() error { - c = c.withDefaults() - s := c.Init() - applied := make([]Applyer, 0, c.MaxLength) - for { - actions := c.Actions(s) - action := mrand.Element(actions, func(i int) uint64 { - if actions[i].Weight == 0 { - return 1 - } - return actions[i].Weight - }).(Action) - - var err error - s, err = action.Apply(s) - applied = append(applied, action.Applyer) - - if err != nil { - return CheckerErr{ - Applied: applied, - Err: err, - } - } else if action.Incomplete { - continue - } else if action.Terminate || len(applied) >= c.MaxLength { - return nil - } - } -} - -// RunCase performs a single sequence of Applyers in order, returning a CheckErr -// if one is returned by one of the Applyers. -func (c Checker) RunCase(aa ...Applyer) error { - s := c.Init() - for i := range aa { - var err error - if s, err = aa[i].Apply(s); err != nil { - return CheckerErr{ - Applied: aa[:i+1], - Err: err, - } - } - } - return nil -} diff --git a/mtest/checker_test.go b/mtest/checker_test.go deleted file mode 100644 index cbb5532..0000000 --- a/mtest/checker_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package mtest - -import ( - "errors" - . "testing" - "time" -) - -type testCheckerRunIncr int - -func (i testCheckerRunIncr) Apply(s State) (State, error) { - si := s.(int) + int(i) - if si > 5 { - return nil, errors.New("went over 5") - } - return si, nil -} - -func TestCheckerRun(t *T) { - c := Checker{ - Init: func() State { return 0 }, - Actions: func(State) []Action { - return []Action{ - { - Applyer: testCheckerRunIncr(1), - Weight: 2, - }, - { - Applyer: testCheckerRunIncr(-1), - }, - } - }, - MaxLength: 4, - } - - // 4 Actions should never be able to go over 5 - if err := c.RunUntil(time.Second); err != nil { - t.Fatal(err) - } - - // 20 should always go over 5 eventually - c.MaxLength = 20 - err := c.RunUntil(time.Second) - if err == nil { - t.Fatal("expected error when maxDepth is 20") - } else if len(err.(CheckerErr).Applied) < 6 { - t.Fatalf("strange CheckerErr when maxDepth is 20: %s", err) - } - - t.Logf("got expected error with large maxDepth:\n%s", err) - caseErr := c.RunCase(err.(CheckerErr).Applied...) - if caseErr == nil || err.Error() != caseErr.Error() { - t.Fatalf("unexpected caseErr: %v", caseErr) - } -} diff --git a/mtest/mchk/mchk.go b/mtest/mchk/mchk.go new file mode 100644 index 0000000..21a1468 --- /dev/null +++ b/mtest/mchk/mchk.go @@ -0,0 +1,168 @@ +// Package mchk implements a framework for writing property checker tests, where +// test cases are generated randomly and performed, and failing test cases are +// output in a way so as to easily be able to rerun them. +// +// The central type of the package is Checker. For every Run call on Checker a +// new initial State is generated, and then an Action is generated off of that. +// The Action is applied to the State to obtain a new State, and a new Action is +// generated from there, and so on. If any Action fails it is output along with +// all of the Actions leading up to it. +package mchk + +import ( + "bytes" + "fmt" + "strings" + "time" +) + +// RunErr represents an test case error which was returned by a Checker Run. +// +// The string form of RunErr includes the sequence of Params which can be +// copy-pasted directly into Checker's RunCase method's arguments. +type RunErr struct { + // The sequence of Action Params which generated the error + Params []Params + + // The error returned by the final Action + Err error +} + +func (ce RunErr) Error() string { + typeName := func(p Params) string { + t := fmt.Sprintf("%T", p) + tSplit := strings.SplitN(t, ".", 2) + if len(tSplit) > 1 { + return tSplit[1] // remove the package name + } + return tSplit[0] + } + + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "Test case: []mtest.Params{\n") + for _, p := range ce.Params { + fmt.Fprintf(buf, "\t%s(%#v),\n", typeName(p), p) + } + fmt.Fprintf(buf, "}\n") + fmt.Fprintf(buf, "Generated error: %s\n", ce.Err) + return buf.String() +} + +// State represents the current state of a Checker run. It can be any value +// convenient and useful to the test. +type State interface{} + +// Params represent the parameters to an Action used during a Checker run. It +// should be a static value, meaning no pointers or channels. +type Params interface{} + +// Action describes a change which can take place on a state. +type Action struct { + // Params are defined by the test and affect the behavior of the Action. + Params Params + + // Incomplete can be set to true to indicate that this Action should never + // be the last Action applied, even if that means the length of the Run goes + // over MaxLength. + Incomplete bool + + // Terminate can be set to true to indicate that this Action should always + // be the last Action applied, even if the Run's length hasn't reached + // MaxLength yet. + Terminate bool +} + +// Checker implements a very basic property checker. It generates random test +// cases, attempting to find and print out failing ones. +type Checker struct { + // Init returns the initial state of the test. It should always return the + // exact same value. + Init func() State + + // Next returns a new Action which can be Apply'd to the given State. This + // function should not modify the State in any way. + Next func(State) Action + + // Apply performs the Action's changes to a State, returning the new State. + // After modifying the State this function should also assert that the new + // State is what it's expected to be, returning an error if it's not. + Apply func(State, Action) (State, error) + + // Cleanup is an optional function which can perform any necessary cleanup + // operations on the State. This is called even on error. + Cleanup func(State) + + // MaxLength indicates the maximum number of Actions which can be strung + // together in a single Run. Defaults to 10 if not set. + MaxLength int +} + +func (c Checker) withDefaults() Checker { + if c.MaxLength == 0 { + c.MaxLength = 10 + } + return c +} + +// RunFor performs Runs in a loop until maxDuration has elapsed. +func (c Checker) RunFor(maxDuration time.Duration) error { + doneTimer := time.After(maxDuration) + for { + select { + case <-doneTimer: + return nil + default: + } + + if err := c.Run(); err != nil { + return err + } + } +} + +// Run generates a single sequence of Actions and applies them in order, +// returning nil once the number of Actions performed has reached MaxLength or a +// CheckErr if an error is returned. +func (c Checker) Run() error { + c = c.withDefaults() + s := c.Init() + params := make([]Params, 0, c.MaxLength) + for { + action := c.Next(s) + var err error + s, err = c.Apply(s, action) + params = append(params, action.Params) + + if err != nil { + return RunErr{ + Params: params, + Err: err, + } + } else if action.Incomplete { + continue + } else if action.Terminate || len(params) >= c.MaxLength { + return nil + } + } +} + +// RunCase performs a single sequence of Actions with the given Params. +func (c Checker) RunCase(params ...Params) error { + s := c.Init() + if c.Cleanup != nil { + // wrap in a function so we don't capture the value of s right here + defer func() { + c.Cleanup(s) + }() + } + for i := range params { + var err error + if s, err = c.Apply(s, Action{Params: params[i]}); err != nil { + return RunErr{ + Params: params[:i+1], + Err: err, + } + } + } + return nil +} diff --git a/mtest/mchk/mchk_test.go b/mtest/mchk/mchk_test.go new file mode 100644 index 0000000..2df7c31 --- /dev/null +++ b/mtest/mchk/mchk_test.go @@ -0,0 +1,49 @@ +package mchk + +import ( + "errors" + . "testing" + "time" + + "github.com/mediocregopher/mediocre-go-lib/mrand" +) + +func TestCheckerRun(t *T) { + c := Checker{ + Init: func() State { return 0 }, + Next: func(State) Action { + if mrand.Intn(3) == 0 { + return Action{Params: -1} + } + return Action{Params: 1} + }, + Apply: func(s State, a Action) (State, error) { + si := s.(int) + a.Params.(int) + if si > 5 { + return nil, errors.New("went over 5") + } + return si, nil + }, + MaxLength: 4, + } + + // 4 Actions should never be able to go over 5 + if err := c.RunFor(time.Second); err != nil { + t.Fatal(err) + } + + // 20 should always go over 5 eventually + c.MaxLength = 20 + err := c.RunFor(time.Second) + if err == nil { + t.Fatal("expected error when maxDepth is 20") + } else if len(err.(RunErr).Params) < 6 { + t.Fatalf("strange RunErr when maxDepth is 20: %s", err) + } + + t.Logf("got expected error with large maxDepth:\n%s", err) + caseErr := c.RunCase(err.(RunErr).Params...) + if caseErr == nil || err.Error() != caseErr.Error() { + t.Fatalf("unexpected caseErr: %v", caseErr) + } +}