mediocre-go-lib/mtest/mchk/mchk.go

214 lines
6.2 KiB
Go
Raw Normal View History

// Package mchk implements a framework for writing property checker tests, where
// test cases are generated randomly and performed, and failing test cases are
// output in a way so as to easily be able to rerun them. In addition failing
// test cases are minimized so the smallest possible case is returned.
//
// The central type of the package is Checker. For every Run call on Checker a
// new initial State is generated, and then an Action is generated off of that.
// The Action is applied to the State to obtain a new State, and a new Action is
// generated from there, and so on. If any Action fails it is output along with
// all of the Actions leading up to it.
package mchk
import (
"bytes"
"fmt"
"time"
"github.com/mediocregopher/mediocre-go-lib/mrand"
)
// RunErr represents an test case error which was returned by a Checker Run.
//
// The string form of RunErr includes the sequence of Params which can be
// copy-pasted directly into Checker's RunCase method's arguments.
type RunErr struct {
// The sequence of Action Params which generated the error
Params []Params
// The error returned by the final Action
Err error
}
func (ce RunErr) Error() string {
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "Test case: []mtest.Params{\n")
for _, p := range ce.Params {
fmt.Fprintf(buf, "\t%#v,\n", p)
}
fmt.Fprintf(buf, "}\n")
fmt.Fprintf(buf, "Generated error: %s\n", ce.Err)
return buf.String()
}
// State represents the current state of a Checker run. It can be any value
// convenient and useful to the test.
type State interface{}
// Params represent the parameters to an Action used during a Checker run. It
// should be a static value, meaning no pointers or channels.
type Params interface{}
// Action describes a change which can take place on a state.
type Action struct {
// Params are defined by the test and affect the behavior of the Action.
Params Params
// Incomplete can be set to true to indicate that this Action should never
// be the last Action applied, even if that means the length of the Run goes
// over MaxLength.
Incomplete bool
// Terminate can be set to true to indicate that this Action should always
// be the last Action applied, even if the Run's length hasn't reached
// MaxLength yet.
Terminate bool
}
// Checker implements a very basic property checker. It generates random test
// cases, attempting to find and print out failing ones.
type Checker struct {
// Init returns the initial state of the test. It should always return the
// exact same value.
Init func() State
// Next returns a new Action which can be Apply'd to the given State. This
// function should not modify the State in any way.
Next func(State) Action
// Apply performs the Action's changes to a State, returning the new State.
// After modifying the State this function should also assert that the new
// State is what it's expected to be, returning an error if it's not.
Apply func(State, Action) (State, error)
// Cleanup is an optional function which can perform any necessary cleanup
// operations on the State. This is called even on error.
Cleanup func(State)
// MaxLength indicates the maximum number of Actions which can be strung
// together in a single Run. Defaults to 10 if not set.
MaxLength int
// If true the Run and RunFor methods will return the first erroring Action
// sequence, without trying to remove extraneous Actions from it first.
DontMinimize bool
}
func (c Checker) withDefaults() Checker {
if c.MaxLength == 0 {
c.MaxLength = 10
}
return c
}
// RunFor performs Runs in a loop until maxDuration has elapsed.
func (c Checker) RunFor(maxDuration time.Duration) error {
doneTimer := time.After(maxDuration)
for {
select {
case <-doneTimer:
return nil
default:
}
if err := c.Run(); err != nil {
return err
}
}
}
// Run generates a single sequence of Actions and applies them in order,
// returning nil once the number of Actions performed has reached MaxLength or a
// CheckErr if an error is returned. If an error is to be returned this will
// attempt to minimize the Actions sequence in order to find the smallest
// reproducible test case.
func (c Checker) Run() error {
c = c.withDefaults()
s := c.Init()
params := make([]Params, 0, c.MaxLength)
for {
action := c.Next(s)
var err error
s, err = c.Apply(s, action)
params = append(params, action.Params)
if err != nil && c.DontMinimize {
return RunErr{
Params: params,
Err: err,
}
} else if err != nil {
minParams := c.MinimizeCase(params...)
if minErr := c.RunCase(minParams...); minErr != nil {
// RunCase already wraps errs in RunErrs, so that's not
// necessary here
return minErr
}
// if the minParams didn't return an error here it means the test
// case isn't consistent, as a fallback return the original which
// definitely errored
return RunErr{
Params: params,
Err: err,
}
} else if action.Incomplete {
continue
} else if action.Terminate || len(params) >= c.MaxLength {
return nil
}
}
}
// MinimizeCase repeatedly randomly picks a Param from the set and performs
// RunCase without that Param. It does this until it can't remove a single Param
// without the error ceasing, and returns that minimized set.
func (c Checker) MinimizeCase(params ...Params) []Params {
outer:
for {
if len(params) == 1 {
return params
}
tried := map[int]bool{}
for {
if len(tried) == len(params) {
return params
}
i := mrand.Intn(len(params))
if tried[i] {
continue
}
newParams := make([]Params, 0, len(params)-1)
newParams = append(newParams, params[:i]...)
newParams = append(newParams, params[i+1:]...)
if err := c.RunCase(newParams...); err == nil {
tried[i] = true
continue
}
params = newParams
continue outer
}
}
}
// RunCase performs a single sequence of Actions with the given Params.
func (c Checker) RunCase(params ...Params) error {
s := c.Init()
if c.Cleanup != nil {
// wrap in a function so we don't capture the value of s right here
defer func() {
c.Cleanup(s)
}()
}
for i := range params {
var err error
if s, err = c.Apply(s, Action{Params: params[i]}); err != nil {
return RunErr{
Params: params[:i+1],
Err: err,
}
}
}
return nil
}