mediocre-go-lib/mtest/mchk/mchk.go

// Package mchk implements a framework for writing property checker tests, where
// test cases are generated randomly and performed, and failing test cases are
// output in a way so as to easily be able to rerun them. In addition failing
// test cases are minimized so the smallest possible case is returned.
//
// The central type of the package is Checker. For every Run call on Checker a
// new initial State is generated, and then an Action is generated off of that.
// The Action is applied to the State to obtain a new State, and a new Action is
// generated from there, and so on. If any Action fails it is output along with
// all of the Actions leading up to it.
package mchk

import (
	"bytes"
	"fmt"
	"time"

	"github.com/mediocregopher/mediocre-go-lib/mrand"
)

// RunErr represents an test case error which was returned by a Checker Run.
//
// The string form of RunErr includes the sequence of Params which can be
// copy-pasted directly into Checker's RunCase method's arguments.
type RunErr struct {
	// The sequence of Action Params which generated the error
	Params []Params

	// The error returned by the final Action
	Err error
}

func (ce RunErr) Error() string {
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "Test case: []mtest.Params{\n")
	for _, p := range ce.Params {
		fmt.Fprintf(buf, "\t%#v,\n", p)
	}
	fmt.Fprintf(buf, "}\n")
	fmt.Fprintf(buf, "Generated error: %s\n", ce.Err)
	return buf.String()
}

// State represents the current state of a Checker run. It can be any value
// convenient and useful to the test.
type State interface{}

// Params represent the parameters to an Action used during a Checker run. It
// should be a static value, meaning no pointers or channels.
type Params interface{}

// Action describes a change which can take place on a state.
type Action struct {
	// Params are defined by the test and affect the behavior of the Action.
	Params Params

	// Incomplete can be set to true to indicate that this Action should never
	// be the last Action applied, even if that means the length of the Run goes
	// over MaxLength.
	Incomplete bool

	// Terminate can be set to true to indicate that this Action should always
	// be the last Action applied, even if the Run's length hasn't reached
	// MaxLength yet.
	Terminate bool
}

// Checker implements a very basic property checker. It generates random test
// cases, attempting to find and print out failing ones.
type Checker struct {
	// Init returns the initial state of the test. It should always return the
	// exact same value.
	Init func() State

	// Next returns a new Action which can be Apply'd to the given State. This
	// function should not modify the State in any way.
	Next func(State) Action

	// Apply performs the Action's changes to a State, returning the new State.
	// After modifying the State this function should also assert that the new
	// State is what it's expected to be, returning an error if it's not.
	Apply func(State, Action) (State, error)

	// Cleanup is an optional function which can perform any necessary cleanup
	// operations on the State. This is called even on error.
	Cleanup func(State)

	// MaxLength indicates the maximum number of Actions which can be strung
	// together in a single Run. Defaults to 10 if not set.
	MaxLength int

	// If true the Run and RunFor methods will return the first erroring Action
	// sequence, without trying to remove extraneous Actions from it first.
	DontMinimize bool
}

func (c Checker) withDefaults() Checker {
	if c.MaxLength == 0 {
		c.MaxLength = 10
	}
	return c
}

// RunFor performs Runs in a loop until maxDuration has elapsed.
func (c Checker) RunFor(maxDuration time.Duration) error {
	doneTimer := time.After(maxDuration)
	for {
		select {
		case <-doneTimer:
			return nil
		default:
		}

		if err := c.Run(); err != nil {
			return err
		}
	}
}

// Run generates a single sequence of Actions and applies them in order,
// returning nil once the number of Actions performed has reached MaxLength or a
// CheckErr if an error is returned. If an error is to be returned this will
// attempt to minimize the Actions sequence in order to find the smallest
// reproducible test case.
func (c Checker) Run() error {
	c = c.withDefaults()
	s := c.Init()
	params := make([]Params, 0, c.MaxLength)
	for {
		action := c.Next(s)
		var err error
		s, err = c.Apply(s, action)
		params = append(params, action.Params)

		if err != nil && c.DontMinimize {
			return RunErr{
				Params: params,
				Err:    err,
			}
		} else if err != nil {
			minParams := c.MinimizeCase(params...)
			if minErr := c.RunCase(minParams...); minErr != nil {
				// RunCase already wraps errs in RunErrs, so that's not
				// necessary here
				return minErr
			}
			// if the minParams didn't return an error here it means the test
			// case isn't consistent, as a fallback return the original which
			// definitely errored
			return RunErr{
				Params: params,
				Err:    err,
			}
		} else if action.Incomplete {
			continue
		} else if action.Terminate || len(params) >= c.MaxLength {
			return nil
		}
	}
}

// MinimizeCase repeatedly randomly picks a Param from the set and performs
// RunCase without that Param. It does this until it can't remove a single Param
// without the error ceasing, and returns that minimized set.
func (c Checker) MinimizeCase(params ...Params) []Params {
outer:
	for {
		if len(params) == 1 {
			return params
		}

		tried := map[int]bool{}
		for {
			if len(tried) == len(params) {
				return params
			}
			i := mrand.Intn(len(params))
			if tried[i] {
				continue
			}
			newParams := make([]Params, 0, len(params)-1)
			newParams = append(newParams, params[:i]...)
			newParams = append(newParams, params[i+1:]...)
			if err := c.RunCase(newParams...); err == nil {
				tried[i] = true
				continue
			}
			params = newParams
			continue outer
		}
	}
}

// RunCase performs a single sequence of Actions with the given Params.
func (c Checker) RunCase(params ...Params) error {
	s := c.Init()
	if c.Cleanup != nil {
		// wrap in a function so we don't capture the value of s right here
		defer func() {
			c.Cleanup(s)
		}()
	}
	for i := range params {
		var err error
		if s, err = c.Apply(s, Action{Params: params[i]}); err != nil {
			return RunErr{
				Params: params[:i+1],
				Err:    err,
			}
		}
	}
	return nil
}