Completely refactor gg with new BNF file and decoder

The new gg format is based on a BNF file which can be found in the `gg`
directory. The code for decoding `.gg` files has been refactored to
mirror that file. The result is more resilient parsing, better errors,
and a greater ability to extend the format in the future.

The new decoder is notable in that it does not use a lexer. Both lexing
and parsing are done in a single step.

The format syntax itself has also been modified. Rather than using
semi-colons everywhere, commas are used as separators in tuples.
Additionally the final comma/semi-colon is no longer required.

cmd/eval/main.go

@@ -19,10 +19,7 @@ func main() {
 	opSrc := os.Args[1]
 	inSrc := os.Args[2]
 
-	inVal, err := gg.DecodeSingleValueFromLexer(
+	inVal, err := gg.NewDecoder(bytes.NewBufferString(inSrc)).Next()
-		gg.NewLexer(bytes.NewBufferString(inSrc + ";")),
-	)
-
 	if err != nil {
 		panic(fmt.Sprintf("decoding input: %v", err))
 	}
@@ -32,7 +29,6 @@ func main() {
 		vm.Value{Value: inVal},
 		vm.GlobalScope,
 	)
-
 	if err != nil {
 		panic(fmt.Sprintf("evaluating: %v", err))
 	}

examples/fib.gg

@@ -1,19 +1,24 @@
-out = {
+* A function which accepts a number N and returns the Nth fibonacci number
+{
+    * We are passing a tuple of inputs into a graph here, such that the graph is
+    * evaluated as an anonymous function. That anonymous function uses recur
+    * internally to compute the result.
+    out = {
 
-    decr = { out = add < (in; -1;); };
+        * A little helper function.
+        decr = { out = add < (in, -1) };
 
-    n = tupEl < (in; 0;);
+        * Deconstruct the input tuple into its individual elements, for clarity.
-    a = tupEl < (in; 1;);
+        * There will be a more ergonomic way of doing this one day.
-    b = tupEl < (in; 2;);
+        n = tupEl < (in, 0);
+        a = tupEl < (in, 1);
+        b = tupEl < (in, 2);
 
         out = if < (
-        isZero < n;
+            isZero < n,
-        a;
+            a,
-        recur < (
+            recur < ( decr<n, b, add<(a,b) ),
-            decr < n;
-            b;
-            add < (a;b;);
-        );
         );
 
-} < (in; 0; 1;);
+    } < (in, 0, 1);
+}

gg/decoder.go

@@ -1,353 +1,73 @@
 package gg
 
 import (
-	"errors"
+	"bufio"
 	"fmt"
 	"io"
-	"strconv"
-
-	"github.com/mediocregopher/ginger/graph"
 )
 
-// Type aliases for convenience
+// Decoder reads Value's off of a byte stream.
-type (
+type Decoder struct {
-	Graph    = graph.Graph[Value, Value]
+	br        *bufio.Reader
-	OpenEdge = graph.OpenEdge[Value, Value]
+	brNextLoc Location
-)
 
-// Punctuations which are used in the gg file format.
+	unread   []locatableRune
-const (
+	lastRead locatableRune
-	punctTerm       = ";"
-	punctOp         = "<"
-	punctAssign     = "="
-	punctOpenGraph  = "{"
-	punctCloseGraph = "}"
-	punctOpenTuple  = "("
-	punctCloseTuple = ")"
-)
-
-func decoderErr(tok LexerToken, err error) error {
-	return fmt.Errorf("%s: %w", tok.errPrefix(), err)
 }
 
-func decoderErrf(tok LexerToken, str string, args ...interface{}) error {
+// NewDecoder returns a Decoder which will decode the given stream as a gg
-	return decoderErr(tok, fmt.Errorf(str, args...))
+// formatted stream of a Values.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{
+		br:        bufio.NewReader(r),
+		brNextLoc: Location{Row: 1, Col: 1},
+	}
 }
 
-func isPunct(tok LexerToken, val string) bool {
+func (d *Decoder) readRune() (locatableRune, error) {
-	return tok.Kind == LexerTokenKindPunctuation && tok.Value == val
+	if len(d.unread) > 0 {
-}
+		d.lastRead = d.unread[len(d.unread)-1]
-
+		d.unread = d.unread[:len(d.unread)-1]
-func isTerm(tok LexerToken) bool {
+		return d.lastRead, nil
-	return isPunct(tok, punctTerm)
-}
-
-// decoder is currently only really used to namespace functions related to
-// decoding Graphs. It may later have actual fields added to it, such as for
-// options passed by the caller.
-type decoder struct{}
-
-// returned boolean value indicates if the token following the single token is a
-// term. If a term followed the first token then it is not included in the
-// returned leftover tokens.
-//
-// if termed is false then leftover tokens cannot be empty.
-func (d *decoder) parseSingleValue(
-	toks []LexerToken,
-) (
-	Value, []LexerToken, bool, error,
-) {
-
-	tok, rest := toks[0], toks[1:]
-
-	if len(rest) == 0 {
-		return ZeroValue, nil, false, decoderErrf(tok, "cannot be final token, possibly missing %q", punctTerm)
 	}
 
-	termed := isTerm(rest[0])
+	loc := d.brNextLoc
-
-	if termed {
-		rest = rest[1:]
-	}
-
-	switch tok.Kind {
-
-	case LexerTokenKindName:
-		return Value{Name: &tok.Value, LexerToken: &tok}, rest, termed, nil
-
-	case LexerTokenKindNumber:
-
-		i, err := strconv.ParseInt(tok.Value, 10, 64)
 
+	r, _, err := d.br.ReadRune()
 	if err != nil {
-			return ZeroValue, nil, false, decoderErrf(tok, "parsing %q as integer: %w", tok.Value, err)
+		return d.lastRead, err
 	}
 
-		return Value{Number: &i, LexerToken: &tok}, rest, termed, nil
+	if r == '\n' {
-
+		d.brNextLoc.Row++
-	case LexerTokenKindPunctuation:
+		d.brNextLoc.Col = 1
-		return ZeroValue, nil, false, decoderErrf(tok, "expected value, found punctuation %q", tok.Value)
+	} else {
-
+		d.brNextLoc.Col++
-	default:
-		panic(fmt.Sprintf("unexpected token kind %q", tok.Kind))
 	}
+
+	d.lastRead = locatableRune{loc, r}
+	return d.lastRead, nil
 }
 
-func (d *decoder) parseOpenEdge(
+func (d *Decoder) unreadRune(lr locatableRune) {
-	toks []LexerToken,
+	if d.lastRead != lr {
-) (
+		panic(fmt.Sprintf(
-	*OpenEdge, []LexerToken, error,
+			"unreading rune %#v, but last read rune was %#v", lr, d.lastRead,
-) {
+		))
-
-	if isPunct(toks[0], punctOpenTuple) {
-		return d.parseTuple(toks)
 	}
 
-	var (
+	d.unread = append(d.unread, lr)
-		val    Value
-		termed bool
-		err    error
-	)
-
-	switch {
-
-	case isPunct(toks[0], punctOpenGraph):
-		val, toks, termed, err = d.parseGraphValue(toks, true)
-
-	default:
-		val, toks, termed, err = d.parseSingleValue(toks)
-	}
-
-	if err != nil {
-		return nil, nil, err
-
-	}
-
-	if termed {
-		return graph.ValueOut[Value](ZeroValue, val), toks, nil
-	}
-
-	opTok, toks := toks[0], toks[1:]
-
-	if !isPunct(opTok, punctOp) {
-		return nil, nil, decoderErrf(opTok, "must be %q or %q", punctOp, punctTerm)
-	}
-
-	if len(toks) == 0 {
-		return nil, nil, decoderErrf(opTok, "%q cannot terminate an edge declaration", punctOp)
-	}
-
-	oe, toks, err := d.parseOpenEdge(toks)
-
-	if err != nil {
-		return nil, nil, err
-	}
-
-	oe = graph.TupleOut[Value](val, oe)
-
-	return oe, toks, nil
 }
 
-func (d *decoder) parseTuple(
+func (d *Decoder) nextLoc() Location {
-	toks []LexerToken,
+	if len(d.unread) > 0 {
-) (
+		return d.unread[len(d.unread)-1].Location
-	*OpenEdge, []LexerToken, error,
-) {
-
-	openTok, toks := toks[0], toks[1:]
-
-	var edges []*OpenEdge
-
-	for {
-
-		if len(toks) == 0 {
-			return nil, nil, decoderErrf(openTok, "no matching %q", punctCloseTuple)
-
-		} else if isPunct(toks[0], punctCloseTuple) {
-			toks = toks[1:]
-			break
 	}
 
-		var (
+	return d.brNextLoc
-			oe  *OpenEdge
-			err error
-		)
-
-		oe, toks, err = d.parseOpenEdge(toks)
-
-		if err != nil {
-			return nil, nil, err
-		}
-
-		edges = append(edges, oe)
-	}
-
-	// this is a quirk of the syntax, _technically_ a tuple doesn't need a
-	// term after it, since it can't be used as an edge value, and so
-	// nothing can come after it in the chain.
-	if len(toks) > 0 && isTerm(toks[0]) {
-		toks = toks[1:]
-	}
-
-	return graph.TupleOut[Value](ZeroValue, edges...), toks, nil
 }
 
-// returned boolean value indicates if the token following the graph is a term.
+// Next returns the next top-level value in the stream, or io.EOF.
-// If a term followed the first token then it is not included in the returned
+func (d *Decoder) Next() (Value, error) {
-// leftover tokens.
+	return topLevelTerm.decodeFn(d)
-//
-// if termed is false then leftover tokens cannot be empty.
-func (d *decoder) parseGraphValue(
-	toks []LexerToken, expectWrappers bool,
-) (
-	Value, []LexerToken, bool, error,
-) {
-
-	var openTok LexerToken
-
-	if expectWrappers {
-		openTok, toks = toks[0], toks[1:]
-	}
-
-	g := new(Graph)
-
-	for {
-
-		if len(toks) == 0 {
-
-			if !expectWrappers {
-				break
-			}
-
-			return ZeroValue, nil, false, decoderErrf(openTok, "no matching %q", punctCloseGraph)
-
-		} else if closingTok := toks[0]; isPunct(closingTok, punctCloseGraph) {
-
-			if !expectWrappers {
-				return ZeroValue, nil, false, decoderErrf(closingTok, "unexpected %q", punctCloseGraph)
-			}
-
-			toks = toks[1:]
-
-			if len(toks) == 0 {
-				return ZeroValue, nil, false, decoderErrf(closingTok, "cannot be final token, possibly missing %q", punctTerm)
-			}
-
-			break
-		}
-
-		var err error
-
-		if g, toks, err = d.parseValIn(g, toks); err != nil {
-			return ZeroValue, nil, false, err
-		}
-	}
-
-	val := Value{Graph: g}
-
-	if !expectWrappers {
-		return val, toks, true, nil
-	}
-
-	val.LexerToken = &openTok
-
-	termed := isTerm(toks[0])
-
-	if termed {
-		toks = toks[1:]
-	}
-
-	return val, toks, termed, nil
-}
-
-func (d *decoder) parseValIn(into *Graph, toks []LexerToken) (*Graph, []LexerToken, error) {
-
-	if len(toks) == 0 {
-		return into, nil, nil
-
-	} else if len(toks) < 3 {
-		return nil, nil, decoderErrf(toks[0], `must be of the form "<name> = ..."`)
-	}
-
-	dst := toks[0]
-	eq := toks[1]
-	toks = toks[2:]
-
-	if dst.Kind != LexerTokenKindName {
-		return nil, nil, decoderErrf(dst, "must be a name")
-
-	} else if !isPunct(eq, punctAssign) {
-		return nil, nil, decoderErrf(eq, "must be %q", punctAssign)
-	}
-
-	oe, toks, err := d.parseOpenEdge(toks)
-
-	if err != nil {
-		return nil, nil, err
-	}
-
-	dstVal := Value{Name: &dst.Value, LexerToken: &dst}
-
-	return into.AddValueIn(dstVal, oe), toks, nil
-}
-
-func (d *decoder) readAllTokens(lexer Lexer) ([]LexerToken, error) {
-
-	var toks []LexerToken
-
-	for {
-
-		tok, err := lexer.Next()
-
-		if errors.Is(err, io.EOF) {
-			break
-
-		} else if err != nil {
-			return nil, fmt.Errorf("reading next token: %w", err)
-		}
-
-		toks = append(toks, tok)
-	}
-
-	return toks, nil
-}
-
-func (d *decoder) decode(lexer Lexer) (*Graph, error) {
-
-	toks, err := d.readAllTokens(lexer)
-
-	if err != nil {
-		return nil, err
-	}
-
-	val, _, _, err := d.parseGraphValue(toks, false)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return val.Graph, nil
-}
-
-// DecodeLexer reads lexigraphical tokens from the given Lexer and uses them to
-// construct a Graph according to the rules of the gg file format. DecodeLexer
-// will only return an error if there is a non-EOF file returned from the Lexer,
-// or the tokens read cannot be used to construct a valid Graph.
-func DecodeLexer(lexer Lexer) (*Graph, error) {
-	decoder := &decoder{}
-	return decoder.decode(lexer)
-}
-
-func DecodeSingleValueFromLexer(lexer Lexer) (Value, error) {
-	decoder := &decoder{}
-
-	toks, err := decoder.readAllTokens(lexer)
-
-	if err != nil {
-		return ZeroValue, err
-	}
-
-	val, _, _, err := decoder.parseSingleValue(toks)
-
-	return val, err
 }

gg/decoder_test.go

@@ -1,149 +0,0 @@
-package gg
-
-import (
-	"strconv"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/mediocregopher/ginger/graph"
-)
-
-func TestDecoder(t *testing.T) {
-
-	zeroGraph := new(Graph)
-
-	i := func(i int64) Value {
-		return Value{Number: &i}
-	}
-
-	n := func(n string) Value {
-		return Value{Name: &n}
-	}
-
-	vOut := func(edgeVal, val Value) *OpenEdge {
-		return graph.ValueOut(edgeVal, val)
-	}
-
-	tOut := func(edgeVal Value, ins ...*OpenEdge) *OpenEdge {
-		return graph.TupleOut(edgeVal, ins...)
-	}
-
-	tests := []struct {
-		in  string
-		exp *Graph
-	}{
-		{
-			in:  "",
-			exp: zeroGraph,
-		},
-		{
-			in:  "out = 1;",
-			exp: zeroGraph.AddValueIn(n("out"), vOut(ZeroValue, i(1))),
-		},
-		{
-			in:  "out = incr < 1;",
-			exp: zeroGraph.AddValueIn(n("out"), vOut(n("incr"), i(1))),
-		},
-		{
-			in: "out = a < b < 1;",
-			exp: zeroGraph.AddValueIn(
-				n("out"),
-				tOut(
-					n("a"),
-					vOut(n("b"),
-						i(1)),
-				),
-			),
-		},
-		{
-			in: "out = a < b < (1; c < 2; d < e < 3;);",
-			exp: zeroGraph.AddValueIn(
-				n("out"),
-				tOut(
-					n("a"),
-					tOut(
-						n("b"),
-						vOut(ZeroValue, i(1)),
-						vOut(n("c"), i(2)),
-						tOut(
-							n("d"),
-							vOut(n("e"), i(3)),
-						),
-					),
-				),
-			),
-		},
-		{
-			in: "out = a < b < (1; c < (d < 2; 3;); );",
-			exp: zeroGraph.AddValueIn(
-				n("out"),
-				tOut(
-					n("a"),
-					tOut(
-						n("b"),
-						vOut(ZeroValue, i(1)),
-						tOut(
-							n("c"),
-							vOut(n("d"), i(2)),
-							vOut(ZeroValue, i(3)),
-						),
-					),
-				),
-			),
-		},
-		{
-			in: "out = { a = 1; b = c < d < 2; };",
-			exp: zeroGraph.AddValueIn(
-				n("out"),
-				vOut(
-					ZeroValue,
-					Value{Graph: zeroGraph.
-						AddValueIn(n("a"), vOut(ZeroValue, i(1))).
-						AddValueIn(
-							n("b"),
-							tOut(
-								n("c"),
-								vOut(n("d"), i(2)),
-							),
-						),
-					},
-				),
-			),
-		},
-		{
-			in: "out = a < { b = 1; } < 2;",
-			exp: zeroGraph.AddValueIn(
-				n("out"),
-				tOut(
-					n("a"),
-					vOut(
-						Value{Graph: zeroGraph.
-							AddValueIn(n("b"), vOut(ZeroValue, i(1))),
-						},
-						i(2),
-					),
-				),
-			),
-		},
-		{
-			in: "a = 1; b = 2;",
-			exp: zeroGraph.
-				AddValueIn(n("a"), vOut(ZeroValue, i(1))).
-				AddValueIn(n("b"), vOut(ZeroValue, i(2))),
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-
-			r := &mockReader{body: []byte(test.in)}
-			lexer := NewLexer(r)
-
-			got, err := DecodeLexer(lexer)
-			assert.NoError(t, err)
-			assert.True(t, got.Equal(test.exp), "\nexp:%v\ngot:%v", test.exp, got)
-
-		})
-	}
-}

gg/gg.bnf

@@ -0,0 +1,23 @@
+<digit>           ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+<positive-number> ::= <digit>+
+<negative-number> ::= "-" <positive-number>
+<number>          ::= <negative-number> | <positive-number>
+
+<name> ::= (<letter> | <mark>) (<letter> | <mark> | <digit>)*
+
+<tuple>                      ::= "(" <tuple-tail>
+<tuple-tail>                 ::= ")" | <tuple-open-edge>
+<tuple-open-edge>            ::= <value> <tuple-open-edge-value-tail>
+                               | <tuple> <tuple-open-edge-tail>
+<tuple-open-edge-tail>       ::= ")" | "," <tuple-tail>
+<tuple-open-edge-value-tail> ::= <tuple-open-edge-tail> | "<" <tuple-open-edge>
+
+<graph>                      ::= "{" <graph-tail>
+<graph-tail>                 ::= "}" | <name> "=" <graph-open-edge>
+<graph-open-edge>            ::= <value> <graph-open-edge-value-tail>
+                               | <tuple> <graph-open-edge-tail>
+<graph-open-edge-tail>       ::= "}" | ";" <graph-tail>
+<graph-open-edge-value-tail>  ::= <graph-open-edge-tail> | "<" <graph-open-edge>
+
+<value> ::= <name> | <number> | <graph>
+<gg>    ::= <eof> | <value> <gg>

gg/gg.go

@@ -7,23 +7,20 @@ import (
 	"github.com/mediocregopher/ginger/graph"
 )
 
-// ZeroValue is a Value with no fields set.
+// Type aliases for convenience
-var ZeroValue Value
+type (
+	Graph    = graph.Graph[OptionalValue, Value]
+	OpenEdge = graph.OpenEdge[OptionalValue, Value]
+)
 
 // Value represents a value which can be serialized by the gg text format.
 type Value struct {
+	Location
 
 	// Only one of these fields may be set
 	Name   *string
 	Number *int64
 	Graph  *Graph
-
-	// TODO coming soon!
-	// String *string
-
-	// Optional fields indicating the token which was used to construct this
-	// Value, if any.
-	LexerToken *LexerToken
 }
 
 // Name returns a name Value.
@@ -36,12 +33,6 @@ func Number(n int64) Value {
 	return Value{Number: &n}
 }
 
-// IsZero returns true if the Value is the zero value (none of the sub-value
-// fields are set). LexerToken is ignored for this check.
-func (v Value) IsZero() bool {
-	return v.Equal(ZeroValue)
-}
-
 // Equal returns true if the passed in Value is equivalent, ignoring the
 // LexerToken on either Value.
 //
@@ -50,13 +41,8 @@ func (v Value) Equal(v2g graph.Value) bool {
 
 	v2 := v2g.(Value)
 
-	v.LexerToken, v2.LexerToken = nil, nil
-
 	switch {
 
-	case v == ZeroValue && v2 == ZeroValue:
-		return true
-
 	case v.Name != nil && v2.Name != nil && *v.Name == *v2.Name:
 		return true
 
@@ -85,6 +71,46 @@ func (v Value) String() string {
 		return v.Graph.String()
 
 	default:
-		return "<zero>"
+		panic("no fields set on Value")
 	}
 }
+
+// OptionalValue is a Value which may be unset. This is used for edge values,
+// since edges might not have a value.
+type OptionalValue struct {
+	Value
+	Valid bool
+}
+
+// None is the zero OptionalValue (hello rustaceans).
+var None OptionalValue
+
+// Some wraps a Value to be an OptionalValue.
+func Some(v Value) OptionalValue {
+	return OptionalValue{Valid: true, Value: v}
+}
+
+func (v OptionalValue) String() string {
+	if !v.Valid {
+		return "<none>"
+	}
+	return v.Value.String()
+}
+
+func (v OptionalValue) Equal(v2g graph.Value) bool {
+	var v2 OptionalValue
+
+	if v2Val, ok := v2g.(Value); ok {
+		v2 = Some(v2Val)
+	} else {
+		v2 = v2g.(OptionalValue)
+	}
+
+	if v.Valid != v2.Valid {
+		return false
+	} else if !v.Valid {
+		return true
+	}
+
+	return v.Value.Equal(v2.Value)
+}

gg/lexer.go

@@ -1,292 +0,0 @@
-package gg
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"strings"
-	"unicode"
-)
-
-// LexerLocation describes the location in a file where a particular token was
-// parsed from.
-type LexerLocation struct {
-	Row, Col int
-}
-
-func (l LexerLocation) String() string {
-	return fmt.Sprintf("%d:%d", l.Row, l.Col)
-}
-
-// LexerError is returned by Lexer when an unexpected error occurs parsing a
-// stream of LexerTokens.
-type LexerError struct {
-	Err error
-
-	Location LexerLocation
-}
-
-func (e *LexerError) Error() string {
-	return fmt.Sprintf("%s: %s", e.Location.String(), e.Err.Error())
-}
-
-func (e *LexerError) Unwrap() error {
-	return e.Err
-}
-
-// LexerTokenKind enumerates the different kinds of LexerToken there can be.
-type LexerTokenKind string
-
-// Enumeration of LexerTokenKinds.
-const (
-	LexerTokenKindName        LexerTokenKind = "name"
-	LexerTokenKindNumber      LexerTokenKind = "number"
-	LexerTokenKindPunctuation LexerTokenKind = "punctuation"
-)
-
-// LexerToken describes a lexigraphical token which is used when deserializing
-// Graphs.
-type LexerToken struct {
-	Kind  LexerTokenKind
-	Value string // never empty string
-
-	Location LexerLocation
-}
-
-func (t LexerToken) errPrefix() string {
-	return fmt.Sprintf("%s: at %q", t.Location.String(), t.Value)
-}
-
-// Lexer is used to parse a string stream into a sequence of tokens which can
-// then be parsed by a Parser.
-type Lexer interface {
-
-	// Next will return a LexerToken or a LexerError. io.EOF (wrapped in a
-	// LexerError) is returned if the stream being read from is finished.
-	Next() (LexerToken, error)
-}
-
-type lexer struct {
-	r             *bufio.Reader
-	stringBuilder *strings.Builder
-	err           *LexerError
-
-	// these fields are only needed to keep track of the current "cursor"
-	// position when reading.
-	lastRow, lastCol int
-	prevRune         rune
-}
-
-// NewLexer wraps the io.Reader in a Lexer, which will read the io.Reader as a
-// sequence of utf-8 characters and parse it into a sequence of LexerTokens.
-func NewLexer(r io.Reader) Lexer {
-	return &lexer{
-		r:             bufio.NewReader(r),
-		lastRow:       0,
-		lastCol:       -1,
-		stringBuilder: new(strings.Builder),
-	}
-}
-
-// nextRowCol returns the row and column number which the next rune in the
-// stream would be at.
-func (l *lexer) nextRowCol() (int, int) {
-
-	if l.prevRune == '\n' {
-		return l.lastRow + 1, 0
-	}
-
-	return l.lastRow, l.lastCol + 1
-}
-
-func (l *lexer) fmtErr(err error) *LexerError {
-
-	row, col := l.nextRowCol()
-
-	return &LexerError{
-		Err: err,
-		Location: LexerLocation{
-			Row: row,
-			Col: col,
-		},
-	}
-}
-
-func (l *lexer) fmtErrf(str string, args ...interface{}) *LexerError {
-	return l.fmtErr(fmt.Errorf(str, args...))
-}
-
-// discardRune must _always_ be called only after peekRune.
-func (l *lexer) discardRune() {
-
-	r, _, err := l.r.ReadRune()
-
-	if err != nil {
-		panic(err)
-	}
-
-	l.lastRow, l.lastCol = l.nextRowCol()
-	l.prevRune = r
-}
-
-func (l *lexer) peekRune() (rune, error) {
-
-	r, _, err := l.r.ReadRune()
-
-	if err != nil {
-		return '0', err
-
-	} else if err := l.r.UnreadRune(); err != nil {
-
-		// since the most recent operation on the bufio.Reader was a ReadRune,
-		// UnreadRune should never return an error
-		panic(err)
-	}
-
-	return r, nil
-}
-
-// readWhile reads runes until the given predicate returns false, and returns a
-// LexerToken of the given kind whose Value is comprised of all runes which
-// returned true.
-//
-// If an error is encountered then both the token (or what's been parsed of it
-// so far) and the error are returned.
-func (l *lexer) readWhile(
-	kind LexerTokenKind, pred func(rune) bool,
-) (
-	LexerToken, *LexerError,
-) {
-
-	row, col := l.nextRowCol()
-
-	l.stringBuilder.Reset()
-
-	var lexErr *LexerError
-
-	for {
-
-		r, err := l.peekRune()
-
-		if err != nil {
-			lexErr = l.fmtErrf("peeking next character: %w", err)
-			break
-
-		} else if !pred(r) {
-			break
-		}
-
-		l.stringBuilder.WriteRune(r)
-
-		l.discardRune()
-	}
-
-	return LexerToken{
-		Kind:  kind,
-		Value: l.stringBuilder.String(),
-		Location: LexerLocation{
-			Row: row, Col: col,
-		},
-	}, lexErr
-}
-
-// we only support base-10 integers at the moment.
-func isNumber(r rune) bool {
-	return r == '-' || ('0' <= r && r <= '9')
-}
-
-// next can return a token, an error, or both. If an error is returned then no
-// further calls to next should occur.
-func (l *lexer) next() (LexerToken, *LexerError) {
-
-	for {
-
-		r, err := l.peekRune()
-
-		if err != nil {
-			return LexerToken{}, l.fmtErrf("peeking next character: %w", err)
-		}
-
-		switch {
-
-		case r == '*': // comment
-
-			// comments are everything up until a newline
-			_, err := l.readWhile("", func(r rune) bool {
-				return r != '\n'
-			})
-
-			if err != nil {
-				return LexerToken{}, err
-			}
-
-			// terminating newline will be discarded on next loop
-
-		case r == '"' || r == '`':
-
-			// reserve double-quote and backtick for string parsing.
-			l.discardRune()
-			return LexerToken{}, l.fmtErrf("string parsing not yet implemented")
-
-		case unicode.IsLetter(r):
-			// letters denote the start of a name
-
-			return l.readWhile(LexerTokenKindName, func(r rune) bool {
-
-				if unicode.In(r, unicode.Letter, unicode.Number, unicode.Mark) {
-					return true
-				}
-
-				if r == '-' {
-					return true
-				}
-
-				return false
-			})
-
-		case isNumber(r):
-			return l.readWhile(LexerTokenKindNumber, isNumber)
-
-		case unicode.IsPunct(r) || unicode.IsSymbol(r):
-			// symbols are also considered punctuation
-
-			l.discardRune()
-
-			return LexerToken{
-				Kind:  LexerTokenKindPunctuation,
-				Value: string(r),
-				Location: LexerLocation{
-					Row: l.lastRow,
-					Col: l.lastCol,
-				},
-			}, nil
-
-		case unicode.IsSpace(r):
-			l.discardRune()
-
-		default:
-			return LexerToken{}, l.fmtErrf("unexpected character %q", r)
-		}
-
-	}
-}
-
-func (l *lexer) Next() (LexerToken, error) {
-
-	if l.err != nil {
-		return LexerToken{}, l.err
-	}
-
-	tok, err := l.next()
-
-	if err != nil {
-
-		l.err = err
-
-		if tok.Kind == "" {
-			return LexerToken{}, l.err
-		}
-	}
-
-	return tok, nil
-}

gg/lexer_test.go

@@ -1,150 +0,0 @@
-package gg
-
-import (
-	"errors"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestLexer(t *testing.T) {
-
-	expErr := errors.New("eof")
-
-	tests := []struct {
-		in  string
-		exp []LexerToken
-	}{
-		{in: "", exp: []LexerToken{}},
-		{in: "* fooo\n", exp: []LexerToken{}},
-		{
-			in: "foo",
-			exp: []LexerToken{
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "foo",
-					Location: LexerLocation{Row: 0, Col: 0},
-				},
-			},
-		},
-		{
-			in: "foo bar\nf-o f0O Foo",
-			exp: []LexerToken{
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "foo",
-					Location: LexerLocation{Row: 0, Col: 0},
-				},
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "bar",
-					Location: LexerLocation{Row: 0, Col: 4},
-				},
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "f-o",
-					Location: LexerLocation{Row: 1, Col: 0},
-				},
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "f0O",
-					Location: LexerLocation{Row: 1, Col: 4},
-				},
-				{
-					Kind:     LexerTokenKindName,
-					Value:    "Foo",
-					Location: LexerLocation{Row: 1, Col: 8},
-				},
-			},
-		},
-		{
-			in: "1 100 -100",
-			exp: []LexerToken{
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "1",
-					Location: LexerLocation{Row: 0, Col: 0},
-				},
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "100",
-					Location: LexerLocation{Row: 0, Col: 2},
-				},
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "-100",
-					Location: LexerLocation{Row: 0, Col: 6},
-				},
-			},
-		},
-		{
-			in: "1<2!-3 ()",
-			exp: []LexerToken{
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "1",
-					Location: LexerLocation{Row: 0, Col: 0},
-				},
-				{
-					Kind:     LexerTokenKindPunctuation,
-					Value:    "<",
-					Location: LexerLocation{Row: 0, Col: 1},
-				},
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "2",
-					Location: LexerLocation{Row: 0, Col: 2},
-				},
-				{
-					Kind:     LexerTokenKindPunctuation,
-					Value:    "!",
-					Location: LexerLocation{Row: 0, Col: 3},
-				},
-				{
-					Kind:     LexerTokenKindNumber,
-					Value:    "-3",
-					Location: LexerLocation{Row: 0, Col: 4},
-				},
-				{
-					Kind:     LexerTokenKindPunctuation,
-					Value:    "(",
-					Location: LexerLocation{Row: 0, Col: 7},
-				},
-				{
-					Kind:     LexerTokenKindPunctuation,
-					Value:    ")",
-					Location: LexerLocation{Row: 0, Col: 8},
-				},
-			},
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-
-			lexer := NewLexer(&mockReader{body: []byte(test.in), err: expErr})
-
-			for i := range test.exp {
-				tok, err := lexer.Next()
-				assert.NoError(t, err)
-				assert.Equal(t, test.exp[i], tok, "test.exp[%d]", i)
-			}
-
-			tok, err := lexer.Next()
-			assert.ErrorIs(t, err, expErr)
-			assert.Equal(t, LexerToken{}, tok)
-
-			lexErr := new(LexerError)
-			assert.True(t, errors.As(err, &lexErr))
-
-			inParts := strings.Split(test.in, "\n")
-
-			assert.ErrorIs(t, lexErr, expErr)
-			assert.Equal(t, lexErr.Location.Row, len(inParts)-1)
-			assert.Equal(t, lexErr.Location.Col, len(inParts[len(inParts)-1]))
-		})
-	}
-
-}

gg/location.go

@@ -0,0 +1,36 @@
+package gg
+
+import "fmt"
+
+// Location indicates a position in a stream of bytes identified by column
+// within newline-separated rows.
+type Location struct {
+	Row, Col int
+}
+
+func (l Location) errf(str string, args ...any) LocatedError {
+	return LocatedError{l, fmt.Errorf(str, args...)}
+}
+
+func (l Location) locate() Location { return l }
+
+// LocatedError is an error related to a specific point within a decode gg
+// stream.
+type LocatedError struct {
+	Location
+	Err error
+}
+
+func (e LocatedError) Error() string {
+	return fmt.Sprintf("%d:%d: %v", e.Row, e.Col, e.Err)
+}
+
+type locatableRune struct {
+	Location
+	r rune
+}
+
+type locatableString struct {
+	Location
+	str string
+}

gg/term.go

@@ -0,0 +1,508 @@
+package gg
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/mediocregopher/ginger/graph"
+	"golang.org/x/exp/slices"
+)
+
+var (
+	errNoMatch = errors.New("not found")
+)
+
+type stringerFn func() string
+
+func (fn stringerFn) String() string {
+	return fn()
+}
+
+type stringerStr string
+
+func (str stringerStr) String() string {
+	return string(str)
+}
+
+type term[T any] struct {
+	name     fmt.Stringer
+	decodeFn func(d *Decoder) (T, error)
+}
+
+func (t term[T]) String() string {
+	return t.name.String()
+}
+
+func firstOf[T any](terms ...*term[T]) *term[T] {
+	if len(terms) < 2 {
+		panic("firstOfTerms requires at least 2 terms")
+	}
+
+	return &term[T]{
+		name: stringerFn(func() string {
+			descrs := make([]string, len(terms))
+			for i := range terms {
+				descrs[i] = terms[i].String()
+			}
+			return strings.Join(descrs, " or ")
+		}),
+		decodeFn: func(d *Decoder) (T, error) {
+			var zero T
+			for _, t := range terms {
+				v, err := t.decodeFn(d)
+				if errors.Is(err, errNoMatch) {
+					continue
+				} else if err != nil {
+					return zero, err
+				}
+
+				return v, nil
+			}
+
+			return zero, errNoMatch
+		},
+	}
+}
+
+func seq[Ta, Tb, Tc any](
+	name fmt.Stringer,
+	termA *term[Ta],
+	termB *term[Tb],
+	fn func(Ta, Tb) Tc,
+) *term[Tc] {
+	return &term[Tc]{
+		name: name,
+		decodeFn: func(d *Decoder) (Tc, error) {
+			var zero Tc
+
+			va, err := termA.decodeFn(d)
+			if err != nil {
+				return zero, err
+			}
+
+			vb, err := termB.decodeFn(d)
+			if errors.Is(err, errNoMatch) {
+				return zero, d.nextLoc().errf("expected %v", termB)
+			} else if err != nil {
+				return zero, err
+			}
+
+			return fn(va, vb), nil
+		},
+	}
+}
+
+func prefixed[Ta, Tb any](termA *term[Ta], termB *term[Tb]) *term[Tb] {
+	return seq(termA, termA, termB, func(_ Ta, b Tb) Tb {
+		return b
+	})
+}
+
+func prefixIgnored[Ta, Tb any](termA *term[Ta], termB *term[Tb]) *term[Tb] {
+	return &term[Tb]{
+		name: termB,
+		decodeFn: func(d *Decoder) (Tb, error) {
+			var zero Tb
+			if _, err := termA.decodeFn(d); err != nil {
+				return zero, err
+			}
+			return termB.decodeFn(d)
+		},
+	}
+}
+
+func suffixIgnored[Ta, Tb any](
+	termA *term[Ta], termB *term[Tb],
+) *term[Ta] {
+	return seq(termA, termA, termB, func(a Ta, _ Tb) Ta {
+		return a
+	})
+}
+
+func oneOrMore[T any](t *term[T]) *term[[]T] {
+	return &term[[]T]{
+		name: stringerFn(func() string {
+			return fmt.Sprintf("one or more %v", t)
+		}),
+		decodeFn: func(d *Decoder) ([]T, error) {
+			var vv []T
+			for {
+				v, err := t.decodeFn(d)
+				if errors.Is(err, errNoMatch) {
+					break
+				} else if err != nil {
+					return nil, err
+				}
+
+				vv = append(vv, v)
+			}
+
+			if len(vv) == 0 {
+				return nil, errNoMatch
+			}
+
+			return vv, nil
+		},
+	}
+}
+
+func zeroOrMore[T any](t *term[T]) *term[[]T] {
+	return &term[[]T]{
+		name: stringerFn(func() string {
+			return fmt.Sprintf("zero or more %v", t)
+		}),
+		decodeFn: func(d *Decoder) ([]T, error) {
+			var vv []T
+			for {
+				v, err := t.decodeFn(d)
+				if errors.Is(err, errNoMatch) {
+					break
+				} else if err != nil {
+					return nil, err
+				}
+
+				vv = append(vv, v)
+			}
+
+			return vv, nil
+		},
+	}
+}
+
+func mapTerm[Ta, Tb any](
+	name fmt.Stringer, t *term[Ta], fn func(Ta) Tb,
+) *term[Tb] {
+	return &term[Tb]{
+		name: name,
+		decodeFn: func(d *Decoder) (Tb, error) {
+			var zero Tb
+			va, err := t.decodeFn(d)
+			if err != nil {
+				return zero, err
+			}
+			return fn(va), nil
+		},
+	}
+}
+
+func runePredTerm(
+	name fmt.Stringer, pred func(rune) bool,
+) *term[locatableRune] {
+	return &term[locatableRune]{
+		name: name,
+		decodeFn: func(d *Decoder) (locatableRune, error) {
+			lr, err := d.readRune()
+			if errors.Is(err, io.EOF) {
+				return locatableRune{}, errNoMatch
+			} else if err != nil {
+				return locatableRune{}, err
+			}
+
+			if !pred(lr.r) {
+				d.unreadRune(lr)
+				return locatableRune{}, errNoMatch
+			}
+
+			return lr, nil
+		},
+	}
+}
+
+func runeTerm(r rune) *term[locatableRune] {
+	return runePredTerm(
+		stringerStr(fmt.Sprintf("'%c'", r)),
+		func(r2 rune) bool { return r2 == r },
+	)
+}
+
+func locatableRunesToString(rr []locatableRune) string {
+	str := make([]rune, len(rr))
+	for i := range rr {
+		str[i] = rr[i].r
+	}
+	return string(str)
+}
+
+func runesToStringTerm(
+	t *term[[]locatableRune],
+) *term[locatableString] {
+	return mapTerm(
+		t, t, func(rr []locatableRune) locatableString {
+			return locatableString{rr[0].locate(), locatableRunesToString(rr)}
+		},
+	)
+}
+
+func discard[T any](t *term[T]) *term[struct{}] {
+	return mapTerm(t, t, func(_ T) struct{} { return struct{}{} })
+}
+
+var (
+	notNewlineTerm = runePredTerm(
+		stringerStr("not-newline"),
+		func(r rune) bool { return r != '\n' },
+	)
+
+	commentTerm = prefixed(
+		prefixed(runeTerm('*'), zeroOrMore(notNewlineTerm)),
+		runeTerm('\n'),
+	)
+
+	whitespaceTerm = zeroOrMore(firstOf(
+		discard(runePredTerm(stringerStr("whitespace"), unicode.IsSpace)),
+		discard(commentTerm),
+	))
+)
+
+func trimmedTerm[T any](t *term[T]) *term[T] {
+	t = prefixIgnored(whitespaceTerm, t)
+	t = suffixIgnored(t, whitespaceTerm)
+	return t
+}
+
+func trimmedRuneTerm(r rune) *term[locatableRune] {
+	return trimmedTerm(runeTerm(r))
+}
+
+var (
+	digitTerm = runePredTerm(
+		stringerStr("digit"),
+		func(r rune) bool { return '0' <= r && r <= '9' },
+	)
+
+	positiveNumberTerm = runesToStringTerm(oneOrMore(digitTerm))
+
+	negativeNumberTerm = seq(
+		stringerStr("negative-number"),
+		runeTerm('-'),
+		positiveNumberTerm,
+		func(neg locatableRune, posNum locatableString) locatableString {
+			return locatableString{neg.locate(), string(neg.r) + posNum.str}
+		},
+	)
+
+	numberTerm = mapTerm(
+		stringerStr("number"),
+		firstOf(negativeNumberTerm, positiveNumberTerm),
+		func(str locatableString) Value {
+			i, err := strconv.ParseInt(str.str, 10, 64)
+			if err != nil {
+				panic(fmt.Errorf("parsing %q as int: %w", str, err))
+			}
+
+			return Value{Number: &i, Location: str.locate()}
+		},
+	)
+)
+
+var (
+	letterTerm = runePredTerm(
+		stringerStr("letter"),
+		func(r rune) bool {
+			return unicode.In(r, unicode.Letter, unicode.Mark)
+		},
+	)
+
+	letterTailTerm = zeroOrMore(firstOf(letterTerm, digitTerm))
+
+	nameTerm = seq(
+		stringerStr("name"),
+		letterTerm,
+		letterTailTerm,
+		func(head locatableRune, tail []locatableRune) Value {
+			name := string(head.r) + locatableRunesToString(tail)
+			return Value{Name: &name, Location: head.locate()}
+		},
+	)
+)
+
+func openEdgeIntoValue(val Value, oe *OpenEdge) *OpenEdge {
+	switch {
+	case oe == nil:
+		return graph.ValueOut(None, val)
+	case !oe.EdgeValue().Valid:
+		return oe.WithEdgeValue(Some(val))
+	default:
+		return graph.TupleOut(Some(val), oe)
+	}
+}
+
+var graphTerm, valueTerm = func() (*term[Value], *term[Value]) {
+	type tupleState struct {
+		ins []*OpenEdge
+		oe  *OpenEdge
+	}
+
+	type graphState struct {
+		g  *Graph
+		oe *OpenEdge
+	}
+
+	var (
+		rightParenthesis = trimmedRuneTerm(')')
+		tupleEndTerm     = mapTerm(
+			rightParenthesis,
+			rightParenthesis,
+			func(lr locatableRune) tupleState {
+				// if ')', then map that to an empty state. This acts as a
+				// sentinel value to indicate "end of tuple".
+				return tupleState{}
+			},
+		)
+
+		rightCurlyBrace = trimmedRuneTerm('}')
+		graphEndTerm    = mapTerm(
+			rightCurlyBrace,
+			rightCurlyBrace,
+			func(lr locatableRune) graphState {
+				// if '}', then map that to an empty state. This acts as a
+				// sentinel value to indicate "end of graph".
+				return graphState{}
+			},
+		)
+	)
+
+	var (
+		// pre-define these, and then fill in the pointers after, in order to
+		// deal with recursive dependencies between them.
+		valueTerm = new(term[Value])
+
+		tupleTerm                  = new(term[*OpenEdge])
+		tupleTailTerm              = new(term[tupleState])
+		tupleOpenEdgeTerm          = new(term[tupleState])
+		tupleOpenEdgeTailTerm      = new(term[tupleState])
+		tupleOpenEdgeValueTailTerm = new(term[tupleState])
+
+		graphTerm                  = new(term[Value])
+		graphTailTerm              = new(term[graphState])
+		graphOpenEdgeTerm          = new(term[graphState])
+		graphOpenEdgeTailTerm      = new(term[graphState])
+		graphOpenEdgeValueTailTerm = new(term[graphState])
+	)
+
+	*tupleTerm = *seq(
+		stringerStr("tuple"),
+		trimmedRuneTerm('('),
+		tupleTailTerm,
+		func(lr locatableRune, ts tupleState) *OpenEdge {
+			slices.Reverse(ts.ins)
+			return graph.TupleOut(None, ts.ins...)
+		},
+	)
+
+	*tupleTailTerm = *firstOf(
+		tupleEndTerm,
+		mapTerm(
+			tupleOpenEdgeTerm,
+			tupleOpenEdgeTerm,
+			func(ts tupleState) tupleState {
+				ts.ins = append(ts.ins, ts.oe)
+				ts.oe = nil
+				return ts
+			},
+		),
+	)
+
+	*tupleOpenEdgeTerm = *firstOf(
+		seq(
+			valueTerm,
+			valueTerm,
+			tupleOpenEdgeValueTailTerm,
+			func(val Value, ts tupleState) tupleState {
+				ts.oe = openEdgeIntoValue(val, ts.oe)
+				return ts
+			},
+		),
+		seq(
+			tupleTerm,
+			tupleTerm,
+			tupleOpenEdgeTailTerm,
+			func(oe *OpenEdge, ts tupleState) tupleState {
+				ts.oe = oe
+				return ts
+			},
+		),
+	)
+
+	*tupleOpenEdgeTailTerm = *firstOf(
+		tupleEndTerm,
+		prefixed(trimmedRuneTerm(','), tupleTailTerm),
+	)
+
+	*tupleOpenEdgeValueTailTerm = *firstOf(
+		tupleOpenEdgeTailTerm,
+		prefixed(trimmedRuneTerm('<'), tupleOpenEdgeTerm),
+	)
+
+	*graphTerm = *seq(
+		stringerStr("graph"),
+		trimmedRuneTerm('{'),
+		graphTailTerm,
+		func(lr locatableRune, gs graphState) Value {
+			if gs.g == nil {
+				gs.g = new(Graph)
+			}
+
+			return Value{Graph: gs.g, Location: lr.locate()}
+		},
+	)
+
+	*graphTailTerm = *firstOf(
+		graphEndTerm,
+		seq(
+			nameTerm,
+			nameTerm,
+			prefixed(trimmedRuneTerm('='), graphOpenEdgeTerm),
+			func(name Value, gs graphState) graphState {
+				if gs.g == nil {
+					gs.g = new(Graph)
+				}
+
+				gs.g = gs.g.AddValueIn(name, gs.oe)
+				gs.oe = nil
+				return gs
+			},
+		),
+	)
+
+	*graphOpenEdgeTerm = *firstOf(
+		seq(
+			valueTerm,
+			valueTerm,
+			graphOpenEdgeValueTailTerm,
+			func(val Value, gs graphState) graphState {
+				gs.oe = openEdgeIntoValue(val, gs.oe)
+				return gs
+			},
+		),
+		seq(
+			tupleTerm,
+			tupleTerm,
+			graphOpenEdgeTailTerm,
+			func(oe *OpenEdge, gs graphState) graphState {
+				gs.oe = oe
+				return gs
+			},
+		),
+	)
+
+	*graphOpenEdgeTailTerm = *firstOf(
+		graphEndTerm,
+		prefixed(trimmedRuneTerm(';'), graphTailTerm),
+	)
+
+	*graphOpenEdgeValueTailTerm = *firstOf(
+		graphOpenEdgeTailTerm,
+		prefixed(trimmedRuneTerm('<'), graphOpenEdgeTerm),
+	)
+
+	*valueTerm = *firstOf(nameTerm, numberTerm, graphTerm)
+
+	return graphTerm, valueTerm
+}()
+
+var topLevelTerm = trimmedTerm(valueTerm)

gg/term_test.go

@@ -0,0 +1,387 @@
+package gg
+
+import (
+	"bytes"
+	"io"
+	"strconv"
+	"testing"
+
+	"github.com/mediocregopher/ginger/graph"
+	"github.com/stretchr/testify/assert"
+)
+
+func decoderLeftover(d *Decoder) string {
+	unread := make([]rune, len(d.unread))
+	for i := range unread {
+		unread[i] = d.unread[i].r
+	}
+
+	rest, err := io.ReadAll(d.br)
+	if err != nil {
+		panic(err)
+	}
+	return string(unread) + string(rest)
+}
+
+func TestTermDecoding(t *testing.T) {
+	type test struct {
+		in       string
+		exp      Value
+		expErr   string
+		leftover string
+	}
+
+	runTests := func(
+		t *testing.T, name string, term *term[Value], tests []test,
+	) {
+		t.Run(name, func(t *testing.T) {
+			for i, test := range tests {
+				t.Run(strconv.Itoa(i), func(t *testing.T) {
+					dec := NewDecoder(bytes.NewBufferString(test.in))
+					got, err := term.decodeFn(dec)
+					if test.expErr != "" {
+						assert.Error(t, err)
+						assert.Equal(t, test.expErr, err.Error())
+					} else if assert.NoError(t, err) {
+						assert.True(t,
+							test.exp.Equal(got),
+							"\nexp:%v\ngot:%v", test.exp, got,
+						)
+						assert.Equal(t, test.leftover, decoderLeftover(dec))
+					}
+				})
+			}
+		})
+	}
+
+	expNum := func(row, col int, n int64) Value {
+		return Value{Number: &n, Location: Location{row, col}}
+	}
+
+	runTests(t, "number", numberTerm, []test{
+		{in: `0`, exp: expNum(1, 1, 0)},
+		{in: `100`, exp: expNum(1, 1, 100)},
+		{in: `-100`, exp: expNum(1, 1, -100)},
+		{in: `0foo`, exp: expNum(1, 1, 0), leftover: "foo"},
+		{in: `100foo`, exp: expNum(1, 1, 100), leftover: "foo"},
+	})
+
+	expName := func(row, col int, name string) Value {
+		return Value{Name: &name, Location: Location{row, col}}
+	}
+
+	expGraph := func(row, col int, g *Graph) Value {
+		return Value{Graph: g, Location: Location{row, col}}
+	}
+
+	runTests(t, "name", nameTerm, []test{
+		{in: `a`, exp: expName(1, 1, "a")},
+		{in: `ab`, exp: expName(1, 1, "ab")},
+		{in: `ab2c`, exp: expName(1, 1, "ab2c")},
+		{in: `ab2c,`, exp: expName(1, 1, "ab2c"), leftover: ","},
+	})
+
+	runTests(t, "graph", graphTerm, []test{
+		{in: `{}`, exp: expGraph(1, 1, new(Graph))},
+		{in: `{`, expErr: `1:2: expected '}' or name`},
+		{in: `{a}`, expErr: `1:3: expected '='`},
+		{in: `{a=}`, expErr: `1:4: expected name or number or graph or tuple`},
+		{
+			in: `{foo=a}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(None, expName(6, 1, "a")),
+					),
+			),
+		},
+		{
+			in: `{ foo = a }`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(None, expName(6, 1, "a")),
+					),
+			),
+		},
+		{in: `{1=a}`, expErr: `1:2: expected '}' or name`},
+		{in: `{foo=a ,}`, expErr: `1:8: expected '}' or ';' or '<'`},
+		{in: `{foo=a`, expErr: `1:7: expected '}' or ';' or '<'`},
+		{
+			in: `{foo=a<b}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expName(8, 1, "b"),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a< b <c}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.ValueOut(
+								Some(expName(8, 1, "b")),
+								expName(10, 1, "c"),
+							),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo =a<b<c<1 }`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.TupleOut(
+								Some(expName(8, 1, "b")),
+								graph.ValueOut(
+									Some(expName(10, 1, "c")),
+									expNum(12, 1, 1),
+								),
+							),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<b ; }`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expName(8, 1, "b"),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<b;bar=c}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expName(8, 1, "b"),
+						),
+					).
+					AddValueIn(
+						expName(10, 1, "bar"),
+						graph.ValueOut(None, expName(15, 1, "c")),
+					),
+			),
+		},
+		{
+			in: `{foo= a<{ baz=1 } ; bar=c}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expGraph(8, 1, new(Graph).AddValueIn(
+								expName(9, 1, "baz"),
+								graph.ValueOut(None, expNum(13, 1, 1)),
+							)),
+						),
+					).
+					AddValueIn(
+						expName(16, 1, "bar"),
+						graph.ValueOut(None, expName(20, 1, "c")),
+					),
+			),
+		},
+		{
+			in: `{foo= {baz=1} <a; bar=c}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expGraph(8, 1, new(Graph).AddValueIn(
+								expName(9, 1, "baz"),
+								graph.ValueOut(None, expNum(13, 1, 1)),
+							))),
+							expName(6, 1, "a"),
+						),
+					).
+					AddValueIn(
+						expName(16, 1, "bar"),
+						graph.ValueOut(None, expName(20, 1, "c")),
+					),
+			),
+		},
+	})
+
+	runTests(t, "tuple", graphTerm, []test{
+		{
+			in: `{foo=(a)}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(None, expName(6, 1, "a")),
+					),
+			),
+		},
+		{
+			in: `{foo=(a<b)}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expName(8, 1, "b"),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b)}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(
+							Some(expName(6, 1, "a")),
+							expName(8, 1, "b"),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b,c)}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.ValueOut(None, expName(8, 1, "b")),
+							graph.ValueOut(None, expName(10, 1, "c")),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b<c)}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.TupleOut(
+								Some(expName(6, 1, "b")),
+								graph.ValueOut(None, expName(8, 1, "c")),
+							),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b<(c))}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.TupleOut(
+								Some(expName(6, 1, "b")),
+								graph.ValueOut(None, expName(8, 1, "c")),
+							),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b<(c,d<1))}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.TupleOut(
+								Some(expName(6, 1, "b")),
+								graph.ValueOut(None, expName(8, 1, "c")),
+								graph.ValueOut(
+									Some(expName(12, 1, "d")),
+									expNum(10, 1, 1),
+								),
+							),
+						),
+					),
+			),
+		},
+		{
+			in: `{foo=a<(b<( ( (c) ) ))}`,
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.TupleOut(
+							Some(expName(6, 1, "a")),
+							graph.TupleOut(
+								Some(expName(6, 1, "b")),
+								graph.ValueOut(None, expName(8, 1, "c")),
+							),
+						),
+					),
+			),
+		},
+	})
+
+	runTests(t, "comment", graphTerm, []test{
+		{
+			in:  "*\n{}",
+			exp: expGraph(1, 1, new(Graph)),
+		},
+		{
+			in:  "* ignore me!\n{}",
+			exp: expGraph(1, 1, new(Graph)),
+		},
+		{
+			in:  "{* ignore me!\n}",
+			exp: expGraph(1, 1, new(Graph)),
+		},
+		{
+			in: "{foo* ignore me!\n  =  a}",
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(None, expName(6, 1, "a")),
+					),
+			),
+		},
+		{
+			in: "{foo  =  a* ignore me!\n}",
+			exp: expGraph(
+				1, 1, new(Graph).
+					AddValueIn(
+						expName(2, 1, "foo"),
+						graph.ValueOut(None, expName(6, 1, "a")),
+					),
+			),
+		},
+	})
+}

gg/util_test.go

@@ -1,23 +0,0 @@
-package gg
-
-import "io"
-
-type mockReader struct {
-	body []byte
-	err  error
-}
-
-func (r *mockReader) Read(b []byte) (int, error) {
-
-	n := copy(b, r.body)
-	r.body = r.body[n:]
-
-	if len(r.body) == 0 {
-		if r.err == nil {
-			return n, io.EOF
-		}
-		return n, r.err
-	}
-
-	return n, nil
-}

go.mod

@@ -7,5 +7,6 @@ require github.com/stretchr/testify v1.7.0
 require (
 	github.com/davecgh/go-spew v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )

go.sum

@@ -5,6 +5,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

graph/graph.go

@@ -126,7 +126,7 @@ func TupleOut[E, V Value](edgeVal E, ins ...*OpenEdge[E, V]) *OpenEdge[E, V] {
 	if len(ins) == 1 {
 
 		var (
-			zero V
+			zero E
 			in   = ins[0]
 		)
 
@@ -353,7 +353,6 @@ type reducedEdge[Ea, Va Value, Vb any] struct {
 // If a value or edge is connected to multiple times within the root OpenEdge it
 // will only be mapped/reduced a single time, and the result of that single
 // map/reduction will be passed to each dependant operation.
-//
 func MapReduce[Ea, Va Value, Vb any](
 	root *OpenEdge[Ea, Va],
 	mapVal func(Va) (Vb, error),

vm/function.go

@@ -31,11 +31,6 @@ func Identity(val Value) Function {
 	})
 }
 
-type graphFn struct {
-	*gg.Graph
-	scope Scope
-}
-
 var (
 	valNameIn     = Value{Value: gg.Name("in")}
 	valNameOut    = Value{Value: gg.Name("out")}
@@ -104,12 +99,12 @@ func FunctionFromGraph(g *gg.Graph, scope Scope) (Function, error) {
 
 	compileEdge = func(edge *gg.OpenEdge) (edgeFn, error) {
 
-		return graph.MapReduce[gg.Value, gg.Value, edgeFn](
+		return graph.MapReduce[gg.OptionalValue, gg.Value, edgeFn](
 			edge,
 			func(ggVal gg.Value) (edgeFn, error) {
 				return valToEdgeFn(Value{Value: ggVal})
 			},
-			func(ggEdgeVal gg.Value, inEdgeFns []edgeFn) (edgeFn, error) {
+			func(ggEdgeVal gg.OptionalValue, inEdgeFns []edgeFn) (edgeFn, error) {
 
 				if ggEdgeVal.Equal(valNameIf.Value) {
 
@@ -147,7 +142,10 @@ func FunctionFromGraph(g *gg.Graph, scope Scope) (Function, error) {
 					}))
 				}
 
-				edgeVal := Value{Value: ggEdgeVal}
+				var edgeVal Value
+				if ggEdgeVal.Valid {
+					edgeVal.Value = ggEdgeVal.Value
+				}
 
 				if edgeVal.IsZero() {
 					return inEdgeFn, nil

vm/vm.go

@@ -2,6 +2,7 @@
 package vm
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -44,7 +45,7 @@ func (v Value) Equal(v2g graph.Value) bool {
 
 	switch {
 
-	case !v.Value.IsZero() || !v2.Value.IsZero():
+	case (v.Value != (gg.Value{}) || v2.Value != (gg.Value{})):
 		return v.Value.Equal(v2.Value)
 
 	case v.Function != nil || v2.Function != nil:
@@ -81,7 +82,7 @@ func (v Value) String() string {
 		// We can try to get better strings for ops later
 		return "<fn>"
 
-	case !v.Value.IsZero():
+	case v.Value != (gg.Value{}):
 		return v.Value.String()
 
 	default:
@@ -106,15 +107,14 @@ func (v Value) String() string {
 // scope contains pre-defined operations and values which are available during
 // the evaluation.
 func EvaluateSource(opSrc io.Reader, input Value, scope Scope) (Value, error) {
-	lexer := gg.NewLexer(opSrc)
+	v, err := gg.NewDecoder(opSrc).Next()
-
-	g, err := gg.DecodeLexer(lexer)
 	if err != nil {
 		return Value{}, err
+	} else if v.Graph == nil {
+		return Value{}, errors.New("value must be a graph")
 	}
 
-	fn, err := FunctionFromGraph(g, scope.NewScope())
+	fn, err := FunctionFromGraph(v.Graph, scope.NewScope())
-
 	if err != nil {
 		return Value{}, err
 	}

vm/vm_test.go

@@ -10,11 +10,11 @@ import (
 
 func TestVM(t *testing.T) {
 
-	src := `
+	src := `{
-		incr = { out = add < (1; in;); };
+		incr = { out = add < (1, in); };
 
 		out = incr < incr < in;
-	`
+	}`
 
 	var in int64 = 5