Compare commits

...

3 Commits

Author SHA1 Message Date
4870455430 Completely refactor gg with new BNF file and decoder
The new gg format is based on a BNF file which can be found in the `gg`
directory. The code for decoding `.gg` files has been refactored to
mirror that file. The result is more resilient parsing, better errors,
and a greater ability to extend the format in the future.

The new decoder is notable in that it does not use a lexer. Both lexing
and parsing are done in a single step.

The format syntax itself has also been modified. Rather than using
semi-colons everywhere, commas are used as separators in tuples.
Additionally the final comma/semi-colon is no longer required.
2023-10-25 11:31:33 +02:00
21c91731e9 Rename Operation to Function, plus some cleanup 2023-10-16 18:20:01 +02:00
7d0fcbf28a Switch to using nix flakes, and update go to a real version 2023-10-16 18:16:17 +02:00
25 changed files with 1398 additions and 1339 deletions

View File

@ -6,18 +6,16 @@ super-early-alpha-don't-actually-use-this-for-anything development.
## Development
Current efforts on ginger are focused on a golang-based virtual machine, which
will then be used to bootstrap the language. go >=1.18 is required for this vm.
will then be used to bootstrap the language.
If you are on a linux-amd64 machine with nix installed, you can run:
If you are on a machine with nix installed, you can run:
```
nix-shell -A shell
nix develop
```
from the repo root and you will be dropped into a shell with all dependencies
(including the correct go version) in your PATH, ready to use. This could
probably be expanded to other OSs/architectures easily, if you care to do so
please check out the `default.nix` file and submit a PR!
(including the correct go version) in your PATH, ready to use.
## Demo

View File

@ -19,10 +19,7 @@ func main() {
opSrc := os.Args[1]
inSrc := os.Args[2]
inVal, err := gg.DecodeSingleValueFromLexer(
gg.NewLexer(bytes.NewBufferString(inSrc + ";")),
)
inVal, err := gg.NewDecoder(bytes.NewBufferString(inSrc)).Next()
if err != nil {
panic(fmt.Sprintf("decoding input: %v", err))
}
@ -32,7 +29,6 @@ func main() {
vm.Value{Value: inVal},
vm.GlobalScope,
)
if err != nil {
panic(fmt.Sprintf("evaluating: %v", err))
}

View File

@ -1,25 +0,0 @@
{
pkgs ? import (fetchTarball {
name = "nixpkgs-21-11";
url = "https://github.com/NixOS/nixpkgs/archive/a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31.tar.gz";
sha256 = "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02";
}) { },
}: rec {
# https://go.dev/dl/#go1.18beta1
go = fetchTarball {
name = "go1.18beta1";
url = "https://go.dev/dl/go1.18beta1.linux-amd64.tar.gz";
sha256 = "09sb0viv1ybx6adgx4jym1sckdq3mpjkd6albj06hwnchj5rqn40";
};
shell = pkgs.mkShell {
name = "ginger-dev";
buildInputs = [
go
];
};
}

View File

@ -1,19 +1,24 @@
out = {
* A function which accepts a number N and returns the Nth fibonacci number
{
* We are passing a tuple of inputs into a graph here, such that the graph is
* evaluated as an anonymous function. That anonymous function uses recur
* internally to compute the result.
out = {
decr = { out = add < (in; -1;); };
* A little helper function.
decr = { out = add < (in, -1) };
n = tupEl < (in; 0;);
a = tupEl < (in; 1;);
b = tupEl < (in; 2;);
* Deconstruct the input tuple into its individual elements, for clarity.
* There will be a more ergonomic way of doing this one day.
n = tupEl < (in, 0);
a = tupEl < (in, 1);
b = tupEl < (in, 2);
out = if < (
isZero < n;
a;
recur < (
decr < n;
b;
add < (a;b;);
out = if < (
isZero < n,
a,
recur < ( decr<n, b, add<(a,b) ),
);
);
} < (in; 0; 1;);
} < (in, 0, 1);
}

26
flake.lock Normal file
View File

@ -0,0 +1,26 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1696983906,
"narHash": "sha256-L7GyeErguS7Pg4h8nK0wGlcUTbfUMDu+HMf1UcyP72k=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "bd1cde45c77891214131cbbea5b1203e485a9d51",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-23.05",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

44
flake.nix Normal file
View File

@ -0,0 +1,44 @@
{
description = "gotc development environment";
# Nixpkgs / NixOS version to use.
inputs.nixpkgs.url = "nixpkgs/nixos-23.05";
outputs = { self, nixpkgs }:
let
# to work with older version of flakes
lastModifiedDate = self.lastModifiedDate or self.lastModified or "19700101";
# Generate a user-friendly version number.
version = builtins.substring 0 8 lastModifiedDate;
# System types to support.
supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
# Helper function to generate an attrset '{ x86_64-linux = f "x86_64-linux"; ... }'.
forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
# Nixpkgs instantiated for supported system types.
nixpkgsFor = forAllSystems (system: import nixpkgs {
inherit system;
});
in
{
# Add dependencies that are only needed for development
devShells = forAllSystems (system:
let
pkgs = nixpkgsFor.${system};
in {
default = pkgs.mkShell {
buildInputs = [
pkgs.go
pkgs.gotools
pkgs.golangci-lint
];
};
});
};
}

View File

@ -1,353 +1,73 @@
package gg
import (
"errors"
"bufio"
"fmt"
"io"
"strconv"
"github.com/mediocregopher/ginger/graph"
)
// Type aliases for convenience
type (
Graph = graph.Graph[Value, Value]
OpenEdge = graph.OpenEdge[Value, Value]
)
// Decoder reads Value's off of a byte stream.
type Decoder struct {
br *bufio.Reader
brNextLoc Location
// Punctuations which are used in the gg file format.
const (
punctTerm = ";"
punctOp = "<"
punctAssign = "="
punctOpenGraph = "{"
punctCloseGraph = "}"
punctOpenTuple = "("
punctCloseTuple = ")"
)
func decoderErr(tok LexerToken, err error) error {
return fmt.Errorf("%s: %w", tok.errPrefix(), err)
unread []locatableRune
lastRead locatableRune
}
func decoderErrf(tok LexerToken, str string, args ...interface{}) error {
return decoderErr(tok, fmt.Errorf(str, args...))
}
func isPunct(tok LexerToken, val string) bool {
return tok.Kind == LexerTokenKindPunctuation && tok.Value == val
}
func isTerm(tok LexerToken) bool {
return isPunct(tok, punctTerm)
}
// decoder is currently only really used to namespace functions related to
// decoding Graphs. It may later have actual fields added to it, such as for
// options passed by the caller.
type decoder struct{}
// returned boolean value indicates if the token following the single token is a
// term. If a term followed the first token then it is not included in the
// returned leftover tokens.
//
// if termed is false then leftover tokens cannot be empty.
func (d *decoder) parseSingleValue(
toks []LexerToken,
) (
Value, []LexerToken, bool, error,
) {
tok, rest := toks[0], toks[1:]
if len(rest) == 0 {
return ZeroValue, nil, false, decoderErrf(tok, "cannot be final token, possibly missing %q", punctTerm)
}
termed := isTerm(rest[0])
if termed {
rest = rest[1:]
}
switch tok.Kind {
case LexerTokenKindName:
return Value{Name: &tok.Value, LexerToken: &tok}, rest, termed, nil
case LexerTokenKindNumber:
i, err := strconv.ParseInt(tok.Value, 10, 64)
if err != nil {
return ZeroValue, nil, false, decoderErrf(tok, "parsing %q as integer: %w", tok.Value, err)
}
return Value{Number: &i, LexerToken: &tok}, rest, termed, nil
case LexerTokenKindPunctuation:
return ZeroValue, nil, false, decoderErrf(tok, "expected value, found punctuation %q", tok.Value)
default:
panic(fmt.Sprintf("unexpected token kind %q", tok.Kind))
// NewDecoder returns a Decoder which will decode the given stream as a gg
// formatted stream of a Values.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{
br: bufio.NewReader(r),
brNextLoc: Location{Row: 1, Col: 1},
}
}
func (d *decoder) parseOpenEdge(
toks []LexerToken,
) (
*OpenEdge, []LexerToken, error,
) {
if isPunct(toks[0], punctOpenTuple) {
return d.parseTuple(toks)
func (d *Decoder) readRune() (locatableRune, error) {
if len(d.unread) > 0 {
d.lastRead = d.unread[len(d.unread)-1]
d.unread = d.unread[:len(d.unread)-1]
return d.lastRead, nil
}
var (
val Value
termed bool
err error
)
switch {
case isPunct(toks[0], punctOpenGraph):
val, toks, termed, err = d.parseGraphValue(toks, true)
default:
val, toks, termed, err = d.parseSingleValue(toks)
}
loc := d.brNextLoc
r, _, err := d.br.ReadRune()
if err != nil {
return nil, nil, err
return d.lastRead, err
}
if termed {
return graph.ValueOut[Value](ZeroValue, val), toks, nil
if r == '\n' {
d.brNextLoc.Row++
d.brNextLoc.Col = 1
} else {
d.brNextLoc.Col++
}
opTok, toks := toks[0], toks[1:]
if !isPunct(opTok, punctOp) {
return nil, nil, decoderErrf(opTok, "must be %q or %q", punctOp, punctTerm)
}
if len(toks) == 0 {
return nil, nil, decoderErrf(opTok, "%q cannot terminate an edge declaration", punctOp)
}
oe, toks, err := d.parseOpenEdge(toks)
if err != nil {
return nil, nil, err
}
oe = graph.TupleOut[Value](val, oe)
return oe, toks, nil
d.lastRead = locatableRune{loc, r}
return d.lastRead, nil
}
func (d *decoder) parseTuple(
toks []LexerToken,
) (
*OpenEdge, []LexerToken, error,
) {
openTok, toks := toks[0], toks[1:]
var edges []*OpenEdge
for {
if len(toks) == 0 {
return nil, nil, decoderErrf(openTok, "no matching %q", punctCloseTuple)
} else if isPunct(toks[0], punctCloseTuple) {
toks = toks[1:]
break
}
var (
oe *OpenEdge
err error
)
oe, toks, err = d.parseOpenEdge(toks)
if err != nil {
return nil, nil, err
}
edges = append(edges, oe)
func (d *Decoder) unreadRune(lr locatableRune) {
if d.lastRead != lr {
panic(fmt.Sprintf(
"unreading rune %#v, but last read rune was %#v", lr, d.lastRead,
))
}
// this is a quirk of the syntax, _technically_ a tuple doesn't need a
// term after it, since it can't be used as an edge value, and so
// nothing can come after it in the chain.
if len(toks) > 0 && isTerm(toks[0]) {
toks = toks[1:]
}
return graph.TupleOut[Value](ZeroValue, edges...), toks, nil
d.unread = append(d.unread, lr)
}
// returned boolean value indicates if the token following the graph is a term.
// If a term followed the first token then it is not included in the returned
// leftover tokens.
//
// if termed is false then leftover tokens cannot be empty.
func (d *decoder) parseGraphValue(
toks []LexerToken, expectWrappers bool,
) (
Value, []LexerToken, bool, error,
) {
var openTok LexerToken
if expectWrappers {
openTok, toks = toks[0], toks[1:]
func (d *Decoder) nextLoc() Location {
if len(d.unread) > 0 {
return d.unread[len(d.unread)-1].Location
}
g := new(Graph)
for {
if len(toks) == 0 {
if !expectWrappers {
break
}
return ZeroValue, nil, false, decoderErrf(openTok, "no matching %q", punctCloseGraph)
} else if closingTok := toks[0]; isPunct(closingTok, punctCloseGraph) {
if !expectWrappers {
return ZeroValue, nil, false, decoderErrf(closingTok, "unexpected %q", punctCloseGraph)
}
toks = toks[1:]
if len(toks) == 0 {
return ZeroValue, nil, false, decoderErrf(closingTok, "cannot be final token, possibly missing %q", punctTerm)
}
break
}
var err error
if g, toks, err = d.parseValIn(g, toks); err != nil {
return ZeroValue, nil, false, err
}
}
val := Value{Graph: g}
if !expectWrappers {
return val, toks, true, nil
}
val.LexerToken = &openTok
termed := isTerm(toks[0])
if termed {
toks = toks[1:]
}
return val, toks, termed, nil
return d.brNextLoc
}
func (d *decoder) parseValIn(into *Graph, toks []LexerToken) (*Graph, []LexerToken, error) {
if len(toks) == 0 {
return into, nil, nil
} else if len(toks) < 3 {
return nil, nil, decoderErrf(toks[0], `must be of the form "<name> = ..."`)
}
dst := toks[0]
eq := toks[1]
toks = toks[2:]
if dst.Kind != LexerTokenKindName {
return nil, nil, decoderErrf(dst, "must be a name")
} else if !isPunct(eq, punctAssign) {
return nil, nil, decoderErrf(eq, "must be %q", punctAssign)
}
oe, toks, err := d.parseOpenEdge(toks)
if err != nil {
return nil, nil, err
}
dstVal := Value{Name: &dst.Value, LexerToken: &dst}
return into.AddValueIn(dstVal, oe), toks, nil
}
func (d *decoder) readAllTokens(lexer Lexer) ([]LexerToken, error) {
var toks []LexerToken
for {
tok, err := lexer.Next()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
return nil, fmt.Errorf("reading next token: %w", err)
}
toks = append(toks, tok)
}
return toks, nil
}
func (d *decoder) decode(lexer Lexer) (*Graph, error) {
toks, err := d.readAllTokens(lexer)
if err != nil {
return nil, err
}
val, _, _, err := d.parseGraphValue(toks, false)
if err != nil {
return nil, err
}
return val.Graph, nil
}
// DecodeLexer reads lexigraphical tokens from the given Lexer and uses them to
// construct a Graph according to the rules of the gg file format. DecodeLexer
// will only return an error if there is a non-EOF file returned from the Lexer,
// or the tokens read cannot be used to construct a valid Graph.
func DecodeLexer(lexer Lexer) (*Graph, error) {
decoder := &decoder{}
return decoder.decode(lexer)
}
func DecodeSingleValueFromLexer(lexer Lexer) (Value, error) {
decoder := &decoder{}
toks, err := decoder.readAllTokens(lexer)
if err != nil {
return ZeroValue, err
}
val, _, _, err := decoder.parseSingleValue(toks)
return val, err
// Next returns the next top-level value in the stream, or io.EOF.
func (d *Decoder) Next() (Value, error) {
return topLevelTerm.decodeFn(d)
}

View File

@ -1,149 +0,0 @@
package gg
import (
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/mediocregopher/ginger/graph"
)
func TestDecoder(t *testing.T) {
zeroGraph := new(Graph)
i := func(i int64) Value {
return Value{Number: &i}
}
n := func(n string) Value {
return Value{Name: &n}
}
vOut := func(edgeVal, val Value) *OpenEdge {
return graph.ValueOut(edgeVal, val)
}
tOut := func(edgeVal Value, ins ...*OpenEdge) *OpenEdge {
return graph.TupleOut(edgeVal, ins...)
}
tests := []struct {
in string
exp *Graph
}{
{
in: "",
exp: zeroGraph,
},
{
in: "out = 1;",
exp: zeroGraph.AddValueIn(n("out"), vOut(ZeroValue, i(1))),
},
{
in: "out = incr < 1;",
exp: zeroGraph.AddValueIn(n("out"), vOut(n("incr"), i(1))),
},
{
in: "out = a < b < 1;",
exp: zeroGraph.AddValueIn(
n("out"),
tOut(
n("a"),
vOut(n("b"),
i(1)),
),
),
},
{
in: "out = a < b < (1; c < 2; d < e < 3;);",
exp: zeroGraph.AddValueIn(
n("out"),
tOut(
n("a"),
tOut(
n("b"),
vOut(ZeroValue, i(1)),
vOut(n("c"), i(2)),
tOut(
n("d"),
vOut(n("e"), i(3)),
),
),
),
),
},
{
in: "out = a < b < (1; c < (d < 2; 3;); );",
exp: zeroGraph.AddValueIn(
n("out"),
tOut(
n("a"),
tOut(
n("b"),
vOut(ZeroValue, i(1)),
tOut(
n("c"),
vOut(n("d"), i(2)),
vOut(ZeroValue, i(3)),
),
),
),
),
},
{
in: "out = { a = 1; b = c < d < 2; };",
exp: zeroGraph.AddValueIn(
n("out"),
vOut(
ZeroValue,
Value{Graph: zeroGraph.
AddValueIn(n("a"), vOut(ZeroValue, i(1))).
AddValueIn(
n("b"),
tOut(
n("c"),
vOut(n("d"), i(2)),
),
),
},
),
),
},
{
in: "out = a < { b = 1; } < 2;",
exp: zeroGraph.AddValueIn(
n("out"),
tOut(
n("a"),
vOut(
Value{Graph: zeroGraph.
AddValueIn(n("b"), vOut(ZeroValue, i(1))),
},
i(2),
),
),
),
},
{
in: "a = 1; b = 2;",
exp: zeroGraph.
AddValueIn(n("a"), vOut(ZeroValue, i(1))).
AddValueIn(n("b"), vOut(ZeroValue, i(2))),
},
}
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
r := &mockReader{body: []byte(test.in)}
lexer := NewLexer(r)
got, err := DecodeLexer(lexer)
assert.NoError(t, err)
assert.True(t, got.Equal(test.exp), "\nexp:%v\ngot:%v", test.exp, got)
})
}
}

23
gg/gg.bnf Normal file
View File

@ -0,0 +1,23 @@
<digit> ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
<positive-number> ::= <digit>+
<negative-number> ::= "-" <positive-number>
<number> ::= <negative-number> | <positive-number>
<name> ::= (<letter> | <mark>) (<letter> | <mark> | <digit>)*
<tuple> ::= "(" <tuple-tail>
<tuple-tail> ::= ")" | <tuple-open-edge>
<tuple-open-edge> ::= <value> <tuple-open-edge-value-tail>
| <tuple> <tuple-open-edge-tail>
<tuple-open-edge-tail> ::= ")" | "," <tuple-tail>
<tuple-open-edge-value-tail> ::= <tuple-open-edge-tail> | "<" <tuple-open-edge>
<graph> ::= "{" <graph-tail>
<graph-tail> ::= "}" | <name> "=" <graph-open-edge>
<graph-open-edge> ::= <value> <graph-open-edge-value-tail>
| <tuple> <graph-open-edge-tail>
<graph-open-edge-tail> ::= "}" | ";" <graph-tail>
<graph-open-edge-value-tail> ::= <graph-open-edge-tail> | "<" <graph-open-edge>
<value> ::= <name> | <number> | <graph>
<gg> ::= <eof> | <value> <gg>

View File

@ -7,23 +7,20 @@ import (
"github.com/mediocregopher/ginger/graph"
)
// ZeroValue is a Value with no fields set.
var ZeroValue Value
// Type aliases for convenience
type (
Graph = graph.Graph[OptionalValue, Value]
OpenEdge = graph.OpenEdge[OptionalValue, Value]
)
// Value represents a value which can be serialized by the gg text format.
type Value struct {
Location
// Only one of these fields may be set
Name *string
Number *int64
Graph *Graph
// TODO coming soon!
// String *string
// Optional fields indicating the token which was used to construct this
// Value, if any.
LexerToken *LexerToken
}
// Name returns a name Value.
@ -36,12 +33,6 @@ func Number(n int64) Value {
return Value{Number: &n}
}
// IsZero returns true if the Value is the zero value (none of the sub-value
// fields are set). LexerToken is ignored for this check.
func (v Value) IsZero() bool {
return v.Equal(ZeroValue)
}
// Equal returns true if the passed in Value is equivalent, ignoring the
// LexerToken on either Value.
//
@ -50,13 +41,8 @@ func (v Value) Equal(v2g graph.Value) bool {
v2 := v2g.(Value)
v.LexerToken, v2.LexerToken = nil, nil
switch {
case v == ZeroValue && v2 == ZeroValue:
return true
case v.Name != nil && v2.Name != nil && *v.Name == *v2.Name:
return true
@ -85,6 +71,46 @@ func (v Value) String() string {
return v.Graph.String()
default:
return "<zero>"
panic("no fields set on Value")
}
}
// OptionalValue is a Value which may be unset. This is used for edge values,
// since edges might not have a value.
type OptionalValue struct {
Value
Valid bool
}
// None is the zero OptionalValue (hello rustaceans).
var None OptionalValue
// Some wraps a Value to be an OptionalValue.
func Some(v Value) OptionalValue {
return OptionalValue{Valid: true, Value: v}
}
func (v OptionalValue) String() string {
if !v.Valid {
return "<none>"
}
return v.Value.String()
}
func (v OptionalValue) Equal(v2g graph.Value) bool {
var v2 OptionalValue
if v2Val, ok := v2g.(Value); ok {
v2 = Some(v2Val)
} else {
v2 = v2g.(OptionalValue)
}
if v.Valid != v2.Valid {
return false
} else if !v.Valid {
return true
}
return v.Value.Equal(v2.Value)
}

View File

@ -1,292 +0,0 @@
package gg
import (
"bufio"
"fmt"
"io"
"strings"
"unicode"
)
// LexerLocation describes the location in a file where a particular token was
// parsed from.
type LexerLocation struct {
Row, Col int
}
func (l LexerLocation) String() string {
return fmt.Sprintf("%d:%d", l.Row, l.Col)
}
// LexerError is returned by Lexer when an unexpected error occurs parsing a
// stream of LexerTokens.
type LexerError struct {
Err error
Location LexerLocation
}
func (e *LexerError) Error() string {
return fmt.Sprintf("%s: %s", e.Location.String(), e.Err.Error())
}
func (e *LexerError) Unwrap() error {
return e.Err
}
// LexerTokenKind enumerates the different kinds of LexerToken there can be.
type LexerTokenKind string
// Enumeration of LexerTokenKinds.
const (
LexerTokenKindName LexerTokenKind = "name"
LexerTokenKindNumber LexerTokenKind = "number"
LexerTokenKindPunctuation LexerTokenKind = "punctuation"
)
// LexerToken describes a lexigraphical token which is used when deserializing
// Graphs.
type LexerToken struct {
Kind LexerTokenKind
Value string // never empty string
Location LexerLocation
}
func (t LexerToken) errPrefix() string {
return fmt.Sprintf("%s: at %q", t.Location.String(), t.Value)
}
// Lexer is used to parse a string stream into a sequence of tokens which can
// then be parsed by a Parser.
type Lexer interface {
// Next will return a LexerToken or a LexerError. io.EOF (wrapped in a
// LexerError) is returned if the stream being read from is finished.
Next() (LexerToken, error)
}
type lexer struct {
r *bufio.Reader
stringBuilder *strings.Builder
err *LexerError
// these fields are only needed to keep track of the current "cursor"
// position when reading.
lastRow, lastCol int
prevRune rune
}
// NewLexer wraps the io.Reader in a Lexer, which will read the io.Reader as a
// sequence of utf-8 characters and parse it into a sequence of LexerTokens.
func NewLexer(r io.Reader) Lexer {
return &lexer{
r: bufio.NewReader(r),
lastRow: 0,
lastCol: -1,
stringBuilder: new(strings.Builder),
}
}
// nextRowCol returns the row and column number which the next rune in the
// stream would be at.
func (l *lexer) nextRowCol() (int, int) {
if l.prevRune == '\n' {
return l.lastRow + 1, 0
}
return l.lastRow, l.lastCol + 1
}
func (l *lexer) fmtErr(err error) *LexerError {
row, col := l.nextRowCol()
return &LexerError{
Err: err,
Location: LexerLocation{
Row: row,
Col: col,
},
}
}
func (l *lexer) fmtErrf(str string, args ...interface{}) *LexerError {
return l.fmtErr(fmt.Errorf(str, args...))
}
// discardRune must _always_ be called only after peekRune.
func (l *lexer) discardRune() {
r, _, err := l.r.ReadRune()
if err != nil {
panic(err)
}
l.lastRow, l.lastCol = l.nextRowCol()
l.prevRune = r
}
func (l *lexer) peekRune() (rune, error) {
r, _, err := l.r.ReadRune()
if err != nil {
return '0', err
} else if err := l.r.UnreadRune(); err != nil {
// since the most recent operation on the bufio.Reader was a ReadRune,
// UnreadRune should never return an error
panic(err)
}
return r, nil
}
// readWhile reads runes until the given predicate returns false, and returns a
// LexerToken of the given kind whose Value is comprised of all runes which
// returned true.
//
// If an error is encountered then both the token (or what's been parsed of it
// so far) and the error are returned.
func (l *lexer) readWhile(
kind LexerTokenKind, pred func(rune) bool,
) (
LexerToken, *LexerError,
) {
row, col := l.nextRowCol()
l.stringBuilder.Reset()
var lexErr *LexerError
for {
r, err := l.peekRune()
if err != nil {
lexErr = l.fmtErrf("peeking next character: %w", err)
break
} else if !pred(r) {
break
}
l.stringBuilder.WriteRune(r)
l.discardRune()
}
return LexerToken{
Kind: kind,
Value: l.stringBuilder.String(),
Location: LexerLocation{
Row: row, Col: col,
},
}, lexErr
}
// we only support base-10 integers at the moment.
func isNumber(r rune) bool {
return r == '-' || ('0' <= r && r <= '9')
}
// next can return a token, an error, or both. If an error is returned then no
// further calls to next should occur.
func (l *lexer) next() (LexerToken, *LexerError) {
for {
r, err := l.peekRune()
if err != nil {
return LexerToken{}, l.fmtErrf("peeking next character: %w", err)
}
switch {
case r == '*': // comment
// comments are everything up until a newline
_, err := l.readWhile("", func(r rune) bool {
return r != '\n'
})
if err != nil {
return LexerToken{}, err
}
// terminating newline will be discarded on next loop
case r == '"' || r == '`':
// reserve double-quote and backtick for string parsing.
l.discardRune()
return LexerToken{}, l.fmtErrf("string parsing not yet implemented")
case unicode.IsLetter(r):
// letters denote the start of a name
return l.readWhile(LexerTokenKindName, func(r rune) bool {
if unicode.In(r, unicode.Letter, unicode.Number, unicode.Mark) {
return true
}
if r == '-' {
return true
}
return false
})
case isNumber(r):
return l.readWhile(LexerTokenKindNumber, isNumber)
case unicode.IsPunct(r) || unicode.IsSymbol(r):
// symbols are also considered punctuation
l.discardRune()
return LexerToken{
Kind: LexerTokenKindPunctuation,
Value: string(r),
Location: LexerLocation{
Row: l.lastRow,
Col: l.lastCol,
},
}, nil
case unicode.IsSpace(r):
l.discardRune()
default:
return LexerToken{}, l.fmtErrf("unexpected character %q", r)
}
}
}
func (l *lexer) Next() (LexerToken, error) {
if l.err != nil {
return LexerToken{}, l.err
}
tok, err := l.next()
if err != nil {
l.err = err
if tok.Kind == "" {
return LexerToken{}, l.err
}
}
return tok, nil
}

View File

@ -1,150 +0,0 @@
package gg
import (
"errors"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLexer(t *testing.T) {
expErr := errors.New("eof")
tests := []struct {
in string
exp []LexerToken
}{
{in: "", exp: []LexerToken{}},
{in: "* fooo\n", exp: []LexerToken{}},
{
in: "foo",
exp: []LexerToken{
{
Kind: LexerTokenKindName,
Value: "foo",
Location: LexerLocation{Row: 0, Col: 0},
},
},
},
{
in: "foo bar\nf-o f0O Foo",
exp: []LexerToken{
{
Kind: LexerTokenKindName,
Value: "foo",
Location: LexerLocation{Row: 0, Col: 0},
},
{
Kind: LexerTokenKindName,
Value: "bar",
Location: LexerLocation{Row: 0, Col: 4},
},
{
Kind: LexerTokenKindName,
Value: "f-o",
Location: LexerLocation{Row: 1, Col: 0},
},
{
Kind: LexerTokenKindName,
Value: "f0O",
Location: LexerLocation{Row: 1, Col: 4},
},
{
Kind: LexerTokenKindName,
Value: "Foo",
Location: LexerLocation{Row: 1, Col: 8},
},
},
},
{
in: "1 100 -100",
exp: []LexerToken{
{
Kind: LexerTokenKindNumber,
Value: "1",
Location: LexerLocation{Row: 0, Col: 0},
},
{
Kind: LexerTokenKindNumber,
Value: "100",
Location: LexerLocation{Row: 0, Col: 2},
},
{
Kind: LexerTokenKindNumber,
Value: "-100",
Location: LexerLocation{Row: 0, Col: 6},
},
},
},
{
in: "1<2!-3 ()",
exp: []LexerToken{
{
Kind: LexerTokenKindNumber,
Value: "1",
Location: LexerLocation{Row: 0, Col: 0},
},
{
Kind: LexerTokenKindPunctuation,
Value: "<",
Location: LexerLocation{Row: 0, Col: 1},
},
{
Kind: LexerTokenKindNumber,
Value: "2",
Location: LexerLocation{Row: 0, Col: 2},
},
{
Kind: LexerTokenKindPunctuation,
Value: "!",
Location: LexerLocation{Row: 0, Col: 3},
},
{
Kind: LexerTokenKindNumber,
Value: "-3",
Location: LexerLocation{Row: 0, Col: 4},
},
{
Kind: LexerTokenKindPunctuation,
Value: "(",
Location: LexerLocation{Row: 0, Col: 7},
},
{
Kind: LexerTokenKindPunctuation,
Value: ")",
Location: LexerLocation{Row: 0, Col: 8},
},
},
},
}
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
lexer := NewLexer(&mockReader{body: []byte(test.in), err: expErr})
for i := range test.exp {
tok, err := lexer.Next()
assert.NoError(t, err)
assert.Equal(t, test.exp[i], tok, "test.exp[%d]", i)
}
tok, err := lexer.Next()
assert.ErrorIs(t, err, expErr)
assert.Equal(t, LexerToken{}, tok)
lexErr := new(LexerError)
assert.True(t, errors.As(err, &lexErr))
inParts := strings.Split(test.in, "\n")
assert.ErrorIs(t, lexErr, expErr)
assert.Equal(t, lexErr.Location.Row, len(inParts)-1)
assert.Equal(t, lexErr.Location.Col, len(inParts[len(inParts)-1]))
})
}
}

36
gg/location.go Normal file
View File

@ -0,0 +1,36 @@
package gg
import "fmt"
// Location indicates a position in a stream of bytes identified by column
// within newline-separated rows.
type Location struct {
Row, Col int
}
func (l Location) errf(str string, args ...any) LocatedError {
return LocatedError{l, fmt.Errorf(str, args...)}
}
func (l Location) locate() Location { return l }
// LocatedError is an error related to a specific point within a decode gg
// stream.
type LocatedError struct {
Location
Err error
}
func (e LocatedError) Error() string {
return fmt.Sprintf("%d:%d: %v", e.Row, e.Col, e.Err)
}
type locatableRune struct {
Location
r rune
}
type locatableString struct {
Location
str string
}

508
gg/term.go Normal file
View File

@ -0,0 +1,508 @@
package gg
import (
"errors"
"fmt"
"io"
"strconv"
"strings"
"unicode"
"github.com/mediocregopher/ginger/graph"
"golang.org/x/exp/slices"
)
var (
errNoMatch = errors.New("not found")
)
type stringerFn func() string
func (fn stringerFn) String() string {
return fn()
}
type stringerStr string
func (str stringerStr) String() string {
return string(str)
}
type term[T any] struct {
name fmt.Stringer
decodeFn func(d *Decoder) (T, error)
}
func (t term[T]) String() string {
return t.name.String()
}
func firstOf[T any](terms ...*term[T]) *term[T] {
if len(terms) < 2 {
panic("firstOfTerms requires at least 2 terms")
}
return &term[T]{
name: stringerFn(func() string {
descrs := make([]string, len(terms))
for i := range terms {
descrs[i] = terms[i].String()
}
return strings.Join(descrs, " or ")
}),
decodeFn: func(d *Decoder) (T, error) {
var zero T
for _, t := range terms {
v, err := t.decodeFn(d)
if errors.Is(err, errNoMatch) {
continue
} else if err != nil {
return zero, err
}
return v, nil
}
return zero, errNoMatch
},
}
}
func seq[Ta, Tb, Tc any](
name fmt.Stringer,
termA *term[Ta],
termB *term[Tb],
fn func(Ta, Tb) Tc,
) *term[Tc] {
return &term[Tc]{
name: name,
decodeFn: func(d *Decoder) (Tc, error) {
var zero Tc
va, err := termA.decodeFn(d)
if err != nil {
return zero, err
}
vb, err := termB.decodeFn(d)
if errors.Is(err, errNoMatch) {
return zero, d.nextLoc().errf("expected %v", termB)
} else if err != nil {
return zero, err
}
return fn(va, vb), nil
},
}
}
func prefixed[Ta, Tb any](termA *term[Ta], termB *term[Tb]) *term[Tb] {
return seq(termA, termA, termB, func(_ Ta, b Tb) Tb {
return b
})
}
func prefixIgnored[Ta, Tb any](termA *term[Ta], termB *term[Tb]) *term[Tb] {
return &term[Tb]{
name: termB,
decodeFn: func(d *Decoder) (Tb, error) {
var zero Tb
if _, err := termA.decodeFn(d); err != nil {
return zero, err
}
return termB.decodeFn(d)
},
}
}
func suffixIgnored[Ta, Tb any](
termA *term[Ta], termB *term[Tb],
) *term[Ta] {
return seq(termA, termA, termB, func(a Ta, _ Tb) Ta {
return a
})
}
func oneOrMore[T any](t *term[T]) *term[[]T] {
return &term[[]T]{
name: stringerFn(func() string {
return fmt.Sprintf("one or more %v", t)
}),
decodeFn: func(d *Decoder) ([]T, error) {
var vv []T
for {
v, err := t.decodeFn(d)
if errors.Is(err, errNoMatch) {
break
} else if err != nil {
return nil, err
}
vv = append(vv, v)
}
if len(vv) == 0 {
return nil, errNoMatch
}
return vv, nil
},
}
}
func zeroOrMore[T any](t *term[T]) *term[[]T] {
return &term[[]T]{
name: stringerFn(func() string {
return fmt.Sprintf("zero or more %v", t)
}),
decodeFn: func(d *Decoder) ([]T, error) {
var vv []T
for {
v, err := t.decodeFn(d)
if errors.Is(err, errNoMatch) {
break
} else if err != nil {
return nil, err
}
vv = append(vv, v)
}
return vv, nil
},
}
}
func mapTerm[Ta, Tb any](
name fmt.Stringer, t *term[Ta], fn func(Ta) Tb,
) *term[Tb] {
return &term[Tb]{
name: name,
decodeFn: func(d *Decoder) (Tb, error) {
var zero Tb
va, err := t.decodeFn(d)
if err != nil {
return zero, err
}
return fn(va), nil
},
}
}
func runePredTerm(
name fmt.Stringer, pred func(rune) bool,
) *term[locatableRune] {
return &term[locatableRune]{
name: name,
decodeFn: func(d *Decoder) (locatableRune, error) {
lr, err := d.readRune()
if errors.Is(err, io.EOF) {
return locatableRune{}, errNoMatch
} else if err != nil {
return locatableRune{}, err
}
if !pred(lr.r) {
d.unreadRune(lr)
return locatableRune{}, errNoMatch
}
return lr, nil
},
}
}
func runeTerm(r rune) *term[locatableRune] {
return runePredTerm(
stringerStr(fmt.Sprintf("'%c'", r)),
func(r2 rune) bool { return r2 == r },
)
}
func locatableRunesToString(rr []locatableRune) string {
str := make([]rune, len(rr))
for i := range rr {
str[i] = rr[i].r
}
return string(str)
}
func runesToStringTerm(
t *term[[]locatableRune],
) *term[locatableString] {
return mapTerm(
t, t, func(rr []locatableRune) locatableString {
return locatableString{rr[0].locate(), locatableRunesToString(rr)}
},
)
}
func discard[T any](t *term[T]) *term[struct{}] {
return mapTerm(t, t, func(_ T) struct{} { return struct{}{} })
}
var (
notNewlineTerm = runePredTerm(
stringerStr("not-newline"),
func(r rune) bool { return r != '\n' },
)
commentTerm = prefixed(
prefixed(runeTerm('*'), zeroOrMore(notNewlineTerm)),
runeTerm('\n'),
)
whitespaceTerm = zeroOrMore(firstOf(
discard(runePredTerm(stringerStr("whitespace"), unicode.IsSpace)),
discard(commentTerm),
))
)
func trimmedTerm[T any](t *term[T]) *term[T] {
t = prefixIgnored(whitespaceTerm, t)
t = suffixIgnored(t, whitespaceTerm)
return t
}
func trimmedRuneTerm(r rune) *term[locatableRune] {
return trimmedTerm(runeTerm(r))
}
var (
digitTerm = runePredTerm(
stringerStr("digit"),
func(r rune) bool { return '0' <= r && r <= '9' },
)
positiveNumberTerm = runesToStringTerm(oneOrMore(digitTerm))
negativeNumberTerm = seq(
stringerStr("negative-number"),
runeTerm('-'),
positiveNumberTerm,
func(neg locatableRune, posNum locatableString) locatableString {
return locatableString{neg.locate(), string(neg.r) + posNum.str}
},
)
numberTerm = mapTerm(
stringerStr("number"),
firstOf(negativeNumberTerm, positiveNumberTerm),
func(str locatableString) Value {
i, err := strconv.ParseInt(str.str, 10, 64)
if err != nil {
panic(fmt.Errorf("parsing %q as int: %w", str, err))
}
return Value{Number: &i, Location: str.locate()}
},
)
)
var (
letterTerm = runePredTerm(
stringerStr("letter"),
func(r rune) bool {
return unicode.In(r, unicode.Letter, unicode.Mark)
},
)
letterTailTerm = zeroOrMore(firstOf(letterTerm, digitTerm))
nameTerm = seq(
stringerStr("name"),
letterTerm,
letterTailTerm,
func(head locatableRune, tail []locatableRune) Value {
name := string(head.r) + locatableRunesToString(tail)
return Value{Name: &name, Location: head.locate()}
},
)
)
func openEdgeIntoValue(val Value, oe *OpenEdge) *OpenEdge {
switch {
case oe == nil:
return graph.ValueOut(None, val)
case !oe.EdgeValue().Valid:
return oe.WithEdgeValue(Some(val))
default:
return graph.TupleOut(Some(val), oe)
}
}
var graphTerm, valueTerm = func() (*term[Value], *term[Value]) {
type tupleState struct {
ins []*OpenEdge
oe *OpenEdge
}
type graphState struct {
g *Graph
oe *OpenEdge
}
var (
rightParenthesis = trimmedRuneTerm(')')
tupleEndTerm = mapTerm(
rightParenthesis,
rightParenthesis,
func(lr locatableRune) tupleState {
// if ')', then map that to an empty state. This acts as a
// sentinel value to indicate "end of tuple".
return tupleState{}
},
)
rightCurlyBrace = trimmedRuneTerm('}')
graphEndTerm = mapTerm(
rightCurlyBrace,
rightCurlyBrace,
func(lr locatableRune) graphState {
// if '}', then map that to an empty state. This acts as a
// sentinel value to indicate "end of graph".
return graphState{}
},
)
)
var (
// pre-define these, and then fill in the pointers after, in order to
// deal with recursive dependencies between them.
valueTerm = new(term[Value])
tupleTerm = new(term[*OpenEdge])
tupleTailTerm = new(term[tupleState])
tupleOpenEdgeTerm = new(term[tupleState])
tupleOpenEdgeTailTerm = new(term[tupleState])
tupleOpenEdgeValueTailTerm = new(term[tupleState])
graphTerm = new(term[Value])
graphTailTerm = new(term[graphState])
graphOpenEdgeTerm = new(term[graphState])
graphOpenEdgeTailTerm = new(term[graphState])
graphOpenEdgeValueTailTerm = new(term[graphState])
)
*tupleTerm = *seq(
stringerStr("tuple"),
trimmedRuneTerm('('),
tupleTailTerm,
func(lr locatableRune, ts tupleState) *OpenEdge {
slices.Reverse(ts.ins)
return graph.TupleOut(None, ts.ins...)
},
)
*tupleTailTerm = *firstOf(
tupleEndTerm,
mapTerm(
tupleOpenEdgeTerm,
tupleOpenEdgeTerm,
func(ts tupleState) tupleState {
ts.ins = append(ts.ins, ts.oe)
ts.oe = nil
return ts
},
),
)
*tupleOpenEdgeTerm = *firstOf(
seq(
valueTerm,
valueTerm,
tupleOpenEdgeValueTailTerm,
func(val Value, ts tupleState) tupleState {
ts.oe = openEdgeIntoValue(val, ts.oe)
return ts
},
),
seq(
tupleTerm,
tupleTerm,
tupleOpenEdgeTailTerm,
func(oe *OpenEdge, ts tupleState) tupleState {
ts.oe = oe
return ts
},
),
)
*tupleOpenEdgeTailTerm = *firstOf(
tupleEndTerm,
prefixed(trimmedRuneTerm(','), tupleTailTerm),
)
*tupleOpenEdgeValueTailTerm = *firstOf(
tupleOpenEdgeTailTerm,
prefixed(trimmedRuneTerm('<'), tupleOpenEdgeTerm),
)
*graphTerm = *seq(
stringerStr("graph"),
trimmedRuneTerm('{'),
graphTailTerm,
func(lr locatableRune, gs graphState) Value {
if gs.g == nil {
gs.g = new(Graph)
}
return Value{Graph: gs.g, Location: lr.locate()}
},
)
*graphTailTerm = *firstOf(
graphEndTerm,
seq(
nameTerm,
nameTerm,
prefixed(trimmedRuneTerm('='), graphOpenEdgeTerm),
func(name Value, gs graphState) graphState {
if gs.g == nil {
gs.g = new(Graph)
}
gs.g = gs.g.AddValueIn(name, gs.oe)
gs.oe = nil
return gs
},
),
)
*graphOpenEdgeTerm = *firstOf(
seq(
valueTerm,
valueTerm,
graphOpenEdgeValueTailTerm,
func(val Value, gs graphState) graphState {
gs.oe = openEdgeIntoValue(val, gs.oe)
return gs
},
),
seq(
tupleTerm,
tupleTerm,
graphOpenEdgeTailTerm,
func(oe *OpenEdge, gs graphState) graphState {
gs.oe = oe
return gs
},
),
)
*graphOpenEdgeTailTerm = *firstOf(
graphEndTerm,
prefixed(trimmedRuneTerm(';'), graphTailTerm),
)
*graphOpenEdgeValueTailTerm = *firstOf(
graphOpenEdgeTailTerm,
prefixed(trimmedRuneTerm('<'), graphOpenEdgeTerm),
)
*valueTerm = *firstOf(nameTerm, numberTerm, graphTerm)
return graphTerm, valueTerm
}()
var topLevelTerm = trimmedTerm(valueTerm)

387
gg/term_test.go Normal file
View File

@ -0,0 +1,387 @@
package gg
import (
"bytes"
"io"
"strconv"
"testing"
"github.com/mediocregopher/ginger/graph"
"github.com/stretchr/testify/assert"
)
func decoderLeftover(d *Decoder) string {
unread := make([]rune, len(d.unread))
for i := range unread {
unread[i] = d.unread[i].r
}
rest, err := io.ReadAll(d.br)
if err != nil {
panic(err)
}
return string(unread) + string(rest)
}
func TestTermDecoding(t *testing.T) {
type test struct {
in string
exp Value
expErr string
leftover string
}
runTests := func(
t *testing.T, name string, term *term[Value], tests []test,
) {
t.Run(name, func(t *testing.T) {
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
dec := NewDecoder(bytes.NewBufferString(test.in))
got, err := term.decodeFn(dec)
if test.expErr != "" {
assert.Error(t, err)
assert.Equal(t, test.expErr, err.Error())
} else if assert.NoError(t, err) {
assert.True(t,
test.exp.Equal(got),
"\nexp:%v\ngot:%v", test.exp, got,
)
assert.Equal(t, test.leftover, decoderLeftover(dec))
}
})
}
})
}
expNum := func(row, col int, n int64) Value {
return Value{Number: &n, Location: Location{row, col}}
}
runTests(t, "number", numberTerm, []test{
{in: `0`, exp: expNum(1, 1, 0)},
{in: `100`, exp: expNum(1, 1, 100)},
{in: `-100`, exp: expNum(1, 1, -100)},
{in: `0foo`, exp: expNum(1, 1, 0), leftover: "foo"},
{in: `100foo`, exp: expNum(1, 1, 100), leftover: "foo"},
})
expName := func(row, col int, name string) Value {
return Value{Name: &name, Location: Location{row, col}}
}
expGraph := func(row, col int, g *Graph) Value {
return Value{Graph: g, Location: Location{row, col}}
}
runTests(t, "name", nameTerm, []test{
{in: `a`, exp: expName(1, 1, "a")},
{in: `ab`, exp: expName(1, 1, "ab")},
{in: `ab2c`, exp: expName(1, 1, "ab2c")},
{in: `ab2c,`, exp: expName(1, 1, "ab2c"), leftover: ","},
})
runTests(t, "graph", graphTerm, []test{
{in: `{}`, exp: expGraph(1, 1, new(Graph))},
{in: `{`, expErr: `1:2: expected '}' or name`},
{in: `{a}`, expErr: `1:3: expected '='`},
{in: `{a=}`, expErr: `1:4: expected name or number or graph or tuple`},
{
in: `{foo=a}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(None, expName(6, 1, "a")),
),
),
},
{
in: `{ foo = a }`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(None, expName(6, 1, "a")),
),
),
},
{in: `{1=a}`, expErr: `1:2: expected '}' or name`},
{in: `{foo=a ,}`, expErr: `1:8: expected '}' or ';' or '<'`},
{in: `{foo=a`, expErr: `1:7: expected '}' or ';' or '<'`},
{
in: `{foo=a<b}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expName(8, 1, "b"),
),
),
),
},
{
in: `{foo=a< b <c}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.ValueOut(
Some(expName(8, 1, "b")),
expName(10, 1, "c"),
),
),
),
),
},
{
in: `{foo =a<b<c<1 }`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.TupleOut(
Some(expName(8, 1, "b")),
graph.ValueOut(
Some(expName(10, 1, "c")),
expNum(12, 1, 1),
),
),
),
),
),
},
{
in: `{foo=a<b ; }`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expName(8, 1, "b"),
),
),
),
},
{
in: `{foo=a<b;bar=c}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expName(8, 1, "b"),
),
).
AddValueIn(
expName(10, 1, "bar"),
graph.ValueOut(None, expName(15, 1, "c")),
),
),
},
{
in: `{foo= a<{ baz=1 } ; bar=c}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expGraph(8, 1, new(Graph).AddValueIn(
expName(9, 1, "baz"),
graph.ValueOut(None, expNum(13, 1, 1)),
)),
),
).
AddValueIn(
expName(16, 1, "bar"),
graph.ValueOut(None, expName(20, 1, "c")),
),
),
},
{
in: `{foo= {baz=1} <a; bar=c}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expGraph(8, 1, new(Graph).AddValueIn(
expName(9, 1, "baz"),
graph.ValueOut(None, expNum(13, 1, 1)),
))),
expName(6, 1, "a"),
),
).
AddValueIn(
expName(16, 1, "bar"),
graph.ValueOut(None, expName(20, 1, "c")),
),
),
},
})
runTests(t, "tuple", graphTerm, []test{
{
in: `{foo=(a)}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(None, expName(6, 1, "a")),
),
),
},
{
in: `{foo=(a<b)}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expName(8, 1, "b"),
),
),
),
},
{
in: `{foo=a<(b)}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(
Some(expName(6, 1, "a")),
expName(8, 1, "b"),
),
),
),
},
{
in: `{foo=a<(b,c)}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.ValueOut(None, expName(8, 1, "b")),
graph.ValueOut(None, expName(10, 1, "c")),
),
),
),
},
{
in: `{foo=a<(b<c)}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.TupleOut(
Some(expName(6, 1, "b")),
graph.ValueOut(None, expName(8, 1, "c")),
),
),
),
),
},
{
in: `{foo=a<(b<(c))}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.TupleOut(
Some(expName(6, 1, "b")),
graph.ValueOut(None, expName(8, 1, "c")),
),
),
),
),
},
{
in: `{foo=a<(b<(c,d<1))}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.TupleOut(
Some(expName(6, 1, "b")),
graph.ValueOut(None, expName(8, 1, "c")),
graph.ValueOut(
Some(expName(12, 1, "d")),
expNum(10, 1, 1),
),
),
),
),
),
},
{
in: `{foo=a<(b<( ( (c) ) ))}`,
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.TupleOut(
Some(expName(6, 1, "a")),
graph.TupleOut(
Some(expName(6, 1, "b")),
graph.ValueOut(None, expName(8, 1, "c")),
),
),
),
),
},
})
runTests(t, "comment", graphTerm, []test{
{
in: "*\n{}",
exp: expGraph(1, 1, new(Graph)),
},
{
in: "* ignore me!\n{}",
exp: expGraph(1, 1, new(Graph)),
},
{
in: "{* ignore me!\n}",
exp: expGraph(1, 1, new(Graph)),
},
{
in: "{foo* ignore me!\n = a}",
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(None, expName(6, 1, "a")),
),
),
},
{
in: "{foo = a* ignore me!\n}",
exp: expGraph(
1, 1, new(Graph).
AddValueIn(
expName(2, 1, "foo"),
graph.ValueOut(None, expName(6, 1, "a")),
),
),
},
})
}

View File

@ -1,23 +0,0 @@
package gg
import "io"
type mockReader struct {
body []byte
err error
}
func (r *mockReader) Read(b []byte) (int, error) {
n := copy(b, r.body)
r.body = r.body[n:]
if len(r.body) == 0 {
if r.err == nil {
return n, io.EOF
}
return n, r.err
}
return n, nil
}

1
go.mod
View File

@ -7,5 +7,6 @@ require github.com/stretchr/testify v1.7.0
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
)

2
go.sum
View File

@ -5,6 +5,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

View File

@ -126,7 +126,7 @@ func TupleOut[E, V Value](edgeVal E, ins ...*OpenEdge[E, V]) *OpenEdge[E, V] {
if len(ins) == 1 {
var (
zero V
zero E
in = ins[0]
)
@ -353,7 +353,6 @@ type reducedEdge[Ea, Va Value, Vb any] struct {
// If a value or edge is connected to multiple times within the root OpenEdge it
// will only be mapped/reduced a single time, and the result of that single
// map/reduction will be passed to each dependant operation.
//
func MapReduce[Ea, Va Value, Vb any](
root *OpenEdge[Ea, Va],
mapVal func(Va) (Vb, error),

233
vm/function.go Normal file
View File

@ -0,0 +1,233 @@
package vm
import (
"fmt"
"github.com/mediocregopher/ginger/gg"
"github.com/mediocregopher/ginger/graph"
)
// Function is an entity which accepts an argument Value and performs some
// internal processing on that argument to return a resultant Value.
type Function interface {
Perform(Value) Value
}
// FunctionFunc is a function which implements the Function interface.
type FunctionFunc func(Value) Value
// Perform calls the underlying FunctionFunc directly.
func (f FunctionFunc) Perform(arg Value) Value {
return f(arg)
}
// Identity returns an Function which always returns the given Value,
// regardless of the input argument.
//
// TODO this might not be the right name
func Identity(val Value) Function {
return FunctionFunc(func(Value) Value {
return val
})
}
var (
valNameIn = Value{Value: gg.Name("in")}
valNameOut = Value{Value: gg.Name("out")}
valNameIf = Value{Value: gg.Name("if")}
valNameRecur = Value{Value: gg.Name("recur")}
valNumberZero = Value{Value: gg.Number(0)}
)
// FunctionFromGraph wraps the given Graph such that it can be used as an
// Function. The given Scope determines what values outside of the Graph are
// available for use within the Function.
func FunctionFromGraph(g *gg.Graph, scope Scope) (Function, error) {
// edgeFn is distinct from a generic Function in that the Value passed into
// Perform will _always_ be the value of "in" for the overall Function.
//
// edgeFns will wrap each other, passing "in" downwards to the leaf edgeFns.
type edgeFn Function
var compileEdge func(*gg.OpenEdge) (edgeFn, error)
// TODO memoize?
valToEdgeFn := func(val Value) (edgeFn, error) {
if val.Name == nil {
return edgeFn(Identity(val)), nil
}
name := *val.Name
if val.Equal(valNameIn) {
return edgeFn(FunctionFunc(func(inArg Value) Value {
return inArg
})), nil
}
// TODO intercept if and recur?
edgesIn := g.ValueIns(val.Value)
if l := len(edgesIn); l == 0 {
val, err := scope.Resolve(name)
if err != nil {
return nil, fmt.Errorf("resolving name %q from the outer scope: %w", name, err)
}
return edgeFn(Identity(val)), nil
} else if l != 1 {
return nil, fmt.Errorf("resolved name %q to %d input edges, rather than one", name, l)
}
edge := edgesIn[0]
return compileEdge(edge)
}
// "out" resolves to more than a static value, treat the graph as a full
// operation.
// thisFn is used to support recur. It will get filled in with the Function
// which is returned by this function, once that Function is created.
thisFn := new(Function)
compileEdge = func(edge *gg.OpenEdge) (edgeFn, error) {
return graph.MapReduce[gg.OptionalValue, gg.Value, edgeFn](
edge,
func(ggVal gg.Value) (edgeFn, error) {
return valToEdgeFn(Value{Value: ggVal})
},
func(ggEdgeVal gg.OptionalValue, inEdgeFns []edgeFn) (edgeFn, error) {
if ggEdgeVal.Equal(valNameIf.Value) {
if len(inEdgeFns) != 3 {
return nil, fmt.Errorf("'if' requires a 3-tuple argument")
}
return edgeFn(FunctionFunc(func(inArg Value) Value {
if pred := inEdgeFns[0].Perform(inArg); pred.Equal(valNumberZero) {
return inEdgeFns[2].Perform(inArg)
}
return inEdgeFns[1].Perform(inArg)
})), nil
}
// "if" statements (above) are the only case where we want the
// input edges to remain separated, otherwise they should always
// be combined into a single edge whose value is a tuple. Do
// that here.
inEdgeFn := inEdgeFns[0]
if len(inEdgeFns) > 1 {
inEdgeFn = edgeFn(FunctionFunc(func(inArg Value) Value {
tupVals := make([]Value, len(inEdgeFns))
for i := range inEdgeFns {
tupVals[i] = inEdgeFns[i].Perform(inArg)
}
return Tuple(tupVals...)
}))
}
var edgeVal Value
if ggEdgeVal.Valid {
edgeVal.Value = ggEdgeVal.Value
}
if edgeVal.IsZero() {
return inEdgeFn, nil
}
if edgeVal.Equal(valNameRecur) {
return edgeFn(FunctionFunc(func(inArg Value) Value {
return (*thisFn).Perform(inEdgeFn.Perform(inArg))
})), nil
}
if edgeVal.Graph != nil {
opFromGraph, err := FunctionFromGraph(
edgeVal.Graph,
scope.NewScope(),
)
if err != nil {
return nil, fmt.Errorf("compiling graph to operation: %w", err)
}
edgeVal = Value{Function: opFromGraph}
}
// The Function is known at compile-time, so we can wrap it
// directly into an edgeVal using the existing inEdgeFn as the
// input.
if edgeVal.Function != nil {
return edgeFn(FunctionFunc(func(inArg Value) Value {
return edgeVal.Function.Perform(inEdgeFn.Perform(inArg))
})), nil
}
// the edgeVal is not an Function at compile time, and so
// it must become one at runtime. We must resolve edgeVal to an
// edgeFn as well (edgeEdgeFn), and then at runtime that is
// given the inArg and (hopefully) the resultant Function is
// called.
edgeEdgeFn, err := valToEdgeFn(edgeVal)
if err != nil {
return nil, err
}
return edgeFn(FunctionFunc(func(inArg Value) Value {
runtimeEdgeVal := edgeEdgeFn.Perform(inArg)
if runtimeEdgeVal.Graph != nil {
runtimeFn, err := FunctionFromGraph(
runtimeEdgeVal.Graph,
scope.NewScope(),
)
if err != nil {
panic(fmt.Sprintf("compiling graph to operation: %v", err))
}
runtimeEdgeVal = Value{Function: runtimeFn}
}
if runtimeEdgeVal.Function == nil {
panic("edge value must be an operation")
}
return runtimeEdgeVal.Function.Perform(inEdgeFn.Perform(inArg))
})), nil
},
)
}
graphFn, err := valToEdgeFn(valNameOut)
if err != nil {
return nil, err
}
*thisFn = Function(graphFn)
return Function(graphFn), nil
}

235
vm/op.go
View File

@ -1,235 +0,0 @@
package vm
import (
"fmt"
"github.com/mediocregopher/ginger/gg"
"github.com/mediocregopher/ginger/graph"
)
// Operation is an entity which accepts an argument Value and performs some
// internal processing on that argument to return a resultant Value.
type Operation interface {
Perform(Value) Value
}
// OperationFunc is a function which implements the Operation interface.
type OperationFunc func(Value) Value
// Perform calls the underlying OperationFunc directly.
func (f OperationFunc) Perform(arg Value) Value {
return f(arg)
}
// Identity returns an Operation which always returns the given Value,
// regardless of the input argument.
//
// TODO this might not be the right name
func Identity(val Value) Operation {
return OperationFunc(func(Value) Value {
return val
})
}
type graphOp struct {
*gg.Graph
scope Scope
}
var (
valNameIn = Value{Value: gg.Name("in")}
valNameOut = Value{Value: gg.Name("out")}
valNameIf = Value{Value: gg.Name("if")}
valNameRecur = Value{Value: gg.Name("recur")}
valNumberZero = Value{Value: gg.Number(0)}
)
// OperationFromGraph wraps the given Graph such that it can be used as an
// Operation. The given Scope determines what values outside of the Graph are
// available for use within the Operation.
func OperationFromGraph(g *gg.Graph, scope Scope) (Operation, error) {
// edgeOp is distinct from a generic Operation in that the Value passed into
// Perform will _always_ be the value of "in" for the overall Operation.
//
// edgeOps will wrap each other, passing "in" downwards to the leaf edgeOps.
type edgeOp Operation
var compileEdge func(*gg.OpenEdge) (edgeOp, error)
// TODO memoize?
valToEdgeOp := func(val Value) (edgeOp, error) {
if val.Name == nil {
return edgeOp(Identity(val)), nil
}
name := *val.Name
if val.Equal(valNameIn) {
return edgeOp(OperationFunc(func(inArg Value) Value {
return inArg
})), nil
}
// TODO intercept if and recur?
edgesIn := g.ValueIns(val.Value)
if l := len(edgesIn); l == 0 {
val, err := scope.Resolve(name)
if err != nil {
return nil, fmt.Errorf("resolving name %q from the outer scope: %w", name, err)
}
return edgeOp(Identity(val)), nil
} else if l != 1 {
return nil, fmt.Errorf("resolved name %q to %d input edges, rather than one", name, l)
}
edge := edgesIn[0]
return compileEdge(edge)
}
// "out" resolves to more than a static value, treat the graph as a full
// operation.
// thisOp is used to support recur. It will get filled in with the Operation
// which is returned by this function, once that Operation is created.
thisOp := new(Operation)
compileEdge = func(edge *gg.OpenEdge) (edgeOp, error) {
return graph.MapReduce[gg.Value, gg.Value, edgeOp](
edge,
func(ggVal gg.Value) (edgeOp, error) {
return valToEdgeOp(Value{Value: ggVal})
},
func(ggEdgeVal gg.Value, inEdgeOps []edgeOp) (edgeOp, error) {
if ggEdgeVal.Equal(valNameIf.Value) {
if len(inEdgeOps) != 3 {
return nil, fmt.Errorf("'if' requires a 3-tuple argument")
}
return edgeOp(OperationFunc(func(inArg Value) Value {
if pred := inEdgeOps[0].Perform(inArg); pred.Equal(valNumberZero) {
return inEdgeOps[2].Perform(inArg)
}
return inEdgeOps[1].Perform(inArg)
})), nil
}
// "if" statements (above) are the only case where we want the
// input edges to remain separated, otherwise they should always
// be combined into a single edge whose value is a tuple. Do
// that here.
inEdgeOp := inEdgeOps[0]
if len(inEdgeOps) > 1 {
inEdgeOp = edgeOp(OperationFunc(func(inArg Value) Value {
tupVals := make([]Value, len(inEdgeOps))
for i := range inEdgeOps {
tupVals[i] = inEdgeOps[i].Perform(inArg)
}
return Tuple(tupVals...)
}))
}
edgeVal := Value{Value: ggEdgeVal}
if edgeVal.IsZero() {
return inEdgeOp, nil
}
if edgeVal.Equal(valNameRecur) {
return edgeOp(OperationFunc(func(inArg Value) Value {
return (*thisOp).Perform(inEdgeOp.Perform(inArg))
})), nil
}
if edgeVal.Graph != nil {
opFromGraph, err := OperationFromGraph(
edgeVal.Graph,
scope.NewScope(),
)
if err != nil {
return nil, fmt.Errorf("compiling graph to operation: %w", err)
}
edgeVal = Value{Operation: opFromGraph}
}
// The Operation is known at compile-time, so we can wrap it
// directly into an edgeVal using the existing inEdgeOp as the
// input.
if edgeVal.Operation != nil {
return edgeOp(OperationFunc(func(inArg Value) Value {
return edgeVal.Operation.Perform(inEdgeOp.Perform(inArg))
})), nil
}
// the edgeVal is not an Operation at compile time, and so
// it must become one at runtime. We must resolve edgeVal to an
// edgeOp as well (edgeEdgeOp), and then at runtime that is
// given the inArg and (hopefully) the resultant Operation is
// called.
edgeEdgeOp, err := valToEdgeOp(edgeVal)
if err != nil {
return nil, err
}
return edgeOp(OperationFunc(func(inArg Value) Value {
runtimeEdgeVal := edgeEdgeOp.Perform(inArg)
if runtimeEdgeVal.Graph != nil {
runtimeOp, err := OperationFromGraph(
runtimeEdgeVal.Graph,
scope.NewScope(),
)
if err != nil {
panic(fmt.Sprintf("compiling graph to operation: %v", err))
}
runtimeEdgeVal = Value{Operation: runtimeOp}
}
if runtimeEdgeVal.Operation == nil {
panic("edge value must be an operation")
}
return runtimeEdgeVal.Operation.Perform(inEdgeOp.Perform(inArg))
})), nil
},
)
}
graphOp, err := valToEdgeOp(valNameOut)
if err != nil {
return nil, err
}
*thisOp = Operation(graphOp)
return Operation(graphOp), nil
}

View File

@ -2,8 +2,6 @@ package vm
import (
"fmt"
"github.com/mediocregopher/ginger/gg"
)
// Scope encapsulates a set of name->Value mappings.
@ -62,66 +60,3 @@ func (s *scopeWith) Resolve(name string) (Value, error) {
}
return s.Scope.Resolve(name)
}
type graphScope struct {
*gg.Graph
parent Scope
}
/*
TODO I don't think this is actually necessary
// ScopeFromGraph returns a Scope which will use the given Graph for name
// resolution.
//
// When a name is resolved, that name will be looked up in the Graph. The name's
// vertex must have only a single OpenEdge leading to it. That edge will be
// compiled into an Operation and returned.
//
// If a name does not appear in the Graph, then the given parent Scope will be
// used to resolve that name. If the parent Scope is nil then an error is
// returned.
//
// NewScope will return the parent scope, if one is given, or an empty ScopeMap
// if not.
func ScopeFromGraph(g *gg.Graph, parent Scope) Scope {
return &graphScope{
Graph: g,
parent: parent,
}
}
func (g *graphScope) Resolve(name string) (Value, error) {
var ggNameVal gg.Value
ggNameVal.Name = &name
log.Printf("resolving %q", name)
edgesIn := g.ValueIns(ggNameVal)
if l := len(edgesIn); l == 0 && g.parent != nil {
return g.parent.Resolve(name)
} else if l != 1 {
return nil, fmt.Errorf(
"%q must have exactly one input edge, found %d input edges",
name, l,
)
}
return CompileEdge(edgesIn[0], g)
}
func (g *graphScope) NewScope() Scope {
if g.parent == nil {
return ScopeMap{}
}
return g.parent
}
*/

View File

@ -6,9 +6,9 @@ import (
"github.com/mediocregopher/ginger/gg"
)
func globalOp(fn func(Value) (Value, error)) Value {
func globalFn(fn func(Value) (Value, error)) Value {
return Value{
Operation: OperationFunc(func(in Value) Value {
Function: FunctionFunc(func(in Value) Value {
res, err := fn(in)
if err != nil {
panic(err)
@ -22,7 +22,7 @@ func globalOp(fn func(Value) (Value, error)) Value {
// any operation in a ginger program.
var GlobalScope = ScopeMap{
"add": globalOp(func(val Value) (Value, error) {
"add": globalFn(func(val Value) (Value, error) {
var sum int64
@ -39,7 +39,7 @@ var GlobalScope = ScopeMap{
}),
"tupEl": globalOp(func(val Value) (Value, error) {
"tupEl": globalFn(func(val Value) (Value, error) {
tup, i := val.Tuple[0], val.Tuple[1]
@ -47,7 +47,7 @@ var GlobalScope = ScopeMap{
}),
"isZero": globalOp(func(val Value) (Value, error) {
"isZero": globalFn(func(val Value) (Value, error) {
if *val.Number == 0 {
one := int64(1)

View File

@ -2,6 +2,7 @@
package vm
import (
"errors"
"fmt"
"io"
"strings"
@ -13,12 +14,12 @@ import (
// ZeroValue is a Value with no fields set. It is equivalent to the 0-tuple.
var ZeroValue Value
// Value extends a gg.Value to include Operations and Tuples as a possible
// Value extends a gg.Value to include Functions and Tuples as a possible
// types.
type Value struct {
gg.Value
Operation
Function
Tuple []Value
}
@ -44,11 +45,11 @@ func (v Value) Equal(v2g graph.Value) bool {
switch {
case !v.Value.IsZero() || !v2.Value.IsZero():
case (v.Value != (gg.Value{}) || v2.Value != (gg.Value{})):
return v.Value.Equal(v2.Value)
case v.Operation != nil || v2.Operation != nil:
// for now we say that Operations can't be compared. This will probably
case v.Function != nil || v2.Function != nil:
// for now we say that Functions can't be compared. This will probably
// get revisted later.
return false
@ -76,12 +77,12 @@ func (v Value) String() string {
switch {
case v.Operation != nil:
case v.Function != nil:
// We can try to get better strings for ops later
return "<op>"
return "<fn>"
case !v.Value.IsZero():
case v.Value != (gg.Value{}):
return v.Value.String()
default:
@ -100,30 +101,23 @@ func (v Value) String() string {
}
func nameVal(n string) Value {
var val Value
val.Name = &n
return val
}
// EvaluateSource reads and parses the io.Reader as an operation, input is used
// as the argument to the operation, and the resultant value is returned.
//
// scope contains pre-defined operations and values which are available during
// the evaluation.
func EvaluateSource(opSrc io.Reader, input Value, scope Scope) (Value, error) {
lexer := gg.NewLexer(opSrc)
v, err := gg.NewDecoder(opSrc).Next()
if err != nil {
return Value{}, err
} else if v.Graph == nil {
return Value{}, errors.New("value must be a graph")
}
g, err := gg.DecodeLexer(lexer)
fn, err := FunctionFromGraph(v.Graph, scope.NewScope())
if err != nil {
return Value{}, err
}
op, err := OperationFromGraph(g, scope.NewScope())
if err != nil {
return Value{}, err
}
return op.Perform(input), nil
return fn.Perform(input), nil
}

View File

@ -10,11 +10,11 @@ import (
func TestVM(t *testing.T) {
src := `
incr = { out = add < (1; in;); };
src := `{
incr = { out = add < (1, in); };
out = incr < incr < in;
`
}`
var in int64 = 5