Don't treat newlines specially

For MVP newlines aren't going to be used as a syntax terminator, they're
just going to be whitespace. Otherwise the decoding logic gets way more
complicated.
rust
Brian Picciano 2 years ago
parent 1e30ad6959
commit e2ffc37ddc
  1. 15
      gg/lexer.go
  2. 50
      gg/lexer_test.go

@ -201,20 +201,7 @@ func (l *lexer) next() (LexerToken, *LexerError) {
return LexerToken{}, err
}
// terminating newline is deliberately not discarded. Loop and find
// the next token (which will be that newline).
case r == '\n':
// newlines are considered punctuation, not whitespace
l.discardRune()
return LexerToken{
Kind: LexerTokenKindPunctuation,
Value: string(r),
Row: l.lastRow,
Col: l.lastCol,
}, nil
// terminating newline will be discarded on next loop
case r == '"' || r == '`':

@ -35,47 +35,7 @@ func TestLexer(t *testing.T) {
exp []LexerToken
}{
{in: "", exp: []LexerToken{}},
{in: "* fooo", exp: []LexerToken{}},
{
in: "*\n",
exp: []LexerToken{
{
Kind: LexerTokenKindPunctuation,
Value: "\n",
Row: 0, Col: 1,
},
},
},
{
in: "foo\nbar\n\n",
exp: []LexerToken{
{
Kind: LexerTokenKindName,
Value: "foo",
Row: 0, Col: 0,
},
{
Kind: LexerTokenKindPunctuation,
Value: "\n",
Row: 0, Col: 3,
},
{
Kind: LexerTokenKindName,
Value: "bar",
Row: 1, Col: 0,
},
{
Kind: LexerTokenKindPunctuation,
Value: "\n",
Row: 1, Col: 3,
},
{
Kind: LexerTokenKindPunctuation,
Value: "\n",
Row: 2, Col: 0,
},
},
},
{in: "* fooo\n", exp: []LexerToken{}},
{
in: "foo",
exp: []LexerToken{
@ -87,7 +47,7 @@ func TestLexer(t *testing.T) {
},
},
{
in: "foo bar f-o f0O Foo",
in: "foo bar\nf-o f0O Foo",
exp: []LexerToken{
{
Kind: LexerTokenKindName,
@ -102,17 +62,17 @@ func TestLexer(t *testing.T) {
{
Kind: LexerTokenKindName,
Value: "f-o",
Row: 0, Col: 8,
Row: 1, Col: 0,
},
{
Kind: LexerTokenKindName,
Value: "f0O",
Row: 0, Col: 12,
Row: 1, Col: 4,
},
{
Kind: LexerTokenKindName,
Value: "Foo",
Row: 0, Col: 16,
Row: 1, Col: 8,
},
},
},

Loading…
Cancel
Save