Don't treat newlines specially
For MVP newlines aren't going to be used as a syntax terminator, they're just going to be whitespace. Otherwise the decoding logic gets way more complicated.
This commit is contained in:
parent
1e30ad6959
commit
e2ffc37ddc
15
gg/lexer.go
15
gg/lexer.go
@ -201,20 +201,7 @@ func (l *lexer) next() (LexerToken, *LexerError) {
|
||||
return LexerToken{}, err
|
||||
}
|
||||
|
||||
// terminating newline is deliberately not discarded. Loop and find
|
||||
// the next token (which will be that newline).
|
||||
|
||||
case r == '\n':
|
||||
// newlines are considered punctuation, not whitespace
|
||||
|
||||
l.discardRune()
|
||||
|
||||
return LexerToken{
|
||||
Kind: LexerTokenKindPunctuation,
|
||||
Value: string(r),
|
||||
Row: l.lastRow,
|
||||
Col: l.lastCol,
|
||||
}, nil
|
||||
// terminating newline will be discarded on next loop
|
||||
|
||||
case r == '"' || r == '`':
|
||||
|
||||
|
@ -35,47 +35,7 @@ func TestLexer(t *testing.T) {
|
||||
exp []LexerToken
|
||||
}{
|
||||
{in: "", exp: []LexerToken{}},
|
||||
{in: "* fooo", exp: []LexerToken{}},
|
||||
{
|
||||
in: "*\n",
|
||||
exp: []LexerToken{
|
||||
{
|
||||
Kind: LexerTokenKindPunctuation,
|
||||
Value: "\n",
|
||||
Row: 0, Col: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
in: "foo\nbar\n\n",
|
||||
exp: []LexerToken{
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
Value: "foo",
|
||||
Row: 0, Col: 0,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindPunctuation,
|
||||
Value: "\n",
|
||||
Row: 0, Col: 3,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
Value: "bar",
|
||||
Row: 1, Col: 0,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindPunctuation,
|
||||
Value: "\n",
|
||||
Row: 1, Col: 3,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindPunctuation,
|
||||
Value: "\n",
|
||||
Row: 2, Col: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
{in: "* fooo\n", exp: []LexerToken{}},
|
||||
{
|
||||
in: "foo",
|
||||
exp: []LexerToken{
|
||||
@ -87,7 +47,7 @@ func TestLexer(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
in: "foo bar f-o f0O Foo",
|
||||
in: "foo bar\nf-o f0O Foo",
|
||||
exp: []LexerToken{
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
@ -102,17 +62,17 @@ func TestLexer(t *testing.T) {
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
Value: "f-o",
|
||||
Row: 0, Col: 8,
|
||||
Row: 1, Col: 0,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
Value: "f0O",
|
||||
Row: 0, Col: 12,
|
||||
Row: 1, Col: 4,
|
||||
},
|
||||
{
|
||||
Kind: LexerTokenKindName,
|
||||
Value: "Foo",
|
||||
Row: 0, Col: 16,
|
||||
Row: 1, Col: 8,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
Loading…
Reference in New Issue
Block a user