encompass all opening and closing tokens with a single type, and implement them

This commit is contained in:
Brian Picciano 2014-10-18 17:24:42 -04:00
parent c589cb8f08
commit 99b67fa801
2 changed files with 20 additions and 12 deletions

View File

@ -16,8 +16,8 @@ type TokenType int
const ( const (
BareString TokenType = iota BareString TokenType = iota
QuotedString QuotedString
OpenParen Open
CloseParen Close
Err Err
eof eof
) )
@ -162,9 +162,17 @@ func lexWhitespace(l *Lexer) lexerFunc {
case '"': case '"':
return lexQuotedString return lexQuotedString
case '(': case '(':
l.emit(OpenParen) l.emit(Open)
case ')': case ')':
l.emit(CloseParen) l.emit(Close)
case '[':
l.emit(Open)
case ']':
l.emit(Close)
case '{':
l.emit(Open)
case '}':
l.emit(Close)
default: default:
return lexBareString return lexBareString
} }

View File

@ -19,21 +19,21 @@ func TestLexer(t *T) {
"\"foo\nbar\" baz": {{QuotedString, "\"foo\nbar\""}, "\"foo\nbar\" baz": {{QuotedString, "\"foo\nbar\""},
{BareString, "baz"}, {BareString, "baz"},
{eof, ""}}, {eof, ""}},
"( foo bar ) baz": {{OpenParen, "("}, "( foo bar ) baz": {{Open, "("},
{BareString, "foo"}, {BareString, "foo"},
{BareString, "bar"}, {BareString, "bar"},
{CloseParen, ")"}, {Close, ")"},
{BareString, "baz"}, {BareString, "baz"},
{eof, ""}}, {eof, ""}},
"((foo-bar))": {{OpenParen, "("}, "((foo-bar))": {{Open, "("},
{OpenParen, "("}, {Open, "("},
{BareString, "foo-bar"}, {BareString, "foo-bar"},
{CloseParen, ")"}, {Close, ")"},
{CloseParen, ")"}, {Close, ")"},
{eof, ""}}, {eof, ""}},
"(\"foo\nbar\")": {{OpenParen, "("}, "(\"foo\nbar\")": {{Open, "("},
{QuotedString, "\"foo\nbar\""}, {QuotedString, "\"foo\nbar\""},
{CloseParen, ")"}, {Close, ")"},
{eof, ""}}, {eof, ""}},
} }