encompass all opening and closing tokens with a single type, and implement them

This commit is contained in:
Brian Picciano 2014-10-18 17:24:42 -04:00
parent c589cb8f08
commit 99b67fa801
2 changed files with 20 additions and 12 deletions

View File

@ -16,8 +16,8 @@ type TokenType int
const (
BareString TokenType = iota
QuotedString
OpenParen
CloseParen
Open
Close
Err
eof
)
@ -162,9 +162,17 @@ func lexWhitespace(l *Lexer) lexerFunc {
case '"':
return lexQuotedString
case '(':
l.emit(OpenParen)
l.emit(Open)
case ')':
l.emit(CloseParen)
l.emit(Close)
case '[':
l.emit(Open)
case ']':
l.emit(Close)
case '{':
l.emit(Open)
case '}':
l.emit(Close)
default:
return lexBareString
}

View File

@ -19,21 +19,21 @@ func TestLexer(t *T) {
"\"foo\nbar\" baz": {{QuotedString, "\"foo\nbar\""},
{BareString, "baz"},
{eof, ""}},
"( foo bar ) baz": {{OpenParen, "("},
"( foo bar ) baz": {{Open, "("},
{BareString, "foo"},
{BareString, "bar"},
{CloseParen, ")"},
{Close, ")"},
{BareString, "baz"},
{eof, ""}},
"((foo-bar))": {{OpenParen, "("},
{OpenParen, "("},
"((foo-bar))": {{Open, "("},
{Open, "("},
{BareString, "foo-bar"},
{CloseParen, ")"},
{CloseParen, ")"},
{Close, ")"},
{Close, ")"},
{eof, ""}},
"(\"foo\nbar\")": {{OpenParen, "("},
"(\"foo\nbar\")": {{Open, "("},
{QuotedString, "\"foo\nbar\""},
{CloseParen, ")"},
{Close, ")"},
{eof, ""}},
}