feat: improve debug output of tokens

8 files changed, 182 insertions, 68 deletions

diff --git a/lang/token.go b/lang/token.go
index 5205efc..613f2c6 100644
--- a/lang/token.go
+++ b/lang/token.go
@@ -1,9 +1,11 @@
 package lang
 
+//go:generate stringer -type=TokenId
+
 type TokenId int
 
 const (
-	Illegal = iota
+	Illegal TokenId = iota
 	Comment
 	Ident
 
diff --git a/lang/tokenid_string.go b/lang/tokenid_string.go
new file mode 100644
index 0000000..705edc6
--- /dev/null
+++ b/lang/tokenid_string.go
@@ -0,0 +1,115 @@
+// Code generated by "stringer -type=TokenId"; DO NOT EDIT.
+
+package lang
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Illegal-0]
+	_ = x[Comment-1]
+	_ = x[Ident-2]
+	_ = x[Char-3]
+	_ = x[Float-4]
+	_ = x[Imag-5]
+	_ = x[Int-6]
+	_ = x[String-7]
+	_ = x[Add-8]
+	_ = x[Sub-9]
+	_ = x[Mul-10]
+	_ = x[Quo-11]
+	_ = x[Rem-12]
+	_ = x[And-13]
+	_ = x[Or-14]
+	_ = x[Xor-15]
+	_ = x[Shl-16]
+	_ = x[Shr-17]
+	_ = x[AndNot-18]
+	_ = x[Period-19]
+	_ = x[Equal-20]
+	_ = x[Greater-21]
+	_ = x[GreaterEqual-22]
+	_ = x[Land-23]
+	_ = x[Less-24]
+	_ = x[LessEqual-25]
+	_ = x[Lor-26]
+	_ = x[NotEqual-27]
+	_ = x[Define-28]
+	_ = x[Assign-29]
+	_ = x[AddAssign-30]
+	_ = x[SubAssign-31]
+	_ = x[MulAssign-32]
+	_ = x[QuoAssign-33]
+	_ = x[RemAssign-34]
+	_ = x[AndAssign-35]
+	_ = x[OrAssign-36]
+	_ = x[XorAssign-37]
+	_ = x[ShlAssign-38]
+	_ = x[ShrAssign-39]
+	_ = x[AndNotAssign-40]
+	_ = x[Inc-41]
+	_ = x[Dec-42]
+	_ = x[Plus-43]
+	_ = x[Minus-44]
+	_ = x[Addr-45]
+	_ = x[Deref-46]
+	_ = x[BitComp-47]
+	_ = x[Arrow-48]
+	_ = x[Ellipsis-49]
+	_ = x[Not-50]
+	_ = x[Tilde-51]
+	_ = x[Comma-52]
+	_ = x[Semicolon-53]
+	_ = x[Colon-54]
+	_ = x[ParenBlock-55]
+	_ = x[BracketBlock-56]
+	_ = x[BraceBlock-57]
+	_ = x[Break-58]
+	_ = x[Case-59]
+	_ = x[Chan-60]
+	_ = x[Const-61]
+	_ = x[Continue-62]
+	_ = x[Default-63]
+	_ = x[Defer-64]
+	_ = x[Else-65]
+	_ = x[Fallthrough-66]
+	_ = x[For-67]
+	_ = x[Func-68]
+	_ = x[Go-69]
+	_ = x[Goto-70]
+	_ = x[If-71]
+	_ = x[Import-72]
+	_ = x[Interface-73]
+	_ = x[Map-74]
+	_ = x[Package-75]
+	_ = x[Range-76]
+	_ = x[Return-77]
+	_ = x[Select-78]
+	_ = x[Struct-79]
+	_ = x[Switch-80]
+	_ = x[Type-81]
+	_ = x[Var-82]
+	_ = x[Call-83]
+	_ = x[CallX-84]
+	_ = x[EqualSet-85]
+	_ = x[Grow-86]
+	_ = x[Index-87]
+	_ = x[JumpFalse-88]
+	_ = x[JumpSetFalse-89]
+	_ = x[JumpSetTrue-90]
+	_ = x[Label-91]
+	_ = x[New-92]
+}
+
+const _TokenId_name = "IllegalCommentIdentCharFloatImagIntStringAddSubMulQuoRemAndOrXorShlShrAndNotPeriodEqualGreaterGreaterEqualLandLessLessEqualLorNotEqualDefineAssignAddAssignSubAssignMulAssignQuoAssignRemAssignAndAssignOrAssignXorAssignShlAssignShrAssignAndNotAssignIncDecPlusMinusAddrDerefBitCompArrowEllipsisNotTildeCommaSemicolonColonParenBlockBracketBlockBraceBlockBreakCaseChanConstContinueDefaultDeferElseFallthroughForFuncGoGotoIfImportInterfaceMapPackageRangeReturnSelectStructSwitchTypeVarCallCallXEqualSetGrowIndexJumpFalseJumpSetFalseJumpSetTrueLabelNew"
+
+var _TokenId_index = [...]uint16{0, 7, 14, 19, 23, 28, 32, 35, 41, 44, 47, 50, 53, 56, 59, 61, 64, 67, 70, 76, 82, 87, 94, 106, 110, 114, 123, 126, 134, 140, 146, 155, 164, 173, 182, 191, 200, 208, 217, 226, 235, 247, 250, 253, 257, 262, 266, 271, 278, 283, 291, 294, 299, 304, 313, 318, 328, 340, 350, 355, 359, 363, 368, 376, 383, 388, 392, 403, 406, 410, 412, 416, 418, 424, 433, 436, 443, 448, 454, 460, 466, 472, 476, 479, 483, 488, 496, 500, 505, 514, 526, 537, 542, 545}
+
+func (i TokenId) String() string {
+	if i < 0 || i >= TokenId(len(_TokenId_index)-1) {
+		return "TokenId(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _TokenId_name[_TokenId_index[i]:_TokenId_index[i+1]]
+}
diff --git a/parser/compiler.go b/parser/compiler.go
index 886da9b..4594ab9 100644
--- a/parser/compiler.go
+++ b/parser/compiler.go
@@ -212,9 +212,8 @@ func (c *Compiler) Codegen(tokens Tokens) (err error) {
 			}
 
 		case lang.JumpFalse:
-			label := t.Str[10:]
 			var i int64
-			if s, ok := c.symbols[label]; !ok {
+			if s, ok := c.symbols[t.Str]; !ok {
 				// t.Beg contains the position in code which needs to be fixed.
 				t.Beg = len(c.Code)
 				fixList = append(fixList, t)
@@ -224,9 +223,8 @@ func (c *Compiler) Codegen(tokens Tokens) (err error) {
 			emit(int64(t.Pos), vm.JumpFalse, i)
 
 		case lang.JumpSetFalse:
-			label := t.Str[13:]
 			var i int64
-			if s, ok := c.symbols[label]; !ok {
+			if s, ok := c.symbols[t.Str]; !ok {
 				// t.Beg contains the position in code which needs to be fixed.
 				t.Beg = len(c.Code)
 				fixList = append(fixList, t)
@@ -236,9 +234,8 @@ func (c *Compiler) Codegen(tokens Tokens) (err error) {
 			emit(int64(t.Pos), vm.JumpSetFalse, i)
 
 		case lang.JumpSetTrue:
-			label := t.Str[12:]
 			var i int64
-			if s, ok := c.symbols[label]; !ok {
+			if s, ok := c.symbols[t.Str]; !ok {
 				// t.Beg contains the position in code which needs to be fixed.
 				t.Beg = len(c.Code)
 				fixList = append(fixList, t)
@@ -248,9 +245,8 @@ func (c *Compiler) Codegen(tokens Tokens) (err error) {
 			emit(int64(t.Pos), vm.JumpSetTrue, int64(i))
 
 		case lang.Goto:
-			label := t.Str[5:]
 			var i int64
-			if s, ok := c.symbols[label]; !ok {
+			if s, ok := c.symbols[t.Str]; !ok {
 				t.Beg = len(c.Code)
 				fixList = append(fixList, t)
 			} else {
@@ -275,21 +271,9 @@ func (c *Compiler) Codegen(tokens Tokens) (err error) {
 
 	// Finally we fix unresolved labels for jump destinations.
 	for _, t := range fixList {
-		var label string
-		// TODO: this could be simplified.
-		switch t.Id {
-		case lang.Goto:
-			label = t.Str[5:]
-		case lang.JumpFalse:
-			label = t.Str[10:]
-		case lang.JumpSetFalse:
-			label = t.Str[13:]
-		case lang.JumpSetTrue:
-			label = t.Str[12:]
-		}
-		s, ok := c.symbols[label]
+		s, ok := c.symbols[t.Str]
 		if !ok {
-			return fmt.Errorf("label not found: %q", label)
+			return fmt.Errorf("label not found: %q", t.Str)
 		}
 		c.Code[t.Beg][2] = s.value.Data.Int() - int64(t.Beg)
 
diff --git a/parser/expr.go b/parser/expr.go
index 0b2ad2a..73b860e 100644
--- a/parser/expr.go
+++ b/parser/expr.go
@@ -88,18 +88,20 @@ func (p *Parser) ParseExpr(in Tokens) (out Tokens, err error) {
 					log.Println("callExpr:", t2.Str, p.scope, s, ok, sc)
 					if s.kind == symValue {
 						// Store the number of input parameters in the token Beg field.
-						ops = append(ops, scanner.Token{Str: "callX", Id: lang.CallX, Pos: t.Pos, Beg: p.numItems(t.Block(), lang.Comma)})
+						ops = append(ops, scanner.Token{Id: lang.CallX, Pos: t.Pos, Beg: p.numItems(t.Block(), lang.Comma)})
 						break
 					}
 				}
 			}
-			ops = append(ops, scanner.Token{Str: "call", Id: lang.Call, Pos: t.Pos})
+			ops = append(ops, scanner.Token{Id: lang.Call, Pos: t.Pos})
 		case lang.BracketBlock:
 			out = append(out, t)
 			vl++
-			ops = append(ops, scanner.Token{Str: "index", Id: lang.Index, Pos: t.Pos})
+			ops = append(ops, scanner.Token{Id: lang.Index, Pos: t.Pos})
+		case lang.Comment:
+			return out, nil
 		default:
-			return nil, fmt.Errorf("expression not supported yet: %q", t.Str)
+			return nil, fmt.Errorf("expression not supported yet: %v: %q", t.Id, t.Str)
 		}
 		if len(selectors) > 0 {
 			out = append(out, selectors...)
@@ -196,9 +198,9 @@ func (p *Parser) ParseLogical(in Tokens) (out Tokens, err error) {
 	}
 	out = append(out, lhs...)
 	if in[l].Id == lang.Lor {
-		out = append(out, scanner.Token{Id: lang.JumpSetTrue, Str: "JumpSetTrue " + p.scope + "x" + xp})
+		out = append(out, scanner.Token{Id: lang.JumpSetTrue, Str: p.scope + "x" + xp})
 	} else {
-		out = append(out, scanner.Token{Id: lang.JumpSetFalse, Str: "JumpSetFalse " + p.scope + "x" + xp})
+		out = append(out, scanner.Token{Id: lang.JumpSetFalse, Str: p.scope + "x" + xp})
 	}
 	out = append(out, rhs...)
 	out = append(out, scanner.Token{Id: lang.Label, Str: p.scope + "x" + xp})
diff --git a/parser/parse.go b/parser/parse.go
index 36ff125..95f9af3 100644
--- a/parser/parse.go
+++ b/parser/parse.go
@@ -82,10 +82,15 @@ func (p *Parser) ParseStmt(in Tokens) (out Tokens, err error) {
 		return p.ParseFor(in)
 	case lang.Func:
 		return p.ParseFunc(in)
+	case lang.Defer, lang.Go, lang.Fallthrough, lang.Import, lang.Select:
+		return out, fmt.Errorf("not yet implemented: %v", t.Id)
 	case lang.Goto:
 		return p.ParseGoto(in)
 	case lang.If:
 		return p.ParseIf(in)
+	case lang.Package:
+		// TODO: support packages
+		return out, err
 	case lang.Return:
 		return p.ParseReturn(in)
 	case lang.Switch:
@@ -118,7 +123,7 @@ func (p *Parser) ParseBreak(in Tokens) (out Tokens, err error) {
 	default:
 		return nil, fmt.Errorf("invalid break statement")
 	}
-	out = Tokens{{Id: lang.Goto, Str: "goto " + label}}
+	out = Tokens{{Id: lang.Goto, Str: label}}
 	return out, err
 }
 
@@ -136,7 +141,7 @@ func (p *Parser) ParseContinue(in Tokens) (out Tokens, err error) {
 	default:
 		return nil, fmt.Errorf("invalid continue statement")
 	}
-	out = Tokens{{Id: lang.Goto, Str: "goto " + label}}
+	out = Tokens{{Id: lang.Goto, Str: label}}
 	return out, err
 }
 
@@ -145,7 +150,7 @@ func (p *Parser) ParseGoto(in Tokens) (out Tokens, err error) {
 		return nil, fmt.Errorf("invalid goto statement")
 	}
 	// TODO: check validity of user provided label
-	return Tokens{{Id: lang.Goto, Str: "goto " + p.funcScope + "/" + in[1].Str}}, nil
+	return Tokens{{Id: lang.Goto, Str: p.funcScope + "/" + in[1].Str}}, nil
 }
 
 func (p *Parser) ParseFor(in Tokens) (out Tokens, err error) {
@@ -181,7 +186,7 @@ func (p *Parser) ParseFor(in Tokens) (out Tokens, err error) {
 			return nil, err
 		}
 		out = append(out, cond...)
-		out = append(out, scanner.Token{Id: lang.JumpFalse, Str: "JumpFalse " + p.scope + "e"})
+		out = append(out, scanner.Token{Id: lang.JumpFalse, Str: p.scope + "e"})
 	}
 	if body, err = p.Parse(in[len(in)-1].Block()); err != nil {
 		return nil, err
@@ -194,7 +199,7 @@ func (p *Parser) ParseFor(in Tokens) (out Tokens, err error) {
 		out = append(out, post...)
 	}
 	out = append(out,
-		scanner.Token{Id: lang.Goto, Str: "goto " + p.scope + "b"},
+		scanner.Token{Id: lang.Goto, Str: p.scope + "b"},
 		scanner.Token{Id: lang.Label, Str: p.scope + "e"})
 	return out, err
 }
@@ -230,7 +235,7 @@ func (p *Parser) ParseFunc(in Tokens) (out Tokens, err error) {
 	}()
 
 	out = Tokens{
-		{Id: lang.Goto, Str: "goto " + fname + "_end"}, // Skip function definition.
+		{Id: lang.Goto, Str: fname + "_end"}, // Skip function definition.
 		{Id: lang.Label, Pos: in[0].Pos, Str: fname}}
 
 	bi := in.Index(lang.BraceBlock)
@@ -283,7 +288,7 @@ func (p *Parser) ParseIf(in Tokens) (out Tokens, err error) {
 			return nil, err
 		}
 		if sc > 0 {
-			pre = append(pre, scanner.Token{Id: lang.Goto, Str: "goto " + p.scope + "e0"})
+			pre = append(pre, scanner.Token{Id: lang.Goto, Str: p.scope + "e0"})
 		}
 		pre = append(pre, scanner.Token{Id: lang.Label, Str: p.scope + "e" + ssc})
 		out = append(pre, out...)
@@ -313,7 +318,7 @@ func (p *Parser) ParseIf(in Tokens) (out Tokens, err error) {
 			return nil, err
 		}
 		pre = append(pre, cond...)
-		pre = append(pre, scanner.Token{Id: lang.JumpFalse, Str: "JumpFalse " + p.scope + "e" + ssc})
+		pre = append(pre, scanner.Token{Id: lang.JumpFalse, Str: p.scope + "e" + ssc})
 		out = append(pre, out...)
 		i = ifp
 		if i > 1 && in[i].Id == lang.If && in[i-1].Id == lang.Else { // Step over 'else if'.
@@ -412,11 +417,11 @@ func (p *Parser) ParseCaseClause(in Tokens, index, max int, condSwitch bool) (ou
 			if condSwitch {
 				out = append(out, scanner.Token{Id: lang.EqualSet})
 			}
-			out = append(out, scanner.Token{Id: lang.JumpFalse, Str: "JumpFalse " + next})
+			out = append(out, scanner.Token{Id: lang.JumpFalse, Str: next})
 		}
 		out = append(out, body...)
 		if i != len(lcond)-1 || index != max {
-			out = append(out, scanner.Token{Id: lang.Goto, Str: "Goto " + p.scope + "e"})
+			out = append(out, scanner.Token{Id: lang.Goto, Str: p.scope + "e"})
 		}
 	}
 	return out, err
diff --git a/parser/tokens.go b/parser/tokens.go
index e581eb5..51085d9 100644
--- a/parser/tokens.go
+++ b/parser/tokens.go
@@ -1,8 +1,6 @@
 package parser
 
 import (
-	"fmt"
-
 	"github.com/mvertes/parscan/lang"
 	"github.com/mvertes/parscan/scanner"
 )
@@ -11,7 +9,7 @@ type Tokens []scanner.Token
 
 func (toks Tokens) String() (s string) {
 	for _, t := range toks {
-		s += fmt.Sprintf("%#v ", t.Str)
+		s += t.String() + " "
 	}
 	return s
 }
diff --git a/scanner/scan.go b/scanner/scan.go
index f5d56d1..ea7fc37 100644
--- a/scanner/scan.go
+++ b/scanner/scan.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/mvertes/parscan/lang"
@@ -37,6 +38,14 @@ func (t *Token) Name() string {
 	return name
 }
 
+func (t *Token) String() string {
+	s := t.Id.String()
+	if t.Id.IsLiteral() || t.Id.IsBlock() || t.Id == lang.Ident || t.Id == lang.Comment {
+		s += strconv.Quote(t.Str)
+	}
+	return s
+}
+
 // Scanner contains the scanner rules for a language.
 type Scanner struct {
 	*lang.Spec
diff --git a/scanner/scan_test.go b/scanner/scan_test.go
index ff86535..257b1ab 100644
--- a/scanner/scan_test.go
+++ b/scanner/scan_test.go
@@ -1,7 +1,6 @@
 package scanner_test
 
 import (
-	"fmt"
 	"log"
 	"testing"
 
@@ -37,9 +36,9 @@ func TestScan(t *testing.T) {
 
 func tokStr(tokens []scanner.Token) (s string) {
 	for _, t := range tokens {
-		s += fmt.Sprintf("%#v ", t.Str)
+		s += t.String() + " "
 	}
-	return
+	return s
 }
 
 var tests = []struct {
@@ -48,19 +47,19 @@ var tests = []struct {
 	src: "",
 }, { // #01
 	src: "   abc + 5",
-	tok: `"abc" "+" "5" ";" `,
+	tok: `Ident"abc" Add Int"5" Semicolon `,
 }, { // #02
 	src: "abc0+5 ",
-	tok: `"abc0" "+" "5" ";" `,
+	tok: `Ident"abc0" Add Int"5" Semicolon `,
 }, { // #03
 	src: "a+5\na=x-4",
-	tok: `"a" "+" "5" ";" "a" "=" "x" "-" "4" ";" `,
+	tok: `Ident"a" Add Int"5" Semicolon Ident"a" Assign Ident"x" Sub Int"4" Semicolon `,
 }, { // #04
 	src: `return "hello world" + 4`,
-	tok: `"return" "\"hello world\"" "+" "4" ";" `,
+	tok: `Return String"\"hello world\"" Add Int"4" Semicolon `,
 }, { // #05
 	src: `print(4 * (3+7))`,
-	tok: `"print" "(4 * (3+7))" ";" `,
+	tok: `Ident"print" ParenBlock"(4 * (3+7))" Semicolon `,
 }, { // #06
 	src: `"foo`,
 	err: "1:1: block not terminated",
@@ -73,66 +72,66 @@ def "foo truc`,
 	err: "1:1: block not terminated",
 }, { // #09
 	src: `"ab\\"`,
-	tok: `"\"ab\\\\\"" ";" `,
+	tok: `String"\"ab\\\\\"" Semicolon `,
 }, { // #10
 	src: `"ab\\\"`,
 	err: "1:1: block not terminated",
 }, { // #11
 	src: `"ab\\\\"`,
-	tok: `"\"ab\\\\\\\\\"" ";" `,
+	tok: `String"\"ab\\\\\\\\\"" Semicolon `,
 }, { // #12
 	src: `"abc
 def"`,
 	err: "1:1: block not terminated",
 }, { // #13
 	src: "`hello\nworld`",
-	tok: "\"`hello\\nworld`\" \";\" ",
+	tok: "String\"`hello\\nworld`\" Semicolon ",
 }, { // #14
 	src: "2* (3+4",
 	err: "1:4: block not terminated",
 }, { // #15
 	src: `("fo)o")+1`,
-	tok: `"(\"fo)o\")" "+" "1" ";" `,
+	tok: `ParenBlock"(\"fo)o\")" Add Int"1" Semicolon `,
 }, { // #16
 	src: `"foo""bar"`,
-	tok: `"\"foo\"" "\"bar\"" ";" `,
+	tok: `String"\"foo\"" String"\"bar\"" Semicolon `,
 }, { // #17
 	src: "/* a comment */ a = 2",
-	tok: `"/* a comment */" "a" "=" "2" ";" `,
+	tok: `Comment"/* a comment */" Ident"a" Assign Int"2" Semicolon `,
 }, { // #18
 	src: "return // quit\nbegin",
-	tok: `"return" "// quit" ";" "begin" ";" `,
+	tok: `Return Comment"// quit" Semicolon Ident"begin" Semicolon `,
 }, { // #19
 	src: "return // quit",
-	tok: `"return" "// quit" ";" `,
+	tok: `Return Comment"// quit" Semicolon `,
 }, { // #20
 	src: "println(3 /* argh ) */)",
-	tok: `"println" "(3 /* argh ) */)" ";" `,
+	tok: `Ident"println" ParenBlock"(3 /* argh ) */)" Semicolon `,
 }, { // #21
 	src: `println("in f")`,
-	tok: `"println" "(\"in f\")" ";" `,
+	tok: `Ident"println" ParenBlock"(\"in f\")" Semicolon `,
 }, { // #22
 	src: "a, b = 1, 2",
-	tok: `"a" "," "b" "=" "1" "," "2" ";" `,
+	tok: `Ident"a" Comma Ident"b" Assign Int"1" Comma Int"2" Semicolon `,
 }, { // #23
 	src: "1 + \n2 + 3",
-	tok: `"1" "+" "2" "+" "3" ";" `,
+	tok: `Int"1" Add Int"2" Add Int"3" Semicolon `,
 }, { // #24
 	src: "i++\n2 + 3",
-	tok: `"i" "++" ";" "2" "+" "3" ";" `,
+	tok: `Ident"i" Inc Semicolon Int"2" Add Int"3" Semicolon `,
 }, { // #25
 	src: "return\na = 1",
-	tok: `"return" ";" "a" "=" "1" ";" `,
+	tok: `Return Semicolon Ident"a" Assign Int"1" Semicolon `,
 }, { // #26
 	src: "if\na == 2 { return }",
-	tok: `"if" "a" "==" "2" "{ return }" ";" `,
+	tok: `If Ident"a" Equal Int"2" BraceBlock"{ return }" Semicolon `,
 }, { // #27
 	src: "f(4)\nreturn",
-	tok: `"f" "(4)" ";" "return" ";" `,
+	tok: `Ident"f" ParenBlock"(4)" Semicolon Return Semicolon `,
 }, { // #28
 	src: "f(3).\nfield",
-	tok: `"f" "(3)" "." "field" ";" `,
+	tok: `Ident"f" ParenBlock"(3)" Period Ident"field" Semicolon `,
 }, { // #29
 	src: "\n\n\tif i < 1 {return 0}",
-	tok: `"if" "i" "<" "1" "{return 0}" ";" `,
+	tok: `If Ident"i" Less Int"1" BraceBlock"{return 0}" Semicolon `,
 }}