parser: skip comment modes

Refctor node kind names by concatenating category and instance, to
allow better sorting. Comments are now parsed and skipped during
generation of AST.

10 files changed, 141 insertions, 94 deletions

diff --git a/codegen/compiler.go b/codegen/compiler.go
index 48d3d41..3952166 100644
--- a/codegen/compiler.go
+++ b/codegen/compiler.go
@@ -47,7 +47,7 @@ func (c *Compiler) CodeGen(node *parser.Node) (err error) {
 		nd := notes[n]
 
 		switch n.Kind {
-		case parser.FuncDecl:
+		case parser.DeclFunc:
 			fname := n.Child[0].Content()
 			c.addSym(len(c.Code), scope+fname, false, n)
 			scope = pushScope(scope, fname)
@@ -59,9 +59,9 @@ func (c *Compiler) CodeGen(node *parser.Node) (err error) {
 				fnote.fsp++
 			}
 
-		case parser.StmtBloc:
+		case parser.BlockStmt:
 			nd.ipstart = len(c.Code)
-			if a != nil && a.Kind == parser.IfStmt && k == 1 {
+			if a != nil && a.Kind == parser.StmtIf && k == 1 {
 				c.Emit(n, vm1.JumpFalse, 0) // location to be updated in post IfStmt
 			}
 		}
@@ -72,43 +72,43 @@ func (c *Compiler) CodeGen(node *parser.Node) (err error) {
 		x := extNode{c, n, a, k}
 
 		switch n.Kind {
-		case parser.AddOp:
+		case parser.OpAdd:
 			c.Emit(n, vm1.Add)
 
-		case parser.CallExpr:
+		case parser.ExprCall:
 			err = postCallExpr(x)
 
-		case parser.DefOp:
+		case parser.OpDefine:
 			// Define operation, global vars only. TODO: on local frame too
 			l := c.addSym(nil, n.Child[0].Content(), false, n)
 			c.Emit(n, vm1.Assign, int64(l))
 
-		case parser.FuncDecl:
+		case parser.DeclFunc:
 			scope = popScope(scope)
 			fnote = notes[frameNode[len(frameNode)-1]]
 
 		case parser.Ident:
 			ident := n.Content()
-			if len(n.Child) > 0 || a.Kind == parser.FuncDecl {
+			if len(n.Child) > 0 || a.Kind == parser.DeclFunc {
 				break
 			}
 			if s, _, ok := c.getSym(ident, scope); ok {
 				if s.local {
 					c.Emit(n, vm1.Fdup, int64(s.index))
-				} else if a != nil && a.Kind == parser.AssignOp {
+				} else if a != nil && a.Kind == parser.OpAssign {
 					c.Emit(n, vm1.Push, int64(s.index))
 				} else if _, ok := c.Data[s.index].(int); !ok {
 					c.Emit(n, vm1.Dup, int64(s.index))
 				}
 			}
 
-		case parser.IfStmt:
+		case parser.StmtIf:
 			ifBodyStart := notes[n.Child[1]].ipstart
 			ifBodyEnd := notes[n.Child[1]].ipend
 			c.Code[ifBodyStart][2] = int64(ifBodyEnd - ifBodyStart)
 			// TODO: handle 'else'
 
-		case parser.NumberLit:
+		case parser.LiteralNumber:
 			// A literal number can be a float or an integer, or a big number
 			switch v := n.Value().(type) {
 			case int64:
@@ -119,22 +119,22 @@ func (c *Compiler) CodeGen(node *parser.Node) (err error) {
 				err = fmt.Errorf("type not supported: %T\n", v)
 			}
 
-		case parser.ReturnStmt:
+		case parser.StmtReturn:
 			fun := frameNode[len(frameNode)-1]
 			c.Emit(n, vm1.Return, int64(len(n.Child)), int64(len(fun.Child[1].Child)))
 
-		case parser.StmtBloc:
+		case parser.BlockStmt:
 			nd.ipend = len(c.Code)
 
-		case parser.StringLit:
+		case parser.LiteralString:
 			p := len(c.Data)
 			c.Data = append(c.Data, n.Block())
 			c.Emit(n, vm1.Dup, int64(p))
 
-		case parser.InfOp:
+		case parser.OpInferior:
 			c.Emit(n, vm1.Lower)
 
-		case parser.SubOp:
+		case parser.OpSubtract:
 			c.Emit(n, vm1.Sub)
 		}
 
@@ -143,7 +143,7 @@ func (c *Compiler) CodeGen(node *parser.Node) (err error) {
 		}
 
 		// TODO: Fix this temporary hack to compute an entry point
-		if c.Entry < 0 && len(scope) == 0 && n.Kind != parser.FuncDecl {
+		if c.Entry < 0 && len(scope) == 0 && n.Kind != parser.DeclFunc {
 			c.Entry = len(c.Code) - 1
 			if c.Entry >= 0 && len(c.Code) > c.Entry && c.Code[c.Entry][1] == vm1.Return {
 				c.Entry++
diff --git a/codegen/expression.go b/codegen/expression.go
index b73f4da..f3eb38a 100644
--- a/codegen/expression.go
+++ b/codegen/expression.go
@@ -33,7 +33,7 @@ func postCallExpr(x extNode) error {
 
 func usedRet(n *parser.Node) bool {
 	switch n.Kind {
-	case parser.Undefined, parser.StmtBloc:
+	case parser.Undefined, parser.BlockStmt:
 		return false
 	default:
 		return true
diff --git a/lang/golang/go.go b/lang/golang/go.go
index 2ab1781..5075329 100644
--- a/lang/golang/go.go
+++ b/lang/golang/go.go
@@ -43,24 +43,36 @@ var GoScanner = &scanner.Scanner{
 		"`":  "`",
 		"//": "\n",
 	},
+	BlockProp: map[string]uint{
+		"(":  scanner.CharBlock,
+		"{":  scanner.CharBlock,
+		"[":  scanner.CharBlock,
+		`"`:  scanner.CharStr | scanner.StrEsc | scanner.StrNonl,
+		"`":  scanner.CharStr,
+		"'":  scanner.CharStr | scanner.StrEsc,
+		"/*": scanner.CharStr,
+		"//": scanner.CharStr | scanner.ExcludeEnd | scanner.EosValidEnd,
+	},
 }
 
 var GoParser = &parser.Parser{
 	Scanner: GoScanner,
 	Spec: map[string]parser.NodeSpec{
-		".":      {Kind: parser.DotOp, Flags: parser.Call, Order: 3},
-		"*":      {Kind: parser.MulOp, Order: 4},
-		"+":      {Kind: parser.AddOp, Order: 5},
-		"-":      {Kind: parser.SubOp, Order: 5},
-		"<":      {Kind: parser.InfOp, Order: 6},
-		":=":     {Kind: parser.DefOp, Order: 7},
-		"=":      {Kind: parser.AssignOp, Order: 7},
-		"if":     {Kind: parser.IfStmt, Flags: parser.Stmt | parser.ExprSep},
-		"func":   {Kind: parser.FuncDecl, Flags: parser.Decl | parser.Call},
-		"return": {Kind: parser.ReturnStmt, Flags: parser.Stmt},
-		"{..}":   {Kind: parser.StmtBloc, Flags: parser.ExprSep},
-		"(..)":   {Kind: parser.ParBloc, Flags: parser.Call},
-		`".."`:   {Kind: parser.StringLit},
+		".":      {Kind: parser.OpDot, Flags: parser.Call, Order: 3},
+		"*":      {Kind: parser.OpMultiply, Order: 4},
+		"+":      {Kind: parser.OpAdd, Order: 5},
+		"-":      {Kind: parser.OpSubtract, Order: 5},
+		"<":      {Kind: parser.OpInferior, Order: 6},
+		":=":     {Kind: parser.OpDefine, Order: 7},
+		"=":      {Kind: parser.OpAssign, Order: 7},
+		"if":     {Kind: parser.StmtIf, Flags: parser.Stmt | parser.ExprSep},
+		"func":   {Kind: parser.DeclFunc, Flags: parser.Decl | parser.Call},
+		"return": {Kind: parser.StmtReturn, Flags: parser.Stmt},
+		"{..}":   {Kind: parser.BlockStmt, Flags: parser.ExprSep},
+		"(..)":   {Kind: parser.BlockParen, Flags: parser.Call},
+		`".."`:   {Kind: parser.LiteralString},
+		"//..":   {Kind: parser.Comment},
+		"/*..":   {Kind: parser.Comment},
 	},
 }
 
diff --git a/parser/README.md b/parser/README.md
index 2a25fc8..1c02d22 100644
--- a/parser/README.md
+++ b/parser/README.md
@@ -53,7 +53,7 @@ A successful test must be provided to check the status.
 - [ ] full `if` statement (including `else`, `else if`)
 - [x] init expressions in `if` statements
 - [x] statement blocks
-- [ ] comments
+- [x] comments
 - [ ] for statement
 - [ ] switch statement
 - [ ] select statement
diff --git a/parser/kind.go b/parser/kind.go
index d570358..d004471 100644
--- a/parser/kind.go
+++ b/parser/kind.go
@@ -2,46 +2,50 @@ package parser
 
 import "fmt"
 
+// kind defines the AST node kind. Its name is the concatenation
+// of a category (Block, Decl, Expr, Op, Stmt) and an instance name.
 type Kind int
 
 const (
 	Undefined = Kind(iota)
-	FuncDecl
-	CallExpr
-	IfStmt
-	StmtBloc
-	ReturnStmt
+	BlockParen
+	BlockStmt
+	Comment
+	DeclFunc
+	ExprCall
 	Ident
-	StringLit
-	NumberLit
-	ParBloc
-	DotOp
-	MulOp
-	AddOp
-	SubOp
-	AssignOp
-	DefOp
-	InfOp
+	LiteralNumber
+	LiteralString
+	OpAdd
+	OpAssign
+	OpDefine
+	OpDot
+	OpInferior
+	OpMultiply
+	OpSubtract
+	StmtIf
+	StmtReturn
 )
 
 var kindString = [...]string{
-	Undefined:  "Undefined",
-	FuncDecl:   "FuncDecl",
-	CallExpr:   "CallExpr",
-	IfStmt:     "IfStmt",
-	StmtBloc:   "StmtBloc",
-	ReturnStmt: "ReturnStmt",
-	Ident:      "Ident",
-	StringLit:  "StringLit",
-	NumberLit:  "NumberLit",
-	ParBloc:    "ParBloc",
-	DotOp:      "DotOp",
-	MulOp:      "MulOp",
-	AddOp:      "AddOP",
-	SubOp:      "SubOp",
-	AssignOp:   "AssignOp",
-	DefOp:      "DefOp",
-	InfOp:      "InfOp",
+	Undefined:     "Undefined",
+	BlockParen:    "BlockParen",
+	BlockStmt:     "BlockStmt",
+	Comment:       "Comment",
+	DeclFunc:      "DeclFunc",
+	ExprCall:      "ExprCall",
+	Ident:         "Ident",
+	LiteralString: "LiteralString",
+	LiteralNumber: "LiteralNumber",
+	OpAdd:         "OpAdd",
+	OpAssign:      "OpAssign",
+	OpDefine:      "OpDefine",
+	OpDot:         "OpDot",
+	OpInferior:    "OpInferior",
+	OpMultiply:    "OpMultiply",
+	OpSubtract:    "OpSubtract",
+	StmtIf:        "StmtIf",
+	StmtReturn:    "StmtReturn",
 }
 
 func (k Kind) String() string {
diff --git a/parser/parse.go b/parser/parse.go
index 4f186bd..08fbdbd 100644
--- a/parser/parse.go
+++ b/parser/parse.go
@@ -1,6 +1,8 @@
 package parser
 
-import "github.com/gnolang/parscan/scanner"
+import (
+	"github.com/gnolang/parscan/scanner"
+)
 
 const (
 	Stmt = 1 << iota
@@ -43,13 +45,16 @@ func (p *Parser) ParseTokens(tokens []scanner.Token) (roots []*Node, err error)
 			Token: t,
 			Kind:  p.Spec[t.Name()].Kind,
 		}
+		if c.Kind == Comment {
+			continue
+		}
 		if t.IsOperator() && (i == 0 || tokens[i-1].IsOperator()) {
 			unaryOp[c] = true
 		}
 		if c.Kind == Undefined {
 			switch t.Kind() {
 			case scanner.Number:
-				c.Kind = NumberLit
+				c.Kind = LiteralNumber
 			case scanner.Identifier:
 				c.Kind = Ident
 			}
@@ -80,7 +85,7 @@ func (p *Parser) ParseTokens(tokens []scanner.Token) (roots []*Node, err error)
 					}
 					lce.Child = []*Node{{Token: lce.Token, Child: lce.Child, Kind: lce.Kind}}
 					lce.Token = scanner.NewToken("Call", c.Pos())
-					lce.Kind = CallExpr
+					lce.Kind = ExprCall
 				}
 			}
 			tcont := t.Content()
diff --git a/parser/parse_test.go b/parser/parse_test.go
index d13f893..0a41d09 100644
--- a/parser/parse_test.go
+++ b/parser/parse_test.go
@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"log"
 	"os"
 	"testing"
 
@@ -45,27 +46,42 @@ var GoScanner = &scanner.Scanner{
 		"`":  "`",
 		"//": "\n",
 	},
+	BlockProp: map[string]uint{
+		"(":  scanner.CharBlock,
+		"{":  scanner.CharBlock,
+		"[":  scanner.CharBlock,
+		`"`:  scanner.CharStr | scanner.StrEsc | scanner.StrNonl,
+		"`":  scanner.CharStr,
+		"'":  scanner.CharStr | scanner.StrEsc,
+		"/*": scanner.CharStr,
+		"//": scanner.CharStr | scanner.ExcludeEnd | scanner.EosValidEnd,
+	},
 }
 
 var GoParser = &Parser{
 	Scanner: GoScanner,
 	Spec: map[string]NodeSpec{
-		".":      {DotOp, Call, 3},
-		"*":      {MulOp, 0, 4},
-		"+":      {AddOp, 0, 5},
-		"-":      {SubOp, 0, 5},
-		"<":      {InfOp, 0, 6},
-		":=":     {DefOp, 0, 7},
-		"=":      {AssignOp, 0, 7},
-		"if":     {IfStmt, Stmt | ExprSep, 0},
-		"func":   {FuncDecl, Decl | Call, 0},
-		"return": {ReturnStmt, Stmt, 0},
-		"{..}":   {StmtBloc, ExprSep, 0},
-		"(..)":   {ParBloc, Call, 0},
+		".":      {Kind: OpDot, Flags: Call, Order: 3},
+		"*":      {Kind: OpMultiply, Order: 4},
+		"+":      {Kind: OpAdd, Order: 5},
+		"-":      {Kind: OpSubtract, Order: 5},
+		"<":      {Kind: OpInferior, Order: 6},
+		":=":     {Kind: OpDefine, Order: 7},
+		"=":      {Kind: OpAssign, Order: 7},
+		"if":     {Kind: StmtIf, Flags: Stmt | ExprSep},
+		"func":   {Kind: DeclFunc, Flags: Decl | Call},
+		"return": {Kind: StmtReturn, Flags: Stmt},
+		"{..}":   {Kind: BlockStmt, Flags: ExprSep},
+		"(..)":   {Kind: BlockParen, Flags: Call},
+		"//..":   {Kind: Comment},
+		"/*..":   {Kind: Comment},
 	},
 }
 
-func init() { GoParser.Init() }
+func init() {
+	GoParser.Init()
+	log.SetFlags(log.Lshortfile)
+}
 
 func TestParse(t *testing.T) {
 	for _, test := range goTests {
@@ -173,6 +189,9 @@ var goTests = []struct {
 }, { // #25
 	src: "f(i) + f(j)",
 	dot: `digraph ast { 0 [label=""]; 1 [label="+"]; 0 -> 1; 2 [label="Call"]; 1 -> 2; 3 [label="f"]; 2 -> 3; 4 [label="(..)"]; 2 -> 4; 5 [label="i"]; 4 -> 5; 6 [label="Call"]; 1 -> 6; 7 [label="f"]; 6 -> 7; 8 [label="(..)"]; 6 -> 8; 9 [label="j"]; 8 -> 9; }`,
+}, { // #26
+	src: "a := 1 // This is a comment",
+	dot: `digraph ast { 0 [label=""]; 1 [label=":="]; 0 -> 1; 2 [label="a"]; 1 -> 2; 3 [label="1"]; 1 -> 3; }`,
 	//src: "f(i) + f(j)(4)", // not ok
 	/*
 	   }, { // #26
diff --git a/scanner/scan.go b/scanner/scan.go
index 5644932..ac2aa17 100644
--- a/scanner/scan.go
+++ b/scanner/scan.go
@@ -63,8 +63,11 @@ func (t *Token) IsSeparator() bool { return t.kind == Separator }
 
 func (t *Token) Name() string {
 	name := t.content
+	if t.start > 1 {
+		return name[:t.start] + ".."
+	}
 	if t.start > 0 {
-		name = name[:t.start] + ".." + name[len(name)-t.end:]
+		return name[:t.start] + ".." + name[len(name)-t.end:]
 	}
 	return name
 }
@@ -295,7 +298,8 @@ func (sc *Scanner) getStr(src string, nstart int) (s string, ok bool) {
 		}
 		esc = canEscape && r == '\\' && !esc
 	}
-	return
+	ok = prop&EosValidEnd != 0
+	return s, ok
 }
 
 func (sc *Scanner) getBlock(src string, nstart int) (s string, ok bool) {
diff --git a/scanner/scan_test.go b/scanner/scan_test.go
index 9eb079e..9450f62 100644
--- a/scanner/scan_test.go
+++ b/scanner/scan_test.go
@@ -147,6 +147,9 @@ def"`,
 	src: "return // quit\nbegin",
 	tok: `"return" "// quit" " " "begin" `,
 }, { // #19
+	src: "return // quit",
+	tok: `"return" "// quit" `,
+}, { // #20
 	src: "println(3 /* argh ) */)",
 	tok: `"println" "(3 /* argh ) */)" `,
 }}
diff --git a/vm0/vm.go b/vm0/vm.go
index e26e20f..d9c9a8d 100644
--- a/vm0/vm.go
+++ b/vm0/vm.go
@@ -46,8 +46,8 @@ func (i *Interp) Run(node *parser.Node, scope string) (err error) {
 	node.Walk2(nil, 0, func(n, a *parser.Node, k int) (ok bool) {
 		// Node pre-order processing.
 		switch n.Kind {
-		case parser.StmtBloc:
-			if a != nil && a.Kind == parser.IfStmt {
+		case parser.BlockStmt:
+			if a != nil && a.Kind == parser.StmtIf {
 				// Control-flow in 'if' sub-tree
 				if k == 1 {
 					// 'if' first body branch, evaluated when condition is true.
@@ -59,7 +59,7 @@ func (i *Interp) Run(node *parser.Node, scope string) (err error) {
 				// 'else' body branch, evaluated when condition is false.
 				return !i.pop().(bool)
 			}
-		case parser.FuncDecl:
+		case parser.DeclFunc:
 			i.declareFunc(n, scope)
 			return false
 		}
@@ -71,7 +71,7 @@ func (i *Interp) Run(node *parser.Node, scope string) (err error) {
 		}
 		l := len(i.stack)
 		switch n.Kind {
-		case parser.NumberLit:
+		case parser.LiteralNumber:
 			switch v := n.Value().(type) {
 			case int64:
 				i.push(int(v))
@@ -82,27 +82,27 @@ func (i *Interp) Run(node *parser.Node, scope string) (err error) {
 				err = fmt.Errorf("type not supported: %T\n", v)
 				return false
 			}
-		case parser.StringLit:
+		case parser.LiteralString:
 			i.push(n.Block())
-		case parser.InfOp:
+		case parser.OpInferior:
 			i.stack[l-2] = i.stack[l-2].(int) < i.stack[l-1].(int)
 			i.stack = i.stack[:l-1]
-		case parser.AddOp:
+		case parser.OpAdd:
 			i.stack[l-2] = i.stack[l-2].(int) + i.stack[l-1].(int)
 			i.stack = i.stack[:l-1]
-		case parser.SubOp:
+		case parser.OpSubtract:
 			i.stack[l-2] = i.stack[l-2].(int) - i.stack[l-1].(int)
 			i.stack = i.stack[:l-1]
-		case parser.MulOp:
+		case parser.OpMultiply:
 			i.stack[l-2] = i.stack[l-2].(int) * i.stack[l-1].(int)
 			i.stack = i.stack[:l-1]
-		case parser.AssignOp, parser.DefOp:
+		case parser.OpAssign, parser.OpDefine:
 			i.stack[i.stack[l-2].(int)] = i.stack[l-1]
 			i.stack = i.stack[:l-2]
-		case parser.ReturnStmt:
+		case parser.StmtReturn:
 			stop = true
 			return false
-		case parser.CallExpr:
+		case parser.ExprCall:
 			i.push(len(n.Child[1].Child)) // number of arguments to call
 			i.callFunc(n)
 		case parser.Ident: