From a21b9b12ad865a19ff687645082f9093c4101039 Mon Sep 17 00:00:00 2001
From: Marc Vertes <mvertes@free.fr>
Date: Fri, 8 Sep 2023 11:48:35 +0200
Subject: scanner: automatic semi-colon insertion at EOF

---
 scanner/scan.go | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

(limited to 'scanner/scan.go')

diff --git a/scanner/scan.go b/scanner/scan.go
index 5579dc3..e0a32ca 100644
--- a/scanner/scan.go
+++ b/scanner/scan.go
@@ -49,6 +49,15 @@ type Token struct {
 	value   any
 }
 
+type Tokens []*Token
+
+func (tks Tokens) String() (s string) {
+	for _, t := range tks {
+		s += fmt.Sprintf("%#v ", t.content)
+	}
+	return
+}
+
 func (t *Token) Kind() Kind        { return t.kind }
 func (t *Token) Content() string   { return t.content }
 func (t *Token) Start() int        { return t.start }
@@ -127,7 +136,7 @@ func (sc *Scanner) Init() {
 
 func isNum(r rune) bool { return '0' <= r && r <= '9' }
 
-func (sc *Scanner) Scan(src string) (tokens []*Token, err error) {
+func (sc *Scanner) Scan(src string) (tokens Tokens, err error) {
 	offset := 0
 	s := src
 	for len(s) > 0 {
@@ -139,7 +148,7 @@ func (sc *Scanner) Scan(src string) (tokens []*Token, err error) {
 			break
 		}
 		skip := false
-		if t.kind == Separator && t.content == " " && len(sc.SkipSemi) > 0 {
+		if len(tokens) > 0 && len(sc.SkipSemi) > 0 && t.kind == Separator && t.content == " " {
 			// Check for automatic semi-colon insertion after newline.
 			last := tokens[len(tokens)-1]
 			if last.kind == Identifier && sc.SkipSemi[last.content] ||
@@ -157,6 +166,14 @@ func (sc *Scanner) Scan(src string) (tokens []*Token, err error) {
 			tokens = append(tokens, t)
 		}
 	}
+	// Insertion of semi-colon at the end of the token stream.
+	if len(tokens) > 0 && len(sc.SkipSemi) > 0 {
+		last := tokens[len(tokens)-1]
+		if !(last.kind == Identifier && sc.SkipSemi[last.content] ||
+			last.kind == Operator && !sc.SkipSemi[last.content]) {
+			tokens = append(tokens, &Token{kind: Separator, content: ";"})
+		}
+	}
 	return tokens, nil
 }
 
@@ -351,3 +368,13 @@ func (sc *Scanner) getBlock(src string, nstart int) (s string, ok bool) {
 	ok = prop&EosValidEnd != 0
 	return s, ok
 }
+
+// Index returns the index of the first instance with content s in tokens, or -1 if not found.
+func Index(tokens []*Token, s string) int {
+	for i, t := range tokens {
+		if t.content == s {
+			return i
+		}
+	}
+	return -1
+}
-- 
cgit v1.2.3