chore: refactor token types to avoid mutate scanner tokens

Mutating scanner tokens or reusing scanner token attributes to
store other metadata is a hack. Introduce a new parser token
type with arbitrary args. The next step will be to use the arg
field instead of scanner token fields.

1 files changed, 15 insertions, 1 deletions

diff --git a/parser/tokens.go b/parser/tokens.go
index ac08e2b..9ae655b 100644
--- a/parser/tokens.go
+++ b/parser/tokens.go
@@ -7,8 +7,14 @@ import (
 	"github.com/mvertes/parscan/scanner"
 )
 
+// Token represents a parser token.
+type Token struct {
+	scanner.Token
+	Arg []any
+}
+
 // Tokens represents slice of tokens.
-type Tokens []scanner.Token
+type Tokens []Token
 
 func (toks Tokens) String() (s string) {
 	var sb strings.Builder
@@ -62,3 +68,11 @@ func (toks Tokens) SplitStart(tok lang.Token) (result []Tokens) {
 		toks = toks[i+1:]
 	}
 }
+
+func newIdent(name string, pos int, arg ...any) Token {
+	return Token{Token: scanner.Token{Tok: lang.Ident, Pos: pos, Str: name}, Arg: arg}
+}
+
+func newToken(tok lang.Token, pos int, arg ...any) Token {
+	return Token{Token: scanner.Token{Tok: tok, Pos: pos}, Arg: arg}
+}