From 9bf668e7114bb92a0b072db5d4e092c0b8f964c4 Mon Sep 17 00:00:00 2001
From: Marc Vertes <mvertes@free.fr>
Date: Thu, 22 Jan 2026 11:33:32 +0100
Subject: chore: refactor token types to avoid mutate scanner tokens

Mutating scanner tokens or reusing scanner token attributes to
store other metadata is a hack. Introduce a new parser token
type with arbitrary args. The next step will be to use the arg
field instead of scanner token fields.
---
 parser/tokens.go | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(limited to 'parser/tokens.go')

diff --git a/parser/tokens.go b/parser/tokens.go
index ac08e2b..9ae655b 100644
--- a/parser/tokens.go
+++ b/parser/tokens.go
@@ -7,8 +7,14 @@ import (
 	"github.com/mvertes/parscan/scanner"
 )
 
+// Token represents a parser token.
+type Token struct {
+	scanner.Token
+	Arg []any
+}
+
 // Tokens represents slice of tokens.
-type Tokens []scanner.Token
+type Tokens []Token
 
 func (toks Tokens) String() (s string) {
 	var sb strings.Builder
@@ -62,3 +68,11 @@ func (toks Tokens) SplitStart(tok lang.Token) (result []Tokens) {
 		toks = toks[i+1:]
 	}
 }
+
+func newIdent(name string, pos int, arg ...any) Token {
+	return Token{Token: scanner.Token{Tok: lang.Ident, Pos: pos, Str: name}, Arg: arg}
+}
+
+func newToken(tok lang.Token, pos int, arg ...any) Token {
+	return Token{Token: scanner.Token{Tok: tok, Pos: pos}, Arg: arg}
+}
-- 
cgit v1.2.3