feat: improve debug output of tokens

1 files changed, 9 insertions, 0 deletions

diff --git a/scanner/scan.go b/scanner/scan.go
index f5d56d1..ea7fc37 100644
--- a/scanner/scan.go
+++ b/scanner/scan.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/mvertes/parscan/lang"
@@ -37,6 +38,14 @@ func (t *Token) Name() string {
 	return name
 }
 
+func (t *Token) String() string {
+	s := t.Id.String()
+	if t.Id.IsLiteral() || t.Id.IsBlock() || t.Id == lang.Ident || t.Id == lang.Comment {
+		s += strconv.Quote(t.Str)
+	}
+	return s
+}
+
 // Scanner contains the scanner rules for a language.
 type Scanner struct {
 	*lang.Spec