first commit

1 files changed, 95 insertions, 0 deletions

diff --git a/scanner/scan_test.go b/scanner/scan_test.go
new file mode 100644
index 0000000..dd48faf
--- /dev/null
+++ b/scanner/scan_test.go
@@ -0,0 +1,95 @@
+package scanner
+
+import (
+	"fmt"
+	"testing"
+)
+
+var GoScanner = &Scanner{
+	CharProp: [ASCIILen]uint{
+		'\t': CharSep,
+		'\n': CharLineSep,
+		' ':  CharSep,
+		'!':  CharOp,
+		'"':  CharStr | StrEsc | StrNonl,
+		'%':  CharOp,
+		'&':  CharOp,
+		'\'': CharStr | StrEsc,
+		'(':  CharBlock,
+		'*':  CharOp,
+		'+':  CharOp,
+		',':  CharGroupSep,
+		'-':  CharOp,
+		'`':  CharStr,
+		'.':  CharOp,
+		'/':  CharOp,
+		':':  CharOp,
+		';':  CharGroupSep,
+		'<':  CharOp,
+		'=':  CharOp,
+		'>':  CharOp,
+		'[':  CharBlock,
+		'^':  CharOp,
+		'{':  CharBlock,
+		'|':  CharOp,
+		'~':  CharOp,
+	},
+	End: map[string]string{
+		"(":  ")",
+		"{":  "}",
+		"[":  "]",
+		"/*": "*/",
+		`"`:  `"`,
+		"'":  "'",
+		"`":  "`",
+		"//": "\n",
+	},
+}
+
+func TestScan(t *testing.T) {
+	tests := []struct{ src, result, errStr string }{
+		// Simple tokens: separators, identifiers, numbers, operators.
+		{"", "[]", ""},
+		{"   abc + 5", "[{3 1 abc 0 0} {7 3 + 0 0} {9 2 5 0 0}]", ""},
+		{"abc0+5 ", "[{0 1 abc0 0 0} {4 3 + 0 0} {5 2 5 0 0}]", ""},
+		{"a+5\na=x-4", "[{0 1 a 0 0} {1 3 + 0 0} {2 2 5 0 0} {3 4   0 0} {4 1 a 0 0} {5 3 = 0 0} {6 1 x 0 0} {7 3 - 0 0} {8 2 4 0 0}]", ""},
+
+		// Strings.
+		{`return "hello world" + 4`, `[{0 1 return 0 0} {7 5 "hello world" 1 1} {21 3 + 0 0} {23 2 4 0 0}]`, ""},
+		{`print(4 * (3+7))`, "[{0 1 print 0 0} {5 6 (4 * (3+7)) 1 1}]", ""},
+		{`"foo`, "[]", "1:1: block not terminated"},
+		{`abc
+def "foo truc`, "[]", "2:6: block not terminated"},
+		{`"ab\"`, "[]", "1:1: block not terminated"},
+		{`"ab\\"`, `[{0 5 "ab\\" 1 1}]`, ""},
+		{`"ab\\\"`, "[]", "1:1: block not terminated"},
+		{`"ab\\\\"`, `[{0 5 "ab\\\\" 1 1}]`, ""},
+		{`"abc
+def"`, "[]", "1:1: block not terminated"},
+		{"`hello\nworld`", "[{0 5 `hello\nworld` 1 1}]", ""},
+
+		// Nested blocks.
+		// {`f("a)bc")+1, 3)`, "[{0 1 f } {1 6 (\"a)bc\", 3) (}]", ""},
+		{"2* (3+4", "[]", "1:4: block not terminated"},
+		{`("fo)o")+1`, "[{0 6 (\"fo)o\") 1 1} {8 3 + 0 0} {9 2 1 0 0}]", ""},
+		{`"foo""bar"`, "[{0 5 \"foo\" 1 1} {5 5 \"bar\" 1 1}]", ""},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			errStr := ""
+			token, err := GoScanner.Scan(test.src)
+			if err != nil {
+				errStr = err.Error()
+			}
+			if errStr != test.errStr {
+				t.Errorf("got error %#v, want error %#v", errStr, test.errStr)
+			}
+			result := fmt.Sprintf("%v", token)
+			t.Logf("%#v\n%v %v\n", test.src, result, errStr)
+			if result != test.result {
+				t.Errorf("got %#v, want %#v", result, test.result)
+			}
+		})
+	}
+}