diff options
Diffstat (limited to 'lang/spec.go')
| -rw-r--r-- | lang/spec.go | 15 |
1 files changed, 10 insertions, 5 deletions
diff --git a/lang/spec.go b/lang/spec.go index a910f70..92d90f7 100644 --- a/lang/spec.go +++ b/lang/spec.go @@ -1,5 +1,7 @@ +// Package lang provides tokens for possibly multiple languages. package lang +// Lexical properties of tokens to allow scanning. const ( CharIllegal = 1 << iota CharOp @@ -16,26 +18,29 @@ const ( EosValidEnd // end of input string terminates block or string token ) +// ASCIILen is the length of the ASCII characters set. const ASCIILen = 1 << 7 // 128 +// TokenProp represent token properties for parsing. type TokenProp struct { - TokenId + Token SkipSemi bool // automatic semicolon insertion after newline Precedence int // operator precedence } +// Spec represents the token specification for scanning. type Spec struct { CharProp [ASCIILen]uint // special Character properties End map[string]string // end delimiters, indexed by start BlockProp map[string]uint // block properties TokenProps map[string]TokenProp // token properties DotNum bool // true if a number can start with '.' - IdAscii bool // true if an identifier can be in ASCII only - Num_ bool // true if a number can contain _ character + IdentASCII bool // true if an identifier can be in ASCII only + NumUnder bool // true if a number can contain _ character } -// HasInit stores if a statement may contain a simple init statement -var HasInit = map[TokenId]bool{ +// HasInit stores if a statement may contain a simple init statement. +var HasInit = map[Token]bool{ Case: true, For: true, If: true, |