From 1bff92c52b27b9a516599e172fe9852c3d99be38 Mon Sep 17 00:00:00 2001 From: Marc Vertes Date: Tue, 2 Apr 2024 11:27:13 +0200 Subject: chore: add linters and some lint fixes (#8) * chore: add linters and some lint fixes Configure some golangci-lint linters to get the code quality right. Apply the first fixes. Next step will be to add github actions to run lint and tests in github CI. * chore: more lint, fixed comments and variable names. no semantic change. * chore: add Makefile This makefile is intended to be used as a local substitute to github actions. --- lang/golang/go.go | 122 +++++++++++++++++++++++++------------------------ lang/spec.go | 15 ++++-- lang/token.go | 64 ++++++++++++++++---------- lang/token_string.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++ lang/tokenid_string.go | 115 ---------------------------------------------- 5 files changed, 228 insertions(+), 203 deletions(-) create mode 100644 lang/token_string.go delete mode 100644 lang/tokenid_string.go (limited to 'lang') diff --git a/lang/golang/go.go b/lang/golang/go.go index 5b0ffa6..47baee2 100644 --- a/lang/golang/go.go +++ b/lang/golang/go.go @@ -1,7 +1,9 @@ +// Package golang provides the lexical specification of Go language. package golang import "github.com/mvertes/parscan/lang" +// GoSpec contains the lexical specification of Go. var GoSpec = &lang.Spec{ CharProp: [lang.ASCIILen]uint{ '\t': lang.CharSep, @@ -53,73 +55,73 @@ var GoSpec = &lang.Spec{ }, TokenProps: map[string]lang.TokenProp{ // Block tokens (can be nested) - "{..}": {TokenId: lang.BraceBlock}, - "[..]": {TokenId: lang.BracketBlock}, - "(..)": {TokenId: lang.ParenBlock}, + "{..}": {Token: lang.BraceBlock}, + "[..]": {Token: lang.BracketBlock}, + "(..)": {Token: lang.ParenBlock}, // String tokens (not nested) - "//..": {TokenId: lang.Comment}, - "/*..": {TokenId: lang.Comment}, - `".."`: {TokenId: lang.String}, - "`..`": {TokenId: lang.String}, + "//..": {Token: lang.Comment}, + "/*..": {Token: lang.Comment}, + `".."`: {Token: lang.String}, + "`..`": {Token: lang.String}, // Separators - ",": {TokenId: lang.Comma}, - ";": {TokenId: lang.Semicolon}, - ".": {TokenId: lang.Period}, - ":": {TokenId: lang.Colon}, + ",": {Token: lang.Comma}, + ";": {Token: lang.Semicolon}, + ".": {Token: lang.Period}, + ":": {Token: lang.Colon}, // Operators - "&": {TokenId: lang.And, Precedence: 1}, - "*": {TokenId: lang.Mul, Precedence: 1}, - "/": {TokenId: lang.Quo, Precedence: 1}, - "%": {TokenId: lang.Rem, Precedence: 1}, - "<<": {TokenId: lang.Shl, Precedence: 1}, - ">>": {TokenId: lang.Shr, Precedence: 1}, - "+": {TokenId: lang.Add, Precedence: 2}, - "-": {TokenId: lang.Sub, Precedence: 2}, - "=": {TokenId: lang.Assign, Precedence: 6}, - "+=": {TokenId: lang.AddAssign, Precedence: 6}, - "<": {TokenId: lang.Less, Precedence: 3}, - ">": {TokenId: lang.Greater, Precedence: 3}, - "^": {TokenId: lang.Xor, Precedence: 2}, - "~": {TokenId: lang.Tilde}, - "&&": {TokenId: lang.Land, Precedence: 4}, - "||": {TokenId: lang.Lor, Precedence: 5}, - ":=": {TokenId: lang.Define, Precedence: 6}, - "==": {TokenId: lang.Equal, Precedence: 3}, - "<=": {TokenId: lang.LessEqual, Precedence: 3}, - ">=": {TokenId: lang.GreaterEqual, Precedence: 3}, - "->": {TokenId: lang.Arrow}, - "!": {TokenId: lang.Not}, - "++": {TokenId: lang.Inc, SkipSemi: true}, - "--": {TokenId: lang.Dec, SkipSemi: true}, + "&": {Token: lang.And, Precedence: 1}, + "*": {Token: lang.Mul, Precedence: 1}, + "/": {Token: lang.Quo, Precedence: 1}, + "%": {Token: lang.Rem, Precedence: 1}, + "<<": {Token: lang.Shl, Precedence: 1}, + ">>": {Token: lang.Shr, Precedence: 1}, + "+": {Token: lang.Add, Precedence: 2}, + "-": {Token: lang.Sub, Precedence: 2}, + "=": {Token: lang.Assign, Precedence: 6}, + "+=": {Token: lang.AddAssign, Precedence: 6}, + "<": {Token: lang.Less, Precedence: 3}, + ">": {Token: lang.Greater, Precedence: 3}, + "^": {Token: lang.Xor, Precedence: 2}, + "~": {Token: lang.Tilde}, + "&&": {Token: lang.Land, Precedence: 4}, + "||": {Token: lang.Lor, Precedence: 5}, + ":=": {Token: lang.Define, Precedence: 6}, + "==": {Token: lang.Equal, Precedence: 3}, + "<=": {Token: lang.LessEqual, Precedence: 3}, + ">=": {Token: lang.GreaterEqual, Precedence: 3}, + "->": {Token: lang.Arrow}, + "!": {Token: lang.Not}, + "++": {Token: lang.Inc, SkipSemi: true}, + "--": {Token: lang.Dec, SkipSemi: true}, // Reserved keywords - "break": {TokenId: lang.Break}, - "case": {TokenId: lang.Case, SkipSemi: true}, - "chan": {TokenId: lang.Chan, SkipSemi: true}, - "const": {TokenId: lang.Const, SkipSemi: true}, - "continue": {TokenId: lang.Continue}, - "default": {TokenId: lang.Case, SkipSemi: true}, - "defer": {TokenId: lang.Defer, SkipSemi: true}, - "else": {TokenId: lang.Else, SkipSemi: true}, - "fallthrough": {TokenId: lang.Fallthrough}, - "for": {TokenId: lang.For, SkipSemi: true}, - "func": {TokenId: lang.Func, SkipSemi: true}, - "go": {TokenId: lang.Go, SkipSemi: true}, - "goto": {TokenId: lang.Goto, SkipSemi: true}, - "if": {TokenId: lang.If, SkipSemi: true}, - "import": {TokenId: lang.Import, SkipSemi: true}, - "interface": {TokenId: lang.Interface, SkipSemi: true}, - "map": {TokenId: lang.Map, SkipSemi: true}, - "package": {TokenId: lang.Package, SkipSemi: true}, - "range": {TokenId: lang.Range, SkipSemi: true}, - "return": {TokenId: lang.Return}, - "select": {TokenId: lang.Select, SkipSemi: true}, - "struct": {TokenId: lang.Struct, SkipSemi: true}, - "switch": {TokenId: lang.Switch, SkipSemi: true}, - "type": {TokenId: lang.Type, SkipSemi: true}, - "var": {TokenId: lang.Var, SkipSemi: true}, + "break": {Token: lang.Break}, + "case": {Token: lang.Case, SkipSemi: true}, + "chan": {Token: lang.Chan, SkipSemi: true}, + "const": {Token: lang.Const, SkipSemi: true}, + "continue": {Token: lang.Continue}, + "default": {Token: lang.Case, SkipSemi: true}, + "defer": {Token: lang.Defer, SkipSemi: true}, + "else": {Token: lang.Else, SkipSemi: true}, + "fallthrough": {Token: lang.Fallthrough}, + "for": {Token: lang.For, SkipSemi: true}, + "func": {Token: lang.Func, SkipSemi: true}, + "go": {Token: lang.Go, SkipSemi: true}, + "goto": {Token: lang.Goto, SkipSemi: true}, + "if": {Token: lang.If, SkipSemi: true}, + "import": {Token: lang.Import, SkipSemi: true}, + "interface": {Token: lang.Interface, SkipSemi: true}, + "map": {Token: lang.Map, SkipSemi: true}, + "package": {Token: lang.Package, SkipSemi: true}, + "range": {Token: lang.Range, SkipSemi: true}, + "return": {Token: lang.Return}, + "select": {Token: lang.Select, SkipSemi: true}, + "struct": {Token: lang.Struct, SkipSemi: true}, + "switch": {Token: lang.Switch, SkipSemi: true}, + "type": {Token: lang.Type, SkipSemi: true}, + "var": {Token: lang.Var, SkipSemi: true}, }, } diff --git a/lang/spec.go b/lang/spec.go index a910f70..92d90f7 100644 --- a/lang/spec.go +++ b/lang/spec.go @@ -1,5 +1,7 @@ +// Package lang provides tokens for possibly multiple languages. package lang +// Lexical properties of tokens to allow scanning. const ( CharIllegal = 1 << iota CharOp @@ -16,26 +18,29 @@ const ( EosValidEnd // end of input string terminates block or string token ) +// ASCIILen is the length of the ASCII characters set. const ASCIILen = 1 << 7 // 128 +// TokenProp represent token properties for parsing. type TokenProp struct { - TokenId + Token SkipSemi bool // automatic semicolon insertion after newline Precedence int // operator precedence } +// Spec represents the token specification for scanning. type Spec struct { CharProp [ASCIILen]uint // special Character properties End map[string]string // end delimiters, indexed by start BlockProp map[string]uint // block properties TokenProps map[string]TokenProp // token properties DotNum bool // true if a number can start with '.' - IdAscii bool // true if an identifier can be in ASCII only - Num_ bool // true if a number can contain _ character + IdentASCII bool // true if an identifier can be in ASCII only + NumUnder bool // true if a number can contain _ character } -// HasInit stores if a statement may contain a simple init statement -var HasInit = map[TokenId]bool{ +// HasInit stores if a statement may contain a simple init statement. +var HasInit = map[Token]bool{ Case: true, For: true, If: true, diff --git a/lang/token.go b/lang/token.go index 613f2c6..7ad7bf1 100644 --- a/lang/token.go +++ b/lang/token.go @@ -1,23 +1,25 @@ package lang -//go:generate stringer -type=TokenId +//go:generate stringer -type=Token -type TokenId int +// Token represents a lexical token. +type Token int +// All known tokens for the set of supported languages. const ( - Illegal TokenId = iota + Illegal Token = iota Comment Ident - // Literal values + // Literal values. Char Float Imag Int String - // Binary operators (except indicated) - // Arithmetic and bitwise binary operators + // Binary operators (except indicated). + // Arithmetic and bitwise binary operators. Add // + Sub // - Mul // * @@ -31,7 +33,7 @@ const ( AndNot // &^ Period // . - // Binary operators returning a boolean + // Binary operators returning a boolean. Equal // == Greater // > GreaterEqual // >= @@ -41,7 +43,7 @@ const ( Lor // || NotEqual // != - // Assigment operators (arithmetic and bitwise) + // Assigment operators (arithmetic and bitwise). Define // := Assign // = AddAssign // += @@ -58,7 +60,7 @@ const ( Inc // ++ Dec // -- - // Unary operations + // Unary operations. Plus // unary + Minus // unary - Addr // unary & @@ -69,17 +71,17 @@ const ( Not // unary ! Tilde // unary ~ (underlying type) - // Separators (punctuation) + // Separators (punctuation). Comma // , Semicolon // ; Colon // : - // Block tokens + // Block tokens. ParenBlock // (..) BracketBlock // [..] BraceBlock // {..} - // Reserved keywords + // Reserved keywords. Break Case Chan @@ -106,7 +108,7 @@ const ( Type Var - // Internal virtual machine tokens (no corresponding keyword) + // Internal virtual machine tokens (no corresponding keyword). Call CallX EqualSet @@ -119,8 +121,9 @@ const ( New ) -// TODO: define UnaryOp per language -var UnaryOp = map[TokenId]TokenId{ +// UnaryOp contains the set of unary operators. +// TODO: define UnaryOp per language. +var UnaryOp = map[Token]Token{ Add: Plus, // + And: Addr, // & Not: Not, // ! @@ -130,11 +133,26 @@ var UnaryOp = map[TokenId]TokenId{ Xor: BitComp, // ^ } -func (t TokenId) IsKeyword() bool { return t >= Break && t <= Var } -func (t TokenId) IsLiteral() bool { return t >= Char && t <= String } -func (t TokenId) IsOperator() bool { return t >= Add && t <= Tilde } -func (t TokenId) IsBlock() bool { return t >= ParenBlock && t <= BraceBlock } -func (t TokenId) IsBoolOp() bool { return t >= Equal && t <= NotEqual || t == Not } -func (t TokenId) IsBinaryOp() bool { return t >= Add && t <= NotEqual } -func (t TokenId) IsUnaryOp() bool { return t >= Plus && t <= Tilde } -func (t TokenId) IsLogicalOp() bool { return t == Land || t == Lor } +// IsKeyword returns true if t is a keyword. +func (t Token) IsKeyword() bool { return t >= Break && t <= Var } + +// IsLiteral returns true if t is a literal value. +func (t Token) IsLiteral() bool { return t >= Char && t <= String } + +// IsOperator returns true if t is an operator. +func (t Token) IsOperator() bool { return t >= Add && t <= Tilde } + +// IsBlock returns true if t is a block kind of token. +func (t Token) IsBlock() bool { return t >= ParenBlock && t <= BraceBlock } + +// IsBoolOp returns true if t is boolean operator. +func (t Token) IsBoolOp() bool { return t >= Equal && t <= NotEqual || t == Not } + +// IsBinaryOp returns true if t is a binary operator (takes 2 operands). +func (t Token) IsBinaryOp() bool { return t >= Add && t <= NotEqual } + +// IsUnaryOp returns true if t is an unary operator (takes 1 operand). +func (t Token) IsUnaryOp() bool { return t >= Plus && t <= Tilde } + +// IsLogicalOp returns true if t is a logical operator. +func (t Token) IsLogicalOp() bool { return t == Land || t == Lor } diff --git a/lang/token_string.go b/lang/token_string.go new file mode 100644 index 0000000..6b19fca --- /dev/null +++ b/lang/token_string.go @@ -0,0 +1,115 @@ +// Code generated by "stringer -type=Token"; DO NOT EDIT. + +package lang + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Illegal-0] + _ = x[Comment-1] + _ = x[Ident-2] + _ = x[Char-3] + _ = x[Float-4] + _ = x[Imag-5] + _ = x[Int-6] + _ = x[String-7] + _ = x[Add-8] + _ = x[Sub-9] + _ = x[Mul-10] + _ = x[Quo-11] + _ = x[Rem-12] + _ = x[And-13] + _ = x[Or-14] + _ = x[Xor-15] + _ = x[Shl-16] + _ = x[Shr-17] + _ = x[AndNot-18] + _ = x[Period-19] + _ = x[Equal-20] + _ = x[Greater-21] + _ = x[GreaterEqual-22] + _ = x[Land-23] + _ = x[Less-24] + _ = x[LessEqual-25] + _ = x[Lor-26] + _ = x[NotEqual-27] + _ = x[Define-28] + _ = x[Assign-29] + _ = x[AddAssign-30] + _ = x[SubAssign-31] + _ = x[MulAssign-32] + _ = x[QuoAssign-33] + _ = x[RemAssign-34] + _ = x[AndAssign-35] + _ = x[OrAssign-36] + _ = x[XorAssign-37] + _ = x[ShlAssign-38] + _ = x[ShrAssign-39] + _ = x[AndNotAssign-40] + _ = x[Inc-41] + _ = x[Dec-42] + _ = x[Plus-43] + _ = x[Minus-44] + _ = x[Addr-45] + _ = x[Deref-46] + _ = x[BitComp-47] + _ = x[Arrow-48] + _ = x[Ellipsis-49] + _ = x[Not-50] + _ = x[Tilde-51] + _ = x[Comma-52] + _ = x[Semicolon-53] + _ = x[Colon-54] + _ = x[ParenBlock-55] + _ = x[BracketBlock-56] + _ = x[BraceBlock-57] + _ = x[Break-58] + _ = x[Case-59] + _ = x[Chan-60] + _ = x[Const-61] + _ = x[Continue-62] + _ = x[Default-63] + _ = x[Defer-64] + _ = x[Else-65] + _ = x[Fallthrough-66] + _ = x[For-67] + _ = x[Func-68] + _ = x[Go-69] + _ = x[Goto-70] + _ = x[If-71] + _ = x[Import-72] + _ = x[Interface-73] + _ = x[Map-74] + _ = x[Package-75] + _ = x[Range-76] + _ = x[Return-77] + _ = x[Select-78] + _ = x[Struct-79] + _ = x[Switch-80] + _ = x[Type-81] + _ = x[Var-82] + _ = x[Call-83] + _ = x[CallX-84] + _ = x[EqualSet-85] + _ = x[Grow-86] + _ = x[Index-87] + _ = x[JumpFalse-88] + _ = x[JumpSetFalse-89] + _ = x[JumpSetTrue-90] + _ = x[Label-91] + _ = x[New-92] +} + +const _Token_name = "IllegalCommentIdentCharFloatImagIntStringAddSubMulQuoRemAndOrXorShlShrAndNotPeriodEqualGreaterGreaterEqualLandLessLessEqualLorNotEqualDefineAssignAddAssignSubAssignMulAssignQuoAssignRemAssignAndAssignOrAssignXorAssignShlAssignShrAssignAndNotAssignIncDecPlusMinusAddrDerefBitCompArrowEllipsisNotTildeCommaSemicolonColonParenBlockBracketBlockBraceBlockBreakCaseChanConstContinueDefaultDeferElseFallthroughForFuncGoGotoIfImportInterfaceMapPackageRangeReturnSelectStructSwitchTypeVarCallCallXEqualSetGrowIndexJumpFalseJumpSetFalseJumpSetTrueLabelNew" + +var _Token_index = [...]uint16{0, 7, 14, 19, 23, 28, 32, 35, 41, 44, 47, 50, 53, 56, 59, 61, 64, 67, 70, 76, 82, 87, 94, 106, 110, 114, 123, 126, 134, 140, 146, 155, 164, 173, 182, 191, 200, 208, 217, 226, 235, 247, 250, 253, 257, 262, 266, 271, 278, 283, 291, 294, 299, 304, 313, 318, 328, 340, 350, 355, 359, 363, 368, 376, 383, 388, 392, 403, 406, 410, 412, 416, 418, 424, 433, 436, 443, 448, 454, 460, 466, 472, 476, 479, 483, 488, 496, 500, 505, 514, 526, 537, 542, 545} + +func (i Token) String() string { + if i < 0 || i >= Token(len(_Token_index)-1) { + return "Token(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Token_name[_Token_index[i]:_Token_index[i+1]] +} diff --git a/lang/tokenid_string.go b/lang/tokenid_string.go deleted file mode 100644 index 705edc6..0000000 --- a/lang/tokenid_string.go +++ /dev/null @@ -1,115 +0,0 @@ -// Code generated by "stringer -type=TokenId"; DO NOT EDIT. - -package lang - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[Illegal-0] - _ = x[Comment-1] - _ = x[Ident-2] - _ = x[Char-3] - _ = x[Float-4] - _ = x[Imag-5] - _ = x[Int-6] - _ = x[String-7] - _ = x[Add-8] - _ = x[Sub-9] - _ = x[Mul-10] - _ = x[Quo-11] - _ = x[Rem-12] - _ = x[And-13] - _ = x[Or-14] - _ = x[Xor-15] - _ = x[Shl-16] - _ = x[Shr-17] - _ = x[AndNot-18] - _ = x[Period-19] - _ = x[Equal-20] - _ = x[Greater-21] - _ = x[GreaterEqual-22] - _ = x[Land-23] - _ = x[Less-24] - _ = x[LessEqual-25] - _ = x[Lor-26] - _ = x[NotEqual-27] - _ = x[Define-28] - _ = x[Assign-29] - _ = x[AddAssign-30] - _ = x[SubAssign-31] - _ = x[MulAssign-32] - _ = x[QuoAssign-33] - _ = x[RemAssign-34] - _ = x[AndAssign-35] - _ = x[OrAssign-36] - _ = x[XorAssign-37] - _ = x[ShlAssign-38] - _ = x[ShrAssign-39] - _ = x[AndNotAssign-40] - _ = x[Inc-41] - _ = x[Dec-42] - _ = x[Plus-43] - _ = x[Minus-44] - _ = x[Addr-45] - _ = x[Deref-46] - _ = x[BitComp-47] - _ = x[Arrow-48] - _ = x[Ellipsis-49] - _ = x[Not-50] - _ = x[Tilde-51] - _ = x[Comma-52] - _ = x[Semicolon-53] - _ = x[Colon-54] - _ = x[ParenBlock-55] - _ = x[BracketBlock-56] - _ = x[BraceBlock-57] - _ = x[Break-58] - _ = x[Case-59] - _ = x[Chan-60] - _ = x[Const-61] - _ = x[Continue-62] - _ = x[Default-63] - _ = x[Defer-64] - _ = x[Else-65] - _ = x[Fallthrough-66] - _ = x[For-67] - _ = x[Func-68] - _ = x[Go-69] - _ = x[Goto-70] - _ = x[If-71] - _ = x[Import-72] - _ = x[Interface-73] - _ = x[Map-74] - _ = x[Package-75] - _ = x[Range-76] - _ = x[Return-77] - _ = x[Select-78] - _ = x[Struct-79] - _ = x[Switch-80] - _ = x[Type-81] - _ = x[Var-82] - _ = x[Call-83] - _ = x[CallX-84] - _ = x[EqualSet-85] - _ = x[Grow-86] - _ = x[Index-87] - _ = x[JumpFalse-88] - _ = x[JumpSetFalse-89] - _ = x[JumpSetTrue-90] - _ = x[Label-91] - _ = x[New-92] -} - -const _TokenId_name = "IllegalCommentIdentCharFloatImagIntStringAddSubMulQuoRemAndOrXorShlShrAndNotPeriodEqualGreaterGreaterEqualLandLessLessEqualLorNotEqualDefineAssignAddAssignSubAssignMulAssignQuoAssignRemAssignAndAssignOrAssignXorAssignShlAssignShrAssignAndNotAssignIncDecPlusMinusAddrDerefBitCompArrowEllipsisNotTildeCommaSemicolonColonParenBlockBracketBlockBraceBlockBreakCaseChanConstContinueDefaultDeferElseFallthroughForFuncGoGotoIfImportInterfaceMapPackageRangeReturnSelectStructSwitchTypeVarCallCallXEqualSetGrowIndexJumpFalseJumpSetFalseJumpSetTrueLabelNew" - -var _TokenId_index = [...]uint16{0, 7, 14, 19, 23, 28, 32, 35, 41, 44, 47, 50, 53, 56, 59, 61, 64, 67, 70, 76, 82, 87, 94, 106, 110, 114, 123, 126, 134, 140, 146, 155, 164, 173, 182, 191, 200, 208, 217, 226, 235, 247, 250, 253, 257, 262, 266, 271, 278, 283, 291, 294, 299, 304, 313, 318, 328, 340, 350, 355, 359, 363, 368, 376, 383, 388, 392, 403, 406, 410, 412, 416, 418, 424, 433, 436, 443, 448, 454, 460, 466, 472, 476, 479, 483, 488, 496, 500, 505, 514, 526, 537, 542, 545} - -func (i TokenId) String() string { - if i < 0 || i >= TokenId(len(_TokenId_index)-1) { - return "TokenId(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _TokenId_name[_TokenId_index[i]:_TokenId_index[i+1]] -} -- cgit v1.2.3