diff options
Diffstat (limited to 'vendor/github.com/d5/tengo/compiler/scanner')
3 files changed, 696 insertions, 0 deletions
diff --git a/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go b/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go new file mode 100644 index 00000000..379f0196 --- /dev/null +++ b/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go @@ -0,0 +1,6 @@ +package scanner + +import "github.com/d5/tengo/compiler/source" + +// ErrorHandler is an error handler for the scanner. +type ErrorHandler func(pos source.FilePos, msg string) diff --git a/vendor/github.com/d5/tengo/compiler/scanner/mode.go b/vendor/github.com/d5/tengo/compiler/scanner/mode.go new file mode 100644 index 00000000..f67ceaf8 --- /dev/null +++ b/vendor/github.com/d5/tengo/compiler/scanner/mode.go @@ -0,0 +1,10 @@ +package scanner + +// Mode represents a scanner mode. +type Mode int + +// List of scanner modes. +const ( + ScanComments Mode = 1 << iota + DontInsertSemis +) diff --git a/vendor/github.com/d5/tengo/compiler/scanner/scanner.go b/vendor/github.com/d5/tengo/compiler/scanner/scanner.go new file mode 100644 index 00000000..387cd8ee --- /dev/null +++ b/vendor/github.com/d5/tengo/compiler/scanner/scanner.go @@ -0,0 +1,680 @@ +/* + Scanner reads the Tengo source text and tokenize them. + + Scanner is a modified version of Go's scanner implementation. + + Copyright 2009 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +*/ + +package scanner + +import ( + "fmt" + "unicode" + "unicode/utf8" + + "github.com/d5/tengo/compiler/source" + "github.com/d5/tengo/compiler/token" +) + +// byte order mark +const bom = 0xFEFF + +// Scanner reads the Tengo source text. +type Scanner struct { + file *source.File // source file handle + src []byte // source + ch rune // current character + offset int // character offset + readOffset int // reading offset (position after current character) + lineOffset int // current line offset + insertSemi bool // insert a semicolon before next newline + errorHandler ErrorHandler // error reporting; or nil + errorCount int // number of errors encountered + mode Mode +} + +// NewScanner creates a Scanner. +func NewScanner(file *source.File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner { + if file.Size != len(src) { + panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size, len(src))) + } + + s := &Scanner{ + file: file, + src: src, + errorHandler: errorHandler, + ch: ' ', + mode: mode, + } + + s.next() + if s.ch == bom { + s.next() // ignore BOM at file beginning + } + + return s +} + +// ErrorCount returns the number of errors. +func (s *Scanner) ErrorCount() int { + return s.errorCount +} + +// Scan returns a token, token literal and its position. +func (s *Scanner) Scan() (tok token.Token, literal string, pos source.Pos) { + s.skipWhitespace() + + pos = s.file.FileSetPos(s.offset) + + insertSemi := false + + // determine token value + switch ch := s.ch; { + case isLetter(ch): + literal = s.scanIdentifier() + tok = token.Lookup(literal) + switch tok { + case token.Ident, token.Break, token.Continue, token.Return, token.Export, token.True, token.False, token.Undefined: + insertSemi = true + } + case '0' <= ch && ch <= '9': + insertSemi = true + tok, literal = s.scanNumber(false) + default: + s.next() // always make progress + + switch ch { + case -1: // EOF + if s.insertSemi { + s.insertSemi = false // EOF consumed + return token.Semicolon, "\n", pos + } + tok = token.EOF + case '\n': + // we only reach here if s.insertSemi was set in the first place + s.insertSemi = false // newline consumed + return token.Semicolon, "\n", pos + case '"': + insertSemi = true + tok = token.String + literal = s.scanString() + case '\'': + insertSemi = true + tok = token.Char + literal = s.scanRune() + case '`': + insertSemi = true + tok = token.String + literal = s.scanRawString() + case ':': + tok = s.switch2(token.Colon, token.Define) + case '.': + if '0' <= s.ch && s.ch <= '9' { + insertSemi = true + tok, literal = s.scanNumber(true) + } else { + tok = token.Period + if s.ch == '.' && s.peek() == '.' { + s.next() + s.next() // consume last '.' + tok = token.Ellipsis + } + } + case ',': + tok = token.Comma + case '?': + tok = token.Question + case ';': + tok = token.Semicolon + literal = ";" + case '(': + tok = token.LParen + case ')': + insertSemi = true + tok = token.RParen + case '[': + tok = token.LBrack + case ']': + insertSemi = true + tok = token.RBrack + case '{': + tok = token.LBrace + case '}': + insertSemi = true + tok = token.RBrace + case '+': + tok = s.switch3(token.Add, token.AddAssign, '+', token.Inc) + if tok == token.Inc { + insertSemi = true + } + case '-': + tok = s.switch3(token.Sub, token.SubAssign, '-', token.Dec) + if tok == token.Dec { + insertSemi = true + } + case '*': + tok = s.switch2(token.Mul, token.MulAssign) + case '/': + if s.ch == '/' || s.ch == '*' { + // comment + if s.insertSemi && s.findLineEnd() { + // reset position to the beginning of the comment + s.ch = '/' + s.offset = s.file.Offset(pos) + s.readOffset = s.offset + 1 + s.insertSemi = false // newline consumed + return token.Semicolon, "\n", pos + } + comment := s.scanComment() + if s.mode&ScanComments == 0 { + // skip comment + s.insertSemi = false // newline consumed + return s.Scan() + } + tok = token.Comment + literal = comment + } else { + tok = s.switch2(token.Quo, token.QuoAssign) + } + case '%': + tok = s.switch2(token.Rem, token.RemAssign) + case '^': + tok = s.switch2(token.Xor, token.XorAssign) + case '<': + tok = s.switch4(token.Less, token.LessEq, '<', token.Shl, token.ShlAssign) + case '>': + tok = s.switch4(token.Greater, token.GreaterEq, '>', token.Shr, token.ShrAssign) + case '=': + tok = s.switch2(token.Assign, token.Equal) + case '!': + tok = s.switch2(token.Not, token.NotEqual) + case '&': + if s.ch == '^' { + s.next() + tok = s.switch2(token.AndNot, token.AndNotAssign) + } else { + tok = s.switch3(token.And, token.AndAssign, '&', token.LAnd) + } + case '|': + tok = s.switch3(token.Or, token.OrAssign, '|', token.LOr) + default: + // next reports unexpected BOMs - don't repeat + if ch != bom { + s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch)) + } + insertSemi = s.insertSemi // preserve insertSemi info + tok = token.Illegal + literal = string(ch) + } + } + + if s.mode&DontInsertSemis == 0 { + s.insertSemi = insertSemi + } + + return +} + +func (s *Scanner) next() { + if s.readOffset < len(s.src) { + s.offset = s.readOffset + if s.ch == '\n' { + s.lineOffset = s.offset + s.file.AddLine(s.offset) + } + r, w := rune(s.src[s.readOffset]), 1 + switch { + case r == 0: + s.error(s.offset, "illegal character NUL") + case r >= utf8.RuneSelf: + // not ASCII + r, w = utf8.DecodeRune(s.src[s.readOffset:]) + if r == utf8.RuneError && w == 1 { + s.error(s.offset, "illegal UTF-8 encoding") + } else if r == bom && s.offset > 0 { + s.error(s.offset, "illegal byte order mark") + } + } + s.readOffset += w + s.ch = r + } else { + s.offset = len(s.src) + if s.ch == '\n' { + s.lineOffset = s.offset + s.file.AddLine(s.offset) + } + s.ch = -1 // eof + } +} + +func (s *Scanner) peek() byte { + if s.readOffset < len(s.src) { + return s.src[s.readOffset] + } + + return 0 +} + +func (s *Scanner) error(offset int, msg string) { + if s.errorHandler != nil { + s.errorHandler(s.file.Position(s.file.FileSetPos(offset)), msg) + } + + s.errorCount++ +} + +func (s *Scanner) scanComment() string { + // initial '/' already consumed; s.ch == '/' || s.ch == '*' + offs := s.offset - 1 // position of initial '/' + var numCR int + + if s.ch == '/' { + //-style comment + // (the final '\n' is not considered part of the comment) + s.next() + for s.ch != '\n' && s.ch >= 0 { + if s.ch == '\r' { + numCR++ + } + s.next() + } + goto exit + } + + /*-style comment */ + s.next() + for s.ch >= 0 { + ch := s.ch + if ch == '\r' { + numCR++ + } + s.next() + if ch == '*' && s.ch == '/' { + s.next() + goto exit + } + } + + s.error(offs, "comment not terminated") + +exit: + lit := s.src[offs:s.offset] + + // On Windows, a (//-comment) line may end in "\r\n". + // Remove the final '\r' before analyzing the text for line directives (matching the compiler). + // Remove any other '\r' afterwards (matching the pre-existing behavior of the scanner). + if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' { + lit = lit[:len(lit)-1] + numCR-- + } + + if numCR > 0 { + lit = StripCR(lit, lit[1] == '*') + } + + return string(lit) +} + +func (s *Scanner) findLineEnd() bool { + // initial '/' already consumed + + defer func(offs int) { + // reset scanner state to where it was upon calling findLineEnd + s.ch = '/' + s.offset = offs + s.readOffset = offs + 1 + s.next() // consume initial '/' again + }(s.offset - 1) + + // read ahead until a newline, EOF, or non-comment tok is found + for s.ch == '/' || s.ch == '*' { + if s.ch == '/' { + //-style comment always contains a newline + return true + } + /*-style comment: look for newline */ + s.next() + for s.ch >= 0 { + ch := s.ch + if ch == '\n' { + return true + } + s.next() + if ch == '*' && s.ch == '/' { + s.next() + break + } + } + s.skipWhitespace() // s.insertSemi is set + if s.ch < 0 || s.ch == '\n' { + return true + } + if s.ch != '/' { + // non-comment tok + return false + } + s.next() // consume '/' + } + + return false +} + +func (s *Scanner) scanIdentifier() string { + offs := s.offset + for isLetter(s.ch) || isDigit(s.ch) { + s.next() + } + + return string(s.src[offs:s.offset]) +} + +func (s *Scanner) scanMantissa(base int) { + for digitVal(s.ch) < base { + s.next() + } +} + +func (s *Scanner) scanNumber(seenDecimalPoint bool) (tok token.Token, lit string) { + // digitVal(s.ch) < 10 + offs := s.offset + tok = token.Int + + defer func() { + lit = string(s.src[offs:s.offset]) + }() + + if seenDecimalPoint { + offs-- + tok = token.Float + s.scanMantissa(10) + goto exponent + } + + if s.ch == '0' { + // int or float + offs := s.offset + s.next() + if s.ch == 'x' || s.ch == 'X' { + // hexadecimal int + s.next() + s.scanMantissa(16) + if s.offset-offs <= 2 { + // only scanned "0x" or "0X" + s.error(offs, "illegal hexadecimal number") + } + } else { + // octal int or float + seenDecimalDigit := false + s.scanMantissa(8) + if s.ch == '8' || s.ch == '9' { + // illegal octal int or float + seenDecimalDigit = true + s.scanMantissa(10) + } + if s.ch == '.' || s.ch == 'e' || s.ch == 'E' || s.ch == 'i' { + goto fraction + } + // octal int + if seenDecimalDigit { + s.error(offs, "illegal octal number") + } + } + + return + } + + // decimal int or float + s.scanMantissa(10) + +fraction: + if s.ch == '.' { + tok = token.Float + s.next() + s.scanMantissa(10) + } + +exponent: + if s.ch == 'e' || s.ch == 'E' { + tok = token.Float + s.next() + if s.ch == '-' || s.ch == '+' { + s.next() + } + if digitVal(s.ch) < 10 { + s.scanMantissa(10) + } else { + s.error(offs, "illegal floating-point exponent") + } + } + + return +} + +func (s *Scanner) scanEscape(quote rune) bool { + offs := s.offset + + var n int + var base, max uint32 + switch s.ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: + s.next() + return true + case '0', '1', '2', '3', '4', '5', '6', '7': + n, base, max = 3, 8, 255 + case 'x': + s.next() + n, base, max = 2, 16, 255 + case 'u': + s.next() + n, base, max = 4, 16, unicode.MaxRune + case 'U': + s.next() + n, base, max = 8, 16, unicode.MaxRune + default: + msg := "unknown escape sequence" + if s.ch < 0 { + msg = "escape sequence not terminated" + } + s.error(offs, msg) + return false + } + + var x uint32 + for n > 0 { + d := uint32(digitVal(s.ch)) + if d >= base { + msg := fmt.Sprintf("illegal character %#U in escape sequence", s.ch) + if s.ch < 0 { + msg = "escape sequence not terminated" + } + s.error(s.offset, msg) + return false + } + x = x*base + d + s.next() + n-- + } + + if x > max || 0xD800 <= x && x < 0xE000 { + s.error(offs, "escape sequence is invalid Unicode code point") + return false + } + + return true +} + +func (s *Scanner) scanRune() string { + offs := s.offset - 1 // '\'' opening already consumed + + valid := true + n := 0 + for { + ch := s.ch + if ch == '\n' || ch < 0 { + // only report error if we don't have one already + if valid { + s.error(offs, "rune literal not terminated") + valid = false + } + break + } + s.next() + if ch == '\'' { + break + } + n++ + if ch == '\\' { + if !s.scanEscape('\'') { + valid = false + } + // continue to read to closing quote + } + } + + if valid && n != 1 { + s.error(offs, "illegal rune literal") + } + + return string(s.src[offs:s.offset]) +} + +func (s *Scanner) scanString() string { + offs := s.offset - 1 // '"' opening already consumed + + for { + ch := s.ch + if ch == '\n' || ch < 0 { + s.error(offs, "string literal not terminated") + break + } + s.next() + if ch == '"' { + break + } + if ch == '\\' { + s.scanEscape('"') + } + } + + return string(s.src[offs:s.offset]) +} + +func (s *Scanner) scanRawString() string { + offs := s.offset - 1 // '`' opening already consumed + + hasCR := false + for { + ch := s.ch + if ch < 0 { + s.error(offs, "raw string literal not terminated") + break + } + + s.next() + + if ch == '`' { + break + } + + if ch == '\r' { + hasCR = true + } + } + + lit := s.src[offs:s.offset] + if hasCR { + lit = StripCR(lit, false) + } + + return string(lit) +} + +// StripCR removes carriage return characters. +func StripCR(b []byte, comment bool) []byte { + c := make([]byte, len(b)) + + i := 0 + for j, ch := range b { + // In a /*-style comment, don't strip \r from *\r/ (incl. sequences of \r from *\r\r...\r/) + // since the resulting */ would terminate the comment too early unless the \r is immediately + // following the opening /* in which case it's ok because /*/ is not closed yet. + if ch != '\r' || comment && i > len("/*") && c[i-1] == '*' && j+1 < len(b) && b[j+1] == '/' { + c[i] = ch + i++ + } + } + + return c[:i] +} + +func (s *Scanner) skipWhitespace() { + for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || s.ch == '\r' { + s.next() + } +} + +func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + + return tok0 +} + +func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + + if s.ch == ch2 { + s.next() + return tok2 + } + + return tok0 +} + +func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + + if s.ch == ch2 { + s.next() + if s.ch == '=' { + s.next() + return tok3 + } + + return tok2 + } + + return tok0 +} + +func isLetter(ch rune) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) +} + +func isDigit(ch rune) bool { + return '0' <= ch && ch <= '9' || ch >= utf8.RuneSelf && unicode.IsDigit(ch) +} + +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= ch && ch <= 'f': + return int(ch - 'a' + 10) + case 'A' <= ch && ch <= 'F': + return int(ch - 'A' + 10) + } + + return 16 // larger than any legal digit val +} |