diff options
Diffstat (limited to 'vendor/github.com/d5/tengo/compiler/scanner')
3 files changed, 0 insertions, 696 deletions
diff --git a/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go b/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go deleted file mode 100644 index 379f0196..00000000 --- a/vendor/github.com/d5/tengo/compiler/scanner/error_handler.go +++ /dev/null @@ -1,6 +0,0 @@ -package scanner - -import "github.com/d5/tengo/compiler/source" - -// ErrorHandler is an error handler for the scanner. -type ErrorHandler func(pos source.FilePos, msg string) diff --git a/vendor/github.com/d5/tengo/compiler/scanner/mode.go b/vendor/github.com/d5/tengo/compiler/scanner/mode.go deleted file mode 100644 index f67ceaf8..00000000 --- a/vendor/github.com/d5/tengo/compiler/scanner/mode.go +++ /dev/null @@ -1,10 +0,0 @@ -package scanner - -// Mode represents a scanner mode. -type Mode int - -// List of scanner modes. -const ( - ScanComments Mode = 1 << iota - DontInsertSemis -) diff --git a/vendor/github.com/d5/tengo/compiler/scanner/scanner.go b/vendor/github.com/d5/tengo/compiler/scanner/scanner.go deleted file mode 100644 index 387cd8ee..00000000 --- a/vendor/github.com/d5/tengo/compiler/scanner/scanner.go +++ /dev/null @@ -1,680 +0,0 @@ -/* - Scanner reads the Tengo source text and tokenize them. - - Scanner is a modified version of Go's scanner implementation. - - Copyright 2009 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package scanner - -import ( - "fmt" - "unicode" - "unicode/utf8" - - "github.com/d5/tengo/compiler/source" - "github.com/d5/tengo/compiler/token" -) - -// byte order mark -const bom = 0xFEFF - -// Scanner reads the Tengo source text. -type Scanner struct { - file *source.File // source file handle - src []byte // source - ch rune // current character - offset int // character offset - readOffset int // reading offset (position after current character) - lineOffset int // current line offset - insertSemi bool // insert a semicolon before next newline - errorHandler ErrorHandler // error reporting; or nil - errorCount int // number of errors encountered - mode Mode -} - -// NewScanner creates a Scanner. -func NewScanner(file *source.File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner { - if file.Size != len(src) { - panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size, len(src))) - } - - s := &Scanner{ - file: file, - src: src, - errorHandler: errorHandler, - ch: ' ', - mode: mode, - } - - s.next() - if s.ch == bom { - s.next() // ignore BOM at file beginning - } - - return s -} - -// ErrorCount returns the number of errors. -func (s *Scanner) ErrorCount() int { - return s.errorCount -} - -// Scan returns a token, token literal and its position. -func (s *Scanner) Scan() (tok token.Token, literal string, pos source.Pos) { - s.skipWhitespace() - - pos = s.file.FileSetPos(s.offset) - - insertSemi := false - - // determine token value - switch ch := s.ch; { - case isLetter(ch): - literal = s.scanIdentifier() - tok = token.Lookup(literal) - switch tok { - case token.Ident, token.Break, token.Continue, token.Return, token.Export, token.True, token.False, token.Undefined: - insertSemi = true - } - case '0' <= ch && ch <= '9': - insertSemi = true - tok, literal = s.scanNumber(false) - default: - s.next() // always make progress - - switch ch { - case -1: // EOF - if s.insertSemi { - s.insertSemi = false // EOF consumed - return token.Semicolon, "\n", pos - } - tok = token.EOF - case '\n': - // we only reach here if s.insertSemi was set in the first place - s.insertSemi = false // newline consumed - return token.Semicolon, "\n", pos - case '"': - insertSemi = true - tok = token.String - literal = s.scanString() - case '\'': - insertSemi = true - tok = token.Char - literal = s.scanRune() - case '`': - insertSemi = true - tok = token.String - literal = s.scanRawString() - case ':': - tok = s.switch2(token.Colon, token.Define) - case '.': - if '0' <= s.ch && s.ch <= '9' { - insertSemi = true - tok, literal = s.scanNumber(true) - } else { - tok = token.Period - if s.ch == '.' && s.peek() == '.' { - s.next() - s.next() // consume last '.' - tok = token.Ellipsis - } - } - case ',': - tok = token.Comma - case '?': - tok = token.Question - case ';': - tok = token.Semicolon - literal = ";" - case '(': - tok = token.LParen - case ')': - insertSemi = true - tok = token.RParen - case '[': - tok = token.LBrack - case ']': - insertSemi = true - tok = token.RBrack - case '{': - tok = token.LBrace - case '}': - insertSemi = true - tok = token.RBrace - case '+': - tok = s.switch3(token.Add, token.AddAssign, '+', token.Inc) - if tok == token.Inc { - insertSemi = true - } - case '-': - tok = s.switch3(token.Sub, token.SubAssign, '-', token.Dec) - if tok == token.Dec { - insertSemi = true - } - case '*': - tok = s.switch2(token.Mul, token.MulAssign) - case '/': - if s.ch == '/' || s.ch == '*' { - // comment - if s.insertSemi && s.findLineEnd() { - // reset position to the beginning of the comment - s.ch = '/' - s.offset = s.file.Offset(pos) - s.readOffset = s.offset + 1 - s.insertSemi = false // newline consumed - return token.Semicolon, "\n", pos - } - comment := s.scanComment() - if s.mode&ScanComments == 0 { - // skip comment - s.insertSemi = false // newline consumed - return s.Scan() - } - tok = token.Comment - literal = comment - } else { - tok = s.switch2(token.Quo, token.QuoAssign) - } - case '%': - tok = s.switch2(token.Rem, token.RemAssign) - case '^': - tok = s.switch2(token.Xor, token.XorAssign) - case '<': - tok = s.switch4(token.Less, token.LessEq, '<', token.Shl, token.ShlAssign) - case '>': - tok = s.switch4(token.Greater, token.GreaterEq, '>', token.Shr, token.ShrAssign) - case '=': - tok = s.switch2(token.Assign, token.Equal) - case '!': - tok = s.switch2(token.Not, token.NotEqual) - case '&': - if s.ch == '^' { - s.next() - tok = s.switch2(token.AndNot, token.AndNotAssign) - } else { - tok = s.switch3(token.And, token.AndAssign, '&', token.LAnd) - } - case '|': - tok = s.switch3(token.Or, token.OrAssign, '|', token.LOr) - default: - // next reports unexpected BOMs - don't repeat - if ch != bom { - s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch)) - } - insertSemi = s.insertSemi // preserve insertSemi info - tok = token.Illegal - literal = string(ch) - } - } - - if s.mode&DontInsertSemis == 0 { - s.insertSemi = insertSemi - } - - return -} - -func (s *Scanner) next() { - if s.readOffset < len(s.src) { - s.offset = s.readOffset - if s.ch == '\n' { - s.lineOffset = s.offset - s.file.AddLine(s.offset) - } - r, w := rune(s.src[s.readOffset]), 1 - switch { - case r == 0: - s.error(s.offset, "illegal character NUL") - case r >= utf8.RuneSelf: - // not ASCII - r, w = utf8.DecodeRune(s.src[s.readOffset:]) - if r == utf8.RuneError && w == 1 { - s.error(s.offset, "illegal UTF-8 encoding") - } else if r == bom && s.offset > 0 { - s.error(s.offset, "illegal byte order mark") - } - } - s.readOffset += w - s.ch = r - } else { - s.offset = len(s.src) - if s.ch == '\n' { - s.lineOffset = s.offset - s.file.AddLine(s.offset) - } - s.ch = -1 // eof - } -} - -func (s *Scanner) peek() byte { - if s.readOffset < len(s.src) { - return s.src[s.readOffset] - } - - return 0 -} - -func (s *Scanner) error(offset int, msg string) { - if s.errorHandler != nil { - s.errorHandler(s.file.Position(s.file.FileSetPos(offset)), msg) - } - - s.errorCount++ -} - -func (s *Scanner) scanComment() string { - // initial '/' already consumed; s.ch == '/' || s.ch == '*' - offs := s.offset - 1 // position of initial '/' - var numCR int - - if s.ch == '/' { - //-style comment - // (the final '\n' is not considered part of the comment) - s.next() - for s.ch != '\n' && s.ch >= 0 { - if s.ch == '\r' { - numCR++ - } - s.next() - } - goto exit - } - - /*-style comment */ - s.next() - for s.ch >= 0 { - ch := s.ch - if ch == '\r' { - numCR++ - } - s.next() - if ch == '*' && s.ch == '/' { - s.next() - goto exit - } - } - - s.error(offs, "comment not terminated") - -exit: - lit := s.src[offs:s.offset] - - // On Windows, a (//-comment) line may end in "\r\n". - // Remove the final '\r' before analyzing the text for line directives (matching the compiler). - // Remove any other '\r' afterwards (matching the pre-existing behavior of the scanner). - if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' { - lit = lit[:len(lit)-1] - numCR-- - } - - if numCR > 0 { - lit = StripCR(lit, lit[1] == '*') - } - - return string(lit) -} - -func (s *Scanner) findLineEnd() bool { - // initial '/' already consumed - - defer func(offs int) { - // reset scanner state to where it was upon calling findLineEnd - s.ch = '/' - s.offset = offs - s.readOffset = offs + 1 - s.next() // consume initial '/' again - }(s.offset - 1) - - // read ahead until a newline, EOF, or non-comment tok is found - for s.ch == '/' || s.ch == '*' { - if s.ch == '/' { - //-style comment always contains a newline - return true - } - /*-style comment: look for newline */ - s.next() - for s.ch >= 0 { - ch := s.ch - if ch == '\n' { - return true - } - s.next() - if ch == '*' && s.ch == '/' { - s.next() - break - } - } - s.skipWhitespace() // s.insertSemi is set - if s.ch < 0 || s.ch == '\n' { - return true - } - if s.ch != '/' { - // non-comment tok - return false - } - s.next() // consume '/' - } - - return false -} - -func (s *Scanner) scanIdentifier() string { - offs := s.offset - for isLetter(s.ch) || isDigit(s.ch) { - s.next() - } - - return string(s.src[offs:s.offset]) -} - -func (s *Scanner) scanMantissa(base int) { - for digitVal(s.ch) < base { - s.next() - } -} - -func (s *Scanner) scanNumber(seenDecimalPoint bool) (tok token.Token, lit string) { - // digitVal(s.ch) < 10 - offs := s.offset - tok = token.Int - - defer func() { - lit = string(s.src[offs:s.offset]) - }() - - if seenDecimalPoint { - offs-- - tok = token.Float - s.scanMantissa(10) - goto exponent - } - - if s.ch == '0' { - // int or float - offs := s.offset - s.next() - if s.ch == 'x' || s.ch == 'X' { - // hexadecimal int - s.next() - s.scanMantissa(16) - if s.offset-offs <= 2 { - // only scanned "0x" or "0X" - s.error(offs, "illegal hexadecimal number") - } - } else { - // octal int or float - seenDecimalDigit := false - s.scanMantissa(8) - if s.ch == '8' || s.ch == '9' { - // illegal octal int or float - seenDecimalDigit = true - s.scanMantissa(10) - } - if s.ch == '.' || s.ch == 'e' || s.ch == 'E' || s.ch == 'i' { - goto fraction - } - // octal int - if seenDecimalDigit { - s.error(offs, "illegal octal number") - } - } - - return - } - - // decimal int or float - s.scanMantissa(10) - -fraction: - if s.ch == '.' { - tok = token.Float - s.next() - s.scanMantissa(10) - } - -exponent: - if s.ch == 'e' || s.ch == 'E' { - tok = token.Float - s.next() - if s.ch == '-' || s.ch == '+' { - s.next() - } - if digitVal(s.ch) < 10 { - s.scanMantissa(10) - } else { - s.error(offs, "illegal floating-point exponent") - } - } - - return -} - -func (s *Scanner) scanEscape(quote rune) bool { - offs := s.offset - - var n int - var base, max uint32 - switch s.ch { - case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: - s.next() - return true - case '0', '1', '2', '3', '4', '5', '6', '7': - n, base, max = 3, 8, 255 - case 'x': - s.next() - n, base, max = 2, 16, 255 - case 'u': - s.next() - n, base, max = 4, 16, unicode.MaxRune - case 'U': - s.next() - n, base, max = 8, 16, unicode.MaxRune - default: - msg := "unknown escape sequence" - if s.ch < 0 { - msg = "escape sequence not terminated" - } - s.error(offs, msg) - return false - } - - var x uint32 - for n > 0 { - d := uint32(digitVal(s.ch)) - if d >= base { - msg := fmt.Sprintf("illegal character %#U in escape sequence", s.ch) - if s.ch < 0 { - msg = "escape sequence not terminated" - } - s.error(s.offset, msg) - return false - } - x = x*base + d - s.next() - n-- - } - - if x > max || 0xD800 <= x && x < 0xE000 { - s.error(offs, "escape sequence is invalid Unicode code point") - return false - } - - return true -} - -func (s *Scanner) scanRune() string { - offs := s.offset - 1 // '\'' opening already consumed - - valid := true - n := 0 - for { - ch := s.ch - if ch == '\n' || ch < 0 { - // only report error if we don't have one already - if valid { - s.error(offs, "rune literal not terminated") - valid = false - } - break - } - s.next() - if ch == '\'' { - break - } - n++ - if ch == '\\' { - if !s.scanEscape('\'') { - valid = false - } - // continue to read to closing quote - } - } - - if valid && n != 1 { - s.error(offs, "illegal rune literal") - } - - return string(s.src[offs:s.offset]) -} - -func (s *Scanner) scanString() string { - offs := s.offset - 1 // '"' opening already consumed - - for { - ch := s.ch - if ch == '\n' || ch < 0 { - s.error(offs, "string literal not terminated") - break - } - s.next() - if ch == '"' { - break - } - if ch == '\\' { - s.scanEscape('"') - } - } - - return string(s.src[offs:s.offset]) -} - -func (s *Scanner) scanRawString() string { - offs := s.offset - 1 // '`' opening already consumed - - hasCR := false - for { - ch := s.ch - if ch < 0 { - s.error(offs, "raw string literal not terminated") - break - } - - s.next() - - if ch == '`' { - break - } - - if ch == '\r' { - hasCR = true - } - } - - lit := s.src[offs:s.offset] - if hasCR { - lit = StripCR(lit, false) - } - - return string(lit) -} - -// StripCR removes carriage return characters. -func StripCR(b []byte, comment bool) []byte { - c := make([]byte, len(b)) - - i := 0 - for j, ch := range b { - // In a /*-style comment, don't strip \r from *\r/ (incl. sequences of \r from *\r\r...\r/) - // since the resulting */ would terminate the comment too early unless the \r is immediately - // following the opening /* in which case it's ok because /*/ is not closed yet. - if ch != '\r' || comment && i > len("/*") && c[i-1] == '*' && j+1 < len(b) && b[j+1] == '/' { - c[i] = ch - i++ - } - } - - return c[:i] -} - -func (s *Scanner) skipWhitespace() { - for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || s.ch == '\r' { - s.next() - } -} - -func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token { - if s.ch == '=' { - s.next() - return tok1 - } - - return tok0 -} - -func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token { - if s.ch == '=' { - s.next() - return tok1 - } - - if s.ch == ch2 { - s.next() - return tok2 - } - - return tok0 -} - -func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token { - if s.ch == '=' { - s.next() - return tok1 - } - - if s.ch == ch2 { - s.next() - if s.ch == '=' { - s.next() - return tok3 - } - - return tok2 - } - - return tok0 -} - -func isLetter(ch rune) bool { - return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) -} - -func isDigit(ch rune) bool { - return '0' <= ch && ch <= '9' || ch >= utf8.RuneSelf && unicode.IsDigit(ch) -} - -func digitVal(ch rune) int { - switch { - case '0' <= ch && ch <= '9': - return int(ch - '0') - case 'a' <= ch && ch <= 'f': - return int(ch - 'a' + 10) - case 'A' <= ch && ch <= 'F': - return int(ch - 'A' + 10) - } - - return 16 // larger than any legal digit val -} |