// Copyright 2019 The CC Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cc // import "modernc.org/cc/v3"
import (
"bufio"
"bytes"
"fmt"
goscanner "go/scanner"
"io"
"path/filepath"
"strconv"
"strings"
"sync"
"unicode/utf8"
"modernc.org/mathutil"
"modernc.org/token"
)
const (
clsEOF = iota + 0x80
clsOther
)
const maxASCII = 0x7f
var (
bom = []byte{0xEF, 0xBB, 0xBF}
idDefine = dict.sid("define")
idElif = dict.sid("elif")
idElse = dict.sid("else")
idEndif = dict.sid("endif")
idError = dict.sid("error")
idIf = dict.sid("if")
idIfdef = dict.sid("ifdef")
idIfndef = dict.sid("ifndef")
idInclude = dict.sid("include")
idIncludeNext = dict.sid("include_next")
idLine = dict.sid("line")
idPragma = dict.sid("pragma")
idPragmaOp = dict.sid("_Pragma")
idSpace = dict.sid(" ")
idUndef = dict.sid("undef")
trigraphPrefix = []byte("??")
trigraphs = []struct{ from, to []byte }{
{[]byte("??="), []byte{'#'}},
{[]byte("??("), []byte{'['}},
{[]byte("??/"), []byte{'\\'}},
{[]byte("??)"), []byte{']'}},
{[]byte("??'"), []byte{'^'}},
{[]byte("??<"), []byte{'{'}},
{[]byte("??!"), []byte{'|'}},
{[]byte("??>"), []byte{'}'}},
{[]byte("??-"), []byte{'~'}},
}
)
type tokenFile struct {
*token.File
sync.RWMutex
}
func tokenNewFile(name string, sz int) *tokenFile { return &tokenFile{File: token.NewFile(name, sz)} }
func (f *tokenFile) Position(pos token.Pos) (r token.Position) {
f.RLock()
r = f.File.Position(pos)
f.RUnlock()
return r
}
func (f *tokenFile) PositionFor(pos token.Pos, adjusted bool) (r token.Position) {
f.RLock()
r = f.File.PositionFor(pos, adjusted)
f.RUnlock()
return r
}
func (f *tokenFile) AddLine(off int) {
f.Lock()
f.File.AddLine(off)
f.Unlock()
}
func (f *tokenFile) AddLineInfo(off int, fn string, line int) {
f.Lock()
f.File.AddLineInfo(off, fn, line)
f.Unlock()
}
type node interface {
Pos() token.Pos
}
type dictionary struct {
mu sync.RWMutex
m map[string]StringID
strings []string
}
func newDictionary() (r *dictionary) {
r = &dictionary{m: map[string]StringID{}}
b := make([]byte, 1)
for i := 0; i < 128; i++ {
var s string
if i != 0 {
b[0] = byte(i)
s = string(b)
}
r.m[s] = StringID(i)
r.strings = append(r.strings, s)
dictStrings[i] = s
}
return r
}
func (d *dictionary) id(key []byte) StringID {
switch len(key) {
case 0:
return 0
case 1:
if c := key[0]; c != 0 && c < 128 {
return StringID(c)
}
}
d.mu.Lock()
if n, ok := d.m[string(key)]; ok {
d.mu.Unlock()
return n
}
n := StringID(len(d.strings))
s := string(key)
if int(n) < 256 {
dictStrings[n] = s
}
d.strings = append(d.strings, s)
d.m[s] = n
d.mu.Unlock()
return n
}
func (d *dictionary) sid(key string) StringID {
switch len(key) {
case 0:
return 0
case 1:
if c := key[0]; c != 0 && c < 128 {
return StringID(c)
}
}
d.mu.Lock()
if n, ok := d.m[key]; ok {
d.mu.Unlock()
return n
}
n := StringID(len(d.strings))
if int(n) < 256 {
dictStrings[n] = key
}
d.strings = append(d.strings, key)
d.m[key] = n
d.mu.Unlock()
return n
}
type char struct {
pos int32
c byte
}
// token3 is produced by translation phase 3.
type token3 struct {
char rune
pos int32
value StringID
src StringID
macro StringID
}
func (t token3) Pos() token.Pos { return token.Pos(t.pos) }
func (t token3) String() string { return t.value.String() }
type scanner struct {
bomFix int
bytesBuf []byte
charBuf []char
ctx *context
file *tokenFile
fileOffset int
firstPos token.Pos
lineBuf []byte
lookaheadChar char
lookaheadLine ppLine
mark int
pos token.Pos
r *bufio.Reader
srcBuf []byte
tokenBuf []token3
ungetBuf []char
tok token3
closed bool
preserveWhiteSpace bool
}
func newScanner0(ctx *context, r io.Reader, file *tokenFile, bufSize int) *scanner {
s := &scanner{
ctx: ctx,
file: file,
r: bufio.NewReaderSize(r, bufSize),
}
if r != nil {
s.init()
}
return s
}
func newScanner(ctx *context, r io.Reader, file *tokenFile) *scanner {
bufSize := 1 << 17 // emulate gcc
if n := ctx.cfg.MaxSourceLine; n > 4096 {
bufSize = n
}
return newScanner0(ctx, r, file, bufSize)
}
func (s *scanner) abort() (r byte, b bool) {
if s.mark >= 0 {
if len(s.charBuf) > s.mark {
s.unget(s.lookaheadChar)
for i := len(s.charBuf) - 1; i >= s.mark; i-- {
s.unget(s.charBuf[i])
}
}
s.charBuf = s.charBuf[:s.mark]
return 0, false
}
switch n := len(s.charBuf); n {
case 0: // [] z
c := s.lookaheadChar
s.next()
return s.class(c.c), true
case 1: // [a] z
return s.class(s.charBuf[0].c), true
default: // [a, b, ...], z
c := s.charBuf[0] // a
s.unget(s.lookaheadChar) // z
for i := n - 1; i > 1; i-- {
s.unget(s.charBuf[i]) // ...
}
s.lookaheadChar = s.charBuf[1] // b
s.charBuf = s.charBuf[:1]
return s.class(c.c), true
}
}
func (s *scanner) class(b byte) byte {
switch {
case b == 0:
return clsEOF
case b > maxASCII:
return clsOther
default:
return b
}
}
func (s *scanner) err(n node, msg string, args ...interface{}) { s.errPos(n.Pos(), msg, args...) }
func (s *scanner) errLine(x interface{}, msg string, args ...interface{}) {
var toks []token3
switch x := x.(type) {
case nil:
toks = []token3{{}}
case ppLine:
toks = x.getToks()
default:
panic(internalError())
}
var b strings.Builder
for _, v := range toks {
switch v.char {
case '\n':
// nop
case ' ':
b.WriteByte(' ')
default:
b.WriteString(v.String())
}
}
s.err(toks[0], "%s"+msg, append([]interface{}{b.String()}, args...)...)
}
func (s *scanner) errPos(pos token.Pos, msg string, args ...interface{}) {
if s.ctx.err(s.file.Position(pos), msg, args...) {
s.r.Reset(nil)
s.closed = true
}
}
func (s *scanner) init() *scanner {
if s.r == nil {
return s
}
b, err := s.r.Peek(3)
if err == nil && bytes.Equal(b, bom) {
s.bomFix, _ = s.r.Discard(3)
}
s.tokenBuf = nil
return s
}
func (s *scanner) initScan() (r byte) {
if s.lookaheadChar.pos == 0 {
s.next()
}
s.firstPos = token.Pos(s.lookaheadChar.pos)
s.mark = -1
if len(s.charBuf) > 1<<18 { //DONE benchmark tuned
s.bytesBuf = nil
s.charBuf = nil
s.srcBuf = nil
} else {
s.bytesBuf = s.bytesBuf[:0]
s.charBuf = s.charBuf[:0]
s.srcBuf = s.bytesBuf[:0]
}
return s.class(s.lookaheadChar.c)
}
func (s *scanner) lex() {
s.tok.char = s.scan()
s.tok.pos = int32(s.firstPos)
for _, v := range s.charBuf {
s.srcBuf = append(s.srcBuf, v.c)
}
s.tok.src = dict.id(s.srcBuf)
switch {
case s.tok.char == ' ' && !s.preserveWhiteSpace && !s.ctx.cfg.PreserveWhiteSpace:
s.tok.value = idSpace
case s.tok.char == IDENTIFIER:
for i := 0; i < len(s.charBuf); {
c := s.charBuf[i].c
if c != '\\' {
s.bytesBuf = append(s.bytesBuf, c)
i++
continue
}
i++ // Skip '\\'
var n int
switch s.charBuf[i].c {
case 'u':
n = 4
case 'U':
n = 8
default:
panic(internalError())
}
i++ // Skip 'u' or 'U'
l := len(s.bytesBuf)
for i0 := i; i < i0+n; i++ {
s.bytesBuf = append(s.bytesBuf, s.charBuf[i].c)
}
r, err := strconv.ParseUint(string(s.bytesBuf[l:l+n]), 16, 32)
if err != nil {
panic(internalError())
}
n2 := utf8.EncodeRune(s.bytesBuf[l:], rune(r))
s.bytesBuf = s.bytesBuf[:l+n2]
}
s.tok.value = dict.id(s.bytesBuf)
default:
s.tok.value = s.tok.src
}
switch s.tok.char {
case clsEOF:
s.tok.char = -1
s.tok.pos = int32(s.file.Pos(s.file.Size()))
}
// dbg("lex %q %q", tokName(s.tok.char), s.tok.value)
}
func (s *scanner) next() (r byte) {
if s.lookaheadChar.pos > 0 {
s.charBuf = append(s.charBuf, s.lookaheadChar)
}
if n := len(s.ungetBuf); n != 0 {
s.lookaheadChar = s.ungetBuf[n-1]
s.ungetBuf = s.ungetBuf[:n-1]
return s.class(s.lookaheadChar.c)
}
if len(s.lineBuf) == 0 {
more:
if s.closed || s.fileOffset == s.file.Size() {
s.lookaheadChar.c = 0
s.lookaheadChar.pos = 0
return clsEOF
}
b, err := s.r.ReadSlice('\n')
if err != nil {
if err != io.EOF {
s.errPos(s.pos, "error while reading %s: %s", s.file.Name(), err)
}
if len(b) == 0 {
return clsEOF
}
}
s.file.AddLine(s.fileOffset)
s.fileOffset += s.bomFix
s.bomFix = 0
s.pos = token.Pos(s.fileOffset)
s.fileOffset += len(b)
// [0], 5.1.1.2, 1.1
//
// Physical source file multibyte characters are mapped, in an
// implementation- defined manner, to the source character set
// (introducing new-line characters for end-of-line indicators)
// if necessary. Trigraph sequences are replaced by
// corresponding single-character internal representations.
if !s.ctx.cfg.DisableTrigraphs && bytes.Contains(b, trigraphPrefix) {
for _, v := range trigraphs {
b = bytes.Replace(b, v.from, v.to, -1)
}
}
// [0], 5.1.1.2, 2
//
// Each instance of a backslash character (\) immediately
// followed by a new-line character is deleted, splicing
// physical source lines to form logical source lines. Only
// the last backslash on any physical source line shall be
// eligible for being part of such a splice. A source file that
// is not empty shall end in a new-line character, which shall
// not be immediately preceded by a backslash character before
// any such splicing takes place.
s.lineBuf = b
n := len(b)
switch {
case b[n-1] != '\n':
if s.ctx.cfg.RejectMissingFinalNewline {
s.errPos(s.pos+token.Pos(n), "non empty source file shall end in a new-line character")
}
b = append(b[:n:n], '\n') // bufio.Reader owns the bytes
case n > 1 && b[n-2] == '\\':
if n == 2 {
goto more
}
b = b[:n-2]
n = len(b)
if s.fileOffset == s.file.Size() {
if s.ctx.cfg.RejectFinalBackslash {
s.errPos(s.pos+token.Pos(n+1), "source file final new-line character shall not be preceded by a backslash character")
}
b = append(b[:n:n], '\n') // bufio.Reader owns the bytes
}
case n > 2 && b[n-3] == '\\' && b[n-2] == '\r':
// we've got a windows source that has \r\n line endings.
if n == 3 {
goto more
}
b = b[:n-3]
n = len(b)
if s.fileOffset == s.file.Size() {
if s.ctx.cfg.RejectFinalBackslash {
s.errPos(s.pos+token.Pos(n+1), "source file final new-line character shall not be preceded by a backslash character")
}
b = append(b[:n:n], '\n') // bufio.Reader owns the bytes
}
}
s.lineBuf = b
}
s.pos++
s.lookaheadChar = char{int32(s.pos), s.lineBuf[0]}
s.lineBuf = s.lineBuf[1:]
return s.class(s.lookaheadChar.c)
}
func (s *scanner) unget(c ...char) {
s.ungetBuf = append(s.ungetBuf, c...)
s.lookaheadChar.pos = 0 // Must invalidate lookahead.
}
func (s *scanner) unterminatedComment() rune {
s.errPos(token.Pos(s.file.Size()), "unterminated comment")
n := len(s.charBuf)
s.unget(s.charBuf[n-1]) // \n
s.charBuf = s.charBuf[:n-1]
return ' '
}
// -------------------------------------------------------- Translation phase 3
// [0], 5.1.1.2, 3
//
// The source file is decomposed into preprocessing tokens and sequences of
// white-space characters (including comments). A source file shall not end in
// a partial preprocessing token or in a partial comment. Each comment is
// replaced by one space character. New-line characters are retained. Whether
// each nonempty sequence of white-space characters other than new-line is
// retained or replaced by one space character is implementation-defined.
func (s *scanner) translationPhase3() *ppFile {
r := &ppFile{file: s.file}
if s.file.Size() == 0 {
s.r.Reset(nil)
return r
}
s.nextLine()
r.groups = s.parseGroup()
return r
}
func (s *scanner) parseGroup() (r []ppGroup) {
for {
switch x := s.lookaheadLine.(type) {
case ppGroup:
r = append(r, x)
s.nextLine()
case ppIfGroupDirective:
r = append(r, s.parseIfSection())
default:
return r
}
}
}
func (s *scanner) parseIfSection() *ppIfSection {
return &ppIfSection{
ifGroup: s.parseIfGroup(),
elifGroups: s.parseElifGroup(),
elseGroup: s.parseElseGroup(),
endifLine: s.parseEndifLine(),
}
}
func (s *scanner) parseEndifLine() *ppEndifDirective {
switch x := s.lookaheadLine.(type) {
case *ppEndifDirective:
s.nextLine()
return x
default:
s.errLine(x, fmt.Sprintf(": expected #endif (unexpected %T)", x))
s.nextLine()
return nil
}
}
func (s *scanner) parseElseGroup() *ppElseGroup {
switch x := s.lookaheadLine.(type) {
case *ppElseDirective:
r := &ppElseGroup{elseLine: x}
s.nextLine()
r.groups = s.parseGroup()
return r
default:
return nil
}
}
func (s *scanner) parseElifGroup() (r []*ppElifGroup) {
for {
var g ppElifGroup
switch x := s.lookaheadLine.(type) {
case *ppElifDirective:
g.elif = x
s.nextLine()
g.groups = s.parseGroup()
r = append(r, &g)
default:
return r
}
}
}
func (s *scanner) parseIfGroup() *ppIfGroup {
r := &ppIfGroup{}
switch x := s.lookaheadLine.(type) {
case ppIfGroupDirective:
r.directive = x
default:
s.errLine(x, fmt.Sprintf(": expected if-group (unexpected %T)", x))
}
s.nextLine()
r.groups = s.parseGroup()
return r
}
func (s *scanner) nextLine() {
s.tokenBuf = nil
s.lookaheadLine = s.scanLine()
}
func (s *scanner) scanLine() (r ppLine) {
again:
toks := s.scanToNonBlankToken(nil)
if len(toks) == 0 {
return nil
}
includeNext := false
switch tok := toks[len(toks)-1]; tok.char {
case '#':
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppEmptyDirective{toks: toks}
case IDENTIFIER:
switch tok.value {
case idDefine:
return s.parseDefine(toks)
case idElif:
return s.parseElif(toks)
case idElse:
return s.parseElse(toks)
case idEndif:
return s.parseEndif(toks)
case idIf:
return s.parseIf(toks)
case idIfdef:
return s.parseIfdef(toks)
case idIfndef:
return s.parseIfndef(toks)
case idIncludeNext:
includeNext = true
fallthrough
case idInclude:
// # include pp-tokens new-line
//
// Prevent aliasing of eg. <foo bar.h> and <foo bar.h>.
save := s.preserveWhiteSpace
s.preserveWhiteSpace = true
n := len(toks)
toks := s.scanLineToEOL(toks)
r := &ppIncludeDirective{arg: toks[n : len(toks)-1], toks: toks, includeNext: includeNext}
s.preserveWhiteSpace = save
return r
case idUndef:
return s.parseUndef(toks)
case idLine:
return s.parseLine(toks)
case idError:
// # error pp-tokens_opt new-line
n := len(toks)
toks := s.scanLineToEOL(toks)
msg := toks[n : len(toks)-1]
if len(msg) != 0 && msg[0].char == ' ' {
msg = msg[1:]
}
return &ppErrorDirective{toks: toks, msg: msg}
case idPragma:
return s.parsePragma(toks)
}
}
// # non-directive
return &ppNonDirective{toks: s.scanLineToEOL(toks)}
case '\n':
return &ppTextLine{toks: toks}
case IDENTIFIER:
if tok.value == idPragmaOp {
toks = s.scanToNonBlankToken(toks)
switch tok = toks[len(toks)-1]; tok.char {
case '(':
// ok
default:
s.err(tok, "expected (")
return &ppTextLine{toks: toks}
}
var lit string
toks = s.scanToNonBlankToken(toks)
switch tok = toks[len(toks)-1]; tok.char {
case STRINGLITERAL:
lit = tok.String()
case LONGSTRINGLITERAL:
lit = tok.String()[1:] // [0], 6.9.10, 1
default:
s.err(tok, "expected string literal")
return &ppTextLine{toks: toks}
}
pos := tok.pos
toks = s.scanToNonBlankToken(toks)
switch tok = toks[len(toks)-1]; tok.char {
case ')':
// ok
default:
s.err(tok, "expected )")
return &ppTextLine{toks: toks}
}
s.unget(s.lookaheadChar)
// [0], 6.9.10, 1
lit = lit[1 : len(lit)-1]
lit = strings.ReplaceAll(lit, `\"`, `"`)
lit = strings.ReplaceAll(lit, `\\`, `\`)
lit = "#pragma " + lit + "\n"
for i := len(lit) - 1; i >= 0; i-- {
s.unget(char{pos, lit[i]})
}
goto again
}
fallthrough
default:
return &ppTextLine{toks: s.scanLineToEOL(toks)}
}
}
func (s *scanner) parsePragma(toks []token3) *ppPragmaDirective {
toks = s.scanToNonBlankToken(toks)
n := len(toks)
if toks[n-1].char != '\n' {
toks = s.scanLineToEOL(toks)
}
return &ppPragmaDirective{toks: toks, args: toks[n-1:]}
}
// # line pp-tokens new-line
func (s *scanner) parseLine(toks []token3) *ppLineDirective {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
s.err(tok, "unexpected new-line")
return &ppLineDirective{toks: toks}
default:
toks := s.scanLineToEOL(toks)
last := toks[len(toks)-1]
r := &ppLineDirective{toks: toks, nextPos: int(last.pos) + len(last.src.String())}
toks = toks[:len(toks)-1] // sans new-line
toks = ltrim3(toks)
toks = toks[1:] // Skip '#'
toks = ltrim3(toks)
toks = toks[1:] // Skip "line"
r.args = ltrim3(toks)
return r
}
}
func ltrim3(toks []token3) []token3 {
for len(toks) != 0 && toks[0].char == ' ' {
toks = toks[1:]
}
return toks
}
// # undef identifier new-line
func (s *scanner) parseUndef(toks []token3) *ppUndefDirective {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
s.err(&tok, "expected identifier")
return &ppUndefDirective{toks: toks}
case IDENTIFIER:
name := tok
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppUndefDirective{name: name, toks: toks}
default:
if s.ctx.cfg.RejectUndefExtraTokens {
s.err(&tok, "extra tokens after #undef")
}
return &ppUndefDirective{name: name, toks: s.scanLineToEOL(toks)}
}
default:
s.err(&tok, "expected identifier")
return &ppUndefDirective{toks: s.scanLineToEOL(toks)}
}
}
func (s *scanner) scanLineToEOL(toks []token3) []token3 {
n := len(s.tokenBuf) - len(toks)
for {
s.lex()
s.tokenBuf = append(s.tokenBuf, s.tok)
if s.tok.char == '\n' {
return s.tokenBuf[n:]
}
}
}
// # ifndef identifier new-line
func (s *scanner) parseIfndef(toks []token3) *ppIfndefDirective {
var name StringID
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case IDENTIFIER:
name = tok.value
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppIfndefDirective{name: name, toks: toks}
default:
if s.ctx.cfg.RejectIfndefExtraTokens {
s.err(&tok, "extra tokens after #ifndef")
}
return &ppIfndefDirective{name: name, toks: s.scanLineToEOL(toks)}
}
case '\n':
s.err(tok, "expected identifier")
return &ppIfndefDirective{name: name, toks: toks}
default:
s.err(tok, "expected identifier")
return &ppIfndefDirective{name: name, toks: s.scanLineToEOL(toks)}
}
}
// # ifdef identifier new-line
func (s *scanner) parseIfdef(toks []token3) *ppIfdefDirective {
var name StringID
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case IDENTIFIER:
name = tok.value
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppIfdefDirective{name: name, toks: toks}
default:
if s.ctx.cfg.RejectIfdefExtraTokens {
s.err(&tok, "extra tokens after #ifdef")
}
return &ppIfdefDirective{name: name, toks: s.scanLineToEOL(toks)}
}
case '\n':
s.err(tok, "expected identifier")
return &ppIfdefDirective{name: name, toks: toks}
default:
s.err(tok, "expected identifier")
return &ppIfdefDirective{name: name, toks: s.scanLineToEOL(toks)}
}
}
// # if constant-expression new-line
func (s *scanner) parseIf(toks []token3) *ppIfDirective {
n := len(toks)
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
s.err(tok, "expected expression")
return &ppIfDirective{toks: toks}
default:
toks = s.scanLineToEOL(toks)
expr := toks[n:]
if expr[0].char == ' ' { // sans leading space
expr = expr[1:]
}
expr = expr[:len(expr)-1] // sans '\n'
return &ppIfDirective{toks: toks, expr: expr}
}
}
// # endif new-line
func (s *scanner) parseEndif(toks []token3) *ppEndifDirective {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppEndifDirective{toks}
default:
if s.ctx.cfg.RejectEndifExtraTokens {
s.err(&tok, "extra tokens after #else")
}
return &ppEndifDirective{s.scanLineToEOL(toks)}
}
}
// # else new-line
func (s *scanner) parseElse(toks []token3) *ppElseDirective {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppElseDirective{toks}
default:
if s.ctx.cfg.RejectElseExtraTokens {
s.err(&tok, "extra tokens after #else")
}
return &ppElseDirective{s.scanLineToEOL(toks)}
}
}
// # elif constant-expression new-line
func (s *scanner) parseElif(toks []token3) *ppElifDirective {
n := len(toks)
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
s.err(tok, "expected expression")
return &ppElifDirective{toks, nil}
default:
toks = s.scanLineToEOL(toks)
expr := toks[n:]
if expr[0].char == ' ' { // sans leading space
expr = expr[1:]
}
expr = expr[:len(expr)-1] // sans '\n'
return &ppElifDirective{toks, expr}
}
}
func (s *scanner) parseDefine(toks []token3) ppLine {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case IDENTIFIER:
name := tok
n := len(toks)
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
return &ppDefineObjectMacroDirective{name: name, toks: toks}
case '(':
if toks[n].char == ' ' {
return s.parseDefineObjectMacro(n, name, toks)
}
return s.parseDefineFunctionMacro(name, toks)
default:
return s.parseDefineObjectMacro(n, name, toks)
}
case '\n':
s.err(tok, "expected identifier")
return &ppDefineObjectMacroDirective{toks: toks}
default:
s.err(tok, "expected identifier")
return &ppDefineObjectMacroDirective{toks: s.scanLineToEOL(toks)}
}
}
// # define identifier lparen identifier-list_opt ) replacement-list new-line
// # define identifier lparen ... ) replacement-list new-line
// # define identifier lparen identifier-list , ... ) replacement-list new-line
func (s *scanner) parseDefineFunctionMacro(name token3, toks []token3) *ppDefineFunctionMacroDirective {
// Parse parameters after "#define name(".
var list []token3
variadic := false
namedVariadic := false
again:
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case IDENTIFIER:
more:
list = append(list, tok)
toks = s.scanToNonBlankToken(toks)
switch tok = toks[len(toks)-1]; tok.char {
case ',':
toks = s.scanToNonBlankToken(toks)
switch tok = toks[len(toks)-1]; tok.char {
case IDENTIFIER:
goto more
case DDD:
if toks, variadic = s.parseDDD(toks); !variadic {
goto again
}
case ')':
s.err(tok, "expected parameter name")
default:
s.err(tok, "unexpected %q", &tok)
}
case DDD:
namedVariadic = true
if s.ctx.cfg.RejectInvalidVariadicMacros {
s.err(tok, "expected comma")
}
if toks, variadic = s.parseDDD(toks); !variadic {
goto again
}
case ')':
// ok
case '\n':
s.err(tok, "unexpected new-line")
return &ppDefineFunctionMacroDirective{toks: toks}
case IDENTIFIER:
s.err(tok, "expected comma")
goto more
default:
s.err(tok, "unexpected %q", &tok)
}
case DDD:
if toks, variadic = s.parseDDD(toks); !variadic {
goto again
}
case ',':
s.err(tok, "expected parameter name")
goto again
case ')':
// ok
default:
s.err(tok, "expected parameter name")
goto again
}
// Parse replacement list.
n := len(toks)
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case '\n':
if s.ctx.cfg.RejectFunctionMacroEmptyReplacementList {
s.err(tok, "expected replacement list")
}
return &ppDefineFunctionMacroDirective{name: name, identifierList: list, toks: toks, variadic: variadic, namedVariadic: namedVariadic}
default:
toks = s.scanLineToEOL(toks)
repl := toks[n:] // sans #define identifier
repl = repl[:len(repl)-1] // sans '\n'
// 6.10.3, 7
//
// Any white-space characters preceding or following the
// replacement list of preprocessing tokens are not considered
// part of the replacement list for either form of macro.
repl = trim3(repl)
repl = normalizeHashes(repl)
return &ppDefineFunctionMacroDirective{name: name, identifierList: list, toks: toks, replacementList: repl, variadic: variadic, namedVariadic: namedVariadic}
}
}
func isWhite(char rune) bool {
switch char {
case ' ', '\t', '\n', '\v', '\f':
return true
}
return false
}
func trim3(toks []token3) []token3 {
for len(toks) != 0 && isWhite(toks[0].char) {
toks = toks[1:]
}
for len(toks) != 0 && isWhite(toks[len(toks)-1].char) {
toks = toks[:len(toks)-1]
}
return toks
}
func normalizeHashes(toks []token3) []token3 {
w := 0
var last rune
for _, v := range toks {
switch {
case v.char == PPPASTE:
if isWhite(last) {
w--
}
case isWhite(v.char):
if last == '#' || last == PPPASTE {
continue
}
}
last = v.char
toks[w] = v
w++
}
return toks[:w]
}
func (s *scanner) parseDDD(toks []token3) ([]token3, bool) {
toks = s.scanToNonBlankToken(toks)
switch tok := toks[len(toks)-1]; tok.char {
case ')':
return toks, true
default:
s.err(tok, "expected right parenthesis")
return toks, false
}
}
// # define identifier replacement-list new-line
func (s *scanner) parseDefineObjectMacro(n int, name token3, toks []token3) *ppDefineObjectMacroDirective {
toks = s.scanLineToEOL(toks)
repl := toks[n:] // sans #define identifier
repl = repl[:len(repl)-1] // sans '\n'
// 6.10.3, 7
//
// Any white-space characters preceding or following the replacement
// list of preprocessing tokens are not considered part of the
// replacement list for either form of macro.
repl = trim3(repl)
repl = normalizeHashes(repl)
return &ppDefineObjectMacroDirective{name: name, toks: toks, replacementList: repl}
}
// Return {}, {x} or {' ', x}
func (s *scanner) scanToNonBlankToken(toks []token3) []token3 {
n := len(s.tokenBuf) - len(toks)
for {
s.lex()
if s.tok.char < 0 {
return s.tokenBuf[n:]
}
s.tokenBuf = append(s.tokenBuf, s.tok)
if s.tok.char != ' ' {
return s.tokenBuf[n:]
}
}
}
// ---------------------------------------------------------------------- Cache
// Translation phase4 source.
type source interface {
ppFile() (*ppFile, error)
}
type cachedPPFile struct {
err error
errs goscanner.ErrorList
modTime int64 // time.Time.UnixNano()
pf *ppFile
readyCh chan struct{}
size int
}
func (c *cachedPPFile) ready() *cachedPPFile { close(c.readyCh); return c }
func (c *cachedPPFile) waitFor() (*cachedPPFile, error) { <-c.readyCh; return c, c.err }
func (c *cachedPPFile) ppFile() (*ppFile, error) {
c.waitFor()
if c.err == nil {
return c.pf, nil
}
return nil, c.err
}
type cacheKey struct {
name StringID
sys bool
value StringID
Config3
}
type ppCache struct {
mu sync.RWMutex
m map[cacheKey]*cachedPPFile
}
func newPPCache() *ppCache { return &ppCache{m: map[cacheKey]*cachedPPFile{}} }
func (c *ppCache) get(ctx *context, src Source) (source, error) {
if src.Value != "" {
return c.getValue(ctx, src.Name, src.Value, false, src.DoNotCache)
}
return c.getFile(ctx, src.Name, false, src.DoNotCache)
}
func (c *ppCache) getFile(ctx *context, name string, sys bool, doNotCache bool) (*cachedPPFile, error) {
fi, err := ctx.statFile(name, sys)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("%s is not a regular file", name)
}
if fi.Size() > mathutil.MaxInt {
return nil, fmt.Errorf("%s: file too big", name)
}
size := int(fi.Size())
if !filepath.IsAbs(name) { // Never cache relative paths
f, err := ctx.openFile(name, sys)
if err != nil {
return nil, err
}
defer f.Close()
tf := tokenNewFile(name, size)
ppFile := newScanner(ctx, f, tf).translationPhase3()
cf := &cachedPPFile{pf: ppFile, readyCh: make(chan struct{})}
cf.ready()
return cf, nil
}
modTime := fi.ModTime().UnixNano()
key := cacheKey{dict.sid(name), sys, 0, ctx.cfg.Config3}
c.mu.Lock()
if cf, ok := c.m[key]; ok {
if modTime <= cf.modTime && size == cf.size {
c.mu.Unlock()
if cf.err != nil {
return nil, cf.err
}
r, err := cf.waitFor()
ctx.errs(cf.errs)
return r, err
}
delete(c.m, key)
}
tf := tokenNewFile(name, size)
cf := &cachedPPFile{modTime: modTime, size: size, readyCh: make(chan struct{})}
if !doNotCache {
c.m[key] = cf
}
c.mu.Unlock()
go func() {
defer cf.ready()
f, err := ctx.openFile(name, sys)
if err != nil {
cf.err = err
return
}
defer f.Close()
ctx2 := newContext(ctx.cfg)
cf.pf = newScanner(ctx2, f, tf).translationPhase3()
cf.errs = ctx2.ErrorList
ctx.errs(cf.errs)
}()
return cf.waitFor()
}
func (c *ppCache) getValue(ctx *context, name, value string, sys bool, doNotCache bool) (*cachedPPFile, error) {
key := cacheKey{dict.sid(name), sys, dict.sid(value), ctx.cfg.Config3}
c.mu.Lock()
if cf, ok := c.m[key]; ok {
c.mu.Unlock()
if cf.err != nil {
return nil, cf.err
}
r, err := cf.waitFor()
ctx.errs(cf.errs)
return r, err
}
tf := tokenNewFile(name, len(value))
cf := &cachedPPFile{readyCh: make(chan struct{})}
if !doNotCache {
c.m[key] = cf
}
c.mu.Unlock()
ctx2 := newContext(ctx.cfg)
cf.pf = newScanner(ctx2, strings.NewReader(value), tf).translationPhase3()
cf.errs = ctx2.ErrorList
ctx.errs(cf.errs)
cf.ready()
return cf.waitFor()
}