PL281/Q0/build/kati/eval.go

// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kati

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/golang/glog"
)

type fileState int

const (
	fileExists fileState = iota
	fileNotExists
	fileInconsistent // Modified during kati is running.
)

type accessedMakefile struct {
	Filename string
	Hash     [sha1.Size]byte
	State    fileState
}

type accessCache struct {
	mu sync.Mutex
	m  map[string]*accessedMakefile
}

func newAccessCache() *accessCache {
	return &accessCache{
		m: make(map[string]*accessedMakefile),
	}
}

func (ac *accessCache) update(fn string, hash [sha1.Size]byte, st fileState) string {
	if ac == nil {
		return ""
	}
	ac.mu.Lock()
	defer ac.mu.Unlock()
	rm, present := ac.m[fn]
	if present {
		switch rm.State {
		case fileExists:
			if st != fileExists {
				return fmt.Sprintf("%s was removed after the previous read", fn)
			} else if !bytes.Equal(hash[:], rm.Hash[:]) {
				ac.m[fn].State = fileInconsistent
				return fmt.Sprintf("%s was modified after the previous read", fn)
			}
			return ""
		case fileNotExists:
			if st != fileNotExists {
				ac.m[fn].State = fileInconsistent
				return fmt.Sprintf("%s was created after the previous read", fn)
			}
		case fileInconsistent:
			return ""
		}
		return ""
	}
	ac.m[fn] = &accessedMakefile{
		Filename: fn,
		Hash:     hash,
		State:    st,
	}
	return ""
}

func (ac *accessCache) Slice() []*accessedMakefile {
	if ac == nil {
		return nil
	}
	ac.mu.Lock()
	defer ac.mu.Unlock()
	r := []*accessedMakefile{}
	for _, v := range ac.m {
		r = append(r, v)
	}
	return r
}

type evalResult struct {
	vars        Vars
	rules       []*rule
	ruleVars    map[string]Vars
	accessedMks []*accessedMakefile
	exports     map[string]bool
	vpaths      searchPaths
}

type srcpos struct {
	filename string
	lineno   int
}

func (p srcpos) String() string {
	return fmt.Sprintf("%s:%d", p.filename, p.lineno)
}

// EvalError is an error in kati evaluation.
type EvalError struct {
	Filename string
	Lineno   int
	Err      error
}

func (e EvalError) Error() string {
	return fmt.Sprintf("%s:%d: %v", e.Filename, e.Lineno, e.Err)
}

func (p srcpos) errorf(f string, args ...interface{}) error {
	return EvalError{
		Filename: p.filename,
		Lineno:   p.lineno,
		Err:      fmt.Errorf(f, args...),
	}
}

func (p srcpos) error(err error) error {
	if _, ok := err.(EvalError); ok {
		return err
	}
	return EvalError{
		Filename: p.filename,
		Lineno:   p.lineno,
		Err:      err,
	}
}

// Evaluator manages makefile evaluation.
type Evaluator struct {
	paramVars    []tmpval // $1 => paramVars[1]
	outVars      Vars
	outRules     []*rule
	outRuleVars  map[string]Vars
	vars         Vars
	lastRule     *rule
	currentScope Vars
	cache        *accessCache
	exports      map[string]bool
	vpaths       []vpath

	avoidIO bool
	hasIO   bool
	// delayedOutputs are commands which should run at ninja-time
	// (i.e., info, warning, and error).
	delayedOutputs []string

	srcpos
}

// NewEvaluator creates new Evaluator.
func NewEvaluator(vars map[string]Var) *Evaluator {
	return &Evaluator{
		outVars:     make(Vars),
		vars:        vars,
		outRuleVars: make(map[string]Vars),
		exports:     make(map[string]bool),
	}
}

func (ev *Evaluator) args(buf *evalBuffer, args ...Value) ([][]byte, error) {
	pos := make([]int, 0, len(args))
	for _, arg := range args {
		buf.resetSep()
		err := arg.Eval(buf, ev)
		if err != nil {
			return nil, err
		}
		pos = append(pos, buf.Len())
	}
	v := buf.Bytes()
	buf.args = buf.args[:0]
	s := 0
	for _, p := range pos {
		buf.args = append(buf.args, v[s:p])
		s = p
	}
	return buf.args, nil
}

func (ev *Evaluator) evalAssign(ast *assignAST) error {
	ev.lastRule = nil
	lhs, rhs, err := ev.evalAssignAST(ast)
	if err != nil {
		return err
	}
	if glog.V(1) {
		glog.Infof("ASSIGN: %s=%q (flavor:%q)", lhs, rhs, rhs.Flavor())
	}
	if lhs == "" {
		return ast.errorf("*** empty variable name.")
	}
	ev.outVars.Assign(lhs, rhs)
	return nil
}

func (ev *Evaluator) evalAssignAST(ast *assignAST) (string, Var, error) {
	ev.srcpos = ast.srcpos

	var lhs string
	switch v := ast.lhs.(type) {
	case literal:
		lhs = string(v)
	case tmpval:
		lhs = string(v)
	default:
		buf := newEbuf()
		err := v.Eval(buf, ev)
		if err != nil {
			return "", nil, err
		}
		lhs = string(trimSpaceBytes(buf.Bytes()))
		buf.release()
	}
	rhs, err := ast.evalRHS(ev, lhs)
	if err != nil {
		return "", nil, err
	}
	return lhs, rhs, nil
}

func (ev *Evaluator) setTargetSpecificVar(assign *assignAST, output string) error {
	vars, present := ev.outRuleVars[output]
	if !present {
		vars = make(Vars)
		ev.outRuleVars[output] = vars
	}
	ev.currentScope = vars
	lhs, rhs, err := ev.evalAssignAST(assign)
	if err != nil {
		return err
	}
	if glog.V(1) {
		glog.Infof("rule outputs:%q assign:%q%s%q (flavor:%q)", output, lhs, assign.op, rhs, rhs.Flavor())
	}
	vars.Assign(lhs, &targetSpecificVar{v: rhs, op: assign.op})
	ev.currentScope = nil
	return nil
}

func (ev *Evaluator) evalMaybeRule(ast *maybeRuleAST) error {
	ev.lastRule = nil
	ev.srcpos = ast.srcpos

	if glog.V(1) {
		glog.Infof("maybe rule %s: %q assign:%v", ev.srcpos, ast.expr, ast.assign)
	}

	abuf := newEbuf()
	aexpr := toExpr(ast.expr)
	var rhs expr
	semi := ast.semi
	for i, v := range aexpr {
		var hashFound bool
		var buf evalBuffer
		buf.resetSep()
		switch v.(type) {
		case literal, tmpval:
			s := v.String()
			i := strings.Index(s, "#")
			if i >= 0 {
				hashFound = true
				v = tmpval(trimRightSpaceBytes([]byte(s[:i])))
			}
		}
		err := v.Eval(&buf, ev)
		if err != nil {
			return err
		}
		b := buf.Bytes()
		if ast.isRule {
			abuf.Write(b)
			continue
		}
		eq := findLiteralChar(b, '=', 0, skipVar)
		if eq >= 0 {
			abuf.Write(b[:eq+1])
			if eq+1 < len(b) {
				rhs = append(rhs, tmpval(trimLeftSpaceBytes(b[eq+1:])))
			}
			if i+1 < len(aexpr) {
				rhs = append(rhs, aexpr[i+1:]...)
			}
			if ast.semi != nil {
				rhs = append(rhs, literal(';'))
				sexpr, _, err := parseExpr(ast.semi, nil, parseOp{})
				if err != nil {
					return err
				}
				rhs = append(rhs, toExpr(sexpr)...)
				semi = nil
			}
			break
		}
		abuf.Write(b)
		if hashFound {
			break
		}
	}

	line := abuf.Bytes()
	r := &rule{srcpos: ast.srcpos}
	if glog.V(1) {
		glog.Infof("rule? %s: %q assign:%v rhs:%s", r.srcpos, line, ast.assign, rhs)
	}
	assign, err := r.parse(line, ast.assign, rhs)
	if err != nil {
		ws := newWordScanner(line)
		if ws.Scan() {
			if string(ws.Bytes()) == "override" {
				warnNoPrefix(ast.srcpos, "invalid `override' directive")
				return nil
			}
		}
		return ast.error(err)
	}
	abuf.release()
	if glog.V(1) {
		glog.Infof("rule %q assign:%v rhs:%v=> outputs:%q, inputs:%q", ast.expr, ast.assign, rhs, r.outputs, r.inputs)
	}

	// TODO: Pretty print.
	// glog.V(1).Infof("RULE: %s=%s (%d commands)", lhs, rhs, len(cmds))

	if assign != nil {
		glog.V(1).Infof("target specific var: %#v", assign)
		for _, output := range r.outputs {
			ev.setTargetSpecificVar(assign, output)
		}
		for _, output := range r.outputPatterns {
			ev.setTargetSpecificVar(assign, output.String())
		}
		return nil
	}

	if semi != nil {
		r.cmds = append(r.cmds, string(semi))
	}
	if glog.V(1) {
		glog.Infof("rule outputs:%q cmds:%q", r.outputs, r.cmds)
	}
	ev.lastRule = r
	ev.outRules = append(ev.outRules, r)
	return nil
}

func (ev *Evaluator) evalCommand(ast *commandAST) error {
	ev.srcpos = ast.srcpos
	if ev.lastRule == nil || ev.lastRule.outputs == nil {
		// This could still be an assignment statement. See
		// assign_after_tab.mk.
		if strings.IndexByte(ast.cmd, '=') >= 0 {
			line := trimLeftSpace(ast.cmd)
			mk, err := parseMakefileString(line, ast.srcpos)
			if err != nil {
				return ast.errorf("parse failed: %q: %v", line, err)
			}
			if len(mk.stmts) >= 1 && mk.stmts[len(mk.stmts)-1].(*assignAST) != nil {
				for _, stmt := range mk.stmts {
					err = ev.eval(stmt)
					if err != nil {
						return err
					}
				}
			}
			return nil
		}
		// Or, a comment is OK.
		if strings.TrimSpace(ast.cmd)[0] == '#' {
			return nil
		}
		return ast.errorf("*** commands commence before first target.")
	}
	ev.lastRule.cmds = append(ev.lastRule.cmds, ast.cmd)
	if ev.lastRule.cmdLineno == 0 {
		ev.lastRule.cmdLineno = ast.lineno
	}
	return nil
}

func (ev *Evaluator) paramVar(name string) (Var, error) {
	idx, err := strconv.ParseInt(name, 10, 32)
	if err != nil {
		return nil, fmt.Errorf("param: %s: %v", name, err)
	}
	i := int(idx)
	if i < 0 || i >= len(ev.paramVars) {
		return nil, fmt.Errorf("param: %s out of %d", name, len(ev.paramVars))
	}
	return &automaticVar{value: []byte(ev.paramVars[i])}, nil
}

// LookupVar looks up named variable.
func (ev *Evaluator) LookupVar(name string) Var {
	if ev.currentScope != nil {
		v := ev.currentScope.Lookup(name)
		if v.IsDefined() {
			return v
		}
	}
	v := ev.outVars.Lookup(name)
	if v.IsDefined() {
		return v
	}
	v, err := ev.paramVar(name)
	if err == nil {
		return v
	}
	return ev.vars.Lookup(name)
}

func (ev *Evaluator) lookupVarInCurrentScope(name string) Var {
	if ev.currentScope != nil {
		v := ev.currentScope.Lookup(name)
		return v
	}
	v := ev.outVars.Lookup(name)
	if v.IsDefined() {
		return v
	}
	v, err := ev.paramVar(name)
	if err == nil {
		return v
	}
	return ev.vars.Lookup(name)
}

// EvaluateVar evaluates variable named name.
// Only for a few special uses such as getting SHELL and handling
// export/unexport.
func (ev *Evaluator) EvaluateVar(name string) (string, error) {
	var buf evalBuffer
	buf.resetSep()
	err := ev.LookupVar(name).Eval(&buf, ev)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

func (ev *Evaluator) evalIncludeFile(fname string, mk makefile) error {
	te := traceEvent.begin("include", literal(fname), traceEventMain)
	defer func() {
		traceEvent.end(te)
	}()
	var err error
	makefileList := ev.outVars.Lookup("MAKEFILE_LIST")
	makefileList, err = makefileList.Append(ev, mk.filename)
	if err != nil {
		return err
	}
	ev.outVars.Assign("MAKEFILE_LIST", makefileList)

	for _, stmt := range mk.stmts {
		err = ev.eval(stmt)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ev *Evaluator) evalInclude(ast *includeAST) error {
	ev.lastRule = nil
	ev.srcpos = ast.srcpos

	glog.Infof("%s include %q", ev.srcpos, ast.expr)
	v, _, err := parseExpr([]byte(ast.expr), nil, parseOp{})
	if err != nil {
		return ast.errorf("parse failed: %q: %v", ast.expr, err)
	}
	var buf evalBuffer
	buf.resetSep()
	err = v.Eval(&buf, ev)
	if err != nil {
		return ast.errorf("%v", err)
	}
	pats := splitSpaces(buf.String())
	buf.Reset()

	var files []string
	for _, pat := range pats {
		if strings.Contains(pat, "*") || strings.Contains(pat, "?") {
			matched, err := filepath.Glob(pat)
			if err != nil {
				return ast.errorf("glob error: %s: %v", pat, err)
			}
			files = append(files, matched...)
		} else {
			files = append(files, pat)
		}
	}

	for _, fn := range files {
		fn = trimLeadingCurdir(fn)
		if IgnoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, IgnoreOptionalInclude) {
			continue
		}
		mk, hash, err := makefileCache.parse(fn)
		if os.IsNotExist(err) {
			if ast.op == "include" {
				return ev.errorf("%v\nNOTE: kati does not support generating missing makefiles", err)
			}
			msg := ev.cache.update(fn, hash, fileNotExists)
			if msg != "" {
				warn(ev.srcpos, "%s", msg)
			}
			continue
		}
		msg := ev.cache.update(fn, hash, fileExists)
		if msg != "" {
			warn(ev.srcpos, "%s", msg)
		}
		err = ev.evalIncludeFile(fn, mk)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ev *Evaluator) evalIf(iast *ifAST) error {
	var isTrue bool
	switch iast.op {
	case "ifdef", "ifndef":
		expr := iast.lhs
		buf := newEbuf()
		err := expr.Eval(buf, ev)
		if err != nil {
			return iast.errorf("%v\n expr:%s", err, expr)
		}
		v := ev.LookupVar(buf.String())
		buf.Reset()
		err = v.Eval(buf, ev)
		if err != nil {
			return iast.errorf("%v\n expr:%s=>%s", err, expr, v)
		}
		value := buf.String()
		val := buf.Len()
		buf.release()
		isTrue = (val > 0) == (iast.op == "ifdef")
		if glog.V(1) {
			glog.Infof("%s lhs=%q value=%q => %t", iast.op, iast.lhs, value, isTrue)
		}
	case "ifeq", "ifneq":
		lexpr := iast.lhs
		rexpr := iast.rhs
		buf := newEbuf()
		params, err := ev.args(buf, lexpr, rexpr)
		if err != nil {
			return iast.errorf("%v\n (%s,%s)", err, lexpr, rexpr)
		}
		lhs := string(params[0])
		rhs := string(params[1])
		buf.release()
		isTrue = (lhs == rhs) == (iast.op == "ifeq")
		if glog.V(1) {
			glog.Infof("%s lhs=%q %q rhs=%q %q => %t", iast.op, iast.lhs, lhs, iast.rhs, rhs, isTrue)
		}
	default:
		return iast.errorf("unknown if statement: %q", iast.op)
	}

	var stmts []ast
	if isTrue {
		stmts = iast.trueStmts
	} else {
		stmts = iast.falseStmts
	}
	for _, stmt := range stmts {
		err := ev.eval(stmt)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ev *Evaluator) evalExport(ast *exportAST) error {
	ev.lastRule = nil
	ev.srcpos = ast.srcpos

	v, _, err := parseExpr(ast.expr, nil, parseOp{})
	if err != nil {
		return ast.errorf("failed to parse: %q: %v", string(ast.expr), err)
	}
	var buf evalBuffer
	buf.resetSep()
	err = v.Eval(&buf, ev)
	if err != nil {
		return ast.errorf("%v\n expr:%s", err, v)
	}
	if ast.hasEqual {
		ev.exports[string(trimSpaceBytes(buf.Bytes()))] = ast.export
	} else {
		for _, n := range splitSpacesBytes(buf.Bytes()) {
			ev.exports[string(n)] = ast.export
		}
	}
	return nil
}

func (ev *Evaluator) evalVpath(ast *vpathAST) error {
	ev.lastRule = nil
	ev.srcpos = ast.srcpos

	var ebuf evalBuffer
	ebuf.resetSep()
	err := ast.expr.Eval(&ebuf, ev)
	if err != nil {
		return ast.errorf("%v\n expr:%s", err, ast.expr)
	}
	ws := newWordScanner(ebuf.Bytes())
	if !ws.Scan() {
		ev.vpaths = nil
		return nil
	}
	pat := string(ws.Bytes())
	if !ws.Scan() {
		vpaths := ev.vpaths
		ev.vpaths = nil
		for _, v := range vpaths {
			if v.pattern == pat {
				continue
			}
			ev.vpaths = append(ev.vpaths, v)
		}
		return nil
	}
	// The search path, DIRECTORIES, is a list of directories to be
	// searched, separated by colons (semi-colons on MS-DOS and
	// MS-Windows) or blanks, just like the search path used in the
	// `VPATH' variable.
	var dirs []string
	for {
		for _, dir := range bytes.Split(ws.Bytes(), []byte{':'}) {
			dirs = append(dirs, string(dir))
		}
		if !ws.Scan() {
			break
		}
	}
	ev.vpaths = append(ev.vpaths, vpath{
		pattern: pat,
		dirs:    dirs,
	})
	return nil
}

func (ev *Evaluator) eval(stmt ast) error {
	return stmt.eval(ev)
}

func eval(mk makefile, vars Vars, useCache bool) (er *evalResult, err error) {
	ev := NewEvaluator(vars)
	if useCache {
		ev.cache = newAccessCache()
	}

	makefileList := vars.Lookup("MAKEFILE_LIST")
	if !makefileList.IsDefined() {
		makefileList = &simpleVar{value: []string{""}, origin: "file"}
	}
	makefileList, err = makefileList.Append(ev, mk.filename)
	if err != nil {
		return nil, err
	}
	ev.outVars.Assign("MAKEFILE_LIST", makefileList)

	for _, stmt := range mk.stmts {
		err = ev.eval(stmt)
		if err != nil {
			return nil, err
		}
	}

	vpaths := searchPaths{
		vpaths: ev.vpaths,
	}
	v, found := ev.outVars["VPATH"]
	if found {
		wb := newWbuf()
		err := v.Eval(wb, ev)
		if err != nil {
			return nil, err
		}
		// In the 'VPATH' variable, directory names are separated
		// by colons or blanks. (on windows, semi-colons)
		for _, word := range wb.words {
			for _, dir := range bytes.Split(word, []byte{':'}) {
				vpaths.dirs = append(vpaths.dirs, string(dir))
			}
		}
	}
	glog.Infof("vpaths: %#v", vpaths)

	return &evalResult{
		vars:        ev.outVars,
		rules:       ev.outRules,
		ruleVars:    ev.outRuleVars,
		accessedMks: ev.cache.Slice(),
		exports:     ev.exports,
		vpaths:      vpaths,
	}, nil
}