judge/dsl/parser.go

package dsl

import (
	"fmt"
	"math"
	"strconv"
	"time"
)

type Parser struct {
	tokens []Token
	pos    int
	warns  []string
}

func NewParser(tokens []Token) *Parser {
	return &Parser{tokens: tokens}
}

func (p *Parser) Warnings() []string {
	return p.warns
}

func (p *Parser) warn(msg string) {
	p.warns = append(p.warns, msg)
}

func (p *Parser) peek() Token {
	if p.pos >= len(p.tokens) {
		return Token{Type: TOKEN_EOF}
	}
	return p.tokens[p.pos]
}

func (p *Parser) advance() Token {
	t := p.peek()
	if t.Type != TOKEN_EOF {
		p.pos++
	}
	return t
}

func (p *Parser) expect(tt TokenType) (Token, error) {
	t := p.peek()
	if t.Type != tt {
		return t, fmt.Errorf("%d:%d: expected %s, got %s (%q)", t.Line, t.Col, tt, t.Type, t.Value)
	}
	return p.advance(), nil
}

func (p *Parser) expectIdent(val string) error {
	t := p.peek()
	if t.Type != TOKEN_IDENT || t.Value != val {
		return fmt.Errorf("%d:%d: expected %q, got %q", t.Line, t.Col, val, t.Value)
	}
	p.advance()
	return nil
}

func (p *Parser) isIdent(val string) bool {
	t := p.peek()
	return t.Type == TOKEN_IDENT && t.Value == val
}

func Parse(src string) (*File, []string, error) {
	tokens, err := NewLexer(src).Tokenize()
	if err != nil {
		return nil, nil, err
	}
	parser := NewParser(tokens)
	file, err := parser.parseFile()
	if err != nil {
		return nil, parser.Warnings(), err
	}
	return file, parser.Warnings(), nil
}

func (p *Parser) parseFile() (*File, error) {
	f := &File{}

	for p.peek().Type != TOKEN_EOF {
		t := p.peek()
		if t.Type != TOKEN_IDENT {
			return nil, fmt.Errorf("%d:%d: unexpected token %q", t.Line, t.Col, t.Value)
		}

		switch t.Value {
		case "build":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			if p.peek().Type == TOKEN_LBRACE {
				bc, err := p.parseBuildBlock(s.Value)
				if err != nil {
					return nil, err
				}
				f.Builds = append(f.Builds, bc)
			} else {
				f.Build = s.Value
			}

		case "build_defaults":
			p.advance()
			bc, err := p.parseBuildBlock("")
			if err != nil {
				return nil, err
			}
			f.BuildDefaults = bc

		case "toolchains":
			p.advance()
			specs, err := p.parseToolchainsBlock()
			if err != nil {
				return nil, err
			}
			f.Toolchains = specs

		case "build_linux":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			f.BuildLinux = s.Value

		case "build_windows":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			f.BuildWindows = s.Value

		case "build_darwin":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			f.BuildDarwin = s.Value

		case "binary":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			f.Binary = s.Value

		case "sources":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			f.Sources = s.Value

		case "normalize_crlf":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			b, err := p.parseBool()
			if err != nil {
				return nil, err
			}
			f.NormalizeCRLF = b

		case "trim_trailing_ws":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			b, err := p.parseBool()
			if err != nil {
				return nil, err
			}
			f.TrimTrailingWS = b

		case "timeout":
			p.advance()
			d, err := p.parseDuration()
			if err != nil {
				return nil, err
			}
			f.Timeout = d

		case "memory_limit":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			n, err := p.parseSize()
			if err != nil {
				return nil, err
			}
			f.MemoryLimit = n

		case "group":
			g, err := p.parseGroup(f.Timeout, f.MemoryLimit)
			if err != nil {
				return nil, err
			}
			f.Groups = append(f.Groups, g)

		default:
			return nil, fmt.Errorf("%d:%d: unexpected keyword %q", t.Line, t.Col, t.Value)
		}
	}

	if err := p.validateWeights(f); err != nil {
		return nil, err
	}
	if err := validateBuilds(f); err != nil {
		return nil, err
	}

	return f, nil
}

func validateBuilds(f *File) error {
	hasLegacy := f.Build != "" || f.BuildLinux != "" || f.BuildWindows != "" || f.BuildDarwin != ""
	hasStructured := f.BuildDefaults != nil || len(f.Builds) > 0

	if hasLegacy && hasStructured {
		return fmt.Errorf("cannot mix legacy `build \"shell\"` with structured `build \"name\" { ... }` in the same suite")
	}

	seen := map[string]bool{}
	for _, b := range f.Builds {
		if b.Name == "" {
			return fmt.Errorf("structured build must have a name")
		}
		if seen[b.Name] {
			return fmt.Errorf("duplicate build name %q", b.Name)
		}
		seen[b.Name] = true
	}
	return nil
}

func (p *Parser) validateWeights(f *File) error {
	if len(f.Groups) == 0 {
		return nil
	}
	sum := 0.0
	for _, g := range f.Groups {
		sum += g.Weight
	}
	if math.Abs(sum-1.0) > 0.001 {
		p.warn(fmt.Sprintf("group weights sum to %.4f, expected 1.0", sum))
	}
	return nil
}

func (p *Parser) parseGroup(defaultTimeout time.Duration, defaultMemory int64) (*Group, error) {
	if err := p.expectIdent("group"); err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_LPAREN); err != nil {
		return nil, err
	}
	name, err := p.expect(TOKEN_STRING)
	if err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_RPAREN); err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_LBRACE); err != nil {
		return nil, err
	}

	g := &Group{
		Name:        name.Value,
		Timeout:     defaultTimeout,
		MemoryLimit: defaultMemory,
		Env:         map[string]string{},
		Scoring:     ScoringPartial,
	}

	for !p.isRBrace() {
		t := p.peek()
		if t.Type != TOKEN_IDENT {
			return nil, fmt.Errorf("%d:%d: unexpected token %q in group", t.Line, t.Col, t.Value)
		}

		switch t.Value {
		case "weight":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			w, err := p.parseFloat()
			if err != nil {
				return nil, err
			}
			g.Weight = w

		case "timeout":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			d, err := p.parseDuration()
			if err != nil {
				return nil, err
			}
			g.Timeout = d

		case "memory_limit":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			n, err := p.parseSize()
			if err != nil {
				return nil, err
			}
			g.MemoryLimit = n

		case "scoring":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_IDENT)
			if err != nil {
				return nil, err
			}
			switch s.Value {
			case "partial":
				g.Scoring = ScoringPartial
			case "all_or_none":
				g.Scoring = ScoringAllOrNone
			default:
				return nil, fmt.Errorf("%d:%d: unknown scoring mode %q", s.Line, s.Col, s.Value)
			}

		case "env":
			p.advance()
			if _, err := p.expect(TOKEN_LPAREN); err != nil {
				return nil, err
			}
			key, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_RPAREN); err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			val, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			g.Env[key.Value] = val.Value

		case "wrapper":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			g.Wrapper = s.Value

		case "test":
			test, err := p.parseTest(g.Timeout, g.MemoryLimit)
			if err != nil {
				return nil, err
			}
			g.Tests = append(g.Tests, test)

		case "pattern":
			pat, err := p.parsePattern()
			if err != nil {
				return nil, err
			}
			g.Pattern = pat

		default:
			return nil, fmt.Errorf("%d:%d: unexpected keyword %q in group", t.Line, t.Col, t.Value)
		}
	}

	if _, err := p.expect(TOKEN_RBRACE); err != nil {
		return nil, err
	}
	return g, nil
}

func (p *Parser) parseTest(defaultTimeout time.Duration, defaultMemory int64) (*Test, error) {
	if err := p.expectIdent("test"); err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_LPAREN); err != nil {
		return nil, err
	}
	name, err := p.expect(TOKEN_STRING)
	if err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_RPAREN); err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_LBRACE); err != nil {
		return nil, err
	}

	zero := 0
	test := &Test{
		Name:        name.Value,
		Timeout:     defaultTimeout,
		MemoryLimit: defaultMemory,
		Env:         map[string]string{},
		InFiles:     map[string]string{},
		OutFiles:    map[string]string{},
		ExitCode:    &zero,
		Stdout:      NoMatcher{},
		Stderr:      NoMatcher{},
	}

	for !p.isRBrace() {
		t := p.peek()
		if t.Type != TOKEN_IDENT {
			return nil, fmt.Errorf("%d:%d: unexpected token in test body", t.Line, t.Col)
		}

		switch t.Value {
		case "stdin":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			test.Stdin = &s.Value

		case "stdout":
			p.advance()
			m, err := p.parseMatcherOrAssign()
			if err != nil {
				return nil, err
			}
			test.Stdout = m

		case "stderr":
			p.advance()
			m, err := p.parseMatcherOrAssign()
			if err != nil {
				return nil, err
			}
			test.Stderr = m

		case "args":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			args, err := p.parseStringList()
			if err != nil {
				return nil, err
			}
			test.Args = args

		case "exitCode":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			n, err := p.parseInt()
			if err != nil {
				return nil, err
			}
			test.ExitCode = &n

		case "timeout":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			d, err := p.parseDuration()
			if err != nil {
				return nil, err
			}
			test.Timeout = d

		case "memory_limit":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			n, err := p.parseSize()
			if err != nil {
				return nil, err
			}
			test.MemoryLimit = n

		case "wrapper":
			p.advance()
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			test.Wrapper = s.Value

		case "env":
			p.advance()
			if _, err := p.expect(TOKEN_LPAREN); err != nil {
				return nil, err
			}
			key, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_RPAREN); err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			val, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			test.Env[key.Value] = val.Value

		case "file":
			p.advance()
			if _, err := p.expect(TOKEN_LPAREN); err != nil {
				return nil, err
			}
			fname, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_RPAREN); err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			content, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			test.InFiles[fname.Value] = content.Value

		case "outFile":
			p.advance()
			if _, err := p.expect(TOKEN_LPAREN); err != nil {
				return nil, err
			}
			fname, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_RPAREN); err != nil {
				return nil, err
			}
			if _, err := p.expect(TOKEN_ASSIGN); err != nil {
				return nil, err
			}
			content, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			test.OutFiles[fname.Value] = content.Value

		default:
			return nil, fmt.Errorf("%d:%d: unexpected keyword %q in test", t.Line, t.Col, t.Value)
		}
	}

	if _, err := p.expect(TOKEN_RBRACE); err != nil {
		return nil, err
	}
	return test, nil
}

func (p *Parser) parseMatcherOrAssign() (Matcher, error) {
	t := p.peek()

	if t.Type == TOKEN_ASSIGN {
		p.advance()
		s, err := p.expect(TOKEN_STRING)
		if err != nil {
			return nil, err
		}
		return ExactMatcher{Value: s.Value}, nil
	}

	if t.Type == TOKEN_TILDE {
		p.advance()
		eps, err := p.parseFloat()
		if err != nil {
			return nil, err
		}
		if err := p.expectIdent("of"); err != nil {
			return nil, err
		}
		s, err := p.expect(TOKEN_STRING)
		if err != nil {
			return nil, err
		}
		return NumericEpsMatcher{Epsilon: eps, Value: s.Value}, nil
	}

	if t.Type == TOKEN_IDENT {
		switch t.Value {
		case "contains":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			return ContainsMatcher{Substr: s.Value}, nil

		case "matches":
			p.advance()
			s, err := p.expect(TOKEN_STRING)
			if err != nil {
				return nil, err
			}
			return RegexMatcher{Pattern: s.Value}, nil

		case "anyOrder":
			p.advance()
			if _, err := p.expect(TOKEN_LBRACE); err != nil {
				return nil, err
			}
			var lines []string
			for !p.isRBrace() {
				s, err := p.expect(TOKEN_STRING)
				if err != nil {
					return nil, err
				}
				lines = append(lines, s.Value)
			}
			if _, err := p.expect(TOKEN_RBRACE); err != nil {
				return nil, err
			}
			return AnyOrderMatcher{Lines: lines}, nil
		}
	}

	return nil, fmt.Errorf("%d:%d: expected matcher (=, ~, contains, matches, anyOrder), got %q", t.Line, t.Col, t.Value)
}

func (p *Parser) parsePattern() (*Pattern, error) {
	if err := p.expectIdent("pattern"); err != nil {
		return nil, err
	}
	if _, err := p.expect(TOKEN_LBRACE); err != nil {
		return nil, err
	}

	pat := &Pattern{}
	for !p.isRBrace() {
		t := p.peek()
		if t.Type != TOKEN_IDENT {
			return nil, fmt.Errorf("%d:%d: unexpected token in pattern", t.Line, t.Col)
		}
		p.advance()
		if _, err := p.expect(TOKEN_ASSIGN); err != nil {
			return nil, err
		}
		val, err := p.expect(TOKEN_STRING)
		if err != nil {
			return nil, err
		}
		switch t.Value {
		case "input":
			if pat.DirsGlob != "" {
				pat.InputFile = val.Value
			} else {
				pat.InputGlob = val.Value
			}
		case "output":
			if pat.DirsGlob != "" {
				pat.OutputFile = val.Value
			} else {
				pat.OutputGlob = val.Value
			}
		case "dirs":
			pat.DirsGlob = val.Value
		default:
			return nil, fmt.Errorf("%d:%d: unknown pattern field %q", t.Line, t.Col, t.Value)
		}
	}

	if _, err := p.expect(TOKEN_RBRACE); err != nil {
		return nil, err
	}
	return pat, nil
}

func (p *Parser) parseStringList() ([]string, error) {
	var args []string
	for p.peek().Type == TOKEN_STRING {
		t := p.advance()
		args = append(args, t.Value)
	}
	if len(args) == 0 {
		return nil, fmt.Errorf("%d:%d: expected at least one string", p.peek().Line, p.peek().Col)
	}
	return args, nil
}

func (p *Parser) parseFloat() (float64, error) {
	t := p.peek()
	if t.Type == TOKEN_FLOAT || t.Type == TOKEN_INT {
		p.advance()
		return strconv.ParseFloat(t.Value, 64)
	}
	return 0, fmt.Errorf("%d:%d: expected float, got %s", t.Line, t.Col, t.Type)
}

func (p *Parser) parseBool() (bool, error) {
	t := p.peek()
	if t.Type != TOKEN_IDENT {
		return false, fmt.Errorf("%d:%d: expected true/false, got %s %q", t.Line, t.Col, t.Type, t.Value)
	}
	switch t.Value {
	case "true":
		p.advance()
		return true, nil
	case "false":
		p.advance()
		return false, nil
	default:
		return false, fmt.Errorf("%d:%d: expected true/false, got %q", t.Line, t.Col, t.Value)
	}
}

func (p *Parser) parseInt() (int, error) {
	t, err := p.expect(TOKEN_INT)
	if err != nil {
		return 0, err
	}
	n, err := strconv.Atoi(t.Value)
	if err != nil {
		return 0, fmt.Errorf("%d:%d: invalid int %q", t.Line, t.Col, t.Value)
	}
	return n, nil
}

func (p *Parser) parseSize() (int64, error) {
	t := p.peek()
	switch t.Type {
	case TOKEN_SIZE:
		p.advance()
		return parseSizeLiteral(t.Value, t.Line, t.Col)
	case TOKEN_INT:
		p.advance()
		n, err := strconv.ParseInt(t.Value, 10, 64)
		if err != nil {
			return 0, fmt.Errorf("%d:%d: invalid size %q", t.Line, t.Col, t.Value)
		}
		return n, nil
	default:
		return 0, fmt.Errorf("%d:%d: expected size (e.g. 256MB, 1GiB), got %s %q", t.Line, t.Col, t.Type, t.Value)
	}
}

func parseSizeLiteral(s string, line, col int) (int64, error) {
	i := 0
	for i < len(s) && (s[i] >= '0' && s[i] <= '9') {
		i++
	}
	if i == 0 {
		return 0, fmt.Errorf("%d:%d: invalid size %q", line, col, s)
	}
	numPart := s[:i]
	unit := s[i:]
	n, err := strconv.ParseInt(numPart, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("%d:%d: invalid size %q", line, col, s)
	}
	var mult int64
	switch unit {
	case "", "B":
		mult = 1
	case "K", "KB", "KiB":
		mult = 1024
	case "M", "MB", "MiB":
		mult = 1024 * 1024
	case "G", "GB", "GiB":
		mult = 1024 * 1024 * 1024
	default:
		return 0, fmt.Errorf("%d:%d: unknown size unit %q (use B/K/M/G or KiB/MiB/GiB)", line, col, unit)
	}
	if n < 0 {
		return 0, fmt.Errorf("%d:%d: size must be non-negative", line, col)
	}
	return n * mult, nil
}

func (p *Parser) parseDuration() (time.Duration, error) {
	t := p.peek()
	if t.Type != TOKEN_DURATION {
		return 0, fmt.Errorf("%d:%d: expected duration (e.g. 10s, 2m, 500ms), got %s %q", t.Line, t.Col, t.Type, t.Value)
	}
	p.advance()
	d, err := time.ParseDuration(t.Value)
	if err != nil {
		return 0, fmt.Errorf("%d:%d: invalid duration %q: %w", t.Line, t.Col, t.Value, err)
	}
	return d, nil
}

func (p *Parser) isRBrace() bool {
	return p.peek().Type == TOKEN_RBRACE || p.peek().Type == TOKEN_EOF
}