From 128a64a609b0459c1a4181c17eb87afa1f56bb3a Mon Sep 17 00:00:00 2001 From: Mikhail Kornilovich Date: Sat, 11 Apr 2026 01:51:38 +0300 Subject: [PATCH] add new build system --- .gitea/workflows/build-dsl-smoke.yml | 162 ++++++++++ cmd/cli/main.go | 145 +++++++-- dsl/ast.go | 23 +- dsl/build.go | 187 +++++++++++ dsl/build_parser.go | 355 +++++++++++++++++++++ dsl/build_test.go | 455 +++++++++++++++++++++++++++ dsl/parser.go | 52 ++- example/c-sum-v2/solution.c | 19 ++ example/c-sum-v2/sum.jdg | 82 +++++ reporter/reporter.go | 179 ++++++++--- runner/compiler.go | 198 ++++++++++++ runner/compiler_test.go | 293 +++++++++++++++++ runner/result.go | 41 ++- runner/result_test.go | 57 ++++ runner/runner.go | 350 ++++++++++++++++----- 15 files changed, 2448 insertions(+), 150 deletions(-) create mode 100644 .gitea/workflows/build-dsl-smoke.yml create mode 100644 dsl/build.go create mode 100644 dsl/build_parser.go create mode 100644 dsl/build_test.go create mode 100644 example/c-sum-v2/solution.c create mode 100644 example/c-sum-v2/sum.jdg create mode 100644 runner/compiler.go create mode 100644 runner/compiler_test.go create mode 100644 runner/result_test.go diff --git a/.gitea/workflows/build-dsl-smoke.yml b/.gitea/workflows/build-dsl-smoke.yml new file mode 100644 index 0000000..8d3f48d --- /dev/null +++ b/.gitea/workflows/build-dsl-smoke.yml @@ -0,0 +1,162 @@ +name: build-dsl-smoke +run-name: "Structured build DSL smoke test" + +on: + push: + paths: + - 'dsl/**' + - 'runner/**' + - 'reporter/**' + - 'cmd/cli/**' + - 'example/c-sum-v2/**' + - '.gitea/workflows/build-dsl-smoke.yml' + workflow_dispatch: + +env: + SUITE_FILE: sum.jdg + EXAMPLE_DIR: example/c-sum-v2 + +jobs: + discover: + name: Discover matrix + runs-on: Linux-Runner + timeout-minutes: 5 + outputs: + matrix: ${{ steps.list.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: List matrix from .jdg + id: list + shell: bash + run: | + matrix=$(go run ./cmd/cli --list-matrix "$EXAMPLE_DIR/$SUITE_FILE") + echo "discovered: $matrix" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + build_judge: + name: Build judge + runs-on: Linux-Runner + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Cross-compile judge + shell: bash + run: | + mkdir -p dist + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/judge-linux-amd64 ./cmd/cli + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/judge-windows-amd64.exe ./cmd/cli + ls -la dist/ + + - name: Upload judge binaries + uses: https://github.com/christopherHX/gitea-upload-artifact@v4 + with: + name: judge-bin-csum2 + path: dist/ + retention-days: 1 + + test: + needs: [discover, build_judge] + name: "${{ matrix.cell.build }} / ${{ matrix.cell.toolchain }} / ${{ matrix.cell.platform }}" + + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.discover.outputs.matrix) }} + + runs-on: ${{ matrix.cell.platform == 'windows' && 'Windows-Runner' || 'Linux-Runner' }} + timeout-minutes: 10 + + env: + REPORT_NAME: "report_${{ matrix.cell.build }}_${{ matrix.cell.toolchain }}_${{ matrix.cell.platform }}" + JUDGE_TOOLCHAIN: ${{ matrix.cell.toolchain }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up MSVC environment + if: matrix.cell.toolchain == 'msvc' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Download judge binary + uses: https://github.com/christopherHX/gitea-download-artifact@v4 + with: + name: judge-bin-csum2 + path: judge-bin + + - name: Install judge on PATH + shell: bash + run: | + mkdir -p bin + if [ "${{ matrix.cell.platform }}" = "windows" ]; then + cp judge-bin/judge-windows-amd64.exe bin/judge.exe + else + cp judge-bin/judge-linux-amd64 bin/judge + chmod +x bin/judge + fi + echo "$PWD/bin" >> "$GITHUB_PATH" + + - name: Run judge + shell: bash + working-directory: ${{ env.EXAMPLE_DIR }} + run: | + judge "$SUITE_FILE" . --build=${{ matrix.cell.build }} --json > "$GITHUB_WORKSPACE/${REPORT_NAME}.json" || true + cat "$GITHUB_WORKSPACE/${REPORT_NAME}.json" + judge "$SUITE_FILE" . --build=${{ matrix.cell.build }} + + - name: Upload report + if: ${{ always() }} + uses: https://github.com/christopherHX/gitea-upload-artifact@v4 + with: + name: ${{ env.REPORT_NAME }} + path: ${{ env.REPORT_NAME }}.json + retention-days: 7 + compression-level: 9 + + summary: + needs: [build_judge, test] + if: ${{ always() }} + name: SUMMARY + runs-on: Linux-Runner + timeout-minutes: 5 + + steps: + - name: Download judge binary + uses: https://github.com/christopherHX/gitea-download-artifact@v4 + with: + name: judge-bin-csum2 + path: judge-bin + + - name: Install judge on PATH + shell: bash + run: | + mkdir -p bin + cp judge-bin/judge-linux-amd64 bin/judge + chmod +x bin/judge + echo "$PWD/bin" >> "$GITHUB_PATH" + + - name: Download all reports + uses: https://github.com/christopherHX/gitea-download-artifact@v4 + with: + path: reports + pattern: report_* + + - name: Aggregate + shell: bash + run: judge aggregate reports | tee SUMMARY.md + + - name: Upload summary + uses: https://github.com/christopherHX/gitea-upload-artifact@v4 + with: + name: SUMMARY + path: SUMMARY.md + retention-days: 7 diff --git a/cmd/cli/main.go b/cmd/cli/main.go index b6904d5..3e73253 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "os" "strings" @@ -20,11 +21,16 @@ Flags: --json output as JSON instead of text --wrapper exec wrapper (e.g. "valgrind --error-exitcode=99") --binary name of executable produced by build (overrides .jdg) + --build run only the named structured build (use with matrix CI) + --list-builds print the names of structured builds in the suite as JSON + --list-matrix print the full (build, toolchain, platform) matrix as JSON --help show help Example: judge lab1.jdg ./student-solution judge lab1.jdg ./student-solution --json + judge lab1.jdg ./student-solution --build=sanitized + judge --list-builds lab1.jdg judge aggregate reports/ ` @@ -41,9 +47,20 @@ func main() { return } + if hasFlag(args, "--list-builds") { + runListBuilds(args) + return + } + + if hasFlag(args, "--list-matrix") { + runListMatrix(args) + return + } + jsonOutput := hasFlag(args, "--json") wrapper := flagValue(args, "--wrapper") binary := flagValue(args, "--binary") + targetBuild := flagValue(args, "--build") positional := positionalArgs(args) if len(positional) < 2 { @@ -54,27 +71,17 @@ func main() { testFile := positional[0] solutionDir := positional[1] - src, err := os.ReadFile(testFile) - if err != nil { - fatalf("cannot read %q: %v", testFile, err) - } - - f, warns, err := dsl.Parse(string(src)) - if err != nil { - fatalf("parse error in %q:\n %v", testFile, err) - } - for _, w := range warns { - fmt.Fprintf(os.Stderr, "warning: %s\n", w) - } + f := parseSuite(testFile) if _, err := os.Stat(solutionDir); err != nil { fatalf("solution dir %q not found: %v", solutionDir, err) } r := runner.New(f, runner.Config{ - WorkDir: solutionDir, - BinaryName: binary, - Wrapper: wrapper, + WorkDir: solutionDir, + BinaryName: binary, + Wrapper: wrapper, + TargetBuild: targetBuild, }) result := r.Run() @@ -91,6 +98,98 @@ func main() { } } +func parseSuite(path string) *dsl.File { + src, err := os.ReadFile(path) + if err != nil { + fatalf("cannot read %q: %v", path, err) + } + f, warns, err := dsl.Parse(string(src)) + if err != nil { + fatalf("parse error in %q:\n %v", path, err) + } + for _, w := range warns { + fmt.Fprintf(os.Stderr, "warning: %s\n", w) + } + return f +} + +// runListBuilds prints a JSON array of build names for CI matrix discovery. +// A legacy suite without structured builds reports ["default"] so workflows +// can always iterate `fromJSON` and have exactly one cell. +func runListBuilds(args []string) { + positional := positionalArgs(args) + if len(positional) < 1 { + fatalf("--list-builds requires the path to a .jdg file") + } + f := parseSuite(positional[0]) + + var names []string + if len(f.Builds) == 0 { + names = []string{"default"} + } else { + for _, b := range f.Builds { + names = append(names, b.Name) + } + } + enc := json.NewEncoder(os.Stdout) + if err := enc.Encode(names); err != nil { + fatalf("encode list-builds: %v", err) + } +} + +type matrixEntry struct { + Build string `json:"build"` + Toolchain string `json:"toolchain"` + Platform string `json:"platform"` + Wrapper string `json:"wrapper,omitempty"` +} + +func runListMatrix(args []string) { + positional := positionalArgs(args) + if len(positional) < 1 { + fatalf("--list-matrix requires the path to a .jdg file") + } + f := parseSuite(positional[0]) + + var entries []matrixEntry + + if len(f.Builds) == 0 { + if len(f.Toolchains) == 0 { + entries = append(entries, matrixEntry{Build: "default", Toolchain: "default", Platform: "linux"}) + } else { + for _, tc := range f.Toolchains { + for _, platform := range tc.Platforms { + entries = append(entries, matrixEntry{Build: "default", Toolchain: tc.Name, Platform: platform}) + } + } + } + } else if len(f.Toolchains) == 0 { + fatalf("suite has structured builds but no `toolchains { ... }` block; add one to use --list-matrix") + } else { + for _, b := range f.Builds { + for _, tc := range f.Toolchains { + for _, platform := range tc.Platforms { + eff := b.Resolve(f.BuildDefaults, platform) + if !eff.AppliesTo(platform, tc.Name) { + continue + } + entries = append(entries, matrixEntry{ + Build: b.Name, + Toolchain: tc.Name, + Platform: platform, + Wrapper: eff.Wrapper, + }) + } + } + } + } + + enc := json.NewEncoder(os.Stdout) + if err := enc.Encode(entries); err != nil { + fatalf("encode list-matrix: %v", err) + } +} + func runAggregate(args []string) { if len(args) < 1 { fatalf("usage: judge aggregate ") @@ -128,8 +227,18 @@ func flagValue(args []string, name string) string { } func positionalArgs(args []string) []string { - known := map[string]bool{"--json": true, "--help": true, "-h": true} - withValue := map[string]bool{"--wrapper": true, "--binary": true} + known := map[string]bool{ + "--json": true, + "--help": true, + "-h": true, + "--list-builds": true, + "--list-matrix": true, + } + withValue := map[string]bool{ + "--wrapper": true, + "--binary": true, + "--build": true, + } var out []string skip := false @@ -145,7 +254,7 @@ func positionalArgs(args []string) []string { skip = true continue } - if strings.HasPrefix(a, "--wrapper=") || strings.HasPrefix(a, "--binary=") { + if strings.HasPrefix(a, "--wrapper=") || strings.HasPrefix(a, "--binary=") || strings.HasPrefix(a, "--build=") { continue } out = append(out, a) diff --git a/dsl/ast.go b/dsl/ast.go index cb4a0f9..b9b5b70 100644 --- a/dsl/ast.go +++ b/dsl/ast.go @@ -7,13 +7,18 @@ type File struct { BuildLinux string BuildWindows string BuildDarwin string - Timeout time.Duration - MemoryLimit int64 // bytes; 0 means no limit - Binary string // executable name produced by build (default: solution) - Sources string // glob pattern for source files, expanded as $SOURCES in build - NormalizeCRLF bool // strip \r before matching stdout/stderr/outFiles - TrimTrailingWS bool // trim trailing whitespace on each line before matching + BuildDefaults *BuildConfig + Builds []*BuildConfig + Toolchains []*ToolchainSpec + + Timeout time.Duration + MemoryLimit int64 + Binary string + Sources string + + NormalizeCRLF bool + TrimTrailingWS bool Groups []*Group } @@ -25,7 +30,7 @@ type Group struct { MemoryLimit int64 Env map[string]string Scoring ScoringMode - Wrapper string // exec wrapper command (e.g., "valgrind --error-exitcode=1") + Wrapper string Tests []*Test Pattern *Pattern @@ -34,8 +39,8 @@ type Group struct { type ScoringMode int const ( - ScoringPartial ScoringMode = iota // weight * passed/total (default) - ScoringAllOrNone // weight or 0 + ScoringPartial ScoringMode = iota + ScoringAllOrNone ) type Pattern struct { diff --git a/dsl/build.go b/dsl/build.go new file mode 100644 index 0000000..85b42a0 --- /dev/null +++ b/dsl/build.go @@ -0,0 +1,187 @@ +package dsl + +// BuildProfile is a named compilation profile. The translator maps it into +// compiler-specific flag sets at execution time (e.g. ProfileRelease → "-O2" +// on gnu-like compilers, "/O2" on msvc). +type BuildProfile int + +const ( + ProfileUnset BuildProfile = iota + ProfileRelease + ProfileDebug + ProfileSanitized +) + +func (p BuildProfile) String() string { + switch p { + case ProfileRelease: + return "release" + case ProfileDebug: + return "debug" + case ProfileSanitized: + return "sanitized" + default: + return "unset" + } +} + +// WarningLevel describes how strict the compiler should be about warnings. +type WarningLevel int + +const ( + WarningsUnset WarningLevel = iota + WarningsDefault + WarningsStrict + WarningsPedantic +) + +func (w WarningLevel) String() string { + switch w { + case WarningsDefault: + return "default" + case WarningsStrict: + return "strict" + case WarningsPedantic: + return "pedantic" + default: + return "unset" + } +} + +// BuildConfig describes one structured build variant. It is the new-style +// replacement for the free-form `build "shell-string"` field. +// +// A top-level `build_defaults { ... }` in the suite file produces a +// BuildConfig stored on File.BuildDefaults. Each `build "name" { ... }` +// block produces an entry in File.Builds; the effective configuration used +// by the runner is BuildDefaults merged with the named block, then merged +// with the OS-specific override (Linux / Windows / Darwin) when present. +// +// Zero-valued fields inherit from the parent during merge. Slice and map +// fields accumulate rather than replace. +type BuildConfig struct { + // Name of the variant. Empty on BuildDefaults and on OS-override sub-blocks. + Name string + + Language string // e.g. "c", "c++" + Standard string // e.g. "c11", "c++17" + Sources []string // globs, relative to work dir + Includes []string // include search paths + Output string // binary name (OS-specific extension added automatically) + + Profile BuildProfile + Warnings WarningLevel + Sanitize []string + Wrapper string // e.g. "address", "undefined", "thread" + + Defines map[string]string + Link []string // libraries to link against (e.g. "pthread", "m") + Extra []string // raw passthrough flags + + // Filters — empty means "applies to any". A build is skipped at runtime + // if the current OS or compiler is not in the list. + Platforms []string // "linux", "windows", "darwin" + Compilers []string // "gcc", "clang", "msvc" + + // OS-specific overrides. Only one level of nesting is allowed: these + // sub-configs must not themselves contain Linux/Windows/Darwin blocks. + Linux *BuildConfig + Windows *BuildConfig + Darwin *BuildConfig +} + +// MergeFrom layers src on top of dst in place. Non-zero scalar fields in src +// overwrite dst; slices and maps accumulate. The Name and OS override fields +// on src are intentionally ignored — merging never copies the hierarchy, +// only the leaves. +func (dst *BuildConfig) MergeFrom(src *BuildConfig) { + if src == nil { + return + } + if src.Language != "" { + dst.Language = src.Language + } + if src.Standard != "" { + dst.Standard = src.Standard + } + if src.Output != "" { + dst.Output = src.Output + } + if src.Profile != ProfileUnset { + dst.Profile = src.Profile + } + if src.Warnings != WarningsUnset { + dst.Warnings = src.Warnings + } + if src.Wrapper != "" { + dst.Wrapper = src.Wrapper + } + + dst.Sources = append(dst.Sources, src.Sources...) + dst.Includes = append(dst.Includes, src.Includes...) + dst.Sanitize = append(dst.Sanitize, src.Sanitize...) + dst.Link = append(dst.Link, src.Link...) + dst.Extra = append(dst.Extra, src.Extra...) + dst.Platforms = append(dst.Platforms, src.Platforms...) + dst.Compilers = append(dst.Compilers, src.Compilers...) + + if len(src.Defines) > 0 { + if dst.Defines == nil { + dst.Defines = map[string]string{} + } + for k, v := range src.Defines { + dst.Defines[k] = v + } + } +} + +// Resolve returns the effective BuildConfig for the given OS by merging +// BuildDefaults → this block → the matching OS override. The result is a +// fresh value; the receiver is not mutated. +func (b *BuildConfig) Resolve(defaults *BuildConfig, os string) BuildConfig { + var out BuildConfig + out.MergeFrom(defaults) + out.MergeFrom(b) + out.Name = b.Name + + var osOverride *BuildConfig + switch os { + case "linux": + osOverride = b.Linux + case "windows": + osOverride = b.Windows + case "darwin": + osOverride = b.Darwin + } + out.MergeFrom(osOverride) + + return out +} + +// AppliesTo reports whether this build should run on (os, compiler). +// An empty Platforms/Compilers list means no filter on that axis. +func (b *BuildConfig) AppliesTo(os, compiler string) bool { + if len(b.Platforms) > 0 && !contains(b.Platforms, os) { + return false + } + if len(b.Compilers) > 0 && !contains(b.Compilers, compiler) { + return false + } + return true +} + +func contains(xs []string, x string) bool { + for _, v := range xs { + if v == x { + return true + } + } + return false +} + +type ToolchainSpec struct { + Name string + Platforms []string + Binary string + Class string +} diff --git a/dsl/build_parser.go b/dsl/build_parser.go new file mode 100644 index 0000000..aa204cc --- /dev/null +++ b/dsl/build_parser.go @@ -0,0 +1,355 @@ +package dsl + +import "fmt" + +func (p *Parser) parseBuildBlock(name string) (*BuildConfig, error) { + return p.parseBuildBlockInner(name, false) +} + +func (p *Parser) parseBuildBlockInner(name string, inOSOverride bool) (*BuildConfig, error) { + if _, err := p.expect(TOKEN_LBRACE); err != nil { + return nil, err + } + + bc := &BuildConfig{Name: name} + + for !p.isRBrace() { + t := p.peek() + if t.Type != TOKEN_IDENT { + return nil, fmt.Errorf("%d:%d: unexpected token %q in build block", t.Line, t.Col, t.Value) + } + + switch t.Value { + case "language": + p.advance() + s, err := p.parseAssignString() + if err != nil { + return nil, err + } + bc.Language = s + + case "standard": + p.advance() + s, err := p.parseAssignString() + if err != nil { + return nil, err + } + bc.Standard = s + + case "output": + p.advance() + s, err := p.parseAssignString() + if err != nil { + return nil, err + } + bc.Output = s + + case "wrapper": + p.advance() + s, err := p.parseAssignString() + if err != nil { + return nil, err + } + bc.Wrapper = s + + case "sources": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Sources = xs + + case "includes": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Includes = xs + + case "sanitize": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Sanitize = xs + + case "link": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Link = xs + + case "extra": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Extra = xs + + case "platforms": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + if err := validatePlatformList(xs, t.Line, t.Col); err != nil { + return nil, err + } + bc.Platforms = xs + + case "compilers": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + bc.Compilers = xs + + case "profile": + p.advance() + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return nil, err + } + id, err := p.expect(TOKEN_IDENT) + if err != nil { + return nil, err + } + prof, err := parseProfileIdent(id.Value, id.Line, id.Col) + if err != nil { + return nil, err + } + bc.Profile = prof + + case "warnings": + p.advance() + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return nil, err + } + id, err := p.expect(TOKEN_IDENT) + if err != nil { + return nil, err + } + w, err := parseWarningsIdent(id.Value, id.Line, id.Col) + if err != nil { + return nil, err + } + bc.Warnings = w + + case "define": + p.advance() + if _, err := p.expect(TOKEN_LPAREN); err != nil { + return nil, err + } + key, err := p.expect(TOKEN_STRING) + if err != nil { + return nil, err + } + if _, err := p.expect(TOKEN_RPAREN); err != nil { + return nil, err + } + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return nil, err + } + val, err := p.expect(TOKEN_STRING) + if err != nil { + return nil, err + } + if bc.Defines == nil { + bc.Defines = map[string]string{} + } + bc.Defines[key.Value] = val.Value + + case "linux", "windows", "darwin": + if inOSOverride { + return nil, fmt.Errorf("%d:%d: OS override %q cannot be nested inside another OS override", t.Line, t.Col, t.Value) + } + osName := t.Value + p.advance() + sub, err := p.parseBuildBlockInner("", true) + if err != nil { + return nil, err + } + switch osName { + case "linux": + if bc.Linux != nil { + return nil, fmt.Errorf("%d:%d: duplicate linux override", t.Line, t.Col) + } + bc.Linux = sub + case "windows": + if bc.Windows != nil { + return nil, fmt.Errorf("%d:%d: duplicate windows override", t.Line, t.Col) + } + bc.Windows = sub + case "darwin": + if bc.Darwin != nil { + return nil, fmt.Errorf("%d:%d: duplicate darwin override", t.Line, t.Col) + } + bc.Darwin = sub + } + + default: + return nil, fmt.Errorf("%d:%d: unknown field %q in build block", t.Line, t.Col, t.Value) + } + } + + if _, err := p.expect(TOKEN_RBRACE); err != nil { + return nil, err + } + return bc, nil +} + +func (p *Parser) parseAssignString() (string, error) { + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return "", err + } + s, err := p.expect(TOKEN_STRING) + if err != nil { + return "", err + } + return s.Value, nil +} + +func (p *Parser) parseAssignStringList() ([]string, error) { + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return nil, err + } + return p.parseStringList() +} + +func parseProfileIdent(v string, line, col int) (BuildProfile, error) { + switch v { + case "release": + return ProfileRelease, nil + case "debug": + return ProfileDebug, nil + case "sanitized": + return ProfileSanitized, nil + default: + return ProfileUnset, fmt.Errorf("%d:%d: unknown profile %q (expected release/debug/sanitized)", line, col, v) + } +} + +func parseWarningsIdent(v string, line, col int) (WarningLevel, error) { + switch v { + case "default": + return WarningsDefault, nil + case "strict": + return WarningsStrict, nil + case "pedantic": + return WarningsPedantic, nil + default: + return WarningsUnset, fmt.Errorf("%d:%d: unknown warnings level %q (expected default/strict/pedantic)", line, col, v) + } +} + +func validatePlatformList(xs []string, line, col int) error { + for _, x := range xs { + switch x { + case "linux", "windows", "darwin": + default: + return fmt.Errorf("%d:%d: unknown platform %q (expected linux/windows/darwin)", line, col, x) + } + } + return nil +} + +func (p *Parser) parseToolchainsBlock() ([]*ToolchainSpec, error) { + if _, err := p.expect(TOKEN_LBRACE); err != nil { + return nil, err + } + + var specs []*ToolchainSpec + seen := map[string]bool{} + + for !p.isRBrace() { + nameTok := p.peek() + if nameTok.Type != TOKEN_IDENT && nameTok.Type != TOKEN_STRING { + return nil, fmt.Errorf("%d:%d: expected toolchain name, got %q", nameTok.Line, nameTok.Col, nameTok.Value) + } + p.advance() + name := nameTok.Value + if seen[name] { + return nil, fmt.Errorf("%d:%d: duplicate toolchain %q", nameTok.Line, nameTok.Col, name) + } + seen[name] = true + + spec, err := p.parseToolchainEntry(name) + if err != nil { + return nil, err + } + specs = append(specs, spec) + } + + if _, err := p.expect(TOKEN_RBRACE); err != nil { + return nil, err + } + return specs, nil +} + +func (p *Parser) parseToolchainEntry(name string) (*ToolchainSpec, error) { + if _, err := p.expect(TOKEN_LBRACE); err != nil { + return nil, err + } + + spec := &ToolchainSpec{Name: name} + for !p.isRBrace() { + t := p.peek() + if t.Type != TOKEN_IDENT { + return nil, fmt.Errorf("%d:%d: unexpected token %q in toolchain block", t.Line, t.Col, t.Value) + } + switch t.Value { + case "platforms": + p.advance() + xs, err := p.parseAssignStringList() + if err != nil { + return nil, err + } + if err := validatePlatformList(xs, t.Line, t.Col); err != nil { + return nil, err + } + spec.Platforms = xs + + case "binary": + p.advance() + s, err := p.parseAssignString() + if err != nil { + return nil, err + } + spec.Binary = s + + case "class": + p.advance() + if _, err := p.expect(TOKEN_ASSIGN); err != nil { + return nil, err + } + id, err := p.expect(TOKEN_IDENT) + if err != nil { + return nil, err + } + switch id.Value { + case "gnu", "msvc": + default: + return nil, fmt.Errorf("%d:%d: unknown compiler class %q (expected gnu/msvc)", id.Line, id.Col, id.Value) + } + spec.Class = id.Value + + default: + return nil, fmt.Errorf("%d:%d: unknown field %q in toolchain block", t.Line, t.Col, t.Value) + } + } + + if _, err := p.expect(TOKEN_RBRACE); err != nil { + return nil, err + } + + if len(spec.Platforms) == 0 { + return nil, fmt.Errorf("toolchain %q: platforms is required", name) + } + return spec, nil +} diff --git a/dsl/build_test.go b/dsl/build_test.go new file mode 100644 index 0000000..c1f08f7 --- /dev/null +++ b/dsl/build_test.go @@ -0,0 +1,455 @@ +package dsl + +import ( + "strings" + "testing" +) + +func TestParseBuildBlockMinimal(t *testing.T) { + src := ` +build_defaults { + language = "c" + standard = "c11" + sources = "*.c" + output = "solution" + warnings = strict +} + +build "release" { + profile = release +} + +build "debug" { + profile = debug +} + +group("g1") { + weight = 1.0 + test("t1") { + stdout = "ok\n" + } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + + if f.BuildDefaults == nil { + t.Fatal("BuildDefaults not populated") + } + if f.BuildDefaults.Language != "c" { + t.Errorf("language = %q", f.BuildDefaults.Language) + } + if f.BuildDefaults.Standard != "c11" { + t.Errorf("standard = %q", f.BuildDefaults.Standard) + } + if len(f.BuildDefaults.Sources) != 1 || f.BuildDefaults.Sources[0] != "*.c" { + t.Errorf("sources = %v", f.BuildDefaults.Sources) + } + if f.BuildDefaults.Output != "solution" { + t.Errorf("output = %q", f.BuildDefaults.Output) + } + if f.BuildDefaults.Warnings != WarningsStrict { + t.Errorf("warnings = %v", f.BuildDefaults.Warnings) + } + + if len(f.Builds) != 2 { + t.Fatalf("expected 2 builds, got %d", len(f.Builds)) + } + if f.Builds[0].Name != "release" || f.Builds[0].Profile != ProfileRelease { + t.Errorf("builds[0] = %+v", f.Builds[0]) + } + if f.Builds[1].Name != "debug" || f.Builds[1].Profile != ProfileDebug { + t.Errorf("builds[1] = %+v", f.Builds[1]) + } +} + +func TestParseBuildLegacyStillWorks(t *testing.T) { + src := ` +build "cc -O2 solution.c -o solution" +timeout 5s + +group("g1") { + weight = 1.0 + test("t1") { stdout = "ok\n" } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + if f.Build != "cc -O2 solution.c -o solution" { + t.Errorf("legacy build = %q", f.Build) + } + if f.BuildDefaults != nil || len(f.Builds) != 0 { + t.Errorf("structured fields should be empty for legacy form") + } +} + +func TestParseBuildOSOverride(t *testing.T) { + src := ` +build_defaults { + language = "c" + standard = "c11" + sources = "*.c" + output = "solution" +} + +build "release" { + profile = release + linux { extra = "-fPIC" } + windows { extra = "/bigobj" } +} + +group("g1") { + weight = 1.0 + test("t1") { stdout = "ok\n" } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + b := f.Builds[0] + if b.Linux == nil || len(b.Linux.Extra) != 1 || b.Linux.Extra[0] != "-fPIC" { + t.Errorf("linux override = %+v", b.Linux) + } + if b.Windows == nil || len(b.Windows.Extra) != 1 || b.Windows.Extra[0] != "/bigobj" { + t.Errorf("windows override = %+v", b.Windows) + } + if b.Darwin != nil { + t.Errorf("darwin override should be nil") + } +} + +func TestParseBuildPlatformsFilter(t *testing.T) { + src := ` +build "sanitized" { + profile = sanitized + sanitize = "address" "undefined" + platforms = "linux" + compilers = "gcc" "clang" +} + +group("g1") { + weight = 1.0 + test("t1") { stdout = "ok\n" } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + b := f.Builds[0] + if b.Profile != ProfileSanitized { + t.Errorf("profile = %v", b.Profile) + } + if len(b.Sanitize) != 2 || b.Sanitize[0] != "address" || b.Sanitize[1] != "undefined" { + t.Errorf("sanitize = %v", b.Sanitize) + } + if len(b.Platforms) != 1 || b.Platforms[0] != "linux" { + t.Errorf("platforms = %v", b.Platforms) + } + if len(b.Compilers) != 2 { + t.Errorf("compilers = %v", b.Compilers) + } +} + +func TestParseBuildDefineField(t *testing.T) { + src := ` +build "rel" { + profile = release + define("NDEBUG") = "1" + define("VERSION") = "42" +} + +group("g1") { + weight = 1.0 + test("t1") { stdout = "ok\n" } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + d := f.Builds[0].Defines + if d["NDEBUG"] != "1" || d["VERSION"] != "42" { + t.Errorf("defines = %v", d) + } +} + +func TestParseBuildErrors(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + { + name: "unknown profile", + src: ` +build "x" { profile = ultra } +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "unknown profile", + }, + { + name: "unknown warnings", + src: ` +build "x" { warnings = insane } +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "unknown warnings", + }, + { + name: "unknown platform", + src: ` +build "x" { platforms = "bsd" } +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "unknown platform", + }, + { + name: "nested OS override", + src: ` +build "x" { + linux { + windows { extra = "nope" } + } +} +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "cannot be nested", + }, + { + name: "duplicate OS override", + src: ` +build "x" { + linux { extra = "-a" } + linux { extra = "-b" } +} +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "duplicate linux override", + }, + { + name: "unknown field", + src: ` +build "x" { magic = "yes" } +group("g") { weight=1.0 test("t"){ stdout="ok\n" } } +`, + want: "unknown field", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, _, err := Parse(c.src) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("error %q does not contain %q", err.Error(), c.want) + } + }) + } +} + +func TestParseBuildMixedLegacyAndStructuredRejected(t *testing.T) { + src := ` +build "cc -O2 solution.c -o solution" +build "release" { profile = release } +group("g") { weight = 1.0 test("t") { stdout = "ok\n" } } +` + _, _, err := Parse(src) + if err == nil { + t.Fatal("expected error mixing legacy and structured builds") + } + if !strings.Contains(err.Error(), "cannot mix") { + t.Errorf("error %q does not mention mixing", err.Error()) + } +} + +func TestParseBuildDuplicateNameRejected(t *testing.T) { + src := ` +build "release" { profile = release } +build "release" { profile = debug } +group("g") { weight = 1.0 test("t") { stdout = "ok\n" } } +` + _, _, err := Parse(src) + if err == nil { + t.Fatal("expected error on duplicate build name") + } + if !strings.Contains(err.Error(), "duplicate") { + t.Errorf("error %q does not mention duplicate", err.Error()) + } +} + +func TestBuildConfigResolveMerge(t *testing.T) { + defaults := &BuildConfig{ + Language: "c", + Standard: "c11", + Sources: []string{"*.c"}, + Output: "solution", + Warnings: WarningsStrict, + } + b := &BuildConfig{ + Name: "release", + Profile: ProfileRelease, + Extra: []string{"-DFOO"}, + Linux: &BuildConfig{ + Extra: []string{"-fPIC"}, + }, + Windows: &BuildConfig{ + Extra: []string{"/bigobj"}, + Output: "solution", + }, + } + + linux := b.Resolve(defaults, "linux") + if linux.Language != "c" || linux.Standard != "c11" { + t.Errorf("linux defaults not merged: %+v", linux) + } + if linux.Profile != ProfileRelease { + t.Errorf("linux profile = %v", linux.Profile) + } + if linux.Warnings != WarningsStrict { + t.Errorf("linux warnings = %v", linux.Warnings) + } + if len(linux.Sources) != 1 || linux.Sources[0] != "*.c" { + t.Errorf("linux sources = %v", linux.Sources) + } + if len(linux.Extra) != 2 || linux.Extra[0] != "-DFOO" || linux.Extra[1] != "-fPIC" { + t.Errorf("linux extra = %v", linux.Extra) + } + + windows := b.Resolve(defaults, "windows") + if len(windows.Extra) != 2 || windows.Extra[0] != "-DFOO" || windows.Extra[1] != "/bigobj" { + t.Errorf("windows extra = %v", windows.Extra) + } + + darwin := b.Resolve(defaults, "darwin") + if len(darwin.Extra) != 1 || darwin.Extra[0] != "-DFOO" { + t.Errorf("darwin extra (no override) = %v", darwin.Extra) + } + + if len(b.Extra) != 1 { + t.Errorf("receiver mutated: %v", b.Extra) + } +} + +func TestParseToolchainsBlock(t *testing.T) { + src := ` +toolchains { + gcc { platforms = "linux" } + clang { + platforms = "linux" "windows" + binary = "clang-17" + class = gnu + } + msvc { platforms = "windows" } + nvcc { + platforms = "linux" + class = gnu + } +} + +build "release" { profile = release } + +group("g") { weight = 1.0 test("t") { stdout = "ok\n" } } +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(f.Toolchains) != 4 { + t.Fatalf("expected 4 toolchains, got %d", len(f.Toolchains)) + } + gcc := f.Toolchains[0] + if gcc.Name != "gcc" || len(gcc.Platforms) != 1 || gcc.Platforms[0] != "linux" { + t.Errorf("gcc = %+v", gcc) + } + clang := f.Toolchains[1] + if clang.Name != "clang" || len(clang.Platforms) != 2 || clang.Binary != "clang-17" || clang.Class != "gnu" { + t.Errorf("clang = %+v", clang) + } + if clang.Platforms[0] != "linux" || clang.Platforms[1] != "windows" { + t.Errorf("clang platforms = %v", clang.Platforms) + } + nvcc := f.Toolchains[3] + if nvcc.Name != "nvcc" || len(nvcc.Platforms) != 1 || nvcc.Platforms[0] != "linux" || nvcc.Class != "gnu" { + t.Errorf("nvcc = %+v", nvcc) + } +} + +func TestParseToolchainsErrors(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + { + "missing platforms", + `toolchains { gcc { } } + group("g") { weight=1.0 test("t") { stdout="ok\n" } }`, + "platforms is required", + }, + { + "unknown platform", + `toolchains { gcc { platforms = "bsd" } } + group("g") { weight=1.0 test("t") { stdout="ok\n" } }`, + "unknown platform", + }, + { + "unknown class", + `toolchains { gcc { platforms = "linux" class = llvm } } + group("g") { weight=1.0 test("t") { stdout="ok\n" } }`, + "unknown compiler class", + }, + { + "duplicate toolchain", + `toolchains { + gcc { platforms = "linux" } + gcc { platforms = "linux" } + } + group("g") { weight=1.0 test("t") { stdout="ok\n" } }`, + "duplicate toolchain", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, _, err := Parse(c.src) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), c.want) { + t.Errorf("error %q does not contain %q", err.Error(), c.want) + } + }) + } +} + +func TestBuildConfigAppliesTo(t *testing.T) { + cases := []struct { + name string + b BuildConfig + os, cc string + expected bool + }{ + {"no filters", BuildConfig{}, "linux", "gcc", true}, + {"os match", BuildConfig{Platforms: []string{"linux"}}, "linux", "gcc", true}, + {"os mismatch", BuildConfig{Platforms: []string{"linux"}}, "windows", "msvc", false}, + {"cc match", BuildConfig{Compilers: []string{"gcc", "clang"}}, "linux", "clang", true}, + {"cc mismatch", BuildConfig{Compilers: []string{"gcc"}}, "linux", "msvc", false}, + {"both match", BuildConfig{Platforms: []string{"linux"}, Compilers: []string{"gcc"}}, "linux", "gcc", true}, + {"os ok cc bad", BuildConfig{Platforms: []string{"linux"}, Compilers: []string{"gcc"}}, "linux", "clang", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := c.b.AppliesTo(c.os, c.cc) + if got != c.expected { + t.Errorf("AppliesTo(%q, %q) = %v, want %v", c.os, c.cc, got, c.expected) + } + }) + } +} diff --git a/dsl/parser.go b/dsl/parser.go index ba85de4..e93e9f4 100644 --- a/dsl/parser.go +++ b/dsl/parser.go @@ -91,7 +91,31 @@ func (p *Parser) parseFile() (*File, error) { if err != nil { return nil, err } - f.Build = s.Value + if p.peek().Type == TOKEN_LBRACE { + bc, err := p.parseBuildBlock(s.Value) + if err != nil { + return nil, err + } + f.Builds = append(f.Builds, bc) + } else { + f.Build = s.Value + } + + case "build_defaults": + p.advance() + bc, err := p.parseBuildBlock("") + if err != nil { + return nil, err + } + f.BuildDefaults = bc + + case "toolchains": + p.advance() + specs, err := p.parseToolchainsBlock() + if err != nil { + return nil, err + } + f.Toolchains = specs case "build_linux": p.advance() @@ -195,10 +219,34 @@ func (p *Parser) parseFile() (*File, error) { if err := p.validateWeights(f); err != nil { return nil, err } + if err := validateBuilds(f); err != nil { + return nil, err + } return f, nil } +func validateBuilds(f *File) error { + hasLegacy := f.Build != "" || f.BuildLinux != "" || f.BuildWindows != "" || f.BuildDarwin != "" + hasStructured := f.BuildDefaults != nil || len(f.Builds) > 0 + + if hasLegacy && hasStructured { + return fmt.Errorf("cannot mix legacy `build \"shell\"` with structured `build \"name\" { ... }` in the same suite") + } + + seen := map[string]bool{} + for _, b := range f.Builds { + if b.Name == "" { + return fmt.Errorf("structured build must have a name") + } + if seen[b.Name] { + return fmt.Errorf("duplicate build name %q", b.Name) + } + seen[b.Name] = true + } + return nil +} + func (p *Parser) validateWeights(f *File) error { if len(f.Groups) == 0 { return nil @@ -715,8 +763,6 @@ func (p *Parser) parseInt() (int, error) { return n, nil } -// parseSize accepts either a TOKEN_SIZE (e.g. "256MB", "1GiB", "512K") or a bare -// TOKEN_INT interpreted as bytes. MiB/MB are both 1024² — we use IEC semantics. func (p *Parser) parseSize() (int64, error) { t := p.peek() switch t.Type { diff --git a/example/c-sum-v2/solution.c b/example/c-sum-v2/solution.c new file mode 100644 index 0000000..aa273bb --- /dev/null +++ b/example/c-sum-v2/solution.c @@ -0,0 +1,19 @@ +#include +#include + +int main(void) { + long n; + if (scanf("%ld", &n) != 1) { + return 1; + } + long long sum = 0; + for (long i = 0; i < n; i++) { + long long x; + if (scanf("%lld", &x) != 1) { + return 1; + } + sum += x; + } + printf("%lld\n", sum); + return 0; +} diff --git a/example/c-sum-v2/sum.jdg b/example/c-sum-v2/sum.jdg new file mode 100644 index 0000000..12884ea --- /dev/null +++ b/example/c-sum-v2/sum.jdg @@ -0,0 +1,82 @@ +// Smoke test for the structured build DSL. Same task as example/c-sum +// (sum of N integers) but described declaratively — no shell strings, +// no per-platform `if /I "%CC%"=="msvc"` branching. The compiler is +// selected via the JUDGE_CC env variable set by the CI matrix. + +toolchains { + gcc { platforms = "linux" } + clang { platforms = "linux" "windows" } + msvc { platforms = "windows" } +} + +build_defaults { + language = "c" + standard = "c11" + sources = "solution.c" + output = "solution" + warnings = strict +} + +build "release" { + profile = release +} + +build "debug" { + profile = debug +} + +build "sanitized" { + profile = sanitized + sanitize = "address" "undefined" + platforms = "linux" + compilers = "gcc" "clang" +} + +build "debug-valgrind" { + profile = debug + wrapper = "valgrind --error-exitcode=99 --leak-check=full -q" + platforms = "linux" + compilers = "gcc" +} + +timeout 5s +normalize_crlf = true +trim_trailing_ws = true + +group("basic") { + weight = 0.5 + + test("one number") { + stdin = "1\n42\n" + stdout = "42\n" + } + + test("three numbers") { + stdin = "3\n1 2 3\n" + stdout = "6\n" + } + + test("negatives") { + stdin = "4\n-1 -2 3 5\n" + stdout = "5\n" + } + + test("zero count") { + stdin = "0\n" + stdout = "0\n" + } +} + +group("edge") { + weight = 0.5 + + test("large sum fits in int64") { + stdin = "3\n2000000000 2000000000 2000000000\n" + stdout = "6000000000\n" + } + + test("multiline input") { + stdin = "5\n10\n20\n30\n40\n50\n" + stdout = "150\n" + } +} diff --git a/reporter/reporter.go b/reporter/reporter.go index 021c096..7b50976 100644 --- a/reporter/reporter.go +++ b/reporter/reporter.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "math" "os" "path/filepath" "sort" @@ -13,11 +14,47 @@ import ( ) func Text(w io.Writer, result *runner.SuiteResult) { - if result.BuildLog != "" { - fmt.Fprintf(w, "=== BUILD LOG ===\n%s\n", result.BuildLog) + if len(result.Builds) == 0 { + fmt.Fprintln(w, "(no builds executed)") + return } - for _, gr := range result.Groups { + multi := len(result.Builds) > 1 + if multi { + fmt.Fprintf(w, "=== %d builds ===\n", len(result.Builds)) + for _, b := range result.Builds { + marker := "" + if b.Skipped { + marker = " (skipped)" + } + fmt.Fprintf(w, " • %s%s score=%.4f\n", b.Name, marker, b.TotalScore) + } + fmt.Fprintln(w) + } + + for _, b := range result.Builds { + if multi { + fmt.Fprintf(w, "======== build %q ========\n", b.Name) + } + writeBuildText(w, b, multi) + } + + if multi { + fmt.Fprintf(w, "\n══ AGGREGATE SCORE (min across builds): %.4f / 1.0000 ══\n", result.TotalScore) + } +} + +func writeBuildText(w io.Writer, b *runner.BuildRun, multi bool) { + if b.Skipped { + fmt.Fprintf(w, "skipped: %s\n", b.SkipReason) + return + } + + if b.BuildLog != "" { + fmt.Fprintf(w, "=== BUILD LOG ===\n%s\n", b.BuildLog) + } + + for _, gr := range b.Groups { passed := gr.Passed total := gr.Total pct := 0.0 @@ -53,13 +90,19 @@ func Text(w io.Writer, result *runner.SuiteResult) { fmt.Fprintf(w, "└─\n") } - fmt.Fprintf(w, "\n══ TOTAL SCORE: %.4f / 1.0000 ══\n", result.TotalScore) + if !multi { + fmt.Fprintf(w, "\n══ TOTAL SCORE: %.4f / 1.0000 ══\n", b.TotalScore) + } } func JSON(w io.Writer, result *runner.SuiteResult) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") - return enc.Encode(jsonResult(result)) + + if len(result.Builds) <= 1 { + return enc.Encode(flatResult(result)) + } + return enc.Encode(nestedResult(result)) } type jsonSuiteResult struct { @@ -68,6 +111,21 @@ type jsonSuiteResult struct { Groups []jsonGroupResult `json:"groups"` } +type jsonMultiSuiteResult struct { + TotalScore float64 `json:"total_score"` + Builds map[string]jsonSuiteBuild `json:"builds"` +} + +type jsonSuiteBuild struct { + Name string `json:"name"` + Toolchain string `json:"toolchain,omitempty"` + Skipped bool `json:"skipped,omitempty"` + SkipReason string `json:"skip_reason,omitempty"` + TotalScore float64 `json:"total_score"` + BuildLog string `json:"build_log,omitempty"` + Groups []jsonGroupResult `json:"groups"` +} + type jsonGroupResult struct { Name string `json:"name"` Weight float64 `json:"weight"` @@ -86,6 +144,62 @@ type jsonTestResult struct { Failures []string `json:"failures,omitempty"` } +func flatResult(r *runner.SuiteResult) jsonSuiteResult { + res := jsonSuiteResult{TotalScore: r.TotalScore} + if len(r.Builds) == 0 { + return res + } + b := r.Builds[0] + res.BuildLog = b.BuildLog + for _, gr := range b.Groups { + res.Groups = append(res.Groups, groupJSON(gr)) + } + return res +} + +func nestedResult(r *runner.SuiteResult) jsonMultiSuiteResult { + res := jsonMultiSuiteResult{ + TotalScore: r.TotalScore, + Builds: map[string]jsonSuiteBuild{}, + } + for _, b := range r.Builds { + entry := jsonSuiteBuild{ + Name: b.Name, + Toolchain: b.Toolchain, + Skipped: b.Skipped, + SkipReason: b.SkipReason, + TotalScore: b.TotalScore, + BuildLog: b.BuildLog, + } + for _, gr := range b.Groups { + entry.Groups = append(entry.Groups, groupJSON(gr)) + } + res.Builds[b.Name] = entry + } + return res +} + +func groupJSON(gr *runner.GroupResult) jsonGroupResult { + jgr := jsonGroupResult{ + Name: gr.Name, + Weight: gr.Weight, + Score: gr.Score, + Passed: gr.Passed, + Total: gr.Total, + } + for _, tr := range gr.Tests { + jgr.Tests = append(jgr.Tests, jsonTestResult{ + Name: tr.Name, + Status: tr.Status.String(), + ElapsedMs: tr.Elapsed.Milliseconds(), + PeakMemoryKB: tr.PeakMemory / 1024, + MemoryLimitKB: tr.MemoryLimit / 1024, + Failures: tr.Failures, + }) + } + return jgr +} + func Aggregate(w io.Writer, dir string) error { files, err := filepath.Glob(filepath.Join(dir, "*", "*.json")) if err != nil { @@ -105,24 +219,24 @@ func Aggregate(w io.Writer, dir string) error { } var entries []entry - allPassed := true + minScore := math.Inf(1) for _, f := range files { data, err := os.ReadFile(f) if err != nil { return fmt.Errorf("read %s: %w", f, err) } - var report jsonSuiteResult - if err := json.Unmarshal(data, &report); err != nil { + score, err := extractTotalScore(data) + if err != nil { return fmt.Errorf("parse %s: %w", f, err) } cfg := filepath.Base(filepath.Dir(f)) cfg = strings.TrimPrefix(cfg, "report_") - entries = append(entries, entry{Config: cfg, Score: report.TotalScore}) - if report.TotalScore < 0.9999 { - allPassed = false + entries = append(entries, entry{Config: cfg, Score: score}) + if score < minScore { + minScore = score } } @@ -133,13 +247,24 @@ func Aggregate(w io.Writer, dir string) error { for _, e := range entries { fmt.Fprintf(w, "| %s | %.4f |\n", e.Config, e.Score) } + fmt.Fprintf(w, "| **Overall (min)** | **%.4f** |\n", minScore) - if !allPassed { - return fmt.Errorf("one or more configurations scored below 1.0") + if minScore < 0.9999 { + return fmt.Errorf("minimum score across configurations is %.4f (below 1.0)", minScore) } return nil } +func extractTotalScore(data []byte) (float64, error) { + var header struct { + TotalScore float64 `json:"total_score"` + } + if err := json.Unmarshal(data, &header); err != nil { + return 0, err + } + return header.TotalScore, nil +} + func humanBytes(n int64) string { const ( KiB = 1024 @@ -157,31 +282,3 @@ func humanBytes(n int64) string { return fmt.Sprintf("%dB", n) } } - -func jsonResult(r *runner.SuiteResult) jsonSuiteResult { - res := jsonSuiteResult{ - TotalScore: r.TotalScore, - BuildLog: r.BuildLog, - } - for _, gr := range r.Groups { - jgr := jsonGroupResult{ - Name: gr.Name, - Weight: gr.Weight, - Score: gr.Score, - Passed: gr.Passed, - Total: gr.Total, - } - for _, tr := range gr.Tests { - jgr.Tests = append(jgr.Tests, jsonTestResult{ - Name: tr.Name, - Status: tr.Status.String(), - ElapsedMs: tr.Elapsed.Milliseconds(), - PeakMemoryKB: tr.PeakMemory / 1024, - MemoryLimitKB: tr.MemoryLimit / 1024, - Failures: tr.Failures, - }) - } - res.Groups = append(res.Groups, jgr) - } - return res -} diff --git a/runner/compiler.go b/runner/compiler.go new file mode 100644 index 0000000..4b10031 --- /dev/null +++ b/runner/compiler.go @@ -0,0 +1,198 @@ +package runner + +import ( + "fmt" + "runtime" + "sort" + "strings" + + "github.com/Mond1c/judge/dsl" +) + +type CompilerClass int + +const ( + CompilerUnknown CompilerClass = iota + CompilerGNU + CompilerMSVC +) + +type Toolchain struct { + Class CompilerClass + Binary string + Name string +} + +func ResolveToolchain(ccEnv string) Toolchain { + if ccEnv == "" { + ccEnv = defaultCC() + } + lower := strings.ToLower(ccEnv) + switch lower { + case "gcc", "g++": + return Toolchain{Class: CompilerGNU, Binary: ccEnv, Name: "gcc"} + case "clang", "clang++": + return Toolchain{Class: CompilerGNU, Binary: ccEnv, Name: "clang"} + case "cl", "cl.exe", "msvc": + return Toolchain{Class: CompilerMSVC, Binary: "cl", Name: "msvc"} + case "clang-cl", "clang-cl.exe": + return Toolchain{Class: CompilerMSVC, Binary: "clang-cl", Name: "clang-cl"} + case "cc": + return Toolchain{Class: CompilerGNU, Binary: "cc", Name: "cc"} + default: + return Toolchain{Class: CompilerGNU, Binary: ccEnv, Name: lower} + } +} + +func ResolveToolchainSpec(spec *dsl.ToolchainSpec) Toolchain { + inferred := ResolveToolchain(spec.Name) + binary := spec.Binary + if binary == "" { + binary = inferred.Binary + } + var class CompilerClass + switch spec.Class { + case "gnu": + class = CompilerGNU + case "msvc": + class = CompilerMSVC + default: + class = inferred.Class + } + return Toolchain{Class: class, Binary: binary, Name: spec.Name} +} + +func defaultCC() string { + if runtime.GOOS == "windows" { + return "cl" + } + return "cc" +} + +func Compile(cfg dsl.BuildConfig, tc Toolchain, outputPath string) ([]string, error) { + switch tc.Class { + case CompilerGNU: + return compileGNU(cfg, tc, outputPath), nil + case CompilerMSVC: + return compileMSVC(cfg, tc, outputPath), nil + default: + return nil, fmt.Errorf("unknown compiler class for toolchain %q", tc.Name) + } +} + +func compileGNU(cfg dsl.BuildConfig, tc Toolchain, outputPath string) []string { + argv := []string{tc.Binary} + + if cfg.Standard != "" { + argv = append(argv, "-std="+cfg.Standard) + } + + switch cfg.Profile { + case dsl.ProfileRelease: + argv = append(argv, "-O2") + case dsl.ProfileDebug: + argv = append(argv, "-O0", "-g") + case dsl.ProfileSanitized: + argv = append(argv, "-O1", "-g", "-fno-omit-frame-pointer") + } + + switch cfg.Warnings { + case dsl.WarningsStrict: + argv = append(argv, "-Wall", "-Wextra") + case dsl.WarningsPedantic: + argv = append(argv, "-Wall", "-Wextra", "-Wpedantic") + } + + if len(cfg.Sanitize) > 0 { + argv = append(argv, "-fsanitize="+strings.Join(cfg.Sanitize, ",")) + } + + for _, inc := range cfg.Includes { + argv = append(argv, "-I"+inc) + } + + for _, k := range sortedKeys(cfg.Defines) { + v := cfg.Defines[k] + if v == "" { + argv = append(argv, "-D"+k) + } else { + argv = append(argv, fmt.Sprintf("-D%s=%s", k, v)) + } + } + + argv = append(argv, cfg.Extra...) + argv = append(argv, cfg.Sources...) + argv = append(argv, "-o", outputPath) + + for _, lib := range cfg.Link { + argv = append(argv, "-l"+lib) + } + return argv +} + +func compileMSVC(cfg dsl.BuildConfig, tc Toolchain, outputPath string) []string { + argv := []string{tc.Binary, "/nologo"} + + if cfg.Standard != "" { + argv = append(argv, "/std:"+cfg.Standard) + } + + switch cfg.Profile { + case dsl.ProfileRelease: + argv = append(argv, "/O2") + case dsl.ProfileDebug: + argv = append(argv, "/Od", "/Zi") + case dsl.ProfileSanitized: + argv = append(argv, "/Od", "/Zi", "/fsanitize=address") + } + + switch cfg.Warnings { + case dsl.WarningsStrict: + argv = append(argv, "/W4") + case dsl.WarningsPedantic: + argv = append(argv, "/W4", "/permissive-") + } + + if containsString(cfg.Sanitize, "address") && cfg.Profile != dsl.ProfileSanitized { + argv = append(argv, "/fsanitize=address") + } + + for _, inc := range cfg.Includes { + argv = append(argv, "/I"+inc) + } + + for _, k := range sortedKeys(cfg.Defines) { + v := cfg.Defines[k] + if v == "" { + argv = append(argv, "/D"+k) + } else { + argv = append(argv, fmt.Sprintf("/D%s=%s", k, v)) + } + } + + argv = append(argv, cfg.Extra...) + argv = append(argv, cfg.Sources...) + argv = append(argv, "/Fe:"+outputPath) + return argv +} + +func sortedKeys(m map[string]string) []string { + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func containsString(xs []string, x string) bool { + for _, v := range xs { + if v == x { + return true + } + } + return false +} diff --git a/runner/compiler_test.go b/runner/compiler_test.go new file mode 100644 index 0000000..b7e61a1 --- /dev/null +++ b/runner/compiler_test.go @@ -0,0 +1,293 @@ +package runner + +import ( + "reflect" + "strings" + "testing" + + "github.com/Mond1c/judge/dsl" +) + +func TestResolveToolchainKnown(t *testing.T) { + cases := []struct { + in string + wantClass CompilerClass + wantName string + }{ + {"gcc", CompilerGNU, "gcc"}, + {"g++", CompilerGNU, "gcc"}, + {"clang", CompilerGNU, "clang"}, + {"clang++", CompilerGNU, "clang"}, + {"cl", CompilerMSVC, "msvc"}, + {"cl.exe", CompilerMSVC, "msvc"}, + {"msvc", CompilerMSVC, "msvc"}, + {"clang-cl", CompilerMSVC, "clang-cl"}, + {"cc", CompilerGNU, "cc"}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + tc := ResolveToolchain(c.in) + if tc.Class != c.wantClass { + t.Errorf("class: got %v, want %v", tc.Class, c.wantClass) + } + if tc.Name != c.wantName { + t.Errorf("name: got %q, want %q", tc.Name, c.wantName) + } + }) + } +} + +func TestResolveToolchainSpec(t *testing.T) { + cases := []struct { + name string + spec dsl.ToolchainSpec + wantClass CompilerClass + wantBin string + wantName string + }{ + { + "gcc inferred", + dsl.ToolchainSpec{Name: "gcc", Platforms: []string{"linux"}}, + CompilerGNU, "gcc", "gcc", + }, + { + "msvc inferred", + dsl.ToolchainSpec{Name: "msvc", Platforms: []string{"windows"}}, + CompilerMSVC, "cl", "msvc", + }, + { + "nvcc with explicit class", + dsl.ToolchainSpec{Name: "nvcc", Platforms: []string{"linux"}, Class: "gnu"}, + CompilerGNU, "nvcc", "nvcc", + }, + { + "custom binary override", + dsl.ToolchainSpec{Name: "clang", Platforms: []string{"linux", "windows"}, Binary: "clang-17"}, + CompilerGNU, "clang-17", "clang", + }, + { + "unknown name, explicit class", + dsl.ToolchainSpec{Name: "hipcc", Platforms: []string{"linux"}, Class: "gnu"}, + CompilerGNU, "hipcc", "hipcc", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := ResolveToolchainSpec(&c.spec) + if got.Class != c.wantClass { + t.Errorf("class: got %v, want %v", got.Class, c.wantClass) + } + if got.Binary != c.wantBin { + t.Errorf("binary: got %q, want %q", got.Binary, c.wantBin) + } + if got.Name != c.wantName { + t.Errorf("name: got %q, want %q", got.Name, c.wantName) + } + }) + } +} + +func TestResolveToolchainUnknownFallsBackToGNU(t *testing.T) { + tc := ResolveToolchain("gcc-13") + if tc.Class != CompilerGNU { + t.Errorf("unknown compiler should fall back to GNU, got %v", tc.Class) + } + if tc.Binary != "gcc-13" { + t.Errorf("binary should preserve the original string, got %q", tc.Binary) + } + if tc.Name != "gcc-13" { + t.Errorf("name should be the lowercased original, got %q", tc.Name) + } +} + +func TestCompileGNURelease(t *testing.T) { + cfg := dsl.BuildConfig{ + Language: "c", + Standard: "c11", + Sources: []string{"solution.c"}, + Profile: dsl.ProfileRelease, + Warnings: dsl.WarningsStrict, + } + tc := Toolchain{Class: CompilerGNU, Binary: "gcc", Name: "gcc"} + argv, err := Compile(cfg, tc, "solution") + if err != nil { + t.Fatal(err) + } + want := []string{"gcc", "-std=c11", "-O2", "-Wall", "-Wextra", "solution.c", "-o", "solution"} + if !reflect.DeepEqual(argv, want) { + t.Errorf("argv =\n %v\nwant\n %v", argv, want) + } +} + +func TestCompileGNUDebug(t *testing.T) { + cfg := dsl.BuildConfig{ + Standard: "c11", + Sources: []string{"a.c", "b.c"}, + Profile: dsl.ProfileDebug, + } + tc := Toolchain{Class: CompilerGNU, Binary: "gcc"} + argv, _ := Compile(cfg, tc, "out") + if !containsSubsequence(argv, []string{"-O0", "-g"}) { + t.Errorf("debug flags missing: %v", argv) + } + if !containsSubsequence(argv, []string{"a.c", "b.c", "-o", "out"}) { + t.Errorf("sources and output order wrong: %v", argv) + } +} + +func TestCompileGNUSanitized(t *testing.T) { + cfg := dsl.BuildConfig{ + Standard: "c11", + Sources: []string{"s.c"}, + Profile: dsl.ProfileSanitized, + Sanitize: []string{"address", "undefined"}, + } + tc := Toolchain{Class: CompilerGNU, Binary: "clang"} + argv, _ := Compile(cfg, tc, "s") + joined := strings.Join(argv, " ") + if !strings.Contains(joined, "-fsanitize=address,undefined") { + t.Errorf("sanitize flag missing: %v", argv) + } + if !strings.Contains(joined, "-O1") { + t.Errorf("-O1 missing for sanitized profile: %v", argv) + } +} + +func TestCompileGNUIncludesAndDefinesAndLink(t *testing.T) { + cfg := dsl.BuildConfig{ + Sources: []string{"m.c"}, + Includes: []string{"include", "vendor/inc"}, + Defines: map[string]string{"FOO": "1", "BAR": ""}, + Link: []string{"m", "pthread"}, + Extra: []string{"-fno-strict-aliasing"}, + } + tc := Toolchain{Class: CompilerGNU, Binary: "gcc"} + argv, _ := Compile(cfg, tc, "out") + + joined := strings.Join(argv, " ") + for _, want := range []string{"-Iinclude", "-Ivendor/inc", "-DFOO=1", "-DBAR", "-lm", "-lpthread", "-fno-strict-aliasing"} { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in %v", want, argv) + } + } + + oIdx := indexOf(argv, "-o") + lmIdx := indexOf(argv, "-lm") + if oIdx == -1 || lmIdx == -1 || lmIdx < oIdx { + t.Errorf("-lm must come after -o: %v", argv) + } +} + +func TestCompileGNUDefinesOrderDeterministic(t *testing.T) { + cfg := dsl.BuildConfig{ + Sources: []string{"s.c"}, + Defines: map[string]string{"Z": "1", "A": "2", "M": "3"}, + } + tc := Toolchain{Class: CompilerGNU, Binary: "gcc"} + argv1, _ := Compile(cfg, tc, "s") + for i := 0; i < 20; i++ { + argv2, _ := Compile(cfg, tc, "s") + if !reflect.DeepEqual(argv1, argv2) { + t.Fatalf("defines order not deterministic:\n %v\n %v", argv1, argv2) + } + } +} + +func TestCompileMSVCRelease(t *testing.T) { + cfg := dsl.BuildConfig{ + Standard: "c11", + Sources: []string{"solution.c"}, + Profile: dsl.ProfileRelease, + Warnings: dsl.WarningsStrict, + } + tc := Toolchain{Class: CompilerMSVC, Binary: "cl", Name: "msvc"} + argv, _ := Compile(cfg, tc, "solution.exe") + want := []string{"cl", "/nologo", "/std:c11", "/O2", "/W4", "solution.c", "/Fe:solution.exe"} + if !reflect.DeepEqual(argv, want) { + t.Errorf("argv =\n %v\nwant\n %v", argv, want) + } +} + +func TestCompileMSVCDebug(t *testing.T) { + cfg := dsl.BuildConfig{ + Sources: []string{"s.c"}, + Profile: dsl.ProfileDebug, + } + tc := Toolchain{Class: CompilerMSVC, Binary: "cl"} + argv, _ := Compile(cfg, tc, "s.exe") + joined := strings.Join(argv, " ") + for _, want := range []string{"/Od", "/Zi"} { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in %v", want, argv) + } + } +} + +func TestCompileMSVCSanitizedAddressOnly(t *testing.T) { + cfg := dsl.BuildConfig{ + Sources: []string{"s.c"}, + Profile: dsl.ProfileSanitized, + Sanitize: []string{"address", "undefined", "thread"}, + } + tc := Toolchain{Class: CompilerMSVC, Binary: "cl"} + argv, _ := Compile(cfg, tc, "s.exe") + joined := strings.Join(argv, " ") + if !strings.Contains(joined, "/fsanitize=address") { + t.Errorf("msvc should emit /fsanitize=address for sanitized profile: %v", argv) + } + if strings.Contains(joined, "undefined") || strings.Contains(joined, "thread") { + t.Errorf("msvc should drop unsupported sanitizers, got: %v", argv) + } +} + +func TestCompileMSVCIncludesAndDefines(t *testing.T) { + cfg := dsl.BuildConfig{ + Sources: []string{"m.c"}, + Includes: []string{"include"}, + Defines: map[string]string{"FOO": "1", "BAR": ""}, + } + tc := Toolchain{Class: CompilerMSVC, Binary: "cl"} + argv, _ := Compile(cfg, tc, "m.exe") + joined := strings.Join(argv, " ") + for _, want := range []string{"/Iinclude", "/DFOO=1", "/DBAR"} { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in %v", want, argv) + } + } +} + +func TestCompileUnknownClassErrors(t *testing.T) { + cfg := dsl.BuildConfig{Sources: []string{"s.c"}} + tc := Toolchain{Class: CompilerUnknown, Binary: "weird"} + if _, err := Compile(cfg, tc, "s"); err == nil { + t.Error("expected error for unknown compiler class") + } +} + +func containsSubsequence(haystack, needle []string) bool { + if len(needle) == 0 { + return true + } + for i := 0; i+len(needle) <= len(haystack); i++ { + match := true + for j := range needle { + if haystack[i+j] != needle[j] { + match = false + break + } + } + if match { + return true + } + } + return false +} + +func indexOf(xs []string, x string) int { + for i, v := range xs { + if v == x { + return i + } + } + return -1 +} diff --git a/runner/result.go b/runner/result.go index 8fa40d9..f267182 100644 --- a/runner/result.go +++ b/runner/result.go @@ -2,6 +2,7 @@ package runner import ( "fmt" + "math" "time" ) @@ -40,8 +41,8 @@ type TestResult struct { Status Status Elapsed time.Duration - PeakMemory int64 // bytes; 0 if not measured - MemoryLimit int64 // bytes; 0 if unlimited + PeakMemory int64 + MemoryLimit int64 Failures []string @@ -64,8 +65,40 @@ type GroupResult struct { Total int } -type SuiteResult struct { +type BuildRun struct { + Name string + Toolchain string + Skipped bool + SkipReason string + + BuildLog string Groups []*GroupResult TotalScore float64 - BuildLog string +} + +type SuiteResult struct { + Builds []*BuildRun + + TotalScore float64 +} + +func (r *SuiteResult) AggregateScore() float64 { + if len(r.Builds) == 0 { + return 0 + } + min := math.Inf(1) + anyRan := false + for _, b := range r.Builds { + if b.Skipped { + continue + } + anyRan = true + if b.TotalScore < min { + min = b.TotalScore + } + } + if !anyRan { + return 1.0 + } + return min } diff --git a/runner/result_test.go b/runner/result_test.go new file mode 100644 index 0000000..c072613 --- /dev/null +++ b/runner/result_test.go @@ -0,0 +1,57 @@ +package runner + +import "testing" + +func TestAggregateScoreEmpty(t *testing.T) { + r := &SuiteResult{} + if got := r.AggregateScore(); got != 0 { + t.Errorf("empty aggregate = %v, want 0", got) + } +} + +func TestAggregateScoreSingleBuild(t *testing.T) { + r := &SuiteResult{ + Builds: []*BuildRun{{Name: "release", TotalScore: 0.75}}, + } + if got := r.AggregateScore(); got != 0.75 { + t.Errorf("single build aggregate = %v, want 0.75", got) + } +} + +func TestAggregateScoreTakesMinimum(t *testing.T) { + r := &SuiteResult{ + Builds: []*BuildRun{ + {Name: "release", TotalScore: 1.0}, + {Name: "debug", TotalScore: 0.9}, + {Name: "sanitized", TotalScore: 0.95}, + }, + } + if got := r.AggregateScore(); got != 0.9 { + t.Errorf("aggregate = %v, want 0.9 (minimum)", got) + } +} + +func TestAggregateScoreIgnoresSkipped(t *testing.T) { + r := &SuiteResult{ + Builds: []*BuildRun{ + {Name: "release", TotalScore: 1.0}, + {Name: "sanitized", Skipped: true, SkipReason: "platforms=linux"}, + {Name: "debug", TotalScore: 0.8}, + }, + } + if got := r.AggregateScore(); got != 0.8 { + t.Errorf("aggregate with skipped = %v, want 0.8", got) + } +} + +func TestAggregateScoreAllSkipped(t *testing.T) { + r := &SuiteResult{ + Builds: []*BuildRun{ + {Name: "a", Skipped: true}, + {Name: "b", Skipped: true}, + }, + } + if got := r.AggregateScore(); got != 1.0 { + t.Errorf("all-skipped aggregate = %v, want 1.0", got) + } +} diff --git a/runner/runner.go b/runner/runner.go index a477fca..1bedfc5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -21,6 +21,8 @@ type Config struct { WorkDir string BinaryName string Wrapper string + + TargetBuild string } type Runner struct { @@ -63,41 +65,271 @@ func (r *Runner) Run() *SuiteResult { result := &SuiteResult{} - buildLog, err := r.build() - result.BuildLog = buildLog - if err != nil { - for _, g := range r.file.Groups { - gr := &GroupResult{ - Name: g.Name, - Weight: g.Weight, - Score: 0, - } + if len(r.file.Builds) == 0 { + run := r.runLegacyBuild() + result.Builds = append(result.Builds, run) + } else { + result.Builds = r.runStructuredBuilds() + } - total := len(g.Tests) - if g.Pattern != nil { - total = -1 - } - gr.Total = total - for _, t := range g.Tests { - gr.Tests = append(gr.Tests, &TestResult{ - Name: t.Name, - Status: StatusBuildError, - }) - } - result.Groups = append(result.Groups, gr) - } - return result + result.TotalScore = result.AggregateScore() + return result +} + +// runLegacyBuild handles the classic `build "shell-string"` form. It produces +// a single BuildRun named "default" so that downstream consumers always see +// the same shape. +func (r *Runner) runLegacyBuild() *BuildRun { + run := &BuildRun{Name: "default"} + + if r.cfg.TargetBuild != "" && r.cfg.TargetBuild != "default" { + run.Skipped = true + run.SkipReason = fmt.Sprintf("--build=%q selected, but this suite has no structured builds", r.cfg.TargetBuild) + return run + } + + buildLog, err := r.legacyBuild() + run.BuildLog = buildLog + if err != nil { + r.fillBuildError(run) + return run } r.binary = resolveBinary(r.cfg.WorkDir, filepath.Base(r.binary)) + r.runGroups(run) + return run +} - for _, g := range r.file.Groups { - gr := r.runGroup(g) - result.Groups = append(result.Groups, gr) - result.TotalScore += gr.Score +// runStructuredBuilds handles the new DSL form with one or more named builds. +// Each build is resolved against the current OS and toolchain, compiled via +// the structured translator, and then exercised against every group. +func (r *Runner) resolveRuntimeToolchain() (Toolchain, string) { + goos := runtime.GOOS + wanted := os.Getenv("JUDGE_TOOLCHAIN") + if wanted == "" { + wanted = os.Getenv("JUDGE_CC") + } + for _, spec := range r.file.Toolchains { + if spec.Name == wanted { + return ResolveToolchainSpec(spec), goos + } + } + return ResolveToolchain(wanted), goos +} + +func (r *Runner) runStructuredBuilds() []*BuildRun { + tc, goos := r.resolveRuntimeToolchain() + var runs []*BuildRun + + for _, b := range r.file.Builds { + run := &BuildRun{Name: b.Name, Toolchain: tc.Name} + + if r.cfg.TargetBuild != "" && r.cfg.TargetBuild != b.Name { + // User asked for a different build. Don't include this one + // in the result at all — discovery via --list-builds is the + // caller's responsibility. + continue + } + + effective := b.Resolve(r.file.BuildDefaults, goos) + if !effective.AppliesTo(goos, tc.Name) { + run.Skipped = true + run.SkipReason = fmt.Sprintf("not applicable to %s/%s (platforms=%v, compilers=%v)", goos, tc.Name, effective.Platforms, effective.Compilers) + runs = append(runs, run) + continue + } + + log, binaryPath, err := r.compileStructured(b.Name, effective, tc) + run.BuildLog = log + if err != nil { + run.Groups = r.synthesizeBuildError() + run.TotalScore = 0 + runs = append(runs, run) + continue + } + + prevBinary := r.binary + prevWrapper := r.cfg.Wrapper + r.binary = binaryPath + if r.cfg.Wrapper == "" && effective.Wrapper != "" { + r.cfg.Wrapper = effective.Wrapper + } + r.runGroups(run) + r.binary = prevBinary + r.cfg.Wrapper = prevWrapper + + runs = append(runs, run) } - return result + return runs +} + +// runGroups exercises every group/test in the suite against the currently +// selected binary (r.binary) and records the outcome into run. +func (r *Runner) runGroups(run *BuildRun) { + for _, g := range r.file.Groups { + gr := r.runGroup(g) + run.Groups = append(run.Groups, gr) + run.TotalScore += gr.Score + } +} + +// fillBuildError populates a BuildRun with one failing synthetic test per +// group when the build itself failed. This keeps the reported totals at 0 +// and matches the legacy behaviour. +func (r *Runner) fillBuildError(run *BuildRun) { + run.Groups = r.synthesizeBuildError() +} + +func (r *Runner) synthesizeBuildError() []*GroupResult { + var out []*GroupResult + for _, g := range r.file.Groups { + gr := &GroupResult{ + Name: g.Name, + Weight: g.Weight, + Score: 0, + } + total := len(g.Tests) + if g.Pattern != nil { + total = -1 + } + gr.Total = total + for _, t := range g.Tests { + gr.Tests = append(gr.Tests, &TestResult{ + Name: t.Name, + Status: StatusBuildError, + }) + } + out = append(out, gr) + } + return out +} + +// legacyBuild runs the free-form shell build command. Kept under the old +// name so reviewers can diff against the previous implementation easily. +func (r *Runner) legacyBuild() (string, error) { + buildCmd := r.buildCommand() + + sources, err := r.findSources() + if err != nil { + return "", err + } + if sources != "" { + buildCmd = strings.ReplaceAll(buildCmd, "$SOURCES", sources) + } + + ctx := context.Background() + if r.file.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, r.file.Timeout) + defer cancel() + } + + cmd := shellCommand(ctx, buildCmd) + cmd.Dir = r.cfg.WorkDir + setProcessGroup(cmd) + cmd.Env = os.Environ() + + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + + if err := cmd.Run(); err != nil { + killProcessGroup(cmd) + return out.String(), fmt.Errorf("build failed: %w\n%s", err, out.String()) + } + return out.String(), nil +} + +// compileStructured compiles one structured build via the translator and +// returns the build log plus the absolute path to the produced binary. +// The compiler is invoked via exec.Command directly — no shell involved. +func (r *Runner) compileStructured(name string, cfg dsl.BuildConfig, tc Toolchain) (string, string, error) { + // Expand source globs against the work dir. + sources, err := expandSources(r.cfg.WorkDir, cfg.Sources) + if err != nil { + return "", "", err + } + if len(sources) == 0 { + return "", "", fmt.Errorf("build %q: no sources", name) + } + cfg.Sources = sources + + // Decide output path: /build//[.exe]. + outputName := cfg.Output + if outputName == "" { + outputName = "solution" + } + if runtime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(outputName), ".exe") { + outputName += ".exe" + } + buildDir := filepath.Join(r.cfg.WorkDir, "build", name) + if err := os.MkdirAll(buildDir, 0755); err != nil { + return "", "", fmt.Errorf("mkdir %s: %w", buildDir, err) + } + outputPath := filepath.Join(buildDir, outputName) + + argv, err := Compile(cfg, tc, outputPath) + if err != nil { + return "", "", err + } + + ctx := context.Background() + if r.file.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, r.file.Timeout) + defer cancel() + } + + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Dir = r.cfg.WorkDir + setProcessGroup(cmd) + cmd.Env = os.Environ() + + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + + logPrefix := fmt.Sprintf("$ %s\n", strings.Join(argv, " ")) + if err := cmd.Run(); err != nil { + killProcessGroup(cmd) + return logPrefix + out.String(), "", fmt.Errorf("build %q failed: %w\n%s", name, err, out.String()) + } + return logPrefix + out.String(), outputPath, nil +} + +// expandSources expands each glob in patterns against workDir and returns +// a slice of paths relative to workDir. Globs that match no files cause an +// error — silent zero matches are almost always a typo. +func expandSources(workDir string, patterns []string) ([]string, error) { + var out []string + seen := map[string]bool{} + for _, pat := range patterns { + matches, err := filepath.Glob(filepath.Join(workDir, pat)) + if err != nil { + return nil, fmt.Errorf("glob %q: %w", pat, err) + } + if len(matches) == 0 { + // Fall back to treating it as a literal path. + if _, statErr := os.Stat(filepath.Join(workDir, pat)); statErr == nil { + matches = []string{filepath.Join(workDir, pat)} + } else { + return nil, fmt.Errorf("source glob %q matched no files", pat) + } + } + for _, m := range matches { + rel, err := filepath.Rel(workDir, m) + if err != nil { + rel = m + } + rel = filepath.ToSlash(rel) + if !seen[rel] { + seen[rel] = true + out = append(out, rel) + } + } + } + return out, nil } func (r *Runner) buildCommand() string { @@ -152,40 +384,6 @@ func (r *Runner) findSources() (string, error) { return strings.Join(files, " "), nil } -func (r *Runner) build() (string, error) { - buildCmd := r.buildCommand() - - sources, err := r.findSources() - if err != nil { - return "", err - } - if sources != "" { - buildCmd = strings.ReplaceAll(buildCmd, "$SOURCES", sources) - } - - ctx := context.Background() - if r.file.Timeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, r.file.Timeout) - defer cancel() - } - - cmd := shellCommand(ctx, buildCmd) - cmd.Dir = r.cfg.WorkDir - setProcessGroup(cmd) - cmd.Env = os.Environ() - - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - - if err := cmd.Run(); err != nil { - killProcessGroup(cmd) - return out.String(), fmt.Errorf("build failed: %w\n%s", err, out.String()) - } - return out.String(), nil -} - func shellCommand(ctx context.Context, cmdline string) *exec.Cmd { if runtime.GOOS == "windows" { return exec.CommandContext(ctx, "cmd", "/C", cmdline) @@ -392,21 +590,23 @@ func (r *Runner) runTest(t *dsl.Test) *TestResult { tr.addFailure("%s", f) } - for name, expected := range t.OutFiles { - path := filepath.Join(tmpDir, name) - content, err := os.ReadFile(path) - if err != nil { - tr.addFailure("output file %q not found: %v", name, err) - continue - } - actual := normalizeOutput(string(content), r.file) - for _, f := range applyMatcher(fmt.Sprintf("file(%s)", name), dsl.ExactMatcher{Value: expected}, actual) { - tr.addFailure("%s", f) - } + if len(tr.Failures) > 0 { + tr.Status = StatusFail } - if tr.Status == StatusPass && len(tr.Failures) > 0 { - tr.Status = StatusFail + for name, expected := range t.OutFiles { + actualPath := filepath.Join(tmpDir, name) + data, err := os.ReadFile(actualPath) + if err != nil { + tr.Status = StatusFail + tr.addFailure("output file %q missing: %v", name, err) + continue + } + actual := normalizeOutput(string(data), r.file) + if actual != expected { + tr.Status = StatusFail + tr.addFailure("output file %q mismatch\n expected: %q\n actual: %q", name, expected, actual) + } } return tr