add new build system

runner/runner.go

@@ -21,6 +21,8 @@ type Config struct {
 	WorkDir    string
 	BinaryName string
 	Wrapper    string
+
+	TargetBuild string
 }
 
 type Runner struct {
@@ -63,41 +65,271 @@ func (r *Runner) Run() *SuiteResult {
 
 	result := &SuiteResult{}
 
-	buildLog, err := r.build()
-	result.BuildLog = buildLog
-	if err != nil {
-		for _, g := range r.file.Groups {
-			gr := &GroupResult{
-				Name:   g.Name,
-				Weight: g.Weight,
-				Score:  0,
-			}
+	if len(r.file.Builds) == 0 {
+		run := r.runLegacyBuild()
+		result.Builds = append(result.Builds, run)
+	} else {
+		result.Builds = r.runStructuredBuilds()
+	}
 
-			total := len(g.Tests)
-			if g.Pattern != nil {
-				total = -1
-			}
-			gr.Total = total
-			for _, t := range g.Tests {
-				gr.Tests = append(gr.Tests, &TestResult{
-					Name:   t.Name,
-					Status: StatusBuildError,
-				})
-			}
-			result.Groups = append(result.Groups, gr)
-		}
-		return result
+	result.TotalScore = result.AggregateScore()
+	return result
+}
+
+// runLegacyBuild handles the classic `build "shell-string"` form. It produces
+// a single BuildRun named "default" so that downstream consumers always see
+// the same shape.
+func (r *Runner) runLegacyBuild() *BuildRun {
+	run := &BuildRun{Name: "default"}
+
+	if r.cfg.TargetBuild != "" && r.cfg.TargetBuild != "default" {
+		run.Skipped = true
+		run.SkipReason = fmt.Sprintf("--build=%q selected, but this suite has no structured builds", r.cfg.TargetBuild)
+		return run
+	}
+
+	buildLog, err := r.legacyBuild()
+	run.BuildLog = buildLog
+	if err != nil {
+		r.fillBuildError(run)
+		return run
 	}
 
 	r.binary = resolveBinary(r.cfg.WorkDir, filepath.Base(r.binary))
+	r.runGroups(run)
+	return run
+}
 
-	for _, g := range r.file.Groups {
-		gr := r.runGroup(g)
-		result.Groups = append(result.Groups, gr)
-		result.TotalScore += gr.Score
+// runStructuredBuilds handles the new DSL form with one or more named builds.
+// Each build is resolved against the current OS and toolchain, compiled via
+// the structured translator, and then exercised against every group.
+func (r *Runner) resolveRuntimeToolchain() (Toolchain, string) {
+	goos := runtime.GOOS
+	wanted := os.Getenv("JUDGE_TOOLCHAIN")
+	if wanted == "" {
+		wanted = os.Getenv("JUDGE_CC")
+	}
+	for _, spec := range r.file.Toolchains {
+		if spec.Name == wanted {
+			return ResolveToolchainSpec(spec), goos
+		}
+	}
+	return ResolveToolchain(wanted), goos
+}
+
+func (r *Runner) runStructuredBuilds() []*BuildRun {
+	tc, goos := r.resolveRuntimeToolchain()
+	var runs []*BuildRun
+
+	for _, b := range r.file.Builds {
+		run := &BuildRun{Name: b.Name, Toolchain: tc.Name}
+
+		if r.cfg.TargetBuild != "" && r.cfg.TargetBuild != b.Name {
+			// User asked for a different build. Don't include this one
+			// in the result at all — discovery via --list-builds is the
+			// caller's responsibility.
+			continue
+		}
+
+		effective := b.Resolve(r.file.BuildDefaults, goos)
+		if !effective.AppliesTo(goos, tc.Name) {
+			run.Skipped = true
+			run.SkipReason = fmt.Sprintf("not applicable to %s/%s (platforms=%v, compilers=%v)", goos, tc.Name, effective.Platforms, effective.Compilers)
+			runs = append(runs, run)
+			continue
+		}
+
+		log, binaryPath, err := r.compileStructured(b.Name, effective, tc)
+		run.BuildLog = log
+		if err != nil {
+			run.Groups = r.synthesizeBuildError()
+			run.TotalScore = 0
+			runs = append(runs, run)
+			continue
+		}
+
+		prevBinary := r.binary
+		prevWrapper := r.cfg.Wrapper
+		r.binary = binaryPath
+		if r.cfg.Wrapper == "" && effective.Wrapper != "" {
+			r.cfg.Wrapper = effective.Wrapper
+		}
+		r.runGroups(run)
+		r.binary = prevBinary
+		r.cfg.Wrapper = prevWrapper
+
+		runs = append(runs, run)
 	}
 
-	return result
+	return runs
+}
+
+// runGroups exercises every group/test in the suite against the currently
+// selected binary (r.binary) and records the outcome into run.
+func (r *Runner) runGroups(run *BuildRun) {
+	for _, g := range r.file.Groups {
+		gr := r.runGroup(g)
+		run.Groups = append(run.Groups, gr)
+		run.TotalScore += gr.Score
+	}
+}
+
+// fillBuildError populates a BuildRun with one failing synthetic test per
+// group when the build itself failed. This keeps the reported totals at 0
+// and matches the legacy behaviour.
+func (r *Runner) fillBuildError(run *BuildRun) {
+	run.Groups = r.synthesizeBuildError()
+}
+
+func (r *Runner) synthesizeBuildError() []*GroupResult {
+	var out []*GroupResult
+	for _, g := range r.file.Groups {
+		gr := &GroupResult{
+			Name:   g.Name,
+			Weight: g.Weight,
+			Score:  0,
+		}
+		total := len(g.Tests)
+		if g.Pattern != nil {
+			total = -1
+		}
+		gr.Total = total
+		for _, t := range g.Tests {
+			gr.Tests = append(gr.Tests, &TestResult{
+				Name:   t.Name,
+				Status: StatusBuildError,
+			})
+		}
+		out = append(out, gr)
+	}
+	return out
+}
+
+// legacyBuild runs the free-form shell build command. Kept under the old
+// name so reviewers can diff against the previous implementation easily.
+func (r *Runner) legacyBuild() (string, error) {
+	buildCmd := r.buildCommand()
+
+	sources, err := r.findSources()
+	if err != nil {
+		return "", err
+	}
+	if sources != "" {
+		buildCmd = strings.ReplaceAll(buildCmd, "$SOURCES", sources)
+	}
+
+	ctx := context.Background()
+	if r.file.Timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, r.file.Timeout)
+		defer cancel()
+	}
+
+	cmd := shellCommand(ctx, buildCmd)
+	cmd.Dir = r.cfg.WorkDir
+	setProcessGroup(cmd)
+	cmd.Env = os.Environ()
+
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+
+	if err := cmd.Run(); err != nil {
+		killProcessGroup(cmd)
+		return out.String(), fmt.Errorf("build failed: %w\n%s", err, out.String())
+	}
+	return out.String(), nil
+}
+
+// compileStructured compiles one structured build via the translator and
+// returns the build log plus the absolute path to the produced binary.
+// The compiler is invoked via exec.Command directly — no shell involved.
+func (r *Runner) compileStructured(name string, cfg dsl.BuildConfig, tc Toolchain) (string, string, error) {
+	// Expand source globs against the work dir.
+	sources, err := expandSources(r.cfg.WorkDir, cfg.Sources)
+	if err != nil {
+		return "", "", err
+	}
+	if len(sources) == 0 {
+		return "", "", fmt.Errorf("build %q: no sources", name)
+	}
+	cfg.Sources = sources
+
+	// Decide output path: <workdir>/build/<name>/<output>[.exe].
+	outputName := cfg.Output
+	if outputName == "" {
+		outputName = "solution"
+	}
+	if runtime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(outputName), ".exe") {
+		outputName += ".exe"
+	}
+	buildDir := filepath.Join(r.cfg.WorkDir, "build", name)
+	if err := os.MkdirAll(buildDir, 0755); err != nil {
+		return "", "", fmt.Errorf("mkdir %s: %w", buildDir, err)
+	}
+	outputPath := filepath.Join(buildDir, outputName)
+
+	argv, err := Compile(cfg, tc, outputPath)
+	if err != nil {
+		return "", "", err
+	}
+
+	ctx := context.Background()
+	if r.file.Timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, r.file.Timeout)
+		defer cancel()
+	}
+
+	cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
+	cmd.Dir = r.cfg.WorkDir
+	setProcessGroup(cmd)
+	cmd.Env = os.Environ()
+
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+
+	logPrefix := fmt.Sprintf("$ %s\n", strings.Join(argv, " "))
+	if err := cmd.Run(); err != nil {
+		killProcessGroup(cmd)
+		return logPrefix + out.String(), "", fmt.Errorf("build %q failed: %w\n%s", name, err, out.String())
+	}
+	return logPrefix + out.String(), outputPath, nil
+}
+
+// expandSources expands each glob in patterns against workDir and returns
+// a slice of paths relative to workDir. Globs that match no files cause an
+// error — silent zero matches are almost always a typo.
+func expandSources(workDir string, patterns []string) ([]string, error) {
+	var out []string
+	seen := map[string]bool{}
+	for _, pat := range patterns {
+		matches, err := filepath.Glob(filepath.Join(workDir, pat))
+		if err != nil {
+			return nil, fmt.Errorf("glob %q: %w", pat, err)
+		}
+		if len(matches) == 0 {
+			// Fall back to treating it as a literal path.
+			if _, statErr := os.Stat(filepath.Join(workDir, pat)); statErr == nil {
+				matches = []string{filepath.Join(workDir, pat)}
+			} else {
+				return nil, fmt.Errorf("source glob %q matched no files", pat)
+			}
+		}
+		for _, m := range matches {
+			rel, err := filepath.Rel(workDir, m)
+			if err != nil {
+				rel = m
+			}
+			rel = filepath.ToSlash(rel)
+			if !seen[rel] {
+				seen[rel] = true
+				out = append(out, rel)
+			}
+		}
+	}
+	return out, nil
 }
 
 func (r *Runner) buildCommand() string {
@@ -152,40 +384,6 @@ func (r *Runner) findSources() (string, error) {
 	return strings.Join(files, " "), nil
 }
 
-func (r *Runner) build() (string, error) {
-	buildCmd := r.buildCommand()
-
-	sources, err := r.findSources()
-	if err != nil {
-		return "", err
-	}
-	if sources != "" {
-		buildCmd = strings.ReplaceAll(buildCmd, "$SOURCES", sources)
-	}
-
-	ctx := context.Background()
-	if r.file.Timeout > 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, r.file.Timeout)
-		defer cancel()
-	}
-
-	cmd := shellCommand(ctx, buildCmd)
-	cmd.Dir = r.cfg.WorkDir
-	setProcessGroup(cmd)
-	cmd.Env = os.Environ()
-
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-
-	if err := cmd.Run(); err != nil {
-		killProcessGroup(cmd)
-		return out.String(), fmt.Errorf("build failed: %w\n%s", err, out.String())
-	}
-	return out.String(), nil
-}
-
 func shellCommand(ctx context.Context, cmdline string) *exec.Cmd {
 	if runtime.GOOS == "windows" {
 		return exec.CommandContext(ctx, "cmd", "/C", cmdline)
@@ -392,21 +590,23 @@ func (r *Runner) runTest(t *dsl.Test) *TestResult {
 		tr.addFailure("%s", f)
 	}
 
-	for name, expected := range t.OutFiles {
-		path := filepath.Join(tmpDir, name)
-		content, err := os.ReadFile(path)
-		if err != nil {
-			tr.addFailure("output file %q not found: %v", name, err)
-			continue
-		}
-		actual := normalizeOutput(string(content), r.file)
-		for _, f := range applyMatcher(fmt.Sprintf("file(%s)", name), dsl.ExactMatcher{Value: expected}, actual) {
-			tr.addFailure("%s", f)
-		}
+	if len(tr.Failures) > 0 {
+		tr.Status = StatusFail
 	}
 
-	if tr.Status == StatusPass && len(tr.Failures) > 0 {
-		tr.Status = StatusFail
+	for name, expected := range t.OutFiles {
+		actualPath := filepath.Join(tmpDir, name)
+		data, err := os.ReadFile(actualPath)
+		if err != nil {
+			tr.Status = StatusFail
+			tr.addFailure("output file %q missing: %v", name, err)
+			continue
+		}
+		actual := normalizeOutput(string(data), r.file)
+		if actual != expected {
+			tr.Status = StatusFail
+			tr.addFailure("output file %q mismatch\n  expected: %q\n  actual:   %q", name, expected, actual)
+		}
 	}
 
 	return tr