1. New build system

Reviewed-on: #1

reporter/reporter.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"sort"
@@ -13,11 +14,47 @@ import (
 )
 
 func Text(w io.Writer, result *runner.SuiteResult) {
-	if result.BuildLog != "" {
-		fmt.Fprintf(w, "=== BUILD LOG ===\n%s\n", result.BuildLog)
+	if len(result.Builds) == 0 {
+		fmt.Fprintln(w, "(no builds executed)")
+		return
 	}
 
-	for _, gr := range result.Groups {
+	multi := len(result.Builds) > 1
+	if multi {
+		fmt.Fprintf(w, "=== %d builds ===\n", len(result.Builds))
+		for _, b := range result.Builds {
+			marker := ""
+			if b.Skipped {
+				marker = " (skipped)"
+			}
+			fmt.Fprintf(w, "  • %s%s  score=%.4f\n", b.Name, marker, b.TotalScore)
+		}
+		fmt.Fprintln(w)
+	}
+
+	for _, b := range result.Builds {
+		if multi {
+			fmt.Fprintf(w, "======== build %q ========\n", b.Name)
+		}
+		writeBuildText(w, b, multi)
+	}
+
+	if multi {
+		fmt.Fprintf(w, "\n══ AGGREGATE SCORE (min across builds): %.4f / 1.0000 ══\n", result.TotalScore)
+	}
+}
+
+func writeBuildText(w io.Writer, b *runner.BuildRun, multi bool) {
+	if b.Skipped {
+		fmt.Fprintf(w, "skipped: %s\n", b.SkipReason)
+		return
+	}
+
+	if b.BuildLog != "" {
+		fmt.Fprintf(w, "=== BUILD LOG ===\n%s\n", b.BuildLog)
+	}
+
+	for _, gr := range b.Groups {
 		passed := gr.Passed
 		total := gr.Total
 		pct := 0.0
@@ -53,13 +90,19 @@ func Text(w io.Writer, result *runner.SuiteResult) {
 		fmt.Fprintf(w, "└─\n")
 	}
 
-	fmt.Fprintf(w, "\n══ TOTAL SCORE: %.4f / 1.0000 ══\n", result.TotalScore)
+	if !multi {
+		fmt.Fprintf(w, "\n══ TOTAL SCORE: %.4f / 1.0000 ══\n", b.TotalScore)
+	}
 }
 
 func JSON(w io.Writer, result *runner.SuiteResult) error {
 	enc := json.NewEncoder(w)
 	enc.SetIndent("", "  ")
-	return enc.Encode(jsonResult(result))
+
+	if len(result.Builds) <= 1 {
+		return enc.Encode(flatResult(result))
+	}
+	return enc.Encode(nestedResult(result))
 }
 
 type jsonSuiteResult struct {
@@ -68,6 +111,21 @@ type jsonSuiteResult struct {
 	Groups     []jsonGroupResult `json:"groups"`
 }
 
+type jsonMultiSuiteResult struct {
+	TotalScore float64                   `json:"total_score"`
+	Builds     map[string]jsonSuiteBuild `json:"builds"`
+}
+
+type jsonSuiteBuild struct {
+	Name       string            `json:"name"`
+	Toolchain  string            `json:"toolchain,omitempty"`
+	Skipped    bool              `json:"skipped,omitempty"`
+	SkipReason string            `json:"skip_reason,omitempty"`
+	TotalScore float64           `json:"total_score"`
+	BuildLog   string            `json:"build_log,omitempty"`
+	Groups     []jsonGroupResult `json:"groups"`
+}
+
 type jsonGroupResult struct {
 	Name   string           `json:"name"`
 	Weight float64          `json:"weight"`
@@ -86,6 +144,62 @@ type jsonTestResult struct {
 	Failures      []string `json:"failures,omitempty"`
 }
 
+func flatResult(r *runner.SuiteResult) jsonSuiteResult {
+	res := jsonSuiteResult{TotalScore: r.TotalScore}
+	if len(r.Builds) == 0 {
+		return res
+	}
+	b := r.Builds[0]
+	res.BuildLog = b.BuildLog
+	for _, gr := range b.Groups {
+		res.Groups = append(res.Groups, groupJSON(gr))
+	}
+	return res
+}
+
+func nestedResult(r *runner.SuiteResult) jsonMultiSuiteResult {
+	res := jsonMultiSuiteResult{
+		TotalScore: r.TotalScore,
+		Builds:     map[string]jsonSuiteBuild{},
+	}
+	for _, b := range r.Builds {
+		entry := jsonSuiteBuild{
+			Name:       b.Name,
+			Toolchain:  b.Toolchain,
+			Skipped:    b.Skipped,
+			SkipReason: b.SkipReason,
+			TotalScore: b.TotalScore,
+			BuildLog:   b.BuildLog,
+		}
+		for _, gr := range b.Groups {
+			entry.Groups = append(entry.Groups, groupJSON(gr))
+		}
+		res.Builds[b.Name] = entry
+	}
+	return res
+}
+
+func groupJSON(gr *runner.GroupResult) jsonGroupResult {
+	jgr := jsonGroupResult{
+		Name:   gr.Name,
+		Weight: gr.Weight,
+		Score:  gr.Score,
+		Passed: gr.Passed,
+		Total:  gr.Total,
+	}
+	for _, tr := range gr.Tests {
+		jgr.Tests = append(jgr.Tests, jsonTestResult{
+			Name:          tr.Name,
+			Status:        tr.Status.String(),
+			ElapsedMs:     tr.Elapsed.Milliseconds(),
+			PeakMemoryKB:  tr.PeakMemory / 1024,
+			MemoryLimitKB: tr.MemoryLimit / 1024,
+			Failures:      tr.Failures,
+		})
+	}
+	return jgr
+}
+
 func Aggregate(w io.Writer, dir string) error {
 	files, err := filepath.Glob(filepath.Join(dir, "*", "*.json"))
 	if err != nil {
@@ -105,24 +219,24 @@ func Aggregate(w io.Writer, dir string) error {
 	}
 
 	var entries []entry
-	allPassed := true
+	minScore := math.Inf(1)
 
 	for _, f := range files {
 		data, err := os.ReadFile(f)
 		if err != nil {
 			return fmt.Errorf("read %s: %w", f, err)
 		}
-		var report jsonSuiteResult
-		if err := json.Unmarshal(data, &report); err != nil {
+		score, err := extractTotalScore(data)
+		if err != nil {
 			return fmt.Errorf("parse %s: %w", f, err)
 		}
 
 		cfg := filepath.Base(filepath.Dir(f))
 		cfg = strings.TrimPrefix(cfg, "report_")
 
-		entries = append(entries, entry{Config: cfg, Score: report.TotalScore})
-		if report.TotalScore < 0.9999 {
-			allPassed = false
+		entries = append(entries, entry{Config: cfg, Score: score})
+		if score < minScore {
+			minScore = score
 		}
 	}
 
@@ -133,13 +247,24 @@ func Aggregate(w io.Writer, dir string) error {
 	for _, e := range entries {
 		fmt.Fprintf(w, "| %s | %.4f |\n", e.Config, e.Score)
 	}
+	fmt.Fprintf(w, "| **Overall (min)** | **%.4f** |\n", minScore)
 
-	if !allPassed {
-		return fmt.Errorf("one or more configurations scored below 1.0")
+	if minScore < 0.9999 {
+		return fmt.Errorf("minimum score across configurations is %.4f (below 1.0)", minScore)
 	}
 	return nil
 }
 
+func extractTotalScore(data []byte) (float64, error) {
+	var header struct {
+		TotalScore float64 `json:"total_score"`
+	}
+	if err := json.Unmarshal(data, &header); err != nil {
+		return 0, err
+	}
+	return header.TotalScore, nil
+}
+
 func humanBytes(n int64) string {
 	const (
 		KiB = 1024
@@ -157,31 +282,3 @@ func humanBytes(n int64) string {
 		return fmt.Sprintf("%dB", n)
 	}
 }
-
-func jsonResult(r *runner.SuiteResult) jsonSuiteResult {
-	res := jsonSuiteResult{
-		TotalScore: r.TotalScore,
-		BuildLog:   r.BuildLog,
-	}
-	for _, gr := range r.Groups {
-		jgr := jsonGroupResult{
-			Name:   gr.Name,
-			Weight: gr.Weight,
-			Score:  gr.Score,
-			Passed: gr.Passed,
-			Total:  gr.Total,
-		}
-		for _, tr := range gr.Tests {
-			jgr.Tests = append(jgr.Tests, jsonTestResult{
-				Name:          tr.Name,
-				Status:        tr.Status.String(),
-				ElapsedMs:     tr.Elapsed.Milliseconds(),
-				PeakMemoryKB:  tr.PeakMemory / 1024,
-				MemoryLimitKB: tr.MemoryLimit / 1024,
-				Failures:      tr.Failures,
-			})
-		}
-		res.Groups = append(res.Groups, jgr)
-	}
-	return res
-}