All checks were successful
build-dsl-smoke / Discover matrix (push) Successful in 8s
build-dsl-smoke / Build judge (push) Successful in 11s
build-dsl-smoke / ${{ matrix.cell.build }} / ${{ matrix.cell.toolchain }} / ${{ matrix.cell.platform }} (push) Successful in 5s
memory-limit / Build judge (pull_request) Successful in 10s
build-dsl-smoke / SUMMARY (push) Successful in 3s
memory-limit / Linux / gcc (pull_request) Successful in 9s
memory-limit / Linux / clang (pull_request) Successful in 13s
memory-limit / Windows / clang (pull_request) Successful in 16s
memory-limit / Windows / msvc (pull_request) Successful in 17s
285 lines
6.8 KiB
Go
285 lines
6.8 KiB
Go
package reporter
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/Mond1c/judge/runner"
|
|
)
|
|
|
|
func Text(w io.Writer, result *runner.SuiteResult) {
|
|
if len(result.Builds) == 0 {
|
|
fmt.Fprintln(w, "(no builds executed)")
|
|
return
|
|
}
|
|
|
|
multi := len(result.Builds) > 1
|
|
if multi {
|
|
fmt.Fprintf(w, "=== %d builds ===\n", len(result.Builds))
|
|
for _, b := range result.Builds {
|
|
marker := ""
|
|
if b.Skipped {
|
|
marker = " (skipped)"
|
|
}
|
|
fmt.Fprintf(w, " • %s%s score=%.4f\n", b.Name, marker, b.TotalScore)
|
|
}
|
|
fmt.Fprintln(w)
|
|
}
|
|
|
|
for _, b := range result.Builds {
|
|
if multi {
|
|
fmt.Fprintf(w, "======== build %q ========\n", b.Name)
|
|
}
|
|
writeBuildText(w, b, multi)
|
|
}
|
|
|
|
if multi {
|
|
fmt.Fprintf(w, "\n══ AGGREGATE SCORE (min across builds): %.4f / 1.0000 ══\n", result.TotalScore)
|
|
}
|
|
}
|
|
|
|
func writeBuildText(w io.Writer, b *runner.BuildRun, multi bool) {
|
|
if b.Skipped {
|
|
fmt.Fprintf(w, "skipped: %s\n", b.SkipReason)
|
|
return
|
|
}
|
|
|
|
if b.BuildLog != "" {
|
|
fmt.Fprintf(w, "=== BUILD LOG ===\n%s\n", b.BuildLog)
|
|
}
|
|
|
|
for _, gr := range b.Groups {
|
|
passed := gr.Passed
|
|
total := gr.Total
|
|
pct := 0.0
|
|
if total > 0 {
|
|
pct = float64(passed) / float64(total) * 100
|
|
}
|
|
|
|
fmt.Fprintf(w, "\n┌─ group %q weight=%.2f score=%.4f\n",
|
|
gr.Name, gr.Weight, gr.Score)
|
|
fmt.Fprintf(w, "│ %d/%d passed (%.0f%%)\n", passed, total, pct)
|
|
|
|
for _, tr := range gr.Tests {
|
|
icon := "✓"
|
|
if tr.Status != runner.StatusPass {
|
|
icon = "✗"
|
|
}
|
|
mem := ""
|
|
if tr.PeakMemory > 0 {
|
|
mem = fmt.Sprintf(", %s", humanBytes(tr.PeakMemory))
|
|
if tr.MemoryLimit > 0 {
|
|
mem = fmt.Sprintf(", %s/%s", humanBytes(tr.PeakMemory), humanBytes(tr.MemoryLimit))
|
|
}
|
|
}
|
|
fmt.Fprintf(w, "│ %s [%s] %s (%dms%s)\n",
|
|
icon, tr.Status, tr.Name, tr.Elapsed.Milliseconds(), mem)
|
|
|
|
for _, f := range tr.Failures {
|
|
for _, line := range strings.Split(f, "\n") {
|
|
fmt.Fprintf(w, "│ %s\n", line)
|
|
}
|
|
}
|
|
}
|
|
fmt.Fprintf(w, "└─\n")
|
|
}
|
|
|
|
if !multi {
|
|
fmt.Fprintf(w, "\n══ TOTAL SCORE: %.4f / 1.0000 ══\n", b.TotalScore)
|
|
}
|
|
}
|
|
|
|
func JSON(w io.Writer, result *runner.SuiteResult) error {
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
|
|
if len(result.Builds) <= 1 {
|
|
return enc.Encode(flatResult(result))
|
|
}
|
|
return enc.Encode(nestedResult(result))
|
|
}
|
|
|
|
type jsonSuiteResult struct {
|
|
TotalScore float64 `json:"total_score"`
|
|
BuildLog string `json:"build_log,omitempty"`
|
|
Groups []jsonGroupResult `json:"groups"`
|
|
}
|
|
|
|
type jsonMultiSuiteResult struct {
|
|
TotalScore float64 `json:"total_score"`
|
|
Builds map[string]jsonSuiteBuild `json:"builds"`
|
|
}
|
|
|
|
type jsonSuiteBuild struct {
|
|
Name string `json:"name"`
|
|
Toolchain string `json:"toolchain,omitempty"`
|
|
Skipped bool `json:"skipped,omitempty"`
|
|
SkipReason string `json:"skip_reason,omitempty"`
|
|
TotalScore float64 `json:"total_score"`
|
|
BuildLog string `json:"build_log,omitempty"`
|
|
Groups []jsonGroupResult `json:"groups"`
|
|
}
|
|
|
|
type jsonGroupResult struct {
|
|
Name string `json:"name"`
|
|
Weight float64 `json:"weight"`
|
|
Score float64 `json:"score"`
|
|
Passed int `json:"passed"`
|
|
Total int `json:"total"`
|
|
Tests []jsonTestResult `json:"tests"`
|
|
}
|
|
|
|
type jsonTestResult struct {
|
|
Name string `json:"name"`
|
|
Status string `json:"status"`
|
|
ElapsedMs int64 `json:"elapsed_ms"`
|
|
PeakMemoryKB int64 `json:"peak_memory_kb,omitempty"`
|
|
MemoryLimitKB int64 `json:"memory_limit_kb,omitempty"`
|
|
Failures []string `json:"failures,omitempty"`
|
|
}
|
|
|
|
func flatResult(r *runner.SuiteResult) jsonSuiteResult {
|
|
res := jsonSuiteResult{TotalScore: r.TotalScore}
|
|
if len(r.Builds) == 0 {
|
|
return res
|
|
}
|
|
b := r.Builds[0]
|
|
res.BuildLog = b.BuildLog
|
|
for _, gr := range b.Groups {
|
|
res.Groups = append(res.Groups, groupJSON(gr))
|
|
}
|
|
return res
|
|
}
|
|
|
|
func nestedResult(r *runner.SuiteResult) jsonMultiSuiteResult {
|
|
res := jsonMultiSuiteResult{
|
|
TotalScore: r.TotalScore,
|
|
Builds: map[string]jsonSuiteBuild{},
|
|
}
|
|
for _, b := range r.Builds {
|
|
entry := jsonSuiteBuild{
|
|
Name: b.Name,
|
|
Toolchain: b.Toolchain,
|
|
Skipped: b.Skipped,
|
|
SkipReason: b.SkipReason,
|
|
TotalScore: b.TotalScore,
|
|
BuildLog: b.BuildLog,
|
|
}
|
|
for _, gr := range b.Groups {
|
|
entry.Groups = append(entry.Groups, groupJSON(gr))
|
|
}
|
|
res.Builds[b.Name] = entry
|
|
}
|
|
return res
|
|
}
|
|
|
|
func groupJSON(gr *runner.GroupResult) jsonGroupResult {
|
|
jgr := jsonGroupResult{
|
|
Name: gr.Name,
|
|
Weight: gr.Weight,
|
|
Score: gr.Score,
|
|
Passed: gr.Passed,
|
|
Total: gr.Total,
|
|
}
|
|
for _, tr := range gr.Tests {
|
|
jgr.Tests = append(jgr.Tests, jsonTestResult{
|
|
Name: tr.Name,
|
|
Status: tr.Status.String(),
|
|
ElapsedMs: tr.Elapsed.Milliseconds(),
|
|
PeakMemoryKB: tr.PeakMemory / 1024,
|
|
MemoryLimitKB: tr.MemoryLimit / 1024,
|
|
Failures: tr.Failures,
|
|
})
|
|
}
|
|
return jgr
|
|
}
|
|
|
|
func Aggregate(w io.Writer, dir string) error {
|
|
files, err := filepath.Glob(filepath.Join(dir, "*", "*.json"))
|
|
if err != nil {
|
|
return fmt.Errorf("glob reports: %w", err)
|
|
}
|
|
if len(files) == 0 {
|
|
files, _ = filepath.Glob(filepath.Join(dir, "*.json"))
|
|
}
|
|
if len(files) == 0 {
|
|
return fmt.Errorf("no JSON reports found in %s", dir)
|
|
}
|
|
sort.Strings(files)
|
|
|
|
type entry struct {
|
|
Config string
|
|
Score float64
|
|
}
|
|
|
|
var entries []entry
|
|
minScore := math.Inf(1)
|
|
|
|
for _, f := range files {
|
|
data, err := os.ReadFile(f)
|
|
if err != nil {
|
|
return fmt.Errorf("read %s: %w", f, err)
|
|
}
|
|
score, err := extractTotalScore(data)
|
|
if err != nil {
|
|
return fmt.Errorf("parse %s: %w", f, err)
|
|
}
|
|
|
|
cfg := filepath.Base(filepath.Dir(f))
|
|
cfg = strings.TrimPrefix(cfg, "report_")
|
|
|
|
entries = append(entries, entry{Config: cfg, Score: score})
|
|
if score < minScore {
|
|
minScore = score
|
|
}
|
|
}
|
|
|
|
fmt.Fprintln(w, "# Judge results")
|
|
fmt.Fprintln(w)
|
|
fmt.Fprintln(w, "| Configuration | Score |")
|
|
fmt.Fprintln(w, "|---|---|")
|
|
for _, e := range entries {
|
|
fmt.Fprintf(w, "| %s | %.4f |\n", e.Config, e.Score)
|
|
}
|
|
fmt.Fprintf(w, "| **Overall (min)** | **%.4f** |\n", minScore)
|
|
|
|
if minScore < 0.9999 {
|
|
return fmt.Errorf("minimum score across configurations is %.4f (below 1.0)", minScore)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func extractTotalScore(data []byte) (float64, error) {
|
|
var header struct {
|
|
TotalScore float64 `json:"total_score"`
|
|
}
|
|
if err := json.Unmarshal(data, &header); err != nil {
|
|
return 0, err
|
|
}
|
|
return header.TotalScore, nil
|
|
}
|
|
|
|
func humanBytes(n int64) string {
|
|
const (
|
|
KiB = 1024
|
|
MiB = 1024 * KiB
|
|
GiB = 1024 * MiB
|
|
)
|
|
switch {
|
|
case n >= GiB:
|
|
return fmt.Sprintf("%.2fGiB", float64(n)/float64(GiB))
|
|
case n >= MiB:
|
|
return fmt.Sprintf("%.1fMiB", float64(n)/float64(MiB))
|
|
case n >= KiB:
|
|
return fmt.Sprintf("%.0fKiB", float64(n)/float64(KiB))
|
|
default:
|
|
return fmt.Sprintf("%dB", n)
|
|
}
|
|
}
|