diff --git a/cmd/cli/main.go b/cmd/cli/main.go index b3ef61f..4d7d73f 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -24,6 +24,8 @@ Flags: --build run only the named structured build (use with matrix CI) --list-builds print the names of structured builds in the suite as JSON --list-matrix print the full (build, toolchain, platform) matrix as JSON + --test run tests with specific group/name or glob + --group run all tests from group --help show help Example: @@ -32,6 +34,9 @@ Example: judge lab1.jdg ./student-solution --build=sanitized judge --list-builds lab1.jdg judge aggregate reports/ + judge suite.jdg . --test "basic/*" + judge suite.jdg . --test "*/timeout" + judge suite.jdg . --group "basic" ` func main() { diff --git a/dsl/ast.go b/dsl/ast.go index 658a139..bd78861 100644 --- a/dsl/ast.go +++ b/dsl/ast.go @@ -53,6 +53,16 @@ type Pattern struct { InputFile string OutputFile string + // StageAll, valid only in dir mode, stages every file found in each matched + // directory into the test's working directory (basenames preserved), except + // the OutputFile and ArgsFile. Use it when the program discovers its own + // inputs in the working directory instead of receiving a single input file. + StageAll bool + // ArgsFile, valid only in dir mode, reads the program arguments from this + // file inside each matched directory (whitespace-separated). It takes + // precedence over Args. Placeholders are still substituted. + ArgsFile string + Args []string } diff --git a/dsl/parser.go b/dsl/parser.go index 413f758..b473ff6 100644 --- a/dsl/parser.go +++ b/dsl/parser.go @@ -807,6 +807,18 @@ func (p *Parser) parsePattern() (*Pattern, error) { return nil, err } pat.DirsGlob = val.Value + case "stage": + b, err := p.parseBool() + if err != nil { + return nil, err + } + pat.StageAll = b + case "args_file": + val, err := p.expect(TOKEN_STRING) + if err != nil { + return nil, err + } + pat.ArgsFile = val.Value case "args": xs, err := p.parseStringList() if err != nil { diff --git a/dsl/parser_misc_test.go b/dsl/parser_misc_test.go index 5c357b8..28cb01c 100644 --- a/dsl/parser_misc_test.go +++ b/dsl/parser_misc_test.go @@ -37,6 +37,38 @@ group("g") { } } +func TestParsePatternStageAllMode(t *testing.T) { + src := ` +build "make" +group("g") { + weight = 1.0 + pattern { + dirs = "testdata/test_*" + stage = true + args_file = "args.txt" + output = "answer.txt" + } +} +` + f, _, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + pat := f.Groups[0].Pattern + if pat == nil { + t.Fatal("no pattern") + } + if !pat.StageAll { + t.Error("StageAll = false, want true") + } + if pat.ArgsFile != "args.txt" { + t.Errorf("ArgsFile = %q", pat.ArgsFile) + } + if pat.OutputFile != "answer.txt" { + t.Errorf("OutputFile = %q", pat.OutputFile) + } +} + func TestParsePatternUnknownField(t *testing.T) { src := ` build "make" diff --git a/editor/zed/tree-sitter-jdg/grammar.js b/editor/zed/tree-sitter-jdg/grammar.js index 6177956..5b5a298 100644 --- a/editor/zed/tree-sitter-jdg/grammar.js +++ b/editor/zed/tree-sitter-jdg/grammar.js @@ -298,12 +298,16 @@ module.exports = grammar({ $.pattern_input_field, $.pattern_output_field, $.pattern_dirs_field, + $.pattern_stage_field, + $.pattern_args_file_field, $.pattern_args_field, ), pattern_input_field: $ => seq('input', '=', field('value', $.string)), pattern_output_field: $ => seq('output', '=', field('value', $.string)), pattern_dirs_field: $ => seq('dirs', '=', field('value', $.string)), + pattern_stage_field: $ => seq('stage', '=', field('value', $.bool)), + pattern_args_file_field: $ => seq('args_file', '=', field('value', $.string)), pattern_args_field: $ => seq('args', '=', field('value', repeat1($.string))), bool: $ => choice('true', 'false'), diff --git a/runner/expander.go b/runner/expander.go index ad695e6..5d1b922 100644 --- a/runner/expander.go +++ b/runner/expander.go @@ -36,6 +36,13 @@ func globWithAffixes(pattern string) ([]string, string, string, error) { } func expandGlobPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) { + if pattern.StageAll { + return nil, fmt.Errorf("pattern: `stage` is only valid with `dirs`") + } + if pattern.ArgsFile != "" { + return nil, fmt.Errorf("pattern: `args_file` is only valid with `dirs`") + } + inputIsGlob := strings.Contains(pattern.InputGlob, "*") outputIsGlob := strings.Contains(pattern.OutputGlob, "*") @@ -107,6 +114,22 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) { return nil, fmt.Errorf("no directories matched %q", pattern.DirsGlob) } + if pattern.StageAll { + var tests []*dsl.Test + for _, dir := range dirs { + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + continue + } + t, err := buildStagedDirTest(dir, pattern) + if err != nil { + return nil, err + } + tests = append(tests, t) + } + return tests, nil + } + var cases []patternCase for _, dir := range dirs { info, err := os.Stat(dir) @@ -123,6 +146,72 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) { return buildTests(cases, pattern.Args) } +// buildStagedDirTest builds a single test from a directory, staging every +// regular file in it (except the output and args files) into the test's +// working directory. Program arguments come from ArgsFile (if set) or the +// static Args template; the expected stdout is the content of OutputFile. +func buildStagedDirTest(dir string, pattern *dsl.Pattern) (*dsl.Test, error) { + base := filepath.Base(dir) + zero := 0 + t := &dsl.Test{ + Name: fmt.Sprintf("pattern:%s", base), + Env: map[string]string{}, + InFiles: map[string]string{}, + OutFiles: map[string]string{}, + ExitCode: &zero, + Stdout: dsl.NoMatcher{}, + Stderr: dsl.NoMatcher{}, + } + + skip := map[string]bool{} + if pattern.OutputFile != "" { + skip[pattern.OutputFile] = true + } + if pattern.ArgsFile != "" { + skip[pattern.ArgsFile] = true + } + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read dir %q: %w", dir, err) + } + for _, e := range entries { + if e.IsDir() || skip[e.Name()] { + continue + } + content, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, fmt.Errorf("read staged file %q: %w", e.Name(), err) + } + t.SetInputFile(e.Name(), content) + } + + argTemplate := pattern.Args + if pattern.ArgsFile != "" { + raw, err := os.ReadFile(filepath.Join(dir, pattern.ArgsFile)) + if err != nil { + return nil, fmt.Errorf("read args file %q: %w", pattern.ArgsFile, err) + } + argTemplate = strings.Fields(string(raw)) + } + if len(argTemplate) > 0 { + t.Args = substituteArgs(argTemplate, map[string]string{ + "{name}": base, + "{dir}": dir, + }) + } + + if pattern.OutputFile != "" { + outputContent, err := os.ReadFile(filepath.Join(dir, pattern.OutputFile)) + if err != nil { + return nil, fmt.Errorf("read output %q: %w", pattern.OutputFile, err) + } + t.SetStdout(outputContent) + } + + return t, nil +} + func buildTest(c *patternCase, argTemplate []string, useInputAsFile, useOutputAsFile bool) (*dsl.Test, error) { inputContent, err := os.ReadFile(c.inputPath) if err != nil { diff --git a/runner/expander_test.go b/runner/expander_test.go index 170ffba..342182d 100644 --- a/runner/expander_test.go +++ b/runner/expander_test.go @@ -211,6 +211,100 @@ func TestExpandDirModeWithArgs(t *testing.T) { } } +func TestExpandDirModeStageAll(t *testing.T) { + dir := t.TempDir() + // Two test directories, each with several input files the program discovers + // itself, plus an args file and an expected-stdout file. + writeFile(t, dir, "cases/test_1/1.txt", "10\n") + writeFile(t, dir, "cases/test_1/2.txt", "20\n") + writeFile(t, dir, "cases/test_1/decoy.log", "ignore me\n") + writeFile(t, dir, "cases/test_1/args.txt", "$.txt 1 1 2\n") + writeFile(t, dir, "cases/test_1/answer.txt", "10\n20\n") + writeFile(t, dir, "cases/test_2/a.txt", "x\n") + writeFile(t, dir, "cases/test_2/args.txt", "a.txt\n") + writeFile(t, dir, "cases/test_2/answer.txt", "x\n") + + cwd, _ := os.Getwd() + defer os.Chdir(cwd) + os.Chdir(dir) + + tests, err := expandPattern(&dsl.Pattern{ + DirsGlob: "cases/*", + StageAll: true, + ArgsFile: "args.txt", + OutputFile: "answer.txt", + }) + if err != nil { + t.Fatal(err) + } + if len(tests) != 2 { + t.Fatalf("expected 2 tests, got %d", len(tests)) + } + + var tc1 *dsl.Test + for _, tc := range tests { + if tc.Name == "pattern:test_1" { + tc1 = tc + } + } + if tc1 == nil { + t.Fatalf("test_1 not found among %v", tests) + } + + // All input files staged, including the decoy; answer/args excluded. + for _, name := range []string{"1.txt", "2.txt", "decoy.log"} { + if _, ok := tc1.InFiles[name]; !ok { + t.Errorf("staged InFiles missing %q (have %v)", name, tc1.InFiles) + } + } + if _, ok := tc1.InFiles["answer.txt"]; ok { + t.Error("answer.txt must not be staged") + } + if _, ok := tc1.InFiles["args.txt"]; ok { + t.Error("args.txt must not be staged") + } + + // Args read from args.txt, whitespace-split. + want := []string{"$.txt", "1", "1", "2"} + if strings.Join(tc1.Args, " ") != strings.Join(want, " ") { + t.Errorf("args = %v, want %v", tc1.Args, want) + } + + // Expected stdout is the answer file content. + m, ok := tc1.Stdout.(dsl.ExactMatcher) + if !ok { + t.Fatalf("stdout should be ExactMatcher, got %T", tc1.Stdout) + } + if m.Value != "10\n20\n" { + t.Errorf("stdout = %q, want %q", m.Value, "10\n20\n") + } + + // A successful functional test must exit zero. + if tc1.ExitCode == nil || *tc1.ExitCode != 0 { + t.Errorf("ExitCode = %v, want 0", tc1.ExitCode) + } +} + +func TestExpandStageAllRejectedInGlobMode(t *testing.T) { + _, err := expandPattern(&dsl.Pattern{ + InputGlob: "tests/*.in", + StageAll: true, + }) + if err == nil || !strings.Contains(err.Error(), "stage") { + t.Fatalf("expected stage-only-with-dirs error, got %v", err) + } +} + +func TestExpandArgsFileRejectedInGlobMode(t *testing.T) { + _, err := expandPattern(&dsl.Pattern{ + InputGlob: "tests/*.in", + ArgsFile: "args.txt", + }) + if err == nil || !strings.Contains(err.Error(), "args_file") { + t.Fatalf("expected args_file-only-with-dirs error, got %v", err) + } +} + func TestExpandPatternRejectsAllLiterals(t *testing.T) { _, err := expandPattern(&dsl.Pattern{ InputGlob: "tests/a.in",