Compare commits
1 Commits
v0.2.4
...
1ef3973a15
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ef3973a15 |
@@ -24,6 +24,8 @@ Flags:
|
|||||||
--build <name> run only the named structured build (use with matrix CI)
|
--build <name> run only the named structured build (use with matrix CI)
|
||||||
--list-builds print the names of structured builds in the suite as JSON
|
--list-builds print the names of structured builds in the suite as JSON
|
||||||
--list-matrix print the full (build, toolchain, platform) matrix as JSON
|
--list-matrix print the full (build, toolchain, platform) matrix as JSON
|
||||||
|
--test <name> run tests with specific group/name or glob
|
||||||
|
--group <name> run all tests from <name> group
|
||||||
--help show help
|
--help show help
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
@@ -32,6 +34,9 @@ Example:
|
|||||||
judge lab1.jdg ./student-solution --build=sanitized
|
judge lab1.jdg ./student-solution --build=sanitized
|
||||||
judge --list-builds lab1.jdg
|
judge --list-builds lab1.jdg
|
||||||
judge aggregate reports/
|
judge aggregate reports/
|
||||||
|
judge suite.jdg . --test "basic/*"
|
||||||
|
judge suite.jdg . --test "*/timeout"
|
||||||
|
judge suite.jdg . --group "basic"
|
||||||
`
|
`
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|||||||
10
dsl/ast.go
10
dsl/ast.go
@@ -53,6 +53,16 @@ type Pattern struct {
|
|||||||
InputFile string
|
InputFile string
|
||||||
OutputFile string
|
OutputFile string
|
||||||
|
|
||||||
|
// StageAll, valid only in dir mode, stages every file found in each matched
|
||||||
|
// directory into the test's working directory (basenames preserved), except
|
||||||
|
// the OutputFile and ArgsFile. Use it when the program discovers its own
|
||||||
|
// inputs in the working directory instead of receiving a single input file.
|
||||||
|
StageAll bool
|
||||||
|
// ArgsFile, valid only in dir mode, reads the program arguments from this
|
||||||
|
// file inside each matched directory (whitespace-separated). It takes
|
||||||
|
// precedence over Args. Placeholders are still substituted.
|
||||||
|
ArgsFile string
|
||||||
|
|
||||||
Args []string
|
Args []string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -807,6 +807,18 @@ func (p *Parser) parsePattern() (*Pattern, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
pat.DirsGlob = val.Value
|
pat.DirsGlob = val.Value
|
||||||
|
case "stage":
|
||||||
|
b, err := p.parseBool()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pat.StageAll = b
|
||||||
|
case "args_file":
|
||||||
|
val, err := p.expect(TOKEN_STRING)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pat.ArgsFile = val.Value
|
||||||
case "args":
|
case "args":
|
||||||
xs, err := p.parseStringList()
|
xs, err := p.parseStringList()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -37,6 +37,38 @@ group("g") {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParsePatternStageAllMode(t *testing.T) {
|
||||||
|
src := `
|
||||||
|
build "make"
|
||||||
|
group("g") {
|
||||||
|
weight = 1.0
|
||||||
|
pattern {
|
||||||
|
dirs = "testdata/test_*"
|
||||||
|
stage = true
|
||||||
|
args_file = "args.txt"
|
||||||
|
output = "answer.txt"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`
|
||||||
|
f, _, err := Parse(src)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
pat := f.Groups[0].Pattern
|
||||||
|
if pat == nil {
|
||||||
|
t.Fatal("no pattern")
|
||||||
|
}
|
||||||
|
if !pat.StageAll {
|
||||||
|
t.Error("StageAll = false, want true")
|
||||||
|
}
|
||||||
|
if pat.ArgsFile != "args.txt" {
|
||||||
|
t.Errorf("ArgsFile = %q", pat.ArgsFile)
|
||||||
|
}
|
||||||
|
if pat.OutputFile != "answer.txt" {
|
||||||
|
t.Errorf("OutputFile = %q", pat.OutputFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParsePatternUnknownField(t *testing.T) {
|
func TestParsePatternUnknownField(t *testing.T) {
|
||||||
src := `
|
src := `
|
||||||
build "make"
|
build "make"
|
||||||
|
|||||||
@@ -298,12 +298,16 @@ module.exports = grammar({
|
|||||||
$.pattern_input_field,
|
$.pattern_input_field,
|
||||||
$.pattern_output_field,
|
$.pattern_output_field,
|
||||||
$.pattern_dirs_field,
|
$.pattern_dirs_field,
|
||||||
|
$.pattern_stage_field,
|
||||||
|
$.pattern_args_file_field,
|
||||||
$.pattern_args_field,
|
$.pattern_args_field,
|
||||||
),
|
),
|
||||||
|
|
||||||
pattern_input_field: $ => seq('input', '=', field('value', $.string)),
|
pattern_input_field: $ => seq('input', '=', field('value', $.string)),
|
||||||
pattern_output_field: $ => seq('output', '=', field('value', $.string)),
|
pattern_output_field: $ => seq('output', '=', field('value', $.string)),
|
||||||
pattern_dirs_field: $ => seq('dirs', '=', field('value', $.string)),
|
pattern_dirs_field: $ => seq('dirs', '=', field('value', $.string)),
|
||||||
|
pattern_stage_field: $ => seq('stage', '=', field('value', $.bool)),
|
||||||
|
pattern_args_file_field: $ => seq('args_file', '=', field('value', $.string)),
|
||||||
pattern_args_field: $ => seq('args', '=', field('value', repeat1($.string))),
|
pattern_args_field: $ => seq('args', '=', field('value', repeat1($.string))),
|
||||||
|
|
||||||
bool: $ => choice('true', 'false'),
|
bool: $ => choice('true', 'false'),
|
||||||
|
|||||||
@@ -36,6 +36,13 @@ func globWithAffixes(pattern string) ([]string, string, string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func expandGlobPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
func expandGlobPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
||||||
|
if pattern.StageAll {
|
||||||
|
return nil, fmt.Errorf("pattern: `stage` is only valid with `dirs`")
|
||||||
|
}
|
||||||
|
if pattern.ArgsFile != "" {
|
||||||
|
return nil, fmt.Errorf("pattern: `args_file` is only valid with `dirs`")
|
||||||
|
}
|
||||||
|
|
||||||
inputIsGlob := strings.Contains(pattern.InputGlob, "*")
|
inputIsGlob := strings.Contains(pattern.InputGlob, "*")
|
||||||
outputIsGlob := strings.Contains(pattern.OutputGlob, "*")
|
outputIsGlob := strings.Contains(pattern.OutputGlob, "*")
|
||||||
|
|
||||||
@@ -107,6 +114,22 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
|||||||
return nil, fmt.Errorf("no directories matched %q", pattern.DirsGlob)
|
return nil, fmt.Errorf("no directories matched %q", pattern.DirsGlob)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if pattern.StageAll {
|
||||||
|
var tests []*dsl.Test
|
||||||
|
for _, dir := range dirs {
|
||||||
|
info, err := os.Stat(dir)
|
||||||
|
if err != nil || !info.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t, err := buildStagedDirTest(dir, pattern)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tests = append(tests, t)
|
||||||
|
}
|
||||||
|
return tests, nil
|
||||||
|
}
|
||||||
|
|
||||||
var cases []patternCase
|
var cases []patternCase
|
||||||
for _, dir := range dirs {
|
for _, dir := range dirs {
|
||||||
info, err := os.Stat(dir)
|
info, err := os.Stat(dir)
|
||||||
@@ -123,6 +146,72 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
|||||||
return buildTests(cases, pattern.Args)
|
return buildTests(cases, pattern.Args)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildStagedDirTest builds a single test from a directory, staging every
|
||||||
|
// regular file in it (except the output and args files) into the test's
|
||||||
|
// working directory. Program arguments come from ArgsFile (if set) or the
|
||||||
|
// static Args template; the expected stdout is the content of OutputFile.
|
||||||
|
func buildStagedDirTest(dir string, pattern *dsl.Pattern) (*dsl.Test, error) {
|
||||||
|
base := filepath.Base(dir)
|
||||||
|
zero := 0
|
||||||
|
t := &dsl.Test{
|
||||||
|
Name: fmt.Sprintf("pattern:%s", base),
|
||||||
|
Env: map[string]string{},
|
||||||
|
InFiles: map[string]string{},
|
||||||
|
OutFiles: map[string]string{},
|
||||||
|
ExitCode: &zero,
|
||||||
|
Stdout: dsl.NoMatcher{},
|
||||||
|
Stderr: dsl.NoMatcher{},
|
||||||
|
}
|
||||||
|
|
||||||
|
skip := map[string]bool{}
|
||||||
|
if pattern.OutputFile != "" {
|
||||||
|
skip[pattern.OutputFile] = true
|
||||||
|
}
|
||||||
|
if pattern.ArgsFile != "" {
|
||||||
|
skip[pattern.ArgsFile] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read dir %q: %w", dir, err)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || skip[e.Name()] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
content, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read staged file %q: %w", e.Name(), err)
|
||||||
|
}
|
||||||
|
t.SetInputFile(e.Name(), content)
|
||||||
|
}
|
||||||
|
|
||||||
|
argTemplate := pattern.Args
|
||||||
|
if pattern.ArgsFile != "" {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(dir, pattern.ArgsFile))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read args file %q: %w", pattern.ArgsFile, err)
|
||||||
|
}
|
||||||
|
argTemplate = strings.Fields(string(raw))
|
||||||
|
}
|
||||||
|
if len(argTemplate) > 0 {
|
||||||
|
t.Args = substituteArgs(argTemplate, map[string]string{
|
||||||
|
"{name}": base,
|
||||||
|
"{dir}": dir,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if pattern.OutputFile != "" {
|
||||||
|
outputContent, err := os.ReadFile(filepath.Join(dir, pattern.OutputFile))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read output %q: %w", pattern.OutputFile, err)
|
||||||
|
}
|
||||||
|
t.SetStdout(outputContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
func buildTest(c *patternCase, argTemplate []string, useInputAsFile, useOutputAsFile bool) (*dsl.Test, error) {
|
func buildTest(c *patternCase, argTemplate []string, useInputAsFile, useOutputAsFile bool) (*dsl.Test, error) {
|
||||||
inputContent, err := os.ReadFile(c.inputPath)
|
inputContent, err := os.ReadFile(c.inputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -211,6 +211,100 @@ func TestExpandDirModeWithArgs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExpandDirModeStageAll(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
// Two test directories, each with several input files the program discovers
|
||||||
|
// itself, plus an args file and an expected-stdout file.
|
||||||
|
writeFile(t, dir, "cases/test_1/1.txt", "10\n")
|
||||||
|
writeFile(t, dir, "cases/test_1/2.txt", "20\n")
|
||||||
|
writeFile(t, dir, "cases/test_1/decoy.log", "ignore me\n")
|
||||||
|
writeFile(t, dir, "cases/test_1/args.txt", "$.txt 1 1 2\n")
|
||||||
|
writeFile(t, dir, "cases/test_1/answer.txt", "10\n20\n")
|
||||||
|
writeFile(t, dir, "cases/test_2/a.txt", "x\n")
|
||||||
|
writeFile(t, dir, "cases/test_2/args.txt", "a.txt\n")
|
||||||
|
writeFile(t, dir, "cases/test_2/answer.txt", "x\n")
|
||||||
|
|
||||||
|
cwd, _ := os.Getwd()
|
||||||
|
defer os.Chdir(cwd)
|
||||||
|
os.Chdir(dir)
|
||||||
|
|
||||||
|
tests, err := expandPattern(&dsl.Pattern{
|
||||||
|
DirsGlob: "cases/*",
|
||||||
|
StageAll: true,
|
||||||
|
ArgsFile: "args.txt",
|
||||||
|
OutputFile: "answer.txt",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(tests) != 2 {
|
||||||
|
t.Fatalf("expected 2 tests, got %d", len(tests))
|
||||||
|
}
|
||||||
|
|
||||||
|
var tc1 *dsl.Test
|
||||||
|
for _, tc := range tests {
|
||||||
|
if tc.Name == "pattern:test_1" {
|
||||||
|
tc1 = tc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc1 == nil {
|
||||||
|
t.Fatalf("test_1 not found among %v", tests)
|
||||||
|
}
|
||||||
|
|
||||||
|
// All input files staged, including the decoy; answer/args excluded.
|
||||||
|
for _, name := range []string{"1.txt", "2.txt", "decoy.log"} {
|
||||||
|
if _, ok := tc1.InFiles[name]; !ok {
|
||||||
|
t.Errorf("staged InFiles missing %q (have %v)", name, tc1.InFiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := tc1.InFiles["answer.txt"]; ok {
|
||||||
|
t.Error("answer.txt must not be staged")
|
||||||
|
}
|
||||||
|
if _, ok := tc1.InFiles["args.txt"]; ok {
|
||||||
|
t.Error("args.txt must not be staged")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Args read from args.txt, whitespace-split.
|
||||||
|
want := []string{"$.txt", "1", "1", "2"}
|
||||||
|
if strings.Join(tc1.Args, " ") != strings.Join(want, " ") {
|
||||||
|
t.Errorf("args = %v, want %v", tc1.Args, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expected stdout is the answer file content.
|
||||||
|
m, ok := tc1.Stdout.(dsl.ExactMatcher)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("stdout should be ExactMatcher, got %T", tc1.Stdout)
|
||||||
|
}
|
||||||
|
if m.Value != "10\n20\n" {
|
||||||
|
t.Errorf("stdout = %q, want %q", m.Value, "10\n20\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A successful functional test must exit zero.
|
||||||
|
if tc1.ExitCode == nil || *tc1.ExitCode != 0 {
|
||||||
|
t.Errorf("ExitCode = %v, want 0", tc1.ExitCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpandStageAllRejectedInGlobMode(t *testing.T) {
|
||||||
|
_, err := expandPattern(&dsl.Pattern{
|
||||||
|
InputGlob: "tests/*.in",
|
||||||
|
StageAll: true,
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "stage") {
|
||||||
|
t.Fatalf("expected stage-only-with-dirs error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpandArgsFileRejectedInGlobMode(t *testing.T) {
|
||||||
|
_, err := expandPattern(&dsl.Pattern{
|
||||||
|
InputGlob: "tests/*.in",
|
||||||
|
ArgsFile: "args.txt",
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "args_file") {
|
||||||
|
t.Fatalf("expected args_file-only-with-dirs error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExpandPatternRejectsAllLiterals(t *testing.T) {
|
func TestExpandPatternRejectsAllLiterals(t *testing.T) {
|
||||||
_, err := expandPattern(&dsl.Pattern{
|
_, err := expandPattern(&dsl.Pattern{
|
||||||
InputGlob: "tests/a.in",
|
InputGlob: "tests/a.in",
|
||||||
|
|||||||
Reference in New Issue
Block a user