some fixes
This commit is contained in:
@@ -24,6 +24,8 @@ Flags:
|
||||
--build <name> run only the named structured build (use with matrix CI)
|
||||
--list-builds print the names of structured builds in the suite as JSON
|
||||
--list-matrix print the full (build, toolchain, platform) matrix as JSON
|
||||
--test <name> run tests with specific group/name or glob
|
||||
--group <name> run all tests from <name> group
|
||||
--help show help
|
||||
|
||||
Example:
|
||||
@@ -32,6 +34,9 @@ Example:
|
||||
judge lab1.jdg ./student-solution --build=sanitized
|
||||
judge --list-builds lab1.jdg
|
||||
judge aggregate reports/
|
||||
judge suite.jdg . --test "basic/*"
|
||||
judge suite.jdg . --test "*/timeout"
|
||||
judge suite.jdg . --group "basic"
|
||||
`
|
||||
|
||||
func main() {
|
||||
|
||||
10
dsl/ast.go
10
dsl/ast.go
@@ -53,6 +53,16 @@ type Pattern struct {
|
||||
InputFile string
|
||||
OutputFile string
|
||||
|
||||
// StageAll, valid only in dir mode, stages every file found in each matched
|
||||
// directory into the test's working directory (basenames preserved), except
|
||||
// the OutputFile and ArgsFile. Use it when the program discovers its own
|
||||
// inputs in the working directory instead of receiving a single input file.
|
||||
StageAll bool
|
||||
// ArgsFile, valid only in dir mode, reads the program arguments from this
|
||||
// file inside each matched directory (whitespace-separated). It takes
|
||||
// precedence over Args. Placeholders are still substituted.
|
||||
ArgsFile string
|
||||
|
||||
Args []string
|
||||
}
|
||||
|
||||
|
||||
@@ -807,6 +807,18 @@ func (p *Parser) parsePattern() (*Pattern, error) {
|
||||
return nil, err
|
||||
}
|
||||
pat.DirsGlob = val.Value
|
||||
case "stage":
|
||||
b, err := p.parseBool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pat.StageAll = b
|
||||
case "args_file":
|
||||
val, err := p.expect(TOKEN_STRING)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pat.ArgsFile = val.Value
|
||||
case "args":
|
||||
xs, err := p.parseStringList()
|
||||
if err != nil {
|
||||
|
||||
@@ -37,6 +37,38 @@ group("g") {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePatternStageAllMode(t *testing.T) {
|
||||
src := `
|
||||
build "make"
|
||||
group("g") {
|
||||
weight = 1.0
|
||||
pattern {
|
||||
dirs = "testdata/test_*"
|
||||
stage = true
|
||||
args_file = "args.txt"
|
||||
output = "answer.txt"
|
||||
}
|
||||
}
|
||||
`
|
||||
f, _, err := Parse(src)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
pat := f.Groups[0].Pattern
|
||||
if pat == nil {
|
||||
t.Fatal("no pattern")
|
||||
}
|
||||
if !pat.StageAll {
|
||||
t.Error("StageAll = false, want true")
|
||||
}
|
||||
if pat.ArgsFile != "args.txt" {
|
||||
t.Errorf("ArgsFile = %q", pat.ArgsFile)
|
||||
}
|
||||
if pat.OutputFile != "answer.txt" {
|
||||
t.Errorf("OutputFile = %q", pat.OutputFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePatternUnknownField(t *testing.T) {
|
||||
src := `
|
||||
build "make"
|
||||
|
||||
@@ -298,12 +298,16 @@ module.exports = grammar({
|
||||
$.pattern_input_field,
|
||||
$.pattern_output_field,
|
||||
$.pattern_dirs_field,
|
||||
$.pattern_stage_field,
|
||||
$.pattern_args_file_field,
|
||||
$.pattern_args_field,
|
||||
),
|
||||
|
||||
pattern_input_field: $ => seq('input', '=', field('value', $.string)),
|
||||
pattern_output_field: $ => seq('output', '=', field('value', $.string)),
|
||||
pattern_dirs_field: $ => seq('dirs', '=', field('value', $.string)),
|
||||
pattern_stage_field: $ => seq('stage', '=', field('value', $.bool)),
|
||||
pattern_args_file_field: $ => seq('args_file', '=', field('value', $.string)),
|
||||
pattern_args_field: $ => seq('args', '=', field('value', repeat1($.string))),
|
||||
|
||||
bool: $ => choice('true', 'false'),
|
||||
|
||||
@@ -36,6 +36,13 @@ func globWithAffixes(pattern string) ([]string, string, string, error) {
|
||||
}
|
||||
|
||||
func expandGlobPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
||||
if pattern.StageAll {
|
||||
return nil, fmt.Errorf("pattern: `stage` is only valid with `dirs`")
|
||||
}
|
||||
if pattern.ArgsFile != "" {
|
||||
return nil, fmt.Errorf("pattern: `args_file` is only valid with `dirs`")
|
||||
}
|
||||
|
||||
inputIsGlob := strings.Contains(pattern.InputGlob, "*")
|
||||
outputIsGlob := strings.Contains(pattern.OutputGlob, "*")
|
||||
|
||||
@@ -107,6 +114,22 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
||||
return nil, fmt.Errorf("no directories matched %q", pattern.DirsGlob)
|
||||
}
|
||||
|
||||
if pattern.StageAll {
|
||||
var tests []*dsl.Test
|
||||
for _, dir := range dirs {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
t, err := buildStagedDirTest(dir, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tests = append(tests, t)
|
||||
}
|
||||
return tests, nil
|
||||
}
|
||||
|
||||
var cases []patternCase
|
||||
for _, dir := range dirs {
|
||||
info, err := os.Stat(dir)
|
||||
@@ -123,6 +146,72 @@ func expandDirPattern(pattern *dsl.Pattern) ([]*dsl.Test, error) {
|
||||
return buildTests(cases, pattern.Args)
|
||||
}
|
||||
|
||||
// buildStagedDirTest builds a single test from a directory, staging every
|
||||
// regular file in it (except the output and args files) into the test's
|
||||
// working directory. Program arguments come from ArgsFile (if set) or the
|
||||
// static Args template; the expected stdout is the content of OutputFile.
|
||||
func buildStagedDirTest(dir string, pattern *dsl.Pattern) (*dsl.Test, error) {
|
||||
base := filepath.Base(dir)
|
||||
zero := 0
|
||||
t := &dsl.Test{
|
||||
Name: fmt.Sprintf("pattern:%s", base),
|
||||
Env: map[string]string{},
|
||||
InFiles: map[string]string{},
|
||||
OutFiles: map[string]string{},
|
||||
ExitCode: &zero,
|
||||
Stdout: dsl.NoMatcher{},
|
||||
Stderr: dsl.NoMatcher{},
|
||||
}
|
||||
|
||||
skip := map[string]bool{}
|
||||
if pattern.OutputFile != "" {
|
||||
skip[pattern.OutputFile] = true
|
||||
}
|
||||
if pattern.ArgsFile != "" {
|
||||
skip[pattern.ArgsFile] = true
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dir %q: %w", dir, err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || skip[e.Name()] {
|
||||
continue
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read staged file %q: %w", e.Name(), err)
|
||||
}
|
||||
t.SetInputFile(e.Name(), content)
|
||||
}
|
||||
|
||||
argTemplate := pattern.Args
|
||||
if pattern.ArgsFile != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, pattern.ArgsFile))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read args file %q: %w", pattern.ArgsFile, err)
|
||||
}
|
||||
argTemplate = strings.Fields(string(raw))
|
||||
}
|
||||
if len(argTemplate) > 0 {
|
||||
t.Args = substituteArgs(argTemplate, map[string]string{
|
||||
"{name}": base,
|
||||
"{dir}": dir,
|
||||
})
|
||||
}
|
||||
|
||||
if pattern.OutputFile != "" {
|
||||
outputContent, err := os.ReadFile(filepath.Join(dir, pattern.OutputFile))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read output %q: %w", pattern.OutputFile, err)
|
||||
}
|
||||
t.SetStdout(outputContent)
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func buildTest(c *patternCase, argTemplate []string, useInputAsFile, useOutputAsFile bool) (*dsl.Test, error) {
|
||||
inputContent, err := os.ReadFile(c.inputPath)
|
||||
if err != nil {
|
||||
|
||||
@@ -211,6 +211,100 @@ func TestExpandDirModeWithArgs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandDirModeStageAll(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// Two test directories, each with several input files the program discovers
|
||||
// itself, plus an args file and an expected-stdout file.
|
||||
writeFile(t, dir, "cases/test_1/1.txt", "10\n")
|
||||
writeFile(t, dir, "cases/test_1/2.txt", "20\n")
|
||||
writeFile(t, dir, "cases/test_1/decoy.log", "ignore me\n")
|
||||
writeFile(t, dir, "cases/test_1/args.txt", "$.txt 1 1 2\n")
|
||||
writeFile(t, dir, "cases/test_1/answer.txt", "10\n20\n")
|
||||
writeFile(t, dir, "cases/test_2/a.txt", "x\n")
|
||||
writeFile(t, dir, "cases/test_2/args.txt", "a.txt\n")
|
||||
writeFile(t, dir, "cases/test_2/answer.txt", "x\n")
|
||||
|
||||
cwd, _ := os.Getwd()
|
||||
defer os.Chdir(cwd)
|
||||
os.Chdir(dir)
|
||||
|
||||
tests, err := expandPattern(&dsl.Pattern{
|
||||
DirsGlob: "cases/*",
|
||||
StageAll: true,
|
||||
ArgsFile: "args.txt",
|
||||
OutputFile: "answer.txt",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tests) != 2 {
|
||||
t.Fatalf("expected 2 tests, got %d", len(tests))
|
||||
}
|
||||
|
||||
var tc1 *dsl.Test
|
||||
for _, tc := range tests {
|
||||
if tc.Name == "pattern:test_1" {
|
||||
tc1 = tc
|
||||
}
|
||||
}
|
||||
if tc1 == nil {
|
||||
t.Fatalf("test_1 not found among %v", tests)
|
||||
}
|
||||
|
||||
// All input files staged, including the decoy; answer/args excluded.
|
||||
for _, name := range []string{"1.txt", "2.txt", "decoy.log"} {
|
||||
if _, ok := tc1.InFiles[name]; !ok {
|
||||
t.Errorf("staged InFiles missing %q (have %v)", name, tc1.InFiles)
|
||||
}
|
||||
}
|
||||
if _, ok := tc1.InFiles["answer.txt"]; ok {
|
||||
t.Error("answer.txt must not be staged")
|
||||
}
|
||||
if _, ok := tc1.InFiles["args.txt"]; ok {
|
||||
t.Error("args.txt must not be staged")
|
||||
}
|
||||
|
||||
// Args read from args.txt, whitespace-split.
|
||||
want := []string{"$.txt", "1", "1", "2"}
|
||||
if strings.Join(tc1.Args, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("args = %v, want %v", tc1.Args, want)
|
||||
}
|
||||
|
||||
// Expected stdout is the answer file content.
|
||||
m, ok := tc1.Stdout.(dsl.ExactMatcher)
|
||||
if !ok {
|
||||
t.Fatalf("stdout should be ExactMatcher, got %T", tc1.Stdout)
|
||||
}
|
||||
if m.Value != "10\n20\n" {
|
||||
t.Errorf("stdout = %q, want %q", m.Value, "10\n20\n")
|
||||
}
|
||||
|
||||
// A successful functional test must exit zero.
|
||||
if tc1.ExitCode == nil || *tc1.ExitCode != 0 {
|
||||
t.Errorf("ExitCode = %v, want 0", tc1.ExitCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandStageAllRejectedInGlobMode(t *testing.T) {
|
||||
_, err := expandPattern(&dsl.Pattern{
|
||||
InputGlob: "tests/*.in",
|
||||
StageAll: true,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage") {
|
||||
t.Fatalf("expected stage-only-with-dirs error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandArgsFileRejectedInGlobMode(t *testing.T) {
|
||||
_, err := expandPattern(&dsl.Pattern{
|
||||
InputGlob: "tests/*.in",
|
||||
ArgsFile: "args.txt",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "args_file") {
|
||||
t.Fatalf("expected args_file-only-with-dirs error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandPatternRejectsAllLiterals(t *testing.T) {
|
||||
_, err := expandPattern(&dsl.Pattern{
|
||||
InputGlob: "tests/a.in",
|
||||
|
||||
Reference in New Issue
Block a user