diff --git a/.gitignore b/.gitignore index cfa7910..b0b44c6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,10 @@ *.exe *.json *.out -*.log -*.sh +*.txt + +# User created files. +*.user # Python cache. __pycache__/ diff --git a/README.md b/README.md index eaf43ab..bcc7fa3 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ python main.py --executable-path <путь к исполняемому файлу> --suite <набор тестов> ``` - *Примечание*. В *nix-подобных системах команда `python` может быть недоступна, вместо неё используйте команду `python3`. + *Примечание*. В *nix-подобных системах команда `python` может быть недоступна, вместо неё используйте команду `python3`. 5. Для ускорения отладки рекомендуется создать скрипт, выполняющий шаги 3-4. 6. Для обновления тестов выполнить команду: @@ -35,6 +35,7 @@ |:--:|:--------------------|:-----------------| | 0 | Интро | `intro` | | 1 | Представление чисел | `fixed_floating` | +| 2 | TIFF | `tiff` | ## Скрипт запуска diff --git a/main.py b/main.py index 803074d..81cf129 100755 --- a/main.py +++ b/main.py @@ -5,12 +5,13 @@ import platform import testsuites.intro as intro import testsuites.fixed_floating as fixed_floating +import testsuites.tiff as tiff from testsuites import Testsuite, DynamicWrapper from typing import Dict, Any, List __TESTSUITES: List[Testsuite] = [ - intro.instance, fixed_floating.instance + intro.instance, fixed_floating.instance, tiff.instance ] __SUITENAMES: List[str] = [t.name() for t in __TESTSUITES] diff --git a/testdata/tiff/const.pnm b/testdata/tiff/const.pnm new file mode 100644 index 0000000..521393f Binary files /dev/null and b/testdata/tiff/const.pnm differ diff --git a/testdata/tiff/const.tiff b/testdata/tiff/const.tiff new file mode 100644 index 0000000..fa01413 Binary files /dev/null and b/testdata/tiff/const.tiff differ diff --git a/testdata/tiff/constg.pgm b/testdata/tiff/constg.pgm new file mode 100644 index 0000000..aaca954 Binary files /dev/null and b/testdata/tiff/constg.pgm differ diff --git a/testdata/tiff/constg.tiff b/testdata/tiff/constg.tiff new file mode 100644 index 0000000..f138307 Binary files /dev/null and b/testdata/tiff/constg.tiff differ diff --git a/testsuites/suite.py b/testsuites/suite.py index 02b1237..db233cb 100644 --- a/testsuites/suite.py +++ b/testsuites/suite.py @@ -46,6 +46,7 @@ class Log: class VerdictErrno(enum.Enum): ERROR_SUCCESS = "success" + ERROR_SKIPPED = "skipped" ERROR_RETURNCODE = "program returns wrong returncode" ERROR_ASSERTION = "assertion" ERROR_TIMEOUT = "timeout expired" @@ -53,6 +54,9 @@ class VerdictErrno(enum.Enum): ERROR_STDERR_IS_NOT_EMPTY = "standard error output is not empty" ERROR_TYPE_ERROR = "type error" ERROR_INVALID_FORMAT = "invalid format" + ERROR_FILE_NOT_FOUND = "file not found" + ERROR_STDOUT_EMPTY = "standard output is empty" + ERROR_STDOUT_IS_NOT_EMPTY = "standard output is not empty" ERROR_VALGRIND_MEMCHECK = "valgrind error" ERROR_GDB_ERROR = "GDB error" @@ -72,6 +76,9 @@ class Verdict: def is_failed(self) -> bool: return not self.is_success() + def is_skipped(self) -> bool: + return self.__verdict_errno == VerdictErrno.ERROR_SKIPPED + def verdict_message(self) -> str: return self.__verdict_errno.value @@ -86,9 +93,15 @@ class Verdict: return "no additional information" return self.__what + def what_presented(self) -> bool: + return not self.__what is None + def ok() -> Verdict: return Verdict(VerdictErrno.ERROR_SUCCESS) +def skip() -> Verdict: + return Verdict(VerdictErrno.ERROR_SKIPPED) + class DynamicWrapper(enum.Enum): NO_WRAPPER = "no" VALGRIND_ANALYZER = "valgrind" @@ -136,7 +149,7 @@ class Run: GDB_LOG_FILENAME = "gdb.log" GDB_NO_ERROR_MARKER = "exited normally" - def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True): + def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True, t_skip_when_dynamic_wrapper: Optional[DynamicWrapper] = None): self.__c_timeout = float(c_timeout) self.__c_stdin = c_stdin self.__c_args = c_args @@ -144,6 +157,7 @@ class Run: self.__t_returncode = t_returncode self.__t_stdout = t_stdout self.__t_stderr_empty = t_stderr_empty + self.__t_skip_when_dynamic_wrapper = t_skip_when_dynamic_wrapper def get_timeout(self) -> float: return self.__c_timeout @@ -182,6 +196,11 @@ class Run: def is_stderr_should_be_empty(self) -> bool: return self.__t_stderr_empty + def should_skip(self, dynamic_wrapper: DynamicWrapper) -> bool: + if self.__t_skip_when_dynamic_wrapper is None: + return False + return self.__t_skip_when_dynamic_wrapper == dynamic_wrapper + def run(self, executable_path: str, timeout_factor: float, dynamic_wrapper: DynamicWrapper) -> Optional[Runned]: cmd: List[str] = [] @@ -310,14 +329,16 @@ class Test: if policy == ReturnCodePolicy.ShouldBeZero: if actual_returncode != 0: - return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}") + what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)" + return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}{what_suffix}", c_stderr) elif policy == ReturnCodePolicy.ShouldNotBeZero: if actual_returncode == 0: return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected non-zero returncode, but actual is {actual_returncode}") - elif policy == ReturnCodePolicy.MatchIfPresented and run.expected_returncode_presented(): + elif policy == ReturnCodePolicy.MatchIfPresented: expected_returncode = run.get_expected_returncode() if actual_returncode != expected_returncode: - return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}") + what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)" + return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}{what_suffix}", c_stderr) return ok() @@ -326,7 +347,8 @@ class Test: def __print_verdict(self, i: int, verdict: Verdict, log: Log): log.println(f"{self.__base_message(i)} FAILED.") - log.println(f"{verdict.verdict_message().capitalize()}: {verdict.what()}.") + suffix = f": {verdict.what()}" if verdict.what_presented() else f"" + log.println(f"{verdict.verdict_message().capitalize()}{suffix}.") lines = verdict.extended_what() if verdict.extended_what_is_hint(): assert len(lines) == 1 @@ -339,6 +361,10 @@ class Test: for i, (run, expected) in enumerate(self.__sequence): log.println(self.__base_message(i)) + if run.should_skip(dynamic_wrapper): + log.println(f"{self.__base_message(i)} SKIPPED: {dynamic_wrapper.value} is not supported for this run.") + return skip() + runned = run.run(executable_path, timeout_factor, dynamic_wrapper) if runned is None: @@ -374,18 +400,24 @@ class Result: self.__tests = tests self.__verdicts: List[Verdict] = [] self.__passed = 0 + self.__skipped = 0 def add(self, verdict: Verdict): if verdict.is_success(): self.__passed += 1 + if verdict.is_skipped(): + self.__skipped += 1 self.__verdicts.append(verdict) def n(self) -> int: - return len(self.__verdicts) + return len(self.__verdicts) - self.__skipped def passed(self) -> int: return self.__passed + def skipped(self) -> int: + return self.__skipped + def exitcode(self) -> int: return 0 if self.n() == self.passed() else 1 @@ -401,8 +433,8 @@ class Result: def __get_total_by_category(self, category: str) -> int: total = 0 - for test, _ in zip(self.__tests, self.__verdicts): - if category in test.categories(): + for test, verdict in zip(self.__tests, self.__verdicts): + if not verdict.is_skipped() and category in test.categories(): total += 1 return total @@ -505,7 +537,7 @@ class Tester: end = now() print("=" * 30) - print(f"{result.passed()}/{result.n()} tests passed in {end - start}ms") + print(f"{result.passed()}/{result.n()} ({result.skipped()} skipped) tests passed in {end - start}ms") return result diff --git a/testsuites/tiff.py b/testsuites/tiff.py new file mode 100644 index 0000000..79a053f --- /dev/null +++ b/testsuites/tiff.py @@ -0,0 +1,172 @@ +from .suite import Tester + +from .suite import Run, Runned, Verdict +from testsuites import * + +class __TIFF(Testsuite): + __SUITE_NAME = "tiff" + __TIMEOUT = 1 + __CATEGORIES_TO_ENVNAMES = { "rgb": "RGB", "gray": "GRAY", "neg": "NEG" } + __TESTDATA = os.path.join("testdata", __SUITE_NAME) + + def __init__(self): + super().__init__(self.__SUITE_NAME, PREFIX_ENVIRONMENT_NAME, self.__CATEGORIES_TO_ENVNAMES) + + def get_tester(self) -> Tester: + tester = Tester(self.name()) + + class __Expected(Expected): + __OFFSET_TO_SHOW = 16 + + def __init__(self, output_filename: str, reference_filename: str): + super().__init__() + + self.__output_filename = output_filename + self.__reference_filename = reference_filename + + def test(self, run: Run, runned: Runned) -> Verdict: + if runned.get_stdout() != "": + return Verdict(VerdictErrno.ERROR_STDOUT_IS_NOT_EMPTY, f"below is what was in the stdout", runned.get_stdout()) + + if not os.path.exists(self.__output_filename): + return Verdict(VerdictErrno.ERROR_FILE_NOT_FOUND, f"'{self.__output_filename}' is not created by user's program") + + output = open(self.__output_filename, "rb").read() + reference = open(self.__reference_filename, "rb").read() + + n_output = len(output) + n_reference = len(reference) + n = min(n_reference, n_output) + + verdict_errno = VerdictErrno.ERROR_SUCCESS + what: Optional[str] = None + what_extended: List[str] = [] + fail = False + + for i in range(n): + if output[i] == reference[i]: + continue + + verdict_errno = VerdictErrno.ERROR_ASSERTION + + what = "wrong converting result" + + what_extended.append(f"Comparing '{self.__output_filename}' and '{self.__reference_filename}'...") + if i != 0: + what_extended.append(f". . . . . .") + + k = min(i + self.__OFFSET_TO_SHOW, n) + for j in range(i, k): + what_extended.append("%08X: %02X %02X" % (j, output[j], reference[j])) + + if (k - i) == self.__OFFSET_TO_SHOW: + what_extended.append(f". . . . . .") + + fail = True + + break + + is_reference_bigger = n_reference > n_output + if n_output != n_reference: + if not fail: + verdict_errno = VerdictErrno.ERROR_ASSERTION + what = "wrong file size" + if is_reference_bigger: + what_extended.append(f"Reference file '{self.__reference_filename}' is bigger, than actual file '{self.__output_filename}'.") + else: + what_extended.append(f"Actual file '{self.__output_filename}' is bigger, than reference file '{self.__reference_filename}'.") + + return Verdict(verdict_errno, what, what_extended) + + def __make_path_to(filename: str) -> str: + return os.path.abspath(os.path.join(self.__TESTDATA, filename)) + + def __single_test(input_basename: str, reference_basename: Optional[str], returncode: int = ERROR_SUCCESS, output_filename_override: Optional[str] = None, skip_when_dynamic_wrapper: Optional[DynamicWrapper] = None) -> SingleTest: + input_filename = __make_path_to(input_basename) + output_filename = __make_path_to(f"output.user") if output_filename_override is None else output_filename_override + reference_filename = "" if reference_basename is None else __make_path_to(reference_basename) + neg = returncode != ERROR_SUCCESS + returncode_policy = ReturnCodePolicy.ShouldBeZero if not neg else ReturnCodePolicy.MatchIfPresented + stderr_empty = not neg + run = Run(c_timeout = self.__TIMEOUT, c_stdin = None, c_args = [input_filename, output_filename], t_returncode_policy = returncode_policy, t_returncode = returncode, t_stderr_empty = stderr_empty, t_skip_when_dynamic_wrapper = skip_when_dynamic_wrapper) + expected = None + if not neg: + expected = __Expected(output_filename, reference_filename) + return run, expected + + def __args_single_test(args: List[str]) -> SingleTest: + run = Run(c_timeout = self.__TIMEOUT, c_stdin = None, c_args = args, t_returncode_policy = ReturnCodePolicy.MatchIfPresented, t_returncode = ERROR_ARGUMENTS_INVALID, t_stderr_empty = False, t_skip_when_dynamic_wrapper = DynamicWrapper.GDB_DEBUGGER) + return run, None + + def __neg_single_test(input_basename: str, returncode: int, output_filename_override: Optional[str]) -> SingleTest: + return __single_test(input_basename, None, returncode, output_filename_override, DynamicWrapper.GDB_DEBUGGER) + + def __sequence(input_basename: str, reference_basename: str) -> List[SingleTest]: + return [__single_test(input_basename, reference_basename)] + + def __neg_sequence(input_basename: str, returncode: int, output_filename_override: Optional[str]) -> List[SingleTest]: + return [__neg_single_test(input_basename, returncode, output_filename_override)] + + def __arg_sequence(args: List[str]) -> List[SingleTest]: + return [__args_single_test(args)] + + def __test(name: str, category: str, input_basename: str, reference_basename: str) -> Test: + return Test(name, [category], __sequence(input_basename, reference_basename)) + + def __neg_test(name: str, input_basename: str, returncode: int, output_filename_override: Optional[str]) -> Test: + category = "neg" + return Test(name, [category], __neg_sequence(input_basename, returncode, output_filename_override)) + + def __arg_test(name: str, args: List[str]) -> Test: + category = "neg" + return Test(name, [category], __arg_sequence(args)) + + category_counter: Dict[str, int] = dict((k, 0) for k in self.__CATEGORIES_TO_ENVNAMES.keys()) + + def __testname(category: str, prefix: str, input_basename: str): + category_counter[category] += 1 + + test_basename = input_basename.removesuffix(".tiff") + test_basename = test_basename.removesuffix(".pnm") + test_basename = test_basename.removeprefix(prefix) + test_basename = test_basename.removeprefix("_") + + name_prefix = f"Picture [format = {category}] #{category_counter[category]}" + name_suffix = f"" if test_basename == "" else f": {" ".join(test_basename.split("_")).capitalize()}" + + return f"{name_prefix}{name_suffix}" + + def t1(category: str, prefix: str, input_basename: str, reference_basename: str): + tester.add(__test(__testname(category, prefix, input_basename), category, input_basename, reference_basename)) + + def t(category: str, prefix: str, exception_basename: Set[str] = set(), exception_prefixes: Set[str] = set()): + reference_basename = f"{prefix}.{"pnm" if category == "rgb" else "pgm"}" + + for input_basename in os.listdir(self.__TESTDATA): + if not input_basename.startswith(prefix) or any(input_basename.startswith(p) for p in exception_prefixes): + continue + + if input_basename == reference_basename or input_basename in exception_basename: + continue + + t1(category, prefix, input_basename, reference_basename) + + def n(name: str, input_basename: str, returncode: int, output_filename_override: Optional[str] = None): + tester.add(__neg_test(name, input_basename, returncode, output_filename_override)) + + def na(name: str, args: List[str]): + tester.add(__arg_test(name, args)) + + t("rgb", "const", exception_prefixes = {"constg"}) + t("gray", "constg") + + n("No input file found", "this_file_does_not_exists.tiff", ERROR_CANNOT_OPEN_FILE) + n("Can't create output file", "const_tiff.pnm", ERROR_CANNOT_OPEN_FILE, os.path.abspath(os.path.join(self.__TESTDATA, "this_directory_does_not_exists", "output.pnm"))) + + na("No args", []) + na("1 arg provided", ["argument 1"]) + na("Too much arguments provided", ["argument 1", "argument 2", "argument 3"]) + + return tester + +instance = __TIFF()