ai360-cpp/public-tests-2026
5
0
Fork 0
Code Activity

+ TIFF test suite.

Browse Source
This commit is contained in:
SAVELIY BAKTURIN
2026-04-09 08:34:31 +00:00
parent 0e46af6db8
commit f7b18385e0
9 changed files with 221 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
testsuites/suite.py
View File

@@ -46,6 +46,7 @@ class Log:
class VerdictErrno(enum.Enum):
ERROR_SUCCESS = "success"
ERROR_SKIPPED = "skipped"
ERROR_RETURNCODE = "program returns wrong returncode"
ERROR_ASSERTION = "assertion"
ERROR_TIMEOUT = "timeout expired"
@@ -53,6 +54,9 @@ class VerdictErrno(enum.Enum):
ERROR_STDERR_IS_NOT_EMPTY = "standard error output is not empty"
ERROR_TYPE_ERROR = "type error"
ERROR_INVALID_FORMAT = "invalid format"
ERROR_FILE_NOT_FOUND = "file not found"
ERROR_STDOUT_EMPTY = "standard output is empty"
ERROR_STDOUT_IS_NOT_EMPTY = "standard output is not empty"
ERROR_VALGRIND_MEMCHECK = "valgrind error"
ERROR_GDB_ERROR = "GDB error"
@@ -72,6 +76,9 @@ class Verdict:
def is_failed(self) -> bool:
return not self.is_success()
def is_skipped(self) -> bool:
return self.__verdict_errno == VerdictErrno.ERROR_SKIPPED
def verdict_message(self) -> str:
return self.__verdict_errno.value
@@ -86,9 +93,15 @@ class Verdict:
return "no additional information"
return self.__what
def what_presented(self) -> bool:
return not self.__what is None
def ok() -> Verdict:
return Verdict(VerdictErrno.ERROR_SUCCESS)
def skip() -> Verdict:
return Verdict(VerdictErrno.ERROR_SKIPPED)
class DynamicWrapper(enum.Enum):
NO_WRAPPER = "no"
VALGRIND_ANALYZER = "valgrind"
@@ -136,7 +149,7 @@ class Run:
GDB_LOG_FILENAME = "gdb.log"
GDB_NO_ERROR_MARKER = "exited normally"
def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True):
def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True, t_skip_when_dynamic_wrapper: Optional[DynamicWrapper] = None):
self.__c_timeout = float(c_timeout)
self.__c_stdin = c_stdin
self.__c_args = c_args
@@ -144,6 +157,7 @@ class Run:
self.__t_returncode = t_returncode
self.__t_stdout = t_stdout
self.__t_stderr_empty = t_stderr_empty
self.__t_skip_when_dynamic_wrapper = t_skip_when_dynamic_wrapper
def get_timeout(self) -> float:
return self.__c_timeout
@@ -182,6 +196,11 @@ class Run:
def is_stderr_should_be_empty(self) -> bool:
return self.__t_stderr_empty
def should_skip(self, dynamic_wrapper: DynamicWrapper) -> bool:
if self.__t_skip_when_dynamic_wrapper is None:
return False
return self.__t_skip_when_dynamic_wrapper == dynamic_wrapper
def run(self, executable_path: str, timeout_factor: float, dynamic_wrapper: DynamicWrapper) -> Optional[Runned]:
cmd: List[str] = []
@@ -310,14 +329,16 @@ class Test:
if policy == ReturnCodePolicy.ShouldBeZero:
if actual_returncode != 0:
return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}")
what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)"
return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}{what_suffix}", c_stderr)
elif policy == ReturnCodePolicy.ShouldNotBeZero:
if actual_returncode == 0:
return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected non-zero returncode, but actual is {actual_returncode}")
elif policy == ReturnCodePolicy.MatchIfPresented and run.expected_returncode_presented():
elif policy == ReturnCodePolicy.MatchIfPresented:
expected_returncode = run.get_expected_returncode()
if actual_returncode != expected_returncode:
return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}")
what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)"
return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}{what_suffix}", c_stderr)
return ok()
@@ -326,7 +347,8 @@ class Test:
def __print_verdict(self, i: int, verdict: Verdict, log: Log):
log.println(f"{self.__base_message(i)} FAILED.")
log.println(f"{verdict.verdict_message().capitalize()}: {verdict.what()}.")
suffix = f": {verdict.what()}" if verdict.what_presented() else f""
log.println(f"{verdict.verdict_message().capitalize()}{suffix}.")
lines = verdict.extended_what()
if verdict.extended_what_is_hint():
assert len(lines) == 1
@@ -339,6 +361,10 @@ class Test:
for i, (run, expected) in enumerate(self.__sequence):
log.println(self.__base_message(i))
if run.should_skip(dynamic_wrapper):
log.println(f"{self.__base_message(i)} SKIPPED: {dynamic_wrapper.value} is not supported for this run.")
return skip()
runned = run.run(executable_path, timeout_factor, dynamic_wrapper)
if runned is None:
@@ -374,18 +400,24 @@ class Result:
self.__tests = tests
self.__verdicts: List[Verdict] = []
self.__passed = 0
self.__skipped = 0
def add(self, verdict: Verdict):
if verdict.is_success():
self.__passed += 1
if verdict.is_skipped():
self.__skipped += 1
self.__verdicts.append(verdict)
def n(self) -> int:
return len(self.__verdicts)
return len(self.__verdicts) - self.__skipped
def passed(self) -> int:
return self.__passed
def skipped(self) -> int:
return self.__skipped
def exitcode(self) -> int:
return 0 if self.n() == self.passed() else 1
@@ -401,8 +433,8 @@ class Result:
def __get_total_by_category(self, category: str) -> int:
total = 0
for test, _ in zip(self.__tests, self.__verdicts):
if category in test.categories():
for test, verdict in zip(self.__tests, self.__verdicts):
if not verdict.is_skipped() and category in test.categories():
total += 1
return total
@@ -505,7 +537,7 @@ class Tester:
end = now()
print("=" * 30)
print(f"{result.passed()}/{result.n()} tests passed in {end - start}ms")
print(f"{result.passed()}/{result.n()} ({result.skipped()} skipped) tests passed in {end - start}ms")
return result