+ TIFF test suite.

.gitignore

@@ -3,8 +3,10 @@
 *.exe
 *.json
 *.out
-*.log
-*.sh
+*.txt
+
+# User created files.
+*.user
 
 # Python cache.
 __pycache__/

README.md

@@ -18,7 +18,7 @@
     python main.py --executable-path <путь к исполняемому файлу> --suite <набор тестов>
     ```
 
-    *Примечание*. В *nix-подобных системах команда `python` может быть недоступна, вместо неё используйте команду `python3`. 
+    *Примечание*. В *nix-подобных системах команда `python` может быть недоступна, вместо неё используйте команду `python3`.
 
 5. Для ускорения отладки рекомендуется создать скрипт, выполняющий шаги 3-4.
 6. Для обновления тестов выполнить команду:
@@ -35,6 +35,7 @@
 |:--:|:--------------------|:-----------------|
 | 0  | Интро               | `intro`          |
 | 1  | Представление чисел | `fixed_floating` |
+| 2  | TIFF                | `tiff`           |
 
 ## Скрипт запуска
 

main.py

@@ -5,12 +5,13 @@ import platform
 
 import testsuites.intro as intro
 import testsuites.fixed_floating as fixed_floating
+import testsuites.tiff as tiff
 
 from testsuites import Testsuite, DynamicWrapper
 from typing import Dict, Any, List
 
 __TESTSUITES: List[Testsuite] = [
-	intro.instance, fixed_floating.instance
+	intro.instance, fixed_floating.instance, tiff.instance
 ]
 
 __SUITENAMES: List[str] = [t.name() for t in __TESTSUITES]

testsuites/suite.py

@@ -46,6 +46,7 @@ class Log:
 
 class VerdictErrno(enum.Enum):
 	ERROR_SUCCESS = "success"
+	ERROR_SKIPPED = "skipped"
 	ERROR_RETURNCODE = "program returns wrong returncode"
 	ERROR_ASSERTION = "assertion"
 	ERROR_TIMEOUT = "timeout expired"
@@ -53,6 +54,9 @@ class VerdictErrno(enum.Enum):
 	ERROR_STDERR_IS_NOT_EMPTY = "standard error output is not empty"
 	ERROR_TYPE_ERROR = "type error"
 	ERROR_INVALID_FORMAT = "invalid format"
+	ERROR_FILE_NOT_FOUND = "file not found"
+	ERROR_STDOUT_EMPTY = "standard output is empty"
+	ERROR_STDOUT_IS_NOT_EMPTY = "standard output is not empty"
 	ERROR_VALGRIND_MEMCHECK = "valgrind error"
 	ERROR_GDB_ERROR = "GDB error"
 
@@ -72,6 +76,9 @@ class Verdict:
 	def is_failed(self) -> bool:
 		return not self.is_success()
 
+	def is_skipped(self) -> bool:
+		return self.__verdict_errno == VerdictErrno.ERROR_SKIPPED
+
 	def verdict_message(self) -> str:
 		return self.__verdict_errno.value
 
@@ -86,9 +93,15 @@ class Verdict:
 			return "no additional information"
 		return self.__what
 
+	def what_presented(self) -> bool:
+		return not self.__what is None
+
 def ok() -> Verdict:
 	return Verdict(VerdictErrno.ERROR_SUCCESS)
 
+def skip() -> Verdict:
+	return Verdict(VerdictErrno.ERROR_SKIPPED)
+
 class DynamicWrapper(enum.Enum):
 	NO_WRAPPER = "no"
 	VALGRIND_ANALYZER = "valgrind"
@@ -136,7 +149,7 @@ class Run:
 	GDB_LOG_FILENAME = "gdb.log"
 	GDB_NO_ERROR_MARKER = "exited normally"
 
-	def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True):
+	def __init__(self, c_timeout: Union[float, int], c_stdin: Optional[str], c_args: Optional[List[str]], t_returncode_policy: ReturnCodePolicy, t_returncode: Optional[int] = None, t_stdout: Optional[str] = None, t_stderr_empty: bool = True, t_skip_when_dynamic_wrapper: Optional[DynamicWrapper] = None):
 		self.__c_timeout = float(c_timeout)
 		self.__c_stdin = c_stdin
 		self.__c_args = c_args
@@ -144,6 +157,7 @@ class Run:
 		self.__t_returncode = t_returncode
 		self.__t_stdout = t_stdout
 		self.__t_stderr_empty = t_stderr_empty
+		self.__t_skip_when_dynamic_wrapper = t_skip_when_dynamic_wrapper
 
 	def get_timeout(self) -> float:
 		return self.__c_timeout
@@ -182,6 +196,11 @@ class Run:
 	def is_stderr_should_be_empty(self) -> bool:
 		return self.__t_stderr_empty
 
+	def should_skip(self, dynamic_wrapper: DynamicWrapper) -> bool:
+		if self.__t_skip_when_dynamic_wrapper is None:
+			return False
+		return self.__t_skip_when_dynamic_wrapper == dynamic_wrapper
+
 	def run(self, executable_path: str, timeout_factor: float, dynamic_wrapper: DynamicWrapper) -> Optional[Runned]:
 		cmd: List[str] = []
 
@@ -310,14 +329,16 @@ class Test:
 
 		if policy == ReturnCodePolicy.ShouldBeZero:
 			if actual_returncode != 0:
-				return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}")
+				what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)"
+				return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {0}, but actual is {actual_returncode}{what_suffix}", c_stderr)
 		elif policy == ReturnCodePolicy.ShouldNotBeZero:
 			if actual_returncode == 0:
 				return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected non-zero returncode, but actual is {actual_returncode}")
-		elif policy == ReturnCodePolicy.MatchIfPresented and run.expected_returncode_presented():
+		elif policy == ReturnCodePolicy.MatchIfPresented:
 			expected_returncode = run.get_expected_returncode()
 			if actual_returncode != expected_returncode:
-				return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}")
+				what_suffix = f"" if is_empty_stderr else f" (below is what was in the stderr)"
+				return Verdict(VerdictErrno.ERROR_RETURNCODE, f"expected {expected_returncode}, but actual is {actual_returncode}{what_suffix}", c_stderr)
 
 		return ok()
 
@@ -326,7 +347,8 @@ class Test:
 
 	def __print_verdict(self, i: int, verdict: Verdict, log: Log):
 		log.println(f"{self.__base_message(i)} FAILED.")
-		log.println(f"{verdict.verdict_message().capitalize()}: {verdict.what()}.")
+		suffix = f": {verdict.what()}" if verdict.what_presented() else f""
+		log.println(f"{verdict.verdict_message().capitalize()}{suffix}.")
 		lines = verdict.extended_what()
 		if verdict.extended_what_is_hint():
 			assert len(lines) == 1
@@ -339,6 +361,10 @@ class Test:
 		for i, (run, expected) in enumerate(self.__sequence):
 			log.println(self.__base_message(i))
 
+			if run.should_skip(dynamic_wrapper):
+				log.println(f"{self.__base_message(i)} SKIPPED: {dynamic_wrapper.value} is not supported for this run.")
+				return skip()
+
 			runned = run.run(executable_path, timeout_factor, dynamic_wrapper)
 
 			if runned is None:
@@ -374,18 +400,24 @@ class Result:
 		self.__tests = tests
 		self.__verdicts: List[Verdict] = []
 		self.__passed = 0
+		self.__skipped = 0
 
 	def add(self, verdict: Verdict):
 		if verdict.is_success():
 			self.__passed += 1
+		if verdict.is_skipped():
+			self.__skipped += 1
 		self.__verdicts.append(verdict)
 
 	def n(self) -> int:
-		return len(self.__verdicts)
+		return len(self.__verdicts) - self.__skipped
 
 	def passed(self) -> int:
 		return self.__passed
 
+	def skipped(self) -> int:
+		return self.__skipped
+
 	def exitcode(self) -> int:
 		return 0 if self.n() == self.passed() else 1
 
@@ -401,8 +433,8 @@ class Result:
 	def __get_total_by_category(self, category: str) -> int:
 		total = 0
 
-		for test, _ in zip(self.__tests, self.__verdicts):
-			if category in test.categories():
+		for test, verdict in zip(self.__tests, self.__verdicts):
+			if not verdict.is_skipped() and category in test.categories():
 				total += 1
 
 		return total
@@ -505,7 +537,7 @@ class Tester:
 		end = now()
 
 		print("=" * 30)
-		print(f"{result.passed()}/{result.n()} tests passed in {end - start}ms")
+		print(f"{result.passed()}/{result.n()} ({result.skipped()} skipped) tests passed in {end - start}ms")
 
 		return result
 

testsuites/tiff.py

@@ -0,0 +1,172 @@
+from .suite import Tester
+
+from .suite import Run, Runned, Verdict
+from testsuites import *
+
+class __TIFF(Testsuite):
+	__SUITE_NAME = "tiff"
+	__TIMEOUT = 1
+	__CATEGORIES_TO_ENVNAMES = { "rgb": "RGB", "gray": "GRAY", "neg": "NEG" }
+	__TESTDATA = os.path.join("testdata", __SUITE_NAME)
+
+	def __init__(self):
+		super().__init__(self.__SUITE_NAME, PREFIX_ENVIRONMENT_NAME, self.__CATEGORIES_TO_ENVNAMES)
+
+	def get_tester(self) -> Tester:
+		tester = Tester(self.name())
+
+		class __Expected(Expected):
+			__OFFSET_TO_SHOW = 16
+
+			def __init__(self, output_filename: str, reference_filename: str):
+				super().__init__()
+
+				self.__output_filename = output_filename
+				self.__reference_filename = reference_filename
+
+			def test(self, run: Run, runned: Runned) -> Verdict:
+				if runned.get_stdout() != "":
+					return Verdict(VerdictErrno.ERROR_STDOUT_IS_NOT_EMPTY, f"below is what was in the stdout", runned.get_stdout())
+
+				if not os.path.exists(self.__output_filename):
+					return Verdict(VerdictErrno.ERROR_FILE_NOT_FOUND, f"'{self.__output_filename}' is not created by user's program")
+
+				output = open(self.__output_filename, "rb").read()
+				reference = open(self.__reference_filename, "rb").read()
+
+				n_output = len(output)
+				n_reference = len(reference)
+				n = min(n_reference, n_output)
+
+				verdict_errno = VerdictErrno.ERROR_SUCCESS
+				what: Optional[str] = None
+				what_extended: List[str] = []
+				fail = False
+
+				for i in range(n):
+					if output[i] == reference[i]:
+						continue
+
+					verdict_errno = VerdictErrno.ERROR_ASSERTION
+
+					what = "wrong converting result"
+
+					what_extended.append(f"Comparing '{self.__output_filename}' and '{self.__reference_filename}'...")
+					if i != 0:
+						what_extended.append(f". . . <truncated {i} same bytes> . . .")
+
+					k = min(i + self.__OFFSET_TO_SHOW, n)
+					for j in range(i, k):
+						what_extended.append("%08X: %02X %02X" % (j, output[j], reference[j]))
+
+					if (k - i) == self.__OFFSET_TO_SHOW:
+						what_extended.append(f". . . <truncated output> . . .")
+
+					fail = True
+
+					break
+
+				is_reference_bigger = n_reference > n_output
+				if n_output != n_reference:
+					if not fail:
+						verdict_errno = VerdictErrno.ERROR_ASSERTION
+						what = "wrong file size"
+					if is_reference_bigger:
+						what_extended.append(f"Reference file '{self.__reference_filename}' is bigger, than actual file '{self.__output_filename}'.")
+					else:
+						what_extended.append(f"Actual file '{self.__output_filename}' is bigger, than reference file '{self.__reference_filename}'.")
+
+				return Verdict(verdict_errno, what, what_extended)
+
+		def __make_path_to(filename: str) -> str:
+			return os.path.abspath(os.path.join(self.__TESTDATA, filename))
+
+		def __single_test(input_basename: str, reference_basename: Optional[str], returncode: int = ERROR_SUCCESS, output_filename_override: Optional[str] = None, skip_when_dynamic_wrapper: Optional[DynamicWrapper] = None) -> SingleTest:
+			input_filename = __make_path_to(input_basename)
+			output_filename = __make_path_to(f"output.user") if output_filename_override is None else output_filename_override
+			reference_filename = "" if reference_basename is None else __make_path_to(reference_basename)
+			neg = returncode != ERROR_SUCCESS
+			returncode_policy = ReturnCodePolicy.ShouldBeZero if not neg else ReturnCodePolicy.MatchIfPresented
+			stderr_empty = not neg
+			run = Run(c_timeout = self.__TIMEOUT, c_stdin = None, c_args = [input_filename, output_filename], t_returncode_policy = returncode_policy, t_returncode = returncode, t_stderr_empty = stderr_empty, t_skip_when_dynamic_wrapper = skip_when_dynamic_wrapper)
+			expected = None
+			if not neg:
+				expected = __Expected(output_filename, reference_filename)
+			return run, expected
+
+		def __args_single_test(args: List[str]) -> SingleTest:
+			run = Run(c_timeout = self.__TIMEOUT, c_stdin = None, c_args = args, t_returncode_policy = ReturnCodePolicy.MatchIfPresented, t_returncode = ERROR_ARGUMENTS_INVALID, t_stderr_empty = False, t_skip_when_dynamic_wrapper = DynamicWrapper.GDB_DEBUGGER)
+			return run, None
+
+		def __neg_single_test(input_basename: str, returncode: int, output_filename_override: Optional[str]) -> SingleTest:
+			return __single_test(input_basename, None, returncode, output_filename_override, DynamicWrapper.GDB_DEBUGGER)
+
+		def __sequence(input_basename: str, reference_basename: str) -> List[SingleTest]:
+			return [__single_test(input_basename, reference_basename)]
+
+		def __neg_sequence(input_basename: str, returncode: int, output_filename_override: Optional[str]) -> List[SingleTest]:
+			return [__neg_single_test(input_basename, returncode, output_filename_override)]
+
+		def __arg_sequence(args: List[str]) -> List[SingleTest]:
+			return [__args_single_test(args)]
+
+		def __test(name: str, category: str, input_basename: str, reference_basename: str) -> Test:
+			return Test(name, [category], __sequence(input_basename, reference_basename))
+
+		def __neg_test(name: str, input_basename: str, returncode: int, output_filename_override: Optional[str]) -> Test:
+			category = "neg"
+			return Test(name, [category], __neg_sequence(input_basename, returncode, output_filename_override))
+
+		def __arg_test(name: str, args: List[str]) -> Test:
+			category = "neg"
+			return Test(name, [category], __arg_sequence(args))
+
+		category_counter: Dict[str, int] = dict((k, 0) for k in self.__CATEGORIES_TO_ENVNAMES.keys())
+
+		def __testname(category: str, prefix: str, input_basename: str):
+			category_counter[category] += 1
+
+			test_basename = input_basename.removesuffix(".tiff")
+			test_basename = test_basename.removesuffix(".pnm")
+			test_basename = test_basename.removeprefix(prefix)
+			test_basename = test_basename.removeprefix("_")
+
+			name_prefix = f"Picture [format = {category}] #{category_counter[category]}"
+			name_suffix = f"" if test_basename == "" else f": {" ".join(test_basename.split("_")).capitalize()}"
+
+			return f"{name_prefix}{name_suffix}"
+
+		def t1(category: str, prefix: str, input_basename: str, reference_basename: str):
+			tester.add(__test(__testname(category, prefix, input_basename), category, input_basename, reference_basename))
+
+		def t(category: str, prefix: str, exception_basename: Set[str] = set(), exception_prefixes: Set[str] = set()):
+			reference_basename = f"{prefix}.{"pnm" if category == "rgb" else "pgm"}"
+
+			for input_basename in os.listdir(self.__TESTDATA):
+				if not input_basename.startswith(prefix) or any(input_basename.startswith(p) for p in exception_prefixes):
+					continue
+
+				if input_basename == reference_basename or input_basename in exception_basename:
+					continue
+
+				t1(category, prefix, input_basename, reference_basename)
+
+		def n(name: str, input_basename: str, returncode: int, output_filename_override: Optional[str] = None):
+			tester.add(__neg_test(name, input_basename, returncode, output_filename_override))
+
+		def na(name: str, args: List[str]):
+			tester.add(__arg_test(name, args))
+
+		t("rgb", "const", exception_prefixes = {"constg"})
+		t("gray", "constg")
+
+		n("No input file found", "this_file_does_not_exists.tiff", ERROR_CANNOT_OPEN_FILE)
+		n("Can't create output file", "const_tiff.pnm", ERROR_CANNOT_OPEN_FILE, os.path.abspath(os.path.join(self.__TESTDATA, "this_directory_does_not_exists", "output.pnm")))
+
+		na("No args", [])
+		na("1 arg provided", ["argument 1"])
+		na("Too much arguments provided", ["argument 1", "argument 2", "argument 3"])
+
+		return tester
+
+instance = __TIFF()