diff --git a/testr.py b/testr.py index 695342f..4cec847 100644 --- a/testr.py +++ b/testr.py @@ -1,6 +1,7 @@ import asyncio from pathlib import Path from testr.file_data import DirectorySuite, ExecutableRunner +from testr.runner import TestOptions import argparse parser = argparse.ArgumentParser() @@ -11,14 +12,23 @@ test_data_group = parser.add_mutually_exclusive_group(required=True) test_data_group.add_argument("--testdir", help="Directory for test cases") test_runner_group.add_argument("--exec", help="Executable to run test cases against") +parser.add_argument("--timelim", help="Time limit in seconds", type=float, default=5.0) + args = parser.parse_args() + async def main(): test_suite = DirectorySuite(Path(args.testdir)) test_runner = ExecutableRunner(Path(args.exec)) - async for test_case in test_runner.run_test_suite(test_suite): + async for test_case in test_runner.run_test_suite( + test_suite, + TestOptions( + time_limit=args.timelim + ) + ): print(test_case.code) + if __name__ == "__main__": asyncio.run(main()) diff --git a/testr/file_data.py b/testr/file_data.py index a1e32d0..1f62ece 100644 --- a/testr/file_data.py +++ b/testr/file_data.py @@ -1,5 +1,5 @@ import asyncio -from testr.runner import TestData, TestRunner, TestStatus, TestSuite, StatusCode +from testr.runner import TestData, TestOptions, TestRunner, TestStatus, TestSuite, StatusCode from pathlib import Path @@ -30,7 +30,7 @@ class ExecutableRunner(TestRunner): raise ValueError(f"executable must be a file, got '{executable}'") self.executable = executable - async def run_test(self, data: TestData) -> TestStatus: + async def run_test(self, data: TestData, opts: TestOptions) -> TestStatus: proc = await asyncio.create_subprocess_shell( str(self.executable), stdout=asyncio.subprocess.PIPE, @@ -42,7 +42,7 @@ class ExecutableRunner(TestRunner): try: out_stream, err_stream = await asyncio.wait_for( - proc.communicate(input=input_data.encode()), timeout=5.0) + proc.communicate(input=input_data.encode()), timeout=opts.time_limit) except TimeoutError: proc.kill() return TestStatus(code=StatusCode.TLE, stderr="", stdout="", stdin=input_data) diff --git a/testr/runner.py b/testr/runner.py index dff6dda..cbc4cb5 100644 --- a/testr/runner.py +++ b/testr/runner.py @@ -53,6 +53,21 @@ class TestValidator(ABC): async def validate_output(self, output: str) -> bool: pass +@dataclass +class TestOptions: + """ + Options for running a test suite. + + Attributes + ---------- + + time_limit + Time limit in seconds for each test case. + """ + + time_limit: float + + class TestData(TestInput, TestValidator): """Combined input/output for single test case""" @@ -70,8 +85,8 @@ class TestRunner(ABC): """Runner for test cases.""" @abstractmethod - async def run_test(self, data: TestData) -> TestStatus: pass + async def run_test(self, data: TestData, opts: TestOptions) -> TestStatus: pass - async def run_test_suite(self, data: TestSuite) -> AsyncIterator[TestStatus]: + async def run_test_suite(self, data: TestSuite, opts: TestOptions) -> AsyncIterator[TestStatus]: for test_case in data: - yield await self.run_test(test_case) + yield await self.run_test(test_case, opts)