test: Convert the ethdebug schema conformity tests to unittest

.circleci/config.yml

@@ -948,7 +948,7 @@ jobs:
     <<: *base_ubuntu2404_small
     steps:
       - install_python3:
-          packages: pyyaml jsonschema pytest
+          packages: pyyaml jsonschema
       - checkout
       - run: pylint --version
       - run:
@@ -1639,15 +1639,15 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - install_python3:
-            packages: pyyaml jsonschema pytest
+            packages: pyyaml jsonschema
       - run:
           name: Initialize ethdebug/format schemas
           command: |
             git submodule update --init test/ethdebugSchemaTests/ethdebug-format
       - run:
           name: ETHDebug schema validation
           command: |
-            pytest test/ethdebugSchemaTests --solc-binary-path=/tmp/workspace/solc/solc-static-linux -v
+            python3 test/ethdebugSchemaTests.py --solc-binary-path /tmp/workspace/solc/solc-static-linux
 
   c_ext_benchmarks:
     <<: *base_node_small

test/ethdebugSchemaTests.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+# Runs the tests validating the compiler's ethdebug output against the ethdebug/format schemas.
+# Any test suite located in test/ethdebugSchemaTests/ will be executed automatically by this script.
+
+import sys
+from argparse import ArgumentParser
+from dataclasses import dataclass
+from pathlib import Path
+from unittest import TestLoader, TestSuite, TextTestRunner
+
+TEST_DIR = Path(__file__).parent / "ethdebugSchemaTests"
+
+
+@dataclass(frozen=True)
+class Config:
+    solc_path: Path
+
+
+def test_cases(test_suite):
+    for test in test_suite:
+        if isinstance(test, TestSuite):
+            yield from test_cases(test)
+        else:
+            yield test
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description="Validate the compiler's ethdebug output against the ethdebug/format schemas.")
+    parser.add_argument("--solc-binary-path", type=Path, required=True, help="Path to the solidity compiler binary.")
+    parser.add_argument(
+        "-k",
+        dest="patterns",
+        action="append",
+        default=[],
+        help="Only run the tests whose name matches the pattern, as with `python -m unittest -k`. Can be repeated.",
+    )
+    options = parser.parse_args()
+    assert options.solc_binary_path.is_file(), f"Not a file: {options.solc_binary_path}"
+
+    config = Config(solc_path=options.solc_binary_path)
+
+    # This is equivalent to `python -m unittest discover --start-directory $TEST_DIR -k ...`,
+    # with every test case class given the configuration.
+    test_loader = TestLoader()
+    assert test_loader.testNamePatterns is None
+    if len(options.patterns) > 0:
+        # Unlike `-k`, testNamePatterns matches a pattern without wildcards against the whole
+        # test name, so wrap such a pattern in wildcards to match it as a substring like `-k` does.
+        test_loader.testNamePatterns = [
+            pattern if "*" in pattern else f"*{pattern}*"
+            for pattern in options.patterns
+        ]
+    test_suite = test_loader.discover(start_dir=TEST_DIR)
+    for test_case in test_cases(test_suite):
+        type(test_case).config = config
+    result = TextTestRunner().run(test_suite)
+
+    if not result.wasSuccessful():
+        sys.exit(1)

test/ethdebugSchemaTests/README.md

@@ -11,11 +11,17 @@ git submodule update --init test/ethdebugSchemaTests/ethdebug-format
 Run the tests with:
 
 ```bash
-pytest test/ethdebugSchemaTests --solc-binary-path=build/solc/solc -v
+test/ethdebugSchemaTests.py --solc-binary-path build/solc/solc
 ```
 
-The JSON inputs use `contentFile` entries to keep Solidity examples in regular
-`.sol` fixture files under `sources/`. The pytest harness expands those entries
-to Standard JSON `content` before invoking `solc`.
+The runner discovers every test module in this directory and passes the compiler path to the test cases.
+To run only some tests, use the `-k` option to specify a pattern, as with [`python -m unittest -k`](https://docs.python.org/3/library/unittest.html#cmdoption-unittest-k):
+
+```bash
+test/ethdebugSchemaTests.py --solc-binary-path build/solc/solc -k test_deployed_program_schema
+```
+
+The JSON inputs use `contentFile` entries to keep Solidity examples in regular `.sol` fixture files under `sources/`.
+The tests expand those entries to Standard JSON `content` before invoking `solc`.
 
 To update the schema version, bump the submodule commit and rerun this suite.

test/ethdebugSchemaTests/conftest.py → test/ethdebugSchemaTests/schema_helpers.py

@@ -1,24 +1,10 @@
 from pathlib import Path
 
-import pytest
 import referencing
 import yaml
 from referencing.jsonschema import DRAFT202012
 
 
-def pytest_addoption(parser):
-    parser.addoption("--solc-binary-path", type=Path, required=True, help="Path to the solidity compiler binary.")
-
-
-@pytest.fixture
-def solc_path(request):
-    solc_path = request.config.getoption("--solc-binary-path")
-    assert solc_path.is_file()
-    assert solc_path.exists()
-    return solc_path
-
-
-@pytest.fixture(scope="module")
 def ethdebug_schema_dir():
     schema_dir = Path(__file__).parent / "ethdebug-format" / "schemas"
     assert schema_dir.is_dir(), (
@@ -28,7 +14,6 @@ def ethdebug_schema_dir():
     return schema_dir
 
 
-@pytest.fixture(scope="module")
 def ethdebug_schema_repository(ethdebug_schema_dir):
     registry = referencing.Registry()
     for path in ethdebug_schema_dir.rglob("*.yaml"):

test/ethdebugSchemaTests/test_ethdebug_schema_conformity.py

@@ -2,10 +2,16 @@
 
 import json
 import subprocess
+import unittest
+from functools import cache
 from pathlib import Path
 
 import jsonschema
-import pytest
+
+# NOTE: test/ethdebugSchemaTests.py runs the tests with this directory in the import path.
+# pragma pylint: disable=import-error
+import schema_helpers
+# pragma pylint: enable=import-error
 
 
 def get_nested_value(dictionary, *keys):
@@ -40,14 +46,7 @@ def load_standard_json_input(path):
     return standard_json_input
 
 
-@pytest.fixture(params=["input_file.json"])
-def standard_json_input(request):
-    testfile_dir = Path(__file__).parent
-    return load_standard_json_input(testfile_dir / request.param)
-
-
-@pytest.fixture
-def solc_output(standard_json_input, solc_path):
+def compile_standard_json(solc_path, standard_json_input):
     process = subprocess.run(
         [solc_path, "--standard-json"],
         input=json.dumps(standard_json_input),
@@ -59,76 +58,95 @@ def solc_output(standard_json_input, solc_path):
     return json.loads(process.stdout)
 
 
-@pytest.mark.parametrize("output_selection", ["evm.bytecode.ethdebug", "evm.deployedBytecode.ethdebug"], ids=str)
-def test_program_schema(
-    output_selection,
-    ethdebug_schema_repository,
-    solc_output
-):
-    program_validator = validator("schema:ethdebug/format/program", ethdebug_schema_repository)
-    for _, _, ethdebug_data in ethdebug_programs(solc_output, output_selection):
-        program_validator.validate(ethdebug_data)
-
-
-def test_resources_schema(ethdebug_schema_repository, solc_output):
-    resources_validator = validator("schema:ethdebug/format/info/resources", ethdebug_schema_repository)
-    resources_validator.validate(solc_output["ethdebug"]["resources"])
-
-
-def test_compilation_schema(ethdebug_schema_repository, solc_output):
-    compilation_validator = validator("schema:ethdebug/format/materials/compilation", ethdebug_schema_repository)
-    compilation_validator.validate(solc_output["ethdebug"]["compilation"])
-
-
-@pytest.mark.parametrize(
-    ("output_selection", "environment"),
-    [
-        ("evm.bytecode.ethdebug", "create"),
-        ("evm.deployedBytecode.ethdebug", "call"),
-    ],
-    ids=str
-)
-def test_program_sanity(output_selection, environment, solc_output):
-    source_ids = {source_name: source["id"] for source_name, source in solc_output["sources"].items()}
-
-    for source_name, contract_name, ethdebug_data in ethdebug_programs(solc_output, output_selection):
-        assert ethdebug_data["environment"] == environment
-        assert ethdebug_data["contract"]["name"] == contract_name
-        assert ethdebug_data["contract"]["definition"]["source"]["id"] == source_ids[source_name]
-
-        instructions = ethdebug_data["instructions"]
-        assert len(instructions) > 0
-        assert [instruction["offset"] for instruction in instructions] == sorted(
-            instruction["offset"] for instruction in instructions
-        )
-        assert all(instruction["operation"]["mnemonic"] for instruction in instructions)
-
-
-def test_resources_match_standard_json_sources(solc_output):
-    standard_json_sources = {source_name: source["id"] for source_name, source in solc_output["sources"].items()}
-    ethdebug_sources = {
-        source["path"]: source["id"]
-        for source in solc_output["ethdebug"]["resources"]["compilation"]["sources"]
-    }
-    assert ethdebug_sources == standard_json_sources
-
-
-def test_resources_include_standard_json_source_contents(standard_json_input, solc_output):
-    ethdebug_sources = {
-        source["path"]: source
-        for source in solc_output["ethdebug"]["resources"]["compilation"]["sources"]
-    }
-
-    assert set(ethdebug_sources) == set(standard_json_input["sources"])
-    for source_name, source_input in standard_json_input["sources"].items():
-        assert ethdebug_sources[source_name]["contents"] == source_input["content"]
-        assert ethdebug_sources[source_name]["language"] == "Solidity"
-
-
-def test_resources_include_empty_type_and_pointer_tables(solc_output):
-    assert solc_output["ethdebug"]["resources"]["types"] == {}
-    assert solc_output["ethdebug"]["resources"]["pointers"] == {}
-
-
-def test_resources_and_compilation_share_compilation(solc_output):
-    assert solc_output["ethdebug"]["resources"]["compilation"] == solc_output["ethdebug"]["compilation"]
+@cache
+def ethdebug_schema_repository():
+    return schema_helpers.ethdebug_schema_repository(schema_helpers.ethdebug_schema_dir())
+
+
+class EthdebugSchemaConformityTest(unittest.TestCase):
+    # Set by test/ethdebugSchemaTests.py.
+    config = None
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Compiles the input once for the whole class, since every test reads the same output.
+        """
+        if cls.config is None:
+            raise RuntimeError("No configuration. Run the tests through test/ethdebugSchemaTests.py.")
+        cls.standard_json_input = load_standard_json_input(Path(__file__).parent / "input_file.json")
+        cls.solc_output = compile_standard_json(cls.config.solc_path, cls.standard_json_input)
+
+    def setUp(self):
+        self.ethdebug_schema_repository = ethdebug_schema_repository()
+
+    def check_program_schema(self, output_selection):
+        program_validator = validator("schema:ethdebug/format/program", self.ethdebug_schema_repository)
+        for (_, contract_name, ethdebug_data) in ethdebug_programs(self.solc_output, output_selection):
+            with self.subTest(contract=contract_name):
+                program_validator.validate(ethdebug_data)
+
+    def test_creation_program_schema(self):
+        self.check_program_schema("evm.bytecode.ethdebug")
+
+    def test_deployed_program_schema(self):
+        self.check_program_schema("evm.deployedBytecode.ethdebug")
+
+    def test_resources_schema(self):
+        resources_validator = validator("schema:ethdebug/format/info/resources", self.ethdebug_schema_repository)
+        resources_validator.validate(self.solc_output["ethdebug"]["resources"])
+
+    def test_compilation_schema(self):
+        compilation_validator = validator("schema:ethdebug/format/materials/compilation", self.ethdebug_schema_repository)
+        compilation_validator.validate(self.solc_output["ethdebug"]["compilation"])
+
+    def check_program_sanity(self, output_selection, environment):
+        source_ids = {source_name: source["id"] for (source_name, source) in self.solc_output["sources"].items()}
+
+        for (source_name, contract_name, ethdebug_data) in ethdebug_programs(self.solc_output, output_selection):
+            with self.subTest(contract=contract_name):
+                self.assertEqual(ethdebug_data["environment"], environment)
+                self.assertEqual(ethdebug_data["contract"]["name"], contract_name)
+                self.assertEqual(ethdebug_data["contract"]["definition"]["source"]["id"], source_ids[source_name])
+
+                instructions = ethdebug_data["instructions"]
+                self.assertGreater(len(instructions), 0)
+                self.assertEqual(
+                    [instruction["offset"] for instruction in instructions],
+                    sorted(instruction["offset"] for instruction in instructions),
+                )
+                self.assertTrue(all(instruction["operation"]["mnemonic"] for instruction in instructions))
+
+    def test_creation_program_sanity(self):
+        self.check_program_sanity("evm.bytecode.ethdebug", "create")
+
+    def test_deployed_program_sanity(self):
+        self.check_program_sanity("evm.deployedBytecode.ethdebug", "call")
+
+    def test_resources_match_standard_json_sources(self):
+        standard_json_sources = {
+            source_name: source["id"] for (source_name, source) in self.solc_output["sources"].items()
+        }
+        ethdebug_sources = {
+            source["path"]: source["id"]
+            for source in self.solc_output["ethdebug"]["resources"]["compilation"]["sources"]
+        }
+        self.assertEqual(ethdebug_sources, standard_json_sources)
+
+    def test_resources_include_standard_json_source_contents(self):
+        ethdebug_sources = {
+            source["path"]: source
+            for source in self.solc_output["ethdebug"]["resources"]["compilation"]["sources"]
+        }
+
+        self.assertEqual(set(ethdebug_sources), set(self.standard_json_input["sources"]))
+        for (source_name, source_input) in self.standard_json_input["sources"].items():
+            self.assertEqual(ethdebug_sources[source_name]["contents"], source_input["content"])
+            self.assertEqual(ethdebug_sources[source_name]["language"], "Solidity")
+
+    def test_resources_include_empty_type_and_pointer_tables(self):
+        self.assertEqual(self.solc_output["ethdebug"]["resources"]["types"], {})
+        self.assertEqual(self.solc_output["ethdebug"]["resources"]["pointers"], {})
+
+    def test_resources_and_compilation_share_compilation(self):
+        self.assertEqual(self.solc_output["ethdebug"]["resources"]["compilation"], self.solc_output["ethdebug"]["compilation"])

test/pyscriptTests.py

@@ -26,5 +26,5 @@
     test_suite = TestLoader().discover(start_dir=TEST_DIR)
     result = TextTestRunner().run(test_suite)
 
-    if len(result.errors) > 0 or len(result.failures) > 0:
+    if not result.wasSuccessful():
         sys.exit(1)