feat(forks,plugins): Fork transition test marker (ethereum#1081)

* feat(forks): Adds `supports_blobs` method

* fix(forks): Transition fork comparisons

* refactor,fix(plugins/forks): `fork_transition_test` marker

* docs: add documentation, changelog

* refactor(plugins/forks): Validity markers as classes

* fix: fork set handling

* fix(plugins): config variable usage in other plugins

* fix(plugins/forks): Self-documenting code for validity marker classes

* whitelist

* refactor(forks): suggestions for fork transition marker changes (ethereum#1104)

* refactor(forks): make `name()` an abstractmethod

* refactor(forks): add helper methods to simplify fork comparisons

* Update src/pytest_plugins/forks/forks.py

Co-authored-by: danceratopz <[email protected]>

* Update src/pytest_plugins/forks/forks.py

Co-authored-by: danceratopz <[email protected]>

* Update src/pytest_plugins/forks/forks.py

Co-authored-by: danceratopz <[email protected]>

* fixup

---------

Co-authored-by: danceratopz <[email protected]>

docs/CHANGELOG.md

@@ -78,6 +78,7 @@ Release tarball changes:
 - 🐞 fix(consume): allow absolute paths with `--evm-bin` ([#1052](https://github.com/ethereum/execution-spec-tests/pull/1052)).
 - ✨ Disable EIP-7742 framework changes for Prague ([#1023](https://github.com/ethereum/execution-spec-tests/pull/1023)).
 - ✨ Allow verification of the transaction receipt on executed test transactions ([#1068](https://github.com/ethereum/execution-spec-tests/pull/1068)).
+- ✨ Modify `valid_at_transition_to` marker to add keyword arguments `subsequent_transitions` and `until` to fill a test using multiple transition forks ([#1081](https://github.com/ethereum/execution-spec-tests/pull/1081)).
 - 🐞 fix(consume): use `"HIVE_CHECK_LIVE_PORT"` to signal hive to wait for port 8551 (Engine API port) instead of the 8545 port when running `consume engine` ([#1095](https://github.com/ethereum/execution-spec-tests/pull/1095)).
 
 ### 🔧 EVM Tools

docs/writing_tests/test_markers.md

@@ -10,47 +10,15 @@ These markers are used to specify the forks for which a test is valid.
 
 ### `@pytest.mark.valid_from("FORK_NAME")`
 
-This marker is used to specify the fork from which the test is valid. The test will not be filled for forks before the specified fork.
-
-```python
-import pytest
-
-from ethereum_test_tools import Alloc, StateTestFiller
-
-@pytest.mark.valid_from("London")
-def test_something_only_valid_after_london(
-    state_test: StateTestFiller, 
-    pre: Alloc
-):
-    pass
-```
-
-In this example, the test will only be filled for the London fork and after, e.g. London, Paris, Shanghai, Cancun, etc.
+:::pytest_plugins.forks.forks.ValidFrom
 
 ### `@pytest.mark.valid_until("FORK_NAME")`
 
-This marker is used to specify the fork until which the test is valid. The test will not be filled for forks after the specified fork.
-
-```python
-import pytest
-
-from ethereum_test_tools import Alloc, StateTestFiller
-
-@pytest.mark.valid_until("London")
-def test_something_only_valid_until_london(
-    state_test: StateTestFiller, 
-    pre: Alloc
-):
-    pass
-```
-
-In this example, the test will only be filled for the London fork and before, e.g. London, Berlin, Istanbul, etc.
+:::pytest_plugins.forks.forks.ValidUntil
 
 ### `@pytest.mark.valid_at_transition_to("FORK_NAME")`
 
-This marker is used to specify that a test is only meant to be filled at the transition to the specified fork.
-
-The test usually starts at the fork prior to the specified fork at genesis and at block 5 (for pre-merge forks) or at timestamp 15,000 (for post-merge forks) the fork transition occurs.
+:::pytest_plugins.forks.forks.ValidAtTransitionTo
 
 ## Fork Covariant Markers
 

src/ethereum_test_forks/base_fork.py

@@ -104,29 +104,43 @@ def __call__(
 class BaseForkMeta(ABCMeta):
     """Metaclass for BaseFork."""
 
+    @abstractmethod
     def name(cls) -> str:
-        """To be implemented by the fork base class."""
-        return ""
+        """Return the name of the fork (e.g., Berlin), must be implemented by subclasses."""
+        pass
 
     def __repr__(cls) -> str:
         """Print the name of the fork, instead of the class."""
         return cls.name()
 
+    @staticmethod
+    def _maybe_transitioned(fork_cls: "BaseForkMeta") -> "BaseForkMeta":
+        """Return the transitioned fork, if a transition fork, otherwise return `fork_cls`."""
+        return fork_cls.transitions_to() if hasattr(fork_cls, "transitions_to") else fork_cls
+
+    @staticmethod
+    def _is_subclass_of(a: "BaseForkMeta", b: "BaseForkMeta") -> bool:
+        """Check if `a` is a subclass of `b`, taking fork transitions into account."""
+        a = BaseForkMeta._maybe_transitioned(a)
+        b = BaseForkMeta._maybe_transitioned(b)
+        return issubclass(a, b)
+
     def __gt__(cls, other: "BaseForkMeta") -> bool:
-        """Compare if a fork is newer than some other fork."""
-        return cls != other and other.__subclasscheck__(cls)
+        """Compare if a fork is newer than some other fork (cls > other)."""
+        return cls is not other and BaseForkMeta._is_subclass_of(cls, other)
 
     def __ge__(cls, other: "BaseForkMeta") -> bool:
-        """Compare if a fork is newer than or equal to some other fork."""
-        return other.__subclasscheck__(cls)
+        """Compare if a fork is newer than or equal to some other fork (cls >= other)."""
+        return cls is other or BaseForkMeta._is_subclass_of(cls, other)
 
     def __lt__(cls, other: "BaseForkMeta") -> bool:
-        """Compare if a fork is older than some other fork."""
-        return cls != other and cls.__subclasscheck__(other)
+        """Compare if a fork is older than some other fork (cls < other)."""
+        # "Older" means other is a subclass of cls, but not the same.
+        return cls is not other and BaseForkMeta._is_subclass_of(other, cls)
 
     def __le__(cls, other: "BaseForkMeta") -> bool:
-        """Compare if a fork is older than or equal to some other fork."""
-        return cls.__subclasscheck__(other)
+        """Compare if a fork is older than or equal to some other fork (cls <= other)."""
+        return cls is other or BaseForkMeta._is_subclass_of(other, cls)
 
 
 class BaseFork(ABC, metaclass=BaseForkMeta):
@@ -289,6 +303,12 @@ def blob_base_fee_update_fraction(cls, block_number: int = 0, timestamp: int = 0
         """Return the blob base fee update fraction at a given fork."""
         pass
 
+    @classmethod
+    @abstractmethod
+    def supports_blobs(cls, block_number: int = 0, timestamp: int = 0) -> bool:
+        """Return whether the given fork supports blobs or not."""
+        pass
+
     @classmethod
     @abstractmethod
     def target_blobs_per_block(cls, block_number: int = 0, timestamp: int = 0) -> int:

src/ethereum_test_forks/forks/forks.py

@@ -215,39 +215,46 @@ def blob_gas_price_calculator(
         cls, block_number: int = 0, timestamp: int = 0
     ) -> BlobGasPriceCalculator:
         """Return a callable that calculates the blob gas price at a given fork."""
-        raise NotImplementedError("Blob gas price calculator is not supported in Frontier")
+        raise NotImplementedError(f"Blob gas price calculator is not supported in {cls.name()}")
 
     @classmethod
     def excess_blob_gas_calculator(
         cls, block_number: int = 0, timestamp: int = 0
     ) -> ExcessBlobGasCalculator:
         """Return a callable that calculates the excess blob gas for a block at a given fork."""
-        raise NotImplementedError("Excess blob gas calculator is not supported in Frontier")
+        raise NotImplementedError(f"Excess blob gas calculator is not supported in {cls.name()}")
 
     @classmethod
     def min_base_fee_per_blob_gas(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Return the amount of blob gas used per blob at a given fork."""
-        raise NotImplementedError("Base fee per blob gas is not supported in Frontier")
+        raise NotImplementedError(f"Base fee per blob gas is not supported in {cls.name()}")
 
     @classmethod
     def blob_base_fee_update_fraction(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Return the blob base fee update fraction at a given fork."""
-        raise NotImplementedError("Blob base fee update fraction is not supported in Frontier")
+        raise NotImplementedError(
+            f"Blob base fee update fraction is not supported in {cls.name()}"
+        )
 
     @classmethod
     def blob_gas_per_blob(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Return the amount of blob gas used per blob at a given fork."""
         return 0
 
+    @classmethod
+    def supports_blobs(cls, block_number: int = 0, timestamp: int = 0) -> bool:
+        """Blobs are not supported at Frontier."""
+        return False
+
     @classmethod
     def target_blobs_per_block(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Return the target number of blobs per block at a given fork."""
-        raise NotImplementedError("Target blobs per block is not supported in Frontier")
+        raise NotImplementedError(f"Target blobs per block is not supported in {cls.name()}")
 
     @classmethod
     def max_blobs_per_block(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Return the max number of blobs per block at a given fork."""
-        raise NotImplementedError("Max blobs per block is not supported in Frontier")
+        raise NotImplementedError(f"Max blobs per block is not supported in {cls.name()}")
 
     @classmethod
     def header_requests_required(cls, block_number: int = 0, timestamp: int = 0) -> bool:
@@ -938,6 +945,11 @@ def blob_gas_per_blob(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Blobs are enabled starting from Cancun."""
         return 2**17
 
+    @classmethod
+    def supports_blobs(cls, block_number: int = 0, timestamp: int = 0) -> bool:
+        """At Cancun, blobs support is enabled."""
+        return True
+
     @classmethod
     def target_blobs_per_block(cls, block_number: int = 0, timestamp: int = 0) -> int:
         """Blobs are enabled starting from Cancun, with a static target of 3 blobs."""

src/ethereum_test_forks/helpers.py

@@ -81,16 +81,16 @@ def get_closest_fork_with_solc_support(fork: Fork, solc_version: Version) -> Opt
     )
 
 
-def get_transition_forks() -> List[Fork]:
+def get_transition_forks() -> Set[Fork]:
     """Return all the transition forks."""
-    transition_forks: List[Fork] = []
+    transition_forks: Set[Fork] = set()
 
     for fork_name in transition.__dict__:
         fork = transition.__dict__[fork_name]
         if not isinstance(fork, type):
             continue
         if issubclass(fork, TransitionBaseClass) and issubclass(fork, BaseFork):
-            transition_forks.append(fork)
+            transition_forks.add(fork)
 
     return transition_forks
 
@@ -164,14 +164,14 @@ def transition_fork_from_to(fork_from: Fork, fork_to: Fork) -> Fork | None:
     return None
 
 
-def transition_fork_to(fork_to: Fork) -> List[Fork]:
+def transition_fork_to(fork_to: Fork) -> Set[Fork]:
     """Return transition fork that transitions to the specified fork."""
-    transition_forks: List[Fork] = []
+    transition_forks: Set[Fork] = set()
     for transition_fork in get_transition_forks():
         if not issubclass(transition_fork, TransitionBaseClass):
             continue
         if transition_fork.transitions_to() == fork_to:
-            transition_forks.append(transition_fork)
+            transition_forks.add(transition_fork)
 
     return transition_forks
 

src/ethereum_test_forks/tests/test_forks.py

@@ -17,7 +17,12 @@
     Prague,
     Shanghai,
 )
-from ..forks.transition import BerlinToLondonAt5, ParisToShanghaiAtTime15k
+from ..forks.transition import (
+    BerlinToLondonAt5,
+    CancunToPragueAtTime15k,
+    ParisToShanghaiAtTime15k,
+    ShanghaiToCancunAtTime15k,
+)
 from ..helpers import (
     forks_from,
     forks_from_until,
@@ -40,7 +45,7 @@ def test_transition_forks():
     """Test transition fork utilities."""
     assert transition_fork_from_to(Berlin, London) == BerlinToLondonAt5
     assert transition_fork_from_to(Berlin, Paris) is None
-    assert transition_fork_to(Shanghai) == [ParisToShanghaiAtTime15k]
+    assert transition_fork_to(Shanghai) == {ParisToShanghaiAtTime15k}
 
     # Test forks transitioned to and from
     assert BerlinToLondonAt5.transitions_to() == London
@@ -119,6 +124,9 @@ def test_forks():
     assert cast(Fork, ParisToShanghaiAtTime15k).header_withdrawals_required(0, 15_000) is True
     assert cast(Fork, ParisToShanghaiAtTime15k).header_withdrawals_required() is True
 
+
+def test_fork_comparison():
+    """Test fork comparison operators."""
     # Test fork comparison
     assert Paris > Berlin
     assert not Berlin > Paris
@@ -153,6 +161,50 @@ def test_forks():
     assert fork == Berlin
 
 
+def test_transition_fork_comparison():
+    """
+    Test comparing to a transition fork.
+
+    The comparison logic is based on the logic we use to generate the tests.
+
+    E.g. given transition fork A->B, when filling, and given the from/until markers,
+    we expect the following logic:
+
+    Marker    Comparison   A->B Included
+    --------- ------------ ---------------
+    From A    fork >= A    True
+    Until A   fork <= A    False
+    From B    fork >= B    True
+    Until B   fork <= B    True
+    """
+    assert BerlinToLondonAt5 >= Berlin
+    assert not BerlinToLondonAt5 <= Berlin
+    assert BerlinToLondonAt5 >= London
+    assert BerlinToLondonAt5 <= London
+
+    # Comparisons between transition forks is done against the `transitions_to` fork
+    assert BerlinToLondonAt5 < ParisToShanghaiAtTime15k
+    assert ParisToShanghaiAtTime15k > BerlinToLondonAt5
+    assert BerlinToLondonAt5 == BerlinToLondonAt5
+    assert BerlinToLondonAt5 != ParisToShanghaiAtTime15k
+    assert BerlinToLondonAt5 <= ParisToShanghaiAtTime15k
+    assert ParisToShanghaiAtTime15k >= BerlinToLondonAt5
+
+    assert sorted(
+        {
+            CancunToPragueAtTime15k,
+            ParisToShanghaiAtTime15k,
+            ShanghaiToCancunAtTime15k,
+            BerlinToLondonAt5,
+        }
+    ) == [
+        BerlinToLondonAt5,
+        ParisToShanghaiAtTime15k,
+        ShanghaiToCancunAtTime15k,
+        CancunToPragueAtTime15k,
+    ]
+
+
 def test_get_forks():  # noqa: D103
     all_forks = get_forks()
     assert all_forks[0] == FIRST_DEPLOYED

src/pytest_plugins/execute/execute.py

@@ -95,10 +95,10 @@ def pytest_configure(config):
     command_line_args = "fill " + " ".join(config.invocation_params.args)
     config.stash[metadata_key]["Command-line args"] = f"<code>{command_line_args}</code>"
 
-    if len(config.fork_set) != 1:
+    if len(config.selected_fork_set) != 1:
         pytest.exit(
             f"""
-            Expected exactly one fork to be specified, got {len(config.fork_set)}.
+            Expected exactly one fork to be specified, got {len(config.selected_fork_set)}.
             Make sure to specify exactly one fork using the --fork command line argument.
             """,
             returncode=pytest.ExitCode.USAGE_ERROR,