[DO NOT MERGE] Debug xfails

.github/workflows/MacOS.yml

@@ -562,52 +562,20 @@ jobs:
       run: |
         # Run the tests
         source ${{ github.workspace }}/CppInterOp/.venv/bin/activate
-        cd ${{ github.workspace }}/test/
+        cd ${{ github.workspace }}/test
         echo ::group::Prepare For Testing
         make all
         python -m pip install --upgrade pip
         python -m pip install pytest
         python -m pip install pytest-xdist
         python -m pip install numba
         echo ::endgroup::
-        echo ::group::Run complete test suite
-        set -o pipefail
-        python -m pytest -sv -ra | tee complete_testrun.log 2>&1
-        set +o pipefail
         echo ::group::Crashing Test Logs
-        # See if we don't have a crash that went away
-        # Comment out all xfails but the ones that have a run=False condition.
-        find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \;
-        python -m pytest -n 1 -m "xfail" --runxfail -sv -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
-        git checkout .
+        python -m pytest -n 1 -v -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
         echo ::endgroup::
-        echo ::group::XFAIL Test Logs
-        # Rewrite all xfails that have a run clause to skipif. This way we will
-        # avoid conditionally crashing xfails
-        find . -name "*.py" -exec gsed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\[email protected](condition=not \2/g' {} \;
-        # See if we don't have an xfail that went away
-        python -m pytest --runxfail -sv -ra  | tee test_xfailed.log 2>&1 || true
-        git checkout .
-        echo ::endgroup::
-        echo ::group::Passing Test Logs
-
-        # Run the rest of the non-crashing tests.
-        declare -i RETCODE=0
-
-        set -o pipefail
-        export RETCODE=+$?
-        echo ::endgroup::
-
-        RETCODE=+$?
 
-        echo "Complete Test Suite Summary: \n"
-        tail -n1 complete_testrun.log
-        echo "Crashing Summary: \n"
+        echo "Crashing Summary:"
         tail -n1 test_crashed.log
-        echo "XFAIL Summary:"
-        tail -n1 test_xfailed.log
-        echo "Return Code: ${RETCODE}"
-        exit $RETCODE
 
     - name: Show debug info
       if: ${{ failure() }}

.github/workflows/Ubuntu.yml

@@ -615,55 +615,12 @@ jobs:
         python -m uv pip install pytest-xdist
         python -m uv pip install numba
         echo ::endgroup::
-        echo ::group::Run complete test suite
-        set -o pipefail
-        python -m pytest -sv -ra | tee complete_testrun.log 2>&1
-        set +o pipefail
         echo ::group::Crashing Test Logs
-        # See if we don't have a crash that went away
-        # Comment out all xfails but the ones that have a run=False condition.
-        find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \;
-        python -m pytest -n 1 -m "xfail" --runxfail -sv -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
-        git checkout .
+        python -m pytest -n 1 -v -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
         echo ::endgroup::
-        echo ::group::XFAIL Test Logs
-        # Rewrite all xfails that have a run clause to skipif. This way we will
-        # avoid conditionally crashing xfails
-        find . -name "*.py" -exec sed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\[email protected](condition=not \2/g' {} \;
-        # See if we don't have an xfail that went away
-        python -m pytest --runxfail -sv -ra  | tee test_xfailed.log 2>&1 || true
-        git checkout .
-        echo ::endgroup::
-        echo ::group::Passing Test Logs
-
-        # Run the rest of the non-crashing tests.
-        declare -i RETCODE=0
-
-        set -o pipefail
-
-        echo "Running valgrind on passing tests"
-        CLANG_VERSION="${{ matrix.clang-runtime }}"
-
-        if [[ "${{ matrix.os }}" == *"arm"* ]]; then
-          SUPPRESSION_FILE="../etc/clang${CLANG_VERSION}-valgrind_arm.supp"
-        else
-          SUPPRESSION_FILE="../etc/clang${CLANG_VERSION}-valgrind.supp"
-        fi
-
-        valgrind --show-error-list=yes --error-exitcode=1 --track-origins=yes --gen-suppressions=all --suppressions="${SUPPRESSION_FILE}" --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -sv -ra
-        export RETCODE=+$?
-        echo ::endgroup::
-
-        RETCODE=+$?
 
-        echo "Complete Test Suite Summary: \n"
-        tail -n1 complete_testrun.log
-        echo "Crashing Summary: \n"
+        echo "Crashing Summary:"
         tail -n1 test_crashed.log
-        echo "XFAIL Summary:"
-        tail -n1 test_xfailed.log
-        echo "Return Code: ${RETCODE}"
-        exit $RETCODE
 
     - name: Show debug info
       if: ${{ failure() }}

test/support.py

@@ -73,3 +73,12 @@ def setup_make(targetname):
                                             #else
                                             true
                                             #endif\n""") == 1)
+
+from pytest import mark
+
+proxy = mark.xfail
+def monkey_patch(*args, **kwargs):
+    if "run" in kwargs:
+        del kwargs["run"]
+
+    return proxy(*args, **kwargs)

test/test_aclassloader.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, mark
-from .support import setup_make
+from .support import setup_make, monkey_patch
+
+mark.xfail = monkey_patch
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict"))

test/test_advancedcpp.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86
+from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("advancedcppDict"))

test/test_api.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import ispypy, IS_MAC, IS_LINUX_ARM
+from .support import ispypy, IS_MAC, IS_LINUX_ARM, monkey_patch
+
+mark.xfail = monkey_patch
 
 
 class TestAPI:

test/test_boost.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import mark, raises, skip
-from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM
+from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 noboost = False
 if not (os.path.exists(os.path.join(os.path.sep, 'usr', 'include', 'boost')) or \

test/test_concurrent.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM
+from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM, monkey_patch
+
+mark.xfail = monkey_patch
 
 
 class TestCONCURRENT:

test/test_conversions.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, mark
-from .support import setup_make
+from .support import setup_make, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("conversionsDict"))

test/test_cpp11features.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, mark
-from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM
+from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM, monkey_patch
+
+mark.xfail = monkey_patch
 
 
 currpath = py.path.local(__file__).dirpath()

test/test_crossinheritance.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM
+from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM, monkey_patch
+
+mark.xfail = monkey_patch
 
 
 currpath = py.path.local(__file__).dirpath()

test/test_datatypes.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX
+from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 IS_MAC = IS_MAC_X86 or IS_MAC_ARM
 

test/test_doc_features.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM
+from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("doc_helperDict"))

test/test_eigen.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import mark, raises
-from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86
+from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 inc_paths = [os.path.join(os.path.sep, 'usr', 'include'),
              os.path.join(os.path.sep, 'usr', 'local', 'include')]

test/test_fragile.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC
+from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 
 currpath = py.path.local(__file__).dirpath()

test/test_leakcheck.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import mark, skip
-from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL
+from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("datatypesDict"))

test/test_lowlevel.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC
+from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("datatypesDict"))

test/test_numba.py

@@ -1,7 +1,10 @@
 import py, os, sys
 import math, time
 from pytest import mark, raises
-from .support import setup_make
+from .support import setup_make, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 try:
     import numba

test/test_operators.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, maxvalue, IS_WINDOWS
+from .support import setup_make, pylong, maxvalue, IS_WINDOWS, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("operatorsDict"))

test/test_overloads.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM
+from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("overloadsDict"))

test/test_pythonify.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, IS_MAC
+from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict"))

test/test_pythonization.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, mark
-from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL
+from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("pythonizablesDict"))

test/test_regression.py

@@ -1,6 +1,8 @@
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC
+from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
 
 
 class TestREGRESSION:

test/test_stltypes.py

@@ -1,7 +1,10 @@
 # -*- coding: UTF-8 -*-
 import py, os, sys
 from pytest import raises, skip, mark
-from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC
+from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("stltypesDict"))

test/test_streams.py

@@ -1,6 +1,9 @@
 import py, os, sys
 from pytest import raises, mark
-from .support import setup_make, IS_MAC, IS_CLANG_REPL
+from .support import setup_make, IS_MAC, IS_CLANG_REPL, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("std_streamsDict"))

test/test_templates.py

@@ -1,6 +1,9 @@
 import py, os
 from pytest import raises, mark
-from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC
+from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch
+
+mark.xfail = monkey_patch
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("templatesDict"))