diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index bc2561d6..d74e4fae 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -562,7 +562,7 @@ jobs: run: | # Run the tests source ${{ github.workspace }}/CppInterOp/.venv/bin/activate - cd ${{ github.workspace }}/test/ + cd ${{ github.workspace }}/test echo ::group::Prepare For Testing make all python -m pip install --upgrade pip @@ -570,44 +570,12 @@ jobs: python -m pip install pytest-xdist python -m pip install numba echo ::endgroup:: - echo ::group::Run complete test suite - set -o pipefail - python -m pytest -sv -ra | tee complete_testrun.log 2>&1 - set +o pipefail echo ::group::Crashing Test Logs - # See if we don't have a crash that went away - # Comment out all xfails but the ones that have a run=False condition. - find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \; - python -m pytest -n 1 -m "xfail" --runxfail -sv -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true - git checkout . + python -m pytest -n 1 -v -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true echo ::endgroup:: - echo ::group::XFAIL Test Logs - # Rewrite all xfails that have a run clause to skipif. This way we will - # avoid conditionally crashing xfails - find . -name "*.py" -exec gsed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\1@mark.skipif(condition=not \2/g' {} \; - # See if we don't have an xfail that went away - python -m pytest --runxfail -sv -ra | tee test_xfailed.log 2>&1 || true - git checkout . - echo ::endgroup:: - echo ::group::Passing Test Logs - - # Run the rest of the non-crashing tests. - declare -i RETCODE=0 - - set -o pipefail - export RETCODE=+$? - echo ::endgroup:: - - RETCODE=+$? - echo "Complete Test Suite Summary: \n" - tail -n1 complete_testrun.log - echo "Crashing Summary: \n" + echo "Crashing Summary:" tail -n1 test_crashed.log - echo "XFAIL Summary:" - tail -n1 test_xfailed.log - echo "Return Code: ${RETCODE}" - exit $RETCODE - name: Show debug info if: ${{ failure() }} diff --git a/.github/workflows/Ubuntu.yml b/.github/workflows/Ubuntu.yml index 86f94293..08c27ee6 100644 --- a/.github/workflows/Ubuntu.yml +++ b/.github/workflows/Ubuntu.yml @@ -615,55 +615,12 @@ jobs: python -m uv pip install pytest-xdist python -m uv pip install numba echo ::endgroup:: - echo ::group::Run complete test suite - set -o pipefail - python -m pytest -sv -ra | tee complete_testrun.log 2>&1 - set +o pipefail echo ::group::Crashing Test Logs - # See if we don't have a crash that went away - # Comment out all xfails but the ones that have a run=False condition. - find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \; - python -m pytest -n 1 -m "xfail" --runxfail -sv -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true - git checkout . + python -m pytest -n 1 -v -ra --max-worker-restart 512 | tee test_crashed.log 2>&1 || true echo ::endgroup:: - echo ::group::XFAIL Test Logs - # Rewrite all xfails that have a run clause to skipif. This way we will - # avoid conditionally crashing xfails - find . -name "*.py" -exec sed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\1@mark.skipif(condition=not \2/g' {} \; - # See if we don't have an xfail that went away - python -m pytest --runxfail -sv -ra | tee test_xfailed.log 2>&1 || true - git checkout . - echo ::endgroup:: - echo ::group::Passing Test Logs - - # Run the rest of the non-crashing tests. - declare -i RETCODE=0 - - set -o pipefail - - echo "Running valgrind on passing tests" - CLANG_VERSION="${{ matrix.clang-runtime }}" - - if [[ "${{ matrix.os }}" == *"arm"* ]]; then - SUPPRESSION_FILE="../etc/clang${CLANG_VERSION}-valgrind_arm.supp" - else - SUPPRESSION_FILE="../etc/clang${CLANG_VERSION}-valgrind.supp" - fi - - valgrind --show-error-list=yes --error-exitcode=1 --track-origins=yes --gen-suppressions=all --suppressions="${SUPPRESSION_FILE}" --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -sv -ra - export RETCODE=+$? - echo ::endgroup:: - - RETCODE=+$? - echo "Complete Test Suite Summary: \n" - tail -n1 complete_testrun.log - echo "Crashing Summary: \n" + echo "Crashing Summary:" tail -n1 test_crashed.log - echo "XFAIL Summary:" - tail -n1 test_xfailed.log - echo "Return Code: ${RETCODE}" - exit $RETCODE - name: Show debug info if: ${{ failure() }} diff --git a/test/support.py b/test/support.py index 4c3c8fc7..53137f2f 100644 --- a/test/support.py +++ b/test/support.py @@ -73,3 +73,12 @@ def setup_make(targetname): #else true #endif\n""") == 1) + +from pytest import mark + +proxy = mark.xfail +def monkey_patch(*args, **kwargs): + if "run" in kwargs: + del kwargs["run"] + + return proxy(*args, **kwargs) diff --git a/test/test_aclassloader.py b/test/test_aclassloader.py index 16c21176..c08d4368 100644 --- a/test/test_aclassloader.py +++ b/test/test_aclassloader.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make +from .support import setup_make, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("example01Dict")) diff --git a/test/test_advancedcpp.py b/test/test_advancedcpp.py index 623452da..6db90f64 100644 --- a/test/test_advancedcpp.py +++ b/test/test_advancedcpp.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86 +from .support import setup_make, pylong, IS_WINDOWS, IS_MAC, IS_LINUX, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("advancedcppDict")) diff --git a/test/test_api.py b/test/test_api.py index 24509945..ab7da4f0 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import ispypy, IS_MAC, IS_LINUX_ARM +from .support import ispypy, IS_MAC, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch class TestAPI: diff --git a/test/test_boost.py b/test/test_boost.py index a000de37..d6966c4e 100644 --- a/test/test_boost.py +++ b/test/test_boost.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, raises, skip -from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM +from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, monkey_patch + +mark.xfail = monkey_patch + noboost = False if not (os.path.exists(os.path.join(os.path.sep, 'usr', 'include', 'boost')) or \ diff --git a/test/test_concurrent.py b/test/test_concurrent.py index 5d74d0cd..31f87504 100644 --- a/test/test_concurrent.py +++ b/test/test_concurrent.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM +from .support import IS_MAC_ARM, IS_MAC_X86, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch class TestCONCURRENT: diff --git a/test/test_conversions.py b/test/test_conversions.py index ff69a915..e141dfd7 100644 --- a/test/test_conversions.py +++ b/test/test_conversions.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make +from .support import setup_make, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("conversionsDict")) diff --git a/test/test_cpp11features.py b/test/test_cpp11features.py index f1089eac..60529cac 100644 --- a/test/test_cpp11features.py +++ b/test/test_cpp11features.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM +from .support import setup_make, ispypy, IS_CLANG_REPL, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() diff --git a/test/test_crossinheritance.py b/test/test_crossinheritance.py index 0fc245f4..6384fe5b 100644 --- a/test/test_crossinheritance.py +++ b/test/test_crossinheritance.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM +from .support import setup_make, pylong, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch currpath = py.path.local(__file__).dirpath() diff --git a/test/test_datatypes.py b/test/test_datatypes.py index 8daa08b1..e4223ef5 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX +from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_LINUX, monkey_patch + +mark.xfail = monkey_patch + IS_MAC = IS_MAC_X86 or IS_MAC_ARM diff --git a/test/test_doc_features.py b/test/test_doc_features.py index b350829e..dc579d1f 100644 --- a/test/test_doc_features.py +++ b/test/test_doc_features.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM +from .support import setup_make, ispypy, IS_WINDOWS, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC, IS_MAC_X86, IS_MAC_ARM, IS_LINUX_ARM, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("doc_helperDict")) diff --git a/test/test_eigen.py b/test/test_eigen.py index 187e244b..15327dbb 100644 --- a/test/test_eigen.py +++ b/test/test_eigen.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, raises -from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86 +from .support import setup_make, IS_CLANG_REPL, IS_MAC_X86, monkey_patch + +mark.xfail = monkey_patch + inc_paths = [os.path.join(os.path.sep, 'usr', 'include'), os.path.join(os.path.sep, 'usr', 'local', 'include')] diff --git a/test/test_fragile.py b/test/test_fragile.py index f795b63f..465fc6a2 100644 --- a/test/test_fragile.py +++ b/test/test_fragile.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC +from .support import setup_make, ispypy, IS_LINUX, IS_WINDOWS, IS_MAC_ARM, IS_CLANG_REPL, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() diff --git a/test/test_leakcheck.py b/test/test_leakcheck.py index 837e3d73..e019986c 100644 --- a/test/test_leakcheck.py +++ b/test/test_leakcheck.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import mark, skip -from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL +from .support import setup_make, pylong, pyunicode, IS_CLANG_REPL, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("datatypesDict")) diff --git a/test/test_lowlevel.py b/test/test_lowlevel.py index 8c982795..a5d1e09e 100644 --- a/test/test_lowlevel.py +++ b/test/test_lowlevel.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC +from .support import setup_make, pylong, pyunicode, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("datatypesDict")) diff --git a/test/test_numba.py b/test/test_numba.py index 5077fdc4..a1634305 100644 --- a/test/test_numba.py +++ b/test/test_numba.py @@ -1,7 +1,10 @@ import py, os, sys import math, time from pytest import mark, raises -from .support import setup_make +from .support import setup_make, monkey_patch + +mark.xfail = monkey_patch + try: import numba diff --git a/test/test_operators.py b/test/test_operators.py index ccbd8aa8..a842bbff 100644 --- a/test/test_operators.py +++ b/test/test_operators.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, maxvalue, IS_WINDOWS +from .support import setup_make, pylong, maxvalue, IS_WINDOWS, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("operatorsDict")) diff --git a/test/test_overloads.py b/test/test_overloads.py index 67b85ac0..ce59df9d 100644 --- a/test/test_overloads.py +++ b/test/test_overloads.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM +from .support import setup_make, ispypy, IS_WINDOWS, IS_MAC, IS_MAC_ARM, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("overloadsDict")) diff --git a/test/test_pythonify.py b/test/test_pythonify.py index 40340bc9..45ea9307 100644 --- a/test/test_pythonify.py +++ b/test/test_pythonify.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, IS_MAC +from .support import setup_make, pylong, ispypy, IS_CLANG_REPL, IS_MAC_ARM, IS_MAC_X86, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("example01Dict")) diff --git a/test/test_pythonization.py b/test/test_pythonization.py index 7a57ddb7..a63fe0cc 100644 --- a/test/test_pythonization.py +++ b/test/test_pythonization.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL +from .support import setup_make, pylong, IS_MAC_X86, IS_MAC_ARM, IS_MAC, IS_CLANG_REPL, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("pythonizablesDict")) diff --git a/test/test_regression.py b/test/test_regression.py index fa97be82..7fbc3597 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,6 +1,8 @@ import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC +from .support import setup_make, IS_WINDOWS, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch + +mark.xfail = monkey_patch class TestREGRESSION: diff --git a/test/test_stltypes.py b/test/test_stltypes.py index 5232ba7d..00109de6 100644 --- a/test/test_stltypes.py +++ b/test/test_stltypes.py @@ -1,7 +1,10 @@ # -*- coding: UTF-8 -*- import py, os, sys from pytest import raises, skip, mark -from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC +from .support import setup_make, pylong, pyunicode, maxvalue, ispypy, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("stltypesDict")) diff --git a/test/test_streams.py b/test/test_streams.py index 9a2dc5ac..e7e8692b 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -1,6 +1,9 @@ import py, os, sys from pytest import raises, mark -from .support import setup_make, IS_MAC, IS_CLANG_REPL +from .support import setup_make, IS_MAC, IS_CLANG_REPL, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("std_streamsDict")) diff --git a/test/test_templates.py b/test/test_templates.py index 1627e7a3..8ff0c029 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -1,6 +1,9 @@ import py, os from pytest import raises, mark -from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC +from .support import setup_make, pylong, IS_CLANG_REPL, IS_CLANG_DEBUG, IS_MAC_X86, IS_MAC_ARM, IS_MAC, monkey_patch + +mark.xfail = monkey_patch + currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("templatesDict"))