From 3f296de6abf66e703e840d9966775ebb41b89d52 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Thu, 13 Feb 2025 17:20:20 +0100 Subject: [PATCH] ci: Add test plan that runs CI tests and customize it for each role * Calculate number of managed nodes with this formula: (( number_of_test_playbooks / 10 + 1 )) * Add README explaining how to run the plan locally and remotely Signed-off-by: Sergei Petrosian --- .codespell_ignores | 0 .codespellrc | 10 +++++ .github/workflows/codespell.yml | 17 +++++++++ .github/workflows/tft_citest_bad.yml | 2 +- README.md | 2 +- plans/README-plans.md | 55 ++++++++++++++++++++++++++++ plans/test_playbooks_parallel.fmf | 35 ++++++++++-------- 7 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 .codespell_ignores create mode 100644 .codespellrc create mode 100644 .github/workflows/codespell.yml create mode 100644 plans/README-plans.md diff --git a/.codespell_ignores b/.codespell_ignores new file mode 100644 index 00000000..e69de29b diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..06cadf71 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,10 @@ +[codespell] +check-hidden = true +# Note that `-w` doesn't work when ignore-multiline-regex is set +# https://github.com/codespell-project/codespell/issues/3642 +ignore-multiline-regex = codespell:ignore-begin.*codespell:ignore-end +ignore-words = .codespell_ignores +# skip-file is not available https://github.com/codespell-project/codespell/pull/2759 +# .pandoc_template.html5 contains a typo in Licence that we shouldn't edit +# .README.html is generated from README.md automatically - no need to check spelling +skip = .pandoc_template.html5,.README.html,./tests/files/client.key.nocrypt,./tests/files/client.pem diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000..8eadb612 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,17 @@ +# Codespell configuration is within .codespellrc +--- +name: Codespell +on: # yamllint disable-line rule:truthy + - pull_request +permissions: + contents: read +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Codespell + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/tft_citest_bad.yml b/.github/workflows/tft_citest_bad.yml index 85db1f44..9f48f353 100644 --- a/.github/workflows/tft_citest_bad.yml +++ b/.github/workflows/tft_citest_bad.yml @@ -36,7 +36,7 @@ jobs: RUN_ID=$(gh api "repos/$REPO/actions/workflows/tft.yml/runs?event=issue_comment" \ | jq -r "[.workflow_runs[] | select( .display_title == \"$PR_TITLE\" ) | select( .conclusion == \"failure\" ) | .id][0]") if [ "$RUN_ID" = "null" ]; then - echo "Failed workflow not found, exitting" + echo "Failed workflow not found, exiting" exit 1 fi echo "Re-running workflow $RUN_ID" diff --git a/README.md b/README.md index 0e4a4258..b39e8e19 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # linux-system-roles/network -[![ansible-lint.yml](https://github.com/linux-system-roles/network/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/network/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/network/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/codeql.yml) [![integration.yml](https://github.com/linux-system-roles/network/actions/workflows/integration.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/integration.yml) [![markdownlint.yml](https://github.com/linux-system-roles/network/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/network/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/python-unit-test.yml) [![shellcheck.yml](https://github.com/linux-system-roles/network/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/network/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/tft.yml) [![tft_citest_bad.yml](https://github.com/linux-system-roles/network/actions/workflows/tft_citest_bad.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/tft_citest_bad.yml) [![woke.yml](https://github.com/linux-system-roles/network/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/woke.yml) [![Coverage Status](https://coveralls.io/repos/github/linux-system-roles/network/badge.svg)](https://coveralls.io/github/linux-system-roles/network) [![Code Style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/linux-system-roles/network.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/linux-system-roles/network/context:python) +[![ansible-lint.yml](https://github.com/linux-system-roles/network/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/network/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/network/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/codeql.yml) [![codespell.yml](https://github.com/linux-system-roles/network/actions/workflows/codespell.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/codespell.yml) [![integration.yml](https://github.com/linux-system-roles/network/actions/workflows/integration.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/integration.yml) [![markdownlint.yml](https://github.com/linux-system-roles/network/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/network/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/python-unit-test.yml) [![shellcheck.yml](https://github.com/linux-system-roles/network/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/network/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/tft.yml) [![tft_citest_bad.yml](https://github.com/linux-system-roles/network/actions/workflows/tft_citest_bad.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/tft_citest_bad.yml) [![woke.yml](https://github.com/linux-system-roles/network/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/network/actions/workflows/woke.yml) [![Coverage Status](https://coveralls.io/repos/github/linux-system-roles/network/badge.svg)](https://coveralls.io/github/linux-system-roles/network) [![Code Style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/linux-system-roles/network.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/linux-system-roles/network/context:python) ## Overview diff --git a/plans/README-plans.md b/plans/README-plans.md new file mode 100644 index 00000000..6f532bd6 --- /dev/null +++ b/plans/README-plans.md @@ -0,0 +1,55 @@ +# Introduction CI Testing Plans + +Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tft.yml](https://github.com/linux-system-roles/network/blob/main/.github/workflows/tft.yml) GitHub workflow. + +The `plans/test_playbooks_parallel.fmf` plan is a test plan that runs test playbooks in parallel on multiple managed nodes. +`plans/test_playbooks_parallel.fmf` is generated centrally from `https://github.com/linux-system-roles/.github/`. +The automation calculates the number of managed nodes to provision with this formula: + +```plain +number-of-test-playbooks / 10 + 1 +``` + +The `plans/test_playbooks_parallel.fmf` plan does the following steps: + +1. Provisions systems to be used as a control node and as managed nodes. +2. Does the required preparation on systems. +3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh). + +The [tft.yml](https://github.com/linux-system-roles/network/blob/main/.github/workflows/tft.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access. +This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm). + +## Running Tests + +You can run tests locally with the `tmt try` cli or remotely in Testing Far. + +### Running Tests Locally + +1. Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html). +2. Change to the role repository directory. +3. Modify `plans/test_playbooks_parallel.fmf` to suit your requirements: + 1. Due to [issue #3138](https://github.com/teemtee/tmt/issues/3138), comment out all managed nodes except for one. + 2. Optionally modify environment variables to, e.g. run only specified test playbooks by modifying `SYSTEM_ROLES_ONLY_TESTS`. +4. Enter `tmt try -p plans/test_playbooks_parallel `. + This command identifies the `plans/test_playbooks_parallel.fmf` plan and provisions local VMs, a control node and a managed node. +5. tmt try is in development and does not identify tests from URL automatically, so after provisioning the machines, you must type `t`, `p`, `t` from the interactive prompt to identify tests, run preparation steps, and run the tests. + +### Running in Testing Farm + +1. Install `testing-farm` as described in [Installation](https://gitlab.com/testing-farm/cli/-/blob/main/README.adoc#user-content-installation). +2. Change to the role repository directory. +3. If you want to run tests with edits in your branch, you need to commit and push changes first to some branch. +4. You can uncomment "Inject your ssh public key to test systems" discover step in the plan if you want to troubleshoot tests by SSHing into test systems in Testing Farm. +5. Enter `testing-farm request`. + Edit to your needs. + + ```bash + $ TESTING_FARM_API_TOKEN= \ + testing-farm request --pipeline-type="tmt-multihost" \ + --plan-filter="tag:playbooks_parallel" \ + --git-url "https://github.com//network" \ + --git-ref "" \ + --compose CentOS-Stream-9 \ + -e "SYSTEM_ROLES_ONLY_TESTS=tests_default.yml" \ + --no-wait + ``` diff --git a/plans/test_playbooks_parallel.fmf b/plans/test_playbooks_parallel.fmf index faa01d65..b810af91 100644 --- a/plans/test_playbooks_parallel.fmf +++ b/plans/test_playbooks_parallel.fmf @@ -14,48 +14,55 @@ provision: connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth - name: managed-node2 role: managed_node connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth - name: managed-node3 role: managed_node connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth - name: managed-node4 role: managed_node connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth - name: managed-node5 role: managed_node connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth - name: managed-node6 role: managed_node connection: system hardware: network: - - type: eth - - type: eth + - type: eth + - type: eth + - name: managed-node7 + role: managed_node + connection: system + hardware: + network: + - type: eth + - type: eth environment: ANSIBLE_VER: 2.17 REPO_NAME: network PYTHON_VERSION: 3.12 - SYSTEM_ROLES_ONLY_TESTS: "" + SYSTEM_ROLES_ONLY_TESTS: "" # tests_default.yml TEST_LOCAL_CHANGES: true PR_NUM: "" LINUXSYSTEMROLES_USER: "" @@ -82,8 +89,6 @@ prepare: fi yum install epel-release yum-utils -y yum-config-manager --enable epel epel-debuginfo epel-source - - name: Test NICs - script: lshw -C network discover: - name: Prepare managed node how: fmf