Merge pull request #170 from ImperialCollegeLondon/release/3.0.1

Release candidate 3.0.1 RC 7

.github/workflows/ci.yml → .github/workflows/sdv_ci.yaml

@@ -1,25 +1,33 @@
 name: Test and build
 
-on: [push, pull_request, release]
+# When does this run - new, reopened or updated PRs and when the workflow is called by
+# another workflow, such as the publishing actions.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_call:
+
 
 jobs:
-  # qa:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: actions/setup-python@v5
-  #       with:
-  #         python-version: "3.9"
-  #     - uses: pre-commit/[email protected]
+
+  qa:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: pre-commit/[email protected]
+
 
   test:
-    # needs: qa
+    needs: qa
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ "3.9" , "3.10", "3.11" ]
+        python-version: [ "3.10", "3.11", "3.12" ]
 
     steps:
     - uses: actions/checkout@v4
@@ -39,21 +47,21 @@ jobs:
     - name: Run tests
       run: poetry run pytest --cov-report xml
 
-    - name: Upload coverage reports to Codecov
-      id: codecov
-      if: success() && (matrix.os == 'ubuntu-latest' && matrix.python-version == 3.9)
+
+    - name: Upload coverage to Codecov
+      if: (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10')
       uses: codecov/codecov-action@v3
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   docs_build:
-    # needs: qa
+    needs: qa
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.10"
 
       - name: Install Poetry
         uses: abatilo/[email protected]
@@ -64,4 +72,11 @@ jobs:
         run: poetry install
 
       - name: Build docs using mkdocs
-        run: poetry run mkdocs build --strict
+        run: poetry run mkdocs build --strict
+
+      - name: Check for file modifications due to docs build
+        run: |
+          if git status | grep -q "modified"; then
+            echo "Docs build process has resulted in file changes."
+            exit 1
+          fi

.github/workflows/sdv_publish.yaml

@@ -0,0 +1,103 @@
+name: Publishing
+
+on: 
+  release:
+    types: [published]
+
+jobs:
+  # First, run the standard test suite - for this to work correctly, the workflow needs
+  # to inherit the organisation secrets used to authenticate to CodeCov.
+  # https://github.com/actions/runner/issues/1413
+  test:
+    uses: ./.github/workflows/sdv_ci.yaml
+    secrets: inherit
+
+  # Next, build the package wheel and source releases and add them to the release assets
+  build-wheel:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Build the package - this could use `poetry build` directly but pyproject.toml 
+      # already has the build-system configured to use poetry so `pip` should pick that
+      # up automatically.
+      - name: Build sdist
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+
+      # Upload the build outputs as job artifacts - these will be two files with x.y.z
+      # version numbers:
+      # - pyrealm-x.y.z-py3-none-any.whl
+      # - pyrealm-x.y.z.tar.gz
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/safedata_validator*
+
+      # Add the built files to the release assets, alongside the repo archives 
+      # automatically added by GitHub. These files should then match exactly to the
+      # published files on PyPI.
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: dist/safedata_validator*
+
+  # Now attempt to publish the package to the TestPyPI site, where the pyrealm project
+  # has been configured to allow trusted publishing from this repo and workflow.
+  #
+  # The skip-existing option allows the publication step to pass even when the release
+  # files already exists on PyPI. That suggests something has gone wrong with the
+  # release or the build file staging and the release should not be allowed to continue
+  # to publish on PyPI.
+
+  publish-TestPyPI:
+    needs: build-wheel
+    name: Publish safedata_validator to TestPyPI
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      # Download the built package files from the job artifacts
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact
+          path: dist
+
+      # Information step to show the contents of the job artifacts
+      - name: Display structure of downloaded files
+        run: ls -R dist
+
+      # Use trusted publishing to release the files downloaded into dist to TestPyPI
+      - name: Publish package distributions to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          # skip-existing: true
+
+  # The final job in the workflow is to publish to the real PyPI as long as the release
+  # name does not contain the tag 'test-pypi-only'
+  publish-PyPI:
+    if: ${{ ! contains(github.event.release.name, 'test-pypi-only')}}
+    needs: publish-TestPyPI
+    name: Publish safedata_validator to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      # Download the built package files from the job artifacts
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact
+          path: dist
+
+      # Information step to show the contents of the job artifacts
+      - name: Display structure of downloaded files
+        run: ls -R dist
+
+      # Use trusted publishing to release the files downloaded into dist to PyPI
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.pre-commit-config.yaml

@@ -1,32 +1,27 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-merge-conflict
       - id: debug-statements
-  - repo: https://github.com/PyCQA/isort
-    rev: "5.12.0"
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.6.2
     hooks:
-      - id: isort
-        additional_dependencies: [toml]
-  - repo: https://github.com/psf/black
-    rev: "22.8.0"
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        additional_dependencies: [flake8-docstrings]
+      # Run the linter.
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      # Run the formatter.
+      - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.11.2
     hooks:
       - id: mypy
         additional_dependencies:
           - types-simplejson
           - types-python-dateutil
           - types-requests
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.37.0
+    rev: v0.41.0
     hooks:
     - id: markdownlint

.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.10"
   jobs:
     post_create_environment:
       # Install poetry
@@ -25,4 +25,5 @@ build:
 
 mkdocs:
   configuration: mkdocs.yml
+  fail_on_warning: true
 

.vscode/launch.json

@@ -6,14 +6,14 @@
     "configurations": [
         {
             "name": "Python: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal"
         },
         {
             "name": "Python: Current File with Args",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
@@ -27,7 +27,7 @@
         },
         {
             "name": "_safedata_validate_cli debug",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",

.vscode/settings.json

@@ -1,9 +1,8 @@
 {
-    "python.linting.flake8Enabled": true,
-    "python.linting.mypyEnabled": true,
-    "python.linting.enabled": true,
-    "python.formatting.provider": "black",
-    "editor.formatOnSave": true,
+    "[python]": {
+        "editor.formatOnSave": true,
+        "editor.defaultFormatter": "charliermarsh.ruff"
+    },
     "editor.rulers": [
         88
     ],
@@ -13,9 +12,12 @@
         "-s",
         "test",
         // --no-cov needed so that breakpoints work: https://github.com/microsoft/vscode-python/issues/693
-        "--no-cov"
+        "--no-cov",
     ],
     "workbench.editorAssociations": {
         "*.ipynb": "jupyter-notebook"
     },
+    "[jsonc]": {
+        "editor.defaultFormatter": "vscode.json-language-features"
+    },
 }

docs/data_managers/command_line_tools/command_line_usage/safedata_build_local_gbif.txt

@@ -15,7 +15,7 @@ Build a local GBIF database.
 positional arguments:
   outfile       Filename to use for database file.
 
-optional arguments:
+options:
   -h, --help    show this help message and exit
   -t TIMESTAMP, --timestamp TIMESTAMP
                 The time stamp of a database archive version to use.

docs/data_managers/command_line_tools/command_line_usage/safedata_build_local_ncbi.txt

@@ -15,7 +15,7 @@ Build a local NCBI database.
 positional arguments:
   outfile       Filename to use for database file.
 
-optional arguments:
+options:
   -h, --help    show this help message and exit
   -t TIMESTAMP, --timestamp TIMESTAMP
                 The time stamp of a database archive version to use.

docs/data_managers/command_line_tools/command_line_usage/safedata_metadata_post_metadata.txt

@@ -10,5 +10,5 @@ positional arguments:
   zenodo_json   Path to a Zenodo metadata file
   dataset_json  Path to a dataset metadata file
 
-optional arguments:
+options:
   -h, --help    show this help message and exit

docs/data_managers/command_line_tools/command_line_usage/safedata_metadata_show_resources.txt

@@ -5,5 +5,5 @@ This subcommand simply shows the details of the resources being used and can
 be used to simply display the Zenodo and metadata server details of the
 default or specified resources file.
 
-optional arguments:
+options:
   -h, --help  show this help message and exit

docs/data_managers/command_line_tools/command_line_usage/safedata_metadata_top.txt

@@ -1,5 +1,5 @@
 cl_prompt $ safedata_metadata -h
-usage: safedata_metadata [-h] [-r RESOURCES] [-q]  ...
+usage: safedata_metadata [-h] [-r RESOURCES] [-s] [-q]  ...
 
 Post updated information to a safedata server instance.
 
@@ -16,11 +16,11 @@ positional arguments:
                 Post dataset metadata to a safedata server
     update_resources
                 Update the gazetteer data on safedata server
-    show_resources
-                Show the current resources details
 
-optional arguments:
+options:
   -h, --help    show this help message and exit
   -r RESOURCES, --resources RESOURCES
                 Path to a safedata_validator resource configuration file
+  -s, --show-resources
+                Validate and display the selected resources and exit
   -q, --quiet   Suppress normal information messages.

docs/data_managers/command_line_tools/command_line_usage/safedata_metadata_update_resources.txt

@@ -8,5 +8,5 @@ CSV being used. The server then validates the file contents.
 The resource file being used sets both the metadata server instance IP address and
 provides the paths to the files to be uploaded.
 
-optional arguments:
+options:
   -h, --help  show this help message and exit

docs/data_managers/command_line_tools/command_line_usage/safedata_validate.txt

@@ -1,8 +1,8 @@
 cl_prompt $ safedata_validate -h
-usage: safedata_validate [-h] [-r RESOURCES] [--validate_doi]
+usage: safedata_validate [-h] [-r RESOURCES] [-s] [--validate_doi]
                          [--chunk_size CHUNK_SIZE] [-l LOG] [-j JSON]
                          [--version]
-                         filename
+                         [filename]
 
 Validate a dataset using a command line interface.
 
@@ -30,12 +30,14 @@ Validate a dataset using a command line interface.
     stderr but can be redirected to a file using the `--log` option.
 
 positional arguments:
-  filename      Path to the Excel file to be validated.
+  filename      Path to the Excel file to be validated
 
-optional arguments:
+options:
   -h, --help    show this help message and exit
   -r RESOURCES, --resources RESOURCES
                 A path to a resources configuration file
+  -s, --show-resources
+                Validate and display the selected resources and exit
   --validate_doi
                 Check the validity of any publication DOIs, provided by the
                 user. Requires a web connection.