diff --git a/.env b/.env deleted file mode 100644 index 92300de..0000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -PLUGIN_REL_VERSION=0.1.0 diff --git a/.github/actions/install-dependencies/action.sh b/.github/actions/install-dependencies/action.sh index 4eefb77..8cfccf9 100755 --- a/.github/actions/install-dependencies/action.sh +++ b/.github/actions/install-dependencies/action.sh @@ -5,7 +5,7 @@ python -m pip install --upgrade pip if [[ "${INSTALL_REQUIREMENTS}" == "true" ]]; then echo "Installing code requirements" - pip install -r requirements.lock + pip install -r requirements.txt fi if [[ "${INSTALL_TEST_REQUIREMENTS}" == "true" ]]; then diff --git a/.github/actions/publish_images b/.github/actions/publish_images deleted file mode 100755 index 4c69118..0000000 --- a/.github/actions/publish_images +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -source ".env" - -BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - -ARGS=("$@") - -build_images() { - docker-compose build \ - --build-arg BUILD_DATE="${BUILD_DATE}" \ - bigquery-buildkite-plugin -} - -echo "Building Docker images (latest tag)..." -build_images - -echo "Building Docker images (version tag)..." -export IMAGE_VERSION="${PLUGIN_REL_VERSION}" -build_images - -echo "Logging into Docker Hub..." -echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USER}" --password-stdin - -echo "Pushing images to Docker Hub..." -echo wayfairossdev/bigquery-buildkite-plugin:"${PLUGIN_REL_VERSION}" | xargs -n 1 docker push - -echo "Logging out of Docker Hub..." -docker logout diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7475ae0..10fb62a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,8 +13,4 @@ Please run through before submitting for final review: - [ ] The PR title is descriptive of what changed. - [ ] The description sections above are filled out. - [ ] The CI pipeline is passing -- [ ] The plugin version has been bumped (using [SemVer](https://semver.org/spec/v2.0.0.html)) - - The plugin version should be bumped both in `.env` and in the README. **The plugin version is listed multiple times in the readme. Be sure to do a find/replace to ensure you haven't missed any!** - - [ ] Added the `ready-for-review` label to the PR. \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7a2a79c..5945769 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: test-requirements: "true" - name: Run bandit - run: bandit --ini .bandit -r deploy.py + run: bandit --ini .bandit -r plugin_scripts/deploy.py black: runs-on: ubuntu-latest @@ -55,7 +55,7 @@ jobs: test-requirements: "true" - name: Run black - run: black --check deploy.py tests/test_deploy.py + run: black --check plugin_scripts/deploy.py tests/test_deploy.py flake8: runs-on: ubuntu-latest @@ -72,7 +72,7 @@ jobs: test-requirements: "true" - name: Run flake8 - run: flake8 deploy.py tests/test_deploy.py + run: flake8 plugin_scripts/deploy.py tests/test_deploy.py isort: runs-on: ubuntu-latest @@ -89,7 +89,7 @@ jobs: test-requirements: "true" - name: Run isort - run: isort --recursive --check-only deploy.py tests/test_deploy.py + run: isort --recursive --check-only plugin_scripts/deploy.py tests/test_deploy.py mypy: runs-on: ubuntu-latest @@ -106,4 +106,4 @@ jobs: test-requirements: "true" - name: Run mypy - run: mypy deploy.py tests/test_deploy.py + run: mypy plugin_scripts/deploy.py tests/test_deploy.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 624c84b..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: "Publish" -on: - release: - types: [published] - -jobs: - publish: - name: Publish Docker Images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.3.4 - - run: ./.github/actions/publish_images - env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 33845aa..8f61add 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,16 +4,12 @@ on: push: branches: ["main"] +env: + PYTHON_VERSION: "3.9.1" + jobs: test: - name: Buildkite Plugin Bash Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.3.4 - - run: docker-compose run --rm test - - py-test: - name: Buildkite Plugin Pytest + name: Pytest runs-on: ubuntu-latest steps: - name: Check out code @@ -28,7 +24,7 @@ jobs: test-requirements: "true" - name: Run pytest - run: pytest --cov deploy tests/test_deploy.py --cov-report xml:coverage-${{ env.PYTHON_VERSION }}.xml --junitxml=test-results-${{ env.PYTHON_VERSION }}.xml + run: pytest --cov plugin_scripts/ tests/test_deploy.py --cov-report xml:coverage-${{ env.PYTHON_VERSION }}.xml --junitxml=test-results-${{ env.PYTHON_VERSION }}.xml - name: Upload pytest test results artifact uses: actions/upload-artifact@v2 @@ -49,5 +45,6 @@ jobs: - name: Publish coverage results to Codecov uses: codecov/codecov-action@v2.0.3 with: + token: ${{ secrets.CODECOV_TOKEN }} file: coverage-${{ env.PYTHON_VERSION }}.xml - fail_ci_if_error: true \ No newline at end of file + fail_ci_if_error: true diff --git a/CHANGELOG.md b/CHANGELOG.md index f59e3bd..2d8e59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.0] - 2021-09-12 + +### Added + +Inline Docker + ## [0.1.0] - 2021-07-14 ### Added diff --git a/README.md b/README.md index 8807f58..8325928 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If the version number is not provided then the most recent version of the plugin ```yaml steps: - plugins: - - wayfair-incubator/bigquery#v0.1.0: + - wayfair-incubator/bigquery#v0.2.0: gcp_project: gcp-us-project dataset_schema_directory: schemas/gcp-us-project/dataset ``` @@ -40,14 +40,6 @@ The directory in your repository where are you storing the schemas for your tabl Example: `gcp-us-project/dataset_name` -### `plugin_image_version` (optional, string) - -**ONLY to be used when testing feature branch changes to this plugin from another pipeline** - -The full hash for the latest commit to your feature branch for this plugin. This should match the plugin 'version' you are referencing in your test pipeline. - -Example: `1e602649cebf27b16dc45177ef1552b068fd2f8e` - ## Secret This plugin expects `GCP_SERVICE_ACCOUNT` is placed as environment variable. Make sure to store it [securely](https://buildkite.com/docs/pipelines/secrets)! diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index b740ce6..0000000 --- a/deploy.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -python /app/deploy.py diff --git a/docker-compose.yaml b/docker-compose.yaml index 0361940..d7ca847 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -54,6 +54,7 @@ services: lock-requirements: <<: *bigquery-buildkite-plugin entrypoint: "/bin/bash" + user: root command: "docker/lock_requirements.sh" volumes: diff --git a/docker/devbox.dockerfile b/docker/devbox.dockerfile index 6ec014d..1076100 100644 --- a/docker/devbox.dockerfile +++ b/docker/devbox.dockerfile @@ -26,6 +26,6 @@ USER ${_USER} COPY --chown=${UID}:${GID} ./requirements* /app/ WORKDIR /app -RUN pip install -r requirements.lock -r requirements-test.txt +RUN pip install -r requirements.txt -r requirements-test.txt CMD bash diff --git a/docker/lock_requirements.sh b/docker/lock_requirements.sh index 4ed9a13..8225d1c 100755 --- a/docker/lock_requirements.sh +++ b/docker/lock_requirements.sh @@ -2,7 +2,7 @@ pip uninstall -y -r <(pip freeze) pip install -r requirements.txt -printf "# THIS IS AN AUTOGENERATED LOCKFILE. DO NOT EDIT MANUALLY.\n" >requirements.lock -pip freeze --disable-pip-version-check --all >>requirements.lock +printf "# THIS IS AN AUTOGENERATED LOCKFILE. DO NOT EDIT MANUALLY.\n" >plugin_scripts/requirements.lock +pip freeze --disable-pip-version-check --all >>plugin_scripts/requirements.lock echo "Rebuild containers to verify there are no conflicts." diff --git a/docker/run_tests.sh b/docker/run_tests.sh index ceb3363..f7dcd26 100755 --- a/docker/run_tests.sh +++ b/docker/run_tests.sh @@ -35,19 +35,19 @@ while [[ $# -gt 0 ]]; do done # only generate html locally -pytest --cov deploy tests/test_deploy.py --cov-report html +pytest --cov plugin_scripts/ tests/test_deploy.py --cov-report html echo "Running MyPy..." -mypy deploy.py tests/test_deploy.py +mypy plugin_scripts/deploy.py tests/test_deploy.py echo "Running black..." -black ${BLACK_ACTION} deploy.py tests/test_deploy.py +black ${BLACK_ACTION} plugin_scripts/deploy.py tests/test_deploy.py echo "Running iSort..." -isort ${ISORT_ACTION} deploy.py tests/test_deploy.py +isort ${ISORT_ACTION} plugin_scripts/deploy.py tests/test_deploy.py echo "Running flake8..." -flake8 deploy.py tests/test_deploy.py +flake8 plugin_scripts/deploy.py tests/test_deploy.py echo "Running bandit..." -bandit --ini .bandit --quiet -r deploy.py +bandit --ini .bandit --quiet -r plugin_scripts/deploy.py diff --git a/docker/service.dockerfile b/docker/service.dockerfile deleted file mode 100644 index db71b7a..0000000 --- a/docker/service.dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM python:3.9.7-buster - -ARG _USER="lilchz" -ARG _UID="1001" -ARG _GID="100" -ARG _SHELL="/bin/bash" - -ARG VERSION=${PLUGIN_REL_VERSION} -ARG DESCRIPTION="Docker image for BigQuery Buildkite Plugin" -ARG VCS_URL="https://github.com/wayfair-incubator/bigquery-buildkite-plugin" - -ARG BUILD_DATE -LABEL \ - com.wayfair.name="wayfairossdev/bigquery-buildkite-plugin" \ - com.wayfair.build-date=${BUILD_DATE} \ - com.wayfair.description=${DESCRIPTION} \ - com.wayfair.vsc_url=${VCS_URL} \ - com.wayfair.maintainer="Jash Parekh " \ - com.wayfair.vendor="Wayfair LLC." \ - com.wayfair.version=${VERSION} - -RUN useradd -m -s "${_SHELL}" -N -u "${_UID}" "${_USER}" - -ENV USER ${_USER} -ENV UID ${_UID} -ENV GID ${_GID} -ENV HOME /home/${_USER} -ENV PATH "${HOME}/.local/bin/:${PATH}" -ENV PIP_NO_CACHE_DIR "true" - - -RUN mkdir /app && chown ${UID}:${GID} /app - -ADD . /app -COPY . /app -WORKDIR /app - -RUN chmod +x /app/deploy.sh - -USER ${_USER} - -RUN pip install -r requirements.lock - -ENTRYPOINT [ "/app/deploy.sh" ] diff --git a/hooks/command b/hooks/command index 72e58c8..dcbfa78 100755 --- a/hooks/command +++ b/hooks/command @@ -1,61 +1,116 @@ -#!/usr/bin/env bash -set -eo pipefail +#!/bin/bash +set -euo pipefail -CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." rm -rf /tmp/service_account.json PIPELINE_FILE="/tmp/service_account.json" -# shellcheck source=/dev/null -source "${CUR_DIR}/../.env" - debug_mode="false" if [[ ${BUILDKITE_PLUGIN_BIGQUERY_DEBUG_MODE:-false} =~ (true|on|1) ]]; then + echo "--- :hammer: Enabling debug mode" debug_mode="true" fi -if [[ $BUILDKITE_REPO == git@github.com:wayfair-incubator/bigquery-buildkite-plugin.git ]]; then - # if running integration tests against 'master', use release image version - if [[ $BUILDKITE_BRANCH == 'master' ]]; then - image_with_tag=${PLUGIN_REL_VERSION} - # if running integration tests against feature branch, use devel image version - else - image_with_tag=$BUILDKITE_COMMIT - fi -else - # if using plugin in a separate pipeline, default to release image version unless specified - image_with_tag="wayfairossdev/bigquery-buildkite-plugin:${PLUGIN_REL_VERSION}" -fi +workdir="/workdir" -echo "${BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT}" -echo "${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}" -echo "${image_with_tag}" +default_image="python:3.9.7-buster" +image="${BUILDKITE_PLUGIN_BIGQUERY_CUSTOM_IMAGE:-$default_image}" if [ -z "$BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT" ]; then - echo "ERROR: bigquery project key not set" + echo "ERROR: bigquery project (gcp_project) key not set" exit 1 fi +gcp_project="${BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT}" + if [ -z "${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}" ]; then - echo "ERROR: dataset schema not set" + echo "ERROR: dataset schema (dataset_schema_directory) not set" exit 1 fi +dataset_schema_directory="${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}" + if [ -z "${gcp_service_account}" ]; then - echo "ERROR: gcp service account not set" + echo "ERROR: gcp service account (gcp_service_account) not set" exit 1 fi echo "$gcp_service_account" >"$PIPELINE_FILE" -echo "--- :hammer_and_wrench: Pulling docker image" -docker pull "${image_with_tag}" - -echo "--- :docker: Running docker image" -docker run -t --rm \ - -v /usr/bin/buildkite-agent:/usr/bin/buildkite-agent \ - -v "${PWD}/${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}":/app/"${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}:ro" \ - -e dataset_schema_directory="${BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY}" \ - -e gcp_project="${BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT}" \ - -e "credentials=$(/dev/null 2>&1; then + echo "+++ 🚨 Failed to find buildkite-agent in PATH to mount into container" + exit 1 + else + BUILDKITE_AGENT_BINARY_PATH=$(command -v buildkite-agent) + fi +fi +args+=( + "--env" "BUILDKITE_JOB_ID" + "--env" "BUILDKITE_BUILD_ID" + "--env" "BUILDKITE_AGENT_ACCESS_TOKEN" + "--env" "dataset_schema_directory=$dataset_schema_directory" + "--env" "gcp_project=$gcp_project" + "--env" "credentials=$(&2 + +echo "--- :docker: Running command in ${image}" +echo "$ ${display_command[*]}" >&2 + +if [[ ${debug_mode} == "true" ]]; then + echo "docker create" >&2 + echo "docker cp \"${PWD}/.\" \"dockerContainerID:${workdir}\"" >&2 +fi + +# # For copy-checkout, we have to `docker create`, then `docker cp "${PWD}:${workdir}"`, then `docker start` +DOCKERID=$(docker create "${args[@]}") # substitute any backticks +docker cp "${PWD}/." "${DOCKERID}:${workdir}" +docker cp "${PLUGIN_DIR}/plugin_scripts" "${DOCKERID}:${workdir}/plugin_scripts" +docker start -a "${DOCKERID}" diff --git a/plugin.yml b/plugin.yml index 09bf774..8234cb7 100644 --- a/plugin.yml +++ b/plugin.yml @@ -11,7 +11,7 @@ configuration: type: string gcp_service_account: type: string - plugin_image_version: + custom_image: type: string required: - gcp_project diff --git a/plugin_scripts/__init__.py b/plugin_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deploy.py b/plugin_scripts/deploy.py similarity index 100% rename from deploy.py rename to plugin_scripts/deploy.py diff --git a/requirements.lock b/plugin_scripts/requirements.lock similarity index 64% rename from requirements.lock rename to plugin_scripts/requirements.lock index ff78825..8b6d5c6 100644 --- a/requirements.lock +++ b/plugin_scripts/requirements.lock @@ -2,16 +2,16 @@ cachetools==4.2.2 certifi==2021.5.30 cffi==1.14.6 -charset-normalizer==2.0.1 -gbq==0.1.5 -google-api-core==1.31.0 -google-auth==1.32.1 -google-cloud-bigquery==2.21.0 -google-cloud-core==1.7.1 +charset-normalizer==2.0.4 +gbq==0.2.0 +google-api-core==2.0.1 +google-auth==2.0.2 +google-cloud-bigquery==2.26.0 +google-cloud-core==2.0.0 google-crc32c==1.1.2 -google-resumable-media==1.3.1 +google-resumable-media==2.0.2 googleapis-common-protos==1.53.0 -grpcio==1.38.1 +grpcio==1.40.0 idna==3.2 packaging==21.0 pip==21.0.1 @@ -22,11 +22,10 @@ pyasn1-modules==0.2.8 pycparser==2.20 pydantic==1.8.2 pyparsing==2.4.7 -pytz==2021.1 requests==2.26.0 rsa==4.7.2 setuptools==53.0.0 six==1.16.0 -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 urllib3==1.26.6 wheel==0.36.2 diff --git a/tests/command.bats b/tests/command.bats index e56d3de..0ec5086 100644 --- a/tests/command.bats +++ b/tests/command.bats @@ -1,63 +1,14 @@ -#!/usr/bin/env bats - -load "$BATS_PATH/load.bash" - -source ".env" - -# Uncomment to enable stub debugging -export GIT_STUB_DEBUG=/dev/tty - -setup() { - export BUILDKITE_PLUGIN_BIGQUERY_DEBUG_MODE="true" - export BUILDKITE_REPO="shared/repo" - stub buildkite-agent ${UPLOAD_ARGS} -} - -teardown() { - unstub docker - TMPDIR=$TMPDIR_BACKUP - unset BUILDKITE_PLUGIN_BIGQUERY_DEBUG_MODE - unset BUILDKITE_REPO -} - -@test "Fails when project key not set" { - unset BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT - - stub docker - run $PWD/hooks/command - assert_failure - assert_output --partial "ERROR: bigquery project key not set" -} - -@test "Fails when dataset key not set" { - export BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT="gcp-project" - unset BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY - - stub docker - run $PWD/hooks/command - assert_failure - assert_output --partial "ERROR: dataset schema not set" -} - -@test "Fails when gcp service account key not set" { - export BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT="gcp-project" - export BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY="dataset" - unset BUILDKITE_PLUGIN_BIGQUERY_GCP_SERVICE_ACCOUNT - - stub docker - run $PWD/hooks/command - assert_failure - assert_output --partial "ERROR: gcp service account not set" -} - -@test "Logs start of run" { - export BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT="gcp-project" - export BUILDKITE_PLUGIN_BIGQUERY_DATASET_SCHEMA_DIRECTORY="dataset" - export gcp_service_account="credentials" - - stub docker - - run $PWD/hooks/command - assert_success - assert_output --partial "--- :hammer_and_wrench: Pulling docker image" -} +# #!/usr/bin/env bats +# +# load "$BATS_PATH/load.bash" +# +# # Uncomment to enable stub debugging +# # export GIT_STUB_DEBUG=/dev/tty +# +# @test "Dummy output outputs as expected" { +# # Note there is no export BUILDKITE_PULL_REQUEST, or PULL_REQUEST_RPO +# run $PWD/hooks/command +# +# assert_success +# assert_output --partial "this is the command hook!" +# } \ No newline at end of file diff --git a/tests/test_deploy.py b/tests/test_deploy.py index b72cea8..173aac2 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -1,6 +1,6 @@ import pytest -import deploy +from plugin_scripts import deploy @pytest.fixture