Optimize pipeline run time (#71)

.github/actions/install-dependencies/action.sh

@@ -5,10 +5,10 @@ python -m pip install --upgrade pip
 
 if [[ "${INSTALL_REQUIREMENTS}" == "true"  ]]; then
   echo "Installing code requirements"
-  pip install -r requirements.txt
+  pip install -r plugin_scripts/requirements.lock
 fi
 
 if [[ "${INSTALL_TEST_REQUIREMENTS}" == "true"  ]]; then
   echo "Installing test requirements"
   pip install -r requirements-test.txt
-fi
+fi

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2021-12-16
+
+### Updated
+
+Docker image for pipeline
+Use locked dependencies for pipeline
+
 ## [0.3.0] - 2021-12-05
 
 ### Added

docker/devbox.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.1-buster
+FROM python:3.10.1-slim
 
 ARG _USER="lilchz"
 ARG _UID="1001"

hooks/command

@@ -1,6 +1,29 @@
 #!/bin/bash
 set -euo pipefail
 
+# retry <number-of-retries> <command>
+function retry {
+	local retries=$1
+	shift
+	local attempts=1
+	local status=0
+
+	until "$@"; do
+		status=$?
+		echo "Exited with $status"
+		if ((retries == "0")); then
+			return $status
+		elif ((attempts == retries)); then
+			echo "Failed $attempts retries"
+			return $status
+		else
+			echo "Retrying $((retries - attempts)) more times..."
+			attempts=$((attempts + 1))
+			sleep $(((attempts - 2) * 2))
+		fi
+	done
+}
+
 PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
 
 rm -rf /tmp/service_account.json
@@ -14,7 +37,7 @@ fi
 
 workdir="/workdir"
 
-default_image="python:3.10.0-buster"
+default_image="python:3.10.1-slim"
 image="${BUILDKITE_PLUGIN_BIGQUERY_CUSTOM_IMAGE:-$default_image}"
 
 if [ -z "$BUILDKITE_PLUGIN_BIGQUERY_GCP_PROJECT" ]; then
@@ -98,6 +121,14 @@ command_string="$(
 args+=("${command_string}")
 display_command+=("'${command_string}'")
 
+echo "--- :docker: Pulling ${image}"
+if ! retry "${BUILDKITE_PLUGIN_CLOUD_FUNCTIONS_DOCKER_PULL_RETRIES:-3}" \
+	docker pull "${image}"; then
+	rv=$?
+	echo "--- :docker: Pull failed."
+	exit $rv
+fi
+
 echo '--- :docker: Logging "docker create" command'
 echo "$ docker create" >&2
 

plugin_scripts/requirements.lock

@@ -1,15 +1,15 @@
 # THIS IS AN AUTOGENERATED LOCKFILE. DO NOT EDIT MANUALLY.
 cachetools==4.2.4
 certifi==2021.10.8
-charset-normalizer==2.0.8
+charset-normalizer==2.0.9
 gbq==0.4.0
-google-api-core==2.2.2
+google-api-core==2.3.2
 google-auth==2.3.3
 google-cloud-bigquery==2.31.0
 google-cloud-core==2.2.1
 google-crc32c==1.3.0
 google-resumable-media==2.1.0
-googleapis-common-protos==1.53.0
+googleapis-common-protos==1.54.0
 grpcio==1.42.0
 grpcio-status==1.42.0
 idna==3.3
@@ -24,7 +24,7 @@ pyparsing==3.0.6
 python-dateutil==2.8.2
 requests==2.26.0
 rsa==4.8
-setuptools==59.4.0
+setuptools==59.6.0
 six==1.16.0
 typing_extensions==4.0.1
 urllib3==1.26.7

requirements.txt

@@ -2,7 +2,7 @@ cachetools==4.2.4
 certifi==2021.10.8
 charset-normalizer==2.0.9
 gbq==0.4.0
-google-api-core==2.3.1
+google-api-core==2.3.2
 google-auth==2.3.3
 google-cloud-bigquery==2.31.0
 google-cloud-core==2.2.1