Skip to content

Commit

Permalink
Link to migration bundles
Browse files Browse the repository at this point in the history
Any task bundles that have migrations are linked by the newer task
bundles by a dedicated annotation. See the following example task
bundles:

  tb0 -->  tb1 --> tb2 --> tb3 --> tb4 --> tb5
           |               |
           M               M

where, tb2 and tb3 point to tb1, and tb4 and tb5 point to tb3.

Linking task bundles helps to reduce the number of HTTP requests made to
Quay.io that query which task bundles have migrations.

Terms:

- tb means task bundle.
- M means migration.
  • Loading branch information
tkdchen committed Jan 27, 2025
1 parent 84b6cae commit 110af20
Showing 1 changed file with 39 additions and 2 deletions.
41 changes: 39 additions & 2 deletions hack/build-and-push.sh
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ build_push_task() {
local -r task_bundle=$3
local -r task_file_sha=$4
local -r has_migration=$5
local -r prev_bundle_digest=$6

local -r task_description=$(yq e '.spec.description' "$prepared_task_file" | head -n 1)

Expand All @@ -253,6 +254,8 @@ build_push_task() {
ANNOTATIONS+=("dev.konflux-ci.task.migration=true")
fi

ANNOTATIONS+=("dev.konflux-ci.task/previous-migration-bundle=${prev_bundle_digest}")

local -a ANNOTATION_FLAGS=()
for annotation in "${ANNOTATIONS[@]}"; do
ANNOTATION_FLAGS+=("--annotate" "$(escape_tkn_bundle_arg "$annotation")")
Expand Down Expand Up @@ -431,6 +434,34 @@ attach_migration_file() {
return 0
}

# Find previous bundle that has a migration and output its digest.
find_previous_migration_bundle_digest() {
local -r task_name=${1:?Missing task name}
local -r task_version=${2:?Missing task version}
local -r repo=${TEST_REPO_NAME:-task-${task_name}}

local -r url_list_recent_tags="https://quay.io/api/v1/repository/${QUAY_NAMESPACE}/${repo}/tag/?onlyActiveTags=true&limit=30"
local -r expr_filter_tags=".tags[] | select(.name | test(\"^(${task_name}-)?${task_version}-[0-9a-f]+\$\")) | .manifest_digest"

local prev_bundle_digest has_migration

curl --fail -sL "$url_list_recent_tags" | jq -r "$expr_filter_tags" | \
while read -r manifest_digest; do
curl --fail -sL -o /tmp/manifest.json \
"https://quay.io/v2/${QUAY_NAMESPACE}/${repo}/manifests/${manifest_digest}"
prev_bundle_digest=$(jq -r '.annotations."dev.konflux-ci.task/previous-migration-bundle"' </tmp/manifest.json)
has_migration=$(jq -r '.annotations."dev.konflux-ci.task.migration"' </tmp/manifest.json)
if [ "$has_migration" == "true" ]; then
echo "$manifest_digest"
return
fi
if [ "$prev_bundle_digest" == "null" ] || [ -z "$prev_bundle_digest" ]; then
echo
return
fi
done
}

build_push_tasks() {
local build_data
local task_bundle_with_digest
Expand All @@ -440,6 +471,7 @@ build_push_tasks() {
local task_bundle
local output
local has_migration
local prev_bundle_digest

find task/*/* -maxdepth 0 -type d | awk -F '/' '{ print $0, $2, $3 }' | \
while read -r task_dir task_name task_version
Expand Down Expand Up @@ -479,9 +511,15 @@ build_push_tasks() {
task_bundle_with_digest=${task_bundle}@${digest}
echo "info: use existing $task_bundle_with_digest" 1>&2
else
echo "info: finding the previous task bundle that has a migration" 1>&2
prev_bundle_digest=$(find_previous_migration_bundle_digest "$task_name" "$task_version")

echo "info: push new bundle $task_bundle" 1>&2

output=$(build_push_task "$task_dir" "$prepared_task_file" "$task_bundle" "$task_file_sha" "$has_migration")
output=$(
build_push_task "$task_dir" "$prepared_task_file" "$task_bundle" "$task_file_sha" \
"$has_migration" "$prev_bundle_digest"
)

task_bundle_with_digest=$(grep -m 1 "^Pushed Tekton Bundle to" <<<"$output" 2>/dev/null)
task_bundle_with_digest=${task_bundle_with_digest##* }
Expand All @@ -501,7 +539,6 @@ build_push_tasks() {

build_push_tasks


# Used for build-definitions pull request CI only
if [ -n "$ENABLE_SOURCE_BUILD" ]; then
for pipeline_yaml in "$GENERATED_PIPELINES_DIR"/*.yaml; do
Expand Down

0 comments on commit 110af20

Please sign in to comment.