ci: Adding S3 storage

.github/actions/bitbake-build/action.yml

@@ -1,6 +1,10 @@
 inputs:
+  bitbake_buildname:
+    default: 'build'
   bitbake_prefix:
     default: ''
+  bitbake_machine:
+    default: 'raspberrypi5'
   bitbake_target:
     default: 'core-image-weston-wpe'
   bitbake_source:
@@ -15,27 +19,55 @@ runs:
       run:  |
         echo "Environment:"
         echo "======================================================================"
+        echo "BITBAKE_BUILDNAME=${{ inputs.bitbake_buildname }}"
+        echo "BITBAKE_MACHINE=${{ inputs.bitbake_machine }}"
         echo "BITBAKE_PREFIX=${{ inputs.bitbake_prefix }}"
         echo "BITBAKE_SOURCE=${{ inputs.bitbake_source }}"
         echo "BITBAKE_TARGET=${{ inputs.bitbake_target }}"
         echo "RELEASE=${{ inputs.repo_release }}"
         echo "======================================================================"
+    - name: Generate timestamped prefix
+      id: timestamp
+      shell: bash
+      run: |
+        TIMESTAMP=$(date +"%Y%m%d%H%M")
+        echo "prefix=${TIMESTAMP}" >> $GITHUB_OUTPUT
     - name: Build
       shell: bash
       run:  |
         cd ~/yocto-meta-wpe-image-${{ inputs.repo_release }}
-        rm -rf tmp
         ulimit -n 4096
-        #source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_source }}
-        source setup-environment ${{ inputs.bitbake_source }}
+        source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_buildname}} ${{ inputs.bitbake_machine }} ${{ inputs.bitbake_source }}
+        MAX_RETRIES=5
+        export ${{ inputs.bitbake_prefix }}
+        for i in $(seq 1 $MAX_RETRIES); do
+            echo "Attempt $i to fetch sources for $IMAGE..."
+            if bitbake ${{ inputs.bitbake_target }} --runall=fetch; then
+                echo "All sources fetched successfully."
+                break
+            fi
+
+            if [ $i -lt $MAX_RETRIES ]; then
+                echo "Retrying fetch process..."
+            else
+                echo "Max retries reached. Some sources may still be missing."
+            fi
+        done
         ${{ inputs.bitbake_prefix }} bitbake ${{ inputs.bitbake_target }}
     - name: Artifacts
       if: always()
       shell: bash
-      run:  |
-        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp/deploy/images/*/
+      run: |
+        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}/
+        PREFIX=${{ steps.timestamp.outputs.prefix }}
+        s3cmd put -F ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.rootfs.wic.bmap \
+            s3://yocto/${PREFIX}-${{ inputs.bitbake_buildname }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.wic.bmap
+        s3cmd put -F ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.rootfs.wic.bz2 \
+            s3://yocto/${PREFIX}-${{ inputs.bitbake_buildname }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.wic.bz2
+        s3cmd put -F ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp/deploy/images/${{ inputs.bitbake_machine }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.rootfs-dbg.tar.bz2 \
+            s3://yocto/${PREFIX}-${{ inputs.bitbake_buildname }}/${{ inputs.bitbake_target }}-${{ inputs.bitbake_machine }}.dbg.tar.bz2
     - name: Clean the tmp dir
       if: always()
       shell: bash
       run:  |
-        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp
+        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/${{ inputs.bitbake_buildname }}/tmp

.github/actions/s3-configure/action.yml

@@ -0,0 +1,28 @@
+inputs:
+  access_key:
+    description: "S3 Access Key"
+    required: true
+  secret_key:
+    description: "S3 Secret Key"
+    required: true
+  host_base:
+    description: "S3 Host base"
+    required: false
+    default: "ams3.digitaloceanspaces.com"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure s3cmd
+      shell: bash
+      run: |
+        cat > ~/.s3cfg <<EOL
+        [default]
+        access_key = ${{ inputs.access_key }}
+        secret_key = ${{ inputs.secret_key }}
+        host_base = ${{ inputs.host_base }}
+        host_bucket = %(bucket)s.${{ inputs.host_base }}
+        use_https = True
+        gpg_command = /usr/bin/gpg
+        EOL
+

.github/workflows/test-build.yml

@@ -6,64 +6,73 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/s3-configure
+        env:
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+          S3_HOST_BASE: ${{ secrets.S3_HOST_BASE }}
+        with:
+          access_key: ${S3_ACCESS_KEY}
+          secret_key: ${S3_SECRET_KEY}
+          host_base: ${S3_HOST_BASE}
       - uses: ./.github/actions/bitbake-repo
         with:
           repo_release: 'scarthgap'
 
-  matrix-build-image:
+  matrix-build-image-raspberrypi-stable:
+    strategy:
+      fail-fast: false
+      matrix:
+        machine: ['raspberrypi3-mesa', 'raspberrypi4-64-mesa', 'raspberrypi5']
+        wpe_vers: ['2_46']
+        yocto_rel: ['scarthgap']
+    runs-on: [self-hosted, builder]
+    if: "!contains(github.event.head_commit.message, 'ci skip')"
+    needs: scarthgap-repo
+    steps:
+      - uses: ./.github/actions/bitbake-build
+        with:
+          bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
+          bitbake_machine: ${{ matrix.machine }}
+          bitbake_source: 'poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
+          repo_release: ${{ matrix.yocto_rel }}
+
+
+  matrix-build-image-raspberrypi-nightly:
+    strategy:
+      fail-fast: false
+      matrix:
+        machine: ['raspberrypi3-mesa', 'raspberrypi4-64-mesa', 'raspberrypi5']
+        wpe_vers: ['nightly']
+        yocto_rel: ['scarthgap']
+    continue-on-error: true
+    runs-on: [self-hosted, builder]
+    if: "!contains(github.event.head_commit.message, 'ci skip')"
+    needs: scarthgap-repo
+    steps:
+      - uses: ./.github/actions/bitbake-build
+        with:
+          bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
+          bitbake_machine: ${{ matrix.machine }}
+          bitbake_source: 'poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
+          repo_release: ${{ matrix.yocto_rel }}
+
+  matrix-build-image-freescale:
     strategy:
       fail-fast: false
       matrix:
-        machine: ['raspberrypi3-mesa', 'raspberrypi4-64-mesa', 'raspberrypi5', 'wandboard-mesa']
+        machine: ['wandboard-mesa']
         wpe_vers: ['2_46', 'nightly']
         yocto_rel: ['scarthgap']
     continue-on-error: true
     runs-on: [self-hosted, builder]
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     needs: scarthgap-repo
     steps:
-      #- uses: actions/checkout@v3
       - uses: ./.github/actions/bitbake-build
         with:
-          bitbake_source: 'raspberrypi3-mesa-wpe-${{ matrix.wpe_vers }} ${{ matrix.machine }} poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
+          bitbake_buildname: wpe-${{ matrix.wpe_vers }}-${{ matrix.machine }}
+          bitbake_machine: ${{ matrix.machine }}
+          bitbake_source: 'poky-wayland layers.freescale conf.wpe-${{ matrix.wpe_vers }} --update-config'
           repo_release: ${{ matrix.yocto_rel }}
 
-  # scarthgap-raspberrypi3-mesa-weston-wpe-nightly:
-  #   runs-on: self-hosted
-  #   if: "!contains(github.event.head_commit.message, 'ci skip')"
-  #   steps:
-  #     - uses: ./.github/actions/bitbake-build
-  #       with:
-  #         bitbake_source: 'rpi3-mesa-wpe-nightly raspberrypi3-mesa poky-wayland layers.raspberrypi conf.wpe-nightly --update-config'
-  #         repo_release: 'scarthgap'
-  #   needs: scarthgap-repo
-  #
-  # scarthgap-raspberrypi4-64-mesa-weston-wpe-nightly:
-  #   runs-on: self-hosted
-  #   if: "!contains(github.event.head_commit.message, 'ci skip')"
-  #   steps:
-  #     - uses: ./.github/actions/bitbake-build
-  #       with:
-  #         bitbake_source: 'rpi4-mesa-wpe-nightly raspberrypi4-64-mesa poky-wayland layers.raspberrypi conf.wpe-nightly --update-config'
-  #         repo_release: 'scarthgap'
-  #   needs: scarthgap-repo
-  #
-  # scarthgap-raspberrypi3-mesa-weston-wpe-2_38:
-  #   runs-on: self-hosted
-  #   if: "!contains(github.event.head_commit.message, 'ci skip')"
-  #   steps:
-  #     - uses: ./.github/actions/bitbake-build
-  #       with:
-  #         bitbake_source: 'rpi3-mesa-wpe-2_38 raspberrypi3-mesa poky-wayland layers.raspberrypi conf.wpe-2_38 --update-config'
-  #         repo_release: 'scarthgap'
-  #   needs: scarthgap-repo
-  #
-  # scarthgap-raspberrypi4-64-mesa-weston-wpe-2_38:
-  #   runs-on: self-hosted
-  #   if: "!contains(github.event.head_commit.message, 'ci skip')"
-  #   steps:
-  #     - uses: ./.github/actions/bitbake-build
-  #       with:
-  #         bitbake_source: 'rpi4-mesa-wpe-2_38 raspberrypi4-64-mesa poky-wayland layers.raspberrypi conf.wpe-2_38 --update-config'
-  #         repo_release: 'scarthgap'
-  #   needs: scarthgap-repo