From 4abe6d975a0623a5d8944d0a929db8f2c6653555 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 20:43:17 -0700
Subject: [PATCH 01/49] test

---
 .github/workflows/jenkins_test.yaml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 .github/workflows/jenkins_test.yaml

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
new file mode 100644
index 00000000000000..3bd2074e6a18aa
--- /dev/null
+++ b/.github/workflows/jenkins_test.yaml
@@ -0,0 +1,21 @@
+name: Get Commit Status on PR
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  get-status:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Commit Status
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const response = await github.rest.repos.getCommitStatus({
+              owner: 'commaai',
+              repo: 'openpilot',
+              ref: 'heads/ui_screen_sleep'
+            });
+
+            console.log(response.data);

From 3a38ab3cffc9d6b62b74eb186c0999bd5fe2f58c Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 20:45:25 -0700
Subject: [PATCH 02/49] fix

---
 .github/workflows/jenkins_test.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 3bd2074e6a18aa..f9836962f2414e 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -12,10 +12,13 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
-            const response = await github.rest.repos.getCommitStatus({
+            const response = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
               owner: 'commaai',
               repo: 'openpilot',
-              ref: 'heads/ui_screen_sleep'
+              ref: 'heads/master',
+              headers: {
+                'Accept': 'application/vnd.github+json',
+                'X-GitHub-Api-Version': '2022-11-28'
+              }
             });
-
             console.log(response.data);

From 9fcc21e220b3acb9cb591899c917ab7856c84fd7 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 20:47:41 -0700
Subject: [PATCH 03/49] test

---
 .github/workflows/jenkins_test.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index f9836962f2414e..500006279d5e9d 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -21,4 +21,5 @@ jobs:
                 'X-GitHub-Api-Version': '2022-11-28'
               }
             });
-            console.log(response.data);
+            console.log(response.data.statuses[0].context);
+            console.log(response.data.statuses[0].state);

From dafa0c657d064c8365aae0ff34e5fb7a9506102a Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 20:52:49 -0700
Subject: [PATCH 04/49] test

---
 .github/workflows/jenkins_test.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 500006279d5e9d..8a84d7b3ff5c9e 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -23,3 +23,15 @@ jobs:
             });
             console.log(response.data.statuses[0].context);
             console.log(response.data.statuses[0].state);
+
+      - name: Push to tmp-jenkins branch
+        uses: actions/checkout@v4
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b jenkins_run-${{ github.event.issue.number }}
+          echo "TEST JENKINS 1" >> test_file
+          git add test_file
+          git commit -m "test run 1"
+          GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_run-${{ github.event.issue.number }}
+

From 69b905c036595dad6cdf2a5a3b484b410f3af745 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 20:54:20 -0700
Subject: [PATCH 05/49] fix

---
 .github/workflows/jenkins_test.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 8a84d7b3ff5c9e..3ddc272c054b98 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -24,8 +24,10 @@ jobs:
             console.log(response.data.statuses[0].context);
             console.log(response.data.statuses[0].state);
 
-      - name: Push to tmp-jenkins branch
+      - name: get file
         uses: actions/checkout@v4
+
+      - name: Push to tmp-jenkins branch
         run: |
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"

From 011b8cbd609dd53bc5eaddfd0b24ef85cdd30eb7 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:04:44 -0700
Subject: [PATCH 06/49] test

---
 .github/workflows/jenkins_test.yaml | 35 ++++++++++++++++-------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 3ddc272c054b98..47dde88fc61adc 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -5,9 +5,25 @@ on:
     types: [opened, synchronize]
 
 jobs:
-  get-status:
+  test-jenkins:
+    strategy:
+      matrix:
+        run_id: [1, 2]
     runs-on: ubuntu-latest
     steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Push to tmp-jenkins branch
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b __jenkins_test_run_${{ matrix.run_id }}
+          echo "TEST JENKINS ${{ matrix.run_id }}" >> test_file
+          git add test_file
+          git commit -m "test run ${{ matrix.run_id }}"
+          GIT_LFS_SKIP_PUSH=1 git push -f origin __jenkins_test_run_${{ matrix.run_id }}
+
       - name: Get Commit Status
         uses: actions/github-script@v6
         with:
@@ -15,7 +31,7 @@ jobs:
             const response = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
               owner: 'commaai',
               repo: 'openpilot',
-              ref: 'heads/master',
+              ref: 'heads/__jenkins_test_run_${{ matrix.run_id }}',
               headers: {
                 'Accept': 'application/vnd.github+json',
                 'X-GitHub-Api-Version': '2022-11-28'
@@ -23,17 +39,4 @@ jobs:
             });
             console.log(response.data.statuses[0].context);
             console.log(response.data.statuses[0].state);
-
-      - name: get file
-        uses: actions/checkout@v4
-
-      - name: Push to tmp-jenkins branch
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git checkout -b jenkins_run-${{ github.event.issue.number }}
-          echo "TEST JENKINS 1" >> test_file
-          git add test_file
-          git commit -m "test run 1"
-          GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_run-${{ github.event.issue.number }}
-
+            console.log(response.data.statuses[0].target_url);

From 18b44bbf6f9408ab3a5bd3f228476ffccaeade5d Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:14:35 -0700
Subject: [PATCH 07/49] test

---
 .github/workflows/jenkins_test.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 47dde88fc61adc..a603281498f214 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -37,6 +37,7 @@ jobs:
                 'X-GitHub-Api-Version': '2022-11-28'
               }
             });
-            console.log(response.data.statuses[0].context);
-            console.log(response.data.statuses[0].state);
-            console.log(response.data.statuses[0].target_url);
+            console.log(response);
+            setInterval(() => {
+              console.log('in the loop');
+            }, 5000);

From e4334d08153c27ae205726551d11f93a67eb5559 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:22:11 -0700
Subject: [PATCH 08/49] test

---
 .github/workflows/jenkins_test.yaml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index a603281498f214..bc8f7425cdde60 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -28,6 +28,10 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
+
+            let prev_status = false;
+            setInterval(() => {
+
             const response = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
               owner: 'commaai',
               repo: 'openpilot',
@@ -37,7 +41,13 @@ jobs:
                 'X-GitHub-Api-Version': '2022-11-28'
               }
             });
-            console.log(response);
-            setInterval(() => {
+
+            if ((response.data.statuses === undefined || response.data.statuses == 0) && !prev_status) {
+              console.log('NOT STARTED! exit loop');
+            } else {
+              prev_status = true;
               console.log('in the loop');
-            }, 5000);
+              console.log(response.data.statuses);
+            }
+
+            }, 10000);

From 86c890aec2b991aa71f9fd626b6b23c4930fd4aa Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:23:40 -0700
Subject: [PATCH 09/49] test

---
 .github/workflows/jenkins_test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index bc8f7425cdde60..8acb7815fad117 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -32,7 +32,7 @@ jobs:
             let prev_status = false;
             setInterval(() => {
 
-            const response = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
+            const response = github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
               owner: 'commaai',
               repo: 'openpilot',
               ref: 'heads/__jenkins_test_run_${{ matrix.run_id }}',

From 816c08c3b27ed47d568524ea420431cad2769e0d Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:26:05 -0700
Subject: [PATCH 10/49] test

---
 .github/workflows/jenkins_test.yaml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 8acb7815fad117..a0d0b3b3d08918 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -42,12 +42,7 @@ jobs:
               }
             });
 
-            if ((response.data.statuses === undefined || response.data.statuses == 0) && !prev_status) {
-              console.log('NOT STARTED! exit loop');
-            } else {
-              prev_status = true;
-              console.log('in the loop');
-              console.log(response.data.statuses);
-            }
+            console.log('in the loop');
+            console.log(response.data.statuses);
 
             }, 10000);

From 7cc5ee97479cbb45dd4f4744198d31783fea0b90 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:27:24 -0700
Subject: [PATCH 11/49] test

---
 .github/workflows/jenkins_test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index a0d0b3b3d08918..e2305313b5b50d 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -43,6 +43,6 @@ jobs:
             });
 
             console.log('in the loop');
-            console.log(response.data.statuses);
+            console.log(response);
 
             }, 10000);

From 9d0a9af9b4149807539dc6ac94dfd26fadb051f9 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:29:57 -0700
Subject: [PATCH 12/49] test

---
 .github/workflows/jenkins_test.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index e2305313b5b50d..1ab4c58293e825 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -42,7 +42,11 @@ jobs:
               }
             });
 
+            response.then(function(r) {
+              console.log('in promise');
+              console.log(r);
+            });
+
             console.log('in the loop');
-            console.log(response);
 
             }, 10000);

From 89e5a3c9135c9e418f82f9353f6f5daf534e0362 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 9 Oct 2024 21:34:53 -0700
Subject: [PATCH 13/49] test

---
 .github/workflows/jenkins_test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
index 1ab4c58293e825..9eb027f429c668 100644
--- a/.github/workflows/jenkins_test.yaml
+++ b/.github/workflows/jenkins_test.yaml
@@ -44,7 +44,7 @@ jobs:
 
             response.then(function(r) {
               console.log('in promise');
-              console.log(r);
+              console.log(r.data.statuses);
             });
 
             console.log('in the loop');

From a1947725c1286ba8d12421ceb09a343d750ea879 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 14:57:29 -0700
Subject: [PATCH 14/49] test

---
 .github/workflows/jenkins_test.yaml | 52 -----------------------------
 Jenkinsfile                         | 12 +++++++
 2 files changed, 12 insertions(+), 52 deletions(-)
 delete mode 100644 .github/workflows/jenkins_test.yaml

diff --git a/.github/workflows/jenkins_test.yaml b/.github/workflows/jenkins_test.yaml
deleted file mode 100644
index 9eb027f429c668..00000000000000
--- a/.github/workflows/jenkins_test.yaml
+++ /dev/null
@@ -1,52 +0,0 @@
-name: Get Commit Status on PR
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-jobs:
-  test-jenkins:
-    strategy:
-      matrix:
-        run_id: [1, 2]
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-
-      - name: Push to tmp-jenkins branch
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git checkout -b __jenkins_test_run_${{ matrix.run_id }}
-          echo "TEST JENKINS ${{ matrix.run_id }}" >> test_file
-          git add test_file
-          git commit -m "test run ${{ matrix.run_id }}"
-          GIT_LFS_SKIP_PUSH=1 git push -f origin __jenkins_test_run_${{ matrix.run_id }}
-
-      - name: Get Commit Status
-        uses: actions/github-script@v6
-        with:
-          script: |
-
-            let prev_status = false;
-            setInterval(() => {
-
-            const response = github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
-              owner: 'commaai',
-              repo: 'openpilot',
-              ref: 'heads/__jenkins_test_run_${{ matrix.run_id }}',
-              headers: {
-                'Accept': 'application/vnd.github+json',
-                'X-GitHub-Api-Version': '2022-11-28'
-              }
-            });
-
-            response.then(function(r) {
-              console.log('in promise');
-              console.log(r.data.statuses);
-            });
-
-            console.log('in the loop');
-
-            }, 10000);
diff --git a/Jenkinsfile b/Jenkinsfile
index a98280ccd9476f..13d26136312079 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -170,6 +170,18 @@ node {
     ])
   }
 
+  if (env.BRANCH_NAME == 'jenkins_stress') {
+    sh '''#!/bin/bash
+
+          echo "test"
+          COOKIE_JAR=/tmp/cookies
+          CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+          curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
+      '''
+      currentBuild.result = 'SUCCESS'
+      return
+  }
+
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release3-staging", "tici-needs-can", [], [

From 8f5094854f35dde31c3607b108bd7dca06610e82 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 15:22:15 -0700
Subject: [PATCH 15/49] test this

---
 Jenkinsfile | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 13d26136312079..b4d08182278047 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -164,7 +164,7 @@ node {
                          'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master') {
+  if (env.BRANCH_NAME != 'master' || env.BRANCH_NAME != '__jenkins_stress') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
@@ -173,10 +173,29 @@ node {
   if (env.BRANCH_NAME == 'jenkins_stress') {
     sh '''#!/bin/bash
 
-          echo "test"
           COOKIE_JAR=/tmp/cookies
           CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-          curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
+          for i in $(seq 1 3);
+          do
+            curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
+          done
+
+          while true; do
+            count=0
+            for i in $(seq 1 3);
+            do
+              RES=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json | jq .result)
+              echo $RES
+              if [[ $RES == "FAILURE" ]]; then
+                ((count++))
+              fi
+            done
+            if [[ count -eq 3 ]]; then
+              echo "ALL FAIL"
+              break
+            fi
+            sleep 5
+          done
       '''
       currentBuild.result = 'SUCCESS'
       return

From 9ff576c7c0e39c789a13cc1d2088ada9533004bc Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 15:37:51 -0700
Subject: [PATCH 16/49] test

---
 Jenkinsfile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b4d08182278047..2a4adecbe1dbe7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -184,11 +184,9 @@ node {
             count=0
             for i in $(seq 1 3);
             do
-              RES=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json | jq .result)
+              RES=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
               echo $RES
-              if [[ $RES == "FAILURE" ]]; then
-                ((count++))
-              fi
+              ((count++))
             done
             if [[ count -eq 3 ]]; then
               echo "ALL FAIL"

From 869f3ed2398dbdc8e471077a69b8cdf142e54429 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 15:59:16 -0700
Subject: [PATCH 17/49] fix

---
 Jenkinsfile | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2a4adecbe1dbe7..b2b178c8b1a570 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -175,6 +175,10 @@ node {
 
           COOKIE_JAR=/tmp/cookies
           CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+          # delete previous build
+          curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/[1-3]/doDelete
+
           for i in $(seq 1 3);
           do
             curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
@@ -184,12 +188,18 @@ node {
             count=0
             for i in $(seq 1 3);
             do
-              RES=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
-              echo $RES
+              RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
+              HTTP_CODE=$(tail -n1 <<< "$RES")
+              JSON=$(sed '$ d' <<< "$RES")
+              if [[ $HTTP_CODE == "200" ]]; then
+                echo $JSON
+              else
+                echo "Error getting build $i"
+              fi
               ((count++))
             done
             if [[ count -eq 3 ]]; then
-              echo "ALL FAIL"
+              echo "ALL DONE"
               break
             fi
             sleep 5

From 35e16c9d941eea4c9a96f2ba1319c312a5affb6f Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 16:29:43 -0700
Subject: [PATCH 18/49] better

---
 Jenkinsfile | 46 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b2b178c8b1a570..27a6b7fe995a11 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -173,38 +173,66 @@ node {
   if (env.BRANCH_NAME == 'jenkins_stress') {
     sh '''#!/bin/bash
 
+          # get crumb for CSRF
           COOKIE_JAR=/tmp/cookies
           CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
 
-          # delete previous build
-          curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/[1-3]/doDelete
+          N=5
+          FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber)
+          LAST_RUN=$((FIRST_RUN+N))
 
-          for i in $(seq 1 3);
+          for i in $(seq $FIRST_RUN $LAST_RUN);
           do
+            # start build i
             curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
           done
 
           while true; do
+            sleep 5
+
             count=0
-            for i in $(seq 1 3);
+            FAIL=()
+            PASS=()
+
+            for i in $(seq $FIRST_RUN $LAST_RUN);
             do
               RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
               HTTP_CODE=$(tail -n1 <<< "$RES")
               JSON=$(sed '$ d' <<< "$RES")
+
               if [[ $HTTP_CODE == "200" ]]; then
-                echo $JSON
+                STILL_RUNNING=$(echo $JSON | jq .inProgress)
+                if [[ $STILL_RUNNING == "true" ]]; then
+                  continue
+                fi
+
+                RESULT=$(echo $JSON | jq .result)
+                ((count++))
+
+                if [[ $RESULT == "SUCCESS" ]]; then
+                  PASS+=($i)
+                elif [[ $RESULT == "FAILURE" ]]; then
+                  FAIL+=($i)
+                elif [[ $RESULT == "ABORTED" ]]; then
+                  FAIL+=($i)
+                else
+                  FAIL+=($i)
+                fi
+
               else
                 echo "Error getting build $i"
               fi
-              ((count++))
+
             done
-            if [[ count -eq 3 ]]; then
-              echo "ALL DONE"
+
+            if [[ $count -eq $N ]]; then
+              echo "DONE"
               break
             fi
-            sleep 5
+
           done
       '''
+
       currentBuild.result = 'SUCCESS'
       return
   }

From 2a384f3a1083024f5cb507de10d0005c6bbfc7d8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 17:02:49 -0700
Subject: [PATCH 19/49] test

---
 Jenkinsfile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 27a6b7fe995a11..be808c91f8daf5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -164,7 +164,7 @@ node {
                          'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master' || env.BRANCH_NAME != '__jenkins_stress') {
+  if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != '__jenkins_stress') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
@@ -179,6 +179,7 @@ node {
 
           N=5
           FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber)
+          echo "FIRST RUN IS : $FIRST_RUN"
           LAST_RUN=$((FIRST_RUN+N))
 
           for i in $(seq $FIRST_RUN $LAST_RUN);
@@ -203,6 +204,7 @@ node {
               if [[ $HTTP_CODE == "200" ]]; then
                 STILL_RUNNING=$(echo $JSON | jq .inProgress)
                 if [[ $STILL_RUNNING == "true" ]]; then
+                  echo "build $i still running"
                   continue
                 fi
 
@@ -210,10 +212,13 @@ node {
                 ((count++))
 
                 if [[ $RESULT == "SUCCESS" ]]; then
+                  echo "build $i success"
                   PASS+=($i)
                 elif [[ $RESULT == "FAILURE" ]]; then
+                  echo "build $i fail"
                   FAIL+=($i)
                 elif [[ $RESULT == "ABORTED" ]]; then
+                  echo "build $i abort"
                   FAIL+=($i)
                 else
                   FAIL+=($i)

From b82783566c4735590b7aaaeb4a92d41f3f143f8e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 10 Oct 2024 17:13:49 -0700
Subject: [PATCH 20/49] better

---
 Jenkinsfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index be808c91f8daf5..8ae92e083be0e3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -211,16 +211,17 @@ node {
                 RESULT=$(echo $JSON | jq .result)
                 ((count++))
 
-                if [[ $RESULT == "SUCCESS" ]]; then
+                if [[ $RESULT == '"SUCCESS"' ]]; then
                   echo "build $i success"
                   PASS+=($i)
-                elif [[ $RESULT == "FAILURE" ]]; then
+                elif [[ $RESULT == '"FAILURE"' ]]; then
                   echo "build $i fail"
                   FAIL+=($i)
-                elif [[ $RESULT == "ABORTED" ]]; then
+                elif [[ $RESULT == '"ABORTED"' ]]; then
                   echo "build $i abort"
                   FAIL+=($i)
                 else
+                  echo "build $i $RESULT"
                   FAIL+=($i)
                 fi
 
@@ -231,7 +232,8 @@ node {
             done
 
             if [[ $count -eq $N ]]; then
-              echo "DONE"
+              echo "FAIL : ${FAIL[@]}"
+              echo "PASS : ${PASS[@]}"
               break
             fi
 

From 4ad1d7a776a559ec74c1d7c13e7f9805d4fa1c2d Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 20:50:05 -0700
Subject: [PATCH 21/49] better

---
 .github/workflows/ci_weekly_report.yaml       | 14 ++++-
 ...yaml => ci_weekly_run_github_actions.yaml} |  0
 .github/workflows/ci_weekly_run_jenkins.yaml  | 59 +++++++++++++++++++
 3 files changed, 70 insertions(+), 3 deletions(-)
 rename .github/workflows/{ci_weekly_run.yaml => ci_weekly_run_github_actions.yaml} (100%)
 create mode 100644 .github/workflows/ci_weekly_run_jenkins.yaml

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index 9821283cb57442..372b996f81d0e2 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -26,17 +26,23 @@ jobs:
           matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
           echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
-  ci_matrix_run:
+  ci_matrix_jenkins_run:
+    needs: [ setup ]
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run_jenkins.yaml@master
+    with:
+      build_count: ${{ env.CI_RUNS }}
+
+  ci_matrix_github_actions_run:
     needs: [ setup ]
     strategy:
       fail-fast: false
       matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run_github_actions.yaml@master
     with:
       run_number: ${{ matrix.run_number }}
 
   report:
-    needs: [ci_matrix_run]
+    needs: [ci_matrix_jenkins_run, ci_matrix_github_actions_run]
     runs-on: ubuntu-latest
     if: always()
     steps:
@@ -75,6 +81,7 @@ jobs:
           JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
         run: |
           cat <<EOF >> template.html
+          ## Github Actions CI
           <table>
             <thead>
               <tr>
@@ -99,3 +106,4 @@ jobs:
           echo $JOB_RESULTS | jinja2 template.html > report.html
           echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
           cat report.html >> $GITHUB_STEP_SUMMARY
+          curl https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/jenkins_test_report/jenkins_report >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/ci_weekly_run.yaml b/.github/workflows/ci_weekly_run_github_actions.yaml
similarity index 100%
rename from .github/workflows/ci_weekly_run.yaml
rename to .github/workflows/ci_weekly_run_github_actions.yaml
diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
new file mode 100644
index 00000000000000..95ffa8be34be44
--- /dev/null
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -0,0 +1,59 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      build_count:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.build_count }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  jenkins_run:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Create master/runner branches
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+        git checkout -b jenkins_test_master
+        echo "TEST JENKINS MASTER" >> test_jenkins
+        git add test_jenkins
+        git commit -m 'master jenkins'
+        GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_test_master
+
+        git checkout -b jenkins_test_runner
+        echo "TEST JENKINS RUNNER" >> test_jenkins
+        git add test_jenkins
+        git commit -m 'runner jenkins'
+        GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_test_runner
+
+    - name: Waiting on Jenkins runs
+      uses: actions/github-script@v6
+      with:
+        script: |
+
+          let prev_status = false;
+          const interval setInterval(() => {
+
+            const response = github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
+              owner: 'commaai',
+              repo: 'openpilot',
+              ref: 'heads/jenkins_test_master',
+              headers: {
+                'Accept': 'application/vnd.github+json',
+                'X-GitHub-Api-Version': '2022-11-28'
+              }
+            });
+
+            response.then(function(r) {
+              if (r.data.statuses[0].state != "pending") {
+                clearInterval(interval)
+              }
+            });
+
+          }, 60000);

From fde1ff23464b2aa6100eef03e05df1e540c271a8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 20:50:44 -0700
Subject: [PATCH 22/49] also this

---
 Jenkinsfile | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ae92e083be0e3..017b5ac74e21b6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -170,7 +170,7 @@ node {
     ])
   }
 
-  if (env.BRANCH_NAME == 'jenkins_stress') {
+  if (env.BRANCH_NAME == 'jenkins_stress_NOT_NOW') {
     sh '''#!/bin/bash
 
           # get crumb for CSRF
@@ -179,13 +179,12 @@ node {
 
           N=5
           FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber)
-          echo "FIRST RUN IS : $FIRST_RUN"
           LAST_RUN=$((FIRST_RUN+N))
 
           for i in $(seq $FIRST_RUN $LAST_RUN);
           do
             # start build i
-            curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
+            curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
           done
 
           while true; do
@@ -210,23 +209,20 @@ node {
 
                 RESULT=$(echo $JSON | jq .result)
                 ((count++))
+                echo "build $i $RESULT"
 
                 if [[ $RESULT == '"SUCCESS"' ]]; then
-                  echo "build $i success"
                   PASS+=($i)
                 elif [[ $RESULT == '"FAILURE"' ]]; then
-                  echo "build $i fail"
                   FAIL+=($i)
                 elif [[ $RESULT == '"ABORTED"' ]]; then
-                  echo "build $i abort"
                   FAIL+=($i)
                 else
-                  echo "build $i $RESULT"
                   FAIL+=($i)
                 fi
 
               else
-                echo "Error getting build $i"
+                echo "Error getting status of build $i"
               fi
 
             done

From 2093a1e6f568e92d4301310203f2e01a23cad06e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:14:42 -0700
Subject: [PATCH 23/49] all

---
 Jenkinsfile | 149 +++++++++++++++++++++++++++++++---------------------
 1 file changed, 88 insertions(+), 61 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 017b5ac74e21b6..963a8f134783d7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -172,68 +172,95 @@ node {
 
   if (env.BRANCH_NAME == 'jenkins_stress_NOT_NOW') {
     sh '''#!/bin/bash
-
-          # get crumb for CSRF
-          COOKIE_JAR=/tmp/cookies
-          CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-
-          N=5
-          FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber)
-          LAST_RUN=$((FIRST_RUN+N))
-
-          for i in $(seq $FIRST_RUN $LAST_RUN);
-          do
-            # start build i
-            curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
-          done
-
-          while true; do
-            sleep 5
-
-            count=0
-            FAIL=()
-            PASS=()
-
-            for i in $(seq $FIRST_RUN $LAST_RUN);
-            do
-              RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
-              HTTP_CODE=$(tail -n1 <<< "$RES")
-              JSON=$(sed '$ d' <<< "$RES")
-
-              if [[ $HTTP_CODE == "200" ]]; then
-                STILL_RUNNING=$(echo $JSON | jq .inProgress)
-                if [[ $STILL_RUNNING == "true" ]]; then
-                  echo "build $i still running"
-                  continue
-                fi
-
-                RESULT=$(echo $JSON | jq .result)
-                ((count++))
-                echo "build $i $RESULT"
-
-                if [[ $RESULT == '"SUCCESS"' ]]; then
-                  PASS+=($i)
-                elif [[ $RESULT == '"FAILURE"' ]]; then
-                  FAIL+=($i)
-                elif [[ $RESULT == '"ABORTED"' ]]; then
-                  FAIL+=($i)
-                else
-                  FAIL+=($i)
-                fi
-
-              else
-                echo "Error getting status of build $i"
-              fi
-
-            done
-
-            if [[ $count -eq $N ]]; then
-              echo "FAIL : ${FAIL[@]}"
-              echo "PASS : ${PASS[@]}"
-              break
+      # get crumb for CSRF
+      COOKIE_JAR=/tmp/cookies
+      CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+      N=5
+      FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/api/json | jq .nextBuildNumber)
+      LAST_RUN=$((FIRST_RUN+N))
+
+      for i in $(seq $FIRST_RUN $LAST_RUN);
+      do
+        # start build i
+        curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/build?delay=0sec
+      done
+
+      while true; do
+        sleep 60
+
+        count=0
+        for i in $(seq $FIRST_RUN $LAST_RUN);
+        do
+          RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/api/json)
+          HTTP_CODE=$(tail -n1 <<< "$RES")
+          JSON=$(sed '$ d' <<< "$RES")
+
+          if [[ $HTTP_CODE == "200" ]]; then
+            STILL_RUNNING=$(echo $JSON | jq .inProgress)
+            if [[ $STILL_RUNNING == "true" ]]; then
+              echo "build $i still running"
+              continue
             fi
-
-          done
+            ((count++))
+          else
+            echo "Error getting status of build $i"
+          fi
+        done
+
+        if [[ $count -eq $N ]]; then
+          break
+        fi
+      done
+
+
+      STAGES_NAMES=()
+      while read stage; do
+        STAGES_NAMES[$index]=$stage
+        ((index++))
+      done < <(curl -s -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/lastBuild/wfapi/ | jq .stages[].name)
+      STAGES_COUNT=${#STAGES_NAMES[@]}
+
+      STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
+      STAGES_FAILURES_LOGS=()
+
+      for i in $(seq $FIRST_RUN $LAST_RUN);
+      do
+
+      index=0
+      while read result; do
+        if [[ $result != '"SUCCESS"' ]]; then
+          STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1))
+          STAGES_FAILURES_LOGS[$index]="${STAGES_FAILURES_LOGS[$index]}<a href=\"https://jenkins.comma.life/blue/organizations/jenkins/openpilot/detail/jenkins_test_runner/$i/pipeline/\">Log for run #$(($i-$FIRST_RUN))</a><br>"
+        fi
+        ((index++))
+      done < <(curl https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/wfapi/ | jq .stages[].status)
+
+      done
+
+      TABLE="<table><thead><tr> <th>Stage</th> <th>✅ Passing</th> <th>❌ Failure Details</th> </tr></thead><tbody>"
+      for i in $(seq 0 $(($STAGES_COUNT-1)));
+      do
+        TABLE="${TABLE}<tr>"
+        TABLE="${TABLE}<td>${STAGES_NAMES[$i]}</td>"
+        TABLE="${TABLE}<td>$((100-(${STAGES_FAILURES[$i]}*100/$N)))%</td>"
+        if [[ ${STAGES_FAILURES[$i]} -eq 0 ]]; then
+          TABLE="${TABLE}<td></td>"
+        else
+          TABLE="${TABLE}<td><details>${STAGES_FAILURES_LOGS[$i]}</details></td>"
+        fi
+        TABLE="${TABLE}</tr>"
+      done
+      TABLE="${TABLE}</table>"
+
+      git clone -b master --depth=1 https://github.com/commaai/ci-artifacts
+      cd ci-artifacts
+      git checkout -b "jenkins_test_report"
+      echo "$TABLE" >> jenkins_report
+      git add jenkins_report
+      git commit -m "jenkins report"
+      git remote set-url origin https://${env.CI_ARTIFACTS_TOKEN}@github.com/commaai/ci-artifacts.git
+      git push -f origin jenkins_test_report
       '''
 
       currentBuild.result = 'SUCCESS'

From 6e3651b0eb988e250eb32cf88656b2eb245c09bf Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:30:50 -0700
Subject: [PATCH 24/49] fix

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 963a8f134783d7..4a1772628a60b7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -91,7 +91,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
             device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
-            if (branch != "master" && item.size() == 3 && !hasPathChanged(item[2])) {
+            if (branch != "master" && branch != "jenkins_test_runner" && item.size() == 3 && !hasPathChanged(item[2])) {
               println "Skipping ${item[0]}: no changes in ${item[2]}."
               return;
             } else {
@@ -164,13 +164,13 @@ node {
                          'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != '__jenkins_stress') {
+  if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != 'jenkins_test_runner') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
   }
 
-  if (env.BRANCH_NAME == 'jenkins_stress_NOT_NOW') {
+  if (env.BRANCH_NAME == 'jenkins_test_master') {
     sh '''#!/bin/bash
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies

From 18bf032fbead6a781f85889d19f5da0dc8e8ddec Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:42:04 -0700
Subject: [PATCH 25/49] testing

---
 .github/workflows/ci_weekly_report.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index 372b996f81d0e2..aa2effcfe2dbd2 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -28,7 +28,7 @@ jobs:
 
   ci_matrix_jenkins_run:
     needs: [ setup ]
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run_jenkins.yaml@master
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run_jenkins.yaml@jenkins_stress
     with:
       build_count: ${{ env.CI_RUNS }}
 

From 82d124c8b346be048281d5d9b8074525571de296 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:45:01 -0700
Subject: [PATCH 26/49] fix?

---
 .github/workflows/ci_weekly_run_jenkins.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
index 95ffa8be34be44..034c09e0165594 100644
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -14,6 +14,8 @@ jobs:
   jenkins_run:
     - name: Checkout repository
       uses: actions/checkout@v4
+      with:
+        submodules: true
 
     - name: Create master/runner branches
       run: |

From 8e0148fa94187111eadc43bce95d5d41eb1f93b5 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:48:52 -0700
Subject: [PATCH 27/49] fix

---
 .github/workflows/ci_weekly_run_jenkins.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
index 034c09e0165594..7dc6df871773aa 100644
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -12,6 +12,8 @@ concurrency:
 
 jobs:
   jenkins_run:
+    runs-on: ubuntu-latest
+    steps:
     - name: Checkout repository
       uses: actions/checkout@v4
       with:

From 331669a81eb08e1f7930bc883d13e102b919ba0e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:50:02 -0700
Subject: [PATCH 28/49] fix this also

---
 .github/workflows/ci_weekly_report.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index aa2effcfe2dbd2..0e4c9d4598f2bf 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -37,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run_github_actions.yaml@master
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run_github_actions.yaml@jenkins_stress
     with:
       run_number: ${{ matrix.run_number }}
 

From d37ff5ef34570282531155c3883b4d2a373b9143 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:52:23 -0700
Subject: [PATCH 29/49] test

---
 .github/workflows/ci_weekly_report.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index 0e4c9d4598f2bf..57aaafc2d1f667 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -30,7 +30,7 @@ jobs:
     needs: [ setup ]
     uses: commaai/openpilot/.github/workflows/ci_weekly_run_jenkins.yaml@jenkins_stress
     with:
-      build_count: ${{ env.CI_RUNS }}
+      build_count: ${{ github.event.inputs.ci_runs }}
 
   ci_matrix_github_actions_run:
     needs: [ setup ]

From 86a93a1a2c52e05344e6c2e2cb46661d61021efa Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 16 Oct 2024 21:55:55 -0700
Subject: [PATCH 30/49] js

---
 .github/workflows/ci_weekly_run_jenkins.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
index 7dc6df871773aa..738953ef8e32a5 100644
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -42,7 +42,7 @@ jobs:
         script: |
 
           let prev_status = false;
-          const interval setInterval(() => {
+          const interval = setInterval(() => {
 
             const response = github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
               owner: 'commaai',

From c21f92ae999d6396e734219b7e2a496ee1debb8a Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 10:11:54 -0700
Subject: [PATCH 31/49] better

---
 .github/workflows/ci_weekly_run_github_actions.yaml | 2 +-
 .github/workflows/ci_weekly_run_jenkins.yaml        | 8 +++++---
 Jenkinsfile                                         | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci_weekly_run_github_actions.yaml b/.github/workflows/ci_weekly_run_github_actions.yaml
index c3ce8c42da861e..0b4c623f31db90 100644
--- a/.github/workflows/ci_weekly_run_github_actions.yaml
+++ b/.github/workflows/ci_weekly_run_github_actions.yaml
@@ -1,4 +1,4 @@
-name: weekly CI test run
+name: weekly Github Actions CI test run
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
index 738953ef8e32a5..dd3759eea31008 100644
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -1,4 +1,4 @@
-name: weekly CI test run
+name: weekly Jenkins CI test run
 on:
   workflow_call:
     inputs:
@@ -55,8 +55,10 @@ jobs:
             });
 
             response.then(function(r) {
-              if (r.data.statuses[0].state != "pending") {
-                clearInterval(interval)
+              if (r.data.statuses && r.data.statuses.length) {
+                if (r.data.statuses[0].state != "pending") {
+                  clearInterval(interval)
+                }
               }
             });
 
diff --git a/Jenkinsfile b/Jenkinsfile
index 4a1772628a60b7..1a2eff089f3d03 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -178,7 +178,7 @@ node {
 
       N=5
       FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/api/json | jq .nextBuildNumber)
-      LAST_RUN=$((FIRST_RUN+N))
+      LAST_RUN=$((FIRST_RUN+N-1))
 
       for i in $(seq $FIRST_RUN $LAST_RUN);
       do

From 9862c37d0333aca6a2e96e04777b882abef5a89f Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 10:26:50 -0700
Subject: [PATCH 32/49] better

---
 .github/workflows/ci_weekly_run_jenkins.yaml | 2 ++
 Jenkinsfile                                  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
index dd3759eea31008..5e39bd06ed066d 100644
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ b/.github/workflows/ci_weekly_run_jenkins.yaml
@@ -58,6 +58,8 @@ jobs:
               if (r.data.statuses && r.data.statuses.length) {
                 if (r.data.statuses[0].state != "pending") {
                   clearInterval(interval)
+                } else {
+                  console.log('Still running');
                 }
               }
             });
diff --git a/Jenkinsfile b/Jenkinsfile
index 1a2eff089f3d03..ea28357a3928f2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -253,6 +253,8 @@ node {
       done
       TABLE="${TABLE}</table>"
 
+      git config --global user.email "user@comma.ai"
+      git config --global user.name "Vehicle Researcher"
       git clone -b master --depth=1 https://github.com/commaai/ci-artifacts
       cd ci-artifacts
       git checkout -b "jenkins_test_report"

From 0b4aae71411938bfd842cff99fb7edb67509c873 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 10:43:49 -0700
Subject: [PATCH 33/49] git

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ea28357a3928f2..c0c07b7ccb8af0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -253,15 +253,15 @@ node {
       done
       TABLE="${TABLE}</table>"
 
-      git config --global user.email "user@comma.ai"
-      git config --global user.name "Vehicle Researcher"
       git clone -b master --depth=1 https://github.com/commaai/ci-artifacts
       cd ci-artifacts
+      git config --local user.email "user@comma.ai"
+      git config --local user.name "Vehicle Researcher"
+      git config --local url.https://${{env.CI_ARTIFACTS_TOKEN}}@github.com/.insteadOf https://github.com/
       git checkout -b "jenkins_test_report"
       echo "$TABLE" >> jenkins_report
       git add jenkins_report
       git commit -m "jenkins report"
-      git remote set-url origin https://${env.CI_ARTIFACTS_TOKEN}@github.com/commaai/ci-artifacts.git
       git push -f origin jenkins_test_report
       '''
 

From f33312226a57dfe974326fcda72794f07d8995c8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 10:49:57 -0700
Subject: [PATCH 34/49] fix sub

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c0c07b7ccb8af0..3ab279d646b2d2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -257,7 +257,7 @@ node {
       cd ci-artifacts
       git config --local user.email "user@comma.ai"
       git config --local user.name "Vehicle Researcher"
-      git config --local url.https://${{env.CI_ARTIFACTS_TOKEN}}@github.com/.insteadOf https://github.com/
+      git config --local url.https://${env.CI_ARTIFACTS_TOKEN}@github.com/.insteadOf https://github.com/
       git checkout -b "jenkins_test_report"
       echo "$TABLE" >> jenkins_report
       git add jenkins_report

From 76c6f2091a2774535e530d2db7b413e071fcbee7 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 10:58:58 -0700
Subject: [PATCH 35/49] sub

---
 Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3ab279d646b2d2..8a58ac377ec1c5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -172,6 +172,8 @@ node {
 
   if (env.BRANCH_NAME == 'jenkins_test_master') {
     sh '''#!/bin/bash
+      CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
+
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
       CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
@@ -257,7 +259,7 @@ node {
       cd ci-artifacts
       git config --local user.email "user@comma.ai"
       git config --local user.name "Vehicle Researcher"
-      git config --local url.https://${env.CI_ARTIFACTS_TOKEN}@github.com/.insteadOf https://github.com/
+      git config --local url.https://$CI_ARTIFACTS_TOKEN@github.com/.insteadOf https://github.com/
       git checkout -b "jenkins_test_report"
       echo "$TABLE" >> jenkins_report
       git add jenkins_report

From b0eb82171dfc683acc9d1b6c3ada91aead82116e Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 11:07:28 -0700
Subject: [PATCH 36/49] fix

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8a58ac377ec1c5..b0adbcd5c9ee0b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -171,8 +171,8 @@ node {
   }
 
   if (env.BRANCH_NAME == 'jenkins_test_master') {
-    sh '''#!/bin/bash
-      CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
+    sh """
+      CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
 
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
@@ -265,7 +265,7 @@ node {
       git add jenkins_report
       git commit -m "jenkins report"
       git push -f origin jenkins_test_report
-      '''
+      """
 
       currentBuild.result = 'SUCCESS'
       return

From 5425d961777992b3958f99639d0389504f6469e8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 11:14:38 -0700
Subject: [PATCH 37/49] fix

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b0adbcd5c9ee0b..43e40ed4d0e799 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -171,9 +171,10 @@ node {
   }
 
   if (env.BRANCH_NAME == 'jenkins_test_master') {
-    sh """
+    environment {
       CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
-
+    }
+    sh """
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
       CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')

From b7143ad90c8e629fa8283cb1d87a8fd38b0dc6c2 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 11:40:43 -0700
Subject: [PATCH 38/49] try

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 43e40ed4d0e799..ecad8b345e2530 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -174,7 +174,7 @@ node {
     environment {
       CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
     }
-    sh """
+    sh '''
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
       CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
@@ -266,7 +266,7 @@ node {
       git add jenkins_report
       git commit -m "jenkins report"
       git push -f origin jenkins_test_report
-      """
+      '''
 
       currentBuild.result = 'SUCCESS'
       return

From 5e100e809e4a68894119c7c8d15f06541435ae11 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 11:47:32 -0700
Subject: [PATCH 39/49] env var

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ecad8b345e2530..667b713894b101 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -174,7 +174,8 @@ node {
     environment {
       CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
     }
-    sh '''
+    sh '''#!/bin/bash
+      echo "$CI_ARTIFACTS_TOKEN"
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
       CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')

From ed9591107c6e94db191562eb0fbd7260283cd231 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 12:51:51 -0700
Subject: [PATCH 40/49] try full run

---
 .github/workflows/ci_weekly_run_github_actions.yaml | 4 ----
 Jenkinsfile                                         | 1 -
 2 files changed, 5 deletions(-)

diff --git a/.github/workflows/ci_weekly_run_github_actions.yaml b/.github/workflows/ci_weekly_run_github_actions.yaml
index 0b4c623f31db90..914ad23e211975 100644
--- a/.github/workflows/ci_weekly_run_github_actions.yaml
+++ b/.github/workflows/ci_weekly_run_github_actions.yaml
@@ -11,10 +11,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  selfdrive_tests:
-    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
-    with:
-      run_number: ${{ inputs.run_number }}
   tools_tests:
     uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
     with:
diff --git a/Jenkinsfile b/Jenkinsfile
index 667b713894b101..1a11627377b660 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -175,7 +175,6 @@ node {
       CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
     }
     sh '''#!/bin/bash
-      echo "$CI_ARTIFACTS_TOKEN"
       # get crumb for CSRF
       COOKIE_JAR=/tmp/cookies
       CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')

From ee76d207091e565eeb0774cc10ae990a59c321ff Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 13:36:12 -0700
Subject: [PATCH 41/49] fix this

---
 .github/workflows/ci_weekly_report.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index 57aaafc2d1f667..eab21429716ffc 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -60,7 +60,7 @@ jobs:
               })
             var report = {}
             jobs.slice(1, jobs.length-1).forEach(job => {
-              if (job.conclusion === "skipped") return;
+              if (job.conclusion === "skipped" || job.name.includes("ci_matrix_jenkins_run")) return;
               const jobName = job.name.split(" / ")[2];
               const runRegex = /\((.*?)\)/;
               const run = job.name.match(runRegex)[1];

From 6206c01cdd0a87a28f9e40fbe7beafb872a055b8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Thu, 17 Oct 2024 14:13:43 -0700
Subject: [PATCH 42/49] cleanup

---
 .github/workflows/ci_weekly_report.yaml | 2 +-
 Jenkinsfile                             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index eab21429716ffc..c7ca60d4598db0 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -81,7 +81,7 @@ jobs:
           JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
         run: |
           cat <<EOF >> template.html
-          ## Github Actions CI
+          ### Github Actions CI
           <table>
             <thead>
               <tr>
diff --git a/Jenkinsfile b/Jenkinsfile
index 09e3f0731e9b1a..7928c34728ae6d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -241,7 +241,7 @@ node {
 
       done
 
-      TABLE="<table><thead><tr> <th>Stage</th> <th>✅ Passing</th> <th>❌ Failure Details</th> </tr></thead><tbody>"
+      TABLE="\n### Jenkins CI\n<table><thead><tr> <th>Stage</th> <th>✅ Passing</th> <th>❌ Failure Details</th> </tr></thead><tbody>"
       for i in $(seq 0 $(($STAGES_COUNT-1)));
       do
         TABLE="${TABLE}<tr>"

From 9549de85eaeb86f9a3897b4f38825f0ccc5da5f5 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 10:20:32 -0700
Subject: [PATCH 43/49] simpler

---
 scripts/jenkins_loop_test.sh | 110 +++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100755 scripts/jenkins_loop_test.sh

diff --git a/scripts/jenkins_loop_test.sh b/scripts/jenkins_loop_test.sh
new file mode 100755
index 00000000000000..96539256b5f00c
--- /dev/null
+++ b/scripts/jenkins_loop_test.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+set -e
+
+RED='\033[0;31m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+COOKIE_JAR=/tmp/cookies
+CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+TESTING_BRANCH="jenkins_test_runner"
+API_ROUTE="https://jenkins.comma.life/job/openpilot/job/$TESTING_BRANCH"
+N=1
+TEST_BUILDS=()
+
+# try to find active builds
+ALL_BUILDS=( $(curl -s $API_ROUTE/api/json | jq .builds.[].number || :) )
+for i in $ALL_BUILDS; do
+  running=$(curl -s $API_ROUTE/$i/api/json/ | jq .inProgress)
+  if [[ $running == "false" ]]; then
+    continue
+  fi
+  TEST_BUILDS=${ALL_BUILDS[@]}
+  N=${#TEST_BUILDS[@]}
+  break
+done
+
+# no active builds found
+if [[ ${#TEST_BUILDS[@]} -eq 0 ]]; then
+  # delete all previous builds
+  for i in $ALL_BUILDS; do
+    curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/$i/doDelete
+  done
+
+  # create running branch
+  if [[ ${#ALL_BUILDS[@]} -eq 0 ]]; then
+    TEMP_DIR=$(mktemp -d)
+    GIT_LFS_SKIP_SMUDGE=1 git clone -b master --depth=1 --no-tags git@github.com:commaai/openpilot $TEMP_DIR
+    git -C $TEMP_DIR checkout -b $TESTING_BRANCH
+    echo "TESTING" >> $TEMP_DIR/testing_jenkins
+    git -C $TEMP_DIR add testing_jenkins
+    git -C $TEMP_DIR commit -m "testing"
+    git -C $TEMP_DIR push -f origin $TESTING_BRANCH
+    rm -rf $TEMP_DIR
+    FIRST_RUN=1
+    sleep 60
+  else
+    FIRST_RUN=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber)
+  fi
+  LAST_RUN=$((FIRST_RUN+N-1))
+  TEST_BUILDS=$(seq $FIRST_RUN $LAST_RUN)
+
+  # start N new builds
+  for i in $TEST_BUILDS;
+  do
+    curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec
+  done
+fi
+
+while true; do
+  #sleep 60
+  sleep 5
+
+  count=0
+  for i in $TEST_BUILDS;
+  do
+    RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" $API_ROUTE/$i/api/json)
+    HTTP_CODE=$(tail -n1 <<< "$RES")
+    JSON=$(sed '$ d' <<< "$RES")
+
+    if [[ $HTTP_CODE == "200" ]]; then
+      STILL_RUNNING=$(echo $JSON | jq .inProgress)
+      if [[ $STILL_RUNNING == "true" ]]; then
+        echo "build $i still running"
+        continue
+      fi
+      count=$((count+1))
+    else
+      echo "Error getting status of build $i"
+    fi
+  done
+
+  if [[ $count -eq $N ]]; then
+    break
+  fi
+done
+
+STAGES_NAMES=()
+while read stage; do
+  STAGES_NAMES[$index]=$stage
+  index=$((index+1))
+done < <(curl -s -H "$CRUMB" $API_ROUTE/lastBuild/wfapi/ | jq .stages[].name)
+STAGES_COUNT=${#STAGES_NAMES[@]}
+
+STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
+
+for i in $TEST_BUILDS; do
+index=0
+while read result; do
+  if [[ $result != '"SUCCESS"' ]]; then
+    STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1))
+  fi
+  index=$((index+1))
+done < <(curl -s $API_ROUTE/$i/wfapi/ | jq .stages[].status)
+done
+
+# print results of all builds
+for i in $(seq 0 $(($STAGES_COUNT-1))); do
+  echo -e "${STAGES_NAMES[$i]} : ${RED}${BOLD}$((${STAGES_FAILURES[$i]}*100/$N))% failure rate${NC}"
+done

From ebc7319d5529d874a23efe9957c69c76499a7ab5 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 17:26:13 -0700
Subject: [PATCH 44/49] better

---
 scripts/jenkins_loop_test.sh | 97 +++++++++++++++++++++++-------------
 1 file changed, 61 insertions(+), 36 deletions(-)

diff --git a/scripts/jenkins_loop_test.sh b/scripts/jenkins_loop_test.sh
index 96539256b5f00c..94eee00bd6d711 100755
--- a/scripts/jenkins_loop_test.sh
+++ b/scripts/jenkins_loop_test.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -e
 
+YELLOW='\033[0;33m'
+GREEN='\033[0;32m'
 RED='\033[0;31m'
 BOLD='\033[1m'
 NC='\033[0m'
@@ -10,53 +12,59 @@ CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer
 
 TESTING_BRANCH="jenkins_test_runner"
 API_ROUTE="https://jenkins.comma.life/job/openpilot/job/$TESTING_BRANCH"
-N=1
+N=5
 TEST_BUILDS=()
 
-# try to find active builds
+# Try to find previous builds
 ALL_BUILDS=( $(curl -s $API_ROUTE/api/json | jq .builds.[].number || :) )
-for i in $ALL_BUILDS; do
-  running=$(curl -s $API_ROUTE/$i/api/json/ | jq .inProgress)
-  if [[ $running == "false" ]]; then
-    continue
-  fi
-  TEST_BUILDS=${ALL_BUILDS[@]}
-  N=${#TEST_BUILDS[@]}
-  break
-done
 
-# no active builds found
+# No builds. Create branch
+if [[ ${#ALL_BUILDS[@]} -eq 0 ]]; then
+  TEMP_DIR=$(mktemp -d)
+  GIT_LFS_SKIP_SMUDGE=1 git clone -b master --depth=1 --no-tags git@github.com:commaai/openpilot $TEMP_DIR
+  git -C $TEMP_DIR checkout -b $TESTING_BRANCH
+  echo "TESTING" >> $TEMP_DIR/testing_jenkins
+  git -C $TEMP_DIR add testing_jenkins
+  git -C $TEMP_DIR commit -m "testing"
+  git -C $TEMP_DIR push -f origin $TESTING_BRANCH
+  rm -rf $TEMP_DIR
+  FIRST_RUN=1
+  sleep 90
+else
+  # Found some builds. Check if they are still running
+  for i in $ALL_BUILDS; do
+    running=$(curl -s $API_ROUTE/$i/api/json/ | jq .inProgress)
+    if [[ $running == "false" ]]; then
+      continue
+    fi
+    TEST_BUILDS=${ALL_BUILDS[@]}
+    N=${#TEST_BUILDS[@]}
+    break
+  done
+fi
+
+# No running builds found
 if [[ ${#TEST_BUILDS[@]} -eq 0 ]]; then
-  # delete all previous builds
+  # Delete all previous builds
   for i in $ALL_BUILDS; do
-    curl --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/$i/doDelete
+    curl -s --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/$i/doDelete
   done
 
-  # create running branch
-  if [[ ${#ALL_BUILDS[@]} -eq 0 ]]; then
-    TEMP_DIR=$(mktemp -d)
-    GIT_LFS_SKIP_SMUDGE=1 git clone -b master --depth=1 --no-tags git@github.com:commaai/openpilot $TEMP_DIR
-    git -C $TEMP_DIR checkout -b $TESTING_BRANCH
-    echo "TESTING" >> $TEMP_DIR/testing_jenkins
-    git -C $TEMP_DIR add testing_jenkins
-    git -C $TEMP_DIR commit -m "testing"
-    git -C $TEMP_DIR push -f origin $TESTING_BRANCH
-    rm -rf $TEMP_DIR
-    FIRST_RUN=1
-    sleep 60
-  else
-    FIRST_RUN=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber)
-  fi
+  FIRST_RUN=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber)
   LAST_RUN=$((FIRST_RUN+N-1))
   TEST_BUILDS=$(seq $FIRST_RUN $LAST_RUN)
 
-  # start N new builds
+  # Start N new builds
   for i in $TEST_BUILDS;
   do
-    curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec
+    echo "Starting build $i"
+    curl -s --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec
+    sleep 5
   done
 fi
 
+echo "Testing Jenkins with $N builds:"
+
 while true; do
   #sleep 60
   sleep 5
@@ -71,16 +79,19 @@ while true; do
     if [[ $HTTP_CODE == "200" ]]; then
       STILL_RUNNING=$(echo $JSON | jq .inProgress)
       if [[ $STILL_RUNNING == "true" ]]; then
-        echo "build $i still running"
+        echo -e "Build $i: ${YELLOW}still running${NC}"
         continue
+      else
+        count=$((count+1))
+        echo -e "Build $i: ${GREEN}done${NC}"
       fi
-      count=$((count+1))
     else
-      echo "Error getting status of build $i"
+      echo "No status for build $i"
     fi
   done
+  echo ""
 
-  if [[ $count -eq $N ]]; then
+  if [[ $count -ge $N ]]; then
     break
   fi
 done
@@ -104,7 +115,21 @@ while read result; do
 done < <(curl -s $API_ROUTE/$i/wfapi/ | jq .stages[].status)
 done
 
+echo "=========================================="
+echo "===================DONE==================="
+echo "=========================================="
+
 # print results of all builds
 for i in $(seq 0 $(($STAGES_COUNT-1))); do
-  echo -e "${STAGES_NAMES[$i]} : ${RED}${BOLD}$((${STAGES_FAILURES[$i]}*100/$N))% failure rate${NC}"
+  P=$((${STAGES_FAILURES[$i]}*100/$N))
+  if [[ $P -eq 0 ]]; then
+    P=${GREEN}${BOLD}${P}${NC}
+  else
+    P=${RED}${BOLD}${P}${NC}
+  fi
+  echo -e "${STAGES_NAMES[$i]} : $P% failure rate${NC}"
 done
+
+echo "=========================================="
+echo "=========================================="
+echo "=========================================="

From 1ef5abbdb0ffd42692b73ccca6fea42578bb4dc7 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 18:40:58 -0700
Subject: [PATCH 45/49] fix

---
 scripts/jenkins_loop_test.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scripts/jenkins_loop_test.sh b/scripts/jenkins_loop_test.sh
index 94eee00bd6d711..5a3bceec050e14 100755
--- a/scripts/jenkins_loop_test.sh
+++ b/scripts/jenkins_loop_test.sh
@@ -10,9 +10,9 @@ NC='\033[0m'
 COOKIE_JAR=/tmp/cookies
 CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
 
-TESTING_BRANCH="jenkins_test_runner"
+TESTING_BRANCH="__jenkins_test_loop"
 API_ROUTE="https://jenkins.comma.life/job/openpilot/job/$TESTING_BRANCH"
-N=5
+N=20
 TEST_BUILDS=()
 
 # Try to find previous builds
@@ -32,12 +32,12 @@ if [[ ${#ALL_BUILDS[@]} -eq 0 ]]; then
   sleep 90
 else
   # Found some builds. Check if they are still running
-  for i in $ALL_BUILDS; do
+  for i in ${ALL_BUILDS[@]}; do
     running=$(curl -s $API_ROUTE/$i/api/json/ | jq .inProgress)
     if [[ $running == "false" ]]; then
       continue
     fi
-    TEST_BUILDS=${ALL_BUILDS[@]}
+    TEST_BUILDS=( ${ALL_BUILDS[@]} )
     N=${#TEST_BUILDS[@]}
     break
   done
@@ -46,16 +46,16 @@ fi
 # No running builds found
 if [[ ${#TEST_BUILDS[@]} -eq 0 ]]; then
   # Delete all previous builds
-  for i in $ALL_BUILDS; do
+  for i in ${ALL_BUILDS[@]}; do
     curl -s --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/$i/doDelete
   done
 
   FIRST_RUN=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber)
   LAST_RUN=$((FIRST_RUN+N-1))
-  TEST_BUILDS=$(seq $FIRST_RUN $LAST_RUN)
+  TEST_BUILDS=( $(seq $FIRST_RUN $LAST_RUN) )
 
   # Start N new builds
-  for i in $TEST_BUILDS;
+  for i in ${TEST_BUILDS[@]};
   do
     echo "Starting build $i"
     curl -s --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec
@@ -70,7 +70,7 @@ while true; do
   sleep 5
 
   count=0
-  for i in $TEST_BUILDS;
+  for i in ${TEST_BUILDS[@]};
   do
     RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" $API_ROUTE/$i/api/json)
     HTTP_CODE=$(tail -n1 <<< "$RES")
@@ -105,7 +105,7 @@ STAGES_COUNT=${#STAGES_NAMES[@]}
 
 STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
 
-for i in $TEST_BUILDS; do
+for i in ${TEST_BUILDS[@]}; do
 index=0
 while read result; do
   if [[ $result != '"SUCCESS"' ]]; then
@@ -123,9 +123,9 @@ echo "=========================================="
 for i in $(seq 0 $(($STAGES_COUNT-1))); do
   P=$((${STAGES_FAILURES[$i]}*100/$N))
   if [[ $P -eq 0 ]]; then
-    P=${GREEN}${BOLD}${P}${NC}
+    P=${GREEN}${BOLD}${P}
   else
-    P=${RED}${BOLD}${P}${NC}
+    P=${RED}${BOLD}${P}
   fi
   echo -e "${STAGES_NAMES[$i]} : $P% failure rate${NC}"
 done

From 3d8f808d4a2006e3e8075d7a0cd4e56338ffda77 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 18:42:21 -0700
Subject: [PATCH 46/49] get back

---
 Jenkinsfile | 106 +---------------------------------------------------
 1 file changed, 2 insertions(+), 104 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7928c34728ae6d..42db027f328aac 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -91,7 +91,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
             device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
-            if (branch != "master" && branch != "jenkins_test_runner" && item.size() == 3 && !hasPathChanged(item[2])) {
+            if (branch != "master" && item.size() == 3 && !hasPathChanged(item[2])) {
               println "Skipping ${item[0]}: no changes in ${item[2]}."
               return;
             } else {
@@ -164,114 +164,12 @@ node {
                          'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != 'jenkins_test_runner') {
+  if (env.BRANCH_NAME != 'master') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
   }
 
-  if (env.BRANCH_NAME == 'jenkins_test_master') {
-    environment {
-      CI_ARTIFACTS_TOKEN="${env.CI_ARTIFACTS_TOKEN}"
-    }
-    sh '''#!/bin/bash
-      # get crumb for CSRF
-      COOKIE_JAR=/tmp/cookies
-      CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-
-      N=5
-      FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/api/json | jq .nextBuildNumber)
-      LAST_RUN=$((FIRST_RUN+N-1))
-
-      for i in $(seq $FIRST_RUN $LAST_RUN);
-      do
-        # start build i
-        curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/build?delay=0sec
-      done
-
-      while true; do
-        sleep 60
-
-        count=0
-        for i in $(seq $FIRST_RUN $LAST_RUN);
-        do
-          RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/api/json)
-          HTTP_CODE=$(tail -n1 <<< "$RES")
-          JSON=$(sed '$ d' <<< "$RES")
-
-          if [[ $HTTP_CODE == "200" ]]; then
-            STILL_RUNNING=$(echo $JSON | jq .inProgress)
-            if [[ $STILL_RUNNING == "true" ]]; then
-              echo "build $i still running"
-              continue
-            fi
-            ((count++))
-          else
-            echo "Error getting status of build $i"
-          fi
-        done
-
-        if [[ $count -eq $N ]]; then
-          break
-        fi
-      done
-
-
-      STAGES_NAMES=()
-      while read stage; do
-        STAGES_NAMES[$index]=$stage
-        ((index++))
-      done < <(curl -s -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/lastBuild/wfapi/ | jq .stages[].name)
-      STAGES_COUNT=${#STAGES_NAMES[@]}
-
-      STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
-      STAGES_FAILURES_LOGS=()
-
-      for i in $(seq $FIRST_RUN $LAST_RUN);
-      do
-
-      index=0
-      while read result; do
-        if [[ $result != '"SUCCESS"' ]]; then
-          STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1))
-          STAGES_FAILURES_LOGS[$index]="${STAGES_FAILURES_LOGS[$index]}<a href=\"https://jenkins.comma.life/blue/organizations/jenkins/openpilot/detail/jenkins_test_runner/$i/pipeline/\">Log for run #$(($i-$FIRST_RUN))</a><br>"
-        fi
-        ((index++))
-      done < <(curl https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/wfapi/ | jq .stages[].status)
-
-      done
-
-      TABLE="\n### Jenkins CI\n<table><thead><tr> <th>Stage</th> <th>✅ Passing</th> <th>❌ Failure Details</th> </tr></thead><tbody>"
-      for i in $(seq 0 $(($STAGES_COUNT-1)));
-      do
-        TABLE="${TABLE}<tr>"
-        TABLE="${TABLE}<td>${STAGES_NAMES[$i]}</td>"
-        TABLE="${TABLE}<td>$((100-(${STAGES_FAILURES[$i]}*100/$N)))%</td>"
-        if [[ ${STAGES_FAILURES[$i]} -eq 0 ]]; then
-          TABLE="${TABLE}<td></td>"
-        else
-          TABLE="${TABLE}<td><details>${STAGES_FAILURES_LOGS[$i]}</details></td>"
-        fi
-        TABLE="${TABLE}</tr>"
-      done
-      TABLE="${TABLE}</table>"
-
-      git clone -b master --depth=1 https://github.com/commaai/ci-artifacts
-      cd ci-artifacts
-      git config --local user.email "user@comma.ai"
-      git config --local user.name "Vehicle Researcher"
-      git config --local url.https://$CI_ARTIFACTS_TOKEN@github.com/.insteadOf https://github.com/
-      git checkout -b "jenkins_test_report"
-      echo "$TABLE" >> jenkins_report
-      git add jenkins_report
-      git commit -m "jenkins report"
-      git push -f origin jenkins_test_report
-      '''
-
-      currentBuild.result = 'SUCCESS'
-      return
-  }
-
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release3-staging", "tici-needs-can", [], [

From 00bee5b8c82fead5923caac9eb0b977cd50b08a8 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 18:49:04 -0700
Subject: [PATCH 47/49] concurent build

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 42db027f328aac..06299bc867ac71 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -164,7 +164,7 @@ node {
                          'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master') {
+  if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != '__jenkins_loop_test') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])

From 3c7472394f30d9e243120ec4539325616f0339fc Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 18:55:24 -0700
Subject: [PATCH 48/49] cleanup

---
 .github/workflows/ci_weekly_report.yaml       | 16 ++---
 ...github_actions.yaml => ci_weekly_run.yaml} |  6 +-
 .github/workflows/ci_weekly_run_jenkins.yaml  | 67 -------------------
 3 files changed, 9 insertions(+), 80 deletions(-)
 rename .github/workflows/{ci_weekly_run_github_actions.yaml => ci_weekly_run.yaml} (66%)
 delete mode 100644 .github/workflows/ci_weekly_run_jenkins.yaml

diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml
index c7ca60d4598db0..9821283cb57442 100644
--- a/.github/workflows/ci_weekly_report.yaml
+++ b/.github/workflows/ci_weekly_report.yaml
@@ -26,23 +26,17 @@ jobs:
           matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
           echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
-  ci_matrix_jenkins_run:
-    needs: [ setup ]
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run_jenkins.yaml@jenkins_stress
-    with:
-      build_count: ${{ github.event.inputs.ci_runs }}
-
-  ci_matrix_github_actions_run:
+  ci_matrix_run:
     needs: [ setup ]
     strategy:
       fail-fast: false
       matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run_github_actions.yaml@jenkins_stress
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
     with:
       run_number: ${{ matrix.run_number }}
 
   report:
-    needs: [ci_matrix_jenkins_run, ci_matrix_github_actions_run]
+    needs: [ci_matrix_run]
     runs-on: ubuntu-latest
     if: always()
     steps:
@@ -60,7 +54,7 @@ jobs:
               })
             var report = {}
             jobs.slice(1, jobs.length-1).forEach(job => {
-              if (job.conclusion === "skipped" || job.name.includes("ci_matrix_jenkins_run")) return;
+              if (job.conclusion === "skipped") return;
               const jobName = job.name.split(" / ")[2];
               const runRegex = /\((.*?)\)/;
               const run = job.name.match(runRegex)[1];
@@ -81,7 +75,6 @@ jobs:
           JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
         run: |
           cat <<EOF >> template.html
-          ### Github Actions CI
           <table>
             <thead>
               <tr>
@@ -106,4 +99,3 @@ jobs:
           echo $JOB_RESULTS | jinja2 template.html > report.html
           echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
           cat report.html >> $GITHUB_STEP_SUMMARY
-          curl https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/jenkins_test_report/jenkins_report >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/ci_weekly_run_github_actions.yaml b/.github/workflows/ci_weekly_run.yaml
similarity index 66%
rename from .github/workflows/ci_weekly_run_github_actions.yaml
rename to .github/workflows/ci_weekly_run.yaml
index 914ad23e211975..c3ce8c42da861e 100644
--- a/.github/workflows/ci_weekly_run_github_actions.yaml
+++ b/.github/workflows/ci_weekly_run.yaml
@@ -1,4 +1,4 @@
-name: weekly Github Actions CI test run
+name: weekly CI test run
 on:
   workflow_call:
     inputs:
@@ -11,6 +11,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  selfdrive_tests:
+    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
   tools_tests:
     uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
     with:
diff --git a/.github/workflows/ci_weekly_run_jenkins.yaml b/.github/workflows/ci_weekly_run_jenkins.yaml
deleted file mode 100644
index 5e39bd06ed066d..00000000000000
--- a/.github/workflows/ci_weekly_run_jenkins.yaml
+++ /dev/null
@@ -1,67 +0,0 @@
-name: weekly Jenkins CI test run
-on:
-  workflow_call:
-    inputs:
-      build_count:
-        required: true
-        type: string
-
-concurrency:
-  group: ci-run-${{ inputs.build_count }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  jenkins_run:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - name: Create master/runner branches
-      run: |
-        git config --global user.name "github-actions[bot]"
-        git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
-        git checkout -b jenkins_test_master
-        echo "TEST JENKINS MASTER" >> test_jenkins
-        git add test_jenkins
-        git commit -m 'master jenkins'
-        GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_test_master
-
-        git checkout -b jenkins_test_runner
-        echo "TEST JENKINS RUNNER" >> test_jenkins
-        git add test_jenkins
-        git commit -m 'runner jenkins'
-        GIT_LFS_SKIP_PUSH=1 git push -f origin jenkins_test_runner
-
-    - name: Waiting on Jenkins runs
-      uses: actions/github-script@v6
-      with:
-        script: |
-
-          let prev_status = false;
-          const interval = setInterval(() => {
-
-            const response = github.request('GET /repos/{owner}/{repo}/commits/{ref}/status', {
-              owner: 'commaai',
-              repo: 'openpilot',
-              ref: 'heads/jenkins_test_master',
-              headers: {
-                'Accept': 'application/vnd.github+json',
-                'X-GitHub-Api-Version': '2022-11-28'
-              }
-            });
-
-            response.then(function(r) {
-              if (r.data.statuses && r.data.statuses.length) {
-                if (r.data.statuses[0].state != "pending") {
-                  clearInterval(interval)
-                } else {
-                  console.log('Still running');
-                }
-              }
-            });
-
-          }, 60000);

From 7c9d7ca27f91783588967b819fefec644ed24ce7 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Fri, 18 Oct 2024 19:05:30 -0700
Subject: [PATCH 49/49] name

---
 scripts/jenkins_loop_test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/jenkins_loop_test.sh b/scripts/jenkins_loop_test.sh
index 5a3bceec050e14..83366b8d43bbf2 100755
--- a/scripts/jenkins_loop_test.sh
+++ b/scripts/jenkins_loop_test.sh
@@ -10,7 +10,7 @@ NC='\033[0m'
 COOKIE_JAR=/tmp/cookies
 CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
 
-TESTING_BRANCH="__jenkins_test_loop"
+TESTING_BRANCH="__jenkins_loop_test"
 API_ROUTE="https://jenkins.comma.life/job/openpilot/job/$TESTING_BRANCH"
 N=20
 TEST_BUILDS=()