feat: support comparing between branches

.github/scripts/run_macrobenchmarks.sh

@@ -1,18 +1,17 @@
-#!/usr/bin/env sh
-set -e
+#!/usr/bin/env bash
+
+set -euo pipefail
 
 APP_PKG="com.google.samples.apps.nowinandroid"
 BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks"
 TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner"
 JSON_REPORTS_DIR="benchmarks/json_reports"
 
-run_benchmark() {
-  VERSION_LABEL="$1"   # v1 or v2
-  RUN_NUMBER="$2"      # 1..5
+PATH_APK_VERSION_1="$1"
+PATH_APK_VERSION_2="$2"
 
-  echo "=============================="
-  echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER"
-  echo "=============================="
+install_apk() {
+  adb install -r "$1"
 
   # Clear app data and prepare storage
   adb shell pm clear "$APP_PKG" || true
@@ -20,7 +19,16 @@ run_benchmark() {
   adb shell rm /sdcard/Download/*.json || true
   adb shell rm /sdcard/Download/*.perfetto-trace || true
   adb shell rm /sdcard/Download/*.txt || true
-  
+}
+
+run_benchmark() {
+  VERSION_LABEL="$1"   # v1 or v2
+  RUN_NUMBER="$2"      # 1..5
+
+  echo "=============================="
+  echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER"
+  echo "=============================="
+
   # Run only the Startup Baseline Profile benchmark
   adb shell am instrument -w \
     -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \
@@ -28,7 +36,7 @@ run_benchmark() {
     -e androidx.benchmark.profiling.mode none \
     -e no-isolated-storage true \
     -e additionalTestOutputDir /sdcard/Download \
-    $BENCHMARK_PKG/$TEST_RUNNER
+    "$BENCHMARK_PKG/$TEST_RUNNER"
 
   # Ensure the local directory exists for the pull
   mkdir -p "$JSON_REPORTS_DIR/tmp_results"
@@ -46,7 +54,10 @@ run_benchmark() {
 }
 
 # Alternate runs: v1, v2, v1, v2 ...
-for i in 1 2 3 4 5; do
+for i in {1..5}; do
+  install_apk "$PATH_APK_VERSION_1"
   run_benchmark "v1" "$i"
+
+  install_apk "$PATH_APK_VERSION_2"
   run_benchmark "v2" "$i"
-done
+done

.github/workflows/Build.yaml

@@ -3,39 +3,62 @@ name: Benchmark on Emulator
 on:
   workflow_dispatch:
 
+env:
+  BASELINE_BRANCH: main
+  GRADLE_BUILD_CACHE_DIR: ${{ github.workspace }}/.common-gradle-cache
+
 jobs:
   benchmark-android:
     runs-on: ubuntu-latest
-    timeout-minutes: 45 
+    timeout-minutes: 45
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
       - name: Free Disk Space
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
 
-      - name: Set up JDK 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'zulu'
-          java-version: '17'
-          cache: 'gradle'
-
-      - name: Grant execute permission
-        run: chmod +x gradlew
-
       - name: Enable KVM
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
           sudo udevadm control --reload-rules
           sudo udevadm trigger --name-match=kvm
 
-      - name: Build APKs
+      - name: Checkout branch (Baseline)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.BASELINE_BRANCH }}
+          path: baseline
+
+      - name: Checkout branch (Candidate)
+        uses: actions/checkout@v4
+        with:
+          path: candidate
+
+      - name: Setup JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+        with:
+          # By default, The setup-gradle action will only write to the cache from Jobs on the default (main/master) branch.
+          # Jobs on other branches will read entries from the cache but will not write updated entries.
+          cache-read-only: false
+
+      - name: Build APKs (Baseline)
+        working-directory: ./baseline
         run: |
+          chmod +x ./gradlew
+          ./gradlew :app:assembleDemoBenchmark
+
+      - name: Build APKs (Candidate)
+        working-directory: ./candidate
+        run: |
+          chmod +x ./gradlew
           ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark
 
       - name: Run Benchmarks
@@ -45,22 +68,20 @@ jobs:
           target: google_apis
           arch: x86_64
           force-avd-creation: true
-          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096
-          disable-animations: true 
+          emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096
+          disable-animations: true
           script: |
-            # Install/Copy the built APKs to the emulator
-            adb install app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk
-            adb install benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk
-
-            chmod +x .github/scripts/run_macrobenchmarks.sh
-            .github/scripts/run_macrobenchmarks.sh
+            adb install ./candidate/benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk
+            chmod +x ./candidate/.github/scripts/run_macrobenchmarks.sh
+            ./candidate/.github/scripts/run_macrobenchmarks.sh "./baseline/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" "./candidate/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk"
 
       - name: Run step fit
         run: python3 .github/scripts/step_fit.py
+        working-directory: ./candidate
 
       - name: Upload Artifacts
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: benchmark-results
-          path: benchmarks/json_reports/
+          path: ./candidate/benchmarks/json_reports/