From 79790671f4b404dfdabb84e21e044b9a7a8df294 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Thu, 5 Feb 2026 20:46:33 +0200 Subject: [PATCH] feat: support comparing between branches --- .github/scripts/run_macrobenchmarks.sh | 35 ++++++++----- .github/workflows/Build.yaml | 69 +++++++++++++++++--------- 2 files changed, 68 insertions(+), 36 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 13247a36d..81b0f9796 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -1,18 +1,17 @@ -#!/usr/bin/env sh -set -e +#!/usr/bin/env bash + +set -euo pipefail APP_PKG="com.google.samples.apps.nowinandroid" BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" JSON_REPORTS_DIR="benchmarks/json_reports" -run_benchmark() { - VERSION_LABEL="$1" # v1 or v2 - RUN_NUMBER="$2" # 1..5 +PATH_APK_VERSION_1="$1" +PATH_APK_VERSION_2="$2" - echo "==============================" - echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" - echo "==============================" +install_apk() { + adb install -r "$1" # Clear app data and prepare storage adb shell pm clear "$APP_PKG" || true @@ -20,7 +19,16 @@ run_benchmark() { adb shell rm /sdcard/Download/*.json || true adb shell rm /sdcard/Download/*.perfetto-trace || true adb shell rm /sdcard/Download/*.txt || true - +} + +run_benchmark() { + VERSION_LABEL="$1" # v1 or v2 + RUN_NUMBER="$2" # 1..5 + + echo "==============================" + echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" + echo "==============================" + # Run only the Startup Baseline Profile benchmark adb shell am instrument -w \ -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \ @@ -28,7 +36,7 @@ run_benchmark() { -e androidx.benchmark.profiling.mode none \ -e no-isolated-storage true \ -e additionalTestOutputDir /sdcard/Download \ - $BENCHMARK_PKG/$TEST_RUNNER + "$BENCHMARK_PKG/$TEST_RUNNER" # Ensure the local directory exists for the pull mkdir -p "$JSON_REPORTS_DIR/tmp_results" @@ -46,7 +54,10 @@ run_benchmark() { } # Alternate runs: v1, v2, v1, v2 ... -for i in 1 2 3 4 5; do +for i in {1..5}; do + install_apk "$PATH_APK_VERSION_1" run_benchmark "v1" "$i" + + install_apk "$PATH_APK_VERSION_2" run_benchmark "v2" "$i" -done \ No newline at end of file +done diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index a63de6b5d..413e6dac1 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -3,39 +3,62 @@ name: Benchmark on Emulator on: workflow_dispatch: +env: + BASELINE_BRANCH: main + GRADLE_BUILD_CACHE_DIR: ${{ github.workspace }}/.common-gradle-cache + jobs: benchmark-android: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 45 steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Free Disk Space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: '17' - cache: 'gradle' - - - name: Grant execute permission - run: chmod +x gradlew - - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Build APKs + - name: Checkout branch (Baseline) + uses: actions/checkout@v4 + with: + ref: ${{ env.BASELINE_BRANCH }} + path: baseline + + - name: Checkout branch (Candidate) + uses: actions/checkout@v4 + with: + path: candidate + + - name: Setup JDK 17 + uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v5 + with: + # By default, The setup-gradle action will only write to the cache from Jobs on the default (main/master) branch. + # Jobs on other branches will read entries from the cache but will not write updated entries. + cache-read-only: false + + - name: Build APKs (Baseline) + working-directory: ./baseline run: | + chmod +x ./gradlew + ./gradlew :app:assembleDemoBenchmark + + - name: Build APKs (Candidate) + working-directory: ./candidate + run: | + chmod +x ./gradlew ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - name: Run Benchmarks @@ -45,22 +68,20 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 - disable-animations: true + emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 + disable-animations: true script: | - # Install/Copy the built APKs to the emulator - adb install app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk - adb install benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk - - chmod +x .github/scripts/run_macrobenchmarks.sh - .github/scripts/run_macrobenchmarks.sh + adb install ./candidate/benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk + chmod +x ./candidate/.github/scripts/run_macrobenchmarks.sh + ./candidate/.github/scripts/run_macrobenchmarks.sh "./baseline/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" "./candidate/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" - name: Run step fit run: python3 .github/scripts/step_fit.py + working-directory: ./candidate - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/json_reports/ \ No newline at end of file + path: ./candidate/benchmarks/json_reports/