From f2f5c4ff385d6daeb99ffdefda48e35f7c3e3790 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 16:10:40 +0200 Subject: [PATCH 01/39] yarab this works ngl --- .github/workflows/Build.yaml | 361 ++++++++++++----------------------- 1 file changed, 117 insertions(+), 244 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 7e5a80eea..770a0bf4e 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,286 +1,159 @@ -name: Build +name: "Build APKs + Emulator test (trigger: push to test)" on: - workflow_dispatch: push: branches: - - main - pull_request: + - testNIA + workflow_dispatch: -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true +permissions: + contents: read jobs: - test_and_apk: - name: "Local tests and APKs" + build_and_upload: + name: Build APKs and test APKs runs-on: ubuntu-latest - - permissions: - contents: write - pull-requests: write - security-events: write - timeout-minutes: 60 - + outputs: + apks_artifact: apks steps: - name: Checkout uses: actions/checkout@v4 - - name: Copy CI gradle.properties - run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties + - name: Copy CI gradle.properties (if exists) + run: | + if [ -f .github/ci-gradle.properties ]; then + mkdir -p ~/.gradle + cp .github/ci-gradle.properties ~/.gradle/gradle.properties + fi - - name: Set up JDK 21 + - name: Set up JDK 17 uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: 21 + java-version: 17 - - name: Setup Gradle + - name: Setup Gradle (cache enabled) uses: gradle/actions/setup-gradle@v4 with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - build-scan-publish: true - build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" - build-scan-terms-of-use-agree: "yes" - - - name: Check build-logic - run: ./gradlew :build-logic:convention:check - - - name: Check spotless - run: ./gradlew spotlessCheck --init-script gradle/init.gradle.kts --no-configuration-cache - - - name: Check Dependency Guard - id: dependencyguard_verify - continue-on-error: true - run: ./gradlew dependencyGuard - - - name: Prevent updating Dependency Guard baselines if this is a fork - id: checkfork_dependencyguard - continue-on-error: false - if: steps.dependencyguard_verify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository - run: | - echo "::error::Dependency Guard failed, please update baselines with: ./gradlew dependencyGuardBaseline" && exit 1 - - # Runs if previous job failed - - name: Generate new Dependency Guard baselines if verification failed and it's a PR - id: dependencyguard_baseline - if: steps.dependencyguard_verify.outcome == 'failure' && github.event_name == 'pull_request' - run: | - ./gradlew dependencyGuardBaseline - - - name: Push new Dependency Guard baselines if available - uses: stefanzweifel/git-auto-commit-action@v5 - if: steps.dependencyguard_baseline.outcome == 'success' - with: - file_pattern: '**/dependencies/*.txt' - disable_globbing: true - commit_message: "🤖 Updates baselines for Dependency Guard" - - - name: Update Graphs - run: ./gradlew graphUpdate - continue-on-error: true - - - name: Check Graphs - id: graphs_verify - run: git add -- "**/README.md" && git diff --cached --quiet --exit-code -- "**/README.md" + build-cache: true - - name: Prevent updating graphs if this is a fork - id: checkfork_graphs - continue-on-error: false - if: steps.graphs_verify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository - run: | - echo "::error::Check Graphs failed, please update graphs with: ./gradlew graphUpdate" && exit 1 - - - name: Push new graphs if available - if: steps.graphs_verify.outcome == 'failure' && github.event_name == 'pull_request' - uses: stefanzweifel/git-auto-commit-action@v5 - with: - file_pattern: '**/README.md' - disable_globbing: true - commit_message: "🤖 Updates graphs" + - name: Build all APKs (assemble) + # This builds flavors / variants. It may take a while the first run. + run: ./gradlew :app:assemble -PminifyWithR8=false --no-daemon --no-parallel - - name: Run all local screenshot tests (Roborazzi) - id: screenshotsverify - continue-on-error: true - run: ./gradlew verifyRoborazziDemoDebug - - - name: Prevent pushing new screenshots if this is a fork - id: checkfork_screenshots - continue-on-error: false - if: steps.screenshotsverify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository + - name: Find APKs and test APKs + id: find-apks run: | - echo "::error::Screenshot tests failed, please create a PR in your fork first." - echo "Your fork's CI will take screenshots for your fork." - exit 1 - - # Runs if previous job failed - - name: Generate new screenshots if verification failed and it's a PR - id: screenshotsrecord - if: steps.screenshotsverify.outcome == 'failure' && github.event_name == 'pull_request' - run: | - ./gradlew recordRoborazziDemoDebug - - - name: Push new screenshots if available - uses: stefanzweifel/git-auto-commit-action@v5 - if: steps.screenshotsrecord.outcome == 'success' - with: - file_pattern: '*/*.png' - disable_globbing: true - commit_message: "🤖 Updates screenshots" - - # Run local tests after screenshot tests to avoid wrong UP-TO-DATE. TODO: Ignore screenshots. - - name: Run local tests - run: ./gradlew testDemoDebug :lint:test - - - name: Build all build type and flavor permutations - run: ./gradlew :app:assemble -PminifyWithR8=false - - - name: Upload build outputs (APKs) - uses: actions/upload-artifact@v4 - with: - name: APKs - path: '**/build/outputs/apk/**/*.apk' - - - name: Upload JVM local results (XML) - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: local-test-results - path: '**/build/test-results/test*UnitTest/**.xml' - - - name: Upload screenshot results (PNG) - if: ${{ !cancelled() }} + echo "Searching for APKs under app/build/outputs..." + find ./app/build/outputs -type f -name "*.apk" -print || true + # Collect all APKs (app & androidTest apks) + mkdir -p apk_collection + # find apks (exclude mapping & intermediates) + find ./app/build/outputs -type f -name "*.apk" -not -path "*/mapping/*" -not -path "*/outputs/mapping/*" -exec cp {} apk_collection/ \; + echo "Files copied to apk_collection:" + ls -lah apk_collection || true + if [ $(ls -A apk_collection | wc -l) -eq 0 ]; then + echo "No APKs found! failing." + exit 1 + fi + echo "::set-output name=apk_dir::apk_collection" + + - name: Upload APKs artifact uses: actions/upload-artifact@v4 with: - name: screenshot-test-results - path: '**/build/outputs/roborazzi/*_compare.png' + name: apks + path: apk_collection/** - - name: Check lint - run: ./gradlew :app:lintProdRelease :app-nia-catalog:lintRelease :lint:lint - - - name: Upload lint reports (HTML) - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: lint-reports - path: '**/build/reports/lint-results-*.html' - - - name: Upload lint reports (SARIF) for app module - if: ${{ !cancelled() && hashFiles('app/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './app/' - category: app - - - name: Upload lint reports (SARIF) for app-nia-catalog module - if: ${{ !cancelled() && hashFiles('app-nia-catalog/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './app-nia-catalog/' - category: app-nia-catalog - - - name: Upload lint reports (SARIF) for lint module - if: ${{ !cancelled() && hashFiles('lint/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './lint/' - category: lint - - - name: Check badging - run: ./gradlew :app:checkProdReleaseBadging - - androidTest: + emulator_run: + name: Run emulator, install APKs, run instrumentation & record + needs: build_and_upload runs-on: ubuntu-latest - timeout-minutes: 55 - strategy: - matrix: - api-level: [26, 34] - + timeout-minutes: 60 steps: - - name: Delete unnecessary tools 🔧 - uses: jlumbroso/free-disk-space@v1.3.1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Download built APKs + uses: actions/download-artifact@v4 with: - android: false # Don't remove Android tools - tool-cache: true # Remove image tool cache - rm -rf "$AGENT_TOOLSDIRECTORY" - dotnet: true # rm -rf /usr/share/dotnet - haskell: true # rm -rf /opt/ghc... - swap-storage: true # rm -f /mnt/swapfile (4GiB) - docker-images: false # Takes 16s, enable if needed in the future - large-packages: false # includes google-cloud-sdk and it's slow + name: apks + path: ./apks - - name: Enable KVM group perms + - name: Enable KVM on runner (Linux) + if: runner.os == 'Linux' run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - ls /dev/kvm + sudo udevadm trigger --name-match=kvm || true + ls -la /dev/kvm || true - - name: Checkout - uses: actions/checkout@v4 - - - name: Copy CI gradle.properties - run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties - - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - distribution: 'zulu' - java-version: 21 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - build-scan-publish: true - build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" - build-scan-terms-of-use-agree: "yes" + - name: Prepare artifacts dir + run: mkdir -p artifacts - - name: Build projects and run instrumentation tests + - name: Start emulator, install APKs, run tests and record (ReactiveCircus) uses: reactivecircus/android-emulator-runner@v2 with: - api-level: ${{ matrix.api-level }} + api-level: 30 arch: x86_64 + force-avd-creation: true disable-animations: true - disk-size: 6000M - heap-size: 600M - script: ./gradlew connectedDemoDebugAndroidTest --daemon - - - name: Run local tests (including Roborazzi) for the combined coverage report (only API 30) - if: matrix.api-level == 30 - # There is no need to verify Roborazzi tests to generate coverage. - run: ./gradlew testDemoDebugUnitTest -Proborazzi.test.verify=false # Add Prod if we ever add JVM tests for prod - - # Add `createProdDebugUnitTestCoverageReport` if we ever add JVM tests for prod - - name: Generate coverage reports for Debug variants (only API 30) - if: matrix.api-level == 30 - run: ./gradlew createDemoDebugCombinedCoverageReport - - - name: Upload test reports - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: test-reports-${{ matrix.api-level }} - path: '**/build/reports/androidTests' - - - name: Display local test coverage (only API 30) - if: matrix.api-level == 30 - id: jacoco - uses: madrapps/jacoco-report@v1.7.1 - with: - title: Combined test coverage report - min-coverage-overall: 40 - min-coverage-changed-files: 60 - paths: | - ${{ github.workspace }}/**/build/reports/jacoco/**/*Report.xml - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload local coverage reports (XML + HTML) (only API 30) - if: matrix.api-level == 30 + emulator-options: -no-window -no-boot-anim -noaudio -gpu swiftshader_indirect + script: | + set -euo pipefail + echo "Waiting for emulator..." + adb wait-for-device + until adb -s emulator-5554 shell getprop sys.boot_completed | grep -m1 "1"; do sleep 1; done + echo "Emulator booted." + + # Install all apks found in ./apks (app and test apks) + echo "Installing APKs..." + for f in ./apks/*.apk; do + echo "Installing $f" + adb install -r "$f" || (echo "install failed for $f, continuing" && true) + done + + # Attempt to detect an instrumentation runner automatically + echo "Detecting installed instrumentation..." + INST_LIST="$(adb shell pm list instrumentation || true)" + echo "Instrumentation list: $INST_LIST" + # pick first instrumentation component (format: instrumentation: / (target=...)) + INST=$(echo "$INST_LIST" | sed -n 's/^instrumentation:\s*//p' | head -n1 | awk '{print $1}' | tr -d '\r') + if [ -z "$INST" ]; then + echo "No instrumentation found. Trying to detect test runner via package name fallback..." + # fallback: try to find any package that looks like demo or nowinandroid (best-effort) + PACKAGE=$(adb shell pm list packages | tr -d '\r' | grep -E 'nowinandroid|nia|nowin' | head -n1 | sed 's/package://g' || true) + echo "Detected package: $PACKAGE" + if [ -n "$PACKAGE" ]; then + # list instrumentation for that package specifically + INST=$(adb shell pm list instrumentation | tr -d '\r' | grep "$PACKAGE" | head -n1 | sed -n 's/^instrumentation:\s*//p' | awk '{print $1}' || true) + fi + fi + + if [ -z "$INST" ]; then + echo "Couldn't auto-detect instrumentation runner. As fallback, will run a simple smoke launch." + # attempt to start main activity (best-effort) + adb shell monkey -p com.google.samples.apps.nowinandroid.dev -c android.intent.category.LAUNCHER 1 || true + else + echo "Running instrumentation: $INST" + # Start screenrecord in background on device (30s) + adb shell screenrecord --time-limit 30 /sdcard/test_run.mp4 & + # Run instrumentation - this will run tests already packaged in the test APK + adb shell am instrument -w "$INST" || true + # Give a moment for the recording to finish (screenrecord stops automatically after time-limit) + sleep 5 + # Pull the recording + adb pull /sdcard/test_run.mp4 ./artifacts/test_run.mp4 || true + fi + + # Collect memory and gfx info and logs + adb shell dumpsys meminfo > ./artifacts/dumpsys_meminfo.txt || true + adb shell dumpsys gfxinfo com.google.samples.apps.nowinandroid.dev > ./artifacts/gfxinfo.txt || true + adb logcat -d > ./artifacts/logcat.txt || true + + - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: coverage-reports - if-no-files-found: error - compression-level: 1 - overwrite: false - path: '**/build/reports/jacoco/' + name: emulator-artifacts + path: artifacts/**, apks/** From fa7fff3762bf3131e0465c06d0feb57ce5c1805f Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 17:15:50 +0200 Subject: [PATCH 02/39] yarab ya gemini --- .github/workflows/Build.yaml | 147 +++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 65 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 770a0bf4e..689e3dc54 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,4 +1,4 @@ -name: "Build APKs + Emulator test (trigger: push to test)" +name: Build APKs + Emulator test (push to test) on: push: @@ -6,14 +6,22 @@ on: - testNIA workflow_dispatch: +concurrency: + group: emulator-test-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read +env: + EMULATOR_API_LEVEL: 30 + EMULATOR_ARCH: x86_64 + jobs: build_and_upload: - name: Build APKs and test APKs + name: Build APKs (app + test) runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 45 outputs: apks_artifact: apks steps: @@ -27,35 +35,34 @@ jobs: cp .github/ci-gradle.properties ~/.gradle/gradle.properties fi - - name: Set up JDK 17 + - name: Set up JDK 17 (with Gradle cache) uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: 17 + cache: gradle - - name: Setup Gradle (cache enabled) + - name: Setup Gradle (setup action with caching) uses: gradle/actions/setup-gradle@v4 with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} build-cache: true - - name: Build all APKs (assemble) - # This builds flavors / variants. It may take a while the first run. - run: ./gradlew :app:assemble -PminifyWithR8=false --no-daemon --no-parallel + - name: Build only the needed variants (app + test) + run: | + # Build only app demo debug and its test APK to save time + ./gradlew :app:assembleDemoDebug :app:assembleDemoDebugAndroidTest --no-daemon --no-parallel - - name: Find APKs and test APKs - id: find-apks + - name: Collect APKs + id: collect_apks run: | - echo "Searching for APKs under app/build/outputs..." - find ./app/build/outputs -type f -name "*.apk" -print || true - # Collect all APKs (app & androidTest apks) mkdir -p apk_collection - # find apks (exclude mapping & intermediates) - find ./app/build/outputs -type f -name "*.apk" -not -path "*/mapping/*" -not -path "*/outputs/mapping/*" -exec cp {} apk_collection/ \; - echo "Files copied to apk_collection:" + # copy all apk outputs (app and androidTest apks) + find ./app/build/outputs -type f -name "*.apk" -not -path "*/mapping/*" -exec cp {} apk_collection/ \; || true + echo "Collected files:" ls -lah apk_collection || true if [ $(ls -A apk_collection | wc -l) -eq 0 ]; then - echo "No APKs found! failing." + echo "::error::No APKs found in app/build/outputs; failing." exit 1 fi echo "::set-output name=apk_dir::apk_collection" @@ -67,7 +74,7 @@ jobs: path: apk_collection/** emulator_run: - name: Run emulator, install APKs, run instrumentation & record + name: Run emulator, install APKs, run tests & record needs: build_and_upload runs-on: ubuntu-latest timeout-minutes: 60 @@ -75,13 +82,26 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Download built APKs + - name: Download APKs artifact uses: actions/download-artifact@v4 with: name: apks path: ./apks - - name: Enable KVM on runner (Linux) + - name: Prepare artifacts dir + run: mkdir -p artifacts + + - name: Restore AVD cache + uses: actions/cache@v4 + with: + path: | + ~/.android/avd + ~/.android/adb* + key: avd-${{ env.EMULATOR_API_LEVEL }}-${{ runner.os }}-${{ hashFiles('**/.android/release-not-used') }} + restore-keys: | + avd-${{ env.EMULATOR_API_LEVEL }}-${{ runner.os }}- + + - name: Enable KVM (Linux runners) if: runner.os == 'Linux' run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -89,71 +109,68 @@ jobs: sudo udevadm trigger --name-match=kvm || true ls -la /dev/kvm || true - - name: Prepare artifacts dir - run: mkdir -p artifacts - - - name: Start emulator, install APKs, run tests and record (ReactiveCircus) + - name: Start emulator, install APKs, run tests & record (ReactiveCircus) uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 30 - arch: x86_64 + api-level: ${{ env.EMULATOR_API_LEVEL }} + arch: ${{ env.EMULATOR_ARCH }} force-avd-creation: true disable-animations: true emulator-options: -no-window -no-boot-anim -noaudio -gpu swiftshader_indirect script: | + bash -lc ' set -euo pipefail - echo "Waiting for emulator..." + echo "Waiting for emulator device to appear..." adb wait-for-device - until adb -s emulator-5554 shell getprop sys.boot_completed | grep -m1 "1"; do sleep 1; done + # find emulator serial robustly + EMU=$(adb devices | awk "/emulator/ && !/offline/ {print \$1; exit}") + if [ -z "$EMU" ]; then + # common fallback + EMU=emulator-5554 + fi + echo "Using emulator serial: $EMU" + + # wait for boot completed + until adb -s "$EMU" shell getprop sys.boot_completed | grep -m1 "1"; do sleep 1; done echo "Emulator booted." - # Install all apks found in ./apks (app and test apks) - echo "Installing APKs..." + # Install all APKs produced by build job + echo "Installing APKs from ./apks..." for f in ./apks/*.apk; do + [ -e "$f" ] || continue echo "Installing $f" - adb install -r "$f" || (echo "install failed for $f, continuing" && true) + adb -s "$EMU" install -r "$f" || echo "install failed for $f (continuing)" done - # Attempt to detect an instrumentation runner automatically - echo "Detecting installed instrumentation..." - INST_LIST="$(adb shell pm list instrumentation || true)" - echo "Instrumentation list: $INST_LIST" - # pick first instrumentation component (format: instrumentation: / (target=...)) - INST=$(echo "$INST_LIST" | sed -n 's/^instrumentation:\s*//p' | head -n1 | awk '{print $1}' | tr -d '\r') - if [ -z "$INST" ]; then - echo "No instrumentation found. Trying to detect test runner via package name fallback..." - # fallback: try to find any package that looks like demo or nowinandroid (best-effort) - PACKAGE=$(adb shell pm list packages | tr -d '\r' | grep -E 'nowinandroid|nia|nowin' | head -n1 | sed 's/package://g' || true) - echo "Detected package: $PACKAGE" - if [ -n "$PACKAGE" ]; then - # list instrumentation for that package specifically - INST=$(adb shell pm list instrumentation | tr -d '\r' | grep "$PACKAGE" | head -n1 | sed -n 's/^instrumentation:\s*//p' | awk '{print $1}' || true) - fi - fi + # Start a device screen recording in background (30s) + echo "Starting screenrecord (30s)" + adb -s "$EMU" shell screenrecord --time-limit 30 /sdcard/test_run.mp4 & - if [ -z "$INST" ]; then - echo "Couldn't auto-detect instrumentation runner. As fallback, will run a simple smoke launch." - # attempt to start main activity (best-effort) - adb shell monkey -p com.google.samples.apps.nowinandroid.dev -c android.intent.category.LAUNCHER 1 || true - else + # Attempt to run instrumentation tests via installed instrumentation runner (best effort) + # Try list instrumentation, if exists run first one + INST=$(adb -s "$EMU" shell pm list instrumentation | sed -n "s/^instrumentation://p" | head -n1 | tr -d "\\r") + if [ -n "$INST" ]; then echo "Running instrumentation: $INST" - # Start screenrecord in background on device (30s) - adb shell screenrecord --time-limit 30 /sdcard/test_run.mp4 & - # Run instrumentation - this will run tests already packaged in the test APK - adb shell am instrument -w "$INST" || true - # Give a moment for the recording to finish (screenrecord stops automatically after time-limit) - sleep 5 - # Pull the recording - adb pull /sdcard/test_run.mp4 ./artifacts/test_run.mp4 || true + adb -s "$EMU" shell am instrument -w "$INST" || echo "instrumentation failed (continuing)" + else + echo "No instrumentation found; performing smoke launch using monkey" + adb -s "$EMU" shell monkey -p com.google.samples.apps.nowinandroid.dev -c android.intent.category.LAUNCHER 1 || true fi - # Collect memory and gfx info and logs - adb shell dumpsys meminfo > ./artifacts/dumpsys_meminfo.txt || true - adb shell dumpsys gfxinfo com.google.samples.apps.nowinandroid.dev > ./artifacts/gfxinfo.txt || true - adb logcat -d > ./artifacts/logcat.txt || true + sleep 5 + # try to pull recording (if created) + adb -s "$EMU" pull /sdcard/test_run.mp4 ./artifacts/test_run.mp4 || echo "no recording found (maybe too short)" + + # Collect logs and dumpsys + adb -s "$EMU" shell dumpsys meminfo > ./artifacts/dumpsys_meminfo.txt || true + adb -s "$EMU" shell dumpsys gfxinfo com.google.samples.apps.nowinandroid.dev > ./artifacts/gfxinfo.txt || true + adb -s "$EMU" logcat -d > ./artifacts/logcat.txt || true + ' - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: emulator-artifacts - path: artifacts/**, apks/** + path: | + artifacts/** + apks/** From caaacda603a562442bbfcfbe23ac7eafbb8b96f7 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 18:35:52 +0200 Subject: [PATCH 03/39] maybe yes maybe no --- .github/workflows/Build.yaml | 202 +++++++++++------------------------ 1 file changed, 61 insertions(+), 141 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 689e3dc54..74faa7698 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,176 +1,96 @@ -name: Build APKs + Emulator test (push to test) +name: Macrobenchmark + Screen Record (AVD Cached) on: push: - branches: - - testNIA - workflow_dispatch: - -concurrency: - group: emulator-test-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -env: - EMULATOR_API_LEVEL: 30 - EMULATOR_ARCH: x86_64 + branches: [testNIA] jobs: - build_and_upload: - name: Build APKs (app + test) + macrobenchmark: runs-on: ubuntu-latest - timeout-minutes: 45 - outputs: - apks_artifact: apks + + strategy: + matrix: + api-level: [30] + steps: - name: Checkout uses: actions/checkout@v4 - - name: Copy CI gradle.properties (if exists) + - name: Enable KVM run: | - if [ -f .github/ci-gradle.properties ]; then - mkdir -p ~/.gradle - cp .github/ci-gradle.properties ~/.gradle/gradle.properties - fi + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - - name: Set up JDK 17 (with Gradle cache) + - name: Set up JDK 17 uses: actions/setup-java@v5 with: - distribution: 'zulu' + distribution: zulu java-version: 17 cache: gradle - - name: Setup Gradle (setup action with caching) - uses: gradle/actions/setup-gradle@v4 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - build-cache: true - - - name: Build only the needed variants (app + test) + - name: Build benchmark variants run: | - # Build only app demo debug and its test APK to save time - ./gradlew :app:assembleDemoDebug :app:assembleDemoDebugAndroidTest --no-daemon --no-parallel - - - name: Collect APKs - id: collect_apks - run: | - mkdir -p apk_collection - # copy all apk outputs (app and androidTest apks) - find ./app/build/outputs -type f -name "*.apk" -not -path "*/mapping/*" -exec cp {} apk_collection/ \; || true - echo "Collected files:" - ls -lah apk_collection || true - if [ $(ls -A apk_collection | wc -l) -eq 0 ]; then - echo "::error::No APKs found in app/build/outputs; failing." - exit 1 - fi - echo "::set-output name=apk_dir::apk_collection" - - - name: Upload APKs artifact - uses: actions/upload-artifact@v4 - with: - name: apks - path: apk_collection/** - - emulator_run: - name: Run emulator, install APKs, run tests & record - needs: build_and_upload - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download APKs artifact - uses: actions/download-artifact@v4 - with: - name: apks - path: ./apks - - - name: Prepare artifacts dir - run: mkdir -p artifacts + ./gradlew :app:assembleDemoBenchmark \ + :macrobenchmark:assembleDemoBenchmark - name: Restore AVD cache uses: actions/cache@v4 + id: avd-cache with: path: | ~/.android/avd ~/.android/adb* - key: avd-${{ env.EMULATOR_API_LEVEL }}-${{ runner.os }}-${{ hashFiles('**/.android/release-not-used') }} - restore-keys: | - avd-${{ env.EMULATOR_API_LEVEL }}-${{ runner.os }}- + key: avd-api-${{ matrix.api-level }} - - name: Enable KVM (Linux runners) - if: runner.os == 'Linux' - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm || true - ls -la /dev/kvm || true - - - name: Start emulator, install APKs, run tests & record (ReactiveCircus) + - name: Create AVD snapshot (cache miss only) + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: ${{ matrix.api-level }} + target: google_apis + arch: x86_64 + force-avd-creation: false + disable-animations: true + emulator-options: > + -no-window + -noaudio + -no-boot-anim + -gpu swiftshader_indirect + script: echo "AVD snapshot created" + + - name: Run macrobenchmark + screen record uses: reactivecircus/android-emulator-runner@v2 with: - api-level: ${{ env.EMULATOR_API_LEVEL }} - arch: ${{ env.EMULATOR_ARCH }} - force-avd-creation: true + api-level: ${{ matrix.api-level }} + target: google_apis + arch: x86_64 + force-avd-creation: false disable-animations: true - emulator-options: -no-window -no-boot-anim -noaudio -gpu swiftshader_indirect + emulator-options: > + -no-window + -noaudio + -no-boot-anim + -gpu swiftshader_indirect script: | - bash -lc ' - set -euo pipefail - echo "Waiting for emulator device to appear..." - adb wait-for-device - # find emulator serial robustly - EMU=$(adb devices | awk "/emulator/ && !/offline/ {print \$1; exit}") - if [ -z "$EMU" ]; then - # common fallback - EMU=emulator-5554 - fi - echo "Using emulator serial: $EMU" - - # wait for boot completed - until adb -s "$EMU" shell getprop sys.boot_completed | grep -m1 "1"; do sleep 1; done - echo "Emulator booted." - - # Install all APKs produced by build job - echo "Installing APKs from ./apks..." - for f in ./apks/*.apk; do - [ -e "$f" ] || continue - echo "Installing $f" - adb -s "$EMU" install -r "$f" || echo "install failed for $f (continuing)" - done - - # Start a device screen recording in background (30s) - echo "Starting screenrecord (30s)" - adb -s "$EMU" shell screenrecord --time-limit 30 /sdcard/test_run.mp4 & - - # Attempt to run instrumentation tests via installed instrumentation runner (best effort) - # Try list instrumentation, if exists run first one - INST=$(adb -s "$EMU" shell pm list instrumentation | sed -n "s/^instrumentation://p" | head -n1 | tr -d "\\r") - if [ -n "$INST" ]; then - echo "Running instrumentation: $INST" - adb -s "$EMU" shell am instrument -w "$INST" || echo "instrumentation failed (continuing)" - else - echo "No instrumentation found; performing smoke launch using monkey" - adb -s "$EMU" shell monkey -p com.google.samples.apps.nowinandroid.dev -c android.intent.category.LAUNCHER 1 || true - fi + echo "Start screen recording" + adb shell screenrecord /sdcard/benchmark.mp4 & + + ./gradlew :macrobenchmark:connectedDemoBenchmarkAndroidTest - sleep 5 - # try to pull recording (if created) - adb -s "$EMU" pull /sdcard/test_run.mp4 ./artifacts/test_run.mp4 || echo "no recording found (maybe too short)" + adb shell pkill -INT screenrecord || true + sleep 3 + adb pull /sdcard/benchmark.mp4 benchmark.mp4 - # Collect logs and dumpsys - adb -s "$EMU" shell dumpsys meminfo > ./artifacts/dumpsys_meminfo.txt || true - adb -s "$EMU" shell dumpsys gfxinfo com.google.samples.apps.nowinandroid.dev > ./artifacts/gfxinfo.txt || true - adb -s "$EMU" logcat -d > ./artifacts/logcat.txt || true - ' - - - name: Upload artifacts + - name: Upload benchmark reports uses: actions/upload-artifact@v4 with: - name: emulator-artifacts + name: macrobenchmark-reports path: | - artifacts/** - apks/** + **/build/outputs/benchmarkReports/** + + - name: Upload screen recording + uses: actions/upload-artifact@v4 + with: + name: benchmark-video + path: benchmark.mp4 From adba95e7b20c6d62ce9fefeb245ad77e8a327db6 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 18:47:37 +0200 Subject: [PATCH 04/39] one more time --- .github/workflows/Build.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 74faa7698..da35d30f7 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -31,8 +31,9 @@ jobs: - name: Build benchmark variants run: | - ./gradlew :app:assembleDemoBenchmark \ - :macrobenchmark:assembleDemoBenchmark + ./gradlew \ + :app:assembleDemoBenchmark \ + :benchmark:assembleDemoBenchmark - name: Restore AVD cache uses: actions/cache@v4 @@ -76,7 +77,7 @@ jobs: echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - ./gradlew :macrobenchmark:connectedDemoBenchmarkAndroidTest + ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest adb shell pkill -INT screenrecord || true sleep 3 From b5ae6d4f286a25119ddfbfb801ff59e99c995ffe Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 19:10:46 +0200 Subject: [PATCH 05/39] one more time#2 --- .github/workflows/Build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index da35d30f7..4f5b52479 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -77,7 +77,7 @@ jobs: echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest + ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest || true adb shell pkill -INT screenrecord || true sleep 3 From c2423bcb54eddbd3b302d42b0b701b8398566674 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 19:58:57 +0200 Subject: [PATCH 06/39] again #3 --- .github/workflows/Build.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 4f5b52479..5a419351c 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -77,7 +77,10 @@ jobs: echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest || true + ./gradlew \ + :benchmark:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ + || true adb shell pkill -INT screenrecord || true sleep 3 From 68deefbe3198763ecd51d6470716cde39deff2f9 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 20:12:13 +0200 Subject: [PATCH 07/39] test again --- .github/workflows/Build.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 5a419351c..a00585ef6 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -77,10 +77,8 @@ jobs: echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - ./gradlew \ - :benchmark:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ - || true + ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true + adb shell pkill -INT screenrecord || true sleep 3 From cfa5434496912768e11baed6d38a02a051de5644 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 20:30:31 +0200 Subject: [PATCH 08/39] bruv --- .github/workflows/Build.yaml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index a00585ef6..8c215d951 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,4 +1,4 @@ -name: Macrobenchmark + Screen Record (AVD Cached) +name: Macrobenchmark + Screen Record (Emulator With Display) on: push: @@ -52,12 +52,10 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: false - disable-animations: true + emulator-options: > - -no-window - -noaudio - -no-boot-anim - -gpu swiftshader_indirect + -gpu host + disable-animations: true script: echo "AVD snapshot created" - name: Run macrobenchmark + screen record @@ -67,18 +65,17 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: false - disable-animations: true + emulator-options: > - -no-window - -noaudio - -no-boot-anim - -gpu swiftshader_indirect + -gpu host + disable-animations: true script: | echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - - ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true + # Run benchmarks (suppress emulator error so it runs completely) + ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true adb shell pkill -INT screenrecord || true sleep 3 @@ -88,8 +85,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: macrobenchmark-reports - path: | - **/build/outputs/benchmarkReports/** + path: benchmarks/build/reports/androidTests/connected/ - name: Upload screen recording uses: actions/upload-artifact@v4 From 37a45906a0f5d248cf012e9b543e6c7ba7dcdba8 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 20:59:14 +0200 Subject: [PATCH 09/39] maybe pls --- .github/workflows/Build.yaml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 8c215d951..439d6c65e 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,4 +1,4 @@ -name: Macrobenchmark + Screen Record (Emulator With Display) +name: Macrobenchmark + Screen Record (Emulator API 35) on: push: @@ -10,7 +10,7 @@ jobs: strategy: matrix: - api-level: [30] + api-level: [35] steps: - name: Checkout @@ -18,10 +18,16 @@ jobs: - name: Enable KVM run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm + - name: Install emulator dependencies + run: | + sudo apt-get update + sudo apt-get install -y libpulse0 + - name: Set up JDK 17 uses: actions/setup-java@v5 with: @@ -35,16 +41,16 @@ jobs: :app:assembleDemoBenchmark \ :benchmark:assembleDemoBenchmark - - name: Restore AVD cache + - name: AVD cache uses: actions/cache@v4 id: avd-cache with: path: | - ~/.android/avd + ~/.android/avd/* ~/.android/adb* - key: avd-api-${{ matrix.api-level }} + key: avd-${{ matrix.api-level }} - - name: Create AVD snapshot (cache miss only) + - name: Create AVD snapshot (only on cache miss) if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 with: @@ -52,11 +58,9 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: false - - emulator-options: > - -gpu host + emulator-options: -no-window -gpu host disable-animations: true - script: echo "AVD snapshot created" + script: echo "Snapshot created" - name: Run macrobenchmark + screen record uses: reactivecircus/android-emulator-runner@v2 @@ -65,17 +69,16 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: false - - emulator-options: > - -gpu host + emulator-options: -no-snapshot-save -no-window -gpu host disable-animations: true script: | echo "Start screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - # Run benchmarks (suppress emulator error so it runs completely) + # Run macrobenchmarks (suppress emulator error, non-blocking) ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ + || true adb shell pkill -INT screenrecord || true sleep 3 From e7012382a59092ab30e16d2be552182241afdfb9 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 21:18:35 +0200 Subject: [PATCH 10/39] yarab --- .github/workflows/Build.yaml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 439d6c65e..54297abc1 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -13,20 +13,20 @@ jobs: api-level: [35] steps: - - name: Checkout + - name: Checkout code uses: actions/checkout@v4 - - name: Enable KVM + - name: Enable KVM for emulator run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | \ + sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Install emulator dependencies + - name: Install emulator system dependencies run: | sudo apt-get update - sudo apt-get install -y libpulse0 + sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1-mesa-glx libgl1-mesa-dri - name: Set up JDK 17 uses: actions/setup-java@v5 @@ -35,13 +35,12 @@ jobs: java-version: 17 cache: gradle - - name: Build benchmark variants + - name: Build benchmark APKs run: | - ./gradlew \ - :app:assembleDemoBenchmark \ - :benchmark:assembleDemoBenchmark + ./gradlew :app:assembleDemoBenchmark \ + :benchmark:assembleDemoBenchmark - - name: AVD cache + - name: Cache AVD uses: actions/cache@v4 id: avd-cache with: @@ -50,7 +49,7 @@ jobs: ~/.android/adb* key: avd-${{ matrix.api-level }} - - name: Create AVD snapshot (only on cache miss) + - name: Create emulator snapshot (only if not cached) if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 with: @@ -60,7 +59,7 @@ jobs: force-avd-creation: false emulator-options: -no-window -gpu host disable-animations: true - script: echo "Snapshot created" + script: echo "Snapshot created." - name: Run macrobenchmark + screen record uses: reactivecircus/android-emulator-runner@v2 @@ -72,10 +71,10 @@ jobs: emulator-options: -no-snapshot-save -no-window -gpu host disable-animations: true script: | - echo "Start screen recording" + echo "Starting screen recording" adb shell screenrecord /sdcard/benchmark.mp4 & - # Run macrobenchmarks (suppress emulator error, non-blocking) + # Run macrobenchmark tests (suppress emulator error, non-blocking) ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ || true @@ -84,7 +83,7 @@ jobs: sleep 3 adb pull /sdcard/benchmark.mp4 benchmark.mp4 - - name: Upload benchmark reports + - name: Upload macrobenchmark reports uses: actions/upload-artifact@v4 with: name: macrobenchmark-reports From b4f93128fcd164af8b2c241cfa295fea26fd6c36 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 21:20:42 +0200 Subject: [PATCH 11/39] yarab#2 --- .github/workflows/Build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 54297abc1..a57163f09 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -26,7 +26,7 @@ jobs: - name: Install emulator system dependencies run: | sudo apt-get update - sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1-mesa-glx libgl1-mesa-dri + sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1 libglx-mesa0 libgl1-mesa-dri - name: Set up JDK 17 uses: actions/setup-java@v5 From f1129470aa406641a8d91437c607620b1788fb88 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 21:54:01 +0200 Subject: [PATCH 12/39] yarab#2 --- .github/workflows/Build.yaml | 102 ++++++++++++++++------------------- 1 file changed, 45 insertions(+), 57 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index a57163f09..f46ebcefb 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -7,90 +7,78 @@ on: jobs: macrobenchmark: runs-on: ubuntu-latest - - strategy: - matrix: - api-level: [35] + timeout-minutes: 60 steps: - name: Checkout code uses: actions/checkout@v4 - - name: Enable KVM for emulator - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | \ - sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '17' + cache: 'gradle' - - name: Install emulator system dependencies + - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1 libglx-mesa0 libgl1-mesa-dri - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: zulu - java-version: 17 - cache: gradle + sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1-mesa-glx - name: Build benchmark APKs run: | ./gradlew :app:assembleDemoBenchmark \ - :benchmark:assembleDemoBenchmark - - - name: Cache AVD - uses: actions/cache@v4 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: avd-${{ matrix.api-level }} + :benchmarks:assembleDemoBenchmark \ + -x :app:generateDemoBenchmarkOssLicenses \ + -x :benchmarks:generateDemoBenchmarkOssLicenses - - name: Create emulator snapshot (only if not cached) - if: steps.avd-cache.outputs.cache-hit != 'true' + - name: Run Macrobenchmarks uses: reactivecircus/android-emulator-runner@v2 with: - api-level: ${{ matrix.api-level }} + api-level: 35 target: google_apis arch: x86_64 force-avd-creation: false - emulator-options: -no-window -gpu host - disable-animations: true - script: echo "Snapshot created." - - - name: Run macrobenchmark + screen record - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: ${{ matrix.api-level }} - target: google_apis - arch: x86_64 - force-avd-creation: false - emulator-options: -no-snapshot-save -no-window -gpu host disable-animations: true + emulator-options: -no-window -gpu host -no-audio -no-boot-anim script: | - echo "Starting screen recording" + # Ensure the emulator is fully booted before running commands + adb wait-for-device + + echo "Starting screen recording..." + # Start recording in background adb shell screenrecord /sdcard/benchmark.mp4 & - - # Run macrobenchmark tests (suppress emulator error, non-blocking) - ./gradlew :benchmark:connectedDemoBenchmarkAndroidTest \ + + echo "Running benchmarks..." + # We use the 'benchmarks' module name common in NiA + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ - || true + || echo "Benchmarks failed, but we will still try to pull the video." - adb shell pkill -INT screenrecord || true - sleep 3 - adb pull /sdcard/benchmark.mp4 benchmark.mp4 + echo "Stopping recording..." + # SIGINT (2) allows the MP4 container to close properly + adb shell pkill -2 screenrecord || true + sleep 5 + + echo "Pulling files from device..." + adb pull /sdcard/benchmark.mp4 . || true + + # Pull logcat for extra debugging + adb logcat -d > logcat.txt - - name: Upload macrobenchmark reports + - name: Upload Macrobenchmark Reports + if: always() uses: actions/upload-artifact@v4 with: - name: macrobenchmark-reports - path: benchmarks/build/reports/androidTests/connected/ + name: macrobenchmark-results + path: | + **/build/outputs/connected_android_test_additional_output/**/*.json + **/build/reports/androidTests/connected/ + logcat.txt - - name: Upload screen recording + - name: Upload Screen Recording + if: always() uses: actions/upload-artifact@v4 with: name: benchmark-video - path: benchmark.mp4 + path: benchmark.mp4 \ No newline at end of file From 58b3d02a711e4b063ee028a4b7cfebbdaec15e9e Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 21:58:20 +0200 Subject: [PATCH 13/39] yarab#3 --- .github/workflows/Build.yaml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index f46ebcefb..da36ec65b 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -23,14 +23,14 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1-mesa-glx + # Updated for Ubuntu 24.04: libgl1-mesa-glx is now libgl1 + sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1 libglx-mesa0 - name: Build benchmark APKs run: | ./gradlew :app:assembleDemoBenchmark \ :benchmarks:assembleDemoBenchmark \ - -x :app:generateDemoBenchmarkOssLicenses \ - -x :benchmarks:generateDemoBenchmarkOssLicenses + -x "generate*OssLicenses" - name: Run Macrobenchmarks uses: reactivecircus/android-emulator-runner@v2 @@ -42,29 +42,23 @@ jobs: disable-animations: true emulator-options: -no-window -gpu host -no-audio -no-boot-anim script: | - # Ensure the emulator is fully booted before running commands adb wait-for-device echo "Starting screen recording..." - # Start recording in background adb shell screenrecord /sdcard/benchmark.mp4 & echo "Running benchmarks..." - # We use the 'benchmarks' module name common in NiA ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ || echo "Benchmarks failed, but we will still try to pull the video." echo "Stopping recording..." - # SIGINT (2) allows the MP4 container to close properly adb shell pkill -2 screenrecord || true sleep 5 - echo "Pulling files from device..." - adb pull /sdcard/benchmark.mp4 . || true - - # Pull logcat for extra debugging - adb logcat -d > logcat.txt + echo "Pulling files..." + adb pull /sdcard/benchmark.mp4 . || touch benchmark.mp4 + adb logcat -d > logcat.txt || touch logcat.txt - name: Upload Macrobenchmark Reports if: always() @@ -72,7 +66,7 @@ jobs: with: name: macrobenchmark-results path: | - **/build/outputs/connected_android_test_additional_output/**/*.json + **/build/outputs/**/*.json **/build/reports/androidTests/connected/ logcat.txt From c186975597de4a7bfa3f928f7b9309825dd33e06 Mon Sep 17 00:00:00 2001 From: Moe Date: Tue, 27 Jan 2026 22:01:52 +0200 Subject: [PATCH 14/39] yarab#3 --- .github/workflows/Build.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index da36ec65b..02886b7ac 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -23,14 +23,14 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - # Updated for Ubuntu 24.04: libgl1-mesa-glx is now libgl1 + # Using the verified packages for Ubuntu 24.04 sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1 libglx-mesa0 - name: Build benchmark APKs run: | ./gradlew :app:assembleDemoBenchmark \ :benchmarks:assembleDemoBenchmark \ - -x "generate*OssLicenses" + --stacktrace - name: Run Macrobenchmarks uses: reactivecircus/android-emulator-runner@v2 @@ -45,12 +45,16 @@ jobs: adb wait-for-device echo "Starting screen recording..." + # Using /sdcard/ is standard for screenrecord permissions adb shell screenrecord /sdcard/benchmark.mp4 & echo "Running benchmarks..." + # We add the license exclusion here to prevent the GMS crash during testing ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ + -x :app:generateDemoBenchmarkOssLicenses \ + -x :benchmarks:generateDemoBenchmarkOssLicenses \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ - || echo "Benchmarks failed, but we will still try to pull the video." + || echo "Benchmarks finished with errors, pulling data anyway..." echo "Stopping recording..." adb shell pkill -2 screenrecord || true From 683dfb16008243b9f6f7166627b3ad20002f83d7 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 01:31:37 +0200 Subject: [PATCH 15/39] yarab#4 --- .github/workflows/Build.yaml | 73 ++++++++++++++---------------------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 02886b7ac..f91329c55 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,13 +1,13 @@ -name: Macrobenchmark + Screen Record (Emulator API 35) +name: Benchmark on Emulator on: push: branches: [testNIA] jobs: - macrobenchmark: + benchmark-android: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 45 steps: - name: Checkout code @@ -20,63 +20,48 @@ jobs: java-version: '17' cache: 'gradle' - - name: Install system dependencies + - name: Build Benchmark APKs run: | - sudo apt-get update - # Using the verified packages for Ubuntu 24.04 - sudo apt-get install -y libpulse0 libvulkan1 mesa-utils libgl1 libglx-mesa0 + ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - - name: Build benchmark APKs - run: | - ./gradlew :app:assembleDemoBenchmark \ - :benchmarks:assembleDemoBenchmark \ - --stacktrace + - name: AVD Cache + uses: actions/cache@v4 + id: avd-cache + with: + path: | + ~/.android/avd/* + ~/.android/adb* + key: avd-33-x86_64-google_apis - - name: Run Macrobenchmarks + - name: Run Benchmarks & Record Video uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 35 + api-level: 33 target: google_apis arch: x86_64 force-avd-creation: false - disable-animations: true - emulator-options: -no-window -gpu host -no-audio -no-boot-anim + # We use swiftshader_indirect to avoid the "device not found" crash + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: true script: | - adb wait-for-device + # 1. Start recording + adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - echo "Starting screen recording..." - # Using /sdcard/ is standard for screenrecord permissions - adb shell screenrecord /sdcard/benchmark.mp4 & - - echo "Running benchmarks..." - # We add the license exclusion here to prevent the GMS crash during testing + # 2. Run the tests (Using the APKs we built in STEP A) ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -x :app:generateDemoBenchmarkOssLicenses \ - -x :benchmarks:generateDemoBenchmarkOssLicenses \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR \ - || echo "Benchmarks finished with errors, pulling data anyway..." + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - echo "Stopping recording..." + # 3. Finalize video + sleep 2 adb shell pkill -2 screenrecord || true sleep 5 - - echo "Pulling files..." - adb pull /sdcard/benchmark.mp4 . || touch benchmark.mp4 - adb logcat -d > logcat.txt || touch logcat.txt + adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 - - name: Upload Macrobenchmark Reports + - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: - name: macrobenchmark-results + name: benchmark-package path: | - **/build/outputs/**/*.json - **/build/reports/androidTests/connected/ - logcat.txt - - - name: Upload Screen Recording - if: always() - uses: actions/upload-artifact@v4 - with: - name: benchmark-video - path: benchmark.mp4 \ No newline at end of file + benchmark_video.mp4 + benchmarks/build/outputs/connected_android_test_additional_output/ \ No newline at end of file From ea305cdb925636c38e5c7676680d8958c5fe2ff2 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 01:47:46 +0200 Subject: [PATCH 16/39] yarab#4_gemini --- .github/workflows/Build.yaml | 48 +++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index f91329c55..16451e3e0 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -20,38 +20,52 @@ jobs: java-version: '17' cache: 'gradle' + # 1. CRITICAL: Free up disk space. + # API 30+ emulators need ~10GB. Standard runners are often too full. + - name: Free Disk Space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + # 2. CRITICAL: Enable KVM group permissions + # This ensures the emulator can actually use the CPU's virtualization features. + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Build Benchmark APKs run: | ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - - name: AVD Cache - uses: actions/cache@v4 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: avd-33-x86_64-google_apis - - name: Run Benchmarks & Record Video uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 33 + api-level: 31 # API 31 is lighter and boots faster than 33/35 target: google_apis arch: x86_64 - force-avd-creation: false - # We use swiftshader_indirect to avoid the "device not found" crash - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + force-avd-creation: true # Resetting AVD to ensure no corrupted cache + # Added -accel on and -memory to ensure it has enough power to boot + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -accel on -memory 3072 disable-animations: true script: | - # 1. Start recording + # 1. Wait for boot to be REALLY finished + adb wait-for-device + + # 2. Start recording + echo "Starting video recording..." adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # 2. Run the tests (Using the APKs we built in STEP A) + # 3. Run the tests + echo "Running Macrobenchmarks..." ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - # 3. Finalize video + # 4. Finalize + echo "Pulling video..." sleep 2 adb shell pkill -2 screenrecord || true sleep 5 @@ -61,7 +75,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: benchmark-package + name: benchmark-results path: | benchmark_video.mp4 benchmarks/build/outputs/connected_android_test_additional_output/ \ No newline at end of file From e32e6f3fd577d3ab27e47a141d5d275e084f60e4 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 01:51:32 +0200 Subject: [PATCH 17/39] yarab#5_gemini --- .github/workflows/Build.yaml | 39 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 16451e3e0..482a9d3fc 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -13,6 +13,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Free Disk Space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + # We do NOT delete $AGENT_TOOLSDIRECTORY here because that's where Java goes next + - name: Set up JDK 17 uses: actions/setup-java@v4 with: @@ -20,17 +27,9 @@ jobs: java-version: '17' cache: 'gradle' - # 1. CRITICAL: Free up disk space. - # API 30+ emulators need ~10GB. Standard runners are often too full. - - name: Free Disk Space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Grant execute permission for gradlew + run: chmod +x gradlew - # 2. CRITICAL: Enable KVM group permissions - # This ensures the emulator can actually use the CPU's virtualization features. - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -44,33 +43,31 @@ jobs: - name: Run Benchmarks & Record Video uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 31 # API 31 is lighter and boots faster than 33/35 + api-level: 31 target: google_apis arch: x86_64 - force-avd-creation: true # Resetting AVD to ensure no corrupted cache - # Added -accel on and -memory to ensure it has enough power to boot - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -accel on -memory 3072 + force-avd-creation: true + # Added extra memory to prevent the "offline" crash + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 3072 disable-animations: true script: | - # 1. Wait for boot to be REALLY finished + # Ensure the device is ready adb wait-for-device - # 2. Start recording - echo "Starting video recording..." + # Start recording (3 minute limit) adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # 3. Run the tests - echo "Running Macrobenchmarks..." + # Run the actual tests ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - # 4. Finalize - echo "Pulling video..." + # Pull the video back to the runner sleep 2 adb shell pkill -2 screenrecord || true sleep 5 adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 + # STEP 7: UPLOAD - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 From bb6e3642dbb2ba5e85fa792a0bb818d1dc7dd71d Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 02:05:50 +0200 Subject: [PATCH 18/39] yarab#6_gemini --- .github/workflows/Build.yaml | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 482a9d3fc..781c9b585 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -18,7 +18,6 @@ jobs: sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" - # We do NOT delete $AGENT_TOOLSDIRECTORY here because that's where Java goes next - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -27,7 +26,7 @@ jobs: java-version: '17' cache: 'gradle' - - name: Grant execute permission for gradlew + - name: Grant execute permission run: chmod +x gradlew - name: Enable KVM @@ -37,37 +36,30 @@ jobs: sudo udevadm trigger --name-match=kvm - name: Build Benchmark APKs - run: | - ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark + run: ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - name: Run Benchmarks & Record Video uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 31 + api-level: 30 target: google_apis arch: x86_64 force-avd-creation: true - # Added extra memory to prevent the "offline" crash - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 3072 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 3584 disable-animations: true script: | - # Ensure the device is ready - adb wait-for-device - - # Start recording (3 minute limit) + # 1. Start recording in background adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # Run the actual tests - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + # 2. Run the tests + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - # Pull the video back to the runner + # 3. Pull the video sleep 2 adb shell pkill -2 screenrecord || true sleep 5 adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 - # STEP 7: UPLOAD - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 From 88819ad11e79a3b7710277db8c67ddf44588b58b Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 02:53:40 +0200 Subject: [PATCH 19/39] yarab#7_gemini --- .github/workflows/Build.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 781c9b585..847282e6f 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -45,20 +45,26 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 3584 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | # 1. Start recording in background adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # 2. Run the tests - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + # 2. Run ONLY the StartupBenchmark class (filters out the 15+ tests) + # '|| true' ensures the script continues so we can PULL the video even on failure + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true - # 3. Pull the video - sleep 2 + # 3. Pull the video (Adding extra sleep to ensure recording finishes) + echo "Stopping recording and pulling video..." adb shell pkill -2 screenrecord || true sleep 5 - adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 + adb pull /sdcard/benchmark_video.mp4 . + + # 4. Verify the file exists for the next step + ls -lh benchmark_video.mp4 - name: Upload Artifacts if: always() From 216369a79d7df1d79292b8e2ebbde1b5245f22c2 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 03:05:33 +0200 Subject: [PATCH 20/39] yarab#8_gemini --- .github/workflows/Build.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 847282e6f..d64c89aec 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -48,23 +48,17 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | - # 1. Start recording in background + # 1. Start recording adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # 2. Run ONLY the StartupBenchmark class (filters out the 15+ tests) - # '|| true' ensures the script continues so we can PULL the video even on failure - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true + # 2. Run ONLY the Startup tests (ONE CONTINUOUS LINE - NO BACKSLASHES) + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true - # 3. Pull the video (Adding extra sleep to ensure recording finishes) - echo "Stopping recording and pulling video..." + # 3. Pull the video (This will run even if the tests above fail) + echo "Pulling video file..." adb shell pkill -2 screenrecord || true sleep 5 - adb pull /sdcard/benchmark_video.mp4 . - - # 4. Verify the file exists for the next step - ls -lh benchmark_video.mp4 + adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 - name: Upload Artifacts if: always() From 8134706afc125d54b7a991996c30605cdce7dcd1 Mon Sep 17 00:00:00 2001 From: Moe Date: Wed, 28 Jan 2026 16:23:19 +0200 Subject: [PATCH 21/39] removed video recording --- .github/workflows/Build.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index d64c89aec..94d252ead 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -49,16 +49,16 @@ jobs: disable-animations: true script: | # 1. Start recording - adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & + # adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - # 2. Run ONLY the Startup tests (ONE CONTINUOUS LINE - NO BACKSLASHES) + # 2. Run ONLY the Startup tests ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true # 3. Pull the video (This will run even if the tests above fail) - echo "Pulling video file..." - adb shell pkill -2 screenrecord || true - sleep 5 - adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 + # echo "Pulling video file..." + # adb shell pkill -2 screenrecord || true + # sleep 5 + # adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 - name: Upload Artifacts if: always() @@ -66,5 +66,5 @@ jobs: with: name: benchmark-results path: | - benchmark_video.mp4 + # benchmark_video.mp4 benchmarks/build/outputs/connected_android_test_additional_output/ \ No newline at end of file From 9e0861d7c2a3bc0b29c49eb3e56070c3c85775fd Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Wed, 28 Jan 2026 19:14:39 +0200 Subject: [PATCH 22/39] feat: testing ways to mitigate variability These ways are: alternate benchmarking and the step fit algorithm, I also removed the comments for emulator screen recording, and I now only store the JSON files that we care about without the other perfetto traces and text files --- .github/scripts/step_fit.py | 72 ++++++++++++++++++++++++++++++++++++ .github/workflows/Build.yaml | 61 ++++++++++++++++++++++-------- 2 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 .github/scripts/step_fit.py diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py new file mode 100644 index 000000000..48f062df0 --- /dev/null +++ b/.github/scripts/step_fit.py @@ -0,0 +1,72 @@ +import json +import math +import glob +import os + +# ----------- CONFIG ----------- +RESULTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/collected" +BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" +METRIC_KEY = "timeToInitialDisplayMs" +# ------------------------------ + +def sum_squared_error(values): + avg = sum(values) / len(values) + return sum((v - avg) ** 2 for v in values) + +def step_fit(before, after): + total_squared_error = sum_squared_error(before) + sum_squared_error(after) + step_error = math.sqrt(total_squared_error) / (len(before) + len(after)) + if step_error == 0.0: + return 0.0 + return (sum(before) / len(before) - sum(after) / len(after)) / step_error + +def extract_median_from_file(path): + with open(path, "r") as f: + data = json.load(f) + for bench in data.get("benchmarks", []): + if bench.get("name") == BENCHMARK_NAME: + metrics = bench.get("metrics", {}) + metric = metrics.get(METRIC_KEY, {}) + return metric.get("median") + raise ValueError(f"Metric not found in {path}") + +def main(): + before = [] + after = [] + + json_files = sorted(glob.glob(os.path.join(RESULTS_DIR, "*.json"))) + + if len(json_files) == 0: + raise RuntimeError("No JSON files found.") + + for path in json_files: + median = extract_median_from_file(path) + filename = os.path.basename(path).lower() + if "v1" in filename: + before.append(median) + elif "v2" in filename: + after.append(median) + else: + print(f"Skipping file with unknown label: {filename}") + print(f"{filename}: median={median:.3f} ms") + + if len(before) != 5 or len(after) != 5: + raise RuntimeError(f"Expected 5 runs each, got v1={len(before)}, v2={len(after)}") + + result = step_fit(before, after) + + print("\n-----------------------------") + print(f"v1 medians: {before}") + print(f"v2 medians: {after}") + print(f"Step Fit Result: {result:.4f}") + print("-----------------------------") + + if abs(result) <= 25: + print("➡️ Difference is within noise range (low confidence of real regression)") + elif result > 0: + print("⚠️ v2 is slower than v1 (possible regression)") + else: + print("🚀 v2 is faster than v1 (possible improvement)") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 94d252ead..c3b81c731 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -38,7 +38,7 @@ jobs: - name: Build Benchmark APKs run: ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - - name: Run Benchmarks & Record Video + - name: Run Benchmarks uses: reactivecircus/android-emulator-runner@v2 with: api-level: 30 @@ -48,23 +48,54 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | - # 1. Start recording - # adb shell screenrecord --time-limit 180 /sdcard/benchmark_video.mp4 & - - # 2. Run ONLY the Startup tests - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR || true - - # 3. Pull the video (This will run even if the tests above fail) - # echo "Pulling video file..." - # adb shell pkill -2 screenrecord || true - # sleep 5 - # adb pull /sdcard/benchmark_video.mp4 benchmark_video.mp4 + set -e + + OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" + COLLECTED_DIR="$OUTPUT_DIR/collected" + mkdir -p "$COLLECTED_DIR" + + run_benchmark () { + VERSION_LABEL=$1 # v1 or v2 + RUN_NUMBER=$2 # 1..5 + + echo "==============================" + echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" + echo "==============================" + + # Clear app data to keep runs consistent + adb shell pm clear com.google.samples.apps.nowinandroid || true + + # Run only the Startup benchmark + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + + # Find the newest JSON result file + LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f -printf "%T@ %p\n" | sort -nr | head -n1 | cut -d' ' -f2-) + + if [ -z "$LATEST_JSON" ]; then + echo "Error: No benchmark JSON file found" + exit 1 + fi + + NEW_NAME="$COLLECTED_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" + cp "$LATEST_JSON" "$NEW_NAME" + + echo "Saved result to $NEW_NAME" + } + + # Alternate runs: v1, v2, v1, v2 ... + for i in 1 2 3 4 5; do + run_benchmark "v1" "$i" + run_benchmark "v2" "$i" + done + + - name: Run step fit + run: python3 .github/scripts/step_fit.py - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: name: benchmark-results - path: | - # benchmark_video.mp4 - benchmarks/build/outputs/connected_android_test_additional_output/ \ No newline at end of file + path: benchmarks/build/outputs/connected_android_test_additional_output/collected/ \ No newline at end of file From a783398695e37cbeae29ba6dd67f8f048c84e8c4 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Wed, 28 Jan 2026 19:29:19 +0200 Subject: [PATCH 23/39] fix: collected directory created correctly --- .github/workflows/Build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index c3b81c731..bd9a339aa 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -52,7 +52,6 @@ jobs: OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" COLLECTED_DIR="$OUTPUT_DIR/collected" - mkdir -p "$COLLECTED_DIR" run_benchmark () { VERSION_LABEL=$1 # v1 or v2 @@ -70,6 +69,8 @@ jobs: -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + mkdir -p "$COLLECTED_DIR" + # Find the newest JSON result file LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f -printf "%T@ %p\n" | sort -nr | head -n1 | cut -d' ' -f2-) From 9ce735cbd6b0dd865469772062cb8ae5236f70cb Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Wed, 28 Jan 2026 19:41:58 +0200 Subject: [PATCH 24/39] fix: use bash instead of sh --- .github/workflows/Build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index bd9a339aa..4e60c4327 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -48,6 +48,7 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | + #!/bin/bash set -e OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" From c6e57b42b67b0049e1977db9f2d12d201f9b507e Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Wed, 28 Jan 2026 20:03:03 +0200 Subject: [PATCH 25/39] fix: replace bash code with sh code --- .github/workflows/Build.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 4e60c4327..cc3ef71fd 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -48,13 +48,12 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | - #!/bin/bash set -e OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" COLLECTED_DIR="$OUTPUT_DIR/collected" - run_benchmark () { + run_benchmark() ( VERSION_LABEL=$1 # v1 or v2 RUN_NUMBER=$2 # 1..5 @@ -84,12 +83,14 @@ jobs: cp "$LATEST_JSON" "$NEW_NAME" echo "Saved result to $NEW_NAME" - } + ) # Alternate runs: v1, v2, v1, v2 ... - for i in 1 2 3 4 5; do + i=1 + while [ $i -le 5 ]; do run_benchmark "v1" "$i" run_benchmark "v2" "$i" + i=$((i + 1)) done - name: Run step fit From 455a30d56a022a3c9621ac759bc4c872014a1d42 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Tue, 3 Feb 2026 16:35:00 +0200 Subject: [PATCH 26/39] fix: move benchmarking code into a seperate script also doubled the emulator memory (cause GH Actions runner have a lot of memory so why not), and improved some variable names --- .github/scripts/run_macrobenchmarks.sh | 43 ++++++++++++++++++++++ .github/scripts/step_fit.py | 4 +-- .github/workflows/Build.yaml | 50 +++----------------------- 3 files changed, 49 insertions(+), 48 deletions(-) create mode 100644 .github/scripts/run_macrobenchmarks.sh diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh new file mode 100644 index 000000000..abb929aab --- /dev/null +++ b/.github/scripts/run_macrobenchmarks.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env sh +set -e + +OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" +JSON_REPORTS_DIR="$OUTPUT_DIR/json_reports" + +run_benchmark() { + VERSION_LABEL="$1" # v1 or v2 + RUN_NUMBER="$2" # 1..5 + + echo "==============================" + echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" + echo "==============================" + + # Clear app data to keep runs consistent + adb shell pm clear com.google.samples.apps.nowinandroid || true + + # Run only the Startup benchmark + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + + mkdir -p "$JSON_REPORTS_DIR" + + # Find newest JSON result file + LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f | xargs ls -t | head -n 1) + + if [ -z "$LATEST_JSON" ]; then + echo "Error: No benchmark JSON file found" + exit 1 + fi + + NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" + cp "$LATEST_JSON" "$NEW_JSON_NAME" + + echo "Saved result to $NEW_JSON_NAME" +} + +# Alternate runs: v1, v2, v1, v2 ... +for i in 1 2 3 4 5; do + run_benchmark "v1" "$i" + run_benchmark "v2" "$i" +done \ No newline at end of file diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 48f062df0..28ac5f948 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -4,7 +4,7 @@ import glob import os # ----------- CONFIG ----------- -RESULTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/collected" +JSON_REPORTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ @@ -34,7 +34,7 @@ def main(): before = [] after = [] - json_files = sorted(glob.glob(os.path.join(RESULTS_DIR, "*.json"))) + json_files = sorted(glob.glob(os.path.join(JSON_REPORTS_DIR, "*.json"))) if len(json_files) == 0: raise RuntimeError("No JSON files found.") diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index cc3ef71fd..1d856064f 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -45,54 +45,12 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 8192 disable-animations: true script: | - set -e + chmod +x .github/scripts/run_macrobenchmarks.sh + .github/scripts/run_macrobenchmarks.sh - OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" - COLLECTED_DIR="$OUTPUT_DIR/collected" - - run_benchmark() ( - VERSION_LABEL=$1 # v1 or v2 - RUN_NUMBER=$2 # 1..5 - - echo "==============================" - echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" - echo "==============================" - - # Clear app data to keep runs consistent - adb shell pm clear com.google.samples.apps.nowinandroid || true - - # Run only the Startup benchmark - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - - mkdir -p "$COLLECTED_DIR" - - # Find the newest JSON result file - LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f -printf "%T@ %p\n" | sort -nr | head -n1 | cut -d' ' -f2-) - - if [ -z "$LATEST_JSON" ]; then - echo "Error: No benchmark JSON file found" - exit 1 - fi - - NEW_NAME="$COLLECTED_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - cp "$LATEST_JSON" "$NEW_NAME" - - echo "Saved result to $NEW_NAME" - ) - - # Alternate runs: v1, v2, v1, v2 ... - i=1 - while [ $i -le 5 ]; do - run_benchmark "v1" "$i" - run_benchmark "v2" "$i" - i=$((i + 1)) - done - - name: Run step fit run: python3 .github/scripts/step_fit.py @@ -101,4 +59,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/build/outputs/connected_android_test_additional_output/collected/ \ No newline at end of file + path: benchmarks/build/outputs/connected_android_test_additional_output/json_reports/ \ No newline at end of file From 0d79ede1c270110e68ebc567ba00a4510c74aeb7 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Tue, 3 Feb 2026 16:40:09 +0200 Subject: [PATCH 27/39] ci: edit benchmark workflow to be run manually --- .github/workflows/Build.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 1d856064f..4e26ac85b 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,8 +1,7 @@ name: Benchmark on Emulator on: - push: - branches: [testNIA] + workflow_dispatch: jobs: benchmark-android: From ed7ece40c9a8bdee31fb6b7b2c1db1f3c2642787 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Tue, 3 Feb 2026 17:38:24 +0200 Subject: [PATCH 28/39] fix: edit how metrics json file is located also made an attempt to make the gradle benchmark run faster --- .github/scripts/run_macrobenchmarks.sh | 37 +++++++++++++++++--------- .github/scripts/step_fit.py | 2 +- .github/workflows/Build.yaml | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index abb929aab..339068d25 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -2,7 +2,7 @@ set -e OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" -JSON_REPORTS_DIR="$OUTPUT_DIR/json_reports" +JSON_REPORTS_DIR="benchmarks/build/json_reports" run_benchmark() { VERSION_LABEL="$1" # v1 or v2 @@ -15,23 +15,36 @@ run_benchmark() { # Clear app data to keep runs consistent adb shell pm clear com.google.samples.apps.nowinandroid || true + # Ensure clean slate so only one JSON exists after run + rm -rf "$OUTPUT_DIR" + mkdir -p "$OUTPUT_DIR" + # Run only the Startup benchmark + # We might need to replace gradle with adb later to run the benchmark faster + # but we will need to deal with making sure things are running correctly + # and locating the output JSON files. ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - - mkdir -p "$JSON_REPORTS_DIR" - - # Find newest JSON result file - LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f | xargs ls -t | head -n 1) - - if [ -z "$LATEST_JSON" ]; then - echo "Error: No benchmark JSON file found" + --no-daemon \ + --no-build-cache \ + --rerun-tasks \ + -x assemble \ + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + + JSON_REPORT=$(find "$OUTPUT_DIR" -type f -name "*.json") + COUNT=$(echo "$JSON_REPORT" | wc -l | tr -d ' ') + + if [ "$COUNT" -ne 1 ]; then + echo "Error: Expected exactly 1 JSON file, found $COUNT" + find "$OUTPUT_DIR" -type f -name "*.json" exit 1 fi + # Create JSON reports directory if it doesn't exist + mkdir -p "$JSON_REPORTS_DIR" + NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - cp "$LATEST_JSON" "$NEW_JSON_NAME" + cp "$JSON_REPORT" "$NEW_JSON_NAME" echo "Saved result to $NEW_JSON_NAME" } diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 28ac5f948..9565d9362 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -4,7 +4,7 @@ import glob import os # ----------- CONFIG ----------- -JSON_REPORTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/json_reports" +JSON_REPORTS_DIR = "benchmarks/build/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 4e26ac85b..c0ed9ba4c 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -58,4 +58,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/build/outputs/connected_android_test_additional_output/json_reports/ \ No newline at end of file + path: benchmarks/build/json_reports/ \ No newline at end of file From 0e86bff302b592d3e088a04492942a45eb00091f Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Tue, 3 Feb 2026 18:21:15 +0200 Subject: [PATCH 29/39] fix: remove attempt to speed up gradle --- .github/scripts/run_macrobenchmarks.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 339068d25..b18622081 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -24,12 +24,8 @@ run_benchmark() { # but we will need to deal with making sure things are running correctly # and locating the output JSON files. ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - --no-daemon \ - --no-build-cache \ - --rerun-tasks \ - -x assemble \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR JSON_REPORT=$(find "$OUTPUT_DIR" -type f -name "*.json") COUNT=$(echo "$JSON_REPORT" | wc -l | tr -d ' ') From 89e87e85e318d448cf18c6f2d27f5a5069db3401 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Tue, 3 Feb 2026 19:17:51 +0200 Subject: [PATCH 30/39] fix: use adb to run benchmarks instead of gradle gradle isn't designed to be rerun inside a loop, when I tried that it froze on the second test run, as if it got into a deadlock, and I knew from the start that the way that made more sense was to use adb, it will be a lot faster and hopefully work with our current setup --- .github/scripts/run_macrobenchmarks.sh | 49 +++++++++++++------------- .github/scripts/step_fit.py | 2 +- .github/workflows/Build.yaml | 13 ++++--- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index b18622081..b438a4b8a 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -1,8 +1,10 @@ #!/usr/bin/env sh set -e -OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" -JSON_REPORTS_DIR="benchmarks/build/json_reports" +APP_PKG="com.google.samples.apps.nowinandroid" +BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" +TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" +JSON_REPORTS_DIR="benchmarks/json_reports" run_benchmark() { VERSION_LABEL="$1" # v1 or v2 @@ -12,35 +14,32 @@ run_benchmark() { echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" echo "==============================" - # Clear app data to keep runs consistent - adb shell pm clear com.google.samples.apps.nowinandroid || true - - # Ensure clean slate so only one JSON exists after run - rm -rf "$OUTPUT_DIR" - mkdir -p "$OUTPUT_DIR" + # Clear app data and prepare storage + adb shell pm clear "$APP_PKG" || true + adb shell mkdir -p /sdcard/Download + adb shell rm /sdcard/Download/*.json || true + adb shell rm /sdcard/Download/*.perfetto-trace || true + adb shell rm /sdcard/Download/*.txt || true # Run only the Startup benchmark - # We might need to replace gradle with adb later to run the benchmark faster - # but we will need to deal with making sure things are running correctly - # and locating the output JSON files. - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - - JSON_REPORT=$(find "$OUTPUT_DIR" -type f -name "*.json") - COUNT=$(echo "$JSON_REPORT" | wc -l | tr -d ' ') + adb shell am instrument -w \ + -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -e androidx.benchmark.suppressErrors EMULATOR \ + -e no-isolated-storage true \ + -e additionalTestOutputDir /sdcard/Download \ + $BENCHMARK_PKG/$TEST_RUNNER - if [ "$COUNT" -ne 1 ]; then - echo "Error: Expected exactly 1 JSON file, found $COUNT" - find "$OUTPUT_DIR" -type f -name "*.json" - exit 1 - fi + # Ensure the local directory exists for the pull + mkdir -p "$JSON_REPORTS_DIR/tmp_results" - # Create JSON reports directory if it doesn't exist - mkdir -p "$JSON_REPORTS_DIR" + # Pull the benchmarks output produced on the device + adb pull /sdcard/Download/. "$JSON_REPORTS_DIR/tmp_results" + # Take only the generated JSON file (ignore perfetto traces and text files) + # There should only be one JSON file because of the rm at the start NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - cp "$JSON_REPORT" "$NEW_JSON_NAME" + mv "$JSON_REPORTS_DIR/tmp_results/"*.json "$NEW_JSON_NAME" + rm -rf "$JSON_REPORTS_DIR/tmp_results" echo "Saved result to $NEW_JSON_NAME" } diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 9565d9362..748a14296 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -4,7 +4,7 @@ import glob import os # ----------- CONFIG ----------- -JSON_REPORTS_DIR = "benchmarks/build/json_reports" +JSON_REPORTS_DIR = "benchmarks/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index c0ed9ba4c..a63de6b5d 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -34,8 +34,9 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Build Benchmark APKs - run: ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark + - name: Build APKs + run: | + ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - name: Run Benchmarks uses: reactivecircus/android-emulator-runner@v2 @@ -44,9 +45,13 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 8192 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | + # Install/Copy the built APKs to the emulator + adb install app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk + adb install benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk + chmod +x .github/scripts/run_macrobenchmarks.sh .github/scripts/run_macrobenchmarks.sh @@ -58,4 +63,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/build/json_reports/ \ No newline at end of file + path: benchmarks/json_reports/ \ No newline at end of file From 10c8917ca69f69fe6fd3009b6185760d8ef97360 Mon Sep 17 00:00:00 2001 From: Ahmed Khaled Date: Thu, 5 Feb 2026 01:50:22 +0200 Subject: [PATCH 31/39] perf(ci): speed up running the startup benchmark --- .github/scripts/run_macrobenchmarks.sh | 5 +++-- .../samples/apps/nowinandroid/startup/StartupBenchmark.kt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index b438a4b8a..13247a36d 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -21,10 +21,11 @@ run_benchmark() { adb shell rm /sdcard/Download/*.perfetto-trace || true adb shell rm /sdcard/Download/*.txt || true - # Run only the Startup benchmark + # Run only the Startup Baseline Profile benchmark adb shell am instrument -w \ - -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \ -e androidx.benchmark.suppressErrors EMULATOR \ + -e androidx.benchmark.profiling.mode none \ -e no-isolated-storage true \ -e additionalTestOutputDir /sdcard/Download \ $BENCHMARK_PKG/$TEST_RUNNER diff --git a/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt b/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt index 4aab929b1..c979beeba 100644 --- a/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt +++ b/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt @@ -61,7 +61,7 @@ class StartupBenchmark { metrics = BaselineProfileMetrics.allMetrics, compilationMode = compilationMode, // More iterations result in higher statistical significance. - iterations = 20, + iterations = 5, startupMode = COLD, setupBlock = { pressHome() From 79790671f4b404dfdabb84e21e044b9a7a8df294 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Thu, 5 Feb 2026 20:46:33 +0200 Subject: [PATCH 32/39] feat: support comparing between branches --- .github/scripts/run_macrobenchmarks.sh | 35 ++++++++----- .github/workflows/Build.yaml | 69 +++++++++++++++++--------- 2 files changed, 68 insertions(+), 36 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 13247a36d..81b0f9796 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -1,18 +1,17 @@ -#!/usr/bin/env sh -set -e +#!/usr/bin/env bash + +set -euo pipefail APP_PKG="com.google.samples.apps.nowinandroid" BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" JSON_REPORTS_DIR="benchmarks/json_reports" -run_benchmark() { - VERSION_LABEL="$1" # v1 or v2 - RUN_NUMBER="$2" # 1..5 +PATH_APK_VERSION_1="$1" +PATH_APK_VERSION_2="$2" - echo "==============================" - echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" - echo "==============================" +install_apk() { + adb install -r "$1" # Clear app data and prepare storage adb shell pm clear "$APP_PKG" || true @@ -20,7 +19,16 @@ run_benchmark() { adb shell rm /sdcard/Download/*.json || true adb shell rm /sdcard/Download/*.perfetto-trace || true adb shell rm /sdcard/Download/*.txt || true - +} + +run_benchmark() { + VERSION_LABEL="$1" # v1 or v2 + RUN_NUMBER="$2" # 1..5 + + echo "==============================" + echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" + echo "==============================" + # Run only the Startup Baseline Profile benchmark adb shell am instrument -w \ -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \ @@ -28,7 +36,7 @@ run_benchmark() { -e androidx.benchmark.profiling.mode none \ -e no-isolated-storage true \ -e additionalTestOutputDir /sdcard/Download \ - $BENCHMARK_PKG/$TEST_RUNNER + "$BENCHMARK_PKG/$TEST_RUNNER" # Ensure the local directory exists for the pull mkdir -p "$JSON_REPORTS_DIR/tmp_results" @@ -46,7 +54,10 @@ run_benchmark() { } # Alternate runs: v1, v2, v1, v2 ... -for i in 1 2 3 4 5; do +for i in {1..5}; do + install_apk "$PATH_APK_VERSION_1" run_benchmark "v1" "$i" + + install_apk "$PATH_APK_VERSION_2" run_benchmark "v2" "$i" -done \ No newline at end of file +done diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index a63de6b5d..413e6dac1 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -3,39 +3,62 @@ name: Benchmark on Emulator on: workflow_dispatch: +env: + BASELINE_BRANCH: main + GRADLE_BUILD_CACHE_DIR: ${{ github.workspace }}/.common-gradle-cache + jobs: benchmark-android: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 45 steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Free Disk Space run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: '17' - cache: 'gradle' - - - name: Grant execute permission - run: chmod +x gradlew - - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Build APKs + - name: Checkout branch (Baseline) + uses: actions/checkout@v4 + with: + ref: ${{ env.BASELINE_BRANCH }} + path: baseline + + - name: Checkout branch (Candidate) + uses: actions/checkout@v4 + with: + path: candidate + + - name: Setup JDK 17 + uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v5 + with: + # By default, The setup-gradle action will only write to the cache from Jobs on the default (main/master) branch. + # Jobs on other branches will read entries from the cache but will not write updated entries. + cache-read-only: false + + - name: Build APKs (Baseline) + working-directory: ./baseline run: | + chmod +x ./gradlew + ./gradlew :app:assembleDemoBenchmark + + - name: Build APKs (Candidate) + working-directory: ./candidate + run: | + chmod +x ./gradlew ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - name: Run Benchmarks @@ -45,22 +68,20 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 - disable-animations: true + emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 + disable-animations: true script: | - # Install/Copy the built APKs to the emulator - adb install app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk - adb install benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk - - chmod +x .github/scripts/run_macrobenchmarks.sh - .github/scripts/run_macrobenchmarks.sh + adb install ./candidate/benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk + chmod +x ./candidate/.github/scripts/run_macrobenchmarks.sh + ./candidate/.github/scripts/run_macrobenchmarks.sh "./baseline/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" "./candidate/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" - name: Run step fit run: python3 .github/scripts/step_fit.py + working-directory: ./candidate - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/json_reports/ \ No newline at end of file + path: ./candidate/benchmarks/json_reports/ From 6a84bdd9e9ce1089f64b8c17da578ee2800384c3 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Fri, 6 Feb 2026 14:55:35 +0200 Subject: [PATCH 33/39] rewrite(run_macrobenchmark): remove hardcoded constraints --- .github/scripts/run_macrobenchmarks.sh | 99 +++++++++++++++++--------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 81b0f9796..5cf036716 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -2,62 +2,93 @@ set -euo pipefail +NUMBER_OF_RUNS=5 + APP_PKG="com.google.samples.apps.nowinandroid" BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" -JSON_REPORTS_DIR="benchmarks/json_reports" +EMULATOR_BENCHMARK_RESULT_DIR="/sdcard/Download" + +PATH_APK_BASELINE="${1:-}" +PATH_APK_CANDIDATE="${2:-}" +OUTPUT_DIR="${3:-./macrobenchmark_results}" -PATH_APK_VERSION_1="$1" -PATH_APK_VERSION_2="$2" +TEMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TEMP_DIR}"' EXIT install_apk() { - adb install -r "$1" + local apk_path="${1}" + + adb install -r "${apk_path}" - # Clear app data and prepare storage adb shell pm clear "$APP_PKG" || true - adb shell mkdir -p /sdcard/Download - adb shell rm /sdcard/Download/*.json || true - adb shell rm /sdcard/Download/*.perfetto-trace || true - adb shell rm /sdcard/Download/*.txt || true + adb shell pm clear "${BENCHMARK_PKG}" || true + adb shell mkdir -p "${EMULATOR_BENCHMARK_RESULT_DIR}" + + adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.json || true + adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.perfetto-trace || true + adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.txt || true } run_benchmark() { - VERSION_LABEL="$1" # v1 or v2 - RUN_NUMBER="$2" # 1..5 - - echo "==============================" - echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" - echo "==============================" - - # Run only the Startup Baseline Profile benchmark adb shell am instrument -w \ -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \ -e androidx.benchmark.suppressErrors EMULATOR \ -e androidx.benchmark.profiling.mode none \ -e no-isolated-storage true \ - -e additionalTestOutputDir /sdcard/Download \ + -e additionalTestOutputDir "${EMULATOR_BENCHMARK_RESULT_DIR}" \ "$BENCHMARK_PKG/$TEST_RUNNER" +} - # Ensure the local directory exists for the pull - mkdir -p "$JSON_REPORTS_DIR/tmp_results" - - # Pull the benchmarks output produced on the device - adb pull /sdcard/Download/. "$JSON_REPORTS_DIR/tmp_results" +write_benchmark_result() { + local output_path="${1}" - # Take only the generated JSON file (ignore perfetto traces and text files) - # There should only be one JSON file because of the rm at the start - NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - mv "$JSON_REPORTS_DIR/tmp_results/"*.json "$NEW_JSON_NAME" - rm -rf "$JSON_REPORTS_DIR/tmp_results" + adb pull "${EMULATOR_BENCHMARK_RESULT_DIR}/." "${TEMP_DIR}/pull_out/" - echo "Saved result to $NEW_JSON_NAME" + mv "${TEMP_DIR}/pull_out/"*.json "${output_path}" + rm -rf "${TEMP_DIR}/pull_out/" } +if [[ -z "${PATH_APK_BASELINE}" || -z "${PATH_APK_CANDIDATE}" ]]; then + echo "Usage: $0 [output_dir]" + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}/baseline" "${OUTPUT_DIR}/candidate" + # Alternate runs: v1, v2, v1, v2 ... -for i in {1..5}; do - install_apk "$PATH_APK_VERSION_1" - run_benchmark "v1" "$i" +for ((i=1; i<=${NUMBER_OF_RUNS}; i++)); do + start_time=$(date +%s) - install_apk "$PATH_APK_VERSION_2" - run_benchmark "v2" "$i" + timestamp=$(date +"%Y-%m-%dT%H-%M-%S") + output_filename="${BENCHMARK_PKG}_${timestamp}" + baseline_output_path="${OUTPUT_DIR}/baseline/${output_filename}" + candidate_output_path="${OUTPUT_DIR}/candidate/${output_filename}" + + echo "==============================" + echo "Start iteration (${i} / ${NUMBER_OF_RUNS})" + echo "==============================" + + echo "Starting Baseline Benchmark:" + echo " >> APK file : ${PATH_APK_BASELINE}" + echo " >> Output file path: ${baseline_output_path}" + + install_apk "${PATH_APK_BASELINE}" + run_benchmark + write_benchmark_result "${baseline_output_path}" + + echo "Starting Candidate Benchmark:" + echo " >> APK file : ${PATH_APK_CANDIDATE}" + echo " >> Output file path: ${candidate_output_path}" + + install_apk "${PATH_APK_CANDIDATE}" + run_benchmark + write_benchmark_result "${candidate_output_path}" + + end_time=$(date +%s) + duration=$((end_time - start_time)) + + echo "==============================" + echo "End iteration (${i} / ${NUMBER_OF_RUNS}) took ${duration}s" + echo "==============================" done From c6cafffd373f4ebd891b90953143d149953819a7 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Sat, 7 Feb 2026 17:49:04 +0200 Subject: [PATCH 34/39] refactor(step_fit.py): remove hardcoded paths, and general code cleanup --- .github/scripts/step_fit.py | 106 ++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 748a14296..1c1356947 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -1,63 +1,89 @@ +import argparse import json import math -import glob -import os +import sys +from pathlib import Path # ----------- CONFIG ----------- -JSON_REPORTS_DIR = "benchmarks/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ -def sum_squared_error(values): - avg = sum(values) / len(values) - return sum((v - avg) ** 2 for v in values) +def step_fit(a, b): + def sum_squared_error(values): + avg = sum(values) / len(values) + return sum((v - avg) ** 2 for v in values) -def step_fit(before, after): - total_squared_error = sum_squared_error(before) + sum_squared_error(after) - step_error = math.sqrt(total_squared_error) / (len(before) + len(after)) + if not a or not b: + return 0.0 + + total_squared_error = sum_squared_error(a) + sum_squared_error(b) + step_error = math.sqrt(total_squared_error) / (len(a) + len(b)) if step_error == 0.0: return 0.0 - return (sum(before) / len(before) - sum(after) / len(after)) / step_error - -def extract_median_from_file(path): - with open(path, "r") as f: - data = json.load(f) - for bench in data.get("benchmarks", []): - if bench.get("name") == BENCHMARK_NAME: - metrics = bench.get("metrics", {}) - metric = metrics.get(METRIC_KEY, {}) - return metric.get("median") - raise ValueError(f"Metric not found in {path}") + + return (sum(a) / len(a) - sum(b) / len(b)) / step_error + +def extract_median_from_files(paths): + medians = [] + + for path in paths: + with open(path, "r") as f: + data = json.load(f) + + found = False + for bench in data.get("benchmarks", []): + if bench.get("name") == BENCHMARK_NAME: + metrics = bench.get("metrics", {}) + metric = metrics.get(METRIC_KEY, {}) + medians.append(metric.get("median")) + found = True + + if not found: + raise ValueError(f"Metric not found in {path}") + + return medians def main(): - before = [] - after = [] + parser = argparse.ArgumentParser(prog='Comperator', description='Compare between multiple macrobenchmark test results') + parser.add_argument('baseline_dir', help='Baseline macrobenchmark reports directory') + parser.add_argument('candidate_dir', help='Candidate macrobenchmark reports directory') + args = parser.parse_args() + + baseline_dir = Path(args.baseline_dir) + candidate_dir = Path(args.candidate_dir) + + # Using glob on Path objects + baseline_files = sorted([str(p) for p in baseline_dir.glob("*.json")]) + candidate_files = sorted([str(p) for p in candidate_dir.glob("*.json")]) + + if len(baseline_files) <= 0: + print('ERR: baseline has no macrobenchmark results', file=sys.stderr) + exit(1) - json_files = sorted(glob.glob(os.path.join(JSON_REPORTS_DIR, "*.json"))) + if len(candidate_files) <= 0: + print('ERR: candidate has no macrobenchmark results', file=sys.stderr) + exit(1) - if len(json_files) == 0: - raise RuntimeError("No JSON files found.") + min_len = min(len(baseline_files), len(candidate_files)) + if len(baseline_files) != len(candidate_files): + print(f"WARN: Length mismatch, using first {min_len} samples. baseline: {len(baseline_files)}, candidate: {len(candidate_files)}") - for path in json_files: - median = extract_median_from_file(path) - filename = os.path.basename(path).lower() - if "v1" in filename: - before.append(median) - elif "v2" in filename: - after.append(median) - else: - print(f"Skipping file with unknown label: {filename}") - print(f"{filename}: median={median:.3f} ms") + print('Macrobenchmark Result Mapping:') + print('| Index | Baseline | Candidate |') + print('--------------------------------') + for i in range(min_len): + print(f'{i + 1} {baseline_files[i]} <-> {candidate_files[i]}') - if len(before) != 5 or len(after) != 5: - raise RuntimeError(f"Expected 5 runs each, got v1={len(before)}, v2={len(after)}") + baseline_medians = extract_median_from_files(baseline_files[:min_len]) + candidate_medians = extract_median_from_files(candidate_files[:min_len]) + assert (len(baseline_medians) == len(candidate_medians)) - result = step_fit(before, after) + result = step_fit(baseline_medians, candidate_medians) print("\n-----------------------------") - print(f"v1 medians: {before}") - print(f"v2 medians: {after}") + print(f"Baseline medians : {baseline_medians}") + print(f"Candidate medians: {candidate_medians}") print(f"Step Fit Result: {result:.4f}") print("-----------------------------") From 606a28caf8dd6f87790e325bb5ac3546150e5b02 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Sat, 7 Feb 2026 17:52:42 +0200 Subject: [PATCH 35/39] fix: adapt build workflow to work with new step_fit script --- .github/workflows/Build.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 413e6dac1..edd2c70cf 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -75,13 +75,15 @@ jobs: chmod +x ./candidate/.github/scripts/run_macrobenchmarks.sh ./candidate/.github/scripts/run_macrobenchmarks.sh "./baseline/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" "./candidate/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" - - name: Run step fit - run: python3 .github/scripts/step_fit.py - working-directory: ./candidate + - name: Compare macrobenchmark results + run: | + python3 ./candidate/.github/scripts/step_fit.py \ + "./macrobenchmark_results/baseline" \ + "./macrobenchmark_results/candidate" - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: name: benchmark-results - path: ./candidate/benchmarks/json_reports/ + path: ./macrobenchmark_results/ From 59936a2765d7046810c1eedcddf3611bcfe05c84 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Sat, 7 Feb 2026 17:55:30 +0200 Subject: [PATCH 36/39] chore: Set number of runs to 2 runs only to speed up testing --- .github/scripts/run_macrobenchmarks.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 5cf036716..2fe435b5b 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -2,7 +2,8 @@ set -euo pipefail -NUMBER_OF_RUNS=5 +# TODO: pass number of runs as a commandline argument +NUMBER_OF_RUNS=2 APP_PKG="com.google.samples.apps.nowinandroid" BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" From 929b4f1d32d488cd6c8c1e5baae75b3695efc221 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Sat, 7 Feb 2026 18:09:03 +0200 Subject: [PATCH 37/39] fix(run_macrobenchmark): delete emulator benchmark results directory on each apk install --- .github/scripts/run_macrobenchmarks.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index 2fe435b5b..e15984a67 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -24,11 +24,8 @@ install_apk() { adb shell pm clear "$APP_PKG" || true adb shell pm clear "${BENCHMARK_PKG}" || true + adb shell rm -rf "${EMULATOR_BENCHMARK_RESULT_DIR}" || true adb shell mkdir -p "${EMULATOR_BENCHMARK_RESULT_DIR}" - - adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.json || true - adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.perfetto-trace || true - adb shell rm "${EMULATOR_BENCHMARK_RESULT_DIR}"*.txt || true } run_benchmark() { @@ -62,7 +59,7 @@ for ((i=1; i<=${NUMBER_OF_RUNS}; i++)); do start_time=$(date +%s) timestamp=$(date +"%Y-%m-%dT%H-%M-%S") - output_filename="${BENCHMARK_PKG}_${timestamp}" + output_filename="${BENCHMARK_PKG}_${timestamp}.json" baseline_output_path="${OUTPUT_DIR}/baseline/${output_filename}" candidate_output_path="${OUTPUT_DIR}/candidate/${output_filename}" From 0ef53c77d97aa6480d2e9e7379dfa1cbffff9ccc Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Sun, 8 Feb 2026 14:08:07 +0200 Subject: [PATCH 38/39] feat(step_fit): add filename mapping mismatch detection --- .github/scripts/step_fit.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 1c1356947..96383c50b 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -52,10 +52,8 @@ def main(): baseline_dir = Path(args.baseline_dir) candidate_dir = Path(args.candidate_dir) - - # Using glob on Path objects - baseline_files = sorted([str(p) for p in baseline_dir.glob("*.json")]) - candidate_files = sorted([str(p) for p in candidate_dir.glob("*.json")]) + baseline_files = sorted(baseline_dir.glob("*.json")) + candidate_files = sorted(candidate_dir.glob("*.json")) if len(baseline_files) <= 0: print('ERR: baseline has no macrobenchmark results', file=sys.stderr) @@ -72,9 +70,22 @@ def main(): print('Macrobenchmark Result Mapping:') print('| Index | Baseline | Candidate |') print('--------------------------------') + + mismatch_count = 0 for i in range(min_len): + baseline_filename = baseline_files[i].name.upper() + candidate_filename = candidate_files[i].name.upper() + if baseline_filename != candidate_filename: + mismatch_count += 1 + print('* ', end='') print(f'{i + 1} {baseline_files[i]} <-> {candidate_files[i]}') + print('--------------------------------') + print(f'# Match : {min_len - mismatch_count}') + print(f'# Mismatch: {mismatch_count}') + if mismatch_count > 0: + print("WARN: filename mapping mismatch detected. Output prediction may be incorrect") + baseline_medians = extract_median_from_files(baseline_files[:min_len]) candidate_medians = extract_median_from_files(candidate_files[:min_len]) assert (len(baseline_medians) == len(candidate_medians)) From 67073df4e05345e87fe61e07b3d5921ba2e5f742 Mon Sep 17 00:00:00 2001 From: Tony Medhat Date: Mon, 9 Feb 2026 15:06:54 +0200 Subject: [PATCH 39/39] feat(step_fit): cleanup result logging --- .github/scripts/step_fit.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 96383c50b..ef1691ece 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -85,25 +85,27 @@ def main(): print(f'# Mismatch: {mismatch_count}') if mismatch_count > 0: print("WARN: filename mapping mismatch detected. Output prediction may be incorrect") + print() baseline_medians = extract_median_from_files(baseline_files[:min_len]) candidate_medians = extract_median_from_files(candidate_files[:min_len]) assert (len(baseline_medians) == len(candidate_medians)) - result = step_fit(baseline_medians, candidate_medians) - - print("\n-----------------------------") + print(f"Benchmark : {BENCHMARK_NAME}") + print(f"Metric : {METRIC_KEY}") print(f"Baseline medians : {baseline_medians}") print(f"Candidate medians: {candidate_medians}") - print(f"Step Fit Result: {result:.4f}") print("-----------------------------") + print("Result: ", end="") + result = step_fit(baseline_medians, candidate_medians) if abs(result) <= 25: - print("➡️ Difference is within noise range (low confidence of real regression)") - elif result > 0: - print("⚠️ v2 is slower than v1 (possible regression)") + print("Within noise range", end="") + elif result < 0: + print("POSSIBLE REGRESSION", end="") else: - print("🚀 v2 is faster than v1 (possible improvement)") + print("POSSIBLE IMPROVEMENT", end="") + print(f" (Step fit: {result:.4})") if __name__ == "__main__": main()