diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh new file mode 100644 index 000000000..e15984a67 --- /dev/null +++ b/.github/scripts/run_macrobenchmarks.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# TODO: pass number of runs as a commandline argument +NUMBER_OF_RUNS=2 + +APP_PKG="com.google.samples.apps.nowinandroid" +BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" +TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" +EMULATOR_BENCHMARK_RESULT_DIR="/sdcard/Download" + +PATH_APK_BASELINE="${1:-}" +PATH_APK_CANDIDATE="${2:-}" +OUTPUT_DIR="${3:-./macrobenchmark_results}" + +TEMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TEMP_DIR}"' EXIT + +install_apk() { + local apk_path="${1}" + + adb install -r "${apk_path}" + + adb shell pm clear "$APP_PKG" || true + adb shell pm clear "${BENCHMARK_PKG}" || true + adb shell rm -rf "${EMULATOR_BENCHMARK_RESULT_DIR}" || true + adb shell mkdir -p "${EMULATOR_BENCHMARK_RESULT_DIR}" +} + +run_benchmark() { + adb shell am instrument -w \ + -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark#startupPrecompiledWithBaselineProfile \ + -e androidx.benchmark.suppressErrors EMULATOR \ + -e androidx.benchmark.profiling.mode none \ + -e no-isolated-storage true \ + -e additionalTestOutputDir "${EMULATOR_BENCHMARK_RESULT_DIR}" \ + "$BENCHMARK_PKG/$TEST_RUNNER" +} + +write_benchmark_result() { + local output_path="${1}" + + adb pull "${EMULATOR_BENCHMARK_RESULT_DIR}/." "${TEMP_DIR}/pull_out/" + + mv "${TEMP_DIR}/pull_out/"*.json "${output_path}" + rm -rf "${TEMP_DIR}/pull_out/" +} + +if [[ -z "${PATH_APK_BASELINE}" || -z "${PATH_APK_CANDIDATE}" ]]; then + echo "Usage: $0 [output_dir]" + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}/baseline" "${OUTPUT_DIR}/candidate" + +# Alternate runs: v1, v2, v1, v2 ... +for ((i=1; i<=${NUMBER_OF_RUNS}; i++)); do + start_time=$(date +%s) + + timestamp=$(date +"%Y-%m-%dT%H-%M-%S") + output_filename="${BENCHMARK_PKG}_${timestamp}.json" + baseline_output_path="${OUTPUT_DIR}/baseline/${output_filename}" + candidate_output_path="${OUTPUT_DIR}/candidate/${output_filename}" + + echo "==============================" + echo "Start iteration (${i} / ${NUMBER_OF_RUNS})" + echo "==============================" + + echo "Starting Baseline Benchmark:" + echo " >> APK file : ${PATH_APK_BASELINE}" + echo " >> Output file path: ${baseline_output_path}" + + install_apk "${PATH_APK_BASELINE}" + run_benchmark + write_benchmark_result "${baseline_output_path}" + + echo "Starting Candidate Benchmark:" + echo " >> APK file : ${PATH_APK_CANDIDATE}" + echo " >> Output file path: ${candidate_output_path}" + + install_apk "${PATH_APK_CANDIDATE}" + run_benchmark + write_benchmark_result "${candidate_output_path}" + + end_time=$(date +%s) + duration=$((end_time - start_time)) + + echo "==============================" + echo "End iteration (${i} / ${NUMBER_OF_RUNS}) took ${duration}s" + echo "==============================" +done diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py new file mode 100644 index 000000000..ef1691ece --- /dev/null +++ b/.github/scripts/step_fit.py @@ -0,0 +1,111 @@ +import argparse +import json +import math +import sys +from pathlib import Path + +# ----------- CONFIG ----------- +BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" +METRIC_KEY = "timeToInitialDisplayMs" +# ------------------------------ + +def step_fit(a, b): + def sum_squared_error(values): + avg = sum(values) / len(values) + return sum((v - avg) ** 2 for v in values) + + if not a or not b: + return 0.0 + + total_squared_error = sum_squared_error(a) + sum_squared_error(b) + step_error = math.sqrt(total_squared_error) / (len(a) + len(b)) + if step_error == 0.0: + return 0.0 + + return (sum(a) / len(a) - sum(b) / len(b)) / step_error + +def extract_median_from_files(paths): + medians = [] + + for path in paths: + with open(path, "r") as f: + data = json.load(f) + + found = False + for bench in data.get("benchmarks", []): + if bench.get("name") == BENCHMARK_NAME: + metrics = bench.get("metrics", {}) + metric = metrics.get(METRIC_KEY, {}) + medians.append(metric.get("median")) + found = True + + if not found: + raise ValueError(f"Metric not found in {path}") + + return medians + +def main(): + parser = argparse.ArgumentParser(prog='Comperator', description='Compare between multiple macrobenchmark test results') + parser.add_argument('baseline_dir', help='Baseline macrobenchmark reports directory') + parser.add_argument('candidate_dir', help='Candidate macrobenchmark reports directory') + args = parser.parse_args() + + baseline_dir = Path(args.baseline_dir) + candidate_dir = Path(args.candidate_dir) + baseline_files = sorted(baseline_dir.glob("*.json")) + candidate_files = sorted(candidate_dir.glob("*.json")) + + if len(baseline_files) <= 0: + print('ERR: baseline has no macrobenchmark results', file=sys.stderr) + exit(1) + + if len(candidate_files) <= 0: + print('ERR: candidate has no macrobenchmark results', file=sys.stderr) + exit(1) + + min_len = min(len(baseline_files), len(candidate_files)) + if len(baseline_files) != len(candidate_files): + print(f"WARN: Length mismatch, using first {min_len} samples. baseline: {len(baseline_files)}, candidate: {len(candidate_files)}") + + print('Macrobenchmark Result Mapping:') + print('| Index | Baseline | Candidate |') + print('--------------------------------') + + mismatch_count = 0 + for i in range(min_len): + baseline_filename = baseline_files[i].name.upper() + candidate_filename = candidate_files[i].name.upper() + if baseline_filename != candidate_filename: + mismatch_count += 1 + print('* ', end='') + print(f'{i + 1} {baseline_files[i]} <-> {candidate_files[i]}') + + print('--------------------------------') + print(f'# Match : {min_len - mismatch_count}') + print(f'# Mismatch: {mismatch_count}') + if mismatch_count > 0: + print("WARN: filename mapping mismatch detected. Output prediction may be incorrect") + print() + + baseline_medians = extract_median_from_files(baseline_files[:min_len]) + candidate_medians = extract_median_from_files(candidate_files[:min_len]) + assert (len(baseline_medians) == len(candidate_medians)) + + print(f"Benchmark : {BENCHMARK_NAME}") + print(f"Metric : {METRIC_KEY}") + print(f"Baseline medians : {baseline_medians}") + print(f"Candidate medians: {candidate_medians}") + print("-----------------------------") + print("Result: ", end="") + + result = step_fit(baseline_medians, candidate_medians) + if abs(result) <= 25: + print("Within noise range", end="") + elif result < 0: + print("POSSIBLE REGRESSION", end="") + else: + print("POSSIBLE IMPROVEMENT", end="") + print(f" (Step fit: {result:.4})") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index 7e5a80eea..edd2c70cf 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -1,286 +1,89 @@ -name: Build +name: Benchmark on Emulator on: workflow_dispatch: - push: - branches: - - main - pull_request: -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true +env: + BASELINE_BRANCH: main + GRADLE_BUILD_CACHE_DIR: ${{ github.workspace }}/.common-gradle-cache jobs: - test_and_apk: - name: "Local tests and APKs" + benchmark-android: runs-on: ubuntu-latest - - permissions: - contents: write - pull-requests: write - security-events: write - - timeout-minutes: 60 + timeout-minutes: 45 steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Copy CI gradle.properties - run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties - - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - distribution: 'zulu' - java-version: 21 - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - build-scan-publish: true - build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" - build-scan-terms-of-use-agree: "yes" - - - name: Check build-logic - run: ./gradlew :build-logic:convention:check - - - name: Check spotless - run: ./gradlew spotlessCheck --init-script gradle/init.gradle.kts --no-configuration-cache - - - name: Check Dependency Guard - id: dependencyguard_verify - continue-on-error: true - run: ./gradlew dependencyGuard - - - name: Prevent updating Dependency Guard baselines if this is a fork - id: checkfork_dependencyguard - continue-on-error: false - if: steps.dependencyguard_verify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository + - name: Free Disk Space run: | - echo "::error::Dependency Guard failed, please update baselines with: ./gradlew dependencyGuardBaseline" && exit 1 + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" - # Runs if previous job failed - - name: Generate new Dependency Guard baselines if verification failed and it's a PR - id: dependencyguard_baseline - if: steps.dependencyguard_verify.outcome == 'failure' && github.event_name == 'pull_request' - run: | - ./gradlew dependencyGuardBaseline - - - name: Push new Dependency Guard baselines if available - uses: stefanzweifel/git-auto-commit-action@v5 - if: steps.dependencyguard_baseline.outcome == 'success' - with: - file_pattern: '**/dependencies/*.txt' - disable_globbing: true - commit_message: "🤖 Updates baselines for Dependency Guard" - - - name: Update Graphs - run: ./gradlew graphUpdate - continue-on-error: true - - - name: Check Graphs - id: graphs_verify - run: git add -- "**/README.md" && git diff --cached --quiet --exit-code -- "**/README.md" - - - name: Prevent updating graphs if this is a fork - id: checkfork_graphs - continue-on-error: false - if: steps.graphs_verify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository - run: | - echo "::error::Check Graphs failed, please update graphs with: ./gradlew graphUpdate" && exit 1 - - - name: Push new graphs if available - if: steps.graphs_verify.outcome == 'failure' && github.event_name == 'pull_request' - uses: stefanzweifel/git-auto-commit-action@v5 - with: - file_pattern: '**/README.md' - disable_globbing: true - commit_message: "🤖 Updates graphs" - - - name: Run all local screenshot tests (Roborazzi) - id: screenshotsverify - continue-on-error: true - run: ./gradlew verifyRoborazziDemoDebug - - - name: Prevent pushing new screenshots if this is a fork - id: checkfork_screenshots - continue-on-error: false - if: steps.screenshotsverify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository - run: | - echo "::error::Screenshot tests failed, please create a PR in your fork first." - echo "Your fork's CI will take screenshots for your fork." - exit 1 - - # Runs if previous job failed - - name: Generate new screenshots if verification failed and it's a PR - id: screenshotsrecord - if: steps.screenshotsverify.outcome == 'failure' && github.event_name == 'pull_request' - run: | - ./gradlew recordRoborazziDemoDebug - - - name: Push new screenshots if available - uses: stefanzweifel/git-auto-commit-action@v5 - if: steps.screenshotsrecord.outcome == 'success' - with: - file_pattern: '*/*.png' - disable_globbing: true - commit_message: "🤖 Updates screenshots" - - # Run local tests after screenshot tests to avoid wrong UP-TO-DATE. TODO: Ignore screenshots. - - name: Run local tests - run: ./gradlew testDemoDebug :lint:test - - - name: Build all build type and flavor permutations - run: ./gradlew :app:assemble -PminifyWithR8=false - - - name: Upload build outputs (APKs) - uses: actions/upload-artifact@v4 - with: - name: APKs - path: '**/build/outputs/apk/**/*.apk' - - - name: Upload JVM local results (XML) - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: local-test-results - path: '**/build/test-results/test*UnitTest/**.xml' - - - name: Upload screenshot results (PNG) - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: screenshot-test-results - path: '**/build/outputs/roborazzi/*_compare.png' - - - name: Check lint - run: ./gradlew :app:lintProdRelease :app-nia-catalog:lintRelease :lint:lint - - - name: Upload lint reports (HTML) - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: lint-reports - path: '**/build/reports/lint-results-*.html' - - - name: Upload lint reports (SARIF) for app module - if: ${{ !cancelled() && hashFiles('app/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './app/' - category: app - - - name: Upload lint reports (SARIF) for app-nia-catalog module - if: ${{ !cancelled() && hashFiles('app-nia-catalog/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './app-nia-catalog/' - category: app-nia-catalog - - - name: Upload lint reports (SARIF) for lint module - if: ${{ !cancelled() && hashFiles('lint/**/*.sarif') != '' }} - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: './lint/' - category: lint - - - name: Check badging - run: ./gradlew :app:checkProdReleaseBadging - - androidTest: - runs-on: ubuntu-latest - timeout-minutes: 55 - strategy: - matrix: - api-level: [26, 34] - - steps: - - name: Delete unnecessary tools 🔧 - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: false # Don't remove Android tools - tool-cache: true # Remove image tool cache - rm -rf "$AGENT_TOOLSDIRECTORY" - dotnet: true # rm -rf /usr/share/dotnet - haskell: true # rm -rf /opt/ghc... - swap-storage: true # rm -f /mnt/swapfile (4GiB) - docker-images: false # Takes 16s, enable if needed in the future - large-packages: false # includes google-cloud-sdk and it's slow - - - name: Enable KVM group perms + - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - ls /dev/kvm - - name: Checkout + - name: Checkout branch (Baseline) uses: actions/checkout@v4 + with: + ref: ${{ env.BASELINE_BRANCH }} + path: baseline - - name: Copy CI gradle.properties - run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties + - name: Checkout branch (Candidate) + uses: actions/checkout@v4 + with: + path: candidate - - name: Set up JDK 21 - uses: actions/setup-java@v5 + - name: Setup JDK 17 + uses: actions/setup-java@v4 with: - distribution: 'zulu' - java-version: 21 + distribution: "zulu" + java-version: "17" - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + uses: gradle/actions/setup-gradle@v5 with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - build-scan-publish: true - build-scan-terms-of-use-url: "https://gradle.com/terms-of-service" - build-scan-terms-of-use-agree: "yes" + # By default, The setup-gradle action will only write to the cache from Jobs on the default (main/master) branch. + # Jobs on other branches will read entries from the cache but will not write updated entries. + cache-read-only: false - - name: Build projects and run instrumentation tests + - name: Build APKs (Baseline) + working-directory: ./baseline + run: | + chmod +x ./gradlew + ./gradlew :app:assembleDemoBenchmark + + - name: Build APKs (Candidate) + working-directory: ./candidate + run: | + chmod +x ./gradlew + ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark + + - name: Run Benchmarks uses: reactivecircus/android-emulator-runner@v2 with: - api-level: ${{ matrix.api-level }} + api-level: 30 + target: google_apis arch: x86_64 + force-avd-creation: true + emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true - disk-size: 6000M - heap-size: 600M - script: ./gradlew connectedDemoDebugAndroidTest --daemon - - - name: Run local tests (including Roborazzi) for the combined coverage report (only API 30) - if: matrix.api-level == 30 - # There is no need to verify Roborazzi tests to generate coverage. - run: ./gradlew testDemoDebugUnitTest -Proborazzi.test.verify=false # Add Prod if we ever add JVM tests for prod + script: | + adb install ./candidate/benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk + chmod +x ./candidate/.github/scripts/run_macrobenchmarks.sh + ./candidate/.github/scripts/run_macrobenchmarks.sh "./baseline/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" "./candidate/app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk" - # Add `createProdDebugUnitTestCoverageReport` if we ever add JVM tests for prod - - name: Generate coverage reports for Debug variants (only API 30) - if: matrix.api-level == 30 - run: ./gradlew createDemoDebugCombinedCoverageReport - - - name: Upload test reports - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: test-reports-${{ matrix.api-level }} - path: '**/build/reports/androidTests' + - name: Compare macrobenchmark results + run: | + python3 ./candidate/.github/scripts/step_fit.py \ + "./macrobenchmark_results/baseline" \ + "./macrobenchmark_results/candidate" - - name: Display local test coverage (only API 30) - if: matrix.api-level == 30 - id: jacoco - uses: madrapps/jacoco-report@v1.7.1 - with: - title: Combined test coverage report - min-coverage-overall: 40 - min-coverage-changed-files: 60 - paths: | - ${{ github.workspace }}/**/build/reports/jacoco/**/*Report.xml - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload local coverage reports (XML + HTML) (only API 30) - if: matrix.api-level == 30 + - name: Upload Artifacts + if: always() uses: actions/upload-artifact@v4 with: - name: coverage-reports - if-no-files-found: error - compression-level: 1 - overwrite: false - path: '**/build/reports/jacoco/' + name: benchmark-results + path: ./macrobenchmark_results/ diff --git a/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt b/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt index 4aab929b1..c979beeba 100644 --- a/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt +++ b/benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/startup/StartupBenchmark.kt @@ -61,7 +61,7 @@ class StartupBenchmark { metrics = BaselineProfileMetrics.allMetrics, compilationMode = compilationMode, // More iterations result in higher statistical significance. - iterations = 20, + iterations = 5, startupMode = COLD, setupBlock = { pressHome()