diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh index b18622081..b438a4b8a 100644 --- a/.github/scripts/run_macrobenchmarks.sh +++ b/.github/scripts/run_macrobenchmarks.sh @@ -1,8 +1,10 @@ #!/usr/bin/env sh set -e -OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" -JSON_REPORTS_DIR="benchmarks/build/json_reports" +APP_PKG="com.google.samples.apps.nowinandroid" +BENCHMARK_PKG="com.google.samples.apps.nowinandroid.benchmarks" +TEST_RUNNER="androidx.test.runner.AndroidJUnitRunner" +JSON_REPORTS_DIR="benchmarks/json_reports" run_benchmark() { VERSION_LABEL="$1" # v1 or v2 @@ -12,35 +14,32 @@ run_benchmark() { echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" echo "==============================" - # Clear app data to keep runs consistent - adb shell pm clear com.google.samples.apps.nowinandroid || true - - # Ensure clean slate so only one JSON exists after run - rm -rf "$OUTPUT_DIR" - mkdir -p "$OUTPUT_DIR" + # Clear app data and prepare storage + adb shell pm clear "$APP_PKG" || true + adb shell mkdir -p /sdcard/Download + adb shell rm /sdcard/Download/*.json || true + adb shell rm /sdcard/Download/*.perfetto-trace || true + adb shell rm /sdcard/Download/*.txt || true # Run only the Startup benchmark - # We might need to replace gradle with adb later to run the benchmark faster - # but we will need to deal with making sure things are running correctly - # and locating the output JSON files. - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - - JSON_REPORT=$(find "$OUTPUT_DIR" -type f -name "*.json") - COUNT=$(echo "$JSON_REPORT" | wc -l | tr -d ' ') + adb shell am instrument -w \ + -e class com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -e androidx.benchmark.suppressErrors EMULATOR \ + -e no-isolated-storage true \ + -e additionalTestOutputDir /sdcard/Download \ + $BENCHMARK_PKG/$TEST_RUNNER - if [ "$COUNT" -ne 1 ]; then - echo "Error: Expected exactly 1 JSON file, found $COUNT" - find "$OUTPUT_DIR" -type f -name "*.json" - exit 1 - fi + # Ensure the local directory exists for the pull + mkdir -p "$JSON_REPORTS_DIR/tmp_results" - # Create JSON reports directory if it doesn't exist - mkdir -p "$JSON_REPORTS_DIR" + # Pull the benchmarks output produced on the device + adb pull /sdcard/Download/. "$JSON_REPORTS_DIR/tmp_results" + # Take only the generated JSON file (ignore perfetto traces and text files) + # There should only be one JSON file because of the rm at the start NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - cp "$JSON_REPORT" "$NEW_JSON_NAME" + mv "$JSON_REPORTS_DIR/tmp_results/"*.json "$NEW_JSON_NAME" + rm -rf "$JSON_REPORTS_DIR/tmp_results" echo "Saved result to $NEW_JSON_NAME" } diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 9565d9362..748a14296 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -4,7 +4,7 @@ import glob import os # ----------- CONFIG ----------- -JSON_REPORTS_DIR = "benchmarks/build/json_reports" +JSON_REPORTS_DIR = "benchmarks/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index c0ed9ba4c..a63de6b5d 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -34,8 +34,9 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Build Benchmark APKs - run: ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark + - name: Build APKs + run: | + ./gradlew :app:assembleDemoBenchmark :benchmarks:assembleDemoBenchmark - name: Run Benchmarks uses: reactivecircus/android-emulator-runner@v2 @@ -44,9 +45,13 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 8192 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 disable-animations: true script: | + # Install/Copy the built APKs to the emulator + adb install app/build/outputs/apk/demo/benchmarkRelease/app-demo-benchmarkRelease.apk + adb install benchmarks/build/outputs/apk/demo/benchmarkRelease/benchmarks-demo-benchmarkRelease.apk + chmod +x .github/scripts/run_macrobenchmarks.sh .github/scripts/run_macrobenchmarks.sh @@ -58,4 +63,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/build/json_reports/ \ No newline at end of file + path: benchmarks/json_reports/ \ No newline at end of file