diff --git a/.github/scripts/run_macrobenchmarks.sh b/.github/scripts/run_macrobenchmarks.sh new file mode 100644 index 000000000..abb929aab --- /dev/null +++ b/.github/scripts/run_macrobenchmarks.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env sh +set -e + +OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" +JSON_REPORTS_DIR="$OUTPUT_DIR/json_reports" + +run_benchmark() { + VERSION_LABEL="$1" # v1 or v2 + RUN_NUMBER="$2" # 1..5 + + echo "==============================" + echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" + echo "==============================" + + # Clear app data to keep runs consistent + adb shell pm clear com.google.samples.apps.nowinandroid || true + + # Run only the Startup benchmark + ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ + -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR + + mkdir -p "$JSON_REPORTS_DIR" + + # Find newest JSON result file + LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f | xargs ls -t | head -n 1) + + if [ -z "$LATEST_JSON" ]; then + echo "Error: No benchmark JSON file found" + exit 1 + fi + + NEW_JSON_NAME="$JSON_REPORTS_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" + cp "$LATEST_JSON" "$NEW_JSON_NAME" + + echo "Saved result to $NEW_JSON_NAME" +} + +# Alternate runs: v1, v2, v1, v2 ... +for i in 1 2 3 4 5; do + run_benchmark "v1" "$i" + run_benchmark "v2" "$i" +done \ No newline at end of file diff --git a/.github/scripts/step_fit.py b/.github/scripts/step_fit.py index 48f062df0..28ac5f948 100644 --- a/.github/scripts/step_fit.py +++ b/.github/scripts/step_fit.py @@ -4,7 +4,7 @@ import glob import os # ----------- CONFIG ----------- -RESULTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/collected" +JSON_REPORTS_DIR = "benchmarks/build/outputs/connected_android_test_additional_output/json_reports" BENCHMARK_NAME = "startupPrecompiledWithBaselineProfile" METRIC_KEY = "timeToInitialDisplayMs" # ------------------------------ @@ -34,7 +34,7 @@ def main(): before = [] after = [] - json_files = sorted(glob.glob(os.path.join(RESULTS_DIR, "*.json"))) + json_files = sorted(glob.glob(os.path.join(JSON_REPORTS_DIR, "*.json"))) if len(json_files) == 0: raise RuntimeError("No JSON files found.") diff --git a/.github/workflows/Build.yaml b/.github/workflows/Build.yaml index cc3ef71fd..1d856064f 100644 --- a/.github/workflows/Build.yaml +++ b/.github/workflows/Build.yaml @@ -45,54 +45,12 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: true - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 4096 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -memory 8192 disable-animations: true script: | - set -e + chmod +x .github/scripts/run_macrobenchmarks.sh + .github/scripts/run_macrobenchmarks.sh - OUTPUT_DIR="benchmarks/build/outputs/connected_android_test_additional_output" - COLLECTED_DIR="$OUTPUT_DIR/collected" - - run_benchmark() ( - VERSION_LABEL=$1 # v1 or v2 - RUN_NUMBER=$2 # 1..5 - - echo "==============================" - echo "Running benchmark for $VERSION_LABEL run $RUN_NUMBER" - echo "==============================" - - # Clear app data to keep runs consistent - adb shell pm clear com.google.samples.apps.nowinandroid || true - - # Run only the Startup benchmark - ./gradlew :benchmarks:connectedDemoBenchmarkAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.class=com.google.samples.apps.nowinandroid.startup.StartupBenchmark \ - -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.suppressErrors=EMULATOR - - mkdir -p "$COLLECTED_DIR" - - # Find the newest JSON result file - LATEST_JSON=$(find "$OUTPUT_DIR" -name "*.json" -type f -printf "%T@ %p\n" | sort -nr | head -n1 | cut -d' ' -f2-) - - if [ -z "$LATEST_JSON" ]; then - echo "Error: No benchmark JSON file found" - exit 1 - fi - - NEW_NAME="$COLLECTED_DIR/benchmark_${VERSION_LABEL}_run${RUN_NUMBER}.json" - cp "$LATEST_JSON" "$NEW_NAME" - - echo "Saved result to $NEW_NAME" - ) - - # Alternate runs: v1, v2, v1, v2 ... - i=1 - while [ $i -le 5 ]; do - run_benchmark "v1" "$i" - run_benchmark "v2" "$i" - i=$((i + 1)) - done - - name: Run step fit run: python3 .github/scripts/step_fit.py @@ -101,4 +59,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchmarks/build/outputs/connected_android_test_additional_output/collected/ \ No newline at end of file + path: benchmarks/build/outputs/connected_android_test_additional_output/json_reports/ \ No newline at end of file