Add automated baseline profile generation (#880)

Baseline profile generation is disabled for the PR level Build task. Release tasks require a fresh baseline profile. A new profile is generated using the baseline profile Gradle plugin.

* Prepare for usage of dex layout optimizations which can be actively used once NiA switches to AGP 8.2+.
* Add GMD config to release build
* Switch to macos-latest
* Update names for StartupBenchmark tests to better reflect states
* Stable release and recent GMD device
* Reduce flakiness by adding wait to benchmark
* More convenient waiting for objects
* Rename junit dependency to androidx-junit
* Only run baseline profile benchmarks during GH workflow
* Enable automatic BP generation for only release builds
* Disable BP generation from Build workflow
* Specify modules and skip benchmarking Build workflow

Bug: b/299334172
pull/948/head
Ben Weiss 1 year ago committed by GitHub
parent d4ef172c51
commit aa8ce0e1f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -73,9 +73,19 @@ jobs:
- name: Run local tests
if: always()
run: ./gradlew testDemoDebug testProdDebug :lint:test
# Replace task exclusions with `-Pandroidx.baselineprofile.skipgeneration` when
# https://android-review.googlesource.com/c/platform/frameworks/support/+/2602790 landed in a
# release build
- name: Build all build type and flavor permutations
run: ./gradlew assemble
run: ./gradlew :app:assemble :benchmarks:assemble
-x pixel6Api33ProdNonMinifiedReleaseAndroidTest
-x pixel6Api33ProdNonMinifiedBenchmarkAndroidTest
-x pixel6Api33DemoNonMinifiedReleaseAndroidTest
-x pixel6Api33DemoNonMinifiedBenchmarkAndroidTest
-x collectDemoNonMinifiedReleaseBaselineProfile
-x collectDemoNonMinifiedBenchmarkBaselineProfile
-x collectProdNonMinifiedReleaseBaselineProfile
-x collectProdNonMinifiedBenchmarkBaselineProfile
- name: Upload build outputs (APKs)
uses: actions/upload-artifact@v3

@ -7,8 +7,8 @@ on:
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 45
runs-on: macos-latest
timeout-minutes: 120
steps:
- name: Checkout
@ -26,9 +26,19 @@ jobs:
distribution: 'zulu'
java-version: 17
- name: Build app
run: ./gradlew :app:assembleDemoRelease
- name: Install GMD image for baseline profile generation
run: yes | "$ANDROID_HOME"/cmdline-tools/latest/bin/sdkmanager "system-images;android-33;aosp_atd;x86_64"
- name: Accept Android licenses
run: yes | "$ANDROID_HOME"/cmdline-tools/latest/bin/sdkmanager --licenses || true
- name: Build release variant including baseline profile generation
run: ./gradlew :app:assembleDemoRelease
-Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.enabledRules=BaselineProfile
-Pandroid.testoptions.manageddevices.emulator.gpu="swiftshader_indirect"
-Pandroid.experimental.testOptions.managedDevices.emulator.showKernelLogging=true
-Pandroid.experimental.androidTest.numManagedDeviceShards=1
-Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=1
- name: Create Release
id: create_release
uses: actions/create-release@v1

@ -24,6 +24,7 @@ plugins {
id("jacoco")
alias(libs.plugins.nowinandroid.android.application.firebase)
id("com.google.android.gms.oss-licenses-plugin")
alias(libs.plugins.baselineprofile)
}
android {
@ -43,7 +44,7 @@ android {
debug {
applicationIdSuffix = NiaBuildType.DEBUG.applicationIdSuffix
}
val release by getting {
val release = getByName("release") {
isMinifyEnabled = true
applicationIdSuffix = NiaBuildType.RELEASE.applicationIdSuffix
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
@ -52,6 +53,8 @@ android {
// who clones the code to sign and run the release variant, use the debug signing key.
// TODO: Abstract the signing configuration to a separate file to avoid hardcoding this.
signingConfig = signingConfigs.getByName("debug")
// Ensure Baseline Profile is fresh for release builds.
baselineProfile.automaticGenerationDuringBuild = true
}
create("benchmark") {
// Enable all the optimizations from release build through initWith(release).
@ -121,6 +124,8 @@ dependencies {
implementation(libs.kotlinx.coroutines.guava)
implementation(libs.coil.kt)
baselineProfile(project(":benchmarks"))
// Core functions
testImplementation(projects.core.testing)
testImplementation(projects.core.datastoreTest)
@ -133,3 +138,9 @@ dependencies {
kspTest(libs.hilt.compiler)
}
baselineProfile {
// Don't build on every iteration of a full assemble.
// Instead enable generation directly for the release build variant.
automaticGenerationDuringBuild = false
}

@ -17,6 +17,7 @@ import com.google.samples.apps.nowinandroid.NiaBuildType
import com.google.samples.apps.nowinandroid.configureFlavors
plugins {
alias(libs.plugins.baselineprofile)
alias(libs.plugins.nowinandroid.android.test)
}
@ -62,10 +63,27 @@ android {
)
}
testOptions.managedDevices.devices {
create<com.android.build.api.dsl.ManagedVirtualDevice>("pixel6Api33") {
device = "Pixel 6"
apiLevel = 33
systemImageSource = "aosp"
}
}
targetProjectPath = ":app"
experimentalProperties["android.experimental.self-instrumenting"] = true
}
baselineProfile {
// This specifies the managed devices to use that you run the tests on.
managedDevices += "pixel6Api33"
// Don't use a connected device but rely on a GMD for consistency between local and CI builds.
useConnectedDevices = false
}
dependencies {
implementation(libs.androidx.benchmark.macro)
implementation(libs.androidx.test.core)
@ -75,9 +93,3 @@ dependencies {
implementation(libs.androidx.test.runner)
implementation(libs.androidx.test.uiautomator)
}
androidComponents {
beforeVariants {
it.enable = it.buildType == "benchmark"
}
}

@ -20,6 +20,10 @@ import android.Manifest.permission
import android.os.Build.VERSION.SDK_INT
import android.os.Build.VERSION_CODES.TIRAMISU
import androidx.benchmark.macro.MacrobenchmarkScope
import androidx.test.uiautomator.By
import androidx.test.uiautomator.BySelector
import androidx.test.uiautomator.UiObject2
import androidx.test.uiautomator.Until
/**
* Because the app under test is different from the one running the instrumentation test,
@ -42,3 +46,27 @@ fun MacrobenchmarkScope.allowNotifications() {
device.executeShellCommand(command)
}
}
/**
* Wraps starting the default activity, waiting for it to start and then allowing notifications in
* one convenient call.
*/
fun MacrobenchmarkScope.startActivityAndAllowNotifications() {
startActivityAndWait()
allowNotifications()
}
/**
* Waits for and returns the `niaTopAppBar`
*/
fun MacrobenchmarkScope.getTopAppBar(): UiObject2 {
device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
return device.findObject(By.res("niaTopAppBar"))
}
/**
* Waits for an object on the top app bar, passed in as [selector].
*/
fun MacrobenchmarkScope.waitForObjectOnTopAppBar(selector: BySelector, timeout: Long = 2_000) {
getTopAppBar().wait(Until.hasObject(selector), timeout)
}

@ -0,0 +1,40 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.samples.apps.nowinandroid.baselineprofile
import androidx.benchmark.macro.junit4.BaselineProfileRule
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.bookmarks.goToBookmarksScreen
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
/**
* Baseline Profile of the "Bookmarks" screen
*/
class BookmarksBaselineProfile {
@get:Rule val baselineProfileRule = BaselineProfileRule()
@Test
fun generate() =
baselineProfileRule.collect(PACKAGE_NAME) {
startActivityAndAllowNotifications()
// Navigate to saved screen
goToBookmarksScreen()
}
}

@ -18,43 +18,27 @@ package com.google.samples.apps.nowinandroid.baselineprofile
import androidx.benchmark.macro.junit4.BaselineProfileRule
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.allowNotifications
import com.google.samples.apps.nowinandroid.bookmarks.goToBookmarksScreen
import com.google.samples.apps.nowinandroid.foryou.forYouScrollFeedDownUp
import com.google.samples.apps.nowinandroid.foryou.forYouSelectTopics
import com.google.samples.apps.nowinandroid.foryou.forYouWaitForContent
import com.google.samples.apps.nowinandroid.interests.goToInterestsScreen
import com.google.samples.apps.nowinandroid.interests.interestsScrollTopicsDownUp
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
/**
* Generates a baseline profile which can be copied to `app/src/main/baseline-prof.txt`.
* Baseline Profile of the "For You" screen
*/
class BaselineProfileGenerator {
class ForYouBaselineProfile {
@get:Rule val baselineProfileRule = BaselineProfileRule()
@Test
fun generate() =
baselineProfileRule.collect(PACKAGE_NAME) {
// This block defines the app's critical user journey. Here we are interested in
// optimizing for app startup. But you can also navigate and scroll
// through your most important UI.
allowNotifications()
pressHome()
startActivityAndWait()
allowNotifications()
startActivityAndAllowNotifications()
// Scroll the feed critical user journey
forYouWaitForContent()
forYouSelectTopics(true)
forYouScrollFeedDownUp()
// Navigate to saved screen
goToBookmarksScreen()
// Navigate to interests screen
goToInterestsScreen()
interestsScrollTopicsDownUp()
}
}

@ -0,0 +1,42 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.samples.apps.nowinandroid.baselineprofile
import androidx.benchmark.macro.junit4.BaselineProfileRule
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.interests.goToInterestsScreen
import com.google.samples.apps.nowinandroid.interests.interestsScrollTopicsDownUp
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
/**
* Baseline Profile of the "Interests" screen
*/
class InterestsBaselineProfile {
@get:Rule val baselineProfileRule = BaselineProfileRule()
@Test
fun generate() =
baselineProfileRule.collect(PACKAGE_NAME) {
startActivityAndAllowNotifications()
// Navigate to interests screen
goToInterestsScreen()
interestsScrollTopicsDownUp()
}
}

@ -0,0 +1,40 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.samples.apps.nowinandroid.baselineprofile
import androidx.benchmark.macro.junit4.BaselineProfileRule
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
/**
* Baseline Profile for app startup. This profile also enables using [Dex Layout Optimizations](https://developer.android.com/topic/performance/baselineprofiles/dex-layout-optimizations)
* via the `includeInStartupProfile` parameter.
*/
class StartupBaselineProfile {
@get:Rule val baselineProfileRule = BaselineProfileRule()
@Test
fun generate() =
baselineProfileRule.collect(
PACKAGE_NAME,
includeInStartupProfile = true,
) {
startActivityAndAllowNotifications()
}
}

@ -18,13 +18,13 @@ package com.google.samples.apps.nowinandroid.bookmarks
import androidx.benchmark.macro.MacrobenchmarkScope
import androidx.test.uiautomator.By
import androidx.test.uiautomator.Until
import com.google.samples.apps.nowinandroid.waitForObjectOnTopAppBar
fun MacrobenchmarkScope.goToBookmarksScreen() {
device.findObject(By.text("Saved")).click()
val savedSelector = By.text("Saved")
val savedButton = device.findObject(savedSelector)
savedButton.click()
device.waitForIdle()
// Wait until saved title are shown on screen
device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
val topAppBar = device.findObject(By.res("niaTopAppBar"))
topAppBar.wait(Until.hasObject(By.text("Saved")), 2_000)
waitForObjectOnTopAppBar(savedSelector)
}

@ -22,6 +22,8 @@ import androidx.test.uiautomator.Until
import androidx.test.uiautomator.untilHasChildren
import com.google.samples.apps.nowinandroid.flingElementDownUp
import com.google.samples.apps.nowinandroid.waitAndFindObject
import com.google.samples.apps.nowinandroid.waitForObjectOnTopAppBar
import org.junit.Assert.fail
fun MacrobenchmarkScope.forYouWaitForContent() {
// Wait until content is loaded by checking if topics are loaded
@ -49,6 +51,9 @@ fun MacrobenchmarkScope.forYouSelectTopics(recheckTopicsIfChecked: Boolean = fal
var visited = 0
while (visited < 3) {
if (topics.childCount == 0) {
fail("No topics found, can't generate profile for ForYou page.")
}
// Selecting some topics, which will populate items in the feed.
val topic = topics.children[index % topics.childCount]
// Find the checkable element to figure out whether it's checked or not
@ -99,7 +104,5 @@ fun MacrobenchmarkScope.setAppTheme(isDark: Boolean) {
device.findObject(By.text("OK")).click()
// Wait until the top app bar is visible on screen
device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
val topAppBar = device.findObject(By.res("niaTopAppBar"))
topAppBar.wait(Until.hasObject(By.text("Now in Android")), 2_000)
waitForObjectOnTopAppBar(By.text("Now in Android"))
}

@ -22,7 +22,7 @@ import androidx.benchmark.macro.StartupMode
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.allowNotifications
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@ -47,8 +47,7 @@ class ScrollForYouFeedBenchmark {
setupBlock = {
// Start the app
pressHome()
startActivityAndWait()
allowNotifications()
startActivityAndAllowNotifications()
},
) {
forYouWaitForContent()

@ -20,22 +20,21 @@ import androidx.benchmark.macro.MacrobenchmarkScope
import androidx.test.uiautomator.By
import androidx.test.uiautomator.Until
import com.google.samples.apps.nowinandroid.flingElementDownUp
import com.google.samples.apps.nowinandroid.waitAndFindObject
import com.google.samples.apps.nowinandroid.waitForObjectOnTopAppBar
fun MacrobenchmarkScope.goToInterestsScreen() {
device.findObject(By.text("Interests")).click()
device.waitForIdle()
// Wait until interests are shown on screen
device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
val topAppBar = device.findObject(By.res("niaTopAppBar"))
topAppBar.wait(Until.hasObject(By.text("Interests")), 2_000)
waitForObjectOnTopAppBar(By.text("Interests"))
// Wait until content is loaded by checking if interests are loaded
device.wait(Until.gone(By.res("loadingWheel")), 5_000)
}
fun MacrobenchmarkScope.interestsScrollTopicsDownUp() {
val topicsList = device.waitAndFindObject(By.res("interests:topics"), 2_000)
device.wait(Until.hasObject(By.res("interests:topics")), 5_000)
val topicsList = device.findObject(By.res("interests:topics"))
device.flingElementDownUp(topicsList)
}

@ -23,7 +23,7 @@ import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.uiautomator.By
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.allowNotifications
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@ -47,8 +47,7 @@ class ScrollTopicListBenchmark {
setupBlock = {
// Start the app
pressHome()
startActivityAndWait()
allowNotifications()
startActivityAndAllowNotifications()
// Navigate to interests screen
device.findObject(By.text("Interests")).click()
device.waitForIdle()

@ -23,7 +23,7 @@ import androidx.benchmark.macro.junit4.MacrobenchmarkRule
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.uiautomator.By
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.allowNotifications
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@ -47,8 +47,7 @@ class TopicsScreenRecompositionBenchmark {
setupBlock = {
// Start the app
pressHome()
startActivityAndWait()
allowNotifications()
startActivityAndAllowNotifications()
// Navigate to interests screen
device.findObject(By.text("Interests")).click()
device.waitForIdle()

@ -26,6 +26,7 @@ import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
import com.google.samples.apps.nowinandroid.PACKAGE_NAME
import com.google.samples.apps.nowinandroid.allowNotifications
import com.google.samples.apps.nowinandroid.foryou.forYouWaitForContent
import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@ -41,32 +42,32 @@ class StartupBenchmark {
val benchmarkRule = MacrobenchmarkRule()
@Test
fun startupNoCompilation() = startup(CompilationMode.None())
fun startupWithoutPreCompilation() = startup(CompilationMode.None())
@Test
fun startupBaselineProfileDisabled() = startup(
fun startupWithPartialCompilationAndDisabledBaselineProfile() = startup(
CompilationMode.Partial(baselineProfileMode = Disable, warmupIterations = 1),
)
@Test
fun startupBaselineProfile() = startup(CompilationMode.Partial(baselineProfileMode = Require))
fun startupPrecompiledWithBaselineProfile() =
startup(CompilationMode.Partial(baselineProfileMode = Require))
@Test
fun startupFullCompilation() = startup(CompilationMode.Full())
fun startupFullyPrecompiled() = startup(CompilationMode.Full())
private fun startup(compilationMode: CompilationMode) = benchmarkRule.measureRepeated(
packageName = PACKAGE_NAME,
metrics = listOf(StartupTimingMetric()),
compilationMode = compilationMode,
iterations = 10,
iterations = 20, // More iterations result in higher statistical significance.
startupMode = COLD,
setupBlock = {
pressHome()
allowNotifications()
},
) {
startActivityAndWait()
allowNotifications()
startActivityAndAllowNotifications()
// Waits until the content is ready to capture Time To Full Display
forYouWaitForContent()
}

@ -32,6 +32,8 @@ buildscript {
// Lists all plugins used throughout the project without applying them.
plugins {
alias(libs.plugins.android.application) apply false
alias(libs.plugins.android.test) apply false
alias(libs.plugins.baselineprofile) apply false
alias(libs.plugins.kotlin.jvm) apply false
alias(libs.plugins.kotlin.serialization) apply false
alias(libs.plugins.firebase.crashlytics) apply false

@ -15,6 +15,7 @@ androidxCoreSplashscreen = "1.0.1"
androidxDataStore = "1.0.0"
androidxEspresso = "3.5.1"
androidxHiltNavigationCompose = "1.0.0"
androidxJunit = "1.1.5"
androidxLifecycle = "2.6.2"
androidxMacroBenchmark = "1.2.0"
androidxMetrics = "1.0.0-alpha04"
@ -142,6 +143,7 @@ turbine = { group = "app.cash.turbine", name = "turbine", version.ref = "turbine
# Dependencies of the included build-logic
android-gradlePlugin = { group = "com.android.tools.build", name = "gradle", version.ref = "androidGradlePlugin" }
android-tools-common = { group = "com.android.tools", name = "common", version.ref = "androidTools" }
androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "androidxJunit" }
firebase-crashlytics-gradlePlugin = { group = "com.google.firebase", name = "firebase-crashlytics-gradle", version.ref = "firebaseCrashlyticsPlugin" }
firebase-performance-gradlePlugin = { group = "com.google.firebase", name = "perf-plugin", version.ref = "firebasePerfPlugin" }
kotlin-gradlePlugin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version.ref = "kotlin" }
@ -152,6 +154,7 @@ work-testing = { group = "androidx.work", name = "work-testing", version = "2.8.
android-application = { id = "com.android.application", version.ref = "androidGradlePlugin" }
android-library = { id = "com.android.library", version.ref = "androidGradlePlugin" }
android-test = { id = "com.android.test", version.ref = "androidGradlePlugin" }
baselineprofile = { id = "androidx.baselineprofile", version.ref = "androidxMacroBenchmark"}
firebase-crashlytics = { id = "com.google.firebase.crashlytics", version.ref = "firebaseCrashlyticsPlugin" }
firebase-perf = { id = "com.google.firebase.firebase-perf", version.ref = "firebasePerfPlugin" }
gms = { id = "com.google.gms.google-services", version.ref = "gmsPlugin" }

Loading…
Cancel
Save