For #26792 - Change active Firebase Test Lab emulator from x86 to ARM (#26793)

* For #26792 - Change active Firebase Test Lab emulator from x86 to ARM

Update startup-test kind.yml Jobs for ARM
Update ui-test kind.yml Jobs for ARM
Cleans up and renames Flank configurations for ARM
Updates ui-test.sh for ARM

* Fix BookmarksTest.kt and HomeScreenTest.kt

* Fix intent matcher for SettingsAboutTest

Adds a helper function for Play Store intent

* Fix SettingsSearchTest.kt

* ktlint fix TestHelper.kt and SettingsRobot.kt
home_button_fix
Aaron Train 3 months ago committed by GitHub
parent 2899a66b15
commit 4e99fe8521
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 18
      app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt
  2. 3
      app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt
  3. 5
      app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt
  4. 6
      app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt
  5. 1
      app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt
  6. 19
      app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt
  7. 42
      automation/taskcluster/androidTest/flank-arm-beta.yml
  8. 47
      automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml
  9. 35
      automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml
  10. 41
      automation/taskcluster/androidTest/flank-arm-start-test.yml
  11. 46
      automation/taskcluster/androidTest/flank-arm64-v8a.yml
  12. 48
      automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml
  13. 47
      automation/taskcluster/androidTest/flank-armeabi-v7a.yml
  14. 61
      automation/taskcluster/androidTest/flank-x86-beta.yml
  15. 77
      automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml
  16. 55
      automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml
  17. 61
      automation/taskcluster/androidTest/flank-x86-start-test.yml
  18. 36
      automation/taskcluster/androidTest/flank-x86.yml
  19. 16
      automation/taskcluster/androidTest/ui-test.sh
  20. 12
      taskcluster/ci/startup-test/kind.yml
  21. 32
      taskcluster/ci/ui-test/kind.yml

@ -115,6 +115,12 @@ object TestHelper {
)
}
fun waitUntilSnackbarGone() {
mDevice.findObject(
UiSelector().resourceId("$packageName:id/snackbar_layout"),
).waitUntilGone(waitingTime)
}
fun verifyUrl(urlSubstring: String, resourceName: String, resId: Int) {
waitUntilObjectIsFound(resourceName)
mDevice.findObject(UiSelector().text(urlSubstring)).waitForExists(waitingTime)
@ -252,6 +258,18 @@ object TestHelper {
}
}
fun assertPlayStoreOpens() {
if (isPackageInstalled(Constants.PackageName.GOOGLE_PLAY_SERVICES)) {
try {
intended(toPackage(Constants.PackageName.GOOGLE_PLAY_SERVICES))
} catch (e: AssertionFailedError) {
BrowserRobot().verifyRateOnGooglePlayURL()
}
} else {
BrowserRobot().verifyRateOnGooglePlayURL()
}
}
/**
* Checks whether the latest activity of the application is used for custom tabs or PWAs.
*

@ -26,6 +26,7 @@ import org.mozilla.fenix.helpers.HomeActivityTestRule
import org.mozilla.fenix.helpers.RecyclerViewIdlingResource
import org.mozilla.fenix.helpers.RetryTestRule
import org.mozilla.fenix.helpers.TestAssetHelper
import org.mozilla.fenix.helpers.TestHelper
import org.mozilla.fenix.helpers.TestHelper.longTapSelectItem
import org.mozilla.fenix.ui.robots.bookmarksMenu
import org.mozilla.fenix.ui.robots.browserScreen
@ -526,6 +527,8 @@ class BookmarksTest {
confirmDeletion()
verifyDeleteSnackBarText()
verifyFolderTitle("3")
// On some devices we need to wait for the Snackbar to be gone before continuing
TestHelper.waitUntilSnackbarGone()
}.closeMenu {
}

@ -15,6 +15,7 @@ import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.ext.settings
import org.mozilla.fenix.helpers.AndroidAssetDispatcher
import org.mozilla.fenix.helpers.FeatureSettingsHelper
import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
import org.mozilla.fenix.helpers.RetryTestRule
import org.mozilla.fenix.helpers.TestHelper.mDevice
@ -31,6 +32,7 @@ class SettingsAboutTest {
private lateinit var mDevice: UiDevice
private lateinit var mockWebServer: MockWebServer
private val featureSettingsHelper = FeatureSettingsHelper()
@get:Rule
val activityIntentTestRule = HomeActivityIntentTestRule()
@ -51,6 +53,7 @@ class SettingsAboutTest {
@After
fun tearDown() {
mockWebServer.shutdown()
featureSettingsHelper.resetAllFeatureFlags()
}
// Walks through settings menu and sub-menus to ensure all items are present
@ -70,6 +73,8 @@ class SettingsAboutTest {
// ABOUT
@Test
fun verifyRateOnGooglePlayRedirect() {
val settings = activityIntentTestRule.activity.settings()
settings.shouldShowTotalCookieProtectionCFR = false
homeScreen {
}.openThreeDotMenu {
}.openSettings {

@ -13,11 +13,10 @@ import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.customannotations.SmokeTest
import org.mozilla.fenix.helpers.AndroidAssetDispatcher
import org.mozilla.fenix.helpers.Constants.PackageName.GOOGLE_PLAY_SERVICES
import org.mozilla.fenix.helpers.FeatureSettingsHelper
import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
import org.mozilla.fenix.helpers.TestAssetHelper
import org.mozilla.fenix.helpers.TestHelper.assertNativeAppOpens
import org.mozilla.fenix.helpers.TestHelper
import org.mozilla.fenix.ui.robots.homeScreen
import org.mozilla.fenix.ui.robots.navigationToolbar
@ -76,7 +75,6 @@ class SettingsAdvancedTest {
@Test
fun openLinkInAppTest() {
val defaultWebPage = TestAssetHelper.getGenericAsset(mockWebServer, 3)
val playStoreUrl = "play.google.com/store/apps/details?id=org.mozilla.fenix"
homeScreen {
}.openThreeDotMenu {
@ -91,7 +89,7 @@ class SettingsAdvancedTest {
mDevice.waitForIdle()
clickLinkMatchingText("Mozilla Playstore link")
mDevice.waitForIdle()
assertNativeAppOpens(GOOGLE_PLAY_SERVICES, playStoreUrl)
TestHelper.assertPlayStoreOpens()
}
}
}

@ -35,6 +35,7 @@ class SettingsSearchTest {
}
featureSettingsHelper.setJumpBackCFREnabled(false)
featureSettingsHelper.setShowWallpaperOnboarding(false)
featureSettingsHelper.setTCPCFREnabled(false)
}
@After

@ -6,6 +6,8 @@
package org.mozilla.fenix.ui.robots
import android.content.Intent
import android.net.Uri
import androidx.recyclerview.widget.RecyclerView
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.ViewInteraction
@ -14,7 +16,8 @@ import androidx.test.espresso.assertion.ViewAssertions.matches
import androidx.test.espresso.contrib.RecyclerViewActions
import androidx.test.espresso.intent.Intents.intended
import androidx.test.espresso.intent.matcher.IntentMatchers
import androidx.test.espresso.intent.matcher.IntentMatchers.toPackage
import androidx.test.espresso.intent.matcher.IntentMatchers.hasAction
import androidx.test.espresso.intent.matcher.IntentMatchers.hasData
import androidx.test.espresso.matcher.ViewMatchers
import androidx.test.espresso.matcher.ViewMatchers.Visibility
import androidx.test.espresso.matcher.ViewMatchers.hasDescendant
@ -33,6 +36,7 @@ import androidx.test.uiautomator.UiObject
import androidx.test.uiautomator.UiScrollable
import androidx.test.uiautomator.UiSelector
import androidx.test.uiautomator.Until
import junit.framework.AssertionFailedError
import org.hamcrest.CoreMatchers
import org.hamcrest.Matchers.allOf
import org.hamcrest.Matchers.endsWith
@ -49,6 +53,7 @@ import org.mozilla.fenix.helpers.TestHelper.mDevice
import org.mozilla.fenix.helpers.TestHelper.packageName
import org.mozilla.fenix.helpers.TestHelper.scrollToElementByText
import org.mozilla.fenix.helpers.click
import org.mozilla.fenix.settings.SupportUtils
import org.mozilla.fenix.ui.robots.SettingsRobot.Companion.DEFAULT_APPS_SETTINGS_ACTION
/**
@ -128,7 +133,6 @@ class SettingsRobot {
fun openAboutFirefoxPreview(interact: SettingsSubMenuAboutRobot.() -> Unit):
SettingsSubMenuAboutRobot.Transition {
aboutFirefoxHeading().click()
SettingsSubMenuAboutRobot().interact()
return SettingsSubMenuAboutRobot.Transition()
}
@ -583,7 +587,16 @@ fun clickRateButtonGooglePlay() {
private fun assertGooglePlayRedirect() {
if (isPackageInstalled(GOOGLE_PLAY_SERVICES)) {
intended(toPackage(GOOGLE_PLAY_SERVICES))
try {
intended(
allOf(
hasAction(Intent.ACTION_VIEW),
hasData(Uri.parse(SupportUtils.RATE_APP_URL)),
),
)
} catch (e: AssertionFailedError) {
BrowserRobot().verifyRateOnGooglePlayURL()
}
} else {
BrowserRobot().verifyRateOnGooglePlayURL()
}

@ -0,0 +1,42 @@
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
timeout: 30m
async: false
num-flaky-test-attempts: 1
app: /app/path
test: /test/path
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
device:
- model: Pixel2.arm
version: 30
locale: en_US
flank:
project: GOOGLE_PROJECT
max-test-shards: 1
num-test-runs: 1
output-style: compact
full-junit-result: true

@ -0,0 +1,47 @@
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
timeout: 30m
async: false
num-flaky-test-attempts: 2
app: /app/path
test: /test/path
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
- class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
- class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
- class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
- class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
device:
- model: Pixel2.arm
version: 28
locale: en_US
flank:
project: GOOGLE_PROJECT
max-test-shards: -1
num-test-runs: 1
output-style: compact
full-junit-result: true

@ -0,0 +1,35 @@
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
timeout: 30m
async: false
num-flaky-test-attempts: 1
app: /app/path
test: /test/path
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- package org.mozilla.fenix.screenshots
device:
- model: Pixel2.arm
version: 30
locale: en_US
flank:
project: GOOGLE_PROJECT
max-test-shards: 1
num-test-runs: 1
output-style: compact
full-junit-result: true

@ -0,0 +1,41 @@
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
timeout: 30m
async: false
num-flaky-test-attempts: 1
app: /app/path
test: /test/path
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
device:
- model: Pixel2.arm
version: 30
locale: en_US
flank:
project: GOOGLE_PROJECT
max-test-shards: 1
num-test-runs: 1
output-style: compact
full-junit-result: true

@ -1,31 +1,16 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
async: false
num-flaky-test-attempts: 1
# test and app are the only required args
app: /APP/PATH
test: /TEST/PATH
app: /app/path
test: /test/path
auto-google-login: true
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
@ -33,15 +18,18 @@ gcloud:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- notPackage org.mozilla.fenix.screenshots
- notPackage org.mozilla.fenix.syncintegration
device:
- model: Pixel2
version: 30
- model: Pixel2.arm
version: 30
locale: en_US
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: -1
# num-test-runs- the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
max-test-shards: 50
num-test-runs: 1
output-style: compact
full-junit-result: true

@ -1,48 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
device:
- model: Pixel2
version: 30
flank:
project: GOOGLE_PROJECT
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1

@ -1,47 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# test and app are the only required args
app: /APP/PATH
test: /TEST/PATH
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
device:
- model: Pixel2
version: 30
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: -1
# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
repeat-tests: 1

@ -1,61 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 20m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# The number of times a test execution should be re-attempted if one or more failures occur.
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
num-flaky-test-attempts: 1
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
device:
- model: Pixel2
version: 30
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: 1
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1

@ -1,77 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# The number of times a test execution should be re-attempted if one or more failures occur.
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
num-flaky-test-attempts: 2
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: false
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
- class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
- class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
- class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
- class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
device:
- model: Pixel2
version: 28
locale: en_US
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test. default: 1
max-test-shards: -1
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1
### Output Style flag
## Output style of execution status. May be one of [verbose, multi, single, compact].
## For runs with only one test execution the default value is 'verbose', in other cases
## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
## which don't support ansi codes, to avoid corrupted output use single or verbose.
## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
output-style: compact
### Full Junit Result flag
## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
full-junit-result: true

@ -1,55 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 20m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# The number of times a test execution should be re-attempted if one or more failures occur.
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
num-flaky-test-attempts: 1
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- package org.mozilla.fenix.screenshots
device:
- model: Pixel2
version: 30
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: 1
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1

@ -1,61 +0,0 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# The number of times a test execution should be re-attempted if one or more failures occur.
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
num-flaky-test-attempts: 1
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
test-targets:
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
device:
- model: Pixel2
version: 30
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: 1
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1

@ -1,32 +1,14 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
# Flank Documentation: https://flank.github.io/flank/
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# The number of times a test execution should be re-attempted if one or more failures occur.
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
num-flaky-test-attempts: 1
# test and app are the only required args
app: /app/path
app: /app/path
test: /test/path
auto-google-login: false
@ -48,19 +30,7 @@ gcloud:
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test. default: 1
max-test-shards: -1
# num-test-runs: the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
num-test-runs: 1
### Output Style flag
## Output style of execution status. May be one of [verbose, multi, single, compact].
## For runs with only one test execution the default value is 'verbose', in other cases
## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
## which don't support ansi codes, to avoid corrupted output use single or verbose.
## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
output-style: compact
### Full Junit Result flag
## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
full-junit-result: true

@ -76,16 +76,14 @@ set +e
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
flank_template="${PATH_TEST}/flank-${device_type}.yml"
elif [[ "${device_type}" == "x86-start-test" ]]; then
flank_template="${PATH_TEST}/flank-x86-start-test.yml"
elif [[ "${device_type}" == "arm-start-test" ]]; then
flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml"
elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then
flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml"
elif [[ "${device_type}" == "x86-beta-tests" ]]; then
flank_template="${PATH_TEST}/flank-x86-beta.yml"
elif [[ "${device_type}" == "x86-legacy-api-tests" ]]; then
flank_template="${PATH_TEST}/flank-x86-legacy-api-tests.yml"
flank_template="${PATH_TEST}/flank-arm-start-test.yml"
elif [[ "${device_type}" == "arm-screenshots-tests" ]]; then
flank_template="${PATH_TEST}/flank-arm-screenshots-tests.yml"
elif [[ "${device_type}" == "arm-beta-tests" ]]; then
flank_template="${PATH_TEST}/flank-arm-beta.yml"
elif [[ "${device_type}" == "arm-legacy-api-tests" ]]; then
flank_template="${PATH_TEST}/flank-arm-legacy-api-tests.yml"
else
echo "FAILURE: flank config file not found!"
exitcode=1

@ -45,19 +45,11 @@ task-defaults:
signing-android-test: signing-android-test-nightly
tasks:
nightly-x86:
run:
commands:
- [wget, {artifact-reference: '<signing/public/build/x86/target.apk>'}, '-O', app-x86.apk]
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
- [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1']
treeherder:
symbol: nightly(startup-x86)
nightly-arm:
run:
commands:
- [wget, {artifact-reference: '<signing/public/build/armeabi-v7a/target.apk>'}, '-O', app-arm.apk]
- [wget, {artifact-reference: '<signing/public/build/arm64-v8a/target.apk>'}, '-O', app.apk]
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
- [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1']
- [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1']
treeherder:
symbol: nightly(startup-arm)

@ -74,7 +74,7 @@ task-defaults:
]
run:
commands:
- [wget, {artifact-reference: '<signing/public/build/x86/target.apk>'}, '-O', app.apk]
- [wget, {artifact-reference: '<signing/public/build/arm64-v8a/target.apk>'}, '-O', app.apk]
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
secrets:
- name: project/mobile/fenix/firebase
@ -101,16 +101,16 @@ task-defaults:
worker-type: b-android
tasks:
x86-debug:
arm-debug:
description: Test Fenix
run-on-tasks-for: [github-pull-request, github-push]
run-on-git-branches: ["^((?!releases[_/]).+)$"]
run:
commands:
- [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1']
- [automation/taskcluster/androidTest/ui-test.sh, arm64-v8a, app.apk, android-test.apk, '50']
treeherder:
symbol: debug(ui-test-x86)
screenshots-x86:
symbol: debug(ui-test-arm)
screenshots-arm:
attributes:
screenshots: true
description: Run UI screenshots tests to keep them up to date
@ -118,10 +118,10 @@ tasks:
run-on-git-branches: [main]
run:
commands:
- [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1']
- [automation/taskcluster/androidTest/ui-test.sh, arm-screenshots-tests, app.apk, android-test.apk, '1']
treeherder:
symbol: debug(screenshots-x86)
x86-beta:
symbol: debug(screenshots-arm)
arm-beta:
attributes:
build-type: beta-firebase
description: Test Fenix
@ -132,10 +132,10 @@ tasks:
signing-android-test: signing-android-test-beta
run:
commands:
- [automation/taskcluster/androidTest/ui-test.sh, x86-beta-tests, app.apk, android-test.apk, '50']
- [automation/taskcluster/androidTest/ui-test.sh, arm-beta-tests, app.apk, android-test.apk, '1']
treeherder:
symbol: beta(ui-test-x86-beta)
x86-nightly:
symbol: beta(ui-test-arm-beta)
arm-nightly:
attributes:
build-type: nightly-firebase
description: Test Fenix
@ -145,10 +145,10 @@ tasks:
signing-android-test: signing-android-test-nightly
run:
commands:
- [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app.apk, android-test.apk, '-1']
- [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1']
treeherder:
symbol: nightly(ui-test-x86-nightly)
legacy-x86:
symbol: nightly(ui-test-arm-nightly)
legacy-arm:
attributes:
legacy: true
description: Run select UI tests on older Android API on cron
@ -156,6 +156,6 @@ tasks:
run-on-git-branches: [main]
run:
commands:
- [automation/taskcluster/androidTest/ui-test.sh, x86-legacy-api-tests, app.apk, android-test.apk, '-1']
- [automation/taskcluster/androidTest/ui-test.sh, arm-legacy-api-tests, app.apk, android-test.apk, '1']
treeherder:
symbol: debug(legacy-x86)
symbol: debug(legacy-arm)

Loading…
Cancel
Save