For #26792 - Change active Firebase Test Lab emulator from x86 to ARM (#26793)

* For #26792 - Change active Firebase Test Lab emulator from x86 to ARM Update startup-test kind.yml Jobs for ARM Update ui-test kind.yml Jobs for ARM Cleans up and renames Flank configurations for ARM Updates ui-test.sh for ARM * Fix BookmarksTest.kt and HomeScreenTest.kt * Fix intent matcher for SettingsAboutTest Adds a helper function for Play Store intent * Fix SettingsSearchTest.kt * ktlint fix TestHelper.kt and SettingsRobot.kt
2022-09-14 10:42:25 -04:00 · 2022-09-14 10:42:25 -04:00 · 4e99fe8521
parent 2899a66b15
commit 4e99fe8521
21 changed files with 255 additions and 453 deletions
--- a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt
@ -115,6 +115,12 @@ object TestHelper {
        )
    }

+    fun waitUntilSnackbarGone() {
+        mDevice.findObject(
+            UiSelector().resourceId("$packageName:id/snackbar_layout"),
+        ).waitUntilGone(waitingTime)
+    }
+
    fun verifyUrl(urlSubstring: String, resourceName: String, resId: Int) {
        waitUntilObjectIsFound(resourceName)
        mDevice.findObject(UiSelector().text(urlSubstring)).waitForExists(waitingTime)
@ -252,6 +258,18 @@ object TestHelper {
        }
    }

+    fun assertPlayStoreOpens() {
+        if (isPackageInstalled(Constants.PackageName.GOOGLE_PLAY_SERVICES)) {
+            try {
+                intended(toPackage(Constants.PackageName.GOOGLE_PLAY_SERVICES))
+            } catch (e: AssertionFailedError) {
+                BrowserRobot().verifyRateOnGooglePlayURL()
+            }
+        } else {
+            BrowserRobot().verifyRateOnGooglePlayURL()
+        }
+    }
+
    /**
     * Checks whether the latest activity of the application is used for custom tabs or PWAs.
     *
--- a/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt
@ -26,6 +26,7 @@ import org.mozilla.fenix.helpers.HomeActivityTestRule
 import org.mozilla.fenix.helpers.RecyclerViewIdlingResource
 import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestAssetHelper
+import org.mozilla.fenix.helpers.TestHelper
 import org.mozilla.fenix.helpers.TestHelper.longTapSelectItem
 import org.mozilla.fenix.ui.robots.bookmarksMenu
 import org.mozilla.fenix.ui.robots.browserScreen
@ -526,6 +527,8 @@ class BookmarksTest {
            confirmDeletion()
            verifyDeleteSnackBarText()
            verifyFolderTitle("3")
+            // On some devices we need to wait for the Snackbar to be gone before continuing
+            TestHelper.waitUntilSnackbarGone()
        }.closeMenu {
        }

--- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt
@ -15,6 +15,7 @@ import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.ext.settings
 import org.mozilla.fenix.helpers.AndroidAssetDispatcher
+import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
 import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestHelper.mDevice
@ -31,6 +32,7 @@ class SettingsAboutTest {

    private lateinit var mDevice: UiDevice
    private lateinit var mockWebServer: MockWebServer
+    private val featureSettingsHelper = FeatureSettingsHelper()

    @get:Rule
    val activityIntentTestRule = HomeActivityIntentTestRule()
@ -51,6 +53,7 @@ class SettingsAboutTest {
    @After
    fun tearDown() {
        mockWebServer.shutdown()
+        featureSettingsHelper.resetAllFeatureFlags()
    }

    // Walks through settings menu and sub-menus to ensure all items are present
@ -70,6 +73,8 @@ class SettingsAboutTest {
    // ABOUT
    @Test
    fun verifyRateOnGooglePlayRedirect() {
+        val settings = activityIntentTestRule.activity.settings()
+        settings.shouldShowTotalCookieProtectionCFR = false
        homeScreen {
        }.openThreeDotMenu {
        }.openSettings {
--- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt
@ -13,11 +13,10 @@ import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.customannotations.SmokeTest
 import org.mozilla.fenix.helpers.AndroidAssetDispatcher
-import org.mozilla.fenix.helpers.Constants.PackageName.GOOGLE_PLAY_SERVICES
 import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
 import org.mozilla.fenix.helpers.TestAssetHelper
-import org.mozilla.fenix.helpers.TestHelper.assertNativeAppOpens
+import org.mozilla.fenix.helpers.TestHelper
 import org.mozilla.fenix.ui.robots.homeScreen
 import org.mozilla.fenix.ui.robots.navigationToolbar

@ -76,7 +75,6 @@ class SettingsAdvancedTest {
    @Test
    fun openLinkInAppTest() {
        val defaultWebPage = TestAssetHelper.getGenericAsset(mockWebServer, 3)
-        val playStoreUrl = "play.google.com/store/apps/details?id=org.mozilla.fenix"

        homeScreen {
        }.openThreeDotMenu {
@ -91,7 +89,7 @@ class SettingsAdvancedTest {
            mDevice.waitForIdle()
            clickLinkMatchingText("Mozilla Playstore link")
            mDevice.waitForIdle()
-            assertNativeAppOpens(GOOGLE_PLAY_SERVICES, playStoreUrl)
+            TestHelper.assertPlayStoreOpens()
        }
    }
 }
--- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt
@ -35,6 +35,7 @@ class SettingsSearchTest {
        }
        featureSettingsHelper.setJumpBackCFREnabled(false)
        featureSettingsHelper.setShowWallpaperOnboarding(false)
+        featureSettingsHelper.setTCPCFREnabled(false)
    }

    @After
--- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt
+++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt
@ -6,6 +6,8 @@

 package org.mozilla.fenix.ui.robots

+import android.content.Intent
+import android.net.Uri
 import androidx.recyclerview.widget.RecyclerView
 import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.ViewInteraction
@ -14,7 +16,8 @@ import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.contrib.RecyclerViewActions
 import androidx.test.espresso.intent.Intents.intended
 import androidx.test.espresso.intent.matcher.IntentMatchers
-import androidx.test.espresso.intent.matcher.IntentMatchers.toPackage
+import androidx.test.espresso.intent.matcher.IntentMatchers.hasAction
+import androidx.test.espresso.intent.matcher.IntentMatchers.hasData
 import androidx.test.espresso.matcher.ViewMatchers
 import androidx.test.espresso.matcher.ViewMatchers.Visibility
 import androidx.test.espresso.matcher.ViewMatchers.hasDescendant
@ -33,6 +36,7 @@ import androidx.test.uiautomator.UiObject
 import androidx.test.uiautomator.UiScrollable
 import androidx.test.uiautomator.UiSelector
 import androidx.test.uiautomator.Until
+import junit.framework.AssertionFailedError
 import org.hamcrest.CoreMatchers
 import org.hamcrest.Matchers.allOf
 import org.hamcrest.Matchers.endsWith
@ -49,6 +53,7 @@ import org.mozilla.fenix.helpers.TestHelper.mDevice
 import org.mozilla.fenix.helpers.TestHelper.packageName
 import org.mozilla.fenix.helpers.TestHelper.scrollToElementByText
 import org.mozilla.fenix.helpers.click
+import org.mozilla.fenix.settings.SupportUtils
 import org.mozilla.fenix.ui.robots.SettingsRobot.Companion.DEFAULT_APPS_SETTINGS_ACTION

 /**
@ -128,7 +133,6 @@ class SettingsRobot {
        fun openAboutFirefoxPreview(interact: SettingsSubMenuAboutRobot.() -> Unit):
            SettingsSubMenuAboutRobot.Transition {
            aboutFirefoxHeading().click()
-
            SettingsSubMenuAboutRobot().interact()
            return SettingsSubMenuAboutRobot.Transition()
        }
@ -583,7 +587,16 @@ fun clickRateButtonGooglePlay() {

 private fun assertGooglePlayRedirect() {
    if (isPackageInstalled(GOOGLE_PLAY_SERVICES)) {
-        intended(toPackage(GOOGLE_PLAY_SERVICES))
+        try {
+            intended(
+                allOf(
+                    hasAction(Intent.ACTION_VIEW),
+                    hasData(Uri.parse(SupportUtils.RATE_APP_URL)),
+                ),
+            )
+        } catch (e: AssertionFailedError) {
+            BrowserRobot().verifyRateOnGooglePlayURL()
+        }
    } else {
        BrowserRobot().verifyRateOnGooglePlayURL()
    }
--- a/automation/taskcluster/androidTest/flank-arm-beta.yml
+++ b/automation/taskcluster/androidTest/flank-arm-beta.yml
@ -0,0 +1,42 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+    - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
+    - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
+    - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
+    - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
+    - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
+    - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
+    - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
+
+  device:
+    - model: Pixel2.arm
+      version: 30
+      locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
+
--- a/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml
+++ b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml
@ -0,0 +1,47 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 2
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+    - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
+    - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
+    - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
+    - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
+
+  device:
+    - model: Pixel2.arm
+      version: 28
+      locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: -1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
--- a/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml
+++ b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml
@ -0,0 +1,35 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+   - package org.mozilla.fenix.screenshots
+
+  device:
+   - model: Pixel2.arm
+     version: 30
+     locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
--- a/automation/taskcluster/androidTest/flank-arm-start-test.yml
+++ b/automation/taskcluster/androidTest/flank-arm-start-test.yml
@ -0,0 +1,41 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+   - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
+   - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
+   - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
+   - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
+   - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
+   - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
+   - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
+
+  device:
+   - model: Pixel2.arm
+     version: 30
+     locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
--- a/automation/taskcluster/androidTest/flank-arm64-v8a.yml
+++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml
@ -1,31 +1,16 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
 gcloud:
  results-bucket: fenix_test_artifacts
  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false 
+  async: false
+  num-flaky-test-attempts: 1

-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel
+  app: /app/path
+  test: /test/path

-  # test and app are the only required args
-  app: /APP/PATH
-  test: /TEST/PATH
-
-  auto-google-login: true
+  auto-google-login: false
  use-orchestrator: true
  environment-variables:
    clearPackageData: true
@ -33,15 +18,18 @@ gcloud:
    - /sdcard/screenshots
  performance-metrics: true

+  test-targets:
+    - notPackage org.mozilla.fenix.screenshots
+    - notPackage org.mozilla.fenix.syncintegration
+
  device:
-   - model: Pixel2
-     version: 30
+    - model: Pixel2.arm
+      version: 30
+      locale: en_US

 flank:
  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test.
-  max-test-shards: -1 
-  # num-test-runs- the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
+  max-test-shards: 50
  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
--- a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml
+++ b/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml
@ -1,48 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel 
-
-  # test and app are the only required args
-  app: /app/path 
-  test: /test/path
-
-  auto-google-login: true
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  test-targets:
-   - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
-
-  device:
-   - model: Pixel2
-     version: 30
-
-flank:
-  project: GOOGLE_PROJECT
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
-
--- a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml
+++ b/automation/taskcluster/androidTest/flank-armeabi-v7a.yml
@ -1,47 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false 
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel 
-
-  # test and app are the only required args
-  app: /APP/PATH
-  test: /TEST/PATH
-
-  auto-google-login: true
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  device:
-   - model: Pixel2
-     version: 30
-
-flank:
-  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test.
-  max-test-shards: -1 
-  # repeat tests - the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  repeat-tests: 1
--- a/automation/taskcluster/androidTest/flank-x86-beta.yml
+++ b/automation/taskcluster/androidTest/flank-x86-beta.yml
@ -1,61 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 20m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel 
-
-  # The number of times a test execution should be re-attempted if one or more failures occur.
-  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
-  num-flaky-test-attempts: 1
-
-  # test and app are the only required args
-  app: /app/path 
-  test: /test/path
-
-  auto-google-login: true
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  test-targets:
-    - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
-    - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
-    - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
-    - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
-    - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
-    - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
-    - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
-
-  device:
-   - model: Pixel2
-     version: 30
-
-flank:
-  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test.
-  max-test-shards: 1
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
-
--- a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml
+++ b/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml
@ -1,77 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel
-
-  # The number of times a test execution should be re-attempted if one or more failures occur.
-  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
-  num-flaky-test-attempts: 2
-
-  # test and app are the only required args
-  app: /app/path
-  test: /test/path
-
-  auto-google-login: false
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  test-targets:
-    - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
-    - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
-    - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
-    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
-    - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
-
-  device:
-    - model: Pixel2
-      version: 28
-      locale: en_US
-
-flank:
-  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test. default: 1
-  max-test-shards: -1
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
-  ### Output Style flag
-  ## Output style of execution status. May be one of [verbose, multi, single, compact].
-  ## For runs with only one test execution the default value is 'verbose', in other cases
-  ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
-  ## which don't support ansi codes, to avoid corrupted output use single or verbose.
-  ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
-  output-style: compact
-  ### Full Junit Result flag
-  ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
-  full-junit-result: true
--- a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml
+++ b/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml
@ -1,55 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 20m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel 
-
-  # The number of times a test execution should be re-attempted if one or more failures occur.
-  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
-  num-flaky-test-attempts: 1
-
-  # test and app are the only required args
-  app: /app/path 
-  test: /test/path
-
-  auto-google-login: true
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  test-targets:
-   - package org.mozilla.fenix.screenshots
-
-  device:
-   - model: Pixel2
-     version: 30
-
-flank:
-  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test.
-  max-test-shards: 1
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
-
--- a/automation/taskcluster/androidTest/flank-x86-start-test.yml
+++ b/automation/taskcluster/androidTest/flank-x86-start-test.yml
@ -1,61 +0,0 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
-gcloud:
-  results-bucket: fenix_test_artifacts
-  record-video: true
-
-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
-  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
-  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel
-
-  # The number of times a test execution should be re-attempted if one or more failures occur.
-  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
-  num-flaky-test-attempts: 1
-
-  # test and app are the only required args
-  app: /app/path 
-  test: /test/path
-
-  auto-google-login: true
-  use-orchestrator: true
-  environment-variables:
-    clearPackageData: true
-  directories-to-pull:
-    - /sdcard/screenshots
-  performance-metrics: true
-
-  test-targets:
-   - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
-   - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
-   - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
-   - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
-   - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
-   - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
-   - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
-
-  device:
-   - model: Pixel2
-     version: 30
-
-flank:
-  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test.
-  max-test-shards: 1
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
-
--- a/automation/taskcluster/androidTest/flank-x86.yml
+++ b/automation/taskcluster/androidTest/flank-x86.yml
@ -1,32 +1,14 @@
-# gcloud args match the official gcloud cli
-# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
 gcloud:
  results-bucket: fenix_test_artifacts
  record-video: true

-  # The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
  timeout: 30m
-  # will start test then close socket. no reports will be generated.
-  # to retrieve results later, use the "refresh" command
-  # reports will be generated from /results/matrix_ids.json
-  #async: true 
-  # will start test then leave socket open. reports will be published
-  # to /results
-  # see: https://github.com/TestArmada/flank/issues/339
  async: false
-
-  # results-history-name
-  # by default, set to app name
-  # declare results-history-name to create a separate dropdown menu in Firebase 
-  # see: https://github.com/TestArmada/flank/issues/341
-  #results-history-name: tmp_parallel 
-
-  # The number of times a test execution should be re-attempted if one or more failures occur.
-  # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
  num-flaky-test-attempts: 1

-  # test and app are the only required args
-  app: /app/path 
+  app: /app/path
  test: /test/path

  auto-google-login: false
@ -48,19 +30,7 @@ gcloud:

 flank:
  project: GOOGLE_PROJECT
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test. default: 1
  max-test-shards: -1
-  # num-test-runs: the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
  num-test-runs: 1
-  ### Output Style flag
-  ## Output style of execution status. May be one of [verbose, multi, single, compact].
-  ## For runs with only one test execution the default value is 'verbose', in other cases
-  ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
-  ## which don't support ansi codes, to avoid corrupted output use single or verbose.
-  ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
  output-style: compact
-  ### Full Junit Result flag
-  ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
  full-junit-result: true
--- a/automation/taskcluster/androidTest/ui-test.sh
+++ b/automation/taskcluster/androidTest/ui-test.sh
@ -76,16 +76,14 @@ set +e

 if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
    flank_template="${PATH_TEST}/flank-${device_type}.yml"
-elif [[ "${device_type}" == "x86-start-test" ]]; then
-    flank_template="${PATH_TEST}/flank-x86-start-test.yml"
 elif [[ "${device_type}" == "arm-start-test" ]]; then
-    flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml"
-elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then
-    flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml"
-elif [[ "${device_type}" == "x86-beta-tests" ]]; then
-    flank_template="${PATH_TEST}/flank-x86-beta.yml"
-elif [[ "${device_type}" == "x86-legacy-api-tests" ]]; then
-    flank_template="${PATH_TEST}/flank-x86-legacy-api-tests.yml"
+    flank_template="${PATH_TEST}/flank-arm-start-test.yml"
+elif [[ "${device_type}" == "arm-screenshots-tests" ]]; then
+    flank_template="${PATH_TEST}/flank-arm-screenshots-tests.yml"
+elif [[ "${device_type}" == "arm-beta-tests" ]]; then
+    flank_template="${PATH_TEST}/flank-arm-beta.yml"
+elif [[ "${device_type}" == "arm-legacy-api-tests" ]]; then
+    flank_template="${PATH_TEST}/flank-arm-legacy-api-tests.yml"
 else
    echo "FAILURE: flank config file not found!"
    exitcode=1
--- a/taskcluster/ci/startup-test/kind.yml
+++ b/taskcluster/ci/startup-test/kind.yml
@ -45,19 +45,11 @@ task-defaults:
        signing-android-test: signing-android-test-nightly

 tasks:
-    nightly-x86:
-        run:
-            commands:
-                - [wget, {artifact-reference: '<signing/public/build/x86/target.apk>'}, '-O', app-x86.apk]
-                - [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
-                - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1']
-        treeherder:
-            symbol: nightly(startup-x86)
    nightly-arm:
        run:
            commands:
-                - [wget, {artifact-reference: '<signing/public/build/armeabi-v7a/target.apk>'}, '-O', app-arm.apk]
+                - [wget, {artifact-reference: '<signing/public/build/arm64-v8a/target.apk>'}, '-O', app.apk]
                - [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
-                - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1']
        treeherder:
            symbol: nightly(startup-arm)
--- a/taskcluster/ci/ui-test/kind.yml
+++ b/taskcluster/ci/ui-test/kind.yml
@ -74,7 +74,7 @@ task-defaults:
                ]
    run:
        commands:
-            - [wget, {artifact-reference: '<signing/public/build/x86/target.apk>'}, '-O', app.apk]
+            - [wget, {artifact-reference: '<signing/public/build/arm64-v8a/target.apk>'}, '-O', app.apk]
            - [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
        secrets:
            - name: project/mobile/fenix/firebase
@ -101,16 +101,16 @@ task-defaults:
    worker-type: b-android

 tasks:
-    x86-debug:
+    arm-debug:
        description: Test Fenix
        run-on-tasks-for: [github-pull-request, github-push]
        run-on-git-branches: ["^((?!releases[_/]).+)$"]
        run:
            commands:
-                - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm64-v8a, app.apk, android-test.apk, '50']
        treeherder:
-            symbol: debug(ui-test-x86)
-    screenshots-x86:
+            symbol: debug(ui-test-arm)
+    screenshots-arm:
        attributes:
            screenshots: true
        description: Run UI screenshots tests to keep them up to date
@ -118,10 +118,10 @@ tasks:
        run-on-git-branches: [main]
        run:
            commands:
-                - [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm-screenshots-tests, app.apk, android-test.apk, '1']
        treeherder:
-            symbol: debug(screenshots-x86)
-    x86-beta:
+            symbol: debug(screenshots-arm)
+    arm-beta:
        attributes:
            build-type: beta-firebase
        description: Test Fenix
@ -132,10 +132,10 @@ tasks:
            signing-android-test: signing-android-test-beta
        run:
            commands:
-                - [automation/taskcluster/androidTest/ui-test.sh, x86-beta-tests, app.apk, android-test.apk, '50']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm-beta-tests, app.apk, android-test.apk, '1']
        treeherder:
-            symbol: beta(ui-test-x86-beta)
-    x86-nightly:
+            symbol: beta(ui-test-arm-beta)
+    arm-nightly:
        attributes:
            build-type: nightly-firebase
        description: Test Fenix
@ -145,10 +145,10 @@ tasks:
            signing-android-test: signing-android-test-nightly
        run:
            commands:
-                - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app.apk, android-test.apk, '-1']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1']
        treeherder:
-            symbol: nightly(ui-test-x86-nightly)
-    legacy-x86:
+            symbol: nightly(ui-test-arm-nightly)
+    legacy-arm:
        attributes:
            legacy: true
        description: Run select UI tests on older Android API on cron
@ -156,6 +156,6 @@ tasks:
        run-on-git-branches: [main]
        run:
            commands:
-                - [automation/taskcluster/androidTest/ui-test.sh, x86-legacy-api-tests, app.apk, android-test.apk, '-1']
+                - [automation/taskcluster/androidTest/ui-test.sh, arm-legacy-api-tests, app.apk, android-test.apk, '1']
        treeherder:
-            symbol: debug(legacy-x86)
+            symbol: debug(legacy-arm)