diff --git a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt index 8372a8fc9..f276ffbf6 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt @@ -115,6 +115,12 @@ object TestHelper { ) } + fun waitUntilSnackbarGone() { + mDevice.findObject( + UiSelector().resourceId("$packageName:id/snackbar_layout"), + ).waitUntilGone(waitingTime) + } + fun verifyUrl(urlSubstring: String, resourceName: String, resId: Int) { waitUntilObjectIsFound(resourceName) mDevice.findObject(UiSelector().text(urlSubstring)).waitForExists(waitingTime) @@ -252,6 +258,18 @@ object TestHelper { } } + fun assertPlayStoreOpens() { + if (isPackageInstalled(Constants.PackageName.GOOGLE_PLAY_SERVICES)) { + try { + intended(toPackage(Constants.PackageName.GOOGLE_PLAY_SERVICES)) + } catch (e: AssertionFailedError) { + BrowserRobot().verifyRateOnGooglePlayURL() + } + } else { + BrowserRobot().verifyRateOnGooglePlayURL() + } + } + /** * Checks whether the latest activity of the application is used for custom tabs or PWAs. * diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt index 11f08e722..60de6577d 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt @@ -26,6 +26,7 @@ import org.mozilla.fenix.helpers.HomeActivityTestRule import org.mozilla.fenix.helpers.RecyclerViewIdlingResource import org.mozilla.fenix.helpers.RetryTestRule import org.mozilla.fenix.helpers.TestAssetHelper +import org.mozilla.fenix.helpers.TestHelper import org.mozilla.fenix.helpers.TestHelper.longTapSelectItem import org.mozilla.fenix.ui.robots.bookmarksMenu import org.mozilla.fenix.ui.robots.browserScreen @@ -526,6 +527,8 @@ class BookmarksTest { confirmDeletion() verifyDeleteSnackBarText() verifyFolderTitle("3") + // On some devices we need to wait for the Snackbar to be gone before continuing + TestHelper.waitUntilSnackbarGone() }.closeMenu { } diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt index 5f876df3b..43631edfa 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt @@ -15,6 +15,7 @@ import org.junit.Rule import org.junit.Test import org.mozilla.fenix.ext.settings import org.mozilla.fenix.helpers.AndroidAssetDispatcher +import org.mozilla.fenix.helpers.FeatureSettingsHelper import org.mozilla.fenix.helpers.HomeActivityIntentTestRule import org.mozilla.fenix.helpers.RetryTestRule import org.mozilla.fenix.helpers.TestHelper.mDevice @@ -31,6 +32,7 @@ class SettingsAboutTest { private lateinit var mDevice: UiDevice private lateinit var mockWebServer: MockWebServer + private val featureSettingsHelper = FeatureSettingsHelper() @get:Rule val activityIntentTestRule = HomeActivityIntentTestRule() @@ -51,6 +53,7 @@ class SettingsAboutTest { @After fun tearDown() { mockWebServer.shutdown() + featureSettingsHelper.resetAllFeatureFlags() } // Walks through settings menu and sub-menus to ensure all items are present @@ -70,6 +73,8 @@ class SettingsAboutTest { // ABOUT @Test fun verifyRateOnGooglePlayRedirect() { + val settings = activityIntentTestRule.activity.settings() + settings.shouldShowTotalCookieProtectionCFR = false homeScreen { }.openThreeDotMenu { }.openSettings { diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt index ed65db6ca..5cbefd2ef 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt @@ -13,11 +13,10 @@ import org.junit.Rule import org.junit.Test import org.mozilla.fenix.customannotations.SmokeTest import org.mozilla.fenix.helpers.AndroidAssetDispatcher -import org.mozilla.fenix.helpers.Constants.PackageName.GOOGLE_PLAY_SERVICES import org.mozilla.fenix.helpers.FeatureSettingsHelper import org.mozilla.fenix.helpers.HomeActivityIntentTestRule import org.mozilla.fenix.helpers.TestAssetHelper -import org.mozilla.fenix.helpers.TestHelper.assertNativeAppOpens +import org.mozilla.fenix.helpers.TestHelper import org.mozilla.fenix.ui.robots.homeScreen import org.mozilla.fenix.ui.robots.navigationToolbar @@ -76,7 +75,6 @@ class SettingsAdvancedTest { @Test fun openLinkInAppTest() { val defaultWebPage = TestAssetHelper.getGenericAsset(mockWebServer, 3) - val playStoreUrl = "play.google.com/store/apps/details?id=org.mozilla.fenix" homeScreen { }.openThreeDotMenu { @@ -91,7 +89,7 @@ class SettingsAdvancedTest { mDevice.waitForIdle() clickLinkMatchingText("Mozilla Playstore link") mDevice.waitForIdle() - assertNativeAppOpens(GOOGLE_PLAY_SERVICES, playStoreUrl) + TestHelper.assertPlayStoreOpens() } } } diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt index fa223a273..bfc1b71ed 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt @@ -35,6 +35,7 @@ class SettingsSearchTest { } featureSettingsHelper.setJumpBackCFREnabled(false) featureSettingsHelper.setShowWallpaperOnboarding(false) + featureSettingsHelper.setTCPCFREnabled(false) } @After diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt index d23d71f1c..1b390d29d 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt @@ -6,6 +6,8 @@ package org.mozilla.fenix.ui.robots +import android.content.Intent +import android.net.Uri import androidx.recyclerview.widget.RecyclerView import androidx.test.espresso.Espresso.onView import androidx.test.espresso.ViewInteraction @@ -14,7 +16,8 @@ import androidx.test.espresso.assertion.ViewAssertions.matches import androidx.test.espresso.contrib.RecyclerViewActions import androidx.test.espresso.intent.Intents.intended import androidx.test.espresso.intent.matcher.IntentMatchers -import androidx.test.espresso.intent.matcher.IntentMatchers.toPackage +import androidx.test.espresso.intent.matcher.IntentMatchers.hasAction +import androidx.test.espresso.intent.matcher.IntentMatchers.hasData import androidx.test.espresso.matcher.ViewMatchers import androidx.test.espresso.matcher.ViewMatchers.Visibility import androidx.test.espresso.matcher.ViewMatchers.hasDescendant @@ -33,6 +36,7 @@ import androidx.test.uiautomator.UiObject import androidx.test.uiautomator.UiScrollable import androidx.test.uiautomator.UiSelector import androidx.test.uiautomator.Until +import junit.framework.AssertionFailedError import org.hamcrest.CoreMatchers import org.hamcrest.Matchers.allOf import org.hamcrest.Matchers.endsWith @@ -49,6 +53,7 @@ import org.mozilla.fenix.helpers.TestHelper.mDevice import org.mozilla.fenix.helpers.TestHelper.packageName import org.mozilla.fenix.helpers.TestHelper.scrollToElementByText import org.mozilla.fenix.helpers.click +import org.mozilla.fenix.settings.SupportUtils import org.mozilla.fenix.ui.robots.SettingsRobot.Companion.DEFAULT_APPS_SETTINGS_ACTION /** @@ -128,7 +133,6 @@ class SettingsRobot { fun openAboutFirefoxPreview(interact: SettingsSubMenuAboutRobot.() -> Unit): SettingsSubMenuAboutRobot.Transition { aboutFirefoxHeading().click() - SettingsSubMenuAboutRobot().interact() return SettingsSubMenuAboutRobot.Transition() } @@ -583,7 +587,16 @@ fun clickRateButtonGooglePlay() { private fun assertGooglePlayRedirect() { if (isPackageInstalled(GOOGLE_PLAY_SERVICES)) { - intended(toPackage(GOOGLE_PLAY_SERVICES)) + try { + intended( + allOf( + hasAction(Intent.ACTION_VIEW), + hasData(Uri.parse(SupportUtils.RATE_APP_URL)), + ), + ) + } catch (e: AssertionFailedError) { + BrowserRobot().verifyRateOnGooglePlayURL() + } } else { BrowserRobot().verifyRateOnGooglePlayURL() } diff --git a/automation/taskcluster/androidTest/flank-arm-beta.yml b/automation/taskcluster/androidTest/flank-arm-beta.yml new file mode 100644 index 000000000..20144b44d --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-beta.yml @@ -0,0 +1,42 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest + - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest + - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true + diff --git a/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml new file mode 100644 index 000000000..1cd97257d --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml @@ -0,0 +1,47 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 2 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest + - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest + + device: + - model: Pixel2.arm + version: 28 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: -1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml new file mode 100644 index 000000000..a753062da --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml @@ -0,0 +1,35 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - package org.mozilla.fenix.screenshots + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm-start-test.yml b/automation/taskcluster/androidTest/flank-arm-start-test.yml new file mode 100644 index 000000000..1303eb093 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm-start-test.yml @@ -0,0 +1,41 @@ +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + timeout: 30m + async: false + num-flaky-test-attempts: 1 + + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest + - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest + - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest + - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest + - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest + - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest + + device: + - model: Pixel2.arm + version: 30 + locale: en_US + +flank: + project: GOOGLE_PROJECT + max-test-shards: 1 + num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-arm64-v8a.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml index 58db5c740..09e2b730f 100644 --- a/automation/taskcluster/androidTest/flank-arm64-v8a.yml +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -1,31 +1,16 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ gcloud: results-bucket: fenix_test_artifacts record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false + async: false + num-flaky-test-attempts: 1 - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel + app: /app/path + test: /test/path - # test and app are the only required args - app: /APP/PATH - test: /TEST/PATH - - auto-google-login: true + auto-google-login: false use-orchestrator: true environment-variables: clearPackageData: true @@ -33,15 +18,18 @@ gcloud: - /sdcard/screenshots performance-metrics: true + test-targets: + - notPackage org.mozilla.fenix.screenshots + - notPackage org.mozilla.fenix.syncintegration + device: - - model: Pixel2 - version: 30 + - model: Pixel2.arm + version: 30 + locale: en_US flank: project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # num-test-runs- the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x + max-test-shards: 50 num-test-runs: 1 + output-style: compact + full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml deleted file mode 100644 index b1df75ed8..000000000 --- a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml +++ /dev/null @@ -1,48 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a.yml deleted file mode 100644 index 78a3137c6..000000000 --- a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml +++ /dev/null @@ -1,47 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /APP/PATH - test: /TEST/PATH - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # repeat tests - the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/flank-x86-beta.yml b/automation/taskcluster/androidTest/flank-x86-beta.yml deleted file mode 100644 index 92b6ed0d0..000000000 --- a/automation/taskcluster/androidTest/flank-x86-beta.yml +++ /dev/null @@ -1,61 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 20m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest - - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest - - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest - - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest - - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest - - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml deleted file mode 100644 index 226ff61e5..000000000 --- a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml +++ /dev/null @@ -1,77 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 2 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: false - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest - - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest - - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest - - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest - - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest - - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest - - device: - - model: Pixel2 - version: 28 - locale: en_US - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 - max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. - output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. - full-junit-result: true diff --git a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml deleted file mode 100644 index fd026eaa3..000000000 --- a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml +++ /dev/null @@ -1,55 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 20m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - package org.mozilla.fenix.screenshots - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86-start-test.yml b/automation/taskcluster/androidTest/flank-x86-start-test.yml deleted file mode 100644 index d16489550..000000000 --- a/automation/taskcluster/androidTest/flank-x86-start-test.yml +++ /dev/null @@ -1,61 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 1 - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - test-targets: - - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest - - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest - - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest - - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest - - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest - - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest - - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest - - device: - - model: Pixel2 - version: 30 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: 1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - num-test-runs: 1 - diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index 4226b9a20..30ede8817 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -1,32 +1,14 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +# Flank Documentation: https://flank.github.io/flank/ gcloud: results-bucket: fenix_test_artifacts record-video: true - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # The number of times a test execution should be re-attempted if one or more failures occur. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. num-flaky-test-attempts: 1 - # test and app are the only required args - app: /app/path + app: /app/path test: /test/path auto-google-login: false @@ -48,19 +30,7 @@ gcloud: flank: project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 max-test-shards: -1 - # num-test-runs: the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x num-test-runs: 1 - ### Output Style flag - ## Output style of execution status. May be one of [verbose, multi, single, compact]. - ## For runs with only one test execution the default value is 'verbose', in other cases - ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles - ## which don't support ansi codes, to avoid corrupted output use single or verbose. - ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. output-style: compact - ### Full Junit Result flag - ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. full-junit-result: true diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 20c7efece..0a86dffc1 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -76,16 +76,14 @@ set +e if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then flank_template="${PATH_TEST}/flank-${device_type}.yml" -elif [[ "${device_type}" == "x86-start-test" ]]; then - flank_template="${PATH_TEST}/flank-x86-start-test.yml" elif [[ "${device_type}" == "arm-start-test" ]]; then - flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" -elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" -elif [[ "${device_type}" == "x86-beta-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-beta.yml" -elif [[ "${device_type}" == "x86-legacy-api-tests" ]]; then - flank_template="${PATH_TEST}/flank-x86-legacy-api-tests.yml" + flank_template="${PATH_TEST}/flank-arm-start-test.yml" +elif [[ "${device_type}" == "arm-screenshots-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-screenshots-tests.yml" +elif [[ "${device_type}" == "arm-beta-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-beta.yml" +elif [[ "${device_type}" == "arm-legacy-api-tests" ]]; then + flank_template="${PATH_TEST}/flank-arm-legacy-api-tests.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/startup-test/kind.yml b/taskcluster/ci/startup-test/kind.yml index d852dd3e1..00c69f50a 100644 --- a/taskcluster/ci/startup-test/kind.yml +++ b/taskcluster/ci/startup-test/kind.yml @@ -45,19 +45,11 @@ task-defaults: signing-android-test: signing-android-test-nightly tasks: - nightly-x86: - run: - commands: - - [wget, {artifact-reference: ''}, '-O', app-x86.apk] - - [wget, {artifact-reference: ''}, '-O', android-test.apk] - - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1'] - treeherder: - symbol: nightly(startup-x86) nightly-arm: run: commands: - - [wget, {artifact-reference: ''}, '-O', app-arm.apk] + - [wget, {artifact-reference: ''}, '-O', app.apk] - [wget, {artifact-reference: ''}, '-O', android-test.apk] - - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1'] treeherder: symbol: nightly(startup-arm) diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index df185a7ba..6c1e1f518 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -74,7 +74,7 @@ task-defaults: ] run: commands: - - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', app.apk] - [wget, {artifact-reference: ''}, '-O', android-test.apk] secrets: - name: project/mobile/fenix/firebase @@ -101,16 +101,16 @@ task-defaults: worker-type: b-android tasks: - x86-debug: + arm-debug: description: Test Fenix run-on-tasks-for: [github-pull-request, github-push] run-on-git-branches: ["^((?!releases[_/]).+)$"] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm64-v8a, app.apk, android-test.apk, '50'] treeherder: - symbol: debug(ui-test-x86) - screenshots-x86: + symbol: debug(ui-test-arm) + screenshots-arm: attributes: screenshots: true description: Run UI screenshots tests to keep them up to date @@ -118,10 +118,10 @@ tasks: run-on-git-branches: [main] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-screenshots-tests, app.apk, android-test.apk, '1'] treeherder: - symbol: debug(screenshots-x86) - x86-beta: + symbol: debug(screenshots-arm) + arm-beta: attributes: build-type: beta-firebase description: Test Fenix @@ -132,10 +132,10 @@ tasks: signing-android-test: signing-android-test-beta run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-beta-tests, app.apk, android-test.apk, '50'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-beta-tests, app.apk, android-test.apk, '1'] treeherder: - symbol: beta(ui-test-x86-beta) - x86-nightly: + symbol: beta(ui-test-arm-beta) + arm-nightly: attributes: build-type: nightly-firebase description: Test Fenix @@ -145,10 +145,10 @@ tasks: signing-android-test: signing-android-test-nightly run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1'] treeherder: - symbol: nightly(ui-test-x86-nightly) - legacy-x86: + symbol: nightly(ui-test-arm-nightly) + legacy-arm: attributes: legacy: true description: Run select UI tests on older Android API on cron @@ -156,6 +156,6 @@ tasks: run-on-git-branches: [main] run: commands: - - [automation/taskcluster/androidTest/ui-test.sh, x86-legacy-api-tests, app.apk, android-test.apk, '-1'] + - [automation/taskcluster/androidTest/ui-test.sh, arm-legacy-api-tests, app.apk, android-test.apk, '1'] treeherder: - symbol: debug(legacy-x86) + symbol: debug(legacy-arm)