run 1

app/src/androidTest/java/org/mozilla/fenix/helpers/RetryTestRule.kt

@@ -8,12 +8,23 @@ import androidx.test.espresso.IdlingResourceTimeoutException
 import androidx.test.espresso.NoMatchingViewException
 import androidx.test.uiautomator.UiObjectNotFoundException
 import junit.framework.AssertionFailedError
+import kotlinx.coroutines.runBlocking
 import org.junit.rules.TestRule
 import org.junit.runner.Description
 import org.junit.runners.model.Statement
+import org.mozilla.fenix.components.PermissionStorage
 import org.mozilla.fenix.helpers.IdlingResourceHelper.unregisterAllIdlingResources
+import org.mozilla.fenix.helpers.TestHelper.appContext
 
+/**
+ *  Rule to retry flaky tests for a given number of times, catching some of the more common exceptions.
+ *  The Rule doesn't clear the app state in between retries, so we are doing some cleanup here.
+ *  The @Before and @After methods are not called between retries.
+ *
+ */
 class RetryTestRule(private val retryCount: Int = 5) : TestRule {
+    // Used for clearing all permission data after each test try
+    private val permissionStorage = PermissionStorage(appContext.applicationContext)
 
     @Suppress("TooGenericExceptionCaught", "ComplexMethod")
     override fun apply(base: Statement, description: Description): Statement {
@@ -24,36 +35,57 @@ class RetryTestRule(private val retryCount: Int = 5) : TestRule {
                     break
                 } catch (t: AssertionError) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: AssertionFailedError) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: UiObjectNotFoundException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: NoMatchingViewException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: IdlingResourceTimeoutException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: RuntimeException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: NullPointerException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }

app/src/androidTest/java/org/mozilla/fenix/ui/SitePermissionsTest.kt

@@ -12,17 +12,16 @@ import android.os.Build
 import androidx.core.net.toUri
 import androidx.test.filters.SdkSuppress
 import androidx.test.rule.GrantPermissionRule
-import kotlinx.coroutines.runBlocking
 import org.junit.After
 import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
-import org.mozilla.fenix.components.PermissionStorage
 import org.mozilla.fenix.customannotations.SmokeTest
 import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityTestRule
+import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestHelper.appContext
 import org.mozilla.fenix.ui.robots.browserScreen
 import org.mozilla.fenix.ui.robots.navigationToolbar
@@ -48,22 +47,21 @@ class SitePermissionsTest {
         Manifest.permission.CAMERA
     )
 
+    @Rule
+    @JvmField
+    val retryTestRule = RetryTestRule(3)
+
     @Before
     fun setUp() {
         // disabling the new homepage pop-up that interferes with the tests.
         featureSettingsHelper.setJumpBackCFREnabled(false)
         featureSettingsHelper.deleteSitePermissions(true)
+        featureSettingsHelper.disablePwaCFR(true)
     }
 
     @After
     fun tearDown() {
-        // Clearing all permission data after each test to avoid overlapping data
-        val applicationContext: Context = activityTestRule.activity.applicationContext
-        val permissionStorage = PermissionStorage(applicationContext)
-
-        runBlocking {
-            permissionStorage.deleteAllSitePermissions()
-        }
+        featureSettingsHelper.resetAllFeatureFlags()
     }
 
     @SdkSuppress(maxSdkVersion = Build.VERSION_CODES.P, codeName = "P")

automation/taskcluster/androidTest/flank-x86.yml

@@ -38,12 +38,14 @@ gcloud:
   performance-metrics: true
 
   test-targets:
-   - notPackage org.mozilla.fenix.screenshots
-   - notPackage org.mozilla.fenix.syncintegration
+#   - notPackage org.mozilla.fenix.screenshots
+#   - notPackage org.mozilla.fenix.syncintegration
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
 
   device:
    - model: Pixel2
-     version: 30
+     version: 28
      locale: en_US
 
 flank:
@@ -53,7 +55,7 @@ flank:
   max-test-shards: -1
   # num-test-runs: the amount of times to run the tests.
   # 1 runs the tests once. 10 runs all the tests 10x
-  num-test-runs: 1
+  num-test-runs: 50
   ### Output Style flag
   ## Output style of execution status. May be one of [verbose, multi, single, compact].
   ## For runs with only one test execution the default value is 'verbose', in other cases