Issue #25972, #25971: reduce test failures with more retries on legacy-api-tests

app/src/androidTest/java/org/mozilla/fenix/helpers/RetryTestRule.kt

@@ -8,12 +8,23 @@ import androidx.test.espresso.IdlingResourceTimeoutException
 import androidx.test.espresso.NoMatchingViewException
 import androidx.test.uiautomator.UiObjectNotFoundException
 import junit.framework.AssertionFailedError
+import kotlinx.coroutines.runBlocking
 import org.junit.rules.TestRule
 import org.junit.runner.Description
 import org.junit.runners.model.Statement
+import org.mozilla.fenix.components.PermissionStorage
 import org.mozilla.fenix.helpers.IdlingResourceHelper.unregisterAllIdlingResources
+import org.mozilla.fenix.helpers.TestHelper.appContext
 
+/**
+ *  Rule to retry flaky tests for a given number of times, catching some of the more common exceptions.
+ *  The Rule doesn't clear the app state in between retries, so we are doing some cleanup here.
+ *  The @Before and @After methods are not called between retries.
+ *
+ */
 class RetryTestRule(private val retryCount: Int = 5) : TestRule {
+    // Used for clearing all permission data after each test try
+    private val permissionStorage = PermissionStorage(appContext.applicationContext)
 
     @Suppress("TooGenericExceptionCaught", "ComplexMethod")
     override fun apply(base: Statement, description: Description): Statement {
@@ -24,36 +35,57 @@ class RetryTestRule(private val retryCount: Int = 5) : TestRule {
                     break
                 } catch (t: AssertionError) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: AssertionFailedError) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: UiObjectNotFoundException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: NoMatchingViewException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: IdlingResourceTimeoutException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: RuntimeException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }
                 } catch (t: NullPointerException) {
                     unregisterAllIdlingResources()
+                    runBlocking {
+                        permissionStorage.deleteAllSitePermissions()
+                    }
                     if (i == retryCount) {
                         throw t
                     }

app/src/androidTest/java/org/mozilla/fenix/ui/SitePermissionsTest.kt

@@ -12,17 +12,16 @@ import android.os.Build
 import androidx.core.net.toUri
 import androidx.test.filters.SdkSuppress
 import androidx.test.rule.GrantPermissionRule
-import kotlinx.coroutines.runBlocking
 import org.junit.After
 import org.junit.Assume.assumeTrue
 import org.junit.Before
 import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
-import org.mozilla.fenix.components.PermissionStorage
 import org.mozilla.fenix.customannotations.SmokeTest
 import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityTestRule
+import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestHelper.appContext
 import org.mozilla.fenix.ui.robots.browserScreen
 import org.mozilla.fenix.ui.robots.navigationToolbar
@@ -48,22 +47,21 @@ class SitePermissionsTest {
         Manifest.permission.CAMERA
     )
 
+    @Rule
+    @JvmField
+    val retryTestRule = RetryTestRule(3)
+
     @Before
     fun setUp() {
         // disabling the new homepage pop-up that interferes with the tests.
         featureSettingsHelper.setJumpBackCFREnabled(false)
         featureSettingsHelper.deleteSitePermissions(true)
+        featureSettingsHelper.disablePwaCFR(true)
     }
 
     @After
     fun tearDown() {
-        // Clearing all permission data after each test to avoid overlapping data
-        val applicationContext: Context = activityTestRule.activity.applicationContext
-        val permissionStorage = PermissionStorage(applicationContext)
-
-        runBlocking {
-            permissionStorage.deleteAllSitePermissions()
-        }
+        featureSettingsHelper.resetAllFeatureFlags()
     }
 
     @SdkSuppress(maxSdkVersion = Build.VERSION_CODES.P, codeName = "P")

automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml

@@ -23,7 +23,7 @@ gcloud:
 
   # The number of times a test execution should be re-attempted if one or more failures occur.
   # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
-  num-flaky-test-attempts: 1
+  num-flaky-test-attempts: 2
 
   # test and app are the only required args
   app: /app/path