From 8c1476e3f7378c791b09e3abe54051337d4ce79c Mon Sep 17 00:00:00 2001 From: Aaron Train Date: Tue, 5 Jul 2022 12:36:38 -0400 Subject: [PATCH] No issue: Add new cron target task for legacy API UI tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setups up a new daily cron task to run select UI tests on API 28 (we’ll call it legacy). We don’t necessarily want to invoke Flank twice (serial) with a different configuration (FTL APIs dont support API splitting per test-target) on event CI but we do want them running elsewhere (cron). --- .cron.yml | 6 ++ .../flank-x86-legacy-api-tests.yml | 77 +++++++++++++++++++ automation/taskcluster/androidTest/ui-test.sh | 2 + taskcluster/ci/ui-test/kind.yml | 11 +++ taskcluster/fenix_taskgraph/target_tasks.py | 9 +++ 5 files changed, 105 insertions(+) create mode 100644 automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml diff --git a/.cron.yml b/.cron.yml index c51e01f7e..882cbc7df 100644 --- a/.cron.yml +++ b/.cron.yml @@ -30,3 +30,9 @@ jobs: treeherder-symbol: screenshots-D target-tasks-method: screenshots when: [{weekday: 'Monday', hour: 10, minute: 0}] + - name: legacy-api-ui-tests + job: + type: decision-task + treeherder-symbol: legacy-api-ui + target-tasks-method: legacy_api_ui_tests + when: [{hour: 10, minute: 30}] \ No newline at end of file diff --git a/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml new file mode 100644 index 000000000..2ea4a3c4b --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-legacy-api-tests.yml @@ -0,0 +1,77 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a test execution should be re-attempted if one or more failures occur. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 1 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: false + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest + - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest + - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest + - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest + - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest + + device: + - model: Pixel2 + version: 28 + locale: en_US + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. default: 1 + max-test-shards: -1 + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + ### Output Style flag + ## Output style of execution status. May be one of [verbose, multi, single, compact]. + ## For runs with only one test execution the default value is 'verbose', in other cases + ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles + ## which don't support ansi codes, to avoid corrupted output use single or verbose. + ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. + output-style: compact + ### Full Junit Result flag + ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. + full-junit-result: true diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 8e02e4be0..20c7efece 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -84,6 +84,8 @@ elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" elif [[ "${device_type}" == "x86-beta-tests" ]]; then flank_template="${PATH_TEST}/flank-x86-beta.yml" +elif [[ "${device_type}" == "x86-legacy-api-tests" ]]; then + flank_template="${PATH_TEST}/flank-x86-legacy-api-tests.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index 884b0be0a..87ded9598 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -148,3 +148,14 @@ jobs: - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app.apk, android-test.apk, '-1'] treeherder: symbol: nightly(ui-test-x86-nightly) + legacy-x86: + attributes: + legacy: true + description: Run select UI tests on older Android API on cron + run-on-tasks-for: [] + run-on-git-branches: [main] + run: + commands: + - [automation/taskcluster/androidTest/ui-test.sh, x86-legacy-api-tests, app.apk, android-test.apk, '-1'] + treeherder: + symbol: debug(legacy-x86) diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index dba5387e3..6b432d4d4 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -110,3 +110,12 @@ def target_tasks_screnshots(full_task_graph, parameters, graph_config): return task.attributes.get("screenshots", False) return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] + +@_target_task("legacy_api_ui_tests") +def target_tasks_legacy_api_ui_tests(full_task_graph, parameters, graph_config): + """Select the set of tasks required to run select UI tests on other API.""" + + def filter(task, parameters): + return task.attributes.get("legacy", False) + + return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] \ No newline at end of file