Add taskcluster job for UI tests (#4088)

This commit is contained in:
Richard Pappalardo 2019-07-26 08:08:01 -07:00 committed by GitHub
parent 48aeb19db1
commit b39afe1548
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 383 additions and 22 deletions

14
.gitignore vendored
View File

@ -42,6 +42,10 @@ captures/
*.iml
.idea/
# Vim swap files
*.sw[op]
# Keystore files
# Uncomment the following lines if you do not want to check your keystore files in.
#*.jks
@ -82,3 +86,13 @@ gen-external-apklibs
# Python Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
venv/
# UI test artifacts
.firebase_token*
results/
test_artifacts/
/build/test-tools/google-cloud-sdk/
/build/test-tools/*.jar
/build/test-tools/*.gz

View File

@ -405,7 +405,11 @@ dependencies {
androidTestImplementation Deps.mockwebserver
testImplementation Deps.mozilla_support_test
testImplementation Deps.androidx_junit
testImplementation Deps.robolectric
testImplementation (Deps.robolectric) {
exclude group: 'org.apache.maven'
}
testImplementation 'org.apache.maven:maven-ant-tasks:2.1.3'
implementation Deps.fragment_testing
testImplementation Deps.places_forUnitTests

View File

@ -42,12 +42,12 @@ class SettingsRobot {
private fun assertSettingsView() {
// verify that we are in the correct library view
assertBasicsHeading()
assertAdvancedHeading()
assertPrivacyHeading()
}
private fun assertBasicsHeading() = onView(ViewMatchers.withText("Basics"))
.check(matches(withEffectiveVisibility(Visibility.VISIBLE)))
private fun assertAdvancedHeading() = onView(ViewMatchers.withText("Advanced"))
private fun assertPrivacyHeading() = onView(ViewMatchers.withText("Privacy"))
.check(matches(withEffectiveVisibility(Visibility.VISIBLE)))
private fun goBackButton() = onView(CoreMatchers.allOf(withContentDescription("Navigate up")))

View File

@ -0,0 +1,56 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# test and app are the only required args
app: /APP/PATH
test: /TEST/PATH
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
device:
- model: shamu
version: 21
- model: sailfish
version: 25
- model: sailfish
version: 28
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: -1
# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
repeat-tests: 1
# always run - these tests are inserted at the beginning of every shard
# useful if you need to grant permissions or login before other tests run
#test-targets-always-run:
#- class com.example.app.ExampleUiTest#testPasses
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses

View File

@ -0,0 +1,54 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# test and app are the only required args
app: /APP/PATH
test: /TEST/PATH
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
device:
- model: Nexus9
version: 21
- model: Nexus9
version: 22
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: -1
# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
repeat-tests: 1
# always run - these tests are inserted at the beginning of every shard
# useful if you need to grant permissions or login before other tests run
#test-targets-always-run:
#- class com.example.app.ExampleUiTest#testPasses
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses

View File

@ -0,0 +1,52 @@
# gcloud args match the official gcloud cli
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
gcloud:
results-bucket: fenix_test_artifacts
record-video: true
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
timeout: 30m
# will start test then close socket. no reports will be generated.
# to retrieve results later, use the "refresh" command
# reports will be generated from /results/matrix_ids.json
#async: true
# will start test then leave socket open. reports will be published
# to /results
# see: https://github.com/TestArmada/flank/issues/339
async: false
# results-history-name
# by default, set to app name
# declare results-history-name to create a separate dropdown menu in Firebase
# see: https://github.com/TestArmada/flank/issues/341
#results-history-name: tmp_parallel
# test and app are the only required args
app: /app/path
test: /test/path
auto-google-login: true
use-orchestrator: true
environment-variables:
clearPackageData: true
directories-to-pull:
- /sdcard/screenshots
performance-metrics: true
device:
- model: Nexus7
version: 21
flank:
project: GOOGLE_PROJECT
# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
max-test-shards: -1
# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
repeat-tests: 1
# always run - these tests are inserted at the beginning of every shard
# useful if you need to grant permissions or login before other tests run
#test-targets-always-run:
#- class com.example.app.ExampleUiTest#testPasses
# - class org.mozilla.focus.activty.SwitchContextTest#testPasses

View File

@ -0,0 +1,130 @@
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This script does the following:
# 1. Retrieves gcloud service account token
# 2. Activates gcloud service account
# 3. Connects to google Firebase (using TestArmada's Flank tool)
# 4. Executes UI tests
# 5. Puts test artifacts into the test_artifacts folder
# NOTE:
# Flank supports sharding across multiple devices at a time, but gcloud API
# only supports 1 defined APK per test run.
# If a command fails then do not proceed and fail this script too.
set -e
#########################
# The command line help #
#########################
display_help() {
echo "Usage: $0 Build_Variant [Number_Shards...]"
echo
echo "Examples:"
echo "To run UI tests on ARM device shard (1 test / shard)"
echo "$ ui-test.sh arm -1"
echo
echo "To run UI tests on X86 device (on 3 shards)"
echo "$ ui-test.sh feature x86 3"
echo
}
# Basic parameter check
if [[ $# -lt 1 ]]; then
echo "Error: please provide at least one build variant (arm|x86)"
display_help
exit 1
fi
device_type="$1" # arm | x86
if [[ ! -z "$2" ]]; then
num_shards=$2
fi
JAVA_BIN="/usr/bin/java"
PATH_TEST="./automation/taskcluster/androidTest"
FLANK_BIN="/build/test-tools/flank.jar"
FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml"
FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml"
echo
echo "RETRIEVE SERVICE ACCT TOKEN"
echo
python automation/taskcluster/helper/get-secret.py --json -s project/mobile/fenix/firebase -k firebaseToken -f $GOOGLE_APPLICATION_CREDENTIALS
echo
echo
echo
echo "ACTIVATE SERVICE ACCT"
echo
# this is where the Google Testcloud project ID is set
gcloud config set project "$GOOGLE_PROJECT"
echo
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
echo
echo
# From now on disable exiting on error. If the tests fail we want to continue
# and try to download the artifacts. We will exit with the actual error code later.
set +e
if [[ "${device_type,,}" == "x86" ]]
then
deviceType="X86"
flank_template="$FLANK_CONF_X86"
else
deviceType="Arm"
flank_template="$FLANK_CONF_ARM"
fi
APK_APP="./app/build/outputs/apk/${deviceType,,}/debug/app-${deviceType,,}-debug.apk"
APK_TEST="./app/build/outputs/apk/androidTest/${deviceType,,}/debug/app-${deviceType,,}-debug-androidTest.apk"
# function to exit script with exit code from test run.
# (Only 0 if all test executions passed)
function failure_check() {
if [[ $exitcode -ne 0 ]]; then
echo
echo
echo "ERROR: UI test run failed, please check above URL"
fi
exit $exitcode
}
echo
echo "EXECUTE TEST(S)"
echo
$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT
exitcode=$?
failure_check
echo
echo
echo
echo "COPY ARTIFACTS"
echo
cp -r ./results ./test_artifacts
exitcode=$?
failure_check
echo
echo
echo
echo "RESULTS"
echo
ls -la ./results
echo
echo
echo "All UI test(s) have passed!"
echo
echo

View File

@ -51,8 +51,8 @@ BUILDER = TaskBuilder(
)
def pr_or_push(is_push):
if not is_push and SKIP_TASKS_TRIGGER in PR_TITLE:
def pr():
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
@ -74,10 +74,18 @@ def pr_or_push(is_push):
):
other_tasks[taskcluster.slugId()] = craft_function()
if is_push and SHORT_HEAD_BRANCH == 'master':
return (build_tasks, signing_tasks, other_tasks)
def push():
all_tasks = pr()
other_tasks = all_tasks[-1]
other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task()
if SHORT_HEAD_BRANCH == 'master':
other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
return (build_tasks, signing_tasks, other_tasks)
return all_tasks
def raptor(is_staging):
@ -199,10 +207,10 @@ if __name__ == "__main__":
command = result.command
taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
if command == 'pull-request':
ordered_groups_of_tasks = pr_or_push(False)
elif command == 'push':
ordered_groups_of_tasks = pr_or_push(True)
if command in ('pull-request'):
ordered_groups_of_tasks = pr()
elif command in ('push'):
ordered_groups_of_tasks = push()
elif command == 'raptor':
ordered_groups_of_tasks = raptor(result.staging)
elif command == 'nightly':

View File

@ -4,23 +4,20 @@
import argparse
import base64
import json
import os
import errno
import taskcluster
def write_secret_to_file(path, data, key, base64decode=False, append=False, prefix=''):
def write_secret_to_file(path, data, key, base64decode=False, json_secret=False, append=False, prefix=''):
path = os.path.join(os.path.dirname(__file__), '../../../' + path)
try:
os.makedirs(os.path.dirname(path))
except OSError as error:
if error.errno != errno.EEXIST:
raise
with open(path, 'a' if append else 'w') as f:
value = data['secret'][key]
if base64decode:
value = base64.b64decode(value)
if json_secret:
value = json.dumps(value)
f.write(prefix + value)
@ -37,13 +34,14 @@ def main():
parser.add_argument('-k', dest='key', action="store", help='key of the secret')
parser.add_argument('-f', dest="path", action="store", help='file to save secret to')
parser.add_argument('--decode', dest="decode", action="store_true", default=False, help='base64 decode secret before saving to file')
parser.add_argument('--json', dest="json", action="store_true", default=False, help='serializes the secret to JSON format')
parser.add_argument('--append', dest="append", action="store_true", default=False, help='append secret to existing file')
parser.add_argument('--prefix', dest="prefix", action="store", default="", help='add prefix when writing secret to file')
result = parser.parse_args()
secret = fetch_secret_from_taskcluster(result.secret)
write_secret_to_file(result.path, secret, result.key, result.decode, result.append, result.prefix)
write_secret_to_file(result.path, secret, result.key, result.decode, result.json, result.append, result.prefix)
if __name__ == "__main__":

View File

@ -15,6 +15,8 @@ DEFAULT_EXPIRES_IN = '1 year'
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix'
_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task'
GOOGLE_PROJECT = "moz-fenix"
GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json'
class TaskBuilder(object):
@ -161,6 +163,48 @@ class TaskBuilder(object):
},
)
def craft_ui_tests_task(self):
artifacts = {
"public": {
"type": "directory",
"path": "/build/fenix/results",
"expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
}
}
env_vars = {
"GOOGLE_PROJECT": "moz-fenix",
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
}
gradle_commands = (
'./gradlew --no-daemon clean assembleArmDebug assembleArmDebugAndroidTest',
)
test_commands = (
'automation/taskcluster/androidTest/ui-test.sh arm -1',
)
command = ' && '.join(
cmd
for commands in (gradle_commands, test_commands)
for cmd in commands
if cmd
)
return self._craft_build_ish_task(
name='Fenix - UI test',
description='Execute Gradle tasks for UI tests',
command=command,
scopes=[
'secrets:get:project/mobile/fenix/firebase'
],
artifacts=artifacts,
env_vars=env_vars,
)
def craft_detekt_task(self):
return self._craft_clean_gradle_task(
name='detekt',
@ -268,12 +312,13 @@ class TaskBuilder(object):
def _craft_build_ish_task(
self, name, description, command, dependencies=None, artifacts=None, scopes=None,
routes=None, treeherder=None
routes=None, treeherder=None, env_vars=None,
):
dependencies = [] if dependencies is None else dependencies
artifacts = {} if artifacts is None else artifacts
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
env_vars = {} if env_vars is None else env_vars
checkout_command = ' && '.join([
"export TERM=dumb",
@ -289,11 +334,11 @@ class TaskBuilder(object):
features['chainOfTrust'] = True
if any(scope.startswith('secrets:') for scope in scopes):
features['taskclusterProxy'] = True
payload = {
"features": features,
"env": env_vars,
"maxRunTime": 7200,
"image": "mozillamobile/fenix:1.3",
"image": "mozillamobile/fenix:1.4",
"command": [
"/bin/bash",
"--login",