#! /usr/bin/env python3
#
# Copyright 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Regenerate some ART test related files.
# This script handles only a subset of ART run-tests at the moment; additional
# cases will be added later.
import argparse import copy import collections import itertools import json import logging import os import re import sys import textwrap import xml.dom.minidom
from typing import List, Set, Any, Dict
# Common advisory placed at the top of all generated files.
ADVISORY = f"Generated by `{ME}`. Do not edit manually."
# Default indentation unit.
INDENT = " "
# Indentation unit for XML files.
XML_INDENT = " "
def reindent(str: str, indent: str = ""): """Reindent literal string while removing common leading spaces.""" return textwrap.indent(textwrap.dedent(str), indent)
def copyright_header_text(year: int): """Return the copyright header text used in XML files.""" return reindent(f"""\
Copyright (C) {year} The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. """, "")
# Name of the ART MTS test list containing "eng-only" test modules,
# which require a device-under-test running a `userdebug` or `eng`
# build.
ENG_ONLY_TEST_LIST_NAME = "mts-art-tests-list-eng-only"
# These ART run-tests are new and have not had enough post-submit runs
# to meet pre-submit SLOs. Monitor their post-submit runs before
# removing them from this set (in order to promote them to
# presubmits).
postsubmit_only_tests: Set[str] = set([
])
known_failing_on_hwasan_tests: Set[str] = set([ "CtsJdwpTestCases", # times out
# apexd fails to unmount com.android.runtime on ASan builds. "art_standalone_dexopt_chroot_setup_tests",
])
# ART gtests that do not need root access to the device.
art_gtest_user_module_names = [ "art_libnativebridge_cts_tests", "art_standalone_artd_tests", "art_standalone_cmdline_tests", "art_standalone_compiler_tests", "art_standalone_dex2oat_cts_tests", "art_standalone_dex2oat_tests", "art_standalone_dexdump_tests", "art_standalone_dexlist_tests", "art_standalone_libartbase_tests", "art_standalone_libartpalette_tests", "art_standalone_libartservice_tests", "art_standalone_libarttools_tests", "art_standalone_libdexfile_support_tests", "art_standalone_libdexfile_tests", "art_standalone_libprofile_tests", "art_standalone_oatdump_tests", "art_standalone_odrefresh_tests", "art_standalone_runtime_tests", "art_standalone_sigchain_tests", "libnativebridge-lazy-tests", "libnativebridge-tests", "libnativeloader_test",
]
# ART gtests that need root access to the device.
art_gtest_eng_only_module_names = [ "art_standalone_dexopt_chroot_setup_tests", "art_standalone_dexoptanalyzer_tests", "art_standalone_profman_tests", "libnativeloader_e2e_tests",
]
art_test_module_names = [ "art-test-target",
]
# All supported ART gtests.
art_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names)
# These ART gtests are new and have not had enough post-submit runs
# to meet pre-submit SLOs. Monitor their post-submit runs before
# removing them from this set (in order to promote them to
# presubmits).
art_gtest_postsubmit_only_module_names: List[str] = [
]
# ART gtests not supported in MTS.
art_gtest_modules_excluded_from_mts = [
]
# ART gtests supported in MTS that do not need root access to the device.
art_gtest_mts_user_module_names = [t for t in art_gtest_user_module_names if t not in art_gtest_modules_excluded_from_mts]
# ART gtests supported in presubmits.
art_gtest_presubmit_module_names = [t for t in art_gtest_module_names if t not in art_gtest_postsubmit_only_module_names]
# ART gtests supported in Mainline presubmits.
art_gtest_mainline_presubmit_module_names = list(art_gtest_presubmit_module_names)
# ART gtests supported in postsubmits.
unknown_art_gtest_postsubmit_only_module_names = [t for t in art_gtest_postsubmit_only_module_names if t not in art_gtest_module_names] if unknown_art_gtest_postsubmit_only_module_names:
logging.error(textwrap.dedent("""\
The following `art_gtest_postsubmit_only_module_names` elements are not part of
`art_gtest_module_names`: """) + str(unknown_art_gtest_postsubmit_only_module_names))
sys.exit(1)
art_gtest_postsubmit_module_names = copy.copy(art_gtest_postsubmit_only_module_names)
# Tests exhibiting a flaky behavior, currently exluded from MTS for
# the stake of stability / confidence (b/209958457).
flaky_tests_excluded_from_mts = { "CtsLibcoreFileIOTestCases": [
("android.cts.FileChannelInterProcessLockTest#" + m) for m in [ "test_lockJJZ_Exclusive_asyncChannel", "test_lockJJZ_Exclusive_syncChannel", "test_lock_differentChannelTypes", "test_lockJJZ_Shared_asyncChannel", "test_lockJJZ_Shared_syncChannel",
]
], "CtsLibcoreTestCases": [
("com.android.org.conscrypt.javax.net.ssl.SSLSocketVersionCompatibilityTest#" + m + c) for (m, c) in itertools.product(
[ "test_SSLSocket_interrupt_read_withoutAutoClose", "test_SSLSocket_setSoWriteTimeout",
],
[ "[0: TLSv1.2 client, TLSv1.2 server]", "[1: TLSv1.2 client, TLSv1.3 server]", "[2: TLSv1.3 client, TLSv1.2 server]", "[3: TLSv1.3 client, TLSv1.3 server]",
]
)
] + [
("libcore.dalvik.system.DelegateLastClassLoaderTest#" + m) for m in [ "testLookupOrderNodelegate_getResource", "testLookupOrder_getResource",
]
]
}
# Tests excluded from all test mapping test groups.
#
# Example of admissible values in this dictionary:
#
# "art_standalone_cmdline_tests": ["CmdlineParserTest#TestCompilerOption"],
# "art_standalone_dexopt_chroot_setup_tests": ["DexoptChrootSetupTest#HelloWorld"],
#
failing_tests_excluded_from_test_mapping: Dict[str, List[str]] = {
# Empty.
}
test_declarations_comments = [tests_description + "."]
test_declarations_comments.extend(comments) for c in test_declarations_comments:
append_comment(c) for t in tests:
append_option("compatibility:include-filter", t)
module_controller_configuration_comments = [
f"Enable MainlineTestModuleController for {tests_description}."]
module_controller_configuration_comments.extend(comments) for c in module_controller_configuration_comments:
append_comment(c) for t in tests:
append_option("compatibility:module-arg", f"{t}:enable:true") for t in tests: if t in ["CtsLibcoreTestCases", "CtsLibcoreOjTestCases"]:
append_comment("core-test-mode=mts tells ExpectationBasedFilter to exclude @NonMts Tests")
append_option("compatibility:module-arg", f"{t}:instrumentation-arg:core-test-mode:=mts")
with open(test_list_file, "wb") as f:
logging.debug(f"Writing `{test_list_file}`.")
f.write(xml_str)
class Generator:
def __init__(self, top_dir: str): """Generator of ART test files for an Android source tree anchored at `top_dir`."""
# Path to the Android top source tree.
self.top_dir = top_dir
# Path to the ART directory
self.art_dir = os.path.join(top_dir, "art")
# Path to the ART tests top-level directory.
self.art_test_dir = os.path.join(self.art_dir, "test")
# Path to the MTS configuration directory.
self.mts_config_dir = os.path.join(
top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config")
# Path to the ART JVM TI CTS tests top-level directory.
self.jvmti_cts_test_dir = os.path.join(top_dir, "cts/hostsidetests/jvmti/run-tests")
# Return the list of ART JVM TI CTS tests.
def enumerate_jvmti_cts_tests(self): return sorted([re.sub(r"test-(\d+)", r"CtsJvmtiRunTest\1HostTestCases", cts_jvmti_test_dir) for cts_jvmti_test_dir in os.listdir(self.jvmti_cts_test_dir) if re.match(r"^test-\d+$", cts_jvmti_test_dir)])
# See go/test-mapping#attributes and
# https://source.android.com/docs/core/tests/development/test-mapping
# for more information about Test Mapping test groups.
def gen_tests_dict(tests: List[str],
excluded_test_cases: Dict[str, List[str]],
excluded_test_modules: Set[str] = set(),
suffix: str = "") -> List[Any]: return [
({"name": t + suffix, "options": [
{"exclude-filter": e} for e in excluded_test_cases[t]
]} if t in excluded_test_cases else {"name": t + suffix}) for t in tests if t not in excluded_test_modules
]
# Postsubmits.
postsubmit_tests = art_gtest_postsubmit_module_names
postsubmit_tests_dict = [{"name": t} for t in postsubmit_tests]
postsubmit_tests_dict = gen_tests_dict(postsubmit_tests,
failing_tests_excluded_from_test_mapping)
# Use an `OrderedDict` container to preserve the order in which items are inserted.
# Do not produce an entry for a test group if it is empty.
test_mapping_dict = collections.OrderedDict([
(test_group_name, test_group_dict) for (test_group_name, test_group_dict)
in [
("art-mainline-presubmit", art_mainline_presubmit_tests_dict),
("mainline-presubmit", mainline_presubmit_tests_dict),
("presubmit", presubmit_tests_dict),
("hwasan-presubmit", hwasan_presubmit_tests_dict),
("postsubmit", postsubmit_tests_dict),
] if test_group_dict
])
test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT)
test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING")
with open(test_mapping_file, "w") as f:
logging.debug(f"Writing `{test_mapping_file}`.")
f.write(f"// {ADVISORY}\n")
f.write(test_mapping_contents)
f.write("\n")
configuration = root.createElement("configuration")
root.appendChild(configuration)
configuration.setAttribute( "description",
f"Run {self.test_plan_name()} from a preexisting MTS installation.")
# Included XML files.
included_xml_files = ["mts", self.test_list_name()]
# Special casefor the test plan of shard 03 (ART gtests), where we also
# include ART MTS eng-only tests.
#
# TODO(rpl): Restucture the MTS generation logic to avoid special-casing
# at that level of the generator. if self.shard_num == 3:
included_xml_files.append(ENG_ONLY_TEST_LIST_NAME) for xml_file in included_xml_files:
include = root.createElement("include")
include.setAttribute("name", xml_file)
configuration.appendChild(include)
# Test plan name.
option = root.createElement("option")
option.setAttribute("name", "plan")
option.setAttribute("value", self.test_plan_name())
configuration.appendChild(option)
test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml")
with open(test_plan_file, "wb") as f:
logging.debug(f"Writing `{test_plan_file}`.")
f.write(xml_str)
def regen_test_list_file(self): """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`)."""
configuration_description = \
f"List of ART MTS tests that do not need root access (shard {self.shard_id()})"
test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml")
gen_mts_test_list_file(self.tests, test_list_file, self.copyright_year,
configuration_description, self.tests_description, self.comments)
def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards: int): """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`)."""
root = xml.dom.minidom.Document()
configuration = root.createElement("configuration")
root.appendChild(configuration)
configuration.setAttribute("description", "List of ART MTS tests that do not need root access.")
# Included XML files. for s in range(num_mts_art_run_test_shards):
include = root.createElement("include")
include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}")
configuration.appendChild(include)
mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml")
with open(mts_art_tests_list_user_file, "wb") as f:
logging.debug(f"Writing `{mts_art_tests_list_user_file}`.")
f.write(xml_str)
def regen_art_mts_files(self, art_jvmti_cts_tests: List[str]): """Regenerate ART MTS definition files."""
# Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`)
# and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`).
old_test_plan_shards = sorted([
test_plan_shard for test_plan_shard in os.listdir(self.mts_config_dir) if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)]) for shard in old_test_plan_shards:
shard_path = os.path.join(self.mts_config_dir, shard) if os.path.exists(shard_path):
logging.debug(f"Removing `{shard_path}`.")
os.remove(shard_path)
mts_test_shards: List[Any] = []
# ART run-tests shard(s).
art_run_test_shards = [art_test_module_names] for i in range(len(art_run_test_shards)):
art_tests_shard_i_tests = art_run_test_shards[i]
art_tests_shard_i = self.create_mts_test_shard( "ART run-tests", art_tests_shard_i_tests, i, 2020)
mts_test_shards.append(art_tests_shard_i)
for s in mts_test_shards:
s.regen_test_plan_file()
s.regen_test_list_file()
# Generate the MTS test list file of "eng-only" tests (tests that
# need root access to the device-under-test and are not part of
# "user" test plans).
#
# TODO(rpl): Refactor the MTS file generation logic to better
# handle the special case of "eng-only" tests, which do not play
# well with `MtsTestShard` at the moment).
eng_only_test_list_file = os.path.join(self.mts_config_dir, ENG_ONLY_TEST_LIST_NAME + ".xml")
gen_mts_test_list_file(
art_gtest_eng_only_module_names + art_test_module_names, eng_only_test_list_file,
copyright_year = 2020,
configuration_description = "List of ART MTS tests that need root access.",
tests_description = "ART gtests")
for (num_tests, test_kind, tests_percentage, test_group_name) in [
(num_mainline_presubmit_gtests, "ART gtests", mainline_presubmit_gtests_percentage, "mainline-presubmit"),
(num_presubmit_gtests, "ART gtests", presubmit_gtests_percentage, "presubmit"),
(num_postsubmit_gtests, "ART gtests", postsubmit_gtests_percentage, "postsubmit"),
]:
print(
f" {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.")
print(""" Note: Tests in `*presubmit` test groups are executed in pre- and
post-submit test runs. Tests in the `postsubmit` test group
are only executed in post-submit test runs.""")
# Regenerate ART MTS definition (optional).
# -----------------------------------------
if regen_art_mts:
self.regen_art_mts_files(self.enumerate_jvmti_cts_tests())
def main(): if"ANDROID_BUILD_TOP" not in os.environ:
logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?")
sys.exit(1)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("Regenerate some ART test related files."),
epilog=textwrap.dedent("""\
Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and
optionally the ART MTS (Mainline Test Suite) definition. """))
parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well",
action="store_true")
parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true")
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)