| # -*- coding: utf-8 -*- |
| # Copyright 2010-2015, Google Inc. |
| # All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """A script to run unittests on android. |
| |
| This file searches all tests under --test_bin_dir (the default is the current |
| directory), and run them on android (emulator or actual device). |
| You need to run an android emulator or to connect a dvice, before running this |
| script. |
| """ |
| |
| __author__ = "hidehiko" |
| |
| import errno |
| import logging |
| import multiprocessing |
| import optparse |
| import os |
| import subprocess |
| import time |
| from xml.etree import cElementTree as ElementTree |
| from build_tools import android_util |
| from build_tools.test_tools import gtest_report |
| |
| |
| def FindTestBinaries(test_dir): |
| """Returns the file names of tests.""" |
| logging.info('Gathering binaries in %s.', os.path.abspath(test_dir)) |
| result = [] |
| |
| for f in os.listdir(test_dir): |
| path = os.path.join(test_dir, f) |
| # Test binaries are "executable file" and "its name ends with _test." |
| if (os.access(path, os.R_OK | os.X_OK) and |
| os.path.isfile(path) and |
| f.endswith('_test')): |
| result.append(f) |
| return result |
| |
| |
| def ParseArgs(): |
| """Parses command line options and returns them.""" |
| parser = optparse.OptionParser() |
| parser.add_option('--run_java_test', dest='run_java_test', default=False, |
| action='store_true', |
| help='Runs JUnit tests. [JAVA] options are used.') |
| android_devices_default = os.environ.get('ANDROID_DEVICES', '') |
| parser.add_option('--android_devices', dest='android_devices', |
| default=android_devices_default, |
| help='[JAVA][NATIVE] Comma separated serial numbers ' |
| 'on which devices the tests run. ' |
| 'If not specified all the available devices are used.') |
| parser.add_option('--run_native_test', dest='run_native_test', default=False, |
| action='store_true', |
| help='Runs native tests. [NATIVE] options are used.') |
| parser.add_option('--configuration', dest='configuration', |
| default='Debug', |
| help='[JAVA] Build configuration. Used to build correct ' |
| 'testee binary.') |
| parser.add_option('--app_package_name', dest='app_package_name', |
| default='org.mozc.android.inputmethod.japanese', |
| help='[JAVA] Testee project\'s package name. ' |
| 'This is used to find test reporting XML file.') |
| parser.add_option('--remote_dir', dest='remote_dir', |
| default='/sdcard/mozctest', |
| help='[NATIVE] Working directory on ' |
| 'a SD card on a Android device') |
| parser.add_option('--remote_device', dest='remote_device', |
| default='/dev/block/vold/179:0', |
| help='[NATIVE] Path to the device to be (re)mounted.') |
| parser.add_option('--remote_mount_point', dest='remote_mount_point', |
| default='/sdcard', |
| help='[NATIVE] Mount point for the --remote_device.') |
| parser.add_option('--test_bin_dir', dest='test_bin_dir', default='.', |
| help='[NATIVE] The directory which contains test binaries') |
| parser.add_option('--test_case', dest='testcase', default='', |
| help='[NATIVE] Limits testcases to run.') |
| parser.add_option('--native_abi', dest='abi', default='armeabi', |
| help='[JAVA][NATIVE] ABI of built test executables.') |
| parser.add_option('--mozc_dictionary_data_file', |
| dest='mozc_dictionary_data_file', default=None, |
| help='[NATIVE] Path to system.dictionary file.') |
| parser.add_option('--mozc_connection_data_file', |
| dest='mozc_connection_data_file', default=None, |
| help='[NATIVE] Path to connection.data file.') |
| parser.add_option('--mozc_connection_text_data_file', |
| dest='mozc_connection_text_data_file', default=None, |
| help='[NATIVE] Path to connection_single_column.txt file.') |
| parser.add_option('--mozc_test_connection_data_file', |
| dest='mozc_test_connection_data_file', default=None, |
| help='[NATIVE] Path to test_connection.data file.') |
| parser.add_option('--mozc_test_connection_text_data_file', |
| dest='mozc_test_connection_text_data_file', default=None, |
| help='[NATIVE] Path to connection_single_column.txt file.') |
| parser.add_option('--mozc_data_dir', |
| dest='mozc_data_dir', default=None, |
| help='[NATIVE] Path to data directory.') |
| parser.add_option('--output_report_dir', dest='output_report_dir', |
| default=None, |
| help='[JAVA][NATIVE] Path to output gtest ' |
| 'reporting XML files') |
| parser.add_option('--android_home', dest='android_home', |
| default=None, |
| help='[JAVA][NATIVE] Path to the the SDK.') |
| return parser.parse_args()[0] |
| |
| |
| def AppendPrefixToSuiteName(in_file_name, out_file_name, prefix): |
| """Appends prefix to <testsuite> element's name attribute.""" |
| |
| def AppendPrefix(elem): |
| name = elem.attrib['name'] |
| elem.attrib['name'] = prefix + name |
| tree = ElementTree.parse(in_file_name) |
| root = tree.getroot() |
| # Root element isn't returned by any traversing methods. |
| if root.tag == 'testsuite': |
| AppendPrefix(root) |
| for elem in root.findall('testsuite'): |
| AppendPrefix(elem) |
| with open(out_file_name, 'w') as f: |
| # Note that ElementTree of 2.6 doesn't write XML declaration. |
| f.write('<?xml version="1.0" encoding="utf-8"?>\n') |
| f.write(ElementTree.tostring(root, 'utf-8')) |
| |
| |
| class AndroidDevice(android_util.AndroidDevice): |
| def __init__(self, serial, android_home): |
| android_util.AndroidDevice.__init__(self, serial, android_home) |
| |
| def WaitForMount(self): |
| """Wait until SD card is mounted.""" |
| retry = 10 |
| sleep = 30 |
| for _ in xrange(retry): |
| if self._RunCommand('mount').find('/sdcard') != -1: |
| self.GetLogger().info('SD card has been mounted.') |
| return |
| self.GetLogger().info( |
| 'SD card has not been mounted. Wait and retry. ' |
| 'Don\'t worry. This is typically expected behavior.') |
| time.sleep(sleep) |
| raise IOError('No mounted SD card found. Something went wrong or the ' |
| 'emulator is too slow.') |
| |
| def CopyFile(self, host_path, remote_path, operation): |
| """Copy a file between host and remote. |
| |
| Args: |
| host_path: path in host side. |
| remote_path: path in remote side. |
| operation: 'push' or 'pull'. |
| If 'push', 'adb push host_path remote_path' is executed. |
| If 'pull', 'adb pull remote_path host_path' is executed. |
| """ |
| if operation == 'push': |
| command_args = [os.path.join(self._android_home, 'platform-tools', 'adb'), |
| '-s', self.serial, 'push', host_path, remote_path] |
| elif operation == 'pull': |
| command_args = [os.path.join(self._android_home, 'platform-tools', 'adb'), |
| '-s', self.serial, 'pull', remote_path, host_path] |
| else: |
| raise ValueError('operation parameter must be push or pull but ' |
| '%s is given.' % operation) |
| self.GetLogger().info('Copying at %s: %s', self.serial, command_args) |
| process = subprocess.Popen(command_args) |
| if process.wait() == 0: |
| return |
| raise IOError( |
| 'Failed to copy a file: %s to %s' % (host_path, remote_path)) |
| |
| def _VerifyResultXMLFile(self, xml_path, test_name): |
| test_suites = gtest_report.GetFromXMLFile(xml_path) |
| if not test_suites: |
| error_message = ('[FAIL] Result XML file for %s is invalid; %s' % |
| (test_name, xml_path)) |
| else: |
| if test_suites.fail_num: |
| self.GetLogger().warning('[FAIL] %d of test failures for %s are found', |
| test_suites.fail_num, test_name) |
| error_message = test_suites.GetErrorSummary() |
| else: |
| self.GetLogger().info('[ OK ] All tests for %s succeeded.', test_name) |
| error_message = None |
| return error_message |
| |
| def _CopyAndVerifyResult( |
| self, test_name, remote_result_path, host_result_path): |
| try: |
| self.CopyFile(host_path=host_result_path, |
| remote_path=remote_result_path, |
| operation='pull') |
| self.GetLogger().info('[ OK ] %s result file is successfully pulled.', |
| test_name) |
| error_message = self._VerifyResultXMLFile(host_result_path, test_name) |
| except IOError: |
| self.GetLogger().warning('[FAIL] %s result file is not pulled.', |
| test_name) |
| error_message = ('[FAIL] Result file does not exist. ' |
| 'The process might crash: %s' % test_name) |
| if error_message: |
| self.GetLogger().warning(error_message) |
| return error_message |
| |
| def RunOneTest(self, test_bin_dir, test_name, remote_dir, |
| output_report_dir): |
| """Execute each test.""" |
| try: |
| prefix = '%s::' % (self._GetAvdName()) |
| host_path = os.path.join(test_bin_dir, test_name) |
| remote_path = os.path.join(remote_dir, test_name) |
| remote_report_file_name = '%s.xml' % test_name |
| output_report_file_name = '%s%s.xml' % (prefix, test_name) |
| if output_report_dir: |
| output_report_path = os.path.join(output_report_dir, |
| output_report_file_name) |
| else: |
| output_report_path = os.tmpnam() |
| |
| # Create default test report file. |
| CreateDefaultReportFile(output_report_path, test_name, prefix) |
| |
| # Copy the binary file, run the test, and clean up the executable. |
| self.CopyFile(host_path=host_path, |
| remote_path=remote_path, |
| operation='push') |
| remote_report_path = os.path.abspath( |
| os.path.join(remote_dir, remote_report_file_name)) |
| # We should append colored_log=false to suppress escape sequences on |
| # continuous build. |
| command = ['cd', remote_dir, ';', |
| './' + test_name, '--test_srcdir=.', '--logtostderr', |
| '--gunit_output=xml:%s' % remote_report_path, |
| '--colored_log=false'] |
| self._RunCommand(*command) |
| temporal_report_path = os.tmpnam() |
| error_message = self._CopyAndVerifyResult( |
| test_name, remote_report_path, temporal_report_path) |
| # Successfully the result file is pulled. |
| if not error_message: |
| # Append prefix to testsuite name. |
| # Otherwise duplicate testsuites name will be generated finally. |
| AppendPrefixToSuiteName(temporal_report_path, output_report_path, |
| prefix) |
| return error_message |
| finally: |
| if remote_path: |
| self._RunCommand('rm', remote_path) |
| if remote_report_path: |
| self._RunCommand('rm', remote_report_path) |
| |
| def SetUpTest(self, device, mount_point, remote_dir, |
| dictionary_data, connection_data, connection_text_data, |
| test_connection_data, test_connection_text_data, |
| mozc_data_dir): |
| """Set up the android to run tests.""" |
| self.WaitForMount() |
| # Now, the binary size of unittests are getting bigger and actually, |
| # some of them exceed the /data/data/... quota. |
| # So we'll use sdcard, instead. Unfortunately, sdcard has noexec |
| # attribute |
| # by default, so we remove it by remounting. |
| self._RunCommand('mount', '-o', 'remount', device, mount_point) |
| # Some tests depend on dictionary data. In the product, the |
| # data is set at jni loading time, but it is necessary to somehow |
| # set the data in native tests. So, copy the dictionary data to the |
| # emulator. |
| self.CopyFile(host_path=dictionary_data, |
| remote_path=os.path.join(remote_dir, |
| 'embedded_data', 'dictionary_data'), |
| operation='push') |
| self.CopyFile(host_path=connection_data, |
| remote_path=os.path.join(remote_dir, |
| 'embedded_data', 'connection_data'), |
| operation='push') |
| self.CopyFile(host_path=connection_text_data, |
| remote_path=os.path.join(remote_dir, |
| 'data_manager', 'android', |
| 'connection_single_column.txt'), |
| operation='push') |
| self.CopyFile(host_path=test_connection_data, |
| remote_path=os.path.join(remote_dir, |
| 'data_manager', 'testing', |
| 'connection_data.data'), |
| operation='push') |
| self.CopyFile(host_path=test_connection_text_data, |
| remote_path=os.path.join(remote_dir, |
| 'data_manager', 'testing', |
| 'connection_single_column.txt'), |
| operation='push') |
| # mozc_data_dir contains both generated .h files and test data. |
| # We want only test data and they are in mozc_data_dir/data. |
| # TODO(matsuzakit): Split generated .h files and test data |
| # into separate directories. |
| self.CopyFile(host_path=os.path.join(mozc_data_dir, 'data'), |
| remote_path=os.path.join(remote_dir, 'data'), |
| operation='push') |
| |
| def TearDownTest(self, remote_dir): |
| self._RunCommand('rm', '-r', remote_dir) |
| |
| def RunNativeTests(self, options, abi, binaries): |
| self.GetLogger().info('[NATIVE] Testing device=%s', self.serial) |
| if not self.IsAcceptableAbi(abi): |
| self.GetLogger().info('ABI %s is not compatible with built executables.', |
| abi) |
| return [] |
| if int(self.GetProperty('ro.build.version.sdk')) <= 15: |
| self.GetLogger().info('The device does not support PIE so skip testing.') |
| return [] |
| try: |
| error_messages = [] |
| self.SetUpTest(options.remote_device, options.remote_mount_point, |
| options.remote_dir, |
| options.mozc_dictionary_data_file, |
| options.mozc_connection_data_file, |
| options.mozc_connection_text_data_file, |
| options.mozc_test_connection_data_file, |
| options.mozc_test_connection_text_data_file, |
| options.mozc_data_dir) |
| if options.testcase: |
| test_list = options.testcase.split(',') |
| else: |
| test_list = None |
| self.GetLogger().info('[NATIVE] Testing device=%s, abi=%s', |
| self.serial, abi) |
| for test in binaries: |
| if test_list and test not in test_list: |
| continue |
| self.GetLogger().info('[NATIVE] Testing device=%s, abi=%s, test=%s', |
| self.serial, abi, test) |
| error_message = self.RunOneTest(options.test_bin_dir, test, |
| options.remote_dir, |
| options.output_report_dir) |
| if error_message: |
| error_messages.append(error_message) |
| self.TearDownTest(options.remote_dir) |
| return error_messages |
| except Exception as e: # pylint: disable=broad-except |
| # Catches all the exceptions and returns string instead. |
| # If an exception is thrown, multiprocessing module |
| # complains because exceptions might not be able to be pickled. |
| return [str(e)] |
| |
| def RunJavaTests(self, output_report_dir, configuration, |
| app_package_name, abi): |
| try: |
| if not self.IsAcceptableAbi(abi): |
| self.GetLogger().info( |
| 'ABI %s is not compatible with built executables.', abi) |
| return [] |
| self.GetLogger().info('[JAVA] Testing device=%s', self.serial) |
| prefix = '%s::' % self._GetAvdName() |
| report_file_name_remote = 'gtest-report.xml' |
| report_file_name_host = prefix + 'java_layer.xml' |
| output_report_path = os.path.join(output_report_dir, |
| report_file_name_host) |
| CreateDefaultReportFile(output_report_path, 'JUnit all tests', prefix) |
| # Run 'run-tests' target on specified device. |
| # This target does build (testee and tester), install (both) and run. |
| args = ['ant', 'run-tests', |
| '-Dadb.device.arg=-s %s' % self.serial, |
| '-Dgyp.build_type=%s' % configuration] |
| process = subprocess.Popen(args, cwd='tests') |
| process.wait() |
| if process.wait() != 0: |
| return '[FAIL] [JAVA] run_test target fails' |
| |
| remote_report_path = os.path.join('data', 'data', |
| app_package_name, |
| report_file_name_remote) |
| try: |
| temporal_report_path = os.tmpnam() |
| self.CopyFile(host_path=temporal_report_path, |
| remote_path=remote_report_path, |
| operation='pull') |
| self.GetLogger().info('[ OK ] Java result file is successfully pulled.') |
| # Append prefix to testsuite name. |
| # Otherwise duplicate testsuites name will be generated finally. |
| AppendPrefixToSuiteName(temporal_report_path, output_report_path, |
| prefix) |
| # XML file verification is omitted because |
| # - Java's report XML doesn't fit verifier's expectation. |
| except IOError: |
| self.GetLogger().warning('[FAIL] Java result file is not pulled.') |
| return '[FAIL] Result file does not exist. The process might crash.' |
| |
| # TODO(matsuzakit): Verify reporting XML file here. |
| # gtest_report.GetFromXMLFile() is not applicable becuase of |
| # format difference. |
| return None |
| except Exception as e: # pylint: disable=broad-except |
| # Catches all the exceptions and returns string instead. |
| # If an exception is thrown, multiprocessing module |
| # complains because exceptions might not be able to be pickled. |
| return str(e) |
| |
| @staticmethod |
| def GetDevices(android_home): |
| return [AndroidDevice(super_device.serial, android_home) |
| for super_device in |
| android_util.AndroidDevice.GetDevices(android_home)] |
| |
| |
| def CreateDefaultReportFile(output_report_path, test_name, prefix): |
| # Create default test report file. |
| # It will be used as the result of the test if the test crashes. |
| with open(output_report_path, 'w') as f: |
| f.write("""<?xml version="1.0" encoding="UTF-8"?> |
| <testsuite name="%s%s" tests="1" errors="1"> |
| <testcase name="No reporting XML"> |
| <error message="No reporting XML has been generated. Process crash?" /> |
| </testcase> |
| </testsuite>""" % (prefix, test_name)) |
| |
| |
| def RunTest(device, options, binaries): |
| error_messages = [] |
| if options.run_native_test: |
| native_error_messages = device.RunNativeTests(options, options.abi, |
| binaries) |
| if native_error_messages: |
| error_messages.extend(native_error_messages) |
| if options.run_java_test: |
| error_message = device.RunJavaTests(options.output_report_dir, |
| options.configuration, |
| options.app_package_name, |
| options.abi) |
| if error_message: |
| error_messages.append(error_message) |
| return '\n'.join(error_messages) |
| |
| |
| def main(): |
| # Enable logging.info. |
| logging.getLogger().setLevel(logging.INFO) |
| |
| options = ParseArgs() |
| |
| if not options.android_home: |
| logging.error('--android_home option must be specified.') |
| os.exit(1) |
| |
| if options.run_native_test: |
| binaries = FindTestBinaries(options.test_bin_dir) |
| logging.info(binaries) |
| else: |
| binaries = None |
| |
| # Prepare reporting directory. |
| if options.output_report_dir: |
| options.output_report_dir = os.path.abspath(options.output_report_dir) |
| try: |
| os.makedirs(options.output_report_dir) |
| logging.info('Made directory; %s', options.output_report_dir) |
| except OSError as e: |
| if e.errno == errno.EEXIST: |
| logging.info('%s has existed already.', options.output_report_dir) |
| else: |
| raise |
| |
| android_home = options.android_home |
| |
| devices = AndroidDevice.GetDevices(android_home) |
| logging.info('All the activated devices are %s', |
| [device.serial for device in devices]) |
| if options.android_devices: |
| using_devices = options.android_devices.split(',') |
| devices = [device for device in devices if device.serial in using_devices] |
| logging.info('Filtered to %s', |
| [device.serial for device in devices]) |
| |
| if not devices: |
| logging.info('No devices are specified. Do nothing.') |
| return |
| |
| # Maximum # of devices is 10 so running them in the same time is possible. |
| pool = multiprocessing.Pool(len(devices)) |
| results = [pool.apply_async(RunTest, [device, options, binaries]) |
| for device in devices] |
| pool.close() |
| # result.get() blocks until the test terminates. |
| error_messages = [result.get() for result in results if result.get()] |
| if error_messages: |
| print '[FAIL] Native tests result : Test failures are found;' |
| for message in error_messages: |
| print message |
| else: |
| print '[ OK ] Native tests result : Tests scceeded.' |
| |
| |
| if __name__ == '__main__': |
| main() |
