1#!/usr/bin/env python
2
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17'''This script will run one specific test.'''
18from __future__ import print_function, absolute_import
19
20import os
21import sys
22import atexit
23import inspect
24import logging
25import argparse
26import warnings
27
28import harness
29from harness import util_constants
30from harness import util_log
31from harness import util_warnings
32from harness.util_functions import load_py_module
33from harness.util_lldb import UtilLLDB
34from harness.exception import DisconnectedException
35from harness.exception import TestSuiteException, TestIgnoredException
36from harness.util_timer import Timer
37
38
39class TestState(object):
40    '''Simple mutable mapping (like namedtuple)'''
41    def __init__(self, **kwargs):
42        for key, val in kwargs.items():
43            setattr(self, key, val)
44
45
46def _test_pre_run(state):
47    '''This function is called before a test is executed (setup).
48
49    Args:
50        state: Test suite state collection, instance of TestState.
51
52    Returns:
53        True if the pre_run step completed without error. Currently the pre-run
54        will launch the target test binary on the device and attach an
55        lldb-server to it in platform mode.
56
57    Raises:
58        AssertionError: If an assertion fails.
59        TestSuiteException: Previous processes of this apk required for this
60                            test could not be killed.
61    '''
62    assert state.test
63    assert state.bundle
64
65    log = util_log.get_logger()
66    log.info('running: {0}'.format(state.name))
67
68    # Remove any cached NDK scripts between tests
69    state.bundle.delete_ndk_cache()
70
71    # query our test case for the remote target app it needs
72    # First try the legacy behaviour
73    try:
74        target_name = state.test.get_bundle_target()
75        warnings.warn("get_bundle_target() is deprecated and will be removed soon"
76                      " - use the `bundle_target` dictionary attribute instead")
77    except AttributeError:
78        try:
79            target_name = state.test.bundle_target[state.bundle_type]
80        except KeyError:
81            raise TestIgnoredException()
82
83    if target_name is None:
84        # test case doesn't require a remote process to debug
85        return True
86    else:
87        # find the pid of our remote test process
88        state.pid = state.bundle.launch(target_name)
89        if not state.pid:
90            log.error('unable to get pid of target')
91            return False
92        state.android.kill_servers()
93        # spawn lldb platform on the target device
94        state.android.launch_lldb_platform(state.device_port)
95        return True
96
97
98def _test_post_run(state):
99    '''This function is called after a test is executed (cleanup).
100
101    Args:
102        state: Test suite state collection, instance of TestState.
103
104    Raises:
105        AssertionError: If an assertion fails.
106    '''
107    assert state.test
108    assert state.bundle
109
110    try:
111        target_name = state.test.get_bundle_target()
112        warnings.warn("get_bundle_target() is deprecated and will be removed soon"
113                      " - use the `bundle_target` dictionary attribute instead")
114    except AttributeError:
115        try:
116            target_name = state.test.bundle_target[state.bundle_type]
117        except KeyError:
118            raise TestIgnoredException()
119
120
121    if target_name:
122        if state.bundle.is_apk(target_name):
123            state.android.stop_app(state.bundle.get_package(target_name))
124        else:
125            state.android.kill_process(target_name)
126
127
128def _test_run(state):
129    '''Execute a single test suite.
130
131    Args:
132        state: test suite state collection, instance of TestState.
133
134    Returns:
135        True: if the test case ran successfully and passed.
136        False: if the test case failed or suffered an error.
137
138    Raises:
139        AssertionError: If an assertion fails.
140    '''
141    assert state.lldb
142    assert state.lldb_module
143    assert state.test
144
145    test_failures = state.test.run(state.lldb, state.pid, state.lldb_module)
146
147    if test_failures:
148        log = util_log.get_logger()
149        for test, err in test_failures:
150            log.error('test %s:%s failed: %r' % (state.name, test, err))
151
152        return False
153
154    return True
155
156
157def _initialise_timer(android, interval):
158    '''Start a 'timeout' timer, to catch stalled execution.
159
160    This function will start a timer that will act as a timeout killing this
161    test session if a test becomes un-responsive.
162
163    Args:
164        android: current instance of harness.UtilAndroid
165        interval: the interval for the timeout, in seconds
166
167    Returns:
168        The instance of the Timer class that was created.
169    '''
170
171    def on_timeout():
172        '''This is a callback function that will fire if a test takes longer
173        then a threshold time to complete.'''
174        # Clean up the android properties
175        android.reset_all_props()
176        # pylint: disable=protected-access
177        sys.stdout.flush()
178        # hard exit to force kill all threads that may block our exit
179        os._exit(util_constants.RC_TEST_TIMEOUT)
180
181    timer = Timer(interval, on_timeout)
182    timer.start()
183    atexit.register(Timer.stop, timer)
184    return timer
185
186
187def _quit_test(num, timer):
188    '''This function will exit making sure the timeout thread is killed.
189
190    Args:
191        num: An integer specifying the exit status, 0 meaning "successful
192             termination".
193        timer: The current Timer instance.
194    '''
195    if timer:
196        timer.stop()
197    sys.stdout.flush()
198    sys.exit(num)
199
200
201def _execute_test(state):
202    '''Execute a test suite.
203
204    Args:
205        state: The current TestState object.
206    '''
207    log = util_log.get_logger()
208
209    state.test.setup(state.android)
210    try:
211        if not _test_pre_run(state):
212            raise TestSuiteException('test_pre_run() failed')
213        if not _test_run(state):
214            raise TestSuiteException('test_run() failed')
215        _test_post_run(state)
216        log.info('Test passed')
217
218    finally:
219        state.test.post_run()
220        state.test.teardown(state.android)
221
222
223def _get_test_case_class(module):
224    '''Inspect a test case module and return the test case class.
225
226    Args:
227        module: A loaded test case module.
228    '''
229    # We consider only subclasses of TestCase that have `test_` methods`
230    log = util_log.get_logger()
231    log.debug("loading test suites from %r", module)
232    for name, klass in inspect.getmembers(module, inspect.isclass):
233        for attr in dir(klass):
234            if attr.startswith('test_'):
235                log.info("Found test class %r", name)
236                return klass
237        else:
238            log.debug("class %r has no test_ methods", name)
239    return None
240
241
242def get_test_dir(test_name):
243    ''' Get the directory that contains a test with a given name.
244
245    Returns:
246        A string that is the directory containing the test.
247
248    Raises:
249        TestSuiteException: If a test with this name does not exist.
250    '''
251    tests_dir = os.path.dirname(os.path.realpath(__file__))
252    for sub_dir in os.listdir(tests_dir):
253        current_test_dir = os.path.join(tests_dir, sub_dir)
254        if (os.path.isdir(current_test_dir) and
255            test_name in os.listdir(current_test_dir)):
256            return current_test_dir
257
258    raise TestSuiteException(
259        'unable to find test: {0}'.format(test_name))
260
261
262def main():
263    '''Test runner entry point.'''
264
265    # re-open stdout with no buffering
266    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
267
268    android = None
269    timer = None
270    log = None
271
272    # parse the command line (positional arguments only)
273    truthy = lambda x: x.lower() in ('true', '1')
274    parser = argparse.ArgumentParser("Run a single RenderScript TestSuite against lldb")
275    for name, formatter in (
276       ('test_name', str),
277       ('log_file_path', str),
278       ('adb_path', str),
279       ('lldb_server_path_device', str),
280       ('aosp_product_path', str),
281       ('device_port', int),
282       ('device', str),
283       ('print_to_stdout', truthy),
284       ('verbose', truthy),
285       ('wimpy', truthy),
286       ('timeout', int),
287       ('bundle_type', str),
288    ):
289        parser.add_argument(name, type=formatter)
290
291    args = parser.parse_args()
292
293    try:
294        # create utility classes
295        harness.util_log.initialise(
296            '%s(%s)' % (args.test_name, args.bundle_type),
297            print_to_stdout=args.print_to_stdout,
298            level=logging.INFO if not args.verbose else logging.DEBUG,
299            file_path=args.log_file_path,
300            file_mode='a'
301        )
302        log = util_log.get_logger()
303        log.debug('Logger initialised')
304
305        android = harness.UtilAndroid(args.adb_path,
306                                      args.lldb_server_path_device,
307                                      args.device)
308
309        # start the timeout counter
310        timer = _initialise_timer(android, args.timeout)
311
312        # startup lldb and register teardown handler
313        atexit.register(UtilLLDB.stop)
314        UtilLLDB.start()
315
316        current_test_dir = get_test_dir(args.test_name)
317
318        # load a test case module
319        test_module = load_py_module(os.path.join(current_test_dir,
320                                                  args.test_name))
321
322
323        # inspect the test module and locate our test case class
324        test_class = _get_test_case_class(test_module)
325
326        # if our test inherits from TestBaseRemote, check we have a valid device
327        if (hasattr(test_module, "TestBaseRemote") and
328            issubclass(test_class, test_module.TestBaseRemote)):
329            android.validate_device()
330
331        # create an instance of our test case
332        test_inst = test_class(
333            args.device_port,
334            args.device,
335            timer,
336            args.bundle_type,
337            wimpy=args.wimpy
338        )
339
340        # instantiate a test target bundle
341        bundle = harness.UtilBundle(android, args.aosp_product_path)
342
343        # execute the test case
344        try:
345            for _ in range(2):
346                try:
347                    # create an lldb instance
348                    lldb = UtilLLDB.create_debugger()
349
350                    # create state object to encapsulate instances
351
352                    state = TestState(
353                         android=android,
354                         bundle=bundle,
355                         lldb=lldb,
356                         lldb_module=UtilLLDB.get_module(),
357                         test=test_inst,
358                         pid=None,
359                         name=args.test_name,
360                         device_port=args.device_port,
361                         bundle_type=args.bundle_type
362                    )
363
364                    util_warnings.redirect_warnings()
365
366                    _execute_test(state)
367
368                    # tear down the lldb instance
369                    UtilLLDB.destroy_debugger(lldb)
370                    break
371                except DisconnectedException as error:
372                    log.warning(error)
373                    log.warning('Trying again.')
374            else:
375                log.fatal('Not trying again, maximum retries exceeded.')
376                raise TestSuiteException('Lost connection to lldb-server')
377
378        finally:
379            util_warnings.restore_warnings()
380
381        _quit_test(util_constants.RC_TEST_OK, timer)
382
383    except AssertionError:
384        if log:
385            log.critical('Internal test suite error', exc_info=1)
386        print('Internal test suite error', file=sys.stderr)
387        _quit_test(util_constants.RC_TEST_FATAL, timer)
388
389    except TestIgnoredException:
390        if log:
391            log.warn("test ignored")
392        _quit_test(util_constants.RC_TEST_IGNORED, timer)
393
394    except TestSuiteException as error:
395        if log:
396            log.exception(str(error))
397        else:
398            print(error, file=sys.stderr)
399        _quit_test(util_constants.RC_TEST_FAIL, timer)
400
401    # use a global exception handler to be sure that we will
402    # exit safely and correctly
403    except Exception:
404        if log:
405            log.exception('INTERNAL ERROR')
406        else:
407            import traceback
408            print('Exception {0}'.format(traceback.format_exc()),
409                  file=sys.stderr)
410        _quit_test(util_constants.RC_TEST_FATAL, timer)
411
412    finally:
413        if android:
414            android.reset_all_props()
415        if timer:
416            timer.stop()
417
418
419# execution trampoline
420if __name__ == '__main__':
421    print(' '.join(sys.argv))
422    main()
423