1#!/usr/bin/env python
2# coding: utf-8
3#
4# Copyright (c) 2020-2021 Huawei Device Co., Ltd. All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without modification,
7# are permitted provided that the following conditions are met:
8#
9# 1. Redistributions of source code must retain the above copyright notice, this list of
10#    conditions and the following disclaimer.
11#
12# 2. Redistributions in binary form must reproduce the above copyright notice, this list
13#    of conditions and the following disclaimer in the documentation and/or other materials
14#    provided with the distribution.
15#
16# 3. Neither the name of the copyright holder nor the names of its contributors may be used
17#    to endorse or promote products derived from this software without specific prior written
18#    permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
24# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
29# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
30# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32import os
33import platform
34import shutil
35import subprocess
36import sys
37import time
38import base64
39
40
41class TestConfig(object):
42    CMAKE_GEN_PATH = "cmake-build-debug"
43    WORK_DIR = ""
44    HCGEN = ""
45    TEMP_DIR = 'temp'
46    ERROR_COLOR_PREFIX = "\033[31m"
47    ERROR_COLOR_END = "\033[0m"
48    SHOULD_CLEAN_TEMP = True
49    PERFORMANCE_MAX_COMPILE_TIME_MS = 1000
50
51
52def text_file_compare(file_a, file_target):
53    if not os.path.exists(file_target):
54        return True
55
56    with open(file_a, 'r') as f_a:
57        with open(file_target, 'r') as f_b:
58            a_content = f_a.read().replace(r'\r\n', r'\n')
59            b_content = f_b.read().replace(r'\r\n', r'\n')
60            return a_content == b_content
61
62
63def binary_file_compare(file_a, file_target, skip_size=0,
64                        target_base64_encode=False):
65    if not os.path.exists(file_target):
66        return True
67
68    with open(file_a, 'rb') as f_a:
69        with open(file_target, 'rb') as f_b:
70            a_content = f_a.read()
71            b_content = f_b.read()
72            if target_base64_encode:
73                b_content = base64.b64decode(b_content)
74            return a_content[skip_size:] == b_content[skip_size:]
75
76
77def exec_command(command):
78    return subprocess.getstatusoutput(command)
79
80
81def setup_hcgen_compiler():
82    if len(sys.argv) > 1:
83        hcgen_path = os.path.abspath(sys.argv[1])
84        if hcgen_path.find('hc-gen') >= 0 and os.access(hcgen_path, os.X_OK):
85            TestConfig.HCGEN = hcgen_path
86            print('use specified hsc:' + hcgen_path)
87            return
88
89    source_root = '../../'
90    compiler_name = "hc-gen"
91    if platform.system() == "Windows":
92        source_root = source_root.replace("/", "\\")
93        compiler_name += ".exe"
94
95    source_root = os.path.abspath(os.path.join(TestConfig.WORK_DIR, source_root))
96    hcgen = os.path.join(source_root, compiler_name)
97    if not os.access(hcgen, os.X_OK):
98        hcgen = os.path.join(source_root, TestConfig.CMAKE_GEN_PATH, compiler_name)
99        if not os.access(hcgen, os.X_OK):
100            print("Error: hcgen not found, please make first")
101            exit(1)
102    TestConfig.HCGEN = hcgen
103
104
105def index_case(case_path):
106    cases = []
107    for dir_name in os.listdir(case_path):
108        if os.path.isdir(os.path.join(case_path, dir_name)):
109            cases.append(dir_name)
110    cases.sort()
111    return cases
112
113
114def get_golden_compile_result(mode, case_name):
115    result_file_name = os.path.join(TestConfig.WORK_DIR, case_name,
116                                    'golden_%s_compile_result.txt' % mode)
117    status_prefix = '[compile exit status]:'
118    output_prefix = '[compile console output]:\n'
119    with open(result_file_name, 'r') as result_file:
120        status = result_file.readline()
121        status = status[len(status_prefix):]
122        console_output = result_file.read()
123        console_output = console_output[len(output_prefix):]
124
125        return int(status), console_output.strip()
126
127
128def compile_status_to_str(status):
129    if status:
130        return 'success'
131    else:
132        return 'failed'
133
134
135def test_compile(case_name, mode):
136    output_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR, case_name)
137    if not os.path.exists(output_dir):
138        os.makedirs(output_dir)
139    output_file = os.path.join(output_dir, 'golden')
140    source_file = os.path.join(TestConfig.WORK_DIR, case_name, 'case.hcs')
141    temp_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR)
142
143    if mode == 'text':
144        command = "%s -o %s -t  %s" % (TestConfig.HCGEN, output_file, source_file)
145    else:
146        command = "%s -o %s %s" % (TestConfig.HCGEN, output_file, source_file)
147
148    status, output = exec_command(command)
149    golden_status, golden_output = get_golden_compile_result(mode, case_name)
150    if bool(status) != bool(golden_status):
151        print("%s mode: case %s expect compile %s but %s" %
152              (mode, case_name, compile_status_to_str(status),
153               compile_status_to_str(golden_status)))
154        print("Console output :\n" + output)
155        return False
156
157    output = output.replace(temp_dir, ".").replace(TestConfig.WORK_DIR, "."). \
158        replace('\\', '/').replace(TestConfig.ERROR_COLOR_PREFIX, ""). \
159        replace(TestConfig.ERROR_COLOR_END, "")
160    if output.strip() != golden_output:
161        print("output is different with golden for %s compile:" % mode)
162        print("EXPECT:\n" + golden_output)
163        print("ACTUAL:\n" + output.strip())
164        return False
165
166    return True
167
168
169def binary_code_compile(case_name):
170    compile_result = test_compile(case_name, 'binary')
171    if not compile_result:
172        return False
173
174    compile_start_time = get_current_time_ms()
175
176    case_hcb = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR, case_name, 'golden.hcb')
177    golden_hcb = os.path.join(TestConfig.WORK_DIR, case_name, 'golden.hcb')
178    hcb_header_size = 20  # hcb compare skip hcb header
179    output_compare = \
180        binary_file_compare(case_hcb, golden_hcb, hcb_header_size, True)
181    if not output_compare:
182        print('Error: hcb output mismatch with golden')
183        return False
184
185    compile_finish_time = get_current_time_ms()
186    compile_used_time = compile_finish_time - compile_start_time
187    if compile_used_time > TestConfig.PERFORMANCE_MAX_COMPILE_TIME_MS:
188        print('Error: compile time %d, out of threshold %d ms'
189              % (compile_used_time, TestConfig.PERFORMANCE_MAX_COMPILE_TIME_MS))
190        return False
191
192    decompile_result = test_decompile(case_name)
193
194    return decompile_result
195
196
197def test_text_code_compile(case_name):
198    compile_result = test_compile(case_name, 'text')
199    if not compile_result:
200        return False
201
202    case_c_file = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR, case_name, 'golden.c')
203    golden_c_file = os.path.join(TestConfig.WORK_DIR, case_name, 'golden.c.gen')
204    c_file_compare = text_file_compare(case_c_file, golden_c_file)
205    if not c_file_compare:
206        print("Error: The generated C file mismatch with golden")
207
208    case_header_file = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR, case_name, 'golden.h')
209    golden_header_file = os.path.join(TestConfig.WORK_DIR, case_name, 'golden.h.gen')
210    header_file_compare = \
211        text_file_compare(case_header_file, golden_header_file)
212    if not header_file_compare:
213        print("Error: The generated header file mismatch with golden")
214    return c_file_compare and header_file_compare
215
216
217def test_decompile(case_name):
218    golden_decompile_file_name = \
219        os.path.join(TestConfig.WORK_DIR, case_name, 'golden.d.hcs')
220    if not os.path.exists(golden_decompile_file_name):
221        return True
222
223    output_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR, case_name)
224    output_file = os.path.join(output_dir, 'case.hcs')
225    source_file = os.path.join(output_dir, 'golden.hcb')
226    command = "%s -o %s -d %s" % (TestConfig.HCGEN, output_file, source_file)
227
228    status, output = exec_command(command)
229    if status != 0:
230        print('decompile fail')
231        print(output)
232        return False
233
234    decompile_golden_result = text_file_compare(
235        os.path.join(output_dir, 'case.d.hcs'), golden_decompile_file_name)
236    if not decompile_golden_result:
237        print('Error: case %s decompile hcs mismatch with golden' % case_name)
238        return False
239
240    return True
241
242
243def get_current_time_ms():
244    return int(round(time.time() * 1000))
245
246
247def test_cases(cases):
248    print('[==========] running %d cases form hcgen test' % len(cases))
249    failed_cases = []
250    test_start_time = get_current_time_ms()
251    for case in cases:
252        case_start_time = get_current_time_ms()
253        print('[ RUN      ] %s' % case)
254        binary_compile_result = binary_code_compile(case)
255        text_compile_result = test_text_code_compile(case)
256        case_finish_time = get_current_time_ms()
257        used_time_str = ' (%d ms)' % (case_finish_time - case_start_time)
258        if (not binary_compile_result) or (not text_compile_result):
259            print('[    ERROR ] %s%s' % (case, used_time_str))
260            failed_cases.append(case)
261        else:
262            print('[       OK ] %s%s' % (case, used_time_str))
263    test_finish_time = get_current_time_ms()
264    print('\n[==========] running %d case (%d ms)'
265          % (len(cases), test_finish_time - test_start_time))
266    print('[  PASSED  ] %d cases' % (len(cases) - len(failed_cases)))
267    if len(failed_cases) > 0:
268        TestConfig.SHOULD_CLEAN_TEMP = False
269        print('[  FAILED  ] %d cases, list below:' % len(failed_cases))
270        for case in failed_cases:
271            print('[  FAILED  ] %s' % case)
272
273
274def setup_work_dir():
275    pwd = os.path.abspath(sys.argv[0])
276    pwd = pwd[:pwd.rfind(os.sep)]
277    TestConfig.WORK_DIR = pwd
278
279
280def test_setup():
281    temp_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR)
282    if not os.path.exists(temp_dir):
283        os.mkdir(temp_dir)
284
285
286def test_teardown():
287    if not TestConfig.SHOULD_CLEAN_TEMP:
288        return
289    temp_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR)
290    if os.path.exists(temp_dir):
291        shutil.rmtree(temp_dir)
292
293
294def clean_up():
295    temp_dir = os.path.join(TestConfig.WORK_DIR, TestConfig.TEMP_DIR)
296    if os.path.exists(temp_dir):
297        shutil.rmtree(temp_dir)
298
299
300if __name__ == "__main__":
301    setup_work_dir()
302    clean_up()
303    setup_hcgen_compiler()
304    print("hcgen path : " + TestConfig.HCGEN)
305    cases_list = index_case(TestConfig.WORK_DIR)
306    test_setup()
307    test_cases(cases_list)
308    test_teardown()
309