1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2023 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19import time 20import os 21import sys 22import platform 23import pty 24import threading 25import re 26import traceback 27import select 28import subprocess 29import queue 30import shutil 31 32from collections import defaultdict 33 34sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "mylogger.py")) 35from mylogger import get_logger, parse_json 36 37Log = get_logger("performance") 38 39log_info = Log.info 40log_error = Log.error 41script_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 42 43config = parse_json() 44if not config: 45 log_error("config file: build_example.json not exist") 46 raise FileNotFoundError("config file: build_example.json not exist") 47 48 49class PerformanceAnalyse: 50 51 self.html_tamplate = """ 52 <!DOCTYPE html> 53 <html lang="en"> 54 <head> 55 <style type="text/css" media="screen"> 56 table {{ 57 border-collapse: collapse; 58 width: 80%; 59 max-width: 1200px; 60 margin-bottom: 30px; 61 margin-left: auto; 62 margin-right: auto; 63 table-layout: fixed; 64 }} 65 th, td {{ 66 padding: 10px; 67 text-align: center; 68 font-size: 12px; 69 border: 1px solid #ddd; 70 word-wrap: break-word; 71 }} 72 th {{ 73 background-color: #f2f2f2; 74 font-weight: bold; 75 text-transform: capitalize; 76 }} 77 tr:nth-child(even) {{ 78 background-color: #f9f9f9; 79 }} 80 caption {{ 81 font-size: 24px; 82 margin-bottom: 16px; 83 color: #333; 84 text-transform: uppercase; 85 letter-spacing: 2px; 86 font-family: Arial, sans-serif; 87 text-align: center; 88 text-transform: capitalize; 89 }} 90 .container {{ 91 width: 80%; 92 margin: 0 auto; 93 }} 94 body {{ font-family: Microsoft YaHei,Tahoma,arial,helvetica,sans-serif;padding: 20px;}} 95 h1 {{ text-align: center; }} 96 </style> 97 </head> 98 <body> 99 <div class="container"> 100 <h1>{}</h1> 101 """ 102 103 try: 104 TIMEOUT = int(config.get("performance").get("performance_exec_timeout")) 105 select_timeout = float(config.get("performance").get("performance_select_timeout")) 106 top_count = int(config.get("performance").get("performance_top_count")) 107 overflow = float(config.get("performance").get("performance_overflow")) 108 exclude = config.get("performance").get("exclude") 109 log_info("TIMEOUT:{}".format(TIMEOUT)) 110 log_info("select_timeout:{}".format(select_timeout)) 111 log_info("top_count:{}".format(top_count)) 112 log_info("overflow:{} sec".format(overflow)) 113 except Exception as e: 114 log_error("config file:build_example.json has error:{}".format(e)) 115 raise FileNotFoundError("config file:build_example.json has error:{}".format(e)) 116 117 def __init__(self, performance_cmd, output_path, report_titles, ptyflags=False): 118 self.performance_cmd = script_path + performance_cmd 119 self.output_path = script_path + output_path 120 self.report_title = report_titles 121 self.ptyflag = ptyflags 122 self.out_queue = queue.Queue() 123 self.system_info = list() 124 self.ninjia_trace_list = list() 125 self.gn_exec_li = list() 126 self.gn_script_li = list() 127 self.gn_end_li = list() 128 self.ccache_li = list() 129 self.c_targets_li = list() 130 self.root_dir = None 131 self.gn_dir = None 132 self.gn_script_res = None 133 self.gn_exec_res = None 134 self.cost_time_res = list() 135 self.gn_exec_flag = re.compile(r"File execute times") 136 self.gn_script_flag = re.compile(r"Script execute times") 137 self.gn_end_flag = re.compile(r"Done\. Made \d+ targets from \d+ files in (\d+)ms") 138 self.root_dir_flag = re.compile(r"""loader args.*source_root_dir="([a-zA-Z\d/\\_]+)""""") 139 self.gn_dir_flag = re.compile(r"""loader args.*gn_root_out_dir="([a-zA-Z\d/\\_]+)""""") 140 self.ccache_start_flag = re.compile(r"ccache_dir =") 141 self.ccache_end_flag = re.compile(r"c targets overlap rate statistics") 142 self.c_targets_flag = re.compile(r"c overall build overlap rate") 143 self.build_error = re.compile(r"=====build\s\serror=====") 144 self.ohos_error = re.compile(r"OHOS ERROR") 145 self.total_flag = re.compile(r"Cost time:.*(\d+:\d+:\d+)") 146 self.total_cost_time = None 147 self.error_message = list() 148 self.during_time_dic = { 149 "Preloader": {"start_pattern": re.compile(r"Set cache size"), 150 "end_pattern": re.compile(r"generated compile_standard_whitelist"), 151 "start_time": 0, 152 "end_time": 0 153 }, 154 "Loader": {"start_pattern": re.compile(r"Checking all build args"), 155 "end_pattern": re.compile(r"generate target syscap"), 156 "start_time": 0, 157 "end_time": 0 158 }, 159 "Ninjia": {"start_pattern": re.compile(r"Done\. Made \d+ targets from \d+ files in (\d+)ms"), 160 "end_pattern": re.compile(r"ccache_dir ="), 161 "start_time": 0, 162 "end_time": 0 163 }} 164 self.table_html = "" 165 self.base_html = self.html_tamplate.format(self.report_title) 166 self.remove_out() 167 168 @staticmethod 169 def generate_error_content(table_name, lines): 170 """ 171 Description: generate error html content 172 @parameter table_name: table name 173 @parameter lines: error message 174 """ 175 table_title = table_name.capitalize() 176 lines = ['<br>' + text for text in lines] 177 html_content = '<center><h1>{}</h1><div style="text-align:left;">{}<div></center>'.format(table_title, 178 '\n'.join(lines)) 179 error_html = '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>Ohos Error</title></head><body>{}</body></html>'.format( 180 html_content) 181 return error_html 182 183 def remove_out(self): 184 """ 185 Description: remove out dir 186 """ 187 out_dir = os.path.join(script_path, "out") 188 try: 189 if not os.path.exists(out_dir): 190 return 191 for tmp_dir in os.listdir(out_dir): 192 if tmp_dir in self.exclude: 193 continue 194 if os.path.isdir(os.path.join(out_dir, tmp_dir)): 195 shutil.rmtree(os.path.join(out_dir, tmp_dir)) 196 else: 197 os.remove(os.path.join(out_dir, tmp_dir)) 198 except Exception as e: 199 log_error(e) 200 201 def write_html(self, content): 202 """ 203 Description: convert html str 204 @parameter content: html str 205 """ 206 if not os.path.exists(os.path.dirname(self.output_path)): 207 os.makedirs(os.path.dirname(self.output_path), exist_ok=True) 208 with os.fdopen(os.open(self.output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, os.stat.S_IWUSR), "w", encoding="utf-8") as html_file: 209 html_file.write(content) 210 211 def generate_content(self, table_name, data_rows, switch=False): 212 """ 213 Description: generate html content 214 @parameter table_name: table name 215 @parameter data_rows: two-dimensional array data 216 @parameter switch: change overflow data color 217 """ 218 table_title = table_name.capitalize() 219 if not data_rows[1:]: 220 log_error("【{}】 is None") 221 return False 222 tb_html = """ 223 <table style="width: 100%; max-width: 1200px;"> 224 <caption>{0}</caption> 225 <colgroup> 226 <col style="width: {1}%"/> 227 </colgroup> 228 <thead> 229 <tr class="text-center success" style="font-weight: bold;font-size: 14px;"> 230 """.format(table_title, int(100 / len(data_rows[0]))) 231 232 self.table_html += tb_html 233 234 for header in data_rows[0]: 235 self.table_html += "<th>{}</th>".format(header.capitalize()) 236 self.table_html += "</tr></thead>" 237 238 self.table_html += "<tbody>" 239 if switch: 240 for index, row in enumerate(data_rows[1:]): 241 if float(row[-1]) > float(self.overflow): 242 self.table_html += "<tr style='background-color: #ff7f50;'>" 243 elif float(row[-1]) <= float(self.overflow) and index % 2 == 0: 244 self.table_html += "<tr style='background-color: #f5f5f5;'>" 245 else: 246 self.table_html += "<tr>" 247 for data in row: 248 self.table_html += "<td>{}</td>".format(data) 249 self.table_html += "</tr>" 250 else: 251 for index, row in enumerate(data_rows[1:]): 252 if index % 2 == 0: 253 self.table_html += "<tr style='background-color: #f5f5f5;'>" 254 else: 255 self.table_html += "<tr>" 256 for data in row: 257 self.table_html += "<td>{}</td>".format(data) 258 self.table_html += "</tr>" 259 self.table_html += "</tbody>" 260 261 self.table_html += "</table></div></body></html>" 262 return True 263 264 def read_ninjia_trace_file(self): 265 """ 266 Description: read ninjia trace file 267 """ 268 try: 269 ninja_trace_path = os.path.join(self.root_dir, self.gn_dir, "sorted_action_duration.txt") 270 with open(ninja_trace_path, 'r') as f: 271 for line in f: 272 yield line.strip() 273 except Exception as e: 274 log_error("open ninjia trace file error is:{}".format(e)) 275 276 def process_ninja_trace(self): 277 """ 278 Description: generate ninja trace table data 279 """ 280 data = defaultdict(list) 281 result_list = list() 282 283 for line in self.read_ninjia_trace_file(): 284 line_list = line.split(":") 285 name = line_list[0].strip() 286 if name == "total time": 287 continue 288 duration = int(line_list[1].strip()) 289 data[name].append(duration) 290 291 for key, value in data.items(): 292 result = [key, len(value), max(value)] 293 result_list.append(result) 294 sort_result = sorted(result_list, key=lambda x: x[2], reverse=True) 295 for i in sort_result: 296 i[2] = round(float(i[2]) / 1000, 4) 297 298 self.ninjia_trace_list = sort_result[:self.top_count] 299 300 self.ninjia_trace_list.insert(0, ["Ninjia Trace File", "Call Count", "Ninjia Trace Cost Time(s)"]) 301 302 def process_gn_trace(self): 303 """ 304 Description: generate gn trace table data 305 """ 306 self.gn_exec_res = [[item[2], item[1], round(float(item[0]) / 1000, 4)] for item in self.gn_exec_li if 307 item and re.match(r"[\d.]+", item[0])][ 308 :self.top_count] 309 self.gn_script_res = [[item[2], item[1], round(float(item[0]) / 1000, 4)] for item in self.gn_script_li if 310 item and re.match(r"[\d.]+", item[0])][:self.top_count] 311 312 self.gn_exec_res.insert(0, ["Gn Trace Exec File", "Call Count", "GN Trace Exec Cost Time(s)"]) 313 314 self.gn_script_res.insert(0, ["Gn Trace Script File", "Call Count", "GN Trace Script Cost Time(s)"]) 315 316 def process_ccache_ctargets(self): 317 """ 318 Description: generate gn trace table data 319 """ 320 ccache_res = [] 321 c_targets_res = [] 322 for tmp in self.ccache_li: 323 if ":" in tmp and len(tmp.split(":")) == 2: 324 ccache_res.append(tmp.split(":")) 325 ccache_res.insert(0, ["ccache item", "data"]) 326 327 for item_ in self.c_targets_li: 328 if len(item_.split()) == 6: 329 c_targets_res.append(item_.split()) 330 c_targets_res.insert(0, ["subsystem", "files NO.", " percentage", "builds NO.", "percentage", "verlap rate"]) 331 return ccache_res, c_targets_res 332 333 def process_system(self): 334 """ 335 Description: generate system data 336 """ 337 start_li = [ 338 ["System Information name", "System Value"], 339 ['Python Version', sys.version], 340 ['Cpu Count', os.cpu_count()], 341 ["System Info", platform.platform()] 342 ] 343 344 self.system_info.extend(start_li) 345 try: 346 disk_info = os.statvfs('/') 347 total_disk = round(float(disk_info.f_frsize * disk_info.f_blocks) / (1024 ** 3), 4) 348 349 self.system_info.append(["Disk Size", "{} GB".format(total_disk)]) 350 with open('/proc/meminfo', 'r') as f: 351 lines = f.readlines() 352 total_memory_line = [line for line in lines if line.startswith('MemTotal')] 353 total_memory = round(float(total_memory_line[0].split()[1]) / (1024 ** 2), 4) if total_memory_line else " " 354 355 self.system_info.append(["Total Memory", "{} GB".format(total_memory)]) 356 except Exception as e: 357 log_error(e) 358 359 def process_cost_time(self): 360 """ 361 Description: generate summary table data 362 """ 363 for i in self.during_time_dic.keys(): 364 cost_time = (self.during_time_dic.get(i).get("end_time") - self.during_time_dic.get(i).get( 365 "start_time")) / 10 ** 9 366 new_cost_time = round(float(cost_time), 4) 367 self.cost_time_res.append([i, new_cost_time]) 368 gn_res = re.search(self.gn_end_flag, self.gn_end_li[0]) 369 if gn_res: 370 gn_time = round(float(gn_res.group(1)) / 1000, 4) 371 self.cost_time_res.append(['GN', gn_time]) 372 self.cost_time_res.append(["Total", self.total_cost_time]) 373 self.cost_time_res.insert(0, ["Compile Process Phase", "Cost Time(s)"]) 374 375 def producer(self, execute_cmd, out_queue, timeout=TIMEOUT): 376 """ 377 Description: execute cmd and put cmd result data to queue 378 @parameter execute_cmd: execute cmd 379 @parameter out_queue: save out data 380 @parameter timeout: execute cmd time out 381 @return returncode: returncode 382 """ 383 log_info("exec cmd is :{}".format(" ".join(execute_cmd))) 384 log_info("ptyflag is :{}".format(self.ptyflag)) 385 386 if self.ptyflag: 387 try: 388 master, slave = pty.openpty() 389 proc = subprocess.Popen( 390 execute_cmd, 391 stdin=slave, 392 stdout=slave, 393 stderr=slave, 394 encoding="utf-8", 395 universal_newlines=True, 396 errors='ignore', 397 cwd=script_path 398 399 ) 400 start_time = time.time() 401 incomplete_line = "" 402 while True: 403 if timeout and time.time() - start_time > timeout: 404 raise Exception("exec cmd time out,select") 405 ready_to_read, _, _ = select.select([master, ], [], [], PerformanceAnalyse.select_timeout) 406 for stream in ready_to_read: 407 output_bytes = os.read(stream, 1024) 408 output = output_bytes.decode('utf-8') 409 lines = (incomplete_line + output).split("\n") 410 for line in lines[:-1]: 411 line = line.strip() 412 if line: 413 out_str = "{}".format(time.time_ns()) + "[timestamp]" + line 414 out_queue.put(out_str) 415 incomplete_line = lines[-1] 416 if proc.poll() is not None: 417 out_queue.put(None) 418 break 419 returncode = proc.wait() 420 return returncode 421 except Exception as e: 422 out_queue.put(None) 423 log_error("Producer An error occurred:{}".format(e)) 424 err_str = traceback.format_exc() 425 log_error(err_str) 426 raise e 427 else: 428 try: 429 start_time = time.time() 430 proc = subprocess.Popen( 431 execute_cmd, 432 stdout=subprocess.PIPE, 433 stderr=subprocess.PIPE, 434 encoding="utf-8", 435 universal_newlines=True, 436 errors='ignore', 437 cwd=script_path 438 ) 439 440 while True: 441 if timeout and time.time() - start_time > timeout: 442 raise TimeoutError("exec cmd timeout") 443 ready_to_read, _, _ = select.select([proc.stdout, proc.stderr], [], [], 444 PerformanceAnalyse.select_timeout) 445 for stream in ready_to_read: 446 output = stream.readline().strip() 447 if output: 448 out_str = "{}".format(time.time_ns()) + "[timestamp]" + output 449 out_queue.put(out_str) 450 if proc.poll() is not None: 451 out_queue.put(None) 452 break 453 returncode = proc.wait() 454 return returncode 455 except Exception as e: 456 out_queue.put(None) 457 log_error("Producer An error occurred:{}".format(e)) 458 err_str = traceback.format_exc() 459 log_error(err_str) 460 raise e 461 462 def consumer(self, out_queue, timeout=TIMEOUT): 463 """ 464 Description: get cmd result data from queue 465 @parameter out_queue: save out data 466 @parameter timeout: execute cmd time out 467 """ 468 start_time = time.time() 469 try: 470 line_count = 0 471 gn_exec_start, gn_script, gn_end, ccache_start, ccache_end, c_tagart_end = None, None, None, None, None, None 472 while True: 473 if timeout and time.time() - start_time > timeout: 474 raise TimeoutError("consumer timeout") 475 476 output = out_queue.get() 477 if output is None: 478 log_info(".....................exec end...........................") 479 break 480 line_count += 1 481 log_info(output.split("[timestamp]")[1]) 482 line_mes = " ".join(output.split("[timestamp]")[1].split()[2:]) 483 time_stamp = output.split("[timestamp]")[0] 484 485 if re.search(self.root_dir_flag, output): 486 self.root_dir = re.search(self.root_dir_flag, output).group(1) 487 488 if re.search(self.gn_dir_flag, output): 489 self.gn_dir = re.search(self.gn_dir_flag, output).group(1) 490 491 for key, value in self.during_time_dic.items(): 492 if re.search(value.get("start_pattern"), output): 493 self.during_time_dic.get(key)["start_time"] = int(time_stamp) 494 if re.search(value.get("end_pattern"), output): 495 self.during_time_dic.get(key)["end_time"] = int(time_stamp) 496 497 if re.search(self.gn_exec_flag, output): 498 gn_exec_start = line_count 499 elif re.search(self.gn_script_flag, output): 500 gn_script = line_count 501 elif re.search(self.gn_end_flag, output): 502 gn_end = line_count 503 self.gn_end_li.append(line_mes) 504 elif re.search(self.ccache_start_flag, output): 505 ccache_start = line_count 506 elif re.search(self.ccache_end_flag, output): 507 ccache_end = line_count 508 elif re.search(self.c_targets_flag, output): 509 c_tagart_end = line_count 510 511 if gn_exec_start and line_count > gn_exec_start and not gn_script: 512 self.gn_exec_li.append(line_mes.split()) 513 elif gn_script and line_count > gn_script and not gn_end: 514 self.gn_script_li.append(line_mes.split()) 515 elif ccache_start and line_count > ccache_start and not ccache_end: 516 self.ccache_li.append(line_mes) 517 elif ccache_end and line_count > ccache_end and not c_tagart_end: 518 self.c_targets_li.append(line_mes) 519 520 if re.search(self.ohos_error, output) or re.search(self.build_error, output): 521 self.error_message.append( 522 re.sub(r"\x1b\[[0-9;]*m", "", output.split("[timestamp]")[1].strip())) 523 524 if re.search(self.total_flag, output): 525 total_time_str = re.search(self.total_flag, output).group(1) 526 time_obj = time.strptime(total_time_str, "%H:%M:%S") 527 milliseconds = (time_obj.tm_hour * 3600 + time_obj.tm_min * 60 + time_obj.tm_sec) 528 self.total_cost_time = milliseconds 529 530 except Exception as e: 531 log_error("Consumer An error occurred:{}".format(e)) 532 err_str = traceback.format_exc() 533 log_error(err_str) 534 raise e 535 536 def exec_command_pipe(self, execute_cmd): 537 """ 538 Description: start producer and consumer 539 @parameter execute_cmd: execute cmd 540 """ 541 try: 542 producer_thread = threading.Thread(target=self.producer, args=(execute_cmd, self.out_queue)) 543 consumer_thread = threading.Thread(target=self.consumer, args=(self.out_queue,)) 544 producer_thread.daemon = True 545 consumer_thread.daemon = True 546 producer_thread.start() 547 consumer_thread.start() 548 producer_thread.join() 549 consumer_thread.join() 550 except Exception as e: 551 err_str = traceback.format_exc() 552 log_error(err_str) 553 raise Exception(e) 554 555 def process(self): 556 """ 557 Description: start performance test 558 """ 559 try: 560 cmds = self.performance_cmd.split(" ") 561 self.exec_command_pipe(cmds) 562 if self.error_message: 563 err_html = self.generate_error_content("Ohos Error", self.error_message) 564 self.write_html(err_html) 565 return 566 567 self.process_system() 568 self.process_cost_time() 569 self.process_gn_trace() 570 self.process_ninja_trace() 571 ccache_res, c_targets_res = self.process_ccache_ctargets() 572 573 log_info(self.cost_time_res) 574 log_info(self.gn_exec_res) 575 log_info(self.gn_script_res) 576 log_info(self.ninjia_trace_list) 577 log_info(ccache_res) 578 log_info(c_targets_res) 579 self.generate_content("System Information", self.system_info) 580 self.generate_content("Compile Process Summary", self.cost_time_res) 581 self.generate_content("Gn Trace Exec File", self.gn_exec_res, switch=True) 582 self.generate_content("Gn Trace Script File ", self.gn_script_res, switch=True) 583 self.generate_content("Ninjia Trace File", self.ninjia_trace_list, switch=True) 584 self.generate_content("Ccache Data Statistics", ccache_res) 585 self.generate_content("C Targets Overlap Rate Statistics", c_targets_res) 586 res_html = self.base_html + self.table_html 587 self.write_html(res_html) 588 except Exception as e: 589 err_str = traceback.format_exc() 590 log_error(err_str) 591 err_html = self.generate_error_content("Performance system Error", err_str.split("\n")) 592 self.write_html(err_html) 593 594 595if __name__ == '__main__': 596 performance_script_data = config.get("performance").get("performance_script_data") 597 for item in performance_script_data: 598 cmd = item.get("performance_cmd") 599 path = item.get("output_path") 600 report_title = item.get("report_title") 601 ptyflag = True if item.get("ptyflag").lower() == "true" else False 602 if all([cmd, path, report_title]): 603 performance = PerformanceAnalyse(cmd, path, report_title, ptyflag) 604 performance.process() 605 606