1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2024 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import re 19import copy 20import optparse 21import struct 22import os 23import stat 24 25import parse_functions 26 27TRACE_REGEX_ASYNC = "\s*(\d+)\s+(.*?)\|\d+\|[SFC]\s+:(.*?)\s+:(.*?)\s+(.*?)\s+\]\d+\[\s+\)(\d+)\s*\(\s+(\d+?)-(.*?)\s+" 28TRACE_REGEX_SYNC = "\s*\|\d+\|E\s+:(.*?)\s+:(.*?)\s+(.*?)\s+\]\d+\[\s+\)(\d+)\s*\(\s+(\d+?)-(.*?)\s+" 29text_file = "" 30binary_file = "" 31out_file = "" 32 33CONTENT_TYPE_DEFAULT = 0 34CONTENT_TYPE_EVENTS_FORMAT = 1 35CONTENT_TYPE_CMDLINES = 2 36CONTENT_TYPE_TGIDS = 3 37CONTENT_TYPE_CPU_RAW = 4 38CONTENT_TYPE_HEADER_PAGE = 30 39CONTENT_TYPE_PRINTK_FORMATS = 31 40CONTENT_TYPE_KALLSYMS = 32 41 42INT8_DATA_READ_LEN = 1 43INT16_DATA_READ_LEN = 2 44INT32_DATA_READ_LEN = 4 45INT64_DATA_READ_LEN = 8 46 47READ_PAGE_SIZE = 4096 48 49RB_MISSED_FLAGS = (0x1 << 31) | (1 << 30) 50 51BUFFER_TYPE_PADDING = 29 52BUFFER_TYPE_TIME_EXTEND = 30 53BUFFER_TYPE_TIME_STAMP = 31 54 55events_format = {} 56cmd_lines = {} 57tgids = {} 58 59 60TRACE_TXT_HEADER_FORMAT = """# tracer: nop 61# 62# entries-in-buffer/entries-written: %lu/%lu #P:%d 63# 64# _-----=> irqs-off 65# / _----=> need-resched 66# | / _---=> hardirq/softirq 67# || / _--=> preempt-depth 68# ||| / delay 69# TASK-PID TGID CPU# |||| TIMESTAMP FUNCTION 70# | | | | |||| | | 71""" 72 73 74def parse_options(): 75 global text_file 76 global binary_file 77 global out_file 78 79 usage = "Usage: %prog -t text_file -o out_file or\n%prog -b binary_file -o out_file" 80 desc = "Example: %prog -t my_trace_file.htrace -o my_trace_file.systrace" 81 82 parser = optparse.OptionParser(usage=usage, description=desc) 83 parser.add_option('-t', '--text_file', dest='text_file', 84 help='Name of the text file to be parsed.', metavar='FILE') 85 parser.add_option('-b', '--binary_file', dest='binary_file', 86 help='Name of the binary file to be parsed.', metavar='FILE') 87 parser.add_option('-o', '--out_file', dest='out_file', 88 help='File name after successful parsing.', metavar='FILE') 89 90 options, args = parser.parse_args() 91 92 if options.out_file is not None: 93 out_file = options.out_file 94 else: 95 print("Error: out_file must be specified") 96 exit(-1) 97 if options.text_file is not None: 98 text_file = options.text_file 99 if options.binary_file is not None: 100 binary_file = options.binary_file 101 102 if text_file == '' and binary_file == '': 103 print("Error: You must specify a text or binary file") 104 exit(-1) 105 if text_file != '' and binary_file != '': 106 print("Error: Only one parsed file can be specified") 107 exit(-1) 108 109 110def parse_text_trace_file(): 111 print("start processing text trace file") 112 pattern_async = re.compile(TRACE_REGEX_ASYNC) 113 pattern_sync = re.compile(TRACE_REGEX_SYNC) 114 match_num = 0 115 116 infile_flags = os.O_RDONLY 117 infile_mode = stat.S_IRUSR 118 infile = os.fdopen(os.open(text_file, infile_flags, infile_mode), "r", encoding="utf-8") 119 outfile_flags = os.O_RDWR | os.O_CREAT 120 outfile_mode = stat.S_IRUSR | stat.S_IWUSR 121 outfile = os.fdopen(os.open(out_file, outfile_flags, outfile_mode), "w+", encoding="utf-8") 122 123 for line in infile: 124 reverse_line = line[::-1] 125 trace_match_async = pattern_async.match(reverse_line) 126 trace_match_sync = pattern_sync.match(reverse_line) 127 if trace_match_async: 128 line = line.rstrip(' ') 129 pos = line.rfind(' ') 130 line = "%s%s%s" % (line[:pos], '|', line[pos + 1:]) 131 match_num += 1 132 elif trace_match_sync: 133 line = "%s\n" % (line.rstrip()[:-1]) 134 match_num += 1 135 outfile.write(line) 136 infile.close() 137 outfile.close() 138 print("total matched and modified lines: ", match_num) 139 140 141cpu_raw_read_pos = 0 142TRACE_HEADER_SIZE = 12 143 144 145def parse_trace_header(infile): 146 trace_header = {} 147 trace_header_data = infile.read(TRACE_HEADER_SIZE) 148 trace_header_tuple = struct.unpack('HBHL', trace_header_data) 149 trace_header["magic_number"] = trace_header_tuple[0] 150 trace_header["file_type"] = trace_header_tuple[1] 151 trace_header["version_number"] = trace_header_tuple[2] 152 trace_header["reserved"] = trace_header_tuple[3] 153 return trace_header 154 155 156def parse_page_header(data): 157 global cpu_raw_read_pos 158 page_header = {} 159 160 data_str = data[cpu_raw_read_pos:cpu_raw_read_pos + INT64_DATA_READ_LEN * 2 + INT8_DATA_READ_LEN] 161 data_str_len = len(data_str) 162 if data_str_len == 17: 163 struct_page_header = struct.unpack('QQB', data_str) 164 cpu_raw_read_pos += INT64_DATA_READ_LEN * 2 + INT8_DATA_READ_LEN 165 page_header["time_stamp"] = struct_page_header[0] 166 167 page_header["length"] = struct_page_header[1] 168 page_header["core_id"] = struct_page_header[2] 169 170 return page_header 171 172 173def parse_event_header(data): 174 global cpu_raw_read_pos 175 event_header = {} 176 177 data_str = data[cpu_raw_read_pos:cpu_raw_read_pos + INT32_DATA_READ_LEN + INT16_DATA_READ_LEN] 178 data_str_len = len(data_str) 179 if data_str_len == 6: 180 struct_event_header = struct.unpack('LH', data_str) 181 event_header["time_stamp_offset"] = struct_event_header[0] 182 event_header["size"] = struct_event_header[1] 183 184 cpu_raw_read_pos += INT32_DATA_READ_LEN + INT16_DATA_READ_LEN 185 186 return event_header 187 188 189TRACE_FLAG_IRQS_OFF = 0x01 190TRACE_FLAG_IRQS_NOSUPPORT = 0x02 191TRACE_FLAG_NEED_RESCHED = 0x04 192TRACE_FLAG_HARDIRQ = 0x08 193TRACE_FLAG_SOFTIRQ = 0x10 194TRACE_FLAG_PREEMPT_RESCHED = 0x20 195TRACE_FLAG_NMI = 0x40 196 197 198def trace_flags_to_str(flags, preempt_count): 199 result = "" 200 irqs_off = '.' 201 if flags & TRACE_FLAG_IRQS_OFF != 0: 202 irqs_off = 'd' 203 elif flags & TRACE_FLAG_IRQS_NOSUPPORT != 0: 204 irqs_off = 'X' 205 result += irqs_off 206 207 need_resched = '.' 208 is_need_resched = flags & TRACE_FLAG_NEED_RESCHED 209 is_preempt_resched = flags & TRACE_FLAG_PREEMPT_RESCHED 210 if is_need_resched != 0 and is_preempt_resched != 0: 211 need_resched = 'N' 212 elif is_need_resched != 0: 213 need_resched = 'n' 214 elif is_preempt_resched != 0: 215 need_resched = 'p' 216 result += need_resched 217 218 nmi_flag = flags & TRACE_FLAG_NMI 219 hard_irq = flags & TRACE_FLAG_HARDIRQ 220 soft_irq = flags & TRACE_FLAG_SOFTIRQ 221 irq_char = '.' 222 if nmi_flag != 0 and hard_irq != 0: 223 irq_char = 'Z' 224 elif nmi_flag != 0: 225 irq_char = 'z' 226 elif hard_irq != 0 and soft_irq != 0: 227 irq_char = 'H' 228 elif hard_irq != 0: 229 irq_char = 'h' 230 elif soft_irq != 0: 231 irq_char = 's' 232 result += irq_char 233 234 if preempt_count != 0: 235 result += "0123456789abcdef"[preempt_count & 0x0F] 236 else: 237 result += "." 238 239 return result 240 241 242COMM_STR_MAX = 16 243PID_STR_MAX = 6 244TGID_STR_MAX = 5 245CPU_STR_MAX = 3 246TS_SECS_MIN = 5 247TS_MICRO_SECS = 6 248 249 250def generate_one_event_str(data, cpu_id, time_stamp, one_event): 251 pid = int.from_bytes(one_event["fields"]["common_pid"], byteorder='little') 252 event_str = "" 253 254 cmd_line = cmd_lines.get(pid, "") 255 if pid == 0: 256 event_str += "<idle>" 257 elif cmd_line != "": 258 event_str += cmd_line 259 else: 260 event_str += "<...>" 261 event_str = event_str.rjust(COMM_STR_MAX) 262 event_str += "-" 263 264 event_str += str(pid).ljust(PID_STR_MAX) 265 266 tgid = tgids.get(pid, "") 267 if tgid != "": 268 event_str += "(" + tgid.rjust(TGID_STR_MAX) + ")" 269 else: 270 event_str += "(-----)" 271 272 event_str += " [" + str(cpu_id).zfill(CPU_STR_MAX) + "] " 273 274 flags = int.from_bytes(one_event["fields"]["common_flags"], byteorder='little') 275 preempt_count = int.from_bytes(one_event["fields"]["common_preempt_count"], byteorder='little') 276 if flags | preempt_count != 0: 277 event_str += trace_flags_to_str(flags, preempt_count) + " " 278 else: 279 event_str += ".... " 280 281 if time_stamp % 1000 >= 500: 282 time_stamp_str = str((time_stamp // 1000) + 1) 283 else: 284 time_stamp_str = str(time_stamp // 1000) 285 ts_secs = time_stamp_str[:-6].rjust(TS_SECS_MIN) 286 ts_micro_secs = time_stamp_str[-6:] 287 event_str += ts_secs + "." + ts_micro_secs + ": " 288 289 parse_result = parse_functions.parse(one_event["print_fmt"], data, one_event) 290 if parse_result is None: 291 print("Error: function parse_" + str(one_event["name"]) + " not found") 292 else: 293 event_str += str(one_event["name"]) + ": " + parse_result 294 295 return event_str 296 297 298def parse_one_event(data, event_id, cpu_id, time_stamp): 299 event_format = events_format.get(event_id, "") 300 if event_format == "": 301 return "" 302 303 fields = event_format["fields"] 304 one_event = {} 305 one_event["id"] = event_id 306 one_event["name"] = event_format["name"] 307 one_event["print_fmt"] = event_format["print_fmt"] 308 one_event["fields"] = {} 309 for field in fields: 310 offset = field["offset"] 311 size = field["size"] 312 one_event["fields"][field["name"]] = data[offset:offset + size] 313 314 return generate_one_event_str(data, cpu_id, time_stamp, one_event) 315 316 317RMQ_ENTRY_ALIGN_MASK = 3 318 319 320def parse_cpu_raw_one_page(data, result): 321 global cpu_raw_read_pos 322 end_pos = cpu_raw_read_pos + READ_PAGE_SIZE 323 page_header = parse_page_header(data) 324 325 while cpu_raw_read_pos < end_pos: 326 event_header = parse_event_header(data) 327 if event_header.get("size", 0) == 0: 328 break 329 330 time_stamp = page_header.get("time_stamp", 0) + event_header.get("time_stamp_offset", 0) 331 event_id = struct.unpack('H', data[cpu_raw_read_pos:cpu_raw_read_pos + INT16_DATA_READ_LEN])[0] 332 333 one_event_data = data[cpu_raw_read_pos:cpu_raw_read_pos + event_header.get("size", 0)] 334 one_event_result = parse_one_event(one_event_data, event_id, page_header.get("core_id", 0), time_stamp) 335 if one_event_result != "": 336 result.append([time_stamp, one_event_result]) 337 338 evt_size = ((event_header.get("size", 0) + RMQ_ENTRY_ALIGN_MASK) & (~RMQ_ENTRY_ALIGN_MASK)) 339 cpu_raw_read_pos += evt_size 340 cpu_raw_read_pos = end_pos 341 342 343def parse_cpu_raw(data, data_len, result): 344 global cpu_raw_read_pos 345 cpu_raw_read_pos = 0 346 347 while cpu_raw_read_pos < data_len: 348 parse_cpu_raw_one_page(data, result) 349 350 351def parse_events_format_field(field_line): 352 field_info = field_line.split(";") 353 field_info[0] = field_info[0].lstrip() 354 field_info[1] = field_info[1].lstrip() 355 field_info[2] = field_info[2].lstrip() 356 field_info[3] = field_info[3].lstrip() 357 358 field = {} 359 type_name_pos = field_info[0].rfind(" ") 360 field["type"] = field_info[0][len("field:"):type_name_pos] 361 field["name"] = field_info[0][type_name_pos + 1:] 362 field["offset"] = int(field_info[1][len("offset:"):]) 363 field["size"] = int(field_info[2][len("size:"):]) 364 field["signed"] = field_info[3][len("signed:"):] 365 366 return field 367 368 369def parse_events_format(data): 370 name_line_prefix = "name: " 371 id_line_prefix = "ID: " 372 field_line_prefix = "field:" 373 print_fmt_line_prefix = "print fmt: " 374 375 events_format_lines = data.decode('utf-8').split("\n") 376 event_format = {} 377 event_format["fields"] = [] 378 for line in events_format_lines: 379 line = line.lstrip() 380 if line.startswith(name_line_prefix): 381 event_format["name"] = line[len(name_line_prefix):] 382 elif line.startswith(id_line_prefix): 383 event_format["id"] = int(line[len(id_line_prefix):]) 384 elif line.startswith(field_line_prefix): 385 event_format["fields"].append(parse_events_format_field(line)) 386 elif line.startswith(print_fmt_line_prefix): 387 event_format["print_fmt"] = line[len(print_fmt_line_prefix):] 388 events_format[event_format["id"]] = copy.deepcopy(event_format) 389 event_format["fields"].clear() 390 391 392def parse_cmdlines(data): 393 cmd_lines_list = data.decode('utf-8').split("\n") 394 for cmd_line in cmd_lines_list: 395 pos = cmd_line.find(" ") 396 if pos != -1: 397 cmd_lines[int(cmd_line[:pos])] = cmd_line[pos + 1:] 398 399 400def parse_tgids(data): 401 tgids_lines_list = data.decode('utf-8').split("\n") 402 for tgids_line in tgids_lines_list: 403 pos = tgids_line.find(" ") 404 if pos != -1: 405 tgids[int(tgids_line[:pos])] = tgids_line[pos + 1:] 406 407 408def parse_header_page(data): 409 print("in parse_header_page") 410 411 412def parse_printk_formats(data): 413 print("in parse_printk_formats") 414 415 416def parse_kallsyms(data): 417 print("in parse_kallsyms") 418 419 420def parse_trace_base_data(infile, file_size): 421 while infile.tell() < file_size: 422 data_type = struct.unpack('L', infile.read(INT32_DATA_READ_LEN))[0] 423 data_len = struct.unpack('L', infile.read(INT32_DATA_READ_LEN))[0] 424 data = infile.read(data_len) 425 if data_type == CONTENT_TYPE_HEADER_PAGE: 426 parse_header_page(data) 427 elif data_type == CONTENT_TYPE_CMDLINES: 428 parse_cmdlines(data) 429 elif data_type == CONTENT_TYPE_TGIDS: 430 parse_tgids(data) 431 elif data_type == CONTENT_TYPE_EVENTS_FORMAT: 432 parse_events_format(data) 433 elif data_type == CONTENT_TYPE_PRINTK_FORMATS: 434 parse_printk_formats(data) 435 elif data_type == CONTENT_TYPE_KALLSYMS: 436 parse_kallsyms(data) 437 438 439def parse_trace_events_data(infile, file_size, cpu_nums, result): 440 while infile.tell() < file_size: 441 data_type = struct.unpack('L', infile.read(INT32_DATA_READ_LEN))[0] 442 data_len = struct.unpack('L', infile.read(INT32_DATA_READ_LEN))[0] 443 data = infile.read(data_len) 444 445 if data_type >= CONTENT_TYPE_CPU_RAW and data_type < CONTENT_TYPE_CPU_RAW + cpu_nums: 446 parse_cpu_raw(data, data_len, result) 447 448 449def parse_binary_trace_file(): 450 infile_flags = os.O_RDONLY | os.O_BINARY 451 infile_mode = stat.S_IRUSR 452 infile = os.fdopen(os.open(binary_file, infile_flags, infile_mode), 'rb') 453 454 outfile_flags = os.O_RDWR | os.O_CREAT 455 outfile_mode = stat.S_IRUSR | stat.S_IWUSR 456 outfile = os.fdopen(os.open(out_file, outfile_flags, outfile_mode), 'w', encoding="utf-8") 457 458 trace_header = parse_trace_header(infile) 459 cpu_nums = (trace_header.get("reserved", 0) >> 1) & 0xf 460 461 outfile.write(TRACE_TXT_HEADER_FORMAT) 462 trace_file_size = os.path.getsize(binary_file) 463 parse_trace_base_data(infile, trace_file_size) 464 infile.seek(TRACE_HEADER_SIZE) 465 466 result = [] 467 parse_trace_events_data(infile, trace_file_size, cpu_nums, result) 468 result = sorted(result, key=lambda x: x[0]) 469 for line in result: 470 outfile.write("{}\n".format(line[1])) 471 472 outfile.close() 473 infile.close() 474 475 476def main(): 477 parse_options() 478 479 if text_file != '': 480 parse_text_trace_file() 481 else: 482 parse_binary_trace_file() 483 484 485if __name__ == '__main__': 486 main()