1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import os 17import sys 18import argparse 19import subprocess 20import ssl 21import shutil 22import importlib 23import time 24import pathlib 25from multiprocessing import cpu_count 26from concurrent.futures import ThreadPoolExecutor, as_completed 27from functools import partial 28from urllib.request import urlopen 29import urllib.error 30from scripts.util.file_utils import read_json_file 31 32 33def _run_cmd(cmd: str): 34 res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 35 stderr=subprocess.PIPE) 36 sout, serr = res.communicate() 37 return sout.rstrip().decode('utf-8'), serr, res.returncode 38 39 40def _check_sha256(check_url: str, local_file: str) -> bool: 41 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 42 local_sha256_cmd = 'sha256sum ' + local_file + "|cut -d ' ' -f1" 43 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 44 local_sha256, err, returncode = _run_cmd(local_sha256_cmd) 45 if check_sha256 != local_sha256: 46 print('remote file {}.sha256 is not found, begin check SHASUMS256.txt'.format(check_url)) 47 check_sha256 = _obtain_sha256_by_sha_sums256(check_url) 48 return check_sha256 == local_sha256 49 50 51def _check_sha256_by_mark(args, check_url: str, code_dir: str, unzip_dir: str, unzip_filename: str) -> bool: 52 check_sha256_cmd = 'curl -s -k ' + check_url + '.sha256' 53 check_sha256, err, returncode = _run_cmd(check_sha256_cmd) 54 mark_file_dir = os.path.join(code_dir, unzip_dir) 55 mark_file_name = check_sha256 + '.' + unzip_filename + '.mark' 56 mark_file_path = os.path.join(mark_file_dir, mark_file_name) 57 args.mark_file_path = mark_file_path 58 return os.path.exists(mark_file_path) 59 60 61def _obtain_sha256_by_sha_sums256(check_url: str) -> str: 62 sha_sums256 = 'SHASUMS256.txt' 63 sha_sums256_path = os.path.join(os.path.dirname(check_url), sha_sums256) 64 file_name = os.path.basename(check_url) 65 cmd = 'curl -s -k ' + sha_sums256_path 66 data_sha_sums256, err, returncode = _run_cmd(cmd) 67 check_sha256 = None 68 for line in data_sha_sums256.split('\n'): 69 if file_name in line: 70 check_sha256 = line.split(' ')[0] 71 return check_sha256 72 73 74def _config_parse(config: dict, tool_repo: str, host_os_version: str) -> dict: 75 parse_dict = dict() 76 parse_dict['unzip_dir'] = config.get('unzip_dir') 77 parse_dict['huaweicloud_url'] = tool_repo + config.get('file_path') 78 parse_dict['unzip_filename'] = config.get('unzip_filename') 79 config_os_version = config.get('host_os_version') 80 if config_os_version is not None and config_os_version != host_os_version: 81 parse_dict['huaweicloud_url'].replace(config_os_version, host_os_version, 1) 82 md5_huaweicloud_url_cmd = 'echo ' + parse_dict.get('huaweicloud_url') + "|md5sum|cut -d ' ' -f1" 83 parse_dict['md5_huaweicloud_url'], err, returncode = _run_cmd(md5_huaweicloud_url_cmd) 84 parse_dict['bin_file'] = os.path.basename(parse_dict.get('huaweicloud_url')) 85 return parse_dict 86 87 88def _uncompress(args, src_file: str, code_dir: str, unzip_dir: str, unzip_filename: str, mark_file_path: str): 89 dest_dir = os.path.join(code_dir, unzip_dir) 90 if src_file[-3:] == 'zip': 91 cmd = 'unzip -o {} -d {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 92 elif src_file[-6:] == 'tar.gz': 93 cmd = 'tar -xvzf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 94 else: 95 cmd = 'tar -xvf {} -C {};echo 0 > {}'.format(src_file, dest_dir, mark_file_path) 96 _run_cmd(cmd) 97 98 99def _copy_url(args, task_id: int, url: str, local_file: str, code_dir: str, unzip_dir: str, 100 unzip_filename: str, mark_file_path: str, progress): 101 retry_times = 0 102 max_retry_times = 3 103 while retry_times < max_retry_times: 104 # download files 105 download_buffer_size = 32768 106 progress.console.log('Requesting {}'.format(url)) 107 try: 108 response = urlopen(url) 109 except urllib.error.HTTPError as e: 110 progress.console.log("Failed to open {}, HTTPError: {}".format(url, e.code), style='red') 111 progress.update(task_id, total=int(response.info()["Content-length"])) 112 with open(local_file, "wb") as dest_file: 113 progress.start_task(task_id) 114 for data in iter(partial(response.read, download_buffer_size), b""): 115 dest_file.write(data) 116 progress.update(task_id, advance=len(data)) 117 progress.console.log("Downloaded {}".format(local_file)) 118 119 if os.path.exists(local_file): 120 if _check_sha256(url, local_file): 121 # decompressing files 122 progress.console.log("Decompressing {}".format(local_file)) 123 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 124 progress.console.log("Decompressed {}".format(local_file)) 125 break 126 else: 127 os.remove(local_file) 128 retry_times += 1 129 if retry_times == max_retry_times: 130 print('{}, download failed with three times retry, please check network status. Prebuilts download exit.'.format(local_file)) 131 # todo, merge with copy_url_disable_rich 132 sys.exit(1) 133 134 135def _copy_url_disable_rich(args, url: str, local_file: str, code_dir: str, unzip_dir: str, 136 unzip_filename: str, mark_file_path: str): 137 # download files 138 download_buffer_size = 32768 139 print('Requesting {}, please wait'.format(url)) 140 try: 141 response = urlopen(url) 142 except urllib.error.HTTPError as e: 143 print("Failed to open {}, HTTPError: {}".format(url, e.code)) 144 with open(local_file, "wb") as dest_file: 145 for data in iter(partial(response.read, download_buffer_size), b""): 146 dest_file.write(data) 147 print("Downloaded {}".format(local_file)) 148 149 # decompressing files 150 print("Decompressing {}, please wait".format(local_file)) 151 _uncompress(args, local_file, code_dir, unzip_dir, unzip_filename, mark_file_path) 152 print("Decompressed {}".format(local_file)) 153 154 155def _is_system_component() -> bool: 156 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 157 if pathlib.Path(os.path.join(root_dir, 'interface', 'sdk-js')).exists() or pathlib.Path( 158 os.path.join(root_dir, 'foundation', 'arkui')).exists() or pathlib.Path( 159 os.path.join(root_dir, 'arkcompiler')).exists(): 160 return True 161 else: 162 return False 163 164 165def _hwcloud_download(args, config: dict, bin_dir: str, code_dir: str): 166 try: 167 cnt = cpu_count() 168 except Exception as e: 169 cnt = 1 170 with ThreadPoolExecutor(max_workers=cnt) as pool: 171 tasks = dict() 172 for config_info in config: 173 parse_dict = _config_parse(config_info, args.tool_repo, args.host_os_version) 174 unzip_dir = parse_dict.get('unzip_dir') 175 huaweicloud_url = parse_dict.get('huaweicloud_url') 176 unzip_filename = parse_dict.get('unzip_filename') 177 md5_huaweicloud_url = parse_dict.get('md5_huaweicloud_url') 178 bin_file = parse_dict.get('bin_file') 179 abs_unzip_dir = os.path.join(code_dir, unzip_dir) 180 if not os.path.exists(abs_unzip_dir): 181 os.makedirs(abs_unzip_dir, exist_ok=True) 182 if _check_sha256_by_mark(args, huaweicloud_url, code_dir, unzip_dir, unzip_filename): 183 if not args.disable_rich: 184 args.progress.console.log('{}, Sha256 markword check OK.'.format(huaweicloud_url), style='green') 185 else: 186 print('{}, Sha256 markword check OK.'.format(huaweicloud_url)) 187 else: 188 _run_cmd(('rm -rf {}/{}/*.{}.mark').format(code_dir, unzip_dir, unzip_filename)) 189 _run_cmd(('rm -rf {}/{}/{}').format(code_dir, unzip_dir, unzip_filename)) 190 local_file = os.path.join(bin_dir, '{}.{}'.format(md5_huaweicloud_url, bin_file)) 191 if os.path.exists(local_file): 192 if _check_sha256(huaweicloud_url, local_file): 193 if not args.disable_rich: 194 args.progress.console.log('{}, Sha256 check download OK.'.format(local_file), style='green') 195 else: 196 print('{}, Sha256 check download OK. Start decompression, please wait'.format(local_file)) 197 task = pool.submit(_uncompress, args, local_file, code_dir, 198 unzip_dir, unzip_filename, args.mark_file_path) 199 tasks[task] = os.path.basename(huaweicloud_url) 200 continue 201 else: 202 os.remove(local_file) 203 filename = huaweicloud_url.split("/")[-1] 204 if not args.disable_rich: 205 task_id = args.progress.add_task("download", filename=filename, start=False) 206 task = pool.submit(_copy_url, args, task_id, huaweicloud_url, local_file, code_dir, 207 unzip_dir, unzip_filename, args.mark_file_path, args.progress) 208 tasks[task] = os.path.basename(huaweicloud_url) 209 else: 210 task = pool.submit(_copy_url_disable_rich, args, huaweicloud_url, local_file, code_dir, 211 unzip_dir, unzip_filename, args.mark_file_path) 212 213 for task in as_completed(tasks): 214 if not args.disable_rich: 215 args.progress.console.log('{}, download and decompress completed'.format(tasks.get(task)), 216 style='green') 217 else: 218 print('{}, download and decompress completed'.format(tasks.get(task))) 219 220 221def _npm_install(args): 222 node_path = 'prebuilts/build-tools/common/nodejs/current/bin' 223 os.environ['PATH'] = '{}/{}:{}'.format(args.code_dir, node_path, os.environ.get('PATH')) 224 npm = os.path.join(args.code_dir, node_path, 'npm') 225 if args.skip_ssl: 226 skip_ssl_cmd = '{} config set strict-ssl false;'.format(npm) 227 out, err, retcode = _run_cmd(skip_ssl_cmd) 228 if retcode != 0: 229 return False, err.decode() 230 npm_clean_cmd = '{} cache clean -f'.format(npm) 231 npm_package_lock_cmd = '{} config set package-lock true'.format(npm) 232 out, err, retcode = _run_cmd(npm_clean_cmd) 233 if retcode != 0: 234 return False, err.decode() 235 out, err, retcode = _run_cmd(npm_package_lock_cmd) 236 if retcode != 0: 237 return False, err.decode() 238 print('start npm install, please wait.') 239 for install_info in args.npm_install_config: 240 full_code_path = os.path.join(args.code_dir, install_info) 241 basename = os.path.basename(full_code_path) 242 node_modules_path = os.path.join(full_code_path, "node_modules") 243 npm_cache_dir = os.path.join('~/.npm/_cacache', basename) 244 print('check node_modules is not exist') 245 if os.path.exists(node_modules_path): 246 print('remove node_modules %s' % node_modules_path) 247 _run_cmd(('rm -rf {}'.format(node_modules_path))) 248 if os.path.exists(full_code_path): 249 cmd = ['timeout', '-s', '9', '90s', npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 250 if args.host_platform == 'darwin': 251 cmd = [npm, 'install', '--registry', args.npm_registry, '--cache', npm_cache_dir] 252 if args.unsafe_perm: 253 cmd.append('--unsafe-perm') 254 print("in dir:{}, executing:{}".format(full_code_path, ' '.join(cmd))) 255 proc = subprocess.Popen(cmd, cwd=full_code_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 256 # wait proc Popen with 0.1 second 257 time.sleep(0.1) 258 out, err = proc.communicate() 259 if proc.returncode: 260 return False, err.decode() 261 else: 262 raise Exception("{} not exist, it shouldn't happen, pls check...".format(full_code_path)) 263 return True, None 264 265 266def _node_modules_copy(config: dict, code_dir: str, enable_symlink: bool): 267 for config_info in config: 268 src_dir = os.path.join(code_dir, config_info.get('src')) 269 dest_dir = os.path.join(code_dir, config_info.get('dest')) 270 use_symlink = config_info.get('use_symlink') 271 if os.path.exists(os.path.dirname(dest_dir)): 272 shutil.rmtree(os.path.dirname(dest_dir)) 273 if use_symlink == 'True' and enable_symlink == True: 274 os.makedirs(os.path.dirname(dest_dir), exist_ok=True) 275 os.symlink(src_dir, dest_dir) 276 else: 277 shutil.copytree(src_dir, dest_dir, symlinks=True) 278 279 280def _file_handle(config: dict, code_dir: str, host_platform: str): 281 for config_info in config: 282 src_dir = code_dir + config_info.get('src') 283 dest_dir = code_dir + config_info.get('dest') 284 tmp_dir = config_info.get('tmp') 285 symlink_src = config_info.get('symlink_src') 286 symlink_dest = config_info.get('symlink_dest') 287 rename = config_info.get('rename') 288 if os.path.exists(src_dir): 289 if tmp_dir: 290 tmp_dir = code_dir + tmp_dir 291 shutil.move(src_dir, tmp_dir) 292 cmd = 'mv {}/*.mark {}'.format(dest_dir, tmp_dir) 293 _run_cmd(cmd) 294 if os.path.exists(dest_dir): 295 shutil.rmtree(dest_dir) 296 shutil.move(tmp_dir, dest_dir) 297 elif rename: 298 if os.path.exists(dest_dir) and dest_dir != src_dir: 299 shutil.rmtree(dest_dir) 300 shutil.move(src_dir, dest_dir) 301 if symlink_src and symlink_dest: 302 if os.path.exists(dest_dir + symlink_dest): 303 os.remove(dest_dir + symlink_dest) 304 if host_platform == 'darwin' and os.path.basename(dest_dir) == "nodejs": 305 symlink_src = symlink_src.replace('linux', 'darwin') 306 os.symlink(os.path.basename(symlink_src), dest_dir + symlink_dest) 307 else: 308 _run_cmd('chmod 755 {} -R'.format(dest_dir)) 309 310 311def _import_rich_module(): 312 module = importlib.import_module('rich.progress') 313 progress = module.Progress( 314 module.TextColumn("[bold blue]{task.fields[filename]}", justify="right"), 315 module.BarColumn(bar_width=None), 316 "[progress.percentage]{task.percentage:>3.1f}%", 317 "•", 318 module.DownloadColumn(), 319 "•", 320 module.TransferSpeedColumn(), 321 "•", 322 module.TimeRemainingColumn(), 323 ) 324 return progress 325 326 327def _install(config: dict, code_dir: str): 328 for config_info in config: 329 install_dir = '{}/{}'.format(code_dir, config_info.get('install_dir')) 330 script = config_info.get('script') 331 cmd = '{}/{}'.format(install_dir, script) 332 args = config_info.get('args') 333 for arg in args: 334 for key in arg.keys(): 335 cmd = '{} --{}={}'.format(cmd, key, arg[key]) 336 dest_dir = '{}/{}'.format(code_dir, config_info.get('destdir')) 337 cmd = '{} --destdir={}'.format(cmd, dest_dir) 338 _run_cmd(cmd) 339 340 341def main(): 342 parser = argparse.ArgumentParser() 343 parser.add_argument('--skip-ssl', action='store_true', help='skip ssl authentication') 344 parser.add_argument('--unsafe-perm', action='store_true', help='add "--unsafe-perm" for npm install') 345 parser.add_argument('--disable-rich', action='store_true', help='disable the rich module') 346 parser.add_argument('--enable-symlink', action='store_true', help='enable symlink while copying node_modules') 347 parser.add_argument('--build-arkuix', action='store_true', help='build ArkUI-X SDK') 348 parser.add_argument('--tool-repo', default='https://repo.huaweicloud.com', help='prebuilt file download source') 349 parser.add_argument('--npm-registry', default='https://repo.huaweicloud.com/repository/npm/', 350 help='npm download source') 351 parser.add_argument('--host-cpu', help='host cpu', required=True) 352 parser.add_argument('--host-platform', help='host platform', required=True) 353 parser.add_argument('--host-os-version', help='host version', required=False) 354 parser.add_argument('--config-file', help='prebuilts download config file') 355 args = parser.parse_args() 356 args.code_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 357 if args.skip_ssl: 358 ssl._create_default_https_context = ssl._create_unverified_context 359 360 host_platform = args.host_platform 361 host_cpu = args.host_cpu 362 host_os_version = args.host_os_version 363 tool_repo = args.tool_repo 364 if args.build_arkuix: 365 config_file = os.path.join(args.code_dir, 'build_plugins/prebuilts_download_config.json') 366 elif args.config_file: 367 config_file = args.config_file 368 else: 369 config_file = os.path.join(args.code_dir, 'build/prebuilts_download_config.json') 370 config_info = read_json_file(config_file) 371 if _is_system_component(): 372 args.npm_install_config = config_info.get('npm_install_path') 373 node_modules_copy_config = config_info.get('node_modules_copy') 374 else: 375 args.npm_install_config = [] 376 node_modules_copy_config = [] 377 file_handle_config = config_info.get('file_handle_config') 378 379 args.bin_dir = os.path.join(args.code_dir, config_info.get('prebuilts_download_dir')) 380 if not os.path.exists(args.bin_dir): 381 os.makedirs(args.bin_dir, exist_ok=True) 382 copy_config = config_info.get(host_platform).get(host_cpu).get('copy_config') 383 node_config = config_info.get(host_platform).get('node_config') 384 copy_config.extend(node_config) 385 install_config = config_info.get(host_platform).get(host_cpu).get('install') 386 if host_platform == 'linux': 387 linux_copy_config = config_info.get(host_platform).get(host_cpu).get('linux_copy_config') 388 copy_config.extend(linux_copy_config) 389 elif host_platform == 'darwin': 390 darwin_copy_config = config_info.get(host_platform).get(host_cpu).get('darwin_copy_config') 391 copy_config.extend(darwin_copy_config) 392 if args.disable_rich: 393 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 394 else: 395 args.progress = _import_rich_module() 396 with args.progress: 397 _hwcloud_download(args, copy_config, args.bin_dir, args.code_dir) 398 399 print('download finished') 400 _file_handle(file_handle_config, args.code_dir, args.host_platform) 401 retry_times = 0 402 max_retry_times = 2 403 while retry_times <= max_retry_times: 404 print('npm install try times:', retry_times + 1) 405 result, error = _npm_install(args) 406 if result: 407 print("npm install successfully") 408 break 409 elif retry_times == max_retry_times: 410 for error_info in error.split('\n'): 411 print("npm install error, error info: %s" % error) 412 if error_info.endswith('debug.log'): 413 log_path = error_info.split()[-1] 414 cmd = ['cat', log_path] 415 process_cat = subprocess.Popen(cmd) 416 process_cat.communicate(timeout=60) 417 raise Exception("npm install error with three times, prebuilts download exit") 418 retry_times += 1 419 _node_modules_copy(node_modules_copy_config, args.code_dir, args.enable_symlink) 420 if install_config: 421 _install(install_config, args.code_dir) 422 423 # delete uninstalled tools 424 uninstalled_tools = config_info.get('uninstalled_tools') 425 for tool_path in uninstalled_tools: 426 subprocess.run(['rm', '-rf', tool_path]) 427 428if __name__ == '__main__': 429 sys.exit(main()) 430