1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright (c) 2021 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""
16
17Usage: gen_notice_file --output-image-name system \
18               --notice-file-root xx/NOTICE_FILE \
19               --notice-file-install-path xx/system \
20               --output-title notice_title_string
21
22Generate the project notice files, including both text and xml files.
23
24"""
25from collections import defaultdict
26import argparse
27import hashlib
28import os
29import os.path
30import sys
31import gzip
32import shutil
33import glob
34import re
35import subprocess
36
37sys.path.append(
38    os.path.dirname(os.path.dirname(os.path.dirname(
39        os.path.abspath(__file__)))))
40from scripts.util import build_utils  # noqa: E402
41from scripts.util.file_utils import write_json_file, read_json_file  # noqa: E402
42
43XML_ESCAPE_TABLE = {
44    "&": "&",
45    '"': """,
46    "'": "'",
47    ">": ">",
48    "<": "&lt;",
49}
50
51
52def copy_static_library_notices(options, depfiles: list):
53    valid_notices = []
54    basenames = []
55    # add sort method
56    files = build_utils.get_all_files(options.static_library_notice_dir)
57    files.sort()
58    for file in files:
59        if os.stat(file).st_size == 0:
60            continue
61        if not file.endswith('.a.txt'):
62            continue
63        notice_file_name = os.path.basename(file)
64        if file not in basenames:
65            basenames.append(notice_file_name)
66            valid_notices.append(file)
67            depfiles.append(file)
68
69    for file in valid_notices:
70        if options.image_name == "system":
71            if options.target_cpu == "arm64" or options.target_cpu == "x64":
72                install_dir = "system/lib64"
73            elif options.target_cpu == "arm":
74                install_dir = "system/lib"
75            else:
76                continue
77        elif options.image_name == "sdk":
78            install_dir = "toolchains/lib"
79        elif options.image_name == "ndk":
80            install_dir = "sysroot/usr/lib"
81        else:
82            continue
83        dest = os.path.join(options.notice_root_dir, install_dir,
84                            os.path.basename(file))
85        os.makedirs(os.path.dirname(dest), exist_ok=True)
86        shutil.copyfile(file, dest)
87        if os.path.isfile("{}.json".format(file)):
88            os.makedirs(os.path.dirname("{}.json".format(dest)), exist_ok=True)
89            shutil.copyfile("{}.json".format(file), "{}.json".format(dest))
90
91
92def write_file(file: str, string: str):
93    print(string, file=file)
94
95
96def compute_hash(file: str):
97    sha256 = hashlib.sha256()
98    with open(file, 'rb') as file_fd:
99        for line in file_fd:
100            sha256.update(line)
101    return sha256.hexdigest()
102
103
104def get_entity(text: str):
105    return "".join(XML_ESCAPE_TABLE.get(c, c) for c in text)
106
107
108def generate_txt_notice_files(file_hash: str, input_dir: str, output_filename: str,
109                              notice_title: str):
110    with open(output_filename, "w") as output_file:
111        write_file(output_file, notice_title)
112        for value in file_hash:
113            write_file(output_file, '=' * 60)
114            write_file(output_file, "Notices for file(s):")
115            for filename in value:
116                write_file(
117                    output_file, '/{}'.format(
118                        re.sub('.txt.*', '',
119                               os.path.relpath(filename, input_dir))))
120            write_file(output_file, '-' * 60)
121            write_file(output_file, "Notices for software(s):")
122            software_list = []
123            for filename in value:
124                json_filename = '{}.json'.format(filename)
125                contents = read_json_file(json_filename)
126                if contents is not None and contents not in software_list:
127                    software_list.append(contents)
128            for contens_value in software_list:
129                if len(contens_value) > 0:
130                    if contens_value[0].get('Software'):
131                        software_name = contens_value[0].get('Software').strip()
132                        write_file(output_file, "Software: {}".format(software_name))
133                    else:
134                        write_file(output_file, "Software: ")
135                    if contens_value[0].get('Version'):
136                        version = contens_value[0].get('Version').strip()
137                        write_file(output_file, "Version: {}".format(version))
138                    else:
139                        write_file(output_file, "Version: ")
140                    if contens_value[0].get('Path'):
141                        notice_source_path = contens_value[0].get('Path').strip()
142                        write_file(output_file, "Path: {}".format(notice_source_path))
143                    else:
144                        write_file(output_file, "Path: ")
145            write_file(output_file, '-' * 60)
146            with open(value[0], errors='ignore') as temp_file_hd:
147                write_file(output_file, temp_file_hd.read())
148
149
150def generate_xml_notice_files(files_with_same_hash: dict, input_dir: str,
151                              output_filename: str):
152    id_table = {}
153    for file_key in files_with_same_hash.keys():
154        for filename in files_with_same_hash[file_key]:
155            id_table[filename] = file_key
156    with open(output_filename, "w") as output_file:
157        write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>')
158        write_file(output_file, "<licenses>")
159
160        # Flatten the lists into a single filename list
161        sorted_filenames = sorted(id_table.keys())
162
163        # write out a table of contents
164        for filename in sorted_filenames:
165            stripped_filename = re.sub('.txt.*', '',
166                                       os.path.relpath(filename, input_dir))
167            write_file(
168                output_file, '<file-name contentId="%s">%s</file-name>' %
169                             (id_table.get(filename), stripped_filename))
170
171        write_file(output_file, '')
172        write_file(output_file, '')
173
174        processed_file_keys = []
175        # write the notice file lists
176        for filename in sorted_filenames:
177            file_key = id_table.get(filename)
178            if file_key in processed_file_keys:
179                continue
180            processed_file_keys.append(file_key)
181
182            with open(filename, errors='ignore') as temp_file_hd:
183                write_file(
184                    output_file,
185                    '<file-content contentId="{}"><![CDATA[{}]]></file-content>'
186                        .format(file_key, get_entity(temp_file_hd.read())))
187            write_file(output_file, '')
188
189        # write the file complete node.
190        write_file(output_file, "</licenses>")
191
192
193def compress_file_to_gz(src_file_name: str, gz_file_name: str):
194    with open(src_file_name, mode='rb') as src_file_fd:
195        with gzip.open(gz_file_name, mode='wb') as gz_file_fd:
196            gz_file_fd.writelines(src_file_fd)
197
198
199def handle_zipfile_notices(zip_file: str):
200    notice_file = '{}.txt'.format(zip_file[:-4])
201    with build_utils.temp_dir() as tmp_dir:
202        build_utils.extract_all(zip_file, tmp_dir, no_clobber=False)
203        files = build_utils.get_all_files(tmp_dir)
204        contents = []
205        for file in files:
206            with open(file, 'r') as fd:
207                data = fd.read()
208                if data not in contents:
209                    contents.append(data)
210        with open(notice_file, 'w') as merged_notice:
211            merged_notice.write('\n\n'.join(contents))
212    return notice_file
213
214
215def main():
216    parser = argparse.ArgumentParser()
217    parser.add_argument('--image-name')
218    parser.add_argument('--collected-notice-zipfile',
219                        action='append',
220                        help='zipfile stors collected notice files')
221    parser.add_argument('--notice-root-dir', help='where notice files store')
222    parser.add_argument('--output-notice-txt', help='output notice.txt')
223    parser.add_argument('--output-notice-gz', help='output notice.txt')
224    parser.add_argument('--notice-title', help='title of notice.txt')
225    parser.add_argument('--static-library-notice-dir',
226                        help='path to static library notice files')
227    parser.add_argument('--target-cpu', help='cpu arch')
228    parser.add_argument('--depfile', help='depfile')
229    parser.add_argument('--notice-module-info',
230                        help='module info file for notice target')
231    parser.add_argument('--notice-install-dir',
232                        help='install directories of notice file')
233    parser.add_argument('--lite-product', help='', default="")
234
235    args = parser.parse_args()
236
237    notice_dir = args.notice_root_dir
238    depfiles = []
239    if args.collected_notice_zipfile:
240        for zip_file in args.collected_notice_zipfile:
241            build_utils.extract_all(zip_file, notice_dir, no_clobber=False)
242    else:
243        depfiles += build_utils.get_all_files(notice_dir)
244    # Copy notice of static targets to notice_root_dir
245    if args.static_library_notice_dir:
246        copy_static_library_notices(args, depfiles)
247
248    zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True)
249
250    txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True)
251    txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True)
252
253    outputs = [args.output_notice_txt, args.output_notice_gz]
254    if args.notice_module_info:
255        outputs.append(args.notice_module_info)
256    build_utils.call_and_write_depfile_if_stale(
257        lambda: do_merge_notice(args, zipfiles, txt_files),
258        args,
259        depfile_deps=depfiles,
260        input_paths=depfiles,
261        input_strings=args.notice_title + args.target_cpu,
262        output_paths=(outputs))
263
264
265def do_merge_notice(args, zipfiles: str, txt_files: str):
266    notice_dir = args.notice_root_dir
267    notice_txt = args.output_notice_txt
268    notice_gz = args.output_notice_gz
269    notice_title = args.notice_title
270
271    if not notice_txt.endswith('.txt'):
272        raise Exception(
273            'Error: input variable output_notice_txt must ends with .txt')
274    if not notice_gz.endswith('.xml.gz'):
275        raise Exception(
276            'Error: input variable output_notice_gz must ends with .xml.gz')
277
278    notice_xml = notice_gz.replace('.gz', '')
279
280    files_with_same_hash = defaultdict(list)
281    for file in zipfiles:
282        txt_files.append(handle_zipfile_notices(file))
283
284    for file in txt_files:
285        if os.stat(file).st_size == 0:
286            continue
287        file_hash = compute_hash(file)
288        files_with_same_hash[file_hash].append(file)
289
290    file_sets = [
291        sorted(files_with_same_hash[hash])
292        for hash in sorted(files_with_same_hash.keys())
293    ]
294
295    if file_sets is not None:
296        generate_txt_notice_files(file_sets, notice_dir, notice_txt,
297                                  notice_title)
298
299    if files_with_same_hash is not None:
300        generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml)
301        compress_file_to_gz(notice_xml, args.output_notice_gz)
302
303    if args.notice_module_info:
304        module_install_info_list = []
305        module_install_info = {}
306        module_install_info['type'] = 'notice'
307        module_install_info['source'] = args.output_notice_txt
308        module_install_info['install_enable'] = True
309        module_install_info['dest'] = [
310            os.path.join(args.notice_install_dir,
311                         os.path.basename(args.output_notice_txt))
312        ]
313        module_install_info_list.append(module_install_info)
314        write_json_file(args.notice_module_info, module_install_info_list)
315
316    if args.lite_product:
317        current_dir_cmd = ['pwd']
318        process = subprocess.Popen(current_dir_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
319        stdout, stderr = process.communicate(timeout=600)
320        current_dir = stdout.decode().strip()
321        dest = f"{current_dir}/system/etc/NOTICE.txt"
322        if os.path.isfile(notice_txt):
323            os.makedirs(os.path.dirname(dest), exist_ok=True)
324            shutil.copyfile(notice_txt, dest)
325
326if __name__ == "__main__":
327    main()
328