1 /*
2  * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 package ohos.global.i18n;
17 
18 import java.io.IOException;
19 import java.io.InputStreamReader;
20 import java.io.OutputStreamWriter;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.BufferedReader;
24 import java.io.FileOutputStream;
25 import java.net.URISyntaxException;
26 import java.nio.charset.StandardCharsets;
27 import java.util.concurrent.ArrayBlockingQueue;
28 import java.util.concurrent.ThreadPoolExecutor;
29 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.locks.ReentrantLock;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.Comparator;
34 import java.util.HashMap;
35 import java.util.Map;
36 import java.util.logging.Logger;
37 import java.util.logging.Level;
38 import java.util.regex.Pattern;
39 import java.util.regex.Matcher;
40 
41 import com.ibm.icu.util.ULocale;
42 
43 import net.sf.json.JSONArray;
44 import net.sf.json.JSONObject;
45 
46 /**
47  * This class is used to generate i18n.dat file
48  *
49  * @since 2022-8-22
50  */
51 public class DataFetcher {
52     private static final ReentrantLock LOCK = new ReentrantLock();
53     private static final ArrayList<Fetcher> FETCHERS = new ArrayList<>();
54     private static final ArrayList<String> SCRIPTS = new ArrayList<>(Arrays.asList(
55         "", "Latn", "Hans", "Hant", "Qaag", "Cyrl", "Deva", "Guru"
56     ));
57     private static final ArrayList<String> DATA_TYPES = new ArrayList<>(Arrays.asList(
58         "calendar-gregorian-monthNames-format-abbreviated", "calendar-gregorian-dayNames-format-abbreviated",
59         "time-patterns", "date-patterns", "am-pm-markers", "plural", "number-format", "number-digit",
60         "Time-separator", "default-hour", "stand-alone-abbr-month-names", "standalone-abbr-weekday-names",
61         "format-wide-month-names", "hour-minute-secons-pattern", "full-medium-short-pattern",
62         "format-wide-weeday-names", "standalone-wide-weekday-names", "standalone-wide-month-names",
63         "elapsed-patterns", "week_data", "decimal_plural", "minus-sign"
64     ));
65     private static final HashMap<String, Integer> ID_MAP = new HashMap<>(64);
66     private static final HashMap<String, Integer> LOCALES = new HashMap<>();
67     private static final HashMap<Integer, ArrayList<LocaleConfig>> LOCALE_CONFIGS = new HashMap<>(64);
68     private static final Logger LOG = Logger.getLogger("DataFetcher");
69     private static int sStatus = 0;
70     private static final Pattern RE_LANGUAGE = Pattern.compile("^([a-z]{2,3})-\\*$");
71     private static final int MAX_TIME_TO_WAIT = 10;
72     private static final String SEP = File.separator;
73     private static final int CORE_POOL_SIZE = 1000;
74     private static final int MAX_POOL_SIZE = 1000;
75     private static final int QUEUE_CAPACITY = 2000;
76     private static final Long KEEP_ALIVE_TIME = 1L;
77 
78     static {
addFetchers()79         addFetchers();
80     }
81 
DataFetcher()82     private DataFetcher() {}
83 
84     /**
85      *
86      * Add all required locales from locale.txt and fetch its related data.
87      */
addFetchers()88     private static void addFetchers() {
89         try (BufferedReader fLocales = new BufferedReader(new InputStreamReader(new FileInputStream(
90                 new File(MeasureFormatPatternFetcher.class.getResource("/resource/locales.txt").toURI())),
91                 StandardCharsets.UTF_8))) {
92             String line = "";
93             int count = 0;
94             ULocale[] availableLocales = ULocale.getAvailableLocales();
95             while ((line = fLocales.readLine()) != null) {
96                 String tag = line.trim();
97                 if (LOCALES.containsKey(tag)) {
98                     continue;
99                 }
100                 // special treatment to wildcard
101                 int tempCount = processWildcard(line, availableLocales, count);
102                 if (tempCount > count) {
103                     count = tempCount;
104                     continue;
105                 }
106                 if (!Utils.isValidLanguageTag(tag)) {
107                     LOG.log(Level.SEVERE, String.format("wrong languageTag %s", tag));
108                     sStatus = 1;
109                     return;
110                 }
111                 FETCHERS.add(new Fetcher(tag, LOCK, ID_MAP));
112                 LOCALES.put(tag, count);
113                 ++count;
114             }
115         } catch (URISyntaxException e) {
116             LOG.log(Level.SEVERE, "Add fetchers failed: Url syntax exception");
117             sStatus = 1;
118         } catch (IOException e) {
119             LOG.log(Level.SEVERE, "Add fetchers failed: Io exception");
120             sStatus = 1;
121         }
122     }
123 
processWildcard(String line, ULocale[] availableLocales, int count)124     private static int processWildcard(String line, ULocale[] availableLocales, int count) {
125         String tag = line.trim();
126         int tempCount = count;
127         if ("*".equals(line)) { // special treatment to wildcard xx-*
128             for (ULocale loc : availableLocales) {
129                 String finalLanguageTag = loc.toLanguageTag();
130                 // now we assume en-001 as invalid locale,
131                 if (!LOCALES.containsKey(finalLanguageTag) && Utils.isValidLanguageTag(finalLanguageTag)) {
132                     FETCHERS.add(new Fetcher(finalLanguageTag, LOCK, ID_MAP));
133                     LOCALES.put(tag, tempCount);
134                     ++tempCount;
135                 }
136             }
137             return tempCount;
138         }
139         Matcher matcher = RE_LANGUAGE.matcher(line);
140         if (matcher.matches()) { // special treatment to wildcard language-*
141             String baseName = matcher.group(1);
142             for (ULocale loc : availableLocales) {
143                 String finalLanguageTag = loc.toLanguageTag();
144                 if (loc.getLanguage().equals(baseName) && !LOCALES.containsKey(finalLanguageTag) &&
145                     Utils.isValidLanguageTag(finalLanguageTag)) {
146                     FETCHERS.add(new Fetcher(finalLanguageTag, LOCK, ID_MAP));
147                     LOCALES.put(tag, tempCount);
148                     ++tempCount;
149                 }
150             }
151         }
152         return tempCount;
153     }
154 
checkStatus()155     private static boolean checkStatus() {
156         return sStatus == 0;
157     }
158 
countData(Fetcher currentFetcher, int count, Fetcher fallbackFetcher, ArrayList<LocaleConfig> temp)159     private static int countData(Fetcher currentFetcher, int count,
160             Fetcher fallbackFetcher, ArrayList<LocaleConfig> temp) {
161         String fallbackData = null;
162         for (int i = 0; i < Fetcher.getResourceCount(); i++) {
163             String targetMetaData = Fetcher.getInt2Str().get(i);
164             String myData = currentFetcher.datas.get(i);
165             if (fallbackFetcher != null) {
166                 fallbackData = fallbackFetcher.datas.get(i);
167             } else {
168                 fallbackData = null;
169             }
170             if (!myData.equals(fallbackData)) {
171                 temp.add(new LocaleConfig(targetMetaData, i, ID_MAP.get(myData)));
172                 ++count;
173                 currentFetcher.reservedAdd(1);
174             } else {
175                 currentFetcher.reservedAdd(0);
176             }
177         }
178         return count;
179     }
180 
181     /**
182      * If a locale's data equals to its fallback's data, this locale is excluded
183      * if a meta data of a locale equals to its fallback's data, this meta data is excluded
184      * validLocales keep track of how many locales will be available in dat file.
185      * count indicates how many metaData in total will be available in dat file.
186      *
187      * @return Total number of meta data count
188      */
buildLocaleConfigs()189     private static int buildLocaleConfigs() {
190         Fetcher fallbackFetcher = null;
191         int count = 0;
192         for (Map.Entry<String, Integer> entry : LOCALES.entrySet()) {
193             String languageTag = entry.getKey();
194             int index = entry.getValue();
195             Fetcher currentFetcher = FETCHERS.get(index);
196             ArrayList<LocaleConfig> temp = new ArrayList<>();
197             LOCALE_CONFIGS.put(index, temp);
198             String fallbackLanguageTag = Utils.getFallback(languageTag);
199             // now we need to confirm whether current fetcher's data should be write to i18n.dat
200             // if current fetcher's fallback contains equivalent data, then we don't need current fetcher's data.
201             if (!LOCALES.containsKey(fallbackLanguageTag) || fallbackLanguageTag.equals(languageTag)) {
202                 fallbackFetcher = null;
203             } else {
204                 fallbackFetcher = FETCHERS.get(LOCALES.get(fallbackLanguageTag));
205             }
206             if (currentFetcher.equals(fallbackFetcher)) {
207                 currentFetcher.setIncluded(false);
208             } else {
209                 count = countData(currentFetcher, count, fallbackFetcher, temp);
210             }
211         }
212         return count;
213     }
214 
localeCompare(String locale1, String locale2)215     private static int localeCompare(String locale1, String locale2) {
216         String[] locale1Parts = locale1.split("-");
217         String script1 = "";
218         String region1 = "";
219         if (locale1Parts.length == 2) {
220             if (locale1Parts[1].length() == 2) {
221                 region1 = locale1Parts[1];
222             } else {
223                 script1 = locale1Parts[1];
224             }
225         }
226         if (locale1Parts.length == 3) {
227             script1 = locale1Parts[1];
228             region1 = locale1Parts[2];
229         }
230         String[] locale2Parts = locale2.split("-");
231         String script2 = "";
232         String region2 = "";
233         if (locale2Parts.length == 2) {
234             if (locale2Parts[1].length() == 2) {
235                 region2 = locale2Parts[1];
236             } else {
237                 script2 = locale2Parts[1];
238             }
239         }
240         if (locale2Parts.length == 3) {
241             script2 = locale2Parts[1];
242             region2 = locale2Parts[2];
243         }
244         String lang1 = locale1Parts[0];
245         String lang2 = locale2Parts[0];
246         if (!lang1.equals(lang2)) {
247             return lang1.compareTo(lang2);
248         }
249         if (!script1.equals(script2)) {
250             return SCRIPTS.indexOf(script1) - SCRIPTS.indexOf(script2);
251         }
252         return region1.compareTo(region2);
253     }
254 
writeLocales()255     private static void writeLocales() {
256         JSONArray array = new JSONArray();
257         ArrayList<String> locales = new ArrayList<>(LOCALES.keySet());
258         locales.sort(new Comparator<String>() {
259             @Override
260             public int compare(String o1, String o2) {
261                 return localeCompare(o1, o2);
262             }
263         });
264         for (String locale : locales) {
265             array.add(locale);
266         }
267         JSONObject jsonObject = new JSONObject();
268         jsonObject.put("locales", locales);
269         try (OutputStreamWriter osw = new OutputStreamWriter(
270             new FileOutputStream("tools" + SEP + "i18n-dat-tool" + SEP + "src" + SEP + "main" + SEP + "resource" +
271                 SEP + "locales.json"), StandardCharsets.UTF_8)) {
272             osw.write(jsonObject.toString(2));
273             osw.flush();
274             osw.close();
275         } catch (IOException e) {
276             LOG.log(Level.SEVERE, "Write locales.json failed: IO exception");
277         }
278     }
279 
writeData(String fileName, int index)280     private static void writeData(String fileName, int index) {
281         JSONObject object = new JSONObject();
282         for (Fetcher fetcher : FETCHERS) {
283             String language = fetcher.languageTag;
284             String data = fetcher.datas.get(index);
285             if (data.length() == 0 || !fetcher.getIncluded() || fetcher.reservedGet(index) == 0) {
286                 continue;
287             }
288             String[] values = data.split("_", -1);
289             JSONArray array = new JSONArray();
290             for (String val : values) {
291                 array.add(val);
292             }
293             object.put(language, array);
294         }
295         try (OutputStreamWriter osw = new OutputStreamWriter(
296             new FileOutputStream("tools" + SEP + "i18n-dat-tool" + SEP + "src" + SEP + "main" + SEP + "resource" +
297                 SEP + fileName + ".json"), StandardCharsets.UTF_8)) {
298             osw.write(object.toString(2));
299             osw.flush();
300             osw.close();
301         } catch (IOException e) {
302             LOG.log(Level.SEVERE, "Write file failed: IO exception");
303         }
304     }
305 
getMeasureDataUnit(String[] values)306     private static JSONObject getMeasureDataUnit(String[] values) {
307         String[] units = values[1].split("\\|");
308         String[] types = {"short", "medium", "long", "full"};
309         int pluralNum = 6;
310         int start = 4; // 4 is unit start index
311 
312         JSONObject unitsJson = new JSONObject();
313         for (int i = 0; i < units.length; i++) {
314             JSONObject typedUnitsJson = new JSONObject();
315             for (int j = 0; j < types.length; j++) {
316                 JSONArray pluralUnitJson = new JSONArray();
317                 for (int k = 0; k < pluralNum; k++) {
318                     pluralUnitJson.add(values[start]);
319                     start++;
320                 }
321                 typedUnitsJson.put(types[j], pluralUnitJson);
322             }
323             unitsJson.put(units[i], typedUnitsJson);
324         }
325         return unitsJson;
326     }
327 
writeMeasureData()328     private static void writeMeasureData() {
329         JSONObject object = new JSONObject();
330         for (Fetcher fetcher : FETCHERS) {
331             // 22 is measure data index
332             String data = fetcher.datas.get(22);
333             if (data.length() == 0) {
334                 continue;
335             }
336             String[] values = data.split("_", -1);
337             JSONObject languageJson = new JSONObject();
338             languageJson.put("unit_num", values[0]);  // 0 is unit num index in measure data
339             languageJson.put("unit_set", values[1]);  // 1 is unit set index in measure data
340             languageJson.put("pattern", values[2]);  // 2 is pattern index in measure data
341             languageJson.put("order", values[3]);  // 3 is order index in measure data
342 
343             JSONObject units = getMeasureDataUnit(values);
344             languageJson.put("units", units);
345             object.put(fetcher.languageTag, languageJson);
346         }
347         try (OutputStreamWriter osw = new OutputStreamWriter(
348             new FileOutputStream("tools" + SEP + "i18n-dat-tool" + SEP + "src" + SEP + "main" + SEP + "resource" +
349                 SEP + "measure-format-patterns.json"), StandardCharsets.UTF_8)) {
350             osw.write(object.toString(2));
351             osw.flush();
352             osw.close();
353         } catch (IOException e) {
354             LOG.log(Level.SEVERE, "Write measure data failed: IO exception");
355         }
356     }
357 
358     /**
359      * Main function used to generate i18n.dat file
360      *
361      * @param args Main function's argument
362      */
main(String[] args)363     public static void main(String[] args) {
364         if (!Fetcher.isFetcherStatusOk() || !checkStatus()) {
365             return;
366         }
367         ThreadPoolExecutor exec = new ThreadPoolExecutor(CORE_POOL_SIZE, MAX_POOL_SIZE,
368             KEEP_ALIVE_TIME, TimeUnit.SECONDS, new ArrayBlockingQueue<>(QUEUE_CAPACITY));
369         for (Fetcher fe : FETCHERS) {
370             exec.execute(fe);
371         }
372         exec.shutdown();
373         try {
374             exec.awaitTermination(MAX_TIME_TO_WAIT, TimeUnit.SECONDS);
375         } catch (InterruptedException e) {
376             LOG.log(Level.SEVERE, "main class in DataFetcher interrupted");
377         }
378         buildLocaleConfigs(); // every metaData needs 6 bytes
379         for (Fetcher fetcher : FETCHERS) {
380             if (!fetcher.getIncluded()) {
381                 LOCALES.remove(fetcher.languageTag);
382             }
383         }
384         FETCHERS.sort(null);
385 
386         writeLocales();
387         for (int i = 0; i < DATA_TYPES.size(); i++) {
388             writeData(DATA_TYPES.get(i), i);
389         }
390         writeMeasureData();
391     }
392 }