1 /*
2  * Copyright (c) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef DISTRIBUTED_DB_MODULE_TEST_TOOLS_H
16 #define DISTRIBUTED_DB_MODULE_TEST_TOOLS_H
17 #include <condition_variable>
18 #include <thread>
19 
20 #include "kv_store_delegate.h"
21 #include "kv_store_delegate_manager.h"
22 #include "kv_store_observer_impl.h"
23 #include "distributed_test_sysinfo.h"
24 #include "distributeddb_data_generator.h"
25 #include "log_print.h"
26 #ifdef RUN_MST_ON_TRUNCK // only need it in WAGNER env if run MST
27 #define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name)
28 #endif
29 #define ULL(x) (static_cast<unsigned long long>(x))
30 const int MAX_DIR_LENGTH = 4096; // the max length of directory
31 const int BUF_LEN = 8192;
32 const static std::string TAG = "DistributedTestTools"; // for log
33 const int AUTHORITY = 0755;
34 const int E_OK = 0;
35 const int E_ERROR = -1;
36 const std::string DIRECTOR = "/data/test/getstub/"; // default work dir.
37 static std::condition_variable g_conditionKvVar;
38 
39 struct KvDBParameters {
40     std::string storeId;
41     std::string appId;
42     std::string userId;
KvDBParametersKvDBParameters43     KvDBParameters(std::string storeIdStr, std::string appIdStr, std::string userIdStr)
44         : storeId(storeIdStr), appId(appIdStr), userId(userIdStr)
45     {
46     }
47 };
48 
49 struct KvOption {
50     bool createIfNecessary = true;
51     bool localOnly = false;
52     bool isEncryptedDb = false; // whether need encrypt
53     DistributedDB::CipherType cipher = DistributedDB::CipherType::DEFAULT; // cipher type
54     std::vector<uint8_t> passwd; // cipher password
KvOptionKvOption55     KvOption(bool createIfNecessary1, bool isLocalOnly,
56     bool isEncryptedDb1, DistributedDB::CipherType cipher1, std::vector<uint8_t> passwd1)
57         : createIfNecessary(createIfNecessary1),
58           localOnly(isLocalOnly),
59           isEncryptedDb(isEncryptedDb1),
60           cipher(cipher1), passwd(passwd1)
61     {
62     }
KvOptionKvOption63     KvOption()
64     {}
65 };
66 
67 struct EncrypteAttribute {
68     bool isEncryptedDb = false;
69     std::vector<uint8_t> passwd;
70 };
71 
72 const static KvDBParameters g_kvdbParameter1(DistributedDBDataGenerator::STORE_ID_1,
73     DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1);
74 const static KvDBParameters g_kvdbParameter2(DistributedDBDataGenerator::STORE_ID_2,
75     DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_2);
76 const static KvDBParameters g_kvdbParameter3(DistributedDBDataGenerator::STORE_ID_3,
77     DistributedDBDataGenerator::APP_ID_3, DistributedDBDataGenerator::USER_ID_3);
78 const static KvDBParameters g_kvdbParameter4(DistributedDBDataGenerator::STORE_ID_4,
79     DistributedDBDataGenerator::APP_ID_4, DistributedDBDataGenerator::USER_ID_4);
80 const static KvDBParameters g_kvdbParameter5(DistributedDBDataGenerator::STORE_ID_5,
81     DistributedDBDataGenerator::APP_ID_5, DistributedDBDataGenerator::USER_ID_5);
82 const static KvDBParameters g_kvdbParameter6(DistributedDBDataGenerator::STORE_ID_6,
83     DistributedDBDataGenerator::APP_ID_6, DistributedDBDataGenerator::USER_ID_6);
84 const static KvDBParameters g_kvdbParameter1_1_2(DistributedDBDataGenerator::STORE_ID_1,
85     DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_2);
86 const static KvDBParameters g_kvdbParameter1_2_1(DistributedDBDataGenerator::STORE_ID_1,
87     DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_1);
88 const static KvDBParameters g_kvdbParameter1_2_2(DistributedDBDataGenerator::STORE_ID_1,
89     DistributedDBDataGenerator::APP_ID_2, DistributedDBDataGenerator::USER_ID_2);
90 const static KvDBParameters g_kvdbParameter2_1_1(DistributedDBDataGenerator::STORE_ID_2,
91     DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1);
92 const static KvDBParameters g_kvdbParameter2_1_2(DistributedDBDataGenerator::STORE_ID_2,
93     DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_2);
94 const static KvDBParameters KVDB_PARAMETER_PERFORM(DistributedDBDataGenerator::STORE_ID_PERFORM,
95     DistributedDBDataGenerator::APP_ID_PERFORM, DistributedDBDataGenerator::USER_ID_PERFORM);
96 const static KvOption g_createKvDiskUnencrypted(true, false, false, DistributedDB::CipherType::DEFAULT,
97     DistributedDBDataGenerator::NULL_PASSWD_VECTOR);
98 const static KvOption g_createKvDiskEncrypted(true, false, true, DistributedDB::CipherType::DEFAULT,
99     DistributedDBDataGenerator::PASSWD_VECTOR_1);
100 const static KvOption g_createLocalDiskUnencrypted(true, true, false, DistributedDB::CipherType::DEFAULT,
101     DistributedDBDataGenerator::NULL_PASSWD_VECTOR);
102 const static KvOption g_createLocalDiskEncrypted(true, true, true, DistributedDB::CipherType::DEFAULT,
103     DistributedDBDataGenerator::PASSWD_VECTOR_1);
104 static KvOption g_kvOption = g_createKvDiskEncrypted;
105 bool CompareVector(const std::vector<uint8_t>& first, const std::vector<uint8_t>& second);
106 bool CompareEntriesVector(std::vector<DistributedDB::Entry>& retVec,
107     std::vector<DistributedDB::Entry>& expectVec);
108 void PutUniqueKey(std::vector<DistributedDB::Entry>& entryVec,
109     const DistributedDB::Key &putKey, const DistributedDB::Value &putValue);
110 int Uint8VecToString(std::vector<uint8_t>& vec, std::string& str);
111 int GetIntValue(DistributedDB::Value &value);
112 int RemoveDir(const std::string &directory);
113 int SetDir(const std::string &directory, const int authRight = AUTHORITY);
114 void CopyFile(const std::string &srcFile, const std::string &destFile);
115 void CopyDir(const std::string &srcDir, const std::string &destDir, const int authRight = AUTHORITY);
116 void CheckFileNumber(const std::string &filePath, int &fileCount);
117 DistributedDB::Value GetValueWithInt(int val);
118 std::vector<DistributedDB::Entry> GenRanKeyVal(int putGetTimes, int keyLength, int valueLength, char val);
119 std::vector<DistributedDB::Key> GetKeysFromEntries(std::vector<DistributedDB::Entry> entries, bool random);
120 bool GetRandBool();
121 bool PutEntries(DistributedDB::KvStoreNbDelegate *&delegate, std::vector<DistributedDB::Entry> &entries);
122 
123 using SysTime = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>;
124 using SysDurTime = std::chrono::duration<uint64_t, std::micro>;
125 
126 struct PerformanceData {
127     int putGetTimes;
128     int keyLength;
129     int valueLength;
130     bool putBatch;
131     bool getBatch;
132     bool useClear;
133     bool getSysInfo;
134     bool isLocal;
135     double openDuration;
136     double putDuration;
137     double readPutDuration;
138     double updateDuration;
139     double readUpdateDuration;
140     double deleteDuration;
141     double closeDuration;
PerformanceDataPerformanceData142     PerformanceData(int putGetTimes, int keyLength, int valueLength,
143         bool putBatch, bool getBatch, bool useClear, bool getSysInfo, bool isLocal)
144         : putGetTimes(putGetTimes), keyLength(keyLength), valueLength(valueLength),
145           putBatch(putBatch), getBatch(getBatch), useClear(useClear),
146           getSysInfo(getSysInfo), isLocal(isLocal),
147           openDuration(0.0), putDuration(0.0), readPutDuration(0.0), updateDuration(0.0),
148           readUpdateDuration(0.0), deleteDuration(0.0), closeDuration(0.0)
149     {
150     }
151 };
152 
153 struct Duration {
154     double putDuration = 0.0;
155     double readDuration = 0.0;
156     double updateDuration = 0.0;
157     double deleteDuration = 0.0;
158     Duration() = default;
ClearDuration159     void Clear()
160     {
161         putDuration = readDuration = updateDuration = deleteDuration = 0.0;
162     }
163 };
164 
165 struct BackupDuration {
166     double exportDuration = 0.0;
167     double importDuration = 0.0;
168     BackupDuration() = default;
BackupDurationBackupDuration169     BackupDuration(double exportDur, double importDur)
170     {
171         exportDuration = exportDur;
172         importDuration = importDur;
173     }
174     BackupDuration operator+(const BackupDuration &backupDur) const
175     {
176         return BackupDuration(exportDuration + backupDur.exportDuration, importDuration + backupDur.importDuration);
177     }
ClearBackupDuration178     void Clear()
179     {
180         exportDuration = importDuration = 0;
181     }
182 };
183 
184 enum KvDbType {
185     ENCRYED = 0,
186     UNENCRYED = 1
187 };
188 
189 enum OperRecordNum {
190     SINGLE = 1,
191     SMALL_BATCH = 100,
192     BATCH = 128,
193 };
194 
195 enum class OperType {
196     PUT,
197     PUT_LOCAL,
198     UPDATE,
199     UPDATE_LOCAL,
200     DELETE,
201     DELETE_LOCAL
202 };
203 
204 struct KvPerfData {
205     KvDbType kvDbType;
206     int testCnt;
207     OperRecordNum operRecordNum;
208     int keyLength;
209     int valueLength;
210     bool isPresetRecords;
211     int presetRecordsCnt;
212     std::vector<Duration> allCrudDur;
KvPerfDataKvPerfData213     KvPerfData(KvDbType kvDbType, int testCnt, OperRecordNum operRecordNum, int keyLength, int valueLength,
214         bool isPresetRecords, int presetRecordsCnt)
215         : kvDbType(kvDbType), testCnt(testCnt), operRecordNum(operRecordNum),
216           keyLength(keyLength), valueLength(valueLength),
217           isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt)
218     {
219     }
220 };
221 
222 struct RekeyTypesDur {
223     double nullPasswdToPasswd1 = 0.0;
224     double passwd1ToPasswd1 = 0.0;
225     double passwd1ToPasswd2 = 0.0;
226     double passwd2ToNullPasswd = 0.0;
227     RekeyTypesDur() = default;
RekeyTypesDurRekeyTypesDur228     RekeyTypesDur(double nullPasswdToPasswd1, double passwd1ToPasswd1,
229         double passwd1ToPasswd2, double passwd2ToNullPasswd)
230         : nullPasswdToPasswd1(nullPasswdToPasswd1), passwd1ToPasswd1(passwd1ToPasswd1),
231           passwd1ToPasswd2(passwd1ToPasswd2), passwd2ToNullPasswd(passwd2ToNullPasswd)
232     {
233     }
234     RekeyTypesDur operator+(const RekeyTypesDur &rekeyTypesDur) const
235     {
236         return RekeyTypesDur(nullPasswdToPasswd1 + rekeyTypesDur.nullPasswdToPasswd1,
237             passwd1ToPasswd1 + rekeyTypesDur.passwd1ToPasswd1,
238             passwd1ToPasswd2 + rekeyTypesDur.passwd1ToPasswd2,
239             passwd2ToNullPasswd + rekeyTypesDur.passwd2ToNullPasswd);
240     }
241 };
242 
243 struct KvRekeyPerfData {
244     int testCnt;
245     int presetRecordsCnt;
246     int keyLength;
247     int valueLength;
248     std::vector<RekeyTypesDur> allRekeyDur;
KvRekeyPerfDataKvRekeyPerfData249     KvRekeyPerfData(int testCnt, int presetRecordsCnt, int keyLength, int valueLength)
250         : testCnt(testCnt), presetRecordsCnt(presetRecordsCnt), keyLength(keyLength), valueLength(valueLength)
251     {
252     }
253 };
254 
255 struct KvBackupPerfData {
256     KvDbType kvDbType;
257     int testCnt;
258     int presetRecordsCnt;
259     int keyLength;
260     int valueLength;
261     std::vector<BackupDuration> allBackupDur;
KvBackupPerfDataKvBackupPerfData262     KvBackupPerfData(KvDbType kvDbType, int testCnt, int presetRecordsCnt, int keyLength, int valueLength)
263         : kvDbType(kvDbType), testCnt(testCnt), presetRecordsCnt(presetRecordsCnt),
264           keyLength(keyLength), valueLength(valueLength)
265     {
266     }
267 };
268 
269 struct PerImageGallery {
270     uint64_t putDuration = 0;
271     uint64_t readDuration = 0;
ClearPerImageGallery272     void Clear()
273     {
274         putDuration = 0;
275         readDuration = 0;
276     }
277 };
278 struct NbGalleryPerfData {
279     int testCnt;
280     unsigned int keyLength;
281     unsigned int valueLength;
282     bool isLocal;
283     bool isPresetRecords;
284     int presetRecordsCnt;
285     std::vector<PerImageGallery> allCrudDur;
NbGalleryPerfDataNbGalleryPerfData286     NbGalleryPerfData(int testCnt, int keyLength, int valueLength,
287         bool isLocal, bool isPresetRecords, int presetRecordsCnt)
288         : testCnt(testCnt), keyLength(keyLength), valueLength(valueLength),
289           isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt)
290     {
291     }
292 };
293 struct NbLocalPerfData {
294     int testCnt;
295     unsigned int keyLength;
296     unsigned int valueLength;
297     bool isLocal;
298     bool isPresetRecords;
299     int presetRecordsCnt;
300     std::vector<Duration> allCrudDur;
NbLocalPerfDataNbLocalPerfData301     NbLocalPerfData(int testCnt, int keyLength, int valueLength,
302         bool isLocal, bool isPresetRecords, int presetRecordsCnt)
303         : testCnt(testCnt), keyLength(keyLength), valueLength(valueLength),
304           isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt)
305     {
306     }
307 };
308 
309 struct QueryDur {
310     double getEntriesDuration = 0.0;
311     double getResultSetDuration = 0.0;
312     QueryDur() = default;
ClearQueryDur313     void Clear()
314     {
315         getEntriesDuration = getResultSetDuration = 0;
316     }
317 };
318 struct NbSchemaCRUDPerfData {
319     int testCnt;
320     OperRecordNum operRecordNum;
321     unsigned int keyLength;
322     unsigned int valueLength;
323     bool isLocal;
324     bool isPresetRecords;
325     unsigned int presetRecordsCnt;
326     bool isQueryNeeded;
327     bool isIndexSchema;
328     std::vector<Duration> allCrudDur;
329     std::vector<QueryDur> allQueryDur;
NbSchemaCRUDPerfDataNbSchemaCRUDPerfData330     NbSchemaCRUDPerfData(int testCnt, OperRecordNum operRecordNum, int keyLength, int valueLength,
331         bool isLocal, bool isPresetRecords, int presetRecordsCnt, int isQueryNeeded, int isIndexSchema)
332         : testCnt(testCnt), operRecordNum(operRecordNum), keyLength(keyLength), valueLength(valueLength),
333           isLocal(isLocal), isPresetRecords(isPresetRecords), presetRecordsCnt(presetRecordsCnt),
334           isQueryNeeded(isQueryNeeded), isIndexSchema(isIndexSchema)
335     {
336     }
337 };
338 
339 // default kvStoreDelegateManager's config.
340 const DistributedDB::KvStoreConfig KV_CONFIG = {
341     .dataDir = DIRECTOR
342 };
343 
344 class DistributedTestTools final {
345 public:
DistributedTestTools()346     DistributedTestTools() {}
~DistributedTestTools()347     ~DistributedTestTools() {}
348 
349     // Delete the copy and assign constructors
350     DistributedTestTools(const DistributedTestTools &distributeDBTools) = delete;
351     DistributedTestTools& operator=(const DistributedTestTools &distributeDBTools) = delete;
352     DistributedTestTools(DistributedTestTools &&distributeDBTools) = delete;
353     DistributedTestTools& operator=(DistributedTestTools &&distributeDBTools) = delete;
354 
355     // this static method is to compare if the two Value has the same data.
356     static bool IsValueEquals(const DistributedDB::Value &v1, const DistributedDB::Value &v2);
357     static DistributedDB::KvStoreDelegate::Option TransferKvOptionType(const KvOption &optionParam);
358     static DistributedDB::KvStoreDelegate* GetDelegateSuccess(DistributedDB::KvStoreDelegateManager *&outManager,
359         const KvDBParameters &param, const KvOption &optionParam);
360     static DistributedDB::KvStoreDelegate* GetDelegateStatus(DistributedDB::KvStoreDelegateManager *&outManager,
361         DistributedDB::DBStatus &status, const KvDBParameters &param, const KvOption &optionParam);
362 
363     static DistributedDB::DBStatus GetDelegateNotGood(
364         DistributedDB::KvStoreDelegateManager *&outManager, DistributedDB::KvStoreDelegate *&outDelegate,
365         const std::string &storeId, const std::string &appId, const std::string &userId, const KvOption &optionParam);
366 
367     static DistributedDB::DBStatus Put(DistributedDB::KvStoreDelegate &kvStoreDelegate,
368         const DistributedDB::Key &key, const DistributedDB::Value &value);
369 
370     static DistributedDB::DBStatus PutBatch(DistributedDB::KvStoreDelegate &kvStoreDelegate,
371         const std::vector<DistributedDB::Entry> &entries);
372 
373     static DistributedDB::DBStatus Delete(DistributedDB::KvStoreDelegate &kvStoreDelegate,
374         const DistributedDB::Key &key);
375 
376     static DistributedDB::DBStatus DeleteBatch(DistributedDB::KvStoreDelegate &kvStoreDelegate,
377         const std::vector<DistributedDB::Key> &keys);
378 
379     static DistributedDB::DBStatus Clear(DistributedDB::KvStoreDelegate &kvStoreDelegate);
380 
381     static DistributedDB::KvStoreSnapshotDelegate *GetKvStoreSnapshot(DistributedDB::KvStoreDelegate &kvStoreDelegate);
382     static DistributedDB::Value Get(DistributedDB::KvStoreDelegate &kvStoreDelegate, const DistributedDB::Key &key);
383     static DistributedDB::Value Get(DistributedDB::KvStoreSnapshotDelegate &kvStoreSnapshotDelegate,
384         const DistributedDB::Key &key);
385 
386     static std::vector<DistributedDB::Entry> GetEntries(
387         DistributedDB::KvStoreSnapshotDelegate &kvStoreSnapshotDelegate, const DistributedDB::Key &key);
388     static std::vector<DistributedDB::Entry> GetEntries(DistributedDB::KvStoreDelegate &kvStoreDelegate,
389         const DistributedDB::Key &keyPrefix);
390 
391     static DistributedDB::KvStoreSnapshotDelegate *RegisterSnapObserver(DistributedDB::KvStoreDelegate *delegate,
392         DistributedDB::KvStoreObserver *observer);
393     static DistributedDB::DBStatus RegisterObserver(DistributedDB::KvStoreDelegate *delegate,
394         DistributedDB::KvStoreObserver *observer);
395     static DistributedDB::DBStatus UnRegisterObserver(DistributedDB::KvStoreDelegate *delegate,
396         DistributedDB::KvStoreObserver *observer);
397     static bool CalculateOpenPerformance(PerformanceData &performanceData);
398     static bool CalculateInsertPerformance(PerformanceData &performanceData);
399     static bool CalculateGetPutPerformance(PerformanceData &performanceData);
400     static bool CalculateUpdatePerformance(PerformanceData &performanceData);
401     static bool CalculateGetUpdatePerformance(PerformanceData &performanceData);
402     static bool CalculateUseClearPerformance(PerformanceData &performanceData);
403     static bool CalculateTransactionPerformance(PerformanceData &performanceData);
404     static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager,
405         DistributedDB::KvStoreDelegate *&delegate);
406     static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager,
407         DistributedDB::KvStoreNbDelegate *&delegate);
408     static bool VerifyDbRecordCnt(DistributedDB::KvStoreNbDelegate *&delegate, unsigned int recordCnt,
409         bool isLocal = false);
410     static bool VerifyRecordsInDb(DistributedDB::KvStoreNbDelegate *&delegate,
411         std::vector<DistributedDB::Entry> &entriesExpected,
412         const std::vector<uint8_t> &keyPrefix = DistributedDBDataGenerator::KEY_EMPTY, bool isLocal = false);
413     static bool GetRecordCntByKey(const std::string &dbName,
414         const std::string &strSql, std::vector<DistributedDB::Key> &sqlParam, KvOption &option, int &count);
415     static bool QuerySpecifiedData(const std::string &dbName, const std::string &strSql,
416         EncrypteAttribute &attribute, int &count);
417     static bool RepeatCheckAsyncResult(const std::function<bool(void)> &inPred, int repeatLimit,
418         uint32_t repeatInterval);
419     static bool CompareKey(const DistributedDB::Entry &entry1, const DistributedDB::Entry &entry2);
420     static void CopyFile(const std::string &srcFile, const std::string &destFile);
421 };
422 
423 std::string TransferStringToHashHexString(const std::string &origStr);
424 
425 int RemoveDatabaseDirectory(const std::string &directory);
426 
427 bool VerifyObserverResult(const KvStoreObserverImpl &pObserver,
428     int changedTimes, ListType type, const std::list<DistributedDB::Entry> &lst,
429     uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES);
430 bool VerifyObserverResult(const KvStoreObserverImpl &pObserver,
431     int changedTimes, ListType type, const std::vector<DistributedDB::Entry> &vec,
432     uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES);
433 bool VerifyObserverForSchema(const KvStoreObserverImpl &pObserver,
434     int changedTimes, ListType type, const std::vector<DistributedDB::Entry> &expectEntry,
435     uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES);
436 #endif // DISTRIBUTED_DB_MODULE_TEST_TOOLS_H
437