1 /*
2  * Copyright (C) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef NSTACKX_FILE_MANAGER_H
17 #define NSTACKX_FILE_MANAGER_H
18 
19 #include "nstackx_epoll.h"
20 #include "nstackx_list.h"
21 #include "nstackx_dfile_frame.h"
22 #ifdef MBEDTLS_INCLUDED
23 #include "nstackx_mbedtls.h"
24 #else
25 #include "nstackx_openssl.h"
26 #endif
27 #include "nstackx_dfile_config.h"
28 #include "nstackx_util.h"
29 
30 #ifdef __cplusplus
31 extern "C" {
32 #endif
33 
34 #define NSTACKX_FILE_MANAGER_THREAD_NUM 3
35 #define NSTACKX_MAX_DATA_FWRITE_TIMEOUT_COUNT 30
36 #define MAX_SEND_FILE_OPENED_PER_LIST 10 /* at least 1 */
37 
38 #define FILE_MANAGER_EOK  0 /* OK */
39 #define FILE_MANAGER_EMUTEX (-1) /* mutex lock or unlock error */
40 #define FILE_MANAGER_ENOMEM (-2) /* Out of memory */
41 #define FILE_MANAGER_MANAGER_BLIST_EADDFULL (-3) /* try to add node to a full block frame list */
42 #define FILE_MANAGER_LIST_EBLOCK (-3) /* receive a illegal block */
43 #define FILE_MANAGER_FILE_ENOSPC (-3) /* device has no available storage space */
44 #define FILE_MANAGER_FILE_ENOMEM (-4) /* Insufficient kernel memory was available */
45 #define FILE_MANAGER_FILE_ENFILE (-5) /* the number of open file descriptors has been reached */
46 #define FILE_MANAGER_FILE_EACCES (-6) /* permission denied */
47 #define FILE_MANAGER_FILE_ENAMETOOLONG (-7) /* file pathname is too long */
48 #define FILE_MANAGER_FILE_ETXTBSY (-8) /* file is occupied */
49 #define FILE_MANAGER_FILE_EOTHER (-9) /* other error */
50 #define NSTACKX_MEGA_BYTES 1048576
51 #define MEGA_BYTES_TRANSFER_NOTICE_THRESHOLD 20
52 #define NSTACKX_KILO_BYTES 1024
53 #define KILO_BYTES_TRANSFER_NOTICE_THRESHOLD 20480
54 #define FILE_RECV_LIST_LEAST_SIZE 10
55 #define FILE_RECV_LIST_IO_WRITE_THRESHOLD 0.8
56 #define FILE_RECV_LIST_SLOW_START_RATE 2
57 
58 #ifdef NSTACKX_WITH_LITEOS
59 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (64 * 1024 * 1024)
60 #else
61 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (800 * 1024 * 1024)
62 #endif
63 
64 #ifdef BUILD_FOR_WINDOWS
65 #define NSTACKX_INVALID_FD NULL
66 #else
67 #define NSTACKX_INVALID_FD (-1)
68 #endif
69 
70 typedef enum {
71     FILE_MANAGER_INNER_ERROR = 1,
72     FILE_MANAGER_SEND_FAIL,
73     FILE_MANAGER_SEND_WAITING_END,
74     FILE_MANAGER_RECEIVE_FAIL,
75     FILE_MANAGER_RECEIVE_SUCCESS,
76     FILE_MANAGER_IN_PROGRESS,
77     FILE_MANAGER_TRANS_IN_PROGRESS
78 } FileManagerMsgType;
79 
80 typedef struct {
81     uint64_t fileSize;
82     uint64_t startOffset;
83     uint16_t fileId;
84     char *fileName;
85 } FileBaseInfo;
86 
87 typedef struct {
88     FileBaseInfo *fileBasicInfo;
89     uint16_t fileNum;
90     uint16_t transId;
91     uint16_t pathType;
92     uint8_t noSyncFlag;
93 } RecvFileListInfo;
94 
95 typedef struct {
96     char *fileList[NSTACKX_DFILE_MAX_FILE_NUM + 1];
97     uint64_t fileSize[NSTACKX_DFILE_MAX_FILE_NUM];
98     uint64_t startOffset[NSTACKX_DFILE_MAX_FILE_NUM];
99     uint16_t fileNum;
100     uint16_t transId;
101     uint8_t tarFlag;
102     uint8_t smallFlag;
103 } SendFileListInfo;
104 
105 typedef struct {
106     uint16_t fileId;
107     uint64_t fileSize;
108     uint16_t standardBlockSize;
109     uint32_t totalBlockNum;
110     char *fileName;
111 #ifdef BUILD_FOR_WINDOWS
112     FILE *fd;
113 #else
114     int32_t fd;
115 #endif
116     uint8_t *tarData;
117     int32_t errCode;
118     int64_t maxSequenceSend;
119     uint32_t receivedBlockNum;
120     uint64_t fileOffset;
121     uint64_t writeOffset;
122     uint8_t isEndBlockReceived;
123     uint64_t startOffset;
124 } FileInfo;
125 
126 typedef struct {
127     List list;
128     uint16_t fileId;
129     uint32_t blockSequence;
130     uint32_t linkSequence;
131 } SendRetranRequestNode;
132 
133 typedef struct {
134     List head;
135     uint32_t maxSize;
136     uint32_t size;
137     pthread_mutex_t lock;
138 } MutexList;
139 
140 typedef struct {
141     List list;
142     uint32_t sendLen;
143     FileDataFrame *fileDataFrame;
144     uint8_t socketIndex;
145 } BlockFrame;
146 
147 /* Reuse DFileMsg for ease use */
148 typedef DFileMsg FileManagerMsg;
149 typedef void (*FileListMsgReceiver)(uint16_t fileId, FileManagerMsgType msgType, FileManagerMsg *msg, void *context,
150                                     uint16_t transId);
151 
152 typedef struct {
153     FileListMsgReceiver msgReceiver;
154     void *context;
155 } FileListMsgPara;
156 
157 typedef enum {
158     FILE_LIST_TRANSFER_FINISH = 1,
159     FILE_LIST_TRANSFER_CANCEL,
160 } TaskStopType;
161 
162 typedef struct {
163     uint16_t fileId;
164     uint32_t blockSequence;
165     pthread_mutex_t lock;
166 } SendFilesOutSet;
167 
168 typedef struct {
169     List list;
170     uint16_t transId;
171     uint16_t fileNum;
172     FileInfo fileInfo[NSTACKX_DFILE_MAX_FILE_NUM];
173     sem_t semStop;
174     uint32_t runStatus;
175     TaskStopType stopType;
176     uint8_t isOccupied;
177     int32_t errCode;
178     uint16_t sendFileProcessed;
179     SendFilesOutSet newReadOutSet;
180     MutexList sendRetranList; /* DATA:SendRetranBlock */
181     MutexList recvBlockList; /* DATA:BlockFrame */
182     List innerRecvBlockHead; /* DATA:BlockFrame */
183     uint32_t innerRecvSize;
184     uint16_t recvFileProcessed;
185     uint8_t isRecvEmptyFilesCreated;
186     uint8_t socketIndex;
187     EpollDesc epollfd;
188     List *eventNodeChain;
189     FileListMsgReceiver msgReceiver;
190     void *context;
191     CryptPara cryptPara;
192     uint64_t bytesTransferredLastRecord;
193     uint64_t totalBytes;
194     uint8_t hasUnInsetFrame;
195     const char *storagePath; /* only useful for receiver */
196     uint8_t tarFlag;
197     uint8_t smallFlag;
198     uint8_t noSyncFlag;
199     uint8_t tarFinished;
200     const char *tarFile;
201     FILE *tarFd;
202     uint16_t blockOffset;
203     FileDataFrame *tarFrame;
204     FileInfo tarFileInfo;
205     uint32_t bindedSendBlockListIdx;
206     uint16_t maxFrameLength;
207     uint8_t allFileDataReceived;
208     uint32_t dataWriteTimeoutCnt;
209     uint64_t bytesTransferred; /* only useful for non-tar sender */
210 } FileListTask;
211 
212 typedef void (*FileManagerMsgReceiver)(FileManagerMsgType msgType, int32_t errCode, void *context);
213 
214 typedef struct {
215     char *storagePath;
216     uint16_t pathType;
217 } TypedStoragePath;
218 
219 typedef struct {
220     MutexList sendBlockFrameList; /* DATA:BlockFrame */
221     List *sendRetranListTail;
222     sem_t semBlockListNotFull;
223     uint32_t bandingTransNum;
224 } SendBlockFrameListPara;
225 
226 typedef struct {
227     uint32_t runStatus;
228     int32_t errCode;
229     uint8_t isSender;
230     uint16_t maxFrameLength;
231     sem_t semTaskListNotEmpty;
232     char *commonStoragePath;
233     TypedStoragePath pathList[NSTACKX_MAX_STORAGE_PATH_NUM];
234     uint16_t typedPathNum;
235     MutexList taskList; /* DATA:FileListTask */
236     pthread_t fileManagerTid[NSTACKX_FILE_MANAGER_THREAD_NUM];
237     EpollDesc epollfd;
238     List *eventNodeChain;
239     FileManagerMsgReceiver msgReceiver;
240     void *context;
241     uint8_t key[AES_256_KEY_LENGTH];
242     uint32_t keyLen;
243     uint64_t totalBytes;
244     atomic_t totalPackInMemory;
245     uint64_t stoppedTasksBytesTransferred;
246     uint64_t bytesTransferredLastRecord;
247     atomic_t bytesTransferredInCurPeriod;
248     SendBlockFrameListPara sendBlockFrameListPara[NSTACKX_MAX_CLIENT_SEND_THREAD_NUM];
249     uint32_t maxSendBlockListSize;
250     uint16_t sendFrameListNum;
251     uint32_t maxRecvBlockListSize;
252     uint64_t iorBytes;
253     uint64_t iowBytes;
254     uint32_t iorRate;
255     uint32_t iowRate;
256     uint32_t iowMaxRate;
257     uint32_t sendListFullTimes;
258     uint8_t transFlag;
259     uint64_t iowCount; /* io write count in NSTACKX_WLAN_MAX_CONTROL_FRAME_TIMEOUT second */
260     uint8_t recvListOverIo;
261 } FileManager;
262 
263 typedef struct {
264     EpollDesc epollfd;
265     List *eventNodeChain;
266     FileManagerMsgReceiver msgReceiver;
267     void *context;
268 } FileManagerMsgPara;
269 
270 typedef enum {
271     STATE_RECEIVE_ONGOING = 0,
272     STATE_RECEIVE_DONE_FAIL,
273     STATE_RECEIVE_DONE_SUCCESSFULLY,
274 } FileRecvState;
275 
276 typedef struct {
277     List list;
278     TransIdState transIdState;
279     uint16_t transId;
280 } TransStateNode;
281 
282 #define FILE_MANAGE_RUN 0
283 #define FILE_MANAGE_DESTROY 1
284 
285 #define FILE_LIST_STATUS_IDLE 0
286 #define FILE_LIST_STATUS_RUN 1
287 #define FILE_LIST_STATUS_STOP 2
288 
289 #define NSTACKX_MAX_RETRAN_BLOCK_NUM 50000
290 #define NSTACKX_MAX_PROCESSING_TASK_NUM 100
291 
292 #define LIST_TAIL_BLOCK_REPEAT_TIMES 2
293 #define THREAD_QUIT_TRY_TIMES 3
294 #define GCM_AAD_CHAR 'A'
295 
296 #define FILE_MANAGER_THREADS_BINDED_CPU_NUM NSTACKX_FILE_MANAGER_THREAD_NUM
297 
CheckManager(const FileManager * fileManager)298 static inline int32_t CheckManager(const FileManager *fileManager)
299 {
300     if (fileManager == NULL || fileManager->runStatus != FILE_MANAGE_RUN || fileManager->errCode != FILE_MANAGER_EOK) {
301         return NSTACKX_EINVAL;
302     }
303     return NSTACKX_EOK;
304 }
305 
CheckSenderManager(const FileManager * fileManager)306 static inline int32_t CheckSenderManager(const FileManager *fileManager)
307 {
308     if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_TRUE) {
309         return NSTACKX_EINVAL;
310     }
311     return NSTACKX_EOK;
312 }
313 
CheckReceiverManager(FileManager * fileManager)314 static inline int32_t CheckReceiverManager(FileManager *fileManager)
315 {
316     if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_FALSE) {
317         return NSTACKX_EINVAL;
318     }
319     return NSTACKX_EOK;
320 }
321 
CheckFilelist(FileListTask * fileList)322 static inline int32_t CheckFilelist(FileListTask *fileList)
323 {
324     if (fileList == NULL || fileList->runStatus != FILE_LIST_STATUS_RUN || fileList->errCode != FILE_MANAGER_EOK) {
325         return NSTACKX_EFAILED;
326     }
327     return NSTACKX_EOK;
328 }
329 
CheckFilelistNotStop(FileListTask * fileList)330 static inline int32_t CheckFilelistNotStop(FileListTask *fileList)
331 {
332     if (fileList == NULL || fileList->runStatus == FILE_LIST_STATUS_STOP || fileList->errCode != FILE_MANAGER_EOK) {
333         return NSTACKX_EFAILED;
334     }
335     return NSTACKX_EOK;
336 }
337 
338 /* Not thread safe */
339 FileManager *FileManagerCreate(uint8_t isSender, FileManagerMsgPara *msgPara, const uint8_t *key,
340                                uint32_t keyLen, uint16_t connType);
341 
342 /* Not thread safe */
343 void StopFileManagerThreads(FileManager *fileManager);
344 
345 /*
346  * Destroy fileManager and free related resource.
347  * Note: this is not thread safe, and you must call the interface StopFileManagerThreads to stop all related threads
348  *       before call this interface.
349  */
350 void FileManagerDestroy(FileManager *fileManager);
351 
352 /* Not thread safe */
353 int32_t FileManagerSetMaxFrameLength(FileManager *fileManager, uint16_t maxFrameLength);
354 
355 /* Not thread safe */
356 int32_t FileManagerSetRecvParaWithConnType(FileManager *fileManager, uint16_t connectType);
357 
358 /* Not thread safe */
359 int32_t FileManagerSetWritePath(FileManager *fileManager, const char *storagePath);
360 
361 /* Not thread safe */
362 int32_t FileManagerSetWritePathList(FileManager *fileManager, char *path[], uint16_t *pathType, uint16_t pathNum);
363 
364 /* Not thread safe */
365 int32_t FileManagerSendFileTask(FileManager *fileManager, const SendFileListInfo *fileListInfo,
366                                 const FileListMsgPara *msgPara);
367 /* Not thread safe */
368 int32_t FileManagerResetSendOutSet(FileManager *fileManager, uint16_t fileId, uint32_t blockSequence, uint16_t transId);
369 
370 /* Not thread safe */
371 int32_t FileManagerFileRead(FileManager *fileManager, uint32_t tid, BlockFrame **block, int32_t nr);
372 
373 /* Not thread safe */
374 int32_t FileManagerRecvFileTask(FileManager *fileManager, RecvFileListInfo *fileListInfo, FileListMsgPara *msgPara);
375 
376 /* Not thread safe */
377 int32_t FileManagerFileWrite(FileManager *fileManager, FileDataFrame *frame);
378 
379 /*
380  * Stop the transfer of file list specialized by the argument transId and clear related resource.
381  * Para: stopType - this is only meaningful for receiver and is one of the follow integer:
382  *       FILE_LIST_TRANSFER_FINISH, the receiver will remove all error and incompelte files received in this tranfer;
383  *       FILE_LIST_TRANSFER_CANCEL, the receicer will remove all files in this transfer.
384  * Note: this interface is not thread safe
385  */
386 int32_t FileManagerStopTask(FileManager *fileManager, uint16_t transId, TaskStopType stopType);
387 
388 /* Not thread safe */
389 uint8_t FileManagerIsLastBlockRead(FileManager *fileManager, uint16_t transId);
390 
391 /* Not thread safe */
392 uint8_t FileManagerHasPendingData(FileManager *fileManager);
393 
394 /* Not thread safe */
395 int32_t FileManagerGetLastSequence(FileManager *fileManager, uint16_t transId, uint16_t fileId, uint32_t *sequence);
396 
397 /* Not thread safe */
398 uint8_t FileManagerIsRecvBlockWritable(FileManager *fileManager, uint16_t transId);
399 
400 /* Not thread safe */
401 int32_t FileManagerGetTotalBytes(FileManager *fileManager, uint64_t *totalBytes);
402 
403 /* Not thread safe */
404 int32_t FileManagerGetBytesTransferred(FileManager *fileManager, uint64_t *bytesTransferred);
405 
406 /* Not thread safe */
407 int32_t FileManagerGetTransUpdateInfo(FileManager *fileManager, uint16_t transId, uint64_t *totalBytes,
408                                       uint64_t *bytesTransferred);
409 
410 int32_t GetEncryptedDataTarFrame(CryptPara *cryptPara, uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
411 
412 int32_t GetNoEncryptedDataTarFrame(uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
413 
414 int32_t FileManagerGetReceivedFiles(FileManager *fileManager, uint16_t transId, uint16_t fileIdList[],
415                                     uint8_t fileIdSuccessFlag[], uint32_t *fileNum);
416 
417 int32_t FileManagerSetAllDataReceived(FileManager *fileManager, uint16_t transId);
418 
419 uint8_t PushSendBlockFrame(FileManager *fileManager, const FileListTask *fileList, const FileDataFrame *fileDataFrame);
420 
421 void UpdateTarFileListSendStatus(FileListTask *fileList);
422 
423 FileDataFrame *CreateRetranBlockFrame(FileManager *fileManager, FileListTask *fileList);
424 
425 uint8_t PushRetranBlockFrame(FileManager *fileManager, const FileListTask *fileList,
426                              const FileDataFrame *fileDataFrame);
427 void FileManagerCLearReadOutSet(FileListTask *fileList);
428 
429 uint8_t GetBlockHeadFlag(uint8_t isStartFrame, uint8_t isEndFrame);
430 
431 int32_t MutexListInit(MutexList *mutexList, uint32_t maxSize);
432 
433 void MutexListDestory(MutexList *mutexList);
434 
435 int32_t MutexListPopFront(MutexList *mutexList, List **curFront, uint8_t *isPoped);
436 
437 int32_t MutexListAddNode(MutexList *mutexList, List *element, uint8_t isFront);
438 
439 void NotifyFileManagerMsg(const FileManager *fileManager, FileManagerMsgType msgType);
440 
441 void NotifyFileListMsg(const FileListTask *fileList, FileManagerMsgType msgType);
442 
443 void NotifyFileMsg(const FileListTask *fileList, uint16_t fileId, FileManagerMsgType msgType);
444 
445 int32_t ConvertErrCode(int32_t error);
446 
447 int32_t SetFileOffset(FileInfo *fileInfo, uint64_t fileOffset);
448 
449 void CloseFile(FileInfo *fileInfo);
450 
451 uint64_t FileListGetBytesTransferred(const FileListTask *fileList, uint8_t isSender);
452 
453 uint16_t GetStandardBlockSize(const FileManager *fileManager);
454 
455 int32_t SetCryptPara(FileListTask *fileList, const uint8_t key[], uint32_t keyLen);
456 
457 FileListTask *GetFileListById(MutexList *taskList, uint16_t transId, uint8_t *isErrorOccurred);
458 
459 void RefreshBytesTransFerred(FileManager *fileManager, BlockFrame *frame);
460 int32_t GetFileBlockListSize(MutexList *taskList, uint32_t *recvListAllSize, uint32_t *recvInnerAllSize);
461 extern int32_t SetTransIdState(DFileSession *session, uint16_t transId, TransIdState state);
462 extern TransStateNode *GetTransIdState(DFileSession *session, uint16_t transId, TransIdState *state);
463 extern int32_t IsTransIdDone(DFileSession *session, uint16_t transId);
464 extern void ClearTransStateList(DFileSession *session);
465 
466 void FileSync(const FileInfo *fileInfo);
467 char *GetStoragePathByType(FileManager *fileManager, uint16_t pathType);
468 char *GetFullFilePath(const char *path, const char *fileName);
469 
470 #ifdef __cplusplus
471 }
472 #endif
473 
474 #endif /* NSTACKX_FILE_MANAGER_H */
475