1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef NSTACKX_FILE_MANAGER_H
17 #define NSTACKX_FILE_MANAGER_H
18
19 #include "nstackx_epoll.h"
20 #include "nstackx_list.h"
21 #include "nstackx_dfile_frame.h"
22 #ifdef MBEDTLS_INCLUDED
23 #include "nstackx_mbedtls.h"
24 #else
25 #include "nstackx_openssl.h"
26 #endif
27 #include "nstackx_dfile_config.h"
28 #include "nstackx_util.h"
29
30 #ifdef __cplusplus
31 extern "C" {
32 #endif
33
34 #define NSTACKX_FILE_MANAGER_THREAD_NUM 3
35 #define NSTACKX_MAX_DATA_FWRITE_TIMEOUT_COUNT 30
36 #define MAX_SEND_FILE_OPENED_PER_LIST 10 /* at least 1 */
37
38 #define FILE_MANAGER_EOK 0 /* OK */
39 #define FILE_MANAGER_EMUTEX (-1) /* mutex lock or unlock error */
40 #define FILE_MANAGER_ENOMEM (-2) /* Out of memory */
41 #define FILE_MANAGER_MANAGER_BLIST_EADDFULL (-3) /* try to add node to a full block frame list */
42 #define FILE_MANAGER_LIST_EBLOCK (-3) /* receive a illegal block */
43 #define FILE_MANAGER_FILE_ENOSPC (-3) /* device has no available storage space */
44 #define FILE_MANAGER_FILE_ENOMEM (-4) /* Insufficient kernel memory was available */
45 #define FILE_MANAGER_FILE_ENFILE (-5) /* the number of open file descriptors has been reached */
46 #define FILE_MANAGER_FILE_EACCES (-6) /* permission denied */
47 #define FILE_MANAGER_FILE_ENAMETOOLONG (-7) /* file pathname is too long */
48 #define FILE_MANAGER_FILE_ETXTBSY (-8) /* file is occupied */
49 #define FILE_MANAGER_FILE_EOTHER (-9) /* other error */
50 #define NSTACKX_MEGA_BYTES 1048576
51 #define MEGA_BYTES_TRANSFER_NOTICE_THRESHOLD 20
52 #define NSTACKX_KILO_BYTES 1024
53 #define KILO_BYTES_TRANSFER_NOTICE_THRESHOLD 20480
54 #define FILE_RECV_LIST_LEAST_SIZE 10
55 #define FILE_RECV_LIST_IO_WRITE_THRESHOLD 0.8
56 #define FILE_RECV_LIST_SLOW_START_RATE 2
57
58 #ifdef NSTACKX_WITH_LITEOS
59 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (64 * 1024 * 1024)
60 #else
61 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (800 * 1024 * 1024)
62 #endif
63
64 #ifdef BUILD_FOR_WINDOWS
65 #define NSTACKX_INVALID_FD NULL
66 #else
67 #define NSTACKX_INVALID_FD (-1)
68 #endif
69
70 typedef enum {
71 FILE_MANAGER_INNER_ERROR = 1,
72 FILE_MANAGER_SEND_FAIL,
73 FILE_MANAGER_SEND_WAITING_END,
74 FILE_MANAGER_RECEIVE_FAIL,
75 FILE_MANAGER_RECEIVE_SUCCESS,
76 FILE_MANAGER_IN_PROGRESS,
77 FILE_MANAGER_TRANS_IN_PROGRESS
78 } FileManagerMsgType;
79
80 typedef struct {
81 uint64_t fileSize;
82 uint64_t startOffset;
83 uint16_t fileId;
84 char *fileName;
85 } FileBaseInfo;
86
87 typedef struct {
88 FileBaseInfo *fileBasicInfo;
89 uint16_t fileNum;
90 uint16_t transId;
91 uint16_t pathType;
92 uint8_t noSyncFlag;
93 } RecvFileListInfo;
94
95 typedef struct {
96 char *fileList[NSTACKX_DFILE_MAX_FILE_NUM + 1];
97 uint64_t fileSize[NSTACKX_DFILE_MAX_FILE_NUM];
98 uint64_t startOffset[NSTACKX_DFILE_MAX_FILE_NUM];
99 uint16_t fileNum;
100 uint16_t transId;
101 uint8_t tarFlag;
102 uint8_t smallFlag;
103 } SendFileListInfo;
104
105 typedef struct {
106 uint16_t fileId;
107 uint64_t fileSize;
108 uint16_t standardBlockSize;
109 uint32_t totalBlockNum;
110 char *fileName;
111 #ifdef BUILD_FOR_WINDOWS
112 FILE *fd;
113 #else
114 int32_t fd;
115 #endif
116 uint8_t *tarData;
117 int32_t errCode;
118 int64_t maxSequenceSend;
119 uint32_t receivedBlockNum;
120 uint64_t fileOffset;
121 uint64_t writeOffset;
122 uint8_t isEndBlockReceived;
123 uint64_t startOffset;
124 } FileInfo;
125
126 typedef struct {
127 List list;
128 uint16_t fileId;
129 uint32_t blockSequence;
130 uint32_t linkSequence;
131 } SendRetranRequestNode;
132
133 typedef struct {
134 List head;
135 uint32_t maxSize;
136 uint32_t size;
137 pthread_mutex_t lock;
138 } MutexList;
139
140 typedef struct {
141 List list;
142 uint32_t sendLen;
143 FileDataFrame *fileDataFrame;
144 uint8_t socketIndex;
145 } BlockFrame;
146
147 /* Reuse DFileMsg for ease use */
148 typedef DFileMsg FileManagerMsg;
149 typedef void (*FileListMsgReceiver)(uint16_t fileId, FileManagerMsgType msgType, FileManagerMsg *msg, void *context,
150 uint16_t transId);
151
152 typedef struct {
153 FileListMsgReceiver msgReceiver;
154 void *context;
155 } FileListMsgPara;
156
157 typedef enum {
158 FILE_LIST_TRANSFER_FINISH = 1,
159 FILE_LIST_TRANSFER_CANCEL,
160 } TaskStopType;
161
162 typedef struct {
163 uint16_t fileId;
164 uint32_t blockSequence;
165 pthread_mutex_t lock;
166 } SendFilesOutSet;
167
168 typedef struct {
169 List list;
170 uint16_t transId;
171 uint16_t fileNum;
172 FileInfo fileInfo[NSTACKX_DFILE_MAX_FILE_NUM];
173 sem_t semStop;
174 uint32_t runStatus;
175 TaskStopType stopType;
176 uint8_t isOccupied;
177 int32_t errCode;
178 uint16_t sendFileProcessed;
179 SendFilesOutSet newReadOutSet;
180 MutexList sendRetranList; /* DATA:SendRetranBlock */
181 MutexList recvBlockList; /* DATA:BlockFrame */
182 List innerRecvBlockHead; /* DATA:BlockFrame */
183 uint32_t innerRecvSize;
184 uint16_t recvFileProcessed;
185 uint8_t isRecvEmptyFilesCreated;
186 uint8_t socketIndex;
187 EpollDesc epollfd;
188 List *eventNodeChain;
189 FileListMsgReceiver msgReceiver;
190 void *context;
191 CryptPara cryptPara;
192 uint64_t bytesTransferredLastRecord;
193 uint64_t totalBytes;
194 uint8_t hasUnInsetFrame;
195 const char *storagePath; /* only useful for receiver */
196 uint8_t tarFlag;
197 uint8_t smallFlag;
198 uint8_t noSyncFlag;
199 uint8_t tarFinished;
200 const char *tarFile;
201 FILE *tarFd;
202 uint16_t blockOffset;
203 FileDataFrame *tarFrame;
204 FileInfo tarFileInfo;
205 uint32_t bindedSendBlockListIdx;
206 uint16_t maxFrameLength;
207 uint8_t allFileDataReceived;
208 uint32_t dataWriteTimeoutCnt;
209 uint64_t bytesTransferred; /* only useful for non-tar sender */
210 } FileListTask;
211
212 typedef void (*FileManagerMsgReceiver)(FileManagerMsgType msgType, int32_t errCode, void *context);
213
214 typedef struct {
215 char *storagePath;
216 uint16_t pathType;
217 } TypedStoragePath;
218
219 typedef struct {
220 MutexList sendBlockFrameList; /* DATA:BlockFrame */
221 List *sendRetranListTail;
222 sem_t semBlockListNotFull;
223 uint32_t bandingTransNum;
224 } SendBlockFrameListPara;
225
226 typedef struct {
227 uint32_t runStatus;
228 int32_t errCode;
229 uint8_t isSender;
230 uint16_t maxFrameLength;
231 sem_t semTaskListNotEmpty;
232 char *commonStoragePath;
233 TypedStoragePath pathList[NSTACKX_MAX_STORAGE_PATH_NUM];
234 uint16_t typedPathNum;
235 MutexList taskList; /* DATA:FileListTask */
236 pthread_t fileManagerTid[NSTACKX_FILE_MANAGER_THREAD_NUM];
237 EpollDesc epollfd;
238 List *eventNodeChain;
239 FileManagerMsgReceiver msgReceiver;
240 void *context;
241 uint8_t key[AES_256_KEY_LENGTH];
242 uint32_t keyLen;
243 uint64_t totalBytes;
244 atomic_t totalPackInMemory;
245 uint64_t stoppedTasksBytesTransferred;
246 uint64_t bytesTransferredLastRecord;
247 atomic_t bytesTransferredInCurPeriod;
248 SendBlockFrameListPara sendBlockFrameListPara[NSTACKX_MAX_CLIENT_SEND_THREAD_NUM];
249 uint32_t maxSendBlockListSize;
250 uint16_t sendFrameListNum;
251 uint32_t maxRecvBlockListSize;
252 uint64_t iorBytes;
253 uint64_t iowBytes;
254 uint32_t iorRate;
255 uint32_t iowRate;
256 uint32_t iowMaxRate;
257 uint32_t sendListFullTimes;
258 uint8_t transFlag;
259 uint64_t iowCount; /* io write count in NSTACKX_WLAN_MAX_CONTROL_FRAME_TIMEOUT second */
260 uint8_t recvListOverIo;
261 } FileManager;
262
263 typedef struct {
264 EpollDesc epollfd;
265 List *eventNodeChain;
266 FileManagerMsgReceiver msgReceiver;
267 void *context;
268 } FileManagerMsgPara;
269
270 typedef enum {
271 STATE_RECEIVE_ONGOING = 0,
272 STATE_RECEIVE_DONE_FAIL,
273 STATE_RECEIVE_DONE_SUCCESSFULLY,
274 } FileRecvState;
275
276 typedef struct {
277 List list;
278 TransIdState transIdState;
279 uint16_t transId;
280 } TransStateNode;
281
282 #define FILE_MANAGE_RUN 0
283 #define FILE_MANAGE_DESTROY 1
284
285 #define FILE_LIST_STATUS_IDLE 0
286 #define FILE_LIST_STATUS_RUN 1
287 #define FILE_LIST_STATUS_STOP 2
288
289 #define NSTACKX_MAX_RETRAN_BLOCK_NUM 50000
290 #define NSTACKX_MAX_PROCESSING_TASK_NUM 100
291
292 #define LIST_TAIL_BLOCK_REPEAT_TIMES 2
293 #define THREAD_QUIT_TRY_TIMES 3
294 #define GCM_AAD_CHAR 'A'
295
296 #define FILE_MANAGER_THREADS_BINDED_CPU_NUM NSTACKX_FILE_MANAGER_THREAD_NUM
297
CheckManager(const FileManager * fileManager)298 static inline int32_t CheckManager(const FileManager *fileManager)
299 {
300 if (fileManager == NULL || fileManager->runStatus != FILE_MANAGE_RUN || fileManager->errCode != FILE_MANAGER_EOK) {
301 return NSTACKX_EINVAL;
302 }
303 return NSTACKX_EOK;
304 }
305
CheckSenderManager(const FileManager * fileManager)306 static inline int32_t CheckSenderManager(const FileManager *fileManager)
307 {
308 if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_TRUE) {
309 return NSTACKX_EINVAL;
310 }
311 return NSTACKX_EOK;
312 }
313
CheckReceiverManager(FileManager * fileManager)314 static inline int32_t CheckReceiverManager(FileManager *fileManager)
315 {
316 if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_FALSE) {
317 return NSTACKX_EINVAL;
318 }
319 return NSTACKX_EOK;
320 }
321
CheckFilelist(FileListTask * fileList)322 static inline int32_t CheckFilelist(FileListTask *fileList)
323 {
324 if (fileList == NULL || fileList->runStatus != FILE_LIST_STATUS_RUN || fileList->errCode != FILE_MANAGER_EOK) {
325 return NSTACKX_EFAILED;
326 }
327 return NSTACKX_EOK;
328 }
329
CheckFilelistNotStop(FileListTask * fileList)330 static inline int32_t CheckFilelistNotStop(FileListTask *fileList)
331 {
332 if (fileList == NULL || fileList->runStatus == FILE_LIST_STATUS_STOP || fileList->errCode != FILE_MANAGER_EOK) {
333 return NSTACKX_EFAILED;
334 }
335 return NSTACKX_EOK;
336 }
337
338 /* Not thread safe */
339 FileManager *FileManagerCreate(uint8_t isSender, FileManagerMsgPara *msgPara, const uint8_t *key,
340 uint32_t keyLen, uint16_t connType);
341
342 /* Not thread safe */
343 void StopFileManagerThreads(FileManager *fileManager);
344
345 /*
346 * Destroy fileManager and free related resource.
347 * Note: this is not thread safe, and you must call the interface StopFileManagerThreads to stop all related threads
348 * before call this interface.
349 */
350 void FileManagerDestroy(FileManager *fileManager);
351
352 /* Not thread safe */
353 int32_t FileManagerSetMaxFrameLength(FileManager *fileManager, uint16_t maxFrameLength);
354
355 /* Not thread safe */
356 int32_t FileManagerSetRecvParaWithConnType(FileManager *fileManager, uint16_t connectType);
357
358 /* Not thread safe */
359 int32_t FileManagerSetWritePath(FileManager *fileManager, const char *storagePath);
360
361 /* Not thread safe */
362 int32_t FileManagerSetWritePathList(FileManager *fileManager, char *path[], uint16_t *pathType, uint16_t pathNum);
363
364 /* Not thread safe */
365 int32_t FileManagerSendFileTask(FileManager *fileManager, const SendFileListInfo *fileListInfo,
366 const FileListMsgPara *msgPara);
367 /* Not thread safe */
368 int32_t FileManagerResetSendOutSet(FileManager *fileManager, uint16_t fileId, uint32_t blockSequence, uint16_t transId);
369
370 /* Not thread safe */
371 int32_t FileManagerFileRead(FileManager *fileManager, uint32_t tid, BlockFrame **block, int32_t nr);
372
373 /* Not thread safe */
374 int32_t FileManagerRecvFileTask(FileManager *fileManager, RecvFileListInfo *fileListInfo, FileListMsgPara *msgPara);
375
376 /* Not thread safe */
377 int32_t FileManagerFileWrite(FileManager *fileManager, FileDataFrame *frame);
378
379 /*
380 * Stop the transfer of file list specialized by the argument transId and clear related resource.
381 * Para: stopType - this is only meaningful for receiver and is one of the follow integer:
382 * FILE_LIST_TRANSFER_FINISH, the receiver will remove all error and incompelte files received in this tranfer;
383 * FILE_LIST_TRANSFER_CANCEL, the receicer will remove all files in this transfer.
384 * Note: this interface is not thread safe
385 */
386 int32_t FileManagerStopTask(FileManager *fileManager, uint16_t transId, TaskStopType stopType);
387
388 /* Not thread safe */
389 uint8_t FileManagerIsLastBlockRead(FileManager *fileManager, uint16_t transId);
390
391 /* Not thread safe */
392 uint8_t FileManagerHasPendingData(FileManager *fileManager);
393
394 /* Not thread safe */
395 int32_t FileManagerGetLastSequence(FileManager *fileManager, uint16_t transId, uint16_t fileId, uint32_t *sequence);
396
397 /* Not thread safe */
398 uint8_t FileManagerIsRecvBlockWritable(FileManager *fileManager, uint16_t transId);
399
400 /* Not thread safe */
401 int32_t FileManagerGetTotalBytes(FileManager *fileManager, uint64_t *totalBytes);
402
403 /* Not thread safe */
404 int32_t FileManagerGetBytesTransferred(FileManager *fileManager, uint64_t *bytesTransferred);
405
406 /* Not thread safe */
407 int32_t FileManagerGetTransUpdateInfo(FileManager *fileManager, uint16_t transId, uint64_t *totalBytes,
408 uint64_t *bytesTransferred);
409
410 int32_t GetEncryptedDataTarFrame(CryptPara *cryptPara, uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
411
412 int32_t GetNoEncryptedDataTarFrame(uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
413
414 int32_t FileManagerGetReceivedFiles(FileManager *fileManager, uint16_t transId, uint16_t fileIdList[],
415 uint8_t fileIdSuccessFlag[], uint32_t *fileNum);
416
417 int32_t FileManagerSetAllDataReceived(FileManager *fileManager, uint16_t transId);
418
419 uint8_t PushSendBlockFrame(FileManager *fileManager, const FileListTask *fileList, const FileDataFrame *fileDataFrame);
420
421 void UpdateTarFileListSendStatus(FileListTask *fileList);
422
423 FileDataFrame *CreateRetranBlockFrame(FileManager *fileManager, FileListTask *fileList);
424
425 uint8_t PushRetranBlockFrame(FileManager *fileManager, const FileListTask *fileList,
426 const FileDataFrame *fileDataFrame);
427 void FileManagerCLearReadOutSet(FileListTask *fileList);
428
429 uint8_t GetBlockHeadFlag(uint8_t isStartFrame, uint8_t isEndFrame);
430
431 int32_t MutexListInit(MutexList *mutexList, uint32_t maxSize);
432
433 void MutexListDestory(MutexList *mutexList);
434
435 int32_t MutexListPopFront(MutexList *mutexList, List **curFront, uint8_t *isPoped);
436
437 int32_t MutexListAddNode(MutexList *mutexList, List *element, uint8_t isFront);
438
439 void NotifyFileManagerMsg(const FileManager *fileManager, FileManagerMsgType msgType);
440
441 void NotifyFileListMsg(const FileListTask *fileList, FileManagerMsgType msgType);
442
443 void NotifyFileMsg(const FileListTask *fileList, uint16_t fileId, FileManagerMsgType msgType);
444
445 int32_t ConvertErrCode(int32_t error);
446
447 int32_t SetFileOffset(FileInfo *fileInfo, uint64_t fileOffset);
448
449 void CloseFile(FileInfo *fileInfo);
450
451 uint64_t FileListGetBytesTransferred(const FileListTask *fileList, uint8_t isSender);
452
453 uint16_t GetStandardBlockSize(const FileManager *fileManager);
454
455 int32_t SetCryptPara(FileListTask *fileList, const uint8_t key[], uint32_t keyLen);
456
457 FileListTask *GetFileListById(MutexList *taskList, uint16_t transId, uint8_t *isErrorOccurred);
458
459 void RefreshBytesTransFerred(FileManager *fileManager, BlockFrame *frame);
460 int32_t GetFileBlockListSize(MutexList *taskList, uint32_t *recvListAllSize, uint32_t *recvInnerAllSize);
461 extern int32_t SetTransIdState(DFileSession *session, uint16_t transId, TransIdState state);
462 extern TransStateNode *GetTransIdState(DFileSession *session, uint16_t transId, TransIdState *state);
463 extern int32_t IsTransIdDone(DFileSession *session, uint16_t transId);
464 extern void ClearTransStateList(DFileSession *session);
465
466 void FileSync(const FileInfo *fileInfo);
467 char *GetStoragePathByType(FileManager *fileManager, uint16_t pathType);
468 char *GetFullFilePath(const char *path, const char *fileName);
469
470 #ifdef __cplusplus
471 }
472 #endif
473
474 #endif /* NSTACKX_FILE_MANAGER_H */
475