1 /*
2  * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "fps_controller_process.h"
17 
18 #include "dcamera_utils_tools.h"
19 #include "distributed_camera_errno.h"
20 #include "distributed_hardware_log.h"
21 
22 namespace OHOS {
23 namespace DistributedHardware {
~FpsControllerProcess()24 FpsControllerProcess::~FpsControllerProcess()
25 {
26     if (isFpsControllerProcess_) {
27         DHLOGD("~DecodeDataProcess : ReleaseProcessNode.");
28         ReleaseProcessNode();
29     }
30 }
31 
InitNode(const VideoConfigParams & sourceConfig,const VideoConfigParams & targetConfig,VideoConfigParams & processedConfig)32 int32_t FpsControllerProcess::InitNode(const VideoConfigParams& sourceConfig, const VideoConfigParams& targetConfig,
33     VideoConfigParams& processedConfig)
34 {
35     if (targetConfig.GetFrameRate() > MAX_TARGET_FRAME_RATE) {
36         DHLOGE("The target framerate : %{public}d is greater than the max framerate : %{public}d.",
37             targetConfig.GetFrameRate(), MAX_TARGET_FRAME_RATE);
38         return DCAMERA_BAD_TYPE;
39     }
40     sourceConfig_ = sourceConfig;
41     targetConfig_ = targetConfig;
42     targetFrameRate_ = targetConfig_.GetFrameRate();
43 
44     processedConfig_ = sourceConfig;
45     processedConfig = processedConfig_;
46     isFpsControllerProcess_ = true;
47     return DCAMERA_OK;
48 }
49 
ReleaseProcessNode()50 void FpsControllerProcess::ReleaseProcessNode()
51 {
52     DHLOGD("Start release [%{public}zu] node : FPS controller.", nodeRank_);
53     isFpsControllerProcess_ = false;
54     isFirstFrame_ = false;
55     targetFrameRate_ = 0;
56     lastFrameIncomeTimeMs_ = 0;
57     recentFrameTimeSpanMs_ = -1;
58     keepCorrectionCount_ = 0;
59     keepLessThanDoubleCount_ = 0;
60     keepMoreThanDoubleCount_ = 0;
61     frameRateCorrectionFactor_ = 0.0;
62     frameRateOvershootMdf_ = 0;
63     for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
64         incomingFrameTimesMs_[i] = 0;
65     }
66 
67     if (nextDataProcess_ != nullptr) {
68         nextDataProcess_->ReleaseProcessNode();
69         nextDataProcess_ = nullptr;
70     }
71     DHLOGD("Release [%{public}zu] node : FPS controller end.", nodeRank_);
72 }
73 
ProcessData(std::vector<std::shared_ptr<DataBuffer>> & inputBuffers)74 int32_t FpsControllerProcess::ProcessData(std::vector<std::shared_ptr<DataBuffer>>& inputBuffers)
75 {
76     if (inputBuffers.empty() || inputBuffers[0] == nullptr) {
77         DHLOGE("Data buffers is null.");
78         return DCAMERA_BAD_TYPE;
79     }
80     if (!isFpsControllerProcess_) {
81         DHLOGE("Decoder node occurred error.");
82         return DCAMERA_DISABLE_PROCESS;
83     }
84     int64_t timeStampUs = 0;
85     if (!inputBuffers[0]->FindInt64("timeUs", timeStampUs)) {
86         DHLOGE("Find decoder output timestamp failed.");
87         return DCAMERA_BAD_TYPE;
88     }
89 
90     std::lock_guard<std::mutex> lck (mtx);
91     int64_t nowTimeMs = GetNowTimeStampMs();
92     UpdateFPSControllerInfo(nowTimeMs);
93 
94     float curFrameRate = CalculateFrameRate(nowTimeMs);
95     if (IsDropFrame(curFrameRate)) {
96         DHLOGD("frame control, currect frameRate %{public}f, targetRate %{public}d, drop it",
97             curFrameRate, targetFrameRate_);
98         return DCAMERA_OK;
99     }
100 
101     DHLOGD("frame control render PushVideoFrame, frame info width %{public}d height %{public}d, timeStampUs "
102         "%{public}lld, fps %{public}f", sourceConfig_.GetWidth(), sourceConfig_.GetHeight(), (long long)timeStampUs,
103         curFrameRate);
104     return FpsControllerDone(inputBuffers);
105 }
106 
UpdateFPSControllerInfo(int64_t nowMs)107 void FpsControllerProcess::UpdateFPSControllerInfo(int64_t nowMs)
108 {
109     DHLOGD("Frame control, update control info.");
110     if (targetFrameRate_ <= 0) {
111         DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
112         return;
113     }
114 
115     isFirstFrame_ = false;
116     if (lastFrameIncomeTimeMs_ == 0) {
117         DHLOGD("Frame control, income fisrt frame.");
118         isFirstFrame_ = true;
119     }
120     lastFrameIncomeTimeMs_ = nowMs;
121     recentFrameTimeSpanMs_ = nowMs - lastFrameIncomeTimeMs_;
122     DHLOGD("Frame control, lastFrameIncomeTimeMs_ %{public}lld, receive Frame after last frame(ms): %{public}lld",
123         (long long)lastFrameIncomeTimeMs_, (long long)recentFrameTimeSpanMs_);
124     UpdateIncomingFrameTimes(nowMs);
125     UpdateFrameRateCorrectionFactor(nowMs);
126     return;
127 }
128 
UpdateFrameRateCorrectionFactor(int64_t nowMs)129 void FpsControllerProcess::UpdateFrameRateCorrectionFactor(int64_t nowMs)
130 {
131     DHLOGD("Frame control, update FPS correction factor.");
132     if (targetFrameRate_ <= 0) {
133         DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
134         return;
135     }
136     if (isFirstFrame_) {
137         DHLOGD("No frame rate correction factor when the first frame.");
138         return;
139     }
140 
141     const float minDropFrmValue = 0.5;
142     const float maxDropFrmValue = 1.0;
143     const float msPerSecond = 1000;
144     const float maxInstantaneousFrameRateCoefficient = 1.1;
145     float maxInstantaneousFrameRateThreshold = targetFrameRate_ * maxInstantaneousFrameRateCoefficient;
146     float instantaneousFrameRate = msPerSecond / recentFrameTimeSpanMs_;
147     if (instantaneousFrameRate < 0) {
148         instantaneousFrameRate = -instantaneousFrameRate;
149     }
150     if (instantaneousFrameRate <= maxInstantaneousFrameRateThreshold) {
151         frameRateCorrectionFactor_ = minDropFrmValue;
152     } else {
153         if (keepCorrectionCount_ >= VIDEO_FRAME_DROP_INTERVAL) {
154             frameRateCorrectionFactor_ = maxDropFrmValue;
155             keepCorrectionCount_ = 0;
156         } else {
157             frameRateCorrectionFactor_ = 0;
158             keepCorrectionCount_++;
159         }
160         DHLOGD("Frame control, instantaneousFrameRate %{public}.3f is more than maxInstantaneousFrameRateThreshold "
161             "%{public}.3f, keepCorrectionCount %{public}d", instantaneousFrameRate,
162             maxInstantaneousFrameRateThreshold, keepCorrectionCount_);
163     }
164 
165     DHLOGD("Frame control, targetFramerate %{public}d, maxInstantaneousFrameRateThreshold %{public}.3f,"
166         "instantaneousFrameRate %{public}.3f, frameRateCorrectionFactor %{public}.3f", targetFrameRate_,
167         maxInstantaneousFrameRateThreshold, instantaneousFrameRate, frameRateCorrectionFactor_);
168     return;
169 }
170 
UpdateIncomingFrameTimes(int64_t nowMs)171 void FpsControllerProcess::UpdateIncomingFrameTimes(int64_t nowMs)
172 {
173     DHLOGD("Frame control, update incoming frame times array.");
174     if (targetFrameRate_ <= 0) {
175         DHLOGD("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
176         return;
177     }
178     if (isFirstFrame_) {
179         incomingFrameTimesMs_[0] = nowMs;
180         return;
181     }
182 
183     int64_t intervalNewAndFirst = nowMs - incomingFrameTimesMs_[0];
184     if (intervalNewAndFirst < 0) {
185         intervalNewAndFirst = -intervalNewAndFirst;
186     }
187     if (intervalNewAndFirst > FRMAE_MAX_INTERVAL_TIME_WINDOW_MS) {
188         DHLOGD("frame control, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
189             "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
190         for (int i = 0; i < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; i++) {
191             incomingFrameTimesMs_[i] = 0;
192         }
193     } else {
194         DHLOGD("frame control shift, nowMs: %{public}lld mIncomingFrameT[0]: %{public}lld intervalNewAndFirst: "
195             "%{public}lld", (long long)nowMs, (long long)incomingFrameTimesMs_[0], (long long)intervalNewAndFirst);
196         const int32_t windowLeftNum =  2;
197         for (int i = (INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE - windowLeftNum); i >= 0; --i) {
198             incomingFrameTimesMs_[i + 1] = incomingFrameTimesMs_[i];
199         }
200     }
201     incomingFrameTimesMs_[0] = nowMs;
202     return;
203 }
204 
CalculateFrameRate(int64_t nowMs)205 float FpsControllerProcess::CalculateFrameRate(int64_t nowMs)
206 {
207     DHLOGD("Frame control, calculate frame rate.");
208     if (targetFrameRate_ <= 0) {
209         DHLOGE("Frame control, targetFrameRate_ : %{public}d", targetFrameRate_);
210         return 0.0;
211     }
212 
213     int32_t num = 0;
214     int32_t validFramesNumber = 0;
215     if (nowMs < 0) {
216         nowMs = -nowMs;
217     }
218     for (; num < INCOME_FRAME_TIME_HISTORY_WINDOWS_SIZE; num++) {
219         if (incomingFrameTimesMs_[num] <= 0 || nowMs - incomingFrameTimesMs_[num] > FRAME_HISTORY_TIME_WINDOWS_MS) {
220             break;
221         } else {
222             validFramesNumber++;
223         }
224     }
225 
226     const float msPerSecond = 1000;
227     const int32_t minValidCalculatedFrameRatesNum = 2;
228     int32_t minIncomingFrameNum = targetFrameRate_ / MIN_INCOME_FRAME_NUM_COEFFICIENT;
229     if (validFramesNumber > minIncomingFrameNum && validFramesNumber > minValidCalculatedFrameRatesNum) {
230         int64_t validTotalTimeInterval = (nowMs - incomingFrameTimesMs_[num - 1]);
231         if (validTotalTimeInterval < 0) {
232             validTotalTimeInterval = -validTotalTimeInterval;
233         }
234         if (validTotalTimeInterval > 0) {
235             return validFramesNumber * msPerSecond / validTotalTimeInterval + frameRateCorrectionFactor_;
236         }
237     }
238     return static_cast<float>(validFramesNumber);
239 }
240 
IsDropFrame(float incomingFps)241 bool FpsControllerProcess::IsDropFrame(float incomingFps)
242 {
243     DHLOGD("Frame control, IsDropFrame");
244     if (targetFrameRate_ == 0) {
245         DHLOGD("target fps is 0, drop all frame.");
246         return true;
247     }
248     if (incomingFps <= 0) {
249         DHLOGD("incoming fps not more than 0, not drop");
250         return false;
251     }
252     const int32_t incomingFrmRate = static_cast<int32_t>(incomingFps);
253     if (incomingFrmRate > targetFrameRate_) {
254         DHLOGD("incoming fps not more than targetFrameRate_, not drop");
255         return false;
256     }
257     bool isDrop = ReduceFrameRateByUniformStrategy(incomingFrmRate);
258     DHLOGD("drop frame result: %{public}s", isDrop ? "drop" : "no drop");
259     return isDrop;
260 }
261 
ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)262 bool FpsControllerProcess::ReduceFrameRateByUniformStrategy(int32_t incomingFrmRate)
263 {
264     DHLOGD("Frame control, reduce frame rate by uniform rate strategy");
265     if (incomingFrmRate > targetFrameRate_) {
266         DHLOGD("incoming fps not more than targetFrameRate_, not drop");
267         return false;
268     }
269 
270     /*
271      * When the actual incoming frame rate correction value is greater than the target frame
272      * rate, the incoming frames are reduced uniformly.
273      */
274     bool isDrop = false;
275     int32_t overshoot = frameRateOvershootMdf_ + (incomingFrmRate - targetFrameRate_);
276     if (overshoot < 0) {
277         overshoot = 0;
278         frameRateOvershootMdf_ = 0;
279     }
280     if (overshoot && DOUBLE_MULTIPLE * overshoot < incomingFrmRate) {
281         /*
282          * When the actual input frame rate is less than or equal to twice the target frame rate,
283          * one frame is dropped every (incomingFrmRate / overshoot) frames.
284          */
285         if (keepMoreThanDoubleCount_) {
286             keepMoreThanDoubleCount_ = 0;
287             return true;
288         }
289         const int32_t dropVar = incomingFrmRate / overshoot;
290         if (keepLessThanDoubleCount_ >= dropVar) {
291             isDrop = true;
292             frameRateOvershootMdf_ = -(incomingFrmRate % overshoot) / OVERSHOOT_MODIFY_COEFFICIENT;
293             keepLessThanDoubleCount_ = 1;
294         } else {
295             keepLessThanDoubleCount_++;
296         }
297     } else {
298         /*
299          * When the actual frame rate is more than twice the target frame rate or the overshoot is
300          * equal to 0, one frame is reserved every (overshoot / targetFrameRate_) frames.
301          */
302         keepLessThanDoubleCount_ = 0;
303         const int32_t dropVar = overshoot / targetFrameRate_;
304         if (keepMoreThanDoubleCount_ < dropVar) {
305             isDrop = true;
306             keepMoreThanDoubleCount_++;
307         } else {
308             frameRateOvershootMdf_ = overshoot % targetFrameRate_;
309             isDrop = false;
310             keepMoreThanDoubleCount_ = 0;
311         }
312     }
313     return isDrop;
314 }
315 
FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>> & outputBuffers)316 int32_t FpsControllerProcess::FpsControllerDone(std::vector<std::shared_ptr<DataBuffer>>& outputBuffers)
317 {
318     if (outputBuffers.empty()) {
319         DHLOGE("The received data buffers is empty.");
320         return DCAMERA_BAD_VALUE;
321     }
322 
323     if (nextDataProcess_ != nullptr) {
324         DHLOGD("Send to the next node of the FpsController for processing.");
325         int32_t err = nextDataProcess_->ProcessData(outputBuffers);
326         if (err != DCAMERA_OK) {
327             DHLOGE("Someone node after the FpsController processes failed.");
328         }
329         return err;
330     }
331     DHLOGD("The current node is the last node, and Output the processed video buffer");
332     std::shared_ptr<DCameraPipelineSource> targetPipelineSource = callbackPipelineSource_.lock();
333     if (targetPipelineSource == nullptr) {
334         DHLOGE("callbackPipelineSource_ is nullptr.");
335         return DCAMERA_BAD_VALUE;
336     }
337     targetPipelineSource->OnProcessedVideoBuffer(outputBuffers[0]);
338     return DCAMERA_OK;
339 }
340 
GetProperty(const std::string & propertyName,PropertyCarrier & propertyCarrier)341 int32_t FpsControllerProcess::GetProperty(const std::string& propertyName, PropertyCarrier& propertyCarrier)
342 {
343     return DCAMERA_OK;
344 }
345 } // namespace DistributedHardware
346 } // namespace OHOS
347