1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "image_effect_inner.h"
17 
18 #include <cassert>
19 #include <securec.h>
20 #include <algorithm>
21 #include <sync_fence.h>
22 
23 #include "metadata_helper.h"
24 #include "common_utils.h"
25 #include "filter_factory.h"
26 #include "image_sink_filter.h"
27 #include "image_source_filter.h"
28 #include "effect_surface_adapter.h"
29 #include "pipeline_core.h"
30 #include "json_helper.h"
31 #include "efilter_factory.h"
32 #include "external_loader.h"
33 #include "effect_context.h"
34 #include "colorspace_helper.h"
35 #include "memcpy_helper.h"
36 #include "render_task.h"
37 #include "render_environment.h"
38 #include "v1_1/buffer_handle_meta_key_type.h"
39 #include "effect_log.h"
40 #include "effect_trace.h"
41 #include "native_window.h"
42 
43 #define RENDER_QUEUE_SIZE 8
44 #define COMMON_TASK_TAG 0
45 namespace OHOS {
46 namespace Media {
47 namespace Effect {
48 using namespace OHOS::HDI::Display::Graphic::Common;
49 
50 enum class EffectState {
51     IDLE,
52     RUNNING,
53 };
54 
55 const int STRUCT_IMAGE_EFFECT_CONSTANT = 1;
56 const int DESTRUCTOR_IMAGE_EFFECT_CONSTANT = 2;
57 const int VIDEO_SINK_FILTER_STATUS = 3;
58 
59 class ImageEffect::Impl {
60 public:
Impl()61     Impl()
62     {
63         InitPipeline();
64         InitEffectContext();
65     }
66 
67     void CreatePipeline(std::vector<std::shared_ptr<EFilter>> &efilters);
68 private:
69     void InitPipeline();
70     void InitEffectContext();
71 
72 public:
73     std::unique_ptr<EffectSurfaceAdapter> surfaceAdapter_;
74     std::shared_ptr<PipelineCore> pipeline_;
75     std::shared_ptr<ImageSourceFilter> srcFilter_;
76     std::shared_ptr<ImageSinkFilter> sinkFilter_;
77     std::shared_ptr<EffectContext> effectContext_;
78     EffectState effectState_ = EffectState::IDLE;
79 };
80 
InitPipeline()81 void ImageEffect::Impl::InitPipeline()
82 {
83     srcFilter_ = FilterFactory::Instance().CreateFilterWithType<ImageSourceFilter>(GET_FILTER_NAME(ImageSourceFilter));
84     sinkFilter_ = FilterFactory::Instance().CreateFilterWithType<ImageSinkFilter>(GET_FILTER_NAME(ImageSinkFilter));
85     CHECK_AND_RETURN(srcFilter_ != nullptr);
86     CHECK_AND_RETURN(sinkFilter_ != nullptr);
87 }
88 
InitEffectContext()89 void ImageEffect::Impl::InitEffectContext()
90 {
91     effectContext_ = std::make_shared<EffectContext>();
92     effectContext_->memoryManager_ = std::make_shared<EffectMemoryManager>();
93     effectContext_->renderStrategy_ = std::make_shared<RenderStrategy>();
94     effectContext_->capNegotiate_ = std::make_shared<CapabilityNegotiate>();
95     effectContext_->renderEnvironment_ = std::make_shared<RenderEnvironment>();
96     effectContext_->colorSpaceManager_ = std::make_shared<ColorSpaceManager>();
97     effectContext_->metaInfoNegotiate_ = std::make_shared<EfilterMetaInfoNegotiate>();
98 }
99 
CreatePipeline(std::vector<std::shared_ptr<EFilter>> & efilters)100 void ImageEffect::Impl::CreatePipeline(std::vector<std::shared_ptr<EFilter>> &efilters)
101 {
102     pipeline_ = std::make_shared<PipelineCore>();
103     pipeline_->Init(nullptr);
104 
105     CHECK_AND_RETURN_LOG(srcFilter_ != nullptr, "srcFilter is null");
106     std::vector<Filter *> filtersToPipeline; // Note: Filters must be inserted in sequence.
107     filtersToPipeline.push_back(srcFilter_.get());
108     for (const auto &eFilter : efilters) {
109         filtersToPipeline.push_back(eFilter.get());
110     }
111     filtersToPipeline.push_back(sinkFilter_.get());
112 
113     ErrorCode res = pipeline_->AddFilters(filtersToPipeline);
114     CHECK_AND_RETURN_LOG(res == ErrorCode::SUCCESS, "pipeline add filters fail! res=%{public}d", res);
115 
116     res = pipeline_->LinkFilters(filtersToPipeline);
117     CHECK_AND_RETURN_LOG(res == ErrorCode::SUCCESS, "pipeline link filter fail! res=%{public}d", res);
118 }
119 
120 struct EffectParameters {
EffectParametersOHOS::Media::Effect::EffectParameters121     EffectParameters(std::shared_ptr<EffectBuffer> &srcEffectBuffer, std::shared_ptr<EffectBuffer> &dstEffectBuffer,
122         std::map<ConfigType, Plugin::Any> &config, std::shared_ptr<EffectContext> &effectContext)
123         : srcEffectBuffer_(std::move(srcEffectBuffer)),
124           dstEffectBuffer_(std::move(dstEffectBuffer)),
125           config_(std::move(config)),
126           effectContext_(std::move(effectContext)) {};
127     std::shared_ptr<EffectBuffer> &&srcEffectBuffer_;
128     std::shared_ptr<EffectBuffer> &&dstEffectBuffer_;
129     std::map<ConfigType, Plugin::Any> &&config_;
130     std::shared_ptr<EffectContext> &&effectContext_;
131 };
132 
133 enum class RunningType : int32_t {
134     DEFAULT = 0,
135     FOREGROUND = 1,
136     BACKGROUND = 2,
137 };
138 
139 const std::vector<std::string> priorityEFilter_ = {
140     "Crop"
141 };
142 const std::unordered_map<std::string, ConfigType> configTypeTab_ = {
143     { "runningType", ConfigType::IPTYPE },
144 };
145 const std::unordered_map<int32_t, std::vector<IPType>> runningTypeTab_{
146     { std::underlying_type<RunningType>::type(RunningType::FOREGROUND), { IPType::CPU, IPType::GPU } },
147     { std::underlying_type<RunningType>::type(RunningType::BACKGROUND), { IPType::CPU } },
148 };
149 
ImageEffect(const char * name)150 ImageEffect::ImageEffect(const char *name)
151 {
152     imageEffectFlag_ = STRUCT_IMAGE_EFFECT_CONSTANT;
153     impl_ = std::make_shared<Impl>();
154     if (name != nullptr) {
155         name_ = name;
156     }
157     ExternLoader::Instance()->InitExt();
158     ExtInitModule();
159 
160     if (m_renderThread == nullptr) {
161         auto func = [this]() {};
162         m_renderThread = new RenderThread<>(RENDER_QUEUE_SIZE, func);
163         m_renderThread->Start();
164     }
165 }
166 
~ImageEffect()167 ImageEffect::~ImageEffect()
168 {
169     EFFECT_LOGI("ImageEffect destruct enter!");
170     imageEffectFlag_ = DESTRUCTOR_IMAGE_EFFECT_CONSTANT;
171     impl_->surfaceAdapter_ = nullptr;
172     m_renderThread->ClearTask();
173     auto task = std::make_shared<RenderTask<>>([this]() { this->DestroyEGLEnv(); }, COMMON_TASK_TAG,
174         RequestTaskId());
175     m_renderThread->AddTask(task);
176     task->Wait();
177     EFFECT_LOGI("ImageEffect destruct destroy egl env!");
178     ExtDeinitModule();
179     m_renderThread->Stop();
180     delete m_renderThread;
181 
182     impl_->effectContext_->renderEnvironment_ = nullptr;
183     if (toProducerSurface_) {
184         auto res = toProducerSurface_->Disconnect();
185         EFFECT_LOGI("ImageEffect::~ImageEffect disconnect res=%{public}d, id=%{public}" PRIu64,
186             res, toProducerSurface_->GetUniqueId());
187         toProducerSurface_ = nullptr;
188     }
189     fromProducerSurface_ = nullptr;
190     impl_ = nullptr;
191     EFFECT_LOGI("ImageEffect destruct end!");
192 }
193 
AddEFilter(const std::shared_ptr<EFilter> & efilter)194 void ImageEffect::AddEFilter(const std::shared_ptr<EFilter> &efilter)
195 {
196     std::unique_lock<std::mutex> lock(innerEffectMutex);
197     auto priorityEFilter = std::find_if(priorityEFilter_.begin(), priorityEFilter_.end(),
198         [&efilter](const std::string &name) { return name.compare(efilter->GetName()) == 0; });
199     if (priorityEFilter == priorityEFilter_.end()) {
200         efilters_.emplace_back(efilter);
201     } else {
202         auto result =
203             std::find_if(efilters_.rbegin(), efilters_.rend(), [&priorityEFilter](std::shared_ptr<EFilter> &efilter) {
204                 return priorityEFilter->compare(efilter->GetName()) == 0;
205             });
206         if (result == efilters_.rend()) {
207             efilters_.insert(efilters_.begin(), efilter);
208         } else {
209             efilters_.insert(result.base(), efilter);
210         }
211     }
212 
213     impl_->CreatePipeline(efilters_);
214 }
215 
InsertEFilter(const std::shared_ptr<EFilter> & efilter,uint32_t index)216 ErrorCode ImageEffect::InsertEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index)
217 {
218     ErrorCode res = Effect::InsertEFilter(efilter, index);
219     if (res == ErrorCode::SUCCESS) {
220         impl_->CreatePipeline(efilters_);
221     }
222     return res;
223 }
224 
RemoveEFilter(const std::shared_ptr<EFilter> & efilter)225 void ImageEffect::RemoveEFilter(const std::shared_ptr<EFilter> &efilter)
226 {
227     Effect::RemoveEFilter(efilter);
228     impl_->CreatePipeline(efilters_);
229 }
230 
RemoveEFilter(uint32_t index)231 ErrorCode ImageEffect::RemoveEFilter(uint32_t index)
232 {
233     ErrorCode res = Effect::RemoveEFilter(index);
234     if (res == ErrorCode::SUCCESS) {
235         impl_->CreatePipeline(efilters_);
236     }
237     return res;
238 }
239 
ReplaceEFilter(const std::shared_ptr<EFilter> & efilter,uint32_t index)240 ErrorCode ImageEffect::ReplaceEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index)
241 {
242     std::unique_lock<std::mutex> lock(innerEffectMutex);
243     ErrorCode res = Effect::ReplaceEFilter(efilter, index);
244     if (res == ErrorCode::SUCCESS) {
245         impl_->CreatePipeline(efilters_);
246     }
247     return res;
248 }
249 
RequestTaskId()250 unsigned long int ImageEffect::RequestTaskId()
251 {
252     return m_currentTaskId.fetch_add(1);
253 }
254 
SetInputPixelMap(PixelMap * pixelMap)255 ErrorCode ImageEffect::SetInputPixelMap(PixelMap* pixelMap)
256 {
257     std::unique_lock<std::mutex> lock(innerEffectMutex);
258     EFFECT_LOGD("ImageEffect::SetInputPixelMap");
259     CHECK_AND_RETURN_RET_LOG(pixelMap != nullptr, ErrorCode::ERR_INVALID_SRC_PIXELMAP, "invalid source pixelMap");
260     impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
261 
262     ClearDataInfo(inDateInfo_);
263     inDateInfo_.dataType_ = DataType::PIXEL_MAP;
264     inDateInfo_.pixelMap_ = pixelMap;
265     return ErrorCode::SUCCESS;
266 }
267 
ConfigSourceFilter(std::shared_ptr<ImageSourceFilter> & srcFilter,std::shared_ptr<EffectBuffer> & srcBuffer,std::shared_ptr<EffectContext> & context)268 ErrorCode ConfigSourceFilter(std::shared_ptr<ImageSourceFilter> &srcFilter, std::shared_ptr<EffectBuffer> &srcBuffer,
269     std::shared_ptr<EffectContext> &context)
270 {
271     CHECK_AND_RETURN_RET_LOG(srcFilter != nullptr, ErrorCode::ERR_INPUT_NULL, "srcFilter is null");
272 
273     ErrorCode res = srcFilter->SetSource(srcBuffer, context);
274     FALSE_RETURN_MSG_E(res == ErrorCode::SUCCESS, res, "set source fail! res=%{public}d", res);
275 
276     return ErrorCode::SUCCESS;
277 }
278 
ConfigSinkFilter(std::shared_ptr<ImageSinkFilter> & sinkFilter,std::shared_ptr<EffectBuffer> & sinkBuffer)279 ErrorCode ConfigSinkFilter(std::shared_ptr<ImageSinkFilter> &sinkFilter, std::shared_ptr<EffectBuffer> &sinkBuffer)
280 {
281     CHECK_AND_RETURN_RET_LOG(sinkFilter != nullptr, ErrorCode::ERR_INPUT_NULL, "sinkFilter is null");
282 
283     ErrorCode res = sinkFilter->SetSink(sinkBuffer);
284     FALSE_RETURN_MSG_E(res == ErrorCode::SUCCESS, res, "set sink fail! res=%{public}d", res);
285 
286     return ErrorCode::SUCCESS;
287 }
288 
GetConfigIPTypes(const std::map<ConfigType,Plugin::Any> & config,std::vector<IPType> & configIPTypes)289 void GetConfigIPTypes(const std::map<ConfigType, Plugin::Any> &config, std::vector<IPType> &configIPTypes)
290 {
291     auto it = config.find(ConfigType::IPTYPE);
292     if (it == config.end()) {
293         EFFECT_LOGE("ipType config not set! use default config.");
294         configIPTypes = { IPType::CPU, IPType::GPU };
295         return;
296     }
297 
298     ErrorCode result = CommonUtils::ParseAny(it->second, configIPTypes);
299     if (result == ErrorCode::SUCCESS) {
300         return;
301     }
302     EFFECT_LOGE("parse ipType fail! use default config.");
303     configIPTypes = { IPType::CPU, IPType::GPU };
304 }
305 
ChooseIPType(const std::shared_ptr<EffectBuffer> & srcEffectBuffer,const std::shared_ptr<EffectContext> & context,const std::map<ConfigType,Plugin::Any> & config,IPType & runningIPType)306 ErrorCode ChooseIPType(const std::shared_ptr<EffectBuffer> &srcEffectBuffer,
307     const std::shared_ptr<EffectContext> &context, const std::map<ConfigType, Plugin::Any> &config,
308     IPType &runningIPType)
309 {
310     std::vector<IPType> configIPTypes;
311     GetConfigIPTypes(config, configIPTypes);
312 
313     runningIPType = IPType::DEFAULT;
314     IPType priorityIPType = IPType::GPU;
315     IEffectFormat effectFormat = srcEffectBuffer->bufferInfo_->formatType_;
316     const std::vector<std::shared_ptr<Capability>> &caps = context->capNegotiate_->GetCapabilityList();
317     for (const auto &capability : caps) {
318         if (capability == nullptr || capability->pixelFormatCap_ == nullptr) {
319             continue;
320         }
321         std::map<IEffectFormat, std::vector<IPType>> &formats = capability->pixelFormatCap_->formats;
322         if (runningIPType == IPType::GPU) {
323             effectFormat = IEffectFormat::RGBA8888;
324         }
325 
326         auto it = formats.find(effectFormat);
327         if (it == formats.end()) {
328             EFFECT_LOGE("effectFormat not support! effectFormat=%{public}d, name=%{public}s",
329                 effectFormat, capability->name_.c_str());
330             return ErrorCode::SUCCESS;
331         }
332 
333         std::vector<IPType> &ipTypes = it->second;
334         if (std::find(configIPTypes.begin(), configIPTypes.end(), priorityIPType) != configIPTypes.end() &&
335             std::find(ipTypes.begin(), ipTypes.end(), priorityIPType) != ipTypes.end()) {
336             runningIPType = IPType::GPU;
337         } else {
338             if (runningIPType == IPType::DEFAULT) {
339                 runningIPType = IPType::CPU;
340             }
341             return ErrorCode::SUCCESS;
342         }
343     }
344 
345     return ErrorCode::SUCCESS;
346 }
347 
StartPipelineInner(std::shared_ptr<PipelineCore> & pipeline,const EffectParameters & effectParameters,unsigned long int taskId,RenderThread<> * thread)348 ErrorCode StartPipelineInner(std::shared_ptr<PipelineCore> &pipeline, const EffectParameters &effectParameters,
349     unsigned long int taskId, RenderThread<> *thread)
350 {
351     if (thread == nullptr) {
352         EFFECT_LOGE("pipeline Prepare fail! render thread is nullptr");
353         return ErrorCode::ERR_INVALID_OPERATION;
354     }
355     auto prom = std::make_shared<std::promise<ErrorCode>>();
356     std::future<ErrorCode> fut = prom->get_future();
357     auto task = std::make_shared<RenderTask<>>([pipeline, &effectParameters, &prom]() {
358         ErrorCode res = pipeline->Prepare();
359         if (res != ErrorCode::SUCCESS) {
360             EFFECT_LOGE("pipeline Prepare fail! res=%{public}d", res);
361             prom->set_value(res);
362             return;
363         }
364 
365         res = ColorSpaceHelper::ConvertColorSpace(effectParameters.srcEffectBuffer_, effectParameters.effectContext_);
366         if (res != ErrorCode::SUCCESS) {
367             EFFECT_LOGE("StartPipelineInner:ConvertColorSpace fail! res=%{public}d", res);
368             prom->set_value(res);
369             return;
370         }
371 
372         IPType runningIPType;
373         res = ChooseIPType(effectParameters.srcEffectBuffer_, effectParameters.effectContext_, effectParameters.config_,
374             runningIPType);
375         if (res != ErrorCode::SUCCESS) {
376             EFFECT_LOGE("choose running ip type fail! res=%{public}d", res);
377             prom->set_value(res);
378             return;
379         }
380         if (effectParameters.effectContext_->renderEnvironment_->GetEGLStatus() != EGLStatus::READY
381             && runningIPType == IPType::GPU) {
382             effectParameters.effectContext_->renderEnvironment_->Init();
383             effectParameters.effectContext_->renderEnvironment_->Prepare();
384         }
385         effectParameters.effectContext_->ipType_ = runningIPType;
386         effectParameters.effectContext_->memoryManager_->SetIPType(runningIPType);
387 
388         res = pipeline->Start();
389         prom->set_value(res);
390         if (res != ErrorCode::SUCCESS) {
391             EFFECT_LOGE("pipeline start fail! res=%{public}d", res);
392             return;
393         }
394     }, 0, taskId);
395     thread->AddTask(task);
396     task->Wait();
397     ErrorCode res = fut.get();
398     return res;
399 }
400 
StartPipeline(std::shared_ptr<PipelineCore> & pipeline,const EffectParameters & effectParameters,unsigned long int taskId,RenderThread<> * thread)401 ErrorCode StartPipeline(std::shared_ptr<PipelineCore> &pipeline, const EffectParameters &effectParameters,
402     unsigned long int taskId, RenderThread<> *thread)
403 {
404     effectParameters.effectContext_->renderStrategy_->Init(effectParameters.srcEffectBuffer_,
405         effectParameters.dstEffectBuffer_);
406     effectParameters.effectContext_->colorSpaceManager_->Init(effectParameters.srcEffectBuffer_,
407         effectParameters.dstEffectBuffer_);
408     effectParameters.effectContext_->memoryManager_->Init(effectParameters.srcEffectBuffer_,
409         effectParameters.dstEffectBuffer_);
410     ErrorCode res = StartPipelineInner(pipeline, effectParameters, taskId, thread);
411     effectParameters.effectContext_->memoryManager_->Deinit();
412     effectParameters.effectContext_->colorSpaceManager_->Deinit();
413     effectParameters.effectContext_->renderStrategy_->Deinit();
414     effectParameters.effectContext_->capNegotiate_->ClearNegotiateResult();
415     return res;
416 }
417 
Start()418 ErrorCode ImageEffect::Start()
419 {
420     switch (inDateInfo_.dataType_) {
421         case DataType::PIXEL_MAP:
422         case DataType::SURFACE_BUFFER:
423         case DataType::URI:
424         case DataType::PATH:
425         case DataType::PICTURE: {
426             impl_->effectState_ = EffectState::RUNNING;
427             ErrorCode res = this->Render();
428             Stop();
429             return res;
430         }
431         case DataType::SURFACE:
432             impl_->effectState_ = EffectState::RUNNING;
433             if (impl_->surfaceAdapter_) {
434                 impl_->surfaceAdapter_->ConsumerRequestCpuAccess(true);
435             }
436             break;
437         default:
438             EFFECT_LOGE("Not set input data!");
439             return ErrorCode::ERR_NOT_SET_INPUT_DATA;
440     }
441     return ErrorCode::SUCCESS;
442 }
443 
Stop()444 void ImageEffect::Stop()
445 {
446     std::unique_lock<std::mutex> lock(innerEffectMutex);
447     impl_->effectState_ = EffectState::IDLE;
448     if (impl_->surfaceAdapter_) {
449         impl_->surfaceAdapter_->ConsumerRequestCpuAccess(false);
450     }
451     impl_->effectContext_->memoryManager_->ClearMemory();
452 }
453 
SetInputSurfaceBuffer(OHOS::SurfaceBuffer * surfaceBuffer)454 ErrorCode ImageEffect::SetInputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer)
455 {
456     CHECK_AND_RETURN_RET_LOG(surfaceBuffer != nullptr, ErrorCode::ERR_INVALID_SRC_SURFACEBUFFER,
457         "invalid source surface buffer");
458 
459     ClearDataInfo(inDateInfo_);
460     inDateInfo_.dataType_ = DataType::SURFACE_BUFFER;
461     inDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = surfaceBuffer;
462 
463     return ErrorCode::SUCCESS;
464 }
465 
SetOutputSurfaceBuffer(OHOS::SurfaceBuffer * surfaceBuffer)466 ErrorCode ImageEffect::SetOutputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer)
467 {
468     ClearDataInfo(outDateInfo_);
469     if (surfaceBuffer == nullptr) {
470         EFFECT_LOGI("SetOutputSurfaceBuffer: surfaceBuffer set to null!");
471         return ErrorCode::SUCCESS;
472     }
473 
474     outDateInfo_.dataType_ = DataType::SURFACE_BUFFER;
475     outDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = surfaceBuffer;
476 
477     return ErrorCode::SUCCESS;
478 }
479 
SetInputUri(const std::string & uri)480 ErrorCode ImageEffect::SetInputUri(const std::string &uri)
481 {
482     EFFECT_LOGD("ImageEffect::SetInputUri");
483     if (!CommonUtils::EndsWithJPG(uri)) {
484         EFFECT_LOGE("SetInputUri: file type is not support! only support jpg/jpeg.");
485         return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
486     }
487     ClearDataInfo(inDateInfo_);
488     inDateInfo_.dataType_ = DataType::URI;
489     inDateInfo_.uri_ = std::move(uri);
490 
491     return ErrorCode::SUCCESS;
492 }
493 
SetOutputUri(const std::string & uri)494 ErrorCode ImageEffect::SetOutputUri(const std::string &uri)
495 {
496     EFFECT_LOGD("ImageEffect::SetOutputUri");
497     if (uri.empty()) {
498         EFFECT_LOGI("SetOutputUri: uri set to null!");
499         ClearDataInfo(outDateInfo_);
500         return ErrorCode::SUCCESS;
501     }
502 
503     if (!CommonUtils::EndsWithJPG(uri)) {
504         EFFECT_LOGE("SetOutputUri: file type is not support! only support jpg/jpeg.");
505         return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
506     }
507     ClearDataInfo(outDateInfo_);
508     outDateInfo_.dataType_ = DataType::URI;
509     outDateInfo_.uri_ = std::move(uri);
510 
511     return ErrorCode::SUCCESS;
512 }
513 
SetInputPath(const std::string & path)514 ErrorCode ImageEffect::SetInputPath(const std::string &path)
515 {
516     EFFECT_LOGD("ImageEffect::SetInputPath");
517     if (!CommonUtils::EndsWithJPG(path)) {
518         EFFECT_LOGE("SetInputPath: file type is not support! only support jpg/jpeg.");
519         return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
520     }
521     ClearDataInfo(inDateInfo_);
522     inDateInfo_.dataType_ = DataType::PATH;
523     inDateInfo_.path_ = std::move(path);
524 
525     return ErrorCode::SUCCESS;
526 }
527 
SetOutputPath(const std::string & path)528 ErrorCode ImageEffect::SetOutputPath(const std::string &path)
529 {
530     EFFECT_LOGD("ImageEffect::SetOutputPath");
531     if (path.empty()) {
532         EFFECT_LOGI("SetOutputPath: path set to null!");
533         ClearDataInfo(outDateInfo_);
534         return ErrorCode::SUCCESS;
535     }
536 
537     if (!CommonUtils::EndsWithJPG(path)) {
538         EFFECT_LOGE("SetOutputPath: file type is not support! only support jpg/jpeg.");
539         return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
540     }
541     ClearDataInfo(outDateInfo_);
542     outDateInfo_.dataType_ = DataType::PATH;
543     outDateInfo_.path_ = std::move(path);
544 
545     return ErrorCode::SUCCESS;
546 }
547 
CheckToRenderPara(std::shared_ptr<EffectBuffer> & srcEffectBuffer,std::shared_ptr<EffectBuffer> & dstEffectBuffer)548 ErrorCode CheckToRenderPara(std::shared_ptr<EffectBuffer> &srcEffectBuffer,
549     std::shared_ptr<EffectBuffer> &dstEffectBuffer)
550 {
551     CHECK_AND_RETURN_RET_LOG(srcEffectBuffer != nullptr, ErrorCode::ERR_PARSE_FOR_EFFECT_BUFFER_FAIL,
552         "invalid srcEffectBuffer");
553 
554     // allow developers not to set the out parameter.
555     if (dstEffectBuffer == nullptr) {
556         return ErrorCode::SUCCESS;
557     }
558 
559     CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->bufferInfo_ != nullptr && dstEffectBuffer->bufferInfo_ != nullptr,
560         ErrorCode::ERR_BUFFER_INFO_NULL,
561         "buffer info is null! srcBufferInfo=%{public}d, dstBufferInfo=%{public}d",
562         srcEffectBuffer->bufferInfo_ == nullptr, dstEffectBuffer->bufferInfo_ == nullptr);
563     CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->extraInfo_ != nullptr && dstEffectBuffer->extraInfo_ != nullptr,
564         ErrorCode::ERR_EXTRA_INFO_NULL,
565         "extra info is null! srcExtraInfo=%{public}d, dstExtraInfo=%{public}d",
566         srcEffectBuffer->extraInfo_ == nullptr, dstEffectBuffer->extraInfo_ == nullptr);
567 
568     // input and output type is same or not.
569     DataType srcDataType = srcEffectBuffer->extraInfo_->dataType;
570     DataType dtsDataType = dstEffectBuffer->extraInfo_->dataType;
571     std::function<bool(DataType, DataType)> dataTypeCheckFunc = [](DataType srcDataType, DataType dstDataType) {
572         if (srcDataType == dstDataType) {
573             return true;
574         }
575         std::vector<std::pair<DataType, DataType>> extraSupportTab = {
576             { DataType::PIXEL_MAP, DataType::NATIVE_WINDOW },
577             { DataType::PICTURE, DataType::NATIVE_WINDOW },
578         };
579         return extraSupportTab.end() != std::find_if(extraSupportTab.begin(), extraSupportTab.end(),
580             [&srcDataType, &dstDataType](const std::pair<DataType, DataType> &data) {
581             return data.first == srcDataType && data.second == dstDataType;
582         });
583     };
584     CHECK_AND_RETURN_RET_LOG(dataTypeCheckFunc(srcDataType, dtsDataType), ErrorCode::ERR_NOT_SUPPORT_DIFF_DATATYPE,
585         "not supported dataType. srcDataType=%{public}d, dstDataType=%{public}d", srcDataType, dtsDataType);
586 
587     // color space is same or not.
588     if (srcDataType == DataType::PIXEL_MAP && dtsDataType != DataType::NATIVE_WINDOW) {
589         // the format for pixel map is same or not.
590         CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->bufferInfo_->formatType_ == dstEffectBuffer->bufferInfo_->formatType_,
591             ErrorCode::ERR_NOT_SUPPORT_DIFF_FORMAT,
592             "not support different format. srcFormat=%{public}d, dstFormat=%{public}d",
593             srcEffectBuffer->bufferInfo_->formatType_, dstEffectBuffer->bufferInfo_->formatType_);
594 
595         // color space is same or not.
596         EffectColorSpace srcColorSpace = srcEffectBuffer->bufferInfo_->colorSpace_;
597         EffectColorSpace dstColorSpace = dstEffectBuffer->bufferInfo_->colorSpace_;
598         bool isSrcHdr = ColorSpaceHelper::IsHdrColorSpace(srcColorSpace);
599         bool isDstHdr = ColorSpaceHelper::IsHdrColorSpace(dstColorSpace);
600         CHECK_AND_RETURN_RET_LOG(isSrcHdr == isDstHdr, ErrorCode::ERR_NOT_SUPPORT_INPUT_OUTPUT_COLORSPACE,
601             "not support different colorspace. src=%{public}d, dst=%{public}d", srcColorSpace, dstColorSpace);
602     }
603 
604     return ErrorCode::SUCCESS;
605 }
606 
Render()607 ErrorCode ImageEffect::Render()
608 {
609     if (efilters_.empty()) {
610         EFFECT_LOGW("efilters is empty!");
611         return ErrorCode::ERR_NOT_FILTERS_WITH_RENDER;
612     }
613 
614     std::shared_ptr<EffectBuffer> srcEffectBuffer = nullptr;
615     std::shared_ptr<EffectBuffer> dstEffectBuffer = nullptr;
616     ErrorCode res = LockAll(srcEffectBuffer, dstEffectBuffer);
617     if (res != ErrorCode::SUCCESS) {
618         UnLockAll();
619         return res;
620     }
621 
622     res = CheckToRenderPara(srcEffectBuffer, dstEffectBuffer);
623     if (res != ErrorCode::SUCCESS) {
624         UnLockAll();
625         return res;
626     }
627 
628     std::shared_ptr<ImageSourceFilter> &sourceFilter = impl_->srcFilter_;
629     res = ConfigSourceFilter(sourceFilter, srcEffectBuffer, impl_->effectContext_);
630     if (res != ErrorCode::SUCCESS) {
631         UnLockAll();
632         return res;
633     }
634 
635     std::shared_ptr<ImageSinkFilter> &sinkFilter = impl_->sinkFilter_;
636     res = ConfigSinkFilter(sinkFilter, dstEffectBuffer);
637     if (res != ErrorCode::SUCCESS) {
638         UnLockAll();
639         return res;
640     }
641     if (dstEffectBuffer != nullptr) {
642         impl_->effectContext_->renderEnvironment_->SetOutputType(dstEffectBuffer->extraInfo_->dataType);
643     } else {
644         impl_->effectContext_->renderEnvironment_->SetOutputType(srcEffectBuffer->extraInfo_->dataType);
645     }
646 
647     EffectParameters effectParameters(srcEffectBuffer, dstEffectBuffer, config_, impl_->effectContext_);
648     res = StartPipeline(impl_->pipeline_, effectParameters, RequestTaskId(), m_renderThread);
649     if (res != ErrorCode::SUCCESS) {
650         EFFECT_LOGE("StartPipeline fail! res=%{public}d", res);
651         UnLockAll();
652         return res;
653     }
654 
655     UnLockAll();
656     return res;
657 }
658 
Save(EffectJsonPtr & res)659 ErrorCode ImageEffect::Save(EffectJsonPtr &res)
660 {
661     EffectJsonPtr effect = JsonHelper::CreateArray();
662     for (auto it = efilters_.begin(); it != efilters_.end(); it++) {
663         EffectJsonPtr data = JsonHelper::CreateObject();
664         std::shared_ptr<EFilter> efilter = *it;
665         efilter->Save(data);
666         effect->Add(data);
667     }
668 
669     EffectJsonPtr info  = JsonHelper::CreateObject();
670     info->Put("filters", effect);
671     info->Put("name", name_);
672     if (extraInfo_ != nullptr) {
673         extraInfo_->Replace("imageEffect", info);
674         res = extraInfo_;
675     } else {
676         res->Put("imageEffect", info);
677     }
678     return ErrorCode::SUCCESS;
679 }
680 
Restore(std::string & info)681 std::shared_ptr<ImageEffect> ImageEffect::Restore(std::string &info)
682 {
683     const EffectJsonPtr root = JsonHelper::ParseJsonData(info);
684     CHECK_AND_RETURN_RET_LOG(root->HasElement("imageEffect"), nullptr, "Restore: no imageEffect");
685     const EffectJsonPtr &imageInfo = root->GetElement("imageEffect");
686     CHECK_AND_RETURN_RET_LOG(imageInfo != nullptr, nullptr, "Restore: imageInfo is null!");
687     CHECK_AND_RETURN_RET_LOG(imageInfo->HasElement("name"), nullptr, "Restore: imageEffect no name");
688     std::string effectName = imageInfo->GetString("name");
689     CHECK_AND_RETURN_RET_LOG(!effectName.empty(), nullptr, "Restore: imageEffect get name failed");
690 
691     CHECK_AND_RETURN_RET_LOG(imageInfo->HasElement("filters"), nullptr, "Restore: imageEffect no filters");
692     std::vector<EffectJsonPtr> efiltersInfo = imageInfo->GetArray("filters");
693     CHECK_AND_RETURN_RET_LOG(!efiltersInfo.empty(), nullptr, "Restore: filters not array");
694 
695     std::shared_ptr<ImageEffect> imageEffect = std::make_unique<ImageEffect>(effectName.c_str());
696     for (auto &efilterInfo : efiltersInfo) {
697         std::string name = efilterInfo->GetString("name");
698         CHECK_AND_CONTINUE_LOG(!name.empty(), "Restore: [name] not exist");
699         std::shared_ptr<EFilter> efilter = EFilterFactory::Instance()->Restore(name, efilterInfo, nullptr);
700         imageEffect->AddEFilter(efilter);
701     }
702     return imageEffect;
703 }
704 
SetOutputPixelMap(PixelMap * pixelMap)705 ErrorCode ImageEffect::SetOutputPixelMap(PixelMap* pixelMap)
706 {
707     std::unique_lock<std::mutex> lock(innerEffectMutex);
708     EFFECT_LOGD("ImageEffect::SetOutputPixelMap");
709     ClearDataInfo(outDateInfo_);
710     if (pixelMap == nullptr) {
711         EFFECT_LOGI("SetOutputPixelMap: pixelMap set to null!");
712         return ErrorCode::SUCCESS;
713     }
714 
715     outDateInfo_.dataType_ = DataType::PIXEL_MAP;
716     outDateInfo_.pixelMap_ = pixelMap;
717 
718     return ErrorCode::SUCCESS;
719 }
720 
SetOutputSurface(sptr<Surface> & surface)721 ErrorCode ImageEffect::SetOutputSurface(sptr<Surface>& surface)
722 {
723     std::unique_lock<std::mutex> lock(innerEffectMutex);
724     if (surface == nullptr) {
725         EFFECT_LOGE("surface is null.");
726         return ErrorCode::ERR_INPUT_NULL;
727     }
728     outDateInfo_.dataType_ = DataType::SURFACE;
729     toProducerSurface_ = surface;
730 
731     if (impl_->surfaceAdapter_ == nullptr) {
732         impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
733     }
734     impl_->surfaceAdapter_->SetOutputSurfaceDefaultUsage(toProducerSurface_->GetDefaultUsage());
735 
736     toProducerSurface_->SetTransform(GRAPHIC_ROTATE_BUTT);
737     return ErrorCode::SUCCESS;
738 }
739 
UpdateProducerSurfaceInfo()740 void ImageEffect::UpdateProducerSurfaceInfo()
741 {
742     if (impl_->surfaceAdapter_ == nullptr) {
743         EFFECT_LOGE("impl surfaceAdapter is nullptr!");
744         return;
745     }
746     auto transform = impl_->surfaceAdapter_->GetTransform();
747     if (transform == toProducerTransform_) {
748         return;
749     }
750     toProducerTransform_ = transform;
751     EFFECT_LOGI("Set toProducerSurface transform %{public}d, GRAPHIC_ROTATE_270: %{public}d",
752         transform, GRAPHIC_ROTATE_270);
753 
754     if (toProducerSurface_ == nullptr) {
755         EFFECT_LOGE("toProducerSurface_ is nullptr!");
756         return;
757     }
758     toProducerSurface_->SetTransform(transform);
759 }
760 
ConsumerBufferWithGPU(sptr<SurfaceBuffer> & buffer)761 void ImageEffect::ConsumerBufferWithGPU(sptr<SurfaceBuffer>& buffer)
762 {
763     inDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = buffer;
764     GraphicTransformType transform = impl_->surfaceAdapter_->GetTransform();
765     buffer->SetSurfaceBufferTransform(transform);
766     if (impl_->effectState_ == EffectState::RUNNING) {
767         impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
768         this->Render();
769     } else {
770         auto task = std::make_shared<RenderTask<>>([buffer, this, transform]() {
771             if (impl_->effectContext_->renderEnvironment_->GetEGLStatus() != EGLStatus::READY) {
772                 impl_->effectContext_->renderEnvironment_->Init();
773                 impl_->effectContext_->renderEnvironment_->Prepare();
774             }
775             int tex = GLUtils::CreateTextureFromSurfaceBuffer(buffer);
776             impl_->effectContext_->renderEnvironment_->UpdateCanvas();
777             impl_->effectContext_->renderEnvironment_->DrawFrame(tex, transform);
778         }, COMMON_TASK_TAG, RequestTaskId());
779         m_renderThread->AddTask(task);
780         task->Wait();
781     }
782 }
783 
MemoryCopyForSurfaceBuffer(sptr<SurfaceBuffer> & buffer,OHOS::sptr<SurfaceBuffer> & outBuffer)784 void MemoryCopyForSurfaceBuffer(sptr<SurfaceBuffer> &buffer, OHOS::sptr<SurfaceBuffer> &outBuffer)
785 {
786     CopyInfo src = {
787         .bufferInfo = {
788             .width_ = static_cast<uint32_t>(buffer->GetWidth()),
789             .height_ = static_cast<uint32_t>(buffer->GetHeight()),
790             .len_ = buffer->GetSize(),
791             .formatType_ = CommonUtils::SwitchToEffectFormat((GraphicPixelFormat)buffer->GetFormat()),
792             .rowStride_ = static_cast<uint32_t>(buffer->GetStride()),
793         },
794         .data = static_cast<uint8_t *>(buffer->GetVirAddr()),
795     };
796     CopyInfo dst = {
797         .bufferInfo = {
798             .width_ = static_cast<uint32_t>(outBuffer->GetWidth()),
799             .height_ = static_cast<uint32_t>(outBuffer->GetHeight()),
800             .len_ = outBuffer->GetSize(),
801             .formatType_ = CommonUtils::SwitchToEffectFormat((GraphicPixelFormat)outBuffer->GetFormat()),
802             .rowStride_ = static_cast<uint32_t>(outBuffer->GetStride()),
803         },
804         .data = static_cast<uint8_t *>(outBuffer->GetVirAddr()),
805     };
806     MemcpyHelper::CopyData(src, dst);
807 }
808 
IsSurfaceBufferHebc(sptr<SurfaceBuffer> & buffer)809 bool IsSurfaceBufferHebc(sptr<SurfaceBuffer> &buffer)
810 {
811     V1_1::BufferHandleAttrKey key = V1_1::BufferHandleAttrKey::ATTRKEY_ACCESS_TYPE;
812     std::vector<uint8_t> values;
813     auto res = buffer->GetMetadata(key, values);
814     CHECK_AND_RETURN_RET(res == 0, false);
815 
816     V1_1::HebcAccessType hebcAccessType = V1_1::HebcAccessType::HEBC_ACCESS_UNINIT;
817     res = MetadataHelper::ConvertVecToMetadata(values, hebcAccessType);
818     CHECK_AND_RETURN_RET(res == 0, false);
819 
820     if (hebcAccessType == V1_1::HEBC_ACCESS_HW_ONLY) {
821         EFFECT_LOGD("IsSurfaceBufferHebc: surface buffer is Hebc data!");
822         return true;
823     }
824     return false;
825 }
826 
SetSurfaceBufferHebcAccessType(sptr<SurfaceBuffer> & buffer,V1_1::HebcAccessType hebcAccessType)827 void SetSurfaceBufferHebcAccessType(sptr<SurfaceBuffer> &buffer, V1_1::HebcAccessType hebcAccessType)
828 {
829     std::vector<uint8_t> values;
830     auto res = MetadataHelper::ConvertMetadataToVec(hebcAccessType, values);
831     CHECK_AND_RETURN_LOG(res == 0, "SetSurfaceBufferHebcAccessType: ConvertVecToMetadata fail! res=%{public}d", res);
832 
833     V1_1::BufferHandleAttrKey key = V1_1::BufferHandleAttrKey::ATTRKEY_ACCESS_TYPE;
834     res = buffer->SetMetadata(key, values, false);
835     CHECK_AND_RETURN_LOG(res == 0, "SetSurfaceBufferHebcAccessType: SetMetadata fail! res=%{public}d", res);
836 }
837 
CopyMetaData(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer)838 void ImageEffect::CopyMetaData(sptr<SurfaceBuffer> &inBuffer, sptr<SurfaceBuffer> &outBuffer) {
839     std::vector<uint32_t> keys = {};
840     auto res = inBuffer->ListMetadataKeys(keys);
841     CHECK_AND_RETURN_LOG(res == GSError::GSERROR_OK, "CopyMetaData: ListMetadataKeys fail! res=%{public}d", res);
842     for (uint32_t key: keys) {
843         std::vector<uint8_t> values;
844         res = inBuffer->GetMetadata(key, values);
845         if (res != 0) {
846             EFFECT_LOGE("GetMetadata fail! key = %{public}d res = %{public}d", key, res);
847             continue;
848         }
849         auto isNeedUpdate = !(key == VIDEO_SINK_FILTER_STATUS) || !(values[0] == VIDEO_SINK_FILTER_STATUS);
850         impl_->effectContext_->metaInfoNegotiate_->SetNeedUpdate(isNeedUpdate);
851         res = outBuffer->SetMetadata(key, values);
852         if (res != 0) {
853             EFFECT_LOGE("SetMetadata fail! key = %{public}d res = %{public}d", key, res);
854             continue;
855         }
856     }
857 }
858 
OnBufferAvailableToProcess(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,int64_t timestamp,bool isNeedRender)859 bool ImageEffect::OnBufferAvailableToProcess(sptr<SurfaceBuffer> &inBuffer, sptr<SurfaceBuffer> &outBuffer,
860     int64_t timestamp, bool isNeedRender)
861 {
862     bool isNeedSwap = true;
863     if (isNeedRender) {
864         CopyMetaData(inBuffer, outBuffer);
865         inDateInfo_.surfaceBufferInfo_ = {
866             .surfaceBuffer_ = inBuffer,
867             .timestamp_ = timestamp,
868         };
869         outDateInfo_.surfaceBufferInfo_ = {
870             .surfaceBuffer_ = outBuffer,
871             .timestamp_ = timestamp,
872         };
873         impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
874         ErrorCode res = this->Render();
875         isNeedSwap  = (res != ErrorCode::SUCCESS);
876     }
877 
878     auto detRet = GSError::GSERROR_OK;
879     if (isNeedSwap) {
880         EFFECT_TRACE_BEGIN("OnBufferAvailableToProcess::SwapBuffers");
881         detRet = toProducerSurface_->DetachBufferFromQueue(outBuffer);
882         CHECK_AND_RETURN_RET_LOG(detRet == GSError::GSERROR_OK, true,
883                                  "OnBufferAvailableToProcess: detach buffer from producerSurface_ failed");
884         detRet = toProducerSurface_->AttachBufferToQueue(inBuffer);
885         CHECK_AND_RETURN_RET_LOG(detRet == GSError::GSERROR_OK, true,
886                                  "OnBufferAvailableToProcess: attach buffer from producerSurface_ failed");
887         EFFECT_TRACE_END();
888     }
889     return isNeedSwap;
890 }
891 
GetBufferRequestConfig(const sptr<SurfaceBuffer> & buffer)892 BufferRequestConfig ImageEffect::GetBufferRequestConfig(const sptr<SurfaceBuffer>& buffer)
893 {
894     return {
895         .width = buffer->GetWidth(),
896         .height = buffer->GetHeight(),
897         .strideAlignment = 0x8, // default stride is 8 Bytes.
898         .format = buffer->GetFormat(),
899         .usage = buffer->GetUsage(),
900         .timeout = 0,
901         .colorGamut = buffer->GetSurfaceBufferColorGamut(),
902         .transform = buffer->GetSurfaceBufferTransform(),
903     };
904 }
905 
FlushBuffer(sptr<SurfaceBuffer> & flushBuffer,int64_t timestamp,bool isNeedRender)906 void ImageEffect::FlushBuffer(sptr<SurfaceBuffer>& flushBuffer, int64_t timestamp, bool isNeedRender) {
907     if (isNeedRender) {
908         EFFECT_TRACE_BEGIN("FlushBuffer::FlushCache");
909         (void)flushBuffer->FlushCache();
910         EFFECT_TRACE_END();
911     }
912 
913     BufferFlushConfig flushConfig = {
914         .damage = {
915             .w = flushBuffer->GetWidth(),
916             .h = flushBuffer->GetHeight(),
917         },
918         .timestamp = timestamp,
919     };
920     CHECK_AND_RETURN_LOG(imageEffectFlag_ == STRUCT_IMAGE_EFFECT_CONSTANT,
921                          "ImageEffect::OnBufferAvailableWithCPU ImageEffect not exist.");
922     CHECK_AND_RETURN_LOG(toProducerSurface_ != nullptr,
923                          "ImageEffect::OnBufferAvailableWithCPU: toProducerSurface is nullptr.");
924     constexpr int32_t invalidFence = -1;
925     toProducerSurface_->FlushBuffer(flushBuffer, invalidFence, flushConfig);
926 }
927 
OnBufferAvailableWithCPU(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,const OHOS::Rect & damages,int64_t timestamp)928 bool ImageEffect::OnBufferAvailableWithCPU(sptr<SurfaceBuffer>& inBuffer, sptr<SurfaceBuffer>& outBuffer,
929                                            const OHOS::Rect& damages, int64_t timestamp)
930 {
931     CHECK_AND_RETURN_RET_LOG(inBuffer != nullptr, true, "ImageEffect::OnBufferAvailableWithCPU: inBuffer is nullptr.");
932     outDateInfo_.dataType_ = DataType::SURFACE;
933     UpdateProducerSurfaceInfo();
934 
935     sptr<SyncFence> syncFence = SyncFence::INVALID_FENCE;
936 
937     bool isSrcHebcData = IsSurfaceBufferHebc(inBuffer);
938     CHECK_AND_RETURN_RET_LOG(impl_ != nullptr, true, "OnBufferAvailableToProcess: impl is nullptr.");
939     bool isNeedRender = !isSrcHebcData && impl_->effectState_ == EffectState::RUNNING;
940 
941     if (isNeedRender) {
942         EFFECT_TRACE_BEGIN("inBuffer::InvalidateCache");
943         (void)inBuffer->InvalidateCache();
944         EFFECT_TRACE_END();
945     }
946 
947     auto requestConfig = GetBufferRequestConfig(inBuffer);
948 
949     CHECK_AND_RETURN_RET_LOG(toProducerSurface_ != nullptr, true,
950         "OnBufferAvailableWithCPU: toProducerSurface is nullptr.");
951     auto ret = toProducerSurface_->RequestBuffer(outBuffer, syncFence, requestConfig);
952     CHECK_AND_RETURN_RET_LOG(ret == 0 && outBuffer != nullptr, true, "RequestBuffer failed. %{public}d", ret);
953 
954     constexpr uint32_t waitForEver = -1;
955     (void)syncFence->Wait(waitForEver);
956 
957     EFFECT_LOGD("inBuffer: w=%{public}d h=%{public}d stride=%{public}d len=%{public}d usage=%{public}lld",
958         inBuffer->GetWidth(), inBuffer->GetHeight(), inBuffer->GetStride(), inBuffer->GetSize(),
959         static_cast<unsigned long long>(inBuffer->GetUsage()));
960     EFFECT_LOGD("outBuffer: w=%{public}d h=%{public}d stride=%{public}d len=%{public}d usage=%{public}lld",
961         outBuffer->GetWidth(), outBuffer->GetHeight(), outBuffer->GetStride(), outBuffer->GetSize(),
962         static_cast<unsigned long long>(outBuffer->GetUsage()));
963 
964     SetSurfaceBufferHebcAccessType(outBuffer,
965         isSrcHebcData ? V1_1::HebcAccessType::HEBC_ACCESS_HW_ONLY : V1_1::HebcAccessType::HEBC_ACCESS_CPU_ACCESS);
966     bool isNeedSwap = OnBufferAvailableToProcess(inBuffer, outBuffer, timestamp, isNeedRender);
967 
968     auto flushBuffer = (isNeedSwap ? inBuffer : outBuffer);
969     FlushBuffer(flushBuffer, timestamp, isNeedRender);
970 
971     return isNeedSwap;
972 }
973 
ConsumerBufferAvailable(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,const OHOS::Rect & damages,int64_t timestamp)974 bool ImageEffect::ConsumerBufferAvailable(sptr<SurfaceBuffer>& inBuffer, sptr<SurfaceBuffer>& outBuffer,
975     const OHOS::Rect& damages, int64_t timestamp)
976 {
977     CHECK_AND_RETURN_RET_LOG(imageEffectFlag_ == STRUCT_IMAGE_EFFECT_CONSTANT, true,
978         "ImageEffect::OnBufferAvailable ImageEffect not exist.");
979     std::unique_lock<std::mutex> lock(innerEffectMutex);
980     return OnBufferAvailableWithCPU(inBuffer, outBuffer, damages, timestamp);
981 }
982 
GetInputSurface()983 sptr<Surface> ImageEffect::GetInputSurface()
984 {
985     inDateInfo_.dataType_ = DataType::SURFACE;
986     if (fromProducerSurface_ != nullptr) {
987         return fromProducerSurface_;
988     }
989 
990     if (impl_->surfaceAdapter_ == nullptr) {
991         impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
992     }
993 
994     if (impl_->surfaceAdapter_) {
995         fromProducerSurface_ = impl_->surfaceAdapter_->GetProducerSurface();
996     }
997 
998     auto consumerListener = [this](sptr<SurfaceBuffer>& inBuffer,
999         sptr<SurfaceBuffer>& outBuffer, const OHOS::Rect& damages, int64_t timestamp) {
1000         return ConsumerBufferAvailable(inBuffer, outBuffer, damages, timestamp);
1001     };
1002 
1003     if (impl_->surfaceAdapter_) {
1004         impl_->surfaceAdapter_->SetConsumerListener(std::move(consumerListener));
1005     }
1006 
1007     return fromProducerSurface_;
1008 }
1009 
SetOutNativeWindow(OHNativeWindow * nativeWindow)1010 ErrorCode ImageEffect::SetOutNativeWindow(OHNativeWindow *nativeWindow)
1011 {
1012     CHECK_AND_RETURN_RET_LOG(nativeWindow != nullptr, ErrorCode::ERR_INPUT_NULL, "nativeWindow is nullptr");
1013     OHOS::sptr<OHOS::Surface> surface = nativeWindow->surface;
1014     CHECK_AND_RETURN_RET_LOG(surface != nullptr, ErrorCode::ERR_INPUT_NULL, "surface is nullptr");
1015     toProducerSurface_ = surface;
1016     outDateInfo_.dataType_ = DataType::NATIVE_WINDOW;
1017     impl_->effectContext_->renderEnvironment_->InitEngine(nativeWindow);
1018     if (impl_->surfaceAdapter_ == nullptr) {
1019         impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
1020     }
1021     impl_->surfaceAdapter_->SetOutputSurfaceDefaultUsage(toProducerSurface_->GetDefaultUsage());
1022 
1023     toProducerSurface_->SetTransform(GRAPHIC_ROTATE_BUTT);
1024     return ErrorCode::SUCCESS;
1025 }
1026 
Configure(const std::string & key,const Plugin::Any & value)1027 ErrorCode ImageEffect::Configure(const std::string &key, const Plugin::Any &value)
1028 {
1029     auto configTypeIt = std::find_if(configTypeTab_.begin(), configTypeTab_.end(),
1030         [&key](const std::pair<std::string, ConfigType> &item) { return item.first.compare(key) == 0; });
1031 
1032     CHECK_AND_RETURN_RET_LOG(configTypeIt != configTypeTab_.end(), ErrorCode::ERR_UNSUPPORTED_CONFIG_TYPE,
1033         "config key is not support! key=%{public}s", key.c_str());
1034 
1035     ConfigType configType = configTypeIt->second;
1036     switch (configType) {
1037         case ConfigType::IPTYPE: {
1038             int32_t runningType;
1039             ErrorCode result = CommonUtils::ParseAny(value, runningType);
1040             CHECK_AND_RETURN_RET_LOG(result == ErrorCode::SUCCESS, result,
1041                 "parse any fail! expect type is uint32_t! key=%{public}s", key.c_str());
1042             auto it = std::find_if(runningTypeTab_.begin(), runningTypeTab_.end(),
1043                 [&runningType](const std::pair<int32_t, std::vector<IPType>> &item) {
1044                     return item.first == runningType;
1045                 });
1046             CHECK_AND_RETURN_RET_LOG(it != runningTypeTab_.end(), ErrorCode::ERR_UNSUPPORTED_RUNNINGTYPE,
1047                 "not support runningType! key=%{public}s, runningType=%{public}d", key.c_str(), runningType);
1048             config_[configType] = it->second;
1049             break;
1050         }
1051         default:
1052             EFFECT_LOGE("config type is not support! configType=%{public}d", configType);
1053             return ErrorCode::ERR_UNSUPPORTED_CONFIG_TYPE;
1054     }
1055     return ErrorCode::SUCCESS;
1056 }
1057 
ClearDataInfo(DataInfo & dataInfo)1058 void ImageEffect::ClearDataInfo(DataInfo &dataInfo)
1059 {
1060     dataInfo.dataType_ = DataType::UNKNOWN;
1061     dataInfo.pixelMap_ = nullptr;
1062     dataInfo.surfaceBufferInfo_.surfaceBuffer_ = nullptr;
1063     dataInfo.surfaceBufferInfo_.timestamp_ = 0;
1064     dataInfo.uri_ = "";
1065     dataInfo.path_ = "";
1066 }
1067 
IsSameInOutputData(const DataInfo & inDataInfo,const DataInfo & outDataInfo)1068 bool IsSameInOutputData(const DataInfo &inDataInfo, const DataInfo &outDataInfo)
1069 {
1070     if (inDataInfo.dataType_ != outDataInfo.dataType_) {
1071         return false;
1072     }
1073 
1074     switch (inDataInfo.dataType_) {
1075         case DataType::PIXEL_MAP:
1076             return inDataInfo.pixelMap_ == outDataInfo.pixelMap_;
1077         case DataType::SURFACE_BUFFER:
1078             return inDataInfo.surfaceBufferInfo_.surfaceBuffer_ == outDataInfo.surfaceBufferInfo_.surfaceBuffer_;
1079         case DataType::PATH:
1080             return inDataInfo.path_ == outDataInfo.path_;
1081         case DataType::URI:
1082             return inDataInfo.uri_ == outDataInfo.uri_;
1083         case DataType::PICTURE:
1084             return inDataInfo.picture_ == outDataInfo.picture_;
1085         default:
1086             return false;
1087     }
1088 }
1089 
LockAll(std::shared_ptr<EffectBuffer> & srcEffectBuffer,std::shared_ptr<EffectBuffer> & dstEffectBuffer)1090 ErrorCode ImageEffect::LockAll(std::shared_ptr<EffectBuffer> &srcEffectBuffer,
1091     std::shared_ptr<EffectBuffer> &dstEffectBuffer)
1092 {
1093     ErrorCode res = ParseDataInfo(inDateInfo_, srcEffectBuffer, false);
1094     if (res != ErrorCode::SUCCESS) {
1095         EFFECT_LOGE("ParseDataInfo inData fail! res=%{public}d", res);
1096         return res;
1097     }
1098     EFFECT_LOGI("input data set, parse data info success! dataType=%{public}d", inDateInfo_.dataType_);
1099 
1100     if (outDateInfo_.dataType_ != DataType::UNKNOWN && !IsSameInOutputData(inDateInfo_, outDateInfo_)) {
1101         EFFECT_LOGI("output data set, start parse data info. dataType=%{public}d", outDateInfo_.dataType_);
1102         res = ParseDataInfo(outDateInfo_, dstEffectBuffer, true);
1103         if (res != ErrorCode::SUCCESS) {
1104             EFFECT_LOGE("ParseDataInfo outData fail! res=%{public}d", res);
1105             return res;
1106         }
1107         EFFECT_LOGI("output data set, parse data info success! dataType=%{public}d", outDateInfo_.dataType_);
1108     }
1109 
1110     return ErrorCode::SUCCESS;
1111 }
1112 
ParseDataInfo(DataInfo & dataInfo,std::shared_ptr<EffectBuffer> & effectBuffer,bool isOutputData)1113 ErrorCode ImageEffect::ParseDataInfo(DataInfo &dataInfo, std::shared_ptr<EffectBuffer> &effectBuffer,
1114     bool isOutputData)
1115 {
1116     switch (dataInfo.dataType_) {
1117         case DataType::PIXEL_MAP:
1118             return CommonUtils::LockPixelMap(dataInfo.pixelMap_, effectBuffer);
1119         case DataType::SURFACE:
1120         case DataType::SURFACE_BUFFER:
1121             return CommonUtils::ParseSurfaceData(dataInfo.surfaceBufferInfo_.surfaceBuffer_, effectBuffer,
1122                 dataInfo.dataType_, dataInfo.surfaceBufferInfo_.timestamp_);
1123         case DataType::URI:
1124             return CommonUtils::ParseUri(dataInfo.uri_, effectBuffer, isOutputData);
1125         case DataType::PATH:
1126             return CommonUtils::ParsePath(dataInfo.path_, effectBuffer, isOutputData);
1127         case DataType::NATIVE_WINDOW:
1128             return CommonUtils::ParseNativeWindowData(effectBuffer, dataInfo.dataType_);
1129         case DataType::PICTURE:
1130             return CommonUtils::ParsePicture(dataInfo.picture_, effectBuffer);
1131         case DataType::UNKNOWN:
1132             EFFECT_LOGW("dataType is unknown! Data is not set!");
1133             return ErrorCode::ERR_NO_DATA;
1134         default:
1135             EFFECT_LOGW("dataType is not support! dataType=%{public}d", dataInfo.dataType_);
1136             return ErrorCode::ERR_UNSUPPORTED_DATA_TYPE;
1137     }
1138 }
1139 
UnLockAll()1140 void ImageEffect::UnLockAll()
1141 {
1142     UnLockData(inDateInfo_);
1143     UnLockData(outDateInfo_);
1144 }
1145 
UnLockData(DataInfo & dataInfo)1146 void ImageEffect::UnLockData(DataInfo &dataInfo)
1147 {
1148     switch (dataInfo.dataType_) {
1149         case DataType::PIXEL_MAP: {
1150             CommonUtils::UnlockPixelMap(dataInfo.pixelMap_);
1151             return;
1152         }
1153         default:
1154             return;
1155     }
1156 }
1157 
InitEGLEnv()1158 void ImageEffect::InitEGLEnv()
1159 {
1160     EFFECT_TRACE_NAME("ImageEffect::InitEGLEnv");
1161     impl_->effectContext_->renderEnvironment_->Init();
1162     impl_->effectContext_->renderEnvironment_->Prepare();
1163 }
1164 
DestroyEGLEnv()1165 void ImageEffect::DestroyEGLEnv()
1166 {
1167     EFFECT_LOGI("ImageEffect DestroyEGLEnv enter!");
1168     if (impl_->effectContext_->renderEnvironment_ == nullptr) {
1169         return;
1170     }
1171     impl_->effectContext_->renderEnvironment_->ReleaseParam();
1172     impl_->effectContext_->renderEnvironment_->Release();
1173     EFFECT_LOGI("ImageEffect DestroyEGLEnv end!");
1174 }
1175 
SetExtraInfo(EffectJsonPtr res)1176 ErrorCode ImageEffect::SetExtraInfo(EffectJsonPtr res)
1177 {
1178     extraInfo_ = res;
1179     return ErrorCode::SUCCESS;
1180 }
1181 
ExtInitModule()1182 void ImageEffect::ExtInitModule()
1183 {
1184 }
1185 
ExtDeinitModule()1186 void ImageEffect::ExtDeinitModule()
1187 {
1188 }
1189 
SetInputPicture(Picture * picture)1190 ErrorCode ImageEffect::SetInputPicture(Picture *picture)
1191 {
1192     EFFECT_LOGD("ImageEffect::SetInputPicture");
1193     CHECK_AND_RETURN_RET_LOG(picture != nullptr, ErrorCode::ERR_INPUT_NULL,
1194         "ImageEffect::SetInputPicture: picture is null!");
1195 
1196     impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
1197     ClearDataInfo(inDateInfo_);
1198     inDateInfo_.dataType_ = DataType::PICTURE;
1199     inDateInfo_.picture_ = picture;
1200 
1201     return ErrorCode::SUCCESS;
1202 }
1203 
SetOutputPicture(Picture * picture)1204 ErrorCode ImageEffect::SetOutputPicture(Picture *picture)
1205 {
1206     EFFECT_LOGD("ImageEffect::SetOutputPicture");
1207     ClearDataInfo(outDateInfo_);
1208     if (picture == nullptr) {
1209         EFFECT_LOGI("SetOutputPicture:picture set to null!");
1210         return ErrorCode::SUCCESS;
1211     }
1212     outDateInfo_.dataType_ = DataType::PICTURE;
1213     outDateInfo_.picture_ = picture;
1214 
1215     return ErrorCode::SUCCESS;
1216 }
1217 } // namespace Effect
1218 } // namespace Media
1219 } // namespace OHOS