1 /*
2  * Copyright (c) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define GL_GLEXT_PROTOTYPES
17 #define EGL_EGLEXT_PROTOTYPES
18 
19 #include "surface_image.h"
20 
21 #include "securec.h"
22 #include "sandbox_utils.h"
23 #include "surface_utils.h"
24 
25 #include <cinttypes>
26 #include <atomic>
27 #include <sync_fence.h>
28 #include <unistd.h>
29 #include <window.h>
30 
31 #include <EGL/egl.h>
32 #include <EGL/eglext.h>
33 #include <GLES/gl.h>
34 #include <GLES/glext.h>
35 
36 namespace OHOS {
37 namespace {
38 // Get a uniqueID in a process
GetProcessUniqueId()39 static int GetProcessUniqueId()
40 {
41     static std::atomic<int> g_counter { 0 };
42     return g_counter.fetch_add(1);
43 }
44 }
45 
SurfaceImage(uint32_t textureId,uint32_t textureTarget)46 SurfaceImage::SurfaceImage(uint32_t textureId, uint32_t textureTarget)
47     : ConsumerSurface("SurfaceImage-" + std::to_string(GetRealPid()) + "-" + std::to_string(GetProcessUniqueId())),
48       textureId_(textureId),
49       textureTarget_(textureTarget),
50       updateSurfaceImage_(false),
51       eglDisplay_(EGL_NO_DISPLAY),
52       eglContext_(EGL_NO_CONTEXT),
53       currentSurfaceImage_(UINT_MAX),
54       currentSurfaceBuffer_(nullptr),
55       currentTimeStamp_(0)
56 {
57     InitSurfaceImage();
58 }
59 
SurfaceImage()60 SurfaceImage::SurfaceImage()
61     : ConsumerSurface("SurfaceImageConsumer-" + std::to_string(GetRealPid()) +
62     "-" + std::to_string(GetProcessUniqueId())),
63       currentSurfaceBuffer_(nullptr),
64       currentTimeStamp_(0)
65 {
66     InitSurfaceImage();
67 }
68 
~SurfaceImage()69 SurfaceImage::~SurfaceImage()
70 {
71     for (auto it = imageCacheSeqs_.begin(); it != imageCacheSeqs_.end(); it++) {
72         DestroyEGLImage(it->second.eglImage_);
73         DestroyEGLSync(it->second.eglSync_);
74     }
75 }
76 
InitSurfaceImage()77 void SurfaceImage::InitSurfaceImage()
78 {
79     std::string name = "SurfaceImage-" + std::to_string(GetRealPid()) + "-" + std::to_string(GetProcessUniqueId());
80     auto ret = ConsumerSurface::Init();
81     uniqueId_ = GetUniqueId();
82     BLOGI("InitSurfaceImage Init ret: %{public}d, uniqueId: %{public}" PRIu64 ".", ret, uniqueId_);
83     surfaceImageName_ = name;
84 }
85 
UpdateSurfaceInfo(uint32_t seqNum,sptr<SurfaceBuffer> buffer,const sptr<SyncFence> & acquireFence,int64_t timestamp,Rect damage)86 void SurfaceImage::UpdateSurfaceInfo(uint32_t seqNum, sptr<SurfaceBuffer> buffer, const sptr<SyncFence> &acquireFence,
87                                      int64_t timestamp, Rect damage)
88 {
89     // release old buffer
90     int releaseFence = -1;
91     auto iter = imageCacheSeqs_.find(currentSurfaceImage_);
92     if (iter != imageCacheSeqs_.end() && iter->second.eglSync_ != EGL_NO_SYNC_KHR) {
93         // PLANNING: use eglDupNativeFenceFDOHOS in the future.
94         releaseFence = eglDupNativeFenceFDANDROID(eglDisplay_, iter->second.eglSync_);
95     }
96     // There is no need to close this fd, because in function ReleaseBuffer it will be closed.
97     ReleaseBuffer(currentSurfaceBuffer_, releaseFence);
98 
99     currentSurfaceImage_ = seqNum;
100     currentSurfaceBuffer_ = buffer;
101     currentTimeStamp_ = timestamp;
102     currentCrop_ = damage;
103     currentTransformType_ = ConsumerSurface::GetTransform();
104     auto utils = SurfaceUtils::GetInstance();
105     utils->ComputeTransformMatrix(currentTransformMatrix_, TRANSFORM_MATRIX_ELE_COUNT,
106         currentSurfaceBuffer_, currentTransformType_, currentCrop_);
107     utils->ComputeTransformMatrixV2(currentTransformMatrixV2_, TRANSFORM_MATRIX_ELE_COUNT,
108         currentSurfaceBuffer_, currentTransformType_, currentCrop_);
109 
110     // wait on this acquireFence.
111     if (acquireFence != nullptr) {
112         acquireFence->Wait(-1);
113     }
114 }
115 
UpdateSurfaceImage()116 SurfaceError SurfaceImage::UpdateSurfaceImage()
117 {
118     std::lock_guard<std::mutex> lockGuard(opMutex_);
119 
120     // validate egl state
121     SurfaceError ret = ValidateEglState();
122     if (ret != SURFACE_ERROR_OK) {
123         return ret;
124     }
125 
126     // acquire buffer
127     sptr<SurfaceBuffer> buffer = nullptr;
128     sptr<SyncFence> acquireFence = SyncFence::InvalidFence();
129     int64_t timestamp;
130     Rect damage;
131     ret = AcquireBuffer(buffer, acquireFence, timestamp, damage);
132     if (ret != SURFACE_ERROR_OK) {
133         return ret;
134     }
135 
136     ret = UpdateEGLImageAndTexture(buffer);
137     if (ret != SURFACE_ERROR_OK) {
138         ReleaseBuffer(buffer, -1);
139         return ret;
140     }
141 
142     uint32_t seqNum = buffer->GetSeqNum();
143     UpdateSurfaceInfo(seqNum, buffer, acquireFence, timestamp, damage);
144     return SURFACE_ERROR_OK;
145 }
146 
AttachContext(uint32_t textureId)147 SurfaceError SurfaceImage::AttachContext(uint32_t textureId)
148 {
149     std::lock_guard<std::mutex> lockGuard(opMutex_);
150     // validate egl state
151     SurfaceError ret = ValidateEglState();
152     if (ret != SURFACE_ERROR_OK) {
153         return ret;
154     }
155 
156     textureId_ = textureId;
157     auto iter = imageCacheSeqs_.find(currentSurfaceImage_);
158     if (iter != imageCacheSeqs_.end()) {
159         glBindTexture(textureTarget_, textureId);
160         GLenum error = glGetError();
161         if (error != GL_NO_ERROR) {
162             BLOGE("glBindTexture failed, textureTarget:%{public}d, textureId_:%{public}d, error:%{public}d,"
163                 "uniqueId: %{public}" PRIu64 ".", textureTarget_, textureId_, error, uniqueId_);
164             return SURFACE_ERROR_EGL_API_FAILED;
165         }
166         glEGLImageTargetTexture2DOES(textureTarget_, static_cast<GLeglImageOES>(iter->second.eglImage_));
167         error = glGetError();
168         if (error != GL_NO_ERROR) {
169             BLOGE("glEGLImageTargetTexture2DOES failed, textureTarget:%{public}d, error:%{public}d"
170                 "uniqueId: %{public}" PRIu64 ".", textureTarget_, error, uniqueId_);
171             return SURFACE_ERROR_EGL_API_FAILED;
172         }
173     }
174 
175     // If there is no EGLImage, we cannot simply return an error.
176     // Developers can call OH_NativeImage_UpdateSurfaceImage later to achieve their purpose.
177     return SURFACE_ERROR_OK;
178 }
179 
DetachContext()180 SurfaceError SurfaceImage::DetachContext()
181 {
182     std::lock_guard<std::mutex> lockGuard(opMutex_);
183     // validate egl state
184     SurfaceError ret = ValidateEglState();
185     if (ret != SURFACE_ERROR_OK) {
186         return ret;
187     }
188 
189     textureId_ = 0;
190     glBindTexture(textureTarget_, 0);
191     GLenum error = glGetError();
192     if (error != GL_NO_ERROR) {
193         BLOGE("glBindTexture failed, textureTarget:%{public}d, textureId:%{public}d, error:%{public}d"
194             "uniqueId: %{public}" PRIu64 ".", textureTarget_, textureId_, error, uniqueId_);
195         return SURFACE_ERROR_EGL_API_FAILED;
196     }
197     return SURFACE_ERROR_OK;
198 }
199 
GetTimeStamp()200 int64_t SurfaceImage::GetTimeStamp()
201 {
202     std::lock_guard<std::mutex> lockGuard(opMutex_);
203     return currentTimeStamp_;
204 }
205 
GetTransformMatrix(float matrix[16])206 SurfaceError SurfaceImage::GetTransformMatrix(float matrix[16])
207 {
208     std::lock_guard<std::mutex> lockGuard(opMutex_);
209     auto ret = memcpy_s(matrix, sizeof(float) * 16,  // 16 is the length of array
210                         currentTransformMatrix_, sizeof(currentTransformMatrix_));
211     if (ret != EOK) {
212         BLOGE("memcpy_s failed ret: %{public}d, uniqueId: %{public}" PRIu64 ".", ret, uniqueId_);
213         return SURFACE_ERROR_UNKOWN;
214     }
215     return SURFACE_ERROR_OK;
216 }
217 
GetTransformMatrixV2(float matrix[16])218 SurfaceError SurfaceImage::GetTransformMatrixV2(float matrix[16])
219 {
220     std::lock_guard<std::mutex> lockGuard(opMutex_);
221     auto ret = memcpy_s(matrix, sizeof(float) * 16, // 16 is the length of array
222                         currentTransformMatrixV2_, sizeof(currentTransformMatrixV2_));
223     if (ret != EOK) {
224         BLOGE("memcpy_s failed ret: %{public}d, uniqueId: %{public}" PRIu64 ".", ret, uniqueId_);
225         return SURFACE_ERROR_UNKOWN;
226     }
227     return SURFACE_ERROR_OK;
228 }
229 
ValidateEglState()230 SurfaceError SurfaceImage::ValidateEglState()
231 {
232     EGLDisplay disp = eglGetCurrentDisplay();
233     EGLContext context = eglGetCurrentContext();
234 
235     if ((eglDisplay_ != disp && eglDisplay_ != EGL_NO_DISPLAY) || (disp == EGL_NO_DISPLAY)) {
236         BLOGE("EGLDisplay is invalid, errno : 0x%{public}x, uniqueId: %{public}" PRIu64 ".",
237             eglGetError(), uniqueId_);
238         return SURFACE_ERROR_EGL_STATE_UNKONW;
239     }
240     if ((eglContext_ != context && eglContext_ != EGL_NO_CONTEXT) || (context == EGL_NO_CONTEXT)) {
241         BLOGE("EGLContext is invalid, errno : 0x%{public}x, uniqueId: %{public}" PRIu64 ".",
242             eglGetError(), uniqueId_);
243         return SURFACE_ERROR_EGL_STATE_UNKONW;
244     }
245 
246     eglDisplay_ = disp;
247     eglContext_ = context;
248     return SURFACE_ERROR_OK;
249 }
250 
CreateEGLImage(EGLDisplay disp,const sptr<SurfaceBuffer> & buffer)251 EGLImageKHR SurfaceImage::CreateEGLImage(EGLDisplay disp, const sptr<SurfaceBuffer>& buffer)
252 {
253     sptr<SurfaceBuffer> bufferImpl = buffer;
254     NativeWindowBuffer* nBuffer = CreateNativeWindowBufferFromSurfaceBuffer(&bufferImpl);
255     EGLint attrs[] = {
256         EGL_IMAGE_PRESERVED,
257         EGL_TRUE,
258         EGL_NONE,
259     };
260 
261     EGLImageKHR img = eglCreateImageKHR(disp, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_OHOS, nBuffer, attrs);
262     if (img == EGL_NO_IMAGE_KHR) {
263         EGLint error = eglGetError();
264         BLOGE("failed, error %{public}d, uniqueId: %{public}" PRIu64 ".", error, uniqueId_);
265         eglTerminate(disp);
266     }
267     DestroyNativeWindowBuffer(nBuffer);
268     return img;
269 }
270 
CheckImageCacheNeedClean(uint32_t seqNum)271 void SurfaceImage::CheckImageCacheNeedClean(uint32_t seqNum)
272 {
273     for (auto it = imageCacheSeqs_.begin(); it != imageCacheSeqs_.end();) {
274         bool result = true;
275         if (seqNum == it->first) {
276             it++;
277             continue;
278         }
279         if (IsSurfaceBufferInCache(it->first, result) == SURFACE_ERROR_OK && !result) {
280             DestroyEGLImage(it->second.eglImage_);
281             DestroyEGLSync(it->second.eglSync_);
282             it = imageCacheSeqs_.erase(it);
283         } else {
284             it++;
285         }
286     }
287 }
288 
DestroyEGLImage(EGLImageKHR & eglImage)289 void SurfaceImage::DestroyEGLImage(EGLImageKHR &eglImage)
290 {
291     if (eglImage != EGL_NO_IMAGE_KHR) {
292         eglDestroyImageKHR(eglDisplay_, eglImage);
293         eglImage = EGL_NO_IMAGE_KHR;
294     }
295 }
296 
DestroyEGLSync(EGLSyncKHR & eglSync)297 void SurfaceImage::DestroyEGLSync(EGLSyncKHR &eglSync)
298 {
299     if (eglSync != EGL_NO_SYNC_KHR) {
300         eglDestroySyncKHR(eglDisplay_, eglSync);
301         eglSync = EGL_NO_SYNC_KHR;
302     }
303 }
304 
DestroyEGLImageBySeq(uint32_t seqNum)305 void SurfaceImage::DestroyEGLImageBySeq(uint32_t seqNum)
306 {
307     auto iter = imageCacheSeqs_.find(seqNum);
308     if (iter != imageCacheSeqs_.end()) {
309         DestroyEGLImage(iter->second.eglImage_);
310     }
311     imageCacheSeqs_.erase(seqNum);
312 }
313 
NewBufferDestroyEGLImage(bool isNewBuffer,uint32_t seqNum)314 void SurfaceImage::NewBufferDestroyEGLImage(bool isNewBuffer, uint32_t seqNum)
315 {
316     if (isNewBuffer) {
317         DestroyEGLImageBySeq(seqNum);
318     }
319 }
320 
UpdateEGLImageAndTexture(const sptr<SurfaceBuffer> & buffer)321 SurfaceError SurfaceImage::UpdateEGLImageAndTexture(const sptr<SurfaceBuffer>& buffer)
322 {
323     bool isNewBuffer = false;
324     // private function, buffer is always valid.
325     uint32_t seqNum = buffer->GetSeqNum();
326     // If there was no eglImage binding to this buffer, we create a new one.
327     if (imageCacheSeqs_.find(seqNum) == imageCacheSeqs_.end()) {
328         isNewBuffer = true;
329         EGLImageKHR eglImage = CreateEGLImage(eglDisplay_, buffer);
330         if (eglImage == EGL_NO_IMAGE_KHR) {
331             return SURFACE_ERROR_EGL_API_FAILED;
332         }
333         imageCacheSeqs_[seqNum].eglImage_ = eglImage;
334     }
335 
336     auto &image = imageCacheSeqs_[seqNum];
337     glBindTexture(textureTarget_, textureId_);
338     GLenum error = glGetError();
339     if (error != GL_NO_ERROR) {
340         NewBufferDestroyEGLImage(isNewBuffer, seqNum);
341         BLOGE("glBindTexture failed, textureTarget:%{public}d, textureId_:%{public}d, error:%{public}d"
342             "uniqueId: %{public}" PRIu64 ".", textureTarget_, textureId_, error, uniqueId_);
343         return SURFACE_ERROR_EGL_API_FAILED;
344     }
345     glEGLImageTargetTexture2DOES(textureTarget_, static_cast<GLeglImageOES>(image.eglImage_));
346     error = glGetError();
347     if (error != GL_NO_ERROR) {
348         NewBufferDestroyEGLImage(isNewBuffer, seqNum);
349         BLOGE("glEGLImageTargetTexture2DOES failed, textureTarget:%{public}d, error:%{public}d"
350             "uniqueId: %{public}" PRIu64 ".", textureTarget_, error, uniqueId_);
351         return SURFACE_ERROR_EGL_API_FAILED;
352     }
353 
354     // Create fence object for current image
355     auto iter = imageCacheSeqs_.find(currentSurfaceImage_);
356     if (iter != imageCacheSeqs_.end()) {
357         auto &currentImage = iter->second;
358         auto preSync = currentImage.eglSync_;
359         if (preSync != EGL_NO_SYNC_KHR) {
360             eglDestroySyncKHR(eglDisplay_, preSync);
361         }
362         currentImage.eglSync_ = eglCreateSyncKHR(eglDisplay_, EGL_SYNC_NATIVE_FENCE_ANDROID, nullptr);
363         glFlush();
364     }
365 
366     if (isNewBuffer) {
367         CheckImageCacheNeedClean(seqNum);
368     }
369     return SURFACE_ERROR_OK;
370 }
371 
SetOnBufferAvailableListener(void * context,OnBufferAvailableListener listener)372 SurfaceError SurfaceImage::SetOnBufferAvailableListener(void *context, OnBufferAvailableListener listener)
373 {
374     std::lock_guard<std::mutex> lockGuard(opMutex_);
375     if (listener == nullptr) {
376         BLOGE("listener is nullptr, uniqueId: %{public}" PRIu64 ".", uniqueId_);
377         return SURFACE_ERROR_INVALID_PARAM;
378     }
379 
380     listener_ = listener;
381     context_ = context;
382     return SURFACE_ERROR_OK;
383 }
384 
UnsetOnBufferAvailableListener()385 SurfaceError SurfaceImage::UnsetOnBufferAvailableListener()
386 {
387     std::lock_guard<std::mutex> lockGuard(opMutex_);
388     listener_ = nullptr;
389     context_ = nullptr;
390     return SURFACE_ERROR_OK;
391 }
392 
~SurfaceImageListener()393 SurfaceImageListener::~SurfaceImageListener()
394 {
395     BLOGE("~SurfaceImageListener");
396     surfaceImage_ = nullptr;
397 }
398 
OnBufferAvailable()399 void SurfaceImageListener::OnBufferAvailable()
400 {
401     BLOGD("enter OnBufferAvailable");
402     auto surfaceImage = surfaceImage_.promote();
403     if (surfaceImage == nullptr) {
404         BLOGE("surfaceImage promote failed");
405         return;
406     }
407 
408     // check here maybe a messagequeue, flag instead now
409     surfaceImage->OnUpdateBufferAvailableState(true);
410     if (surfaceImage->listener_ != nullptr) {
411         surfaceImage->listener_(surfaceImage->context_);
412     }
413 }
414 
AcquireNativeWindowBuffer(OHNativeWindowBuffer ** nativeWindowBuffer,int32_t * fenceFd)415 SurfaceError SurfaceImage::AcquireNativeWindowBuffer(OHNativeWindowBuffer** nativeWindowBuffer, int32_t* fenceFd)
416 {
417     std::lock_guard<std::mutex> lockGuard(opMutex_);
418     sptr<SurfaceBuffer> buffer = nullptr;
419     sptr<SyncFence> acquireFence = SyncFence::InvalidFence();
420     int64_t timestamp;
421     Rect damage;
422     SurfaceError ret = AcquireBuffer(buffer, acquireFence, timestamp, damage);
423     if (ret != SURFACE_ERROR_OK) {
424         BLOGE("AcquireBuffer failed: %{public}d, uniqueId: %{public}" PRIu64 ".", ret, uniqueId_);
425         return ret;
426     }
427     currentSurfaceBuffer_ = buffer;
428     currentTimeStamp_ = timestamp;
429     currentCrop_ = damage;
430     currentTransformType_ = ConsumerSurface::GetTransform();
431     auto utils = SurfaceUtils::GetInstance();
432     utils->ComputeTransformMatrixV2(currentTransformMatrixV2_, TRANSFORM_MATRIX_ELE_COUNT,
433         currentSurfaceBuffer_, currentTransformType_, currentCrop_);
434 
435     *fenceFd = acquireFence->Dup();
436     OHNativeWindowBuffer *nwBuffer = new OHNativeWindowBuffer();
437     nwBuffer->sfbuffer = buffer;
438     NativeObjectReference(nwBuffer);
439     *nativeWindowBuffer = nwBuffer;
440 
441     return SURFACE_ERROR_OK;
442 }
443 
ReleaseNativeWindowBuffer(OHNativeWindowBuffer * nativeWindowBuffer,int32_t fenceFd)444 SurfaceError SurfaceImage::ReleaseNativeWindowBuffer(OHNativeWindowBuffer* nativeWindowBuffer, int32_t fenceFd)
445 {
446     std::lock_guard<std::mutex> lockGuard(opMutex_);
447     // There is no need to close this fd, because in function ReleaseBuffer it will be closed.
448     SurfaceError ret = ReleaseBuffer(nativeWindowBuffer->sfbuffer, fenceFd);
449     if (ret != SURFACE_ERROR_OK) {
450         BLOGE("ReleaseBuffer failed: %{public}d, uniqueId: %{public}" PRIu64 ".", ret, uniqueId_);
451         return ret;
452     }
453     NativeObjectUnreference(nativeWindowBuffer);
454     return SURFACE_ERROR_OK;
455 }
456 
SetDefaultUsage(uint64_t usage)457 SurfaceError SurfaceImage::SetDefaultUsage(uint64_t usage)
458 {
459     SurfaceError ret = ConsumerSurface::SetDefaultUsage(usage);
460     if (ret != SURFACE_ERROR_OK) {
461         BLOGE("SetDefaultUsage failed: %{public}d, uniqueId: %{public}" PRIu64 ", usage: %{public}" PRIu64 ".", ret,
462             uniqueId_, usage);
463     }
464     return ret;
465 }
466 
SetDefaultSize(int32_t width,int32_t height)467 SurfaceError SurfaceImage::SetDefaultSize(int32_t width, int32_t height)
468 {
469     SurfaceError ret = SetDefaultWidthAndHeight(width, height);
470     if (ret != SURFACE_ERROR_OK) {
471         BLOGE("SetDefaultWidthAndHeight failed: %{public}d, uniqueId: %{public}" PRIu64 ", width: %{public}d, "
472             "height: %{public}d", ret, uniqueId_, width, height);
473     }
474     return ret;
475 }
476 } // namespace OHOS
477