1 /*
2  * Copyright (c) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "jpeg_image_processor.h"
17 
18 #include <cstring>
19 #include <fstream>
20 #include <iostream>
21 #include "jpeglib.h"
22 #include <securec.h>
23 #include <string>
24 
25 #include "dscreen_errcode.h"
26 #include "dscreen_log.h"
27 
28 namespace OHOS {
29 namespace DistributedHardware {
SetOutputSurface(sptr<Surface> surface)30 int32_t JpegImageProcessor::SetOutputSurface(sptr<Surface> surface)
31 {
32     DHLOGI("%{public}s: SetOutputSurface.", DSCREEN_LOG_TAG);
33     if (surface == nullptr) {
34         DHLOGE("%{public}s: SetOutputSurface surface is nullptr.", DSCREEN_LOG_TAG);
35         return ERR_DH_SCREEN_TRANS_NULL_VALUE;
36     }
37     imageSurface_ = surface;
38     return DH_SUCCESS;
39 }
40 
FillDirtyImages2Surface(const std::shared_ptr<DataBuffer> & data,uint8_t * lastFrame)41 int32_t JpegImageProcessor::FillDirtyImages2Surface(const std::shared_ptr<DataBuffer> &data, uint8_t *lastFrame)
42 {
43     DHLOGI("%{public}s: FillDirtyImages2Surface.", DSCREEN_LOG_TAG);
44     if (imageSurface_ == nullptr) {
45         DHLOGE("%{public}s: imageSurface_ is nullptr.", DSCREEN_LOG_TAG);
46         return ERR_DH_SCREEN_SURFACE_INVALIED;
47     }
48     uint32_t lastFrameSize = configParam_.GetScreenWidth() * configParam_.GetScreenHeight() * RGB_CHROMA / TWO;
49     int32_t ret = DecodeDamageData(data, lastFrame);
50     if (ret != DH_SUCCESS) {
51         DHLOGE("%{public}s: Merge dirty failed, ret: %{public}" PRId32, DSCREEN_LOG_TAG, ret);
52         return ret;
53     }
54     sptr<OHOS::SurfaceBuffer> windowSurfaceBuffer = nullptr;
55     int32_t releaseFence = -1;
56     OHOS::BufferRequestConfig requestConfig = {
57         .width = configParam_.GetScreenWidth(),
58         .height = configParam_.GetScreenHeight(),
59         .strideAlignment = STRIDE_ALIGNMENT,
60         .format = GRAPHIC_PIXEL_FMT_YCBCR_420_SP,
61         .usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA,
62     };
63     SurfaceError surfaceErr = imageSurface_->RequestBuffer(windowSurfaceBuffer, releaseFence, requestConfig);
64     if (surfaceErr != SURFACE_ERROR_OK || windowSurfaceBuffer == nullptr) {
65         DHLOGE("%{public}s: imageSurface request buffer failed, surfaceErr: %{public}" PRId32,
66             DSCREEN_LOG_TAG, surfaceErr);
67         imageSurface_->CancelBuffer(windowSurfaceBuffer);
68         return surfaceErr;
69     }
70     uint32_t surfaceBuffeSize = windowSurfaceBuffer->GetSize();
71     auto windowSurfaceAddr = static_cast<uint8_t*>(windowSurfaceBuffer->GetVirAddr());
72     ret = memcpy_s(windowSurfaceAddr, surfaceBuffeSize, lastFrame, lastFrameSize);
73     if (ret != DH_SUCCESS) {
74         DHLOGE("%{public}s: memcpy lastFrame failed,ret: %{public}" PRId32, DSCREEN_LOG_TAG, ret);
75         imageSurface_->CancelBuffer(windowSurfaceBuffer);
76         return ret;
77     }
78     BufferFlushConfig flushConfig = { {0, 0, windowSurfaceBuffer->GetWidth(), windowSurfaceBuffer-> GetHeight()}, 0};
79     surfaceErr = imageSurface_->FlushBuffer(windowSurfaceBuffer, -1, flushConfig);
80     if (surfaceErr != SURFACE_ERROR_OK) {
81         DHLOGE("%{public}s: imageSurface flush buffer failed, surfaceErr: %{public}" PRId32,
82             DSCREEN_LOG_TAG, surfaceErr);
83         imageSurface_->CancelBuffer(windowSurfaceBuffer);
84         return surfaceErr;
85     }
86     DHLOGI("%{public}s: FillDirtyImages2Surface success.", DSCREEN_LOG_TAG);
87     return DH_SUCCESS;
88 }
89 
ProcessDamageSurface(sptr<SurfaceBuffer> & surfaceBuffer,const std::vector<OHOS::Rect> & damages)90 int32_t JpegImageProcessor::ProcessDamageSurface(sptr<SurfaceBuffer> &surfaceBuffer,
91     const std::vector<OHOS::Rect> &damages)
92 {
93     DHLOGI("%{public}s: ProcessDamageSurface.", DSCREEN_LOG_TAG);
94     std::shared_ptr<DataBuffer> dataBuf = std::make_shared<DataBuffer>(configParam_.GetScreenWidth() *
95         configParam_.GetScreenHeight() * RGBA_CHROMA);
96     dataBuf->SetSize(0);
97     for (auto item : damages) {
98         EncodeDamageData(surfaceBuffer, item, dataBuf);
99     }
100     std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
101     if (listener == nullptr) {
102         DHLOGE("%{public}s: Processor listener is null.", DSCREEN_LOG_TAG);
103         imageSurface_->ReleaseBuffer(surfaceBuffer, -1);
104         return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
105     }
106     dataBuf->SetDataType(VIDEO_PART_SCREEN_DATA);
107     listener->OnImageProcessDone(dataBuf);
108     return DH_SUCCESS;
109 }
110 
SetImageProcessListener(std::shared_ptr<IImageSourceProcessorListener> & listener)111 int32_t JpegImageProcessor::SetImageProcessListener(std::shared_ptr<IImageSourceProcessorListener> &listener)
112 {
113     DHLOGI("%{public}s: SetImageProcessorListener.", DSCREEN_LOG_TAG);
114     imageProcessorListener_ = listener;
115     return DH_SUCCESS;
116 }
117 
EncodeDamageData(sptr<SurfaceBuffer> & surfaceBuffer,const OHOS::Rect & damage,std::shared_ptr<DataBuffer> & data)118 void JpegImageProcessor::EncodeDamageData(sptr<SurfaceBuffer> &surfaceBuffer,
119     const OHOS::Rect &damage, std::shared_ptr<DataBuffer> &data)
120 {
121     DHLOGI("%{public}s: EncodeDamageData.", DSCREEN_LOG_TAG);
122     uint32_t partialSize = damage.w * damage.h * RGBA_CHROMA;
123     unsigned char *partialBuffer = new unsigned char[partialSize];
124     unsigned char *partialBufferIdx = partialBuffer;
125     auto surfaceAddrIdx = static_cast<uint8_t*>(surfaceBuffer->GetVirAddr());
126     surfaceAddrIdx += damage.y * configParam_.GetScreenWidth() * RGBA_CHROMA + damage.x * RGBA_CHROMA;
127     for (int32_t row = 0 ; row < damage.h ; row++) {
128         int32_t ret = memcpy_s(partialBufferIdx, damage.w * RGBA_CHROMA, surfaceAddrIdx, damage.w * RGBA_CHROMA);
129         if (ret != DH_SUCCESS) {
130             DHLOGE("%{public}s: get partail data failed.", DSCREEN_LOG_TAG);
131             imageSurface_->ReleaseBuffer(surfaceBuffer, -1);
132             delete [] partialBuffer;
133             return;
134         }
135         partialBufferIdx += damage.w * RGBA_CHROMA;
136         surfaceAddrIdx += configParam_.GetScreenWidth() * RGBA_CHROMA;
137     }
138     uint32_t jpegSize = CompressRgbaToJpeg(damage, partialBuffer, partialSize, data);
139     DHLOGI("CompressRgbaToJpeg end, jpegSize %{public}" PRId32, jpegSize);
140     delete [] partialBuffer;
141 }
142 
DecodeDamageData(const std::shared_ptr<DataBuffer> & data,uint8_t * lastFrame)143 int32_t JpegImageProcessor::DecodeDamageData(const std::shared_ptr<DataBuffer> &data, uint8_t *lastFrame)
144 {
145     DHLOGI("%{public}s: DecodeDamageData.", DSCREEN_LOG_TAG);
146     std::vector<DirtyRect> dirtyRectVec = data->GetDirtyRectVec();
147     int32_t offset = 0;
148     int32_t screenWidth = static_cast<int32_t>(configParam_.GetScreenWidth());
149     int32_t screenHeight = static_cast<int32_t>(configParam_.GetScreenHeight());
150     for (auto item : dirtyRectVec) {
151         if (item.xPos > screenWidth || item.yPos > screenHeight ||
152             item.width > screenWidth - item.xPos || item.height > screenHeight - item.yPos) {
153             DHLOGE("%{public}s: Dirty rect invalid.", DSCREEN_LOG_TAG);
154             return ERR_DH_SCREEN_INPUT_PARAM_INVALID;
155         }
156         uint8_t *jpegData = new uint8_t[item.dirtySize] {0};
157         int32_t ret = data->GetData(offset, item.dirtySize, jpegData);
158         if (ret != DH_SUCCESS) {
159             delete [] jpegData;
160             return ret;
161         }
162         offset += item.dirtySize;
163         uint8_t *dirtyImageData = new uint8_t[item.width * item.height * RGB_CHROMA] {0};
164         DHLOGI("%{public}s: DecompressJpegToNV12.", DSCREEN_LOG_TAG);
165         DecompressJpegToNV12(item.dirtySize, jpegData, dirtyImageData);
166         DHLOGI("%{public}s: DecompressJpegToNV12 success.", DSCREEN_LOG_TAG);
167         ret = ReplaceDamage2LastFrame(lastFrame, dirtyImageData, item);
168         if (ret != DH_SUCCESS) {
169             DHLOGE("ReplaceDamage2LastFrame failed, ret: %{public}" PRId32, ret);
170             delete [] jpegData;
171             delete [] dirtyImageData;
172             return ret;
173         }
174         delete [] jpegData;
175         delete [] dirtyImageData;
176     }
177     DHLOGI("%{public}s: DecodeDamageData success.", DSCREEN_LOG_TAG);
178     return DH_SUCCESS;
179 }
180 
ReplaceDamage2LastFrame(uint8_t * lastFrame,uint8_t * dirtyImageData,const DirtyRect rect)181 int32_t JpegImageProcessor::ReplaceDamage2LastFrame(uint8_t *lastFrame, uint8_t *dirtyImageData, const DirtyRect rect)
182 {
183     DHLOGI("%{public}s: ReplaceDamage2LastFrame.", DSCREEN_LOG_TAG);
184     uint8_t *lastFrameIdx = lastFrame;
185     uint8_t *yData = lastFrameIdx + static_cast<uint32_t>(configParam_.GetScreenWidth() * rect.yPos + rect.xPos);
186     uint8_t *uData = lastFrameIdx + configParam_.GetScreenWidth() * configParam_.GetScreenHeight() +
187                 static_cast<uint32_t>(configParam_.GetScreenWidth() * (rect.yPos / TWO) + rect.xPos);
188     uint8_t *yDirtyData = dirtyImageData;
189     uint8_t *uDirtyData = dirtyImageData + rect.width * rect.height;
190     uint8_t *yTempData = nullptr;
191     uint8_t *uTempData = nullptr;
192     for (int32_t i = 0 ; i < rect.height ; i++) {
193         yTempData = yData + static_cast<uint32_t>(i) * configParam_.GetScreenWidth();
194         int32_t ret = memcpy_s(yTempData, rect.width, yDirtyData, rect.width);
195         if (ret != EOK) {
196             DHLOGE("%{public}s: memcpy yData failed.", DSCREEN_LOG_TAG);
197             return ret;
198         }
199         yDirtyData += static_cast<uint32_t>(rect.width);
200         if (i % TWO) {
201             uTempData = uData + configParam_.GetScreenWidth() * (static_cast<uint32_t>(i) / TWO);
202             ret = memcpy_s(uTempData, rect.width, uDirtyData, rect.width);
203             if (ret != EOK) {
204                 DHLOGE("%{public}s: memcpy uData failed.", DSCREEN_LOG_TAG);
205                 return ret;
206             }
207             uDirtyData += static_cast<uint32_t>(rect.width);
208             }
209     }
210     DHLOGI("%{public}s: ReplaceDamage2LastFrame success.", DSCREEN_LOG_TAG);
211     return DH_SUCCESS;
212 }
213 
CompressRgbaToJpeg(const OHOS::Rect & damage,uint8_t * inputData,uint32_t inputDataSize,std::shared_ptr<DataBuffer> & data)214 uint32_t JpegImageProcessor::CompressRgbaToJpeg(const OHOS::Rect &damage,
215     uint8_t *inputData, uint32_t inputDataSize, std::shared_ptr<DataBuffer> &data)
216 {
217     DHLOGI("%{public}s: CompressRgbaToJpeg.", DSCREEN_LOG_TAG);
218     if (inputDataSize != damage.w * damage.h * RGBA_CHROMA) {
219         return ERR_DH_SCREEN_CODEC_PARTAIL_DATA_ERROR;
220     }
221     jpeg_compress_struct cinfo;
222     jpeg_error_mgr jerr;
223     JSAMPROW row_pointer[1];
224     cinfo.err = jpeg_std_error(&jerr);
225     jpeg_create_compress(&cinfo);
226     unsigned char *outBuffer = nullptr;
227     unsigned long outSize = 0;
228     jpeg_mem_dest(&cinfo, &outBuffer, &outSize);
229     cinfo.image_width = damage.w;
230     cinfo.image_height = damage.h;
231     cinfo.input_components = RGB_CHROMA;
232     cinfo.in_color_space = JCS_RGB;
233     jpeg_set_defaults(&cinfo);
234     jpeg_set_quality(&cinfo, JPEG_QUALITY, TRUE);
235     jpeg_start_compress(&cinfo, TRUE);
236     unsigned char rgb_buffer[damage.w * RGB_CHROMA];
237     unsigned char *pB = inputData;
238     unsigned char *pG = inputData + 1;
239     unsigned char *pR = inputData + TWO;
240     while (cinfo.next_scanline < cinfo.image_height) {
241         int index = 0;
242         for (int i = 0 ; i < damage.w ; i++) {
243             rgb_buffer[index++] = *pB;
244             rgb_buffer[index++] = *pG;
245             rgb_buffer[index++] = *pR;
246             pB += RGBA_CHROMA;
247             pG += RGBA_CHROMA;
248             pR += RGBA_CHROMA;
249         }
250         row_pointer[0] = rgb_buffer;
251         (void)jpeg_write_scanlines(&cinfo, row_pointer, 1);
252     }
253     jpeg_finish_compress(&cinfo);
254     DirtyRect rect = {damage.x, damage.y, damage.w, damage.h, outSize};
255     data->AddData(static_cast<size_t>(outSize), outBuffer);
256     data->AddDirtyRect(rect);
257     jpeg_destroy_compress(&cinfo);
258     if (outBuffer != NULL) {
259         free(outBuffer);
260         outBuffer = NULL;
261     }
262     return (uint32_t)outSize;
263 }
264 
DecompressJpegToNV12(size_t jpegSize,uint8_t * inputData,uint8_t * outputData)265 void JpegImageProcessor::DecompressJpegToNV12(size_t jpegSize, uint8_t *inputData, uint8_t *outputData)
266 {
267     jpeg_decompress_struct cinfo;
268     jpeg_error_mgr jerr;
269     cinfo.err = jpeg_std_error(&jerr);
270     jpeg_create_decompress(&cinfo);
271     jpeg_mem_src(&cinfo, inputData, jpegSize);
272     (void)jpeg_read_header(&cinfo, TRUE);
273     (void)jpeg_start_decompress(&cinfo);
274     int32_t row_stride = static_cast<int32_t>(cinfo.output_width) * cinfo.output_components;
275     JSAMPARRAY buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr)&cinfo, JPOOL_IMAGE, row_stride, 1);
276     uint32_t uvIndex = cinfo.output_width * cinfo.output_height;
277     int32_t i = 0;
278     int32_t yIndex = 0;
279     while (cinfo.output_scanline < cinfo.output_height) {
280         (void)jpeg_read_scanlines(&cinfo, buffer, 1);
281         for (unsigned int j = 0 ; j < cinfo.output_width ; j++) {
282             int32_t y = ((YR_PARAM * buffer[0][j * RGB_CHROMA] + YG_PARAM * buffer[0][j * RGB_CHROMA + 1] +
283                 YB_PARAM * buffer[0][j * RGB_CHROMA + TWO] + UA_PARAM) >> MOVEBITS) + YA_PARAM;
284             int32_t u = ((UB_PARAM * buffer[0][j * RGB_CHROMA + TWO] - UR_PARAM * buffer[0][j * RGB_CHROMA] -
285             UG_PARAM * buffer[0][j * RGB_CHROMA + 1] + UA_PARAM) >> MOVEBITS) + UA_PARAM;
286             int32_t v = ((UB_PARAM * buffer[0][j * RGB_CHROMA] - VG_PARAM * buffer[0][j * RGB_CHROMA + 1] -
287                 VB_PARAM * buffer[0][j * RGB_CHROMA + TWO] + UA_PARAM) >> MOVEBITS) + UA_PARAM;
288             outputData[yIndex++] = static_cast<uint8_t>((y < 0) ? 0 : (y > YUV_PARAM) ? YUV_PARAM : y);
289             if ((i % TWO == 0) && (j % TWO == 0)) {
290                 outputData[uvIndex++] = static_cast<uint8_t>((u < 0) ? 0 : (u > YUV_PARAM) ? YUV_PARAM : u);
291                 outputData[uvIndex++] = static_cast<uint8_t>((v < 0) ? 0 : (v > YUV_PARAM) ? YUV_PARAM : v);
292             }
293         }
294         ++i;
295     }
296     (void)jpeg_finish_decompress(&cinfo);
297     jpeg_destroy_decompress(&cinfo);
298 }
299 } // namespace DistributedHardware
300 } // namespace OHOS