1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ext_encoder.h"
17 #include <algorithm>
18 #include <map>
19
20 #include "include/core/SkImageEncoder.h"
21 #include "src/images/SkImageEncoderFns.h"
22 #include "include/core/SkBitmap.h"
23 #include "pixel_yuv_utils.h"
24 #ifdef IMAGE_COLORSPACE_FLAG
25 #include "color_space.h"
26 #endif
27 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
28 #include "astc_codec.h"
29 #endif
30
31 #include "ext_pixel_convert.h"
32 #include "ext_wstream.h"
33 #include "image_data_statistics.h"
34 #include "image_dfx.h"
35 #include "image_func_timer.h"
36 #include "image_fwk_ext_manager.h"
37 #include "image_log.h"
38 #include "image_system_properties.h"
39 #include "image_type_converter.h"
40 #include "image_utils.h"
41 #include "jpeg_mpf_parser.h"
42 #include "media_errors.h"
43 #include "metadata_accessor.h"
44 #include "metadata_accessor_factory.h"
45 #include "pixel_convert_adapter.h"
46 #include "string_ex.h"
47 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
48 #include "surface_buffer.h"
49 #include "v1_0/buffer_handle_meta_key_type.h"
50 #include "v1_0/cm_color_space.h"
51 #include "v1_0/hdr_static_metadata.h"
52 #include "vpe_utils.h"
53 #include "hdr_helper.h"
54 #endif
55 #include "color_utils.h"
56 #include "tiff_parser.h"
57 #include "image_mime_type.h"
58 #include "securec.h"
59 #include "file_packer_stream.h"
60 #include "memory_manager.h"
61 #ifdef HEIF_HW_ENCODE_ENABLE
62 #include "v2_0/icodec_image.h"
63 #include "iremote_object.h"
64 #include "iproxy_broker.h"
65 #endif
66
67 #undef LOG_DOMAIN
68 #define LOG_DOMAIN LOG_TAG_DOMAIN_ID_PLUGIN
69
70 #undef LOG_TAG
71 #define LOG_TAG "ExtEncoder"
72
73 namespace OHOS {
74 namespace ImagePlugin {
75 using namespace Media;
76 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
77 using namespace HDI::Display::Graphic::Common::V1_0;
78 const std::map<PixelFormat, GraphicPixelFormat> SURFACE_FORMAT_MAP = {
79 { PixelFormat::RGBA_8888, GRAPHIC_PIXEL_FMT_RGBA_8888 },
80 { PixelFormat::NV21, GRAPHIC_PIXEL_FMT_YCRCB_420_SP },
81 { PixelFormat::NV12, GRAPHIC_PIXEL_FMT_YCBCR_420_SP },
82 };
83 #endif
84
85 namespace {
86 constexpr static uint32_t PRIMARY_IMAGE_ITEM_ID = 1;
87 constexpr static uint32_t GAINMAP_IMAGE_ITEM_ID = 2;
88 constexpr static uint32_t TMAP_IMAGE_ITEM_ID = 3;
89 constexpr static uint32_t EXIF_META_ITEM_ID = 4;
90 constexpr static uint32_t DEPTH_MAP_ITEM_ID = 5;
91 constexpr static uint32_t UNREFOCUS_MAP_ITEM_ID = 6;
92 constexpr static uint32_t LINEAR_MAP_ITEM_ID = 7;
93 constexpr static uint32_t FRAGMENT_MAP_ITEM_ID = 8;
94 const static std::string COMPRESS_TYPE_TMAP = "tmap";
95 const static std::string COMPRESS_TYPE_HEVC = "hevc";
96 const static std::string COMPRESS_TYPE_NONE = "none";
97 const static std::string DEFAULT_ASHMEM_TAG = "Heif Encoder Default";
98 const static std::string ICC_ASHMEM_TAG = "Heif Encoder Property";
99 const static std::string IT35_ASHMEM_TAG = "Heif Encoder IT35";
100 const static std::string OUTPUT_ASHMEM_TAG = "Heif Encoder Output";
101 const static std::string IMAGE_DATA_TAG = "Heif Encoder Image";
102 const static std::string HDR_GAINMAP_TAG = "Heif Encoder Gainmap";
103 const static std::string EXIF_ASHMEM_TAG = "Heif Encoder Exif";
104 const static std::string GAINMAP_IMAGE_ITEM_NAME = "Gain map Image";
105 const static std::string DEPTH_MAP_ITEM_NAME = "Depth Map Image";
106 const static std::string UNREFOCUS_MAP_ITEM_NAME = "Unrefocus Map Image";
107 const static std::string LINEAR_MAP_ITEM_NAME = "Linear Map Image";
108 const static std::string FRAGMENT_MAP_ITEM_NAME = "Fragment Map Image";
109 constexpr uint32_t PLACE_HOLDER_LENGTH = 1;
110 const static size_t DEFAULT_OUTPUT_SIZE = 35 * 1024 * 1024; // 35M
111 const static uint16_t STATIC_METADATA_COLOR_SCALE = 50000;
112 const static uint16_t STATIC_METADATA_LUM_SCALE = 10000;
113 const static uint8_t INDEX_ZERO = 0;
114 const static uint8_t INDEX_ONE = 1;
115 const static uint8_t INDEX_TWO = 2;
116 constexpr uint32_t DEFAULT_DENOMINATOR = 1000000;
117 constexpr uint8_t GAINMAP_CHANNEL_MULTI = 3;
118 constexpr uint8_t GAINMAP_CHANNEL_SINGLE = 1;
119 constexpr uint8_t EXIF_PRE_SIZE = 6;
120 constexpr uint32_t JPEG_MARKER_TAG_SIZE = 2;
121 constexpr uint32_t DEPTH_MAP_BYTES = sizeof(float); // float16
122 constexpr uint32_t LINEAR_MAP_BYTES = sizeof(short) * 3 / 2; // 16bit yuv420
123
124 // exif/0/0
125 constexpr uint8_t EXIF_PRE_TAG[EXIF_PRE_SIZE] = {
126 0x45, 0x78, 0x69, 0x66, 0x00, 0x00
127 };
128 }
129
130 struct HeifEncodeItemInfo {
131 uint32_t itemId = 0;
132 std::string itemName = "";
133 std::string itemType = "";
134 };
135
136 static const std::map<std::string, SkEncodedImageFormat> FORMAT_NAME = {
137 {IMAGE_BMP_FORMAT, SkEncodedImageFormat::kBMP},
138 {IMAGE_GIF_FORMAT, SkEncodedImageFormat::kGIF},
139 {IMAGE_ICO_FORMAT, SkEncodedImageFormat::kICO},
140 {IMAGE_JPEG_FORMAT, SkEncodedImageFormat::kJPEG},
141 {IMAGE_PNG_FORMAT, SkEncodedImageFormat::kPNG},
142 {IMAGE_WBMP_FORMAT, SkEncodedImageFormat::kWBMP},
143 {IMAGE_WEBP_FORMAT, SkEncodedImageFormat::kWEBP},
144 {IMAGE_HEIF_FORMAT, SkEncodedImageFormat::kHEIF},
145 {IMAGE_HEIC_FORMAT, SkEncodedImageFormat::kHEIF},
146 };
147
148 static const std::map<AuxiliaryPictureType, std::string> DEFAULT_AUXILIARY_TAG_MAP = {
149 {AuxiliaryPictureType::GAINMAP, AUXILIARY_TAG_GAINMAP},
150 {AuxiliaryPictureType::DEPTH_MAP, AUXILIARY_TAG_DEPTH_MAP_BACK},
151 {AuxiliaryPictureType::UNREFOCUS_MAP, AUXILIARY_TAG_UNREFOCUS_MAP},
152 {AuxiliaryPictureType::LINEAR_MAP, AUXILIARY_TAG_LINEAR_MAP},
153 {AuxiliaryPictureType::FRAGMENT_MAP, AUXILIARY_TAG_FRAGMENT_MAP},
154 };
155
156 static const uint8_t NUM_3 = 3;
157 static const uint8_t NUM_4 = 4;
158
159 #ifdef HEIF_HW_ENCODE_ENABLE
160 using namespace OHOS::HDI::Codec::Image::V2_0;
161 static std::mutex g_codecMtx;
162 static sptr<ICodecImage> g_codecMgr;
163 class CodecHeifDeathRecipient : public IRemoteObject::DeathRecipient {
164 public:
OnRemoteDied(const wptr<OHOS::IRemoteObject> & object)165 void OnRemoteDied(const wptr<OHOS::IRemoteObject>& object) override
166 {
167 IMAGE_LOGW("codec_image_service died");
168 std::lock_guard<std::mutex> lk(g_codecMtx);
169 g_codecMgr = nullptr;
170 }
171 };
172
GetCodecManager()173 static sptr<ICodecImage> GetCodecManager()
174 {
175 std::lock_guard<std::mutex> lk(g_codecMtx);
176 if (g_codecMgr) {
177 return g_codecMgr;
178 }
179 IMAGE_LOGI("need to get ICodecImage");
180 g_codecMgr = ICodecImage::Get();
181 if (g_codecMgr == nullptr) {
182 IMAGE_LOGE("ICodecImage get failed");
183 return nullptr;
184 }
185 bool isDeathRecipientAdded = false;
186 const sptr<OHOS::IRemoteObject> &remote = OHOS::HDI::hdi_objcast<ICodecImage>(g_codecMgr);
187 if (remote) {
188 sptr<CodecHeifDeathRecipient> deathCallBack(new CodecHeifDeathRecipient());
189 isDeathRecipientAdded = remote->AddDeathRecipient(deathCallBack);
190 }
191 if (!isDeathRecipientAdded) {
192 IMAGE_LOGI("failed to add deathRecipient for ICodecImage!");
193 }
194 return g_codecMgr;
195 }
196 #endif
197
ExtEncoder()198 ExtEncoder::ExtEncoder()
199 {
200 }
201
~ExtEncoder()202 ExtEncoder::~ExtEncoder()
203 {
204 }
205
StartEncode(OutputDataStream & outputStream,PlEncodeOptions & option)206 uint32_t ExtEncoder::StartEncode(OutputDataStream &outputStream, PlEncodeOptions &option)
207 {
208 output_ = &outputStream;
209 opts_ = option;
210 return SUCCESS;
211 }
212
AddImage(PixelMap & pixelMap)213 uint32_t ExtEncoder::AddImage(PixelMap &pixelMap)
214 {
215 pixelmap_ = &pixelMap;
216 return SUCCESS;
217 }
218
AddPicture(Media::Picture & picture)219 uint32_t ExtEncoder::AddPicture(Media::Picture &picture)
220 {
221 picture_ = &picture;
222 return SUCCESS;
223 }
224
225 struct TmpBufferHolder {
226 std::unique_ptr<uint8_t[]> buf = nullptr;
227 };
228
229 struct ImageData {
230 uint8_t *dst;
231 uint8_t *pixels;
232 ImageInfo info;
233 };
234
ToSkColorSpace(PixelMap * pixelmap)235 static sk_sp<SkColorSpace> ToSkColorSpace(PixelMap *pixelmap)
236 {
237 #ifdef IMAGE_COLORSPACE_FLAG
238 if (pixelmap->InnerGetGrColorSpacePtr() == nullptr) {
239 return nullptr;
240 }
241 if (pixelmap->InnerGetGrColorSpace().GetColorSpaceName() == ColorManager::ColorSpaceName::NONE) {
242 return SkColorSpace::MakeSRGB();
243 }
244 return pixelmap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
245 #else
246 return nullptr;
247 #endif
248 }
249
ToSkInfo(Media::PixelMap * pixelMap)250 static SkImageInfo ToSkInfo(Media::PixelMap *pixelMap)
251 {
252 ImageInfo info;
253 pixelMap->GetImageInfo(info);
254 SkColorType colorType = ImageTypeConverter::ToSkColorType(info.pixelFormat);
255 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(info.alphaType);
256 sk_sp<SkColorSpace> colorSpace = SkColorSpace::MakeSRGB();
257 #ifdef IMAGE_COLORSPACE_FLAG
258 if (pixelMap->InnerGetGrColorSpacePtr() != nullptr &&
259 pixelMap->InnerGetGrColorSpace().GetColorSpaceName() != ColorManager::ColorSpaceName::NONE) {
260 colorSpace = pixelMap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
261 }
262 #endif
263 return SkImageInfo::Make(info.size.width, info.size.height, colorType, alphaType, colorSpace);
264 }
265
RGBToRGBx(PixelMap * pixelMap,SkImageInfo & skInfo,TmpBufferHolder & holder)266 static uint32_t RGBToRGBx(PixelMap *pixelMap, SkImageInfo &skInfo, TmpBufferHolder &holder)
267 {
268 holder.buf = std::make_unique<uint8_t[]>(skInfo.computeMinByteSize());
269 ExtPixels src = {
270 static_cast<uint8_t*>(pixelMap->GetWritablePixels()),
271 pixelMap->GetCapacity(), pixelMap->GetWidth()*pixelMap->GetHeight(),
272 };
273 ExtPixels dst = {
274 holder.buf.get(), skInfo.computeMinByteSize(), skInfo.width()*skInfo.height(),
275 };
276 return ExtPixelConvert::RGBToRGBx(src, dst);
277 }
278
IsYuvImage(PixelFormat format)279 static bool IsYuvImage(PixelFormat format)
280 {
281 return format == PixelFormat::NV21 || format == PixelFormat::NV12;
282 }
283
YuvToRgbaSkInfo(ImageInfo info,SkImageInfo & skInfo,uint8_t * dstData,Media::PixelMap * pixelMap)284 static uint32_t YuvToRgbaSkInfo(ImageInfo info, SkImageInfo &skInfo, uint8_t * dstData, Media::PixelMap *pixelMap)
285 {
286 uint8_t *srcData = static_cast<uint8_t*>(pixelMap->GetWritablePixels());
287 YUVDataInfo yuvInfo;
288 pixelMap->GetImageYUVInfo(yuvInfo);
289 YuvImageInfo srcInfo = {PixelYuvUtils::ConvertFormat(info.pixelFormat),
290 info.size.width, info.size.height, info.pixelFormat, yuvInfo};
291 YuvImageInfo dstInfo = {PixelYuvUtils::ConvertFormat(PixelFormat::RGBA_8888), info.size.width, info.size.height};
292 if (!PixelConvertAdapter::YUV420ToRGB888(srcData, srcInfo, dstData, dstInfo)) {
293 IMAGE_LOGE("YuvToSkInfo Support YUV format RGB convert failed ");
294 return ERR_IMAGE_ENCODE_FAILED;
295 }
296 auto alpha = pixelMap->GetAlphaType();
297 if (alpha == AlphaType::IMAGE_ALPHA_TYPE_UNKNOWN)
298 alpha = AlphaType::IMAGE_ALPHA_TYPE_OPAQUE;
299 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(alpha);
300 auto cs = ToSkColorSpace(pixelMap);
301 skInfo = SkImageInfo::Make(info.size.width, info.size.height, SkColorType::kRGBA_8888_SkColorType, alphaType, cs);
302 IMAGE_LOGD(" YuvToSkInfo: width:%{public}d, height:%{public}d, alpha:%{public}d \n ",
303 info.size.width, info.size.height, (int32_t)alphaType);
304 return SUCCESS;
305 }
306
pixelToSkInfo(ImageData & image,SkImageInfo & skInfo,Media::PixelMap * pixelMap,TmpBufferHolder & holder,SkEncodedImageFormat format)307 static uint32_t pixelToSkInfo(ImageData &image, SkImageInfo &skInfo, Media::PixelMap *pixelMap,
308 TmpBufferHolder &holder, SkEncodedImageFormat format)
309 {
310 skInfo = ToSkInfo(pixelMap);
311 image.pixels = static_cast<uint8_t*>(pixelMap->GetWritablePixels());
312 if (format == SkEncodedImageFormat::kJPEG &&
313 skInfo.colorType() == SkColorType::kRGB_888x_SkColorType &&
314 pixelMap->GetCapacity() < skInfo.computeMinByteSize()) {
315 uint32_t res = RGBToRGBx(pixelMap, skInfo, holder);
316 if (res != SUCCESS) {
317 IMAGE_LOGE("ExtEncoder::BuildSkBitmap RGB convert failed %{public}d", res);
318 return res;
319 }
320 image.pixels = holder.buf.get();
321 skInfo = skInfo.makeColorType(SkColorType::kRGBA_8888_SkColorType);
322 }
323 return SUCCESS;
324 }
325
IsAstc(const std::string & format)326 bool IsAstc(const std::string &format)
327 {
328 return format.find("image/astc") == 0;
329 }
330
CreateAndWriteBlob(MetadataWStream & tStream,PixelMap * pixelmap,SkWStream & outStream,ImageInfo & imageInfo,PlEncodeOptions & opts)331 static uint32_t CreateAndWriteBlob(MetadataWStream &tStream, PixelMap *pixelmap, SkWStream& outStream,
332 ImageInfo &imageInfo, PlEncodeOptions &opts)
333 {
334 ImageFuncTimer imageFuncTimer("insert exit data (%d, %d)", imageInfo.size.width, imageInfo.size.height);
335 auto metadataAccessor =
336 MetadataAccessorFactory::Create(tStream.GetAddr(), tStream.bytesWritten(), BufferMetadataStream::Dynamic);
337 if (metadataAccessor != nullptr) {
338 auto metadataPtr = pixelmap->GetExifMetadata();
339 metadataAccessor->Set(metadataPtr);
340 if (metadataAccessor->Write() == SUCCESS) {
341 if (metadataAccessor->WriteToOutput(outStream)) {
342 return SUCCESS;
343 }
344 }
345 }
346 if (!outStream.write(tStream.GetAddr(), tStream.bytesWritten())) {
347 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts.format, "Failed to encode image");
348 return ERR_IMAGE_ENCODE_FAILED;
349 }
350 return SUCCESS;
351 }
352
PixelmapEncode(ExtWStream & wStream)353 uint32_t ExtEncoder::PixelmapEncode(ExtWStream& wStream)
354 {
355 uint32_t error;
356 #if defined(_WIN32) || defined(_APPLE) || defined(IOS_PLATFORM) || defined(ANDROID_PLATFORM)
357 IMAGE_LOGD("pixelmapEncode EncodeImageByPixelMap");
358 error = EncodeImageByPixelMap(pixelmap_, opts_.needsPackProperties, wStream);
359 #else
360 switch (opts_.desiredDynamicRange) {
361 case EncodeDynamicRange::AUTO:
362 if (pixelmap_->IsHdr() &&
363 (encodeFormat_ == SkEncodedImageFormat::kJPEG || encodeFormat_ == SkEncodedImageFormat::kHEIF)) {
364 error = EncodeDualVivid(wStream);
365 } else {
366 error = EncodeSdrImage(wStream);
367 }
368 break;
369 case EncodeDynamicRange::SDR:
370 error = EncodeSdrImage(wStream);
371 break;
372 case EncodeDynamicRange::HDR_VIVID_DUAL:
373 error = EncodeDualVivid(wStream);
374 break;
375 case EncodeDynamicRange::HDR_VIVID_SINGLE:
376 error = EncodeSingleVivid(wStream);
377 break;
378 default:
379 error = ERR_IMAGE_ENCODE_FAILED;
380 break;
381 }
382 #endif
383 RecycleResources();
384 return error;
385 }
386
FinalizeEncode()387 uint32_t ExtEncoder::FinalizeEncode()
388 {
389 if ((picture_ == nullptr && pixelmap_ == nullptr) || output_ == nullptr) {
390 return ERR_IMAGE_INVALID_PARAMETER;
391 }
392 ImageDataStatistics imageDataStatistics("[ExtEncoder]FinalizeEncode imageFormat = %s, quality = %d",
393 opts_.format.c_str(), opts_.quality);
394 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
395 if (IsAstc(opts_.format)) {
396 AstcCodec astcEncoder;
397 astcEncoder.SetAstcEncode(output_, opts_, pixelmap_);
398 return astcEncoder.ASTCEncode();
399 }
400 #endif
401 auto iter = FORMAT_NAME.find(LowerStr(opts_.format));
402 if (iter == FORMAT_NAME.end()) {
403 IMAGE_LOGE("ExtEncoder::FinalizeEncode unsupported format %{public}s", opts_.format.c_str());
404 ReportEncodeFault(0, 0, opts_.format, "Unsupported format:" + opts_.format);
405 return ERR_IMAGE_INVALID_PARAMETER;
406 }
407 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
408 if (picture_ != nullptr) {
409 encodeFormat_ = iter->second;
410 return EncodePicture();
411 }
412 #endif
413 ImageInfo imageInfo;
414 pixelmap_->GetImageInfo(imageInfo);
415 imageDataStatistics.AddTitle(", width = %d, height =%d", imageInfo.size.width, imageInfo.size.height);
416 encodeFormat_ = iter->second;
417 ExtWStream wStream(output_);
418 return PixelmapEncode(wStream);
419 }
420
IsHardwareEncodeSupported(const PlEncodeOptions & opts,Media::PixelMap * pixelMap)421 bool ExtEncoder::IsHardwareEncodeSupported(const PlEncodeOptions &opts, Media::PixelMap* pixelMap)
422 {
423 if (pixelMap == nullptr) {
424 IMAGE_LOGE("pixelMap is nullptr");
425 return false;
426 }
427 static const int32_t maxImageSize = 8196;
428 static const int32_t minImageSize = 128;
429 bool isSupport = ImageSystemProperties::GetHardWareEncodeEnabled() && opts.format == "image/jpeg" &&
430 (pixelMap->GetWidth() % 2 == 0) && (pixelMap->GetHeight() % 2 == 0) &&
431 (pixelMap->GetPixelFormat() == PixelFormat::NV12 || pixelMap->GetPixelFormat() == PixelFormat::NV21) &&
432 pixelMap->GetWidth() <= maxImageSize && pixelMap->GetHeight() <= maxImageSize &&
433 pixelMap->GetWidth() >= minImageSize && pixelMap->GetHeight() >= minImageSize;
434 if (!isSupport) {
435 IMAGE_LOGD("hardware encode is not support, dstEncodeFormat:%{public}s, pixelWidth:%{public}d, "
436 "pixelHeight:%{public}d, pixelFormat:%{public}d", opts.format.c_str(), pixelMap->GetWidth(),
437 pixelMap->GetHeight(), pixelMap->GetPixelFormat());
438 }
439 return isSupport;
440 }
441
DoHardWareEncode(SkWStream * skStream)442 uint32_t ExtEncoder::DoHardWareEncode(SkWStream* skStream)
443 {
444 static ImageFwkExtManager imageFwkExtManager;
445 if (imageFwkExtManager.doHardWareEncodeFunc_ != nullptr || imageFwkExtManager.LoadImageFwkExtNativeSo()) {
446 int32_t retCode = imageFwkExtManager.doHardWareEncodeFunc_(skStream, opts_, pixelmap_);
447 if (retCode == SUCCESS) {
448 return SUCCESS;
449 }
450 IMAGE_LOGE("hardware encode failed, retCode is %{public}d", retCode);
451 ImageInfo imageInfo;
452 pixelmap_->GetImageInfo(imageInfo);
453 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, "hardware encode failed");
454 } else {
455 IMAGE_LOGE("hardware encode failed because of load native so failed");
456 }
457 return ERR_IMAGE_ENCODE_FAILED;
458 }
459
DoEncode(SkWStream * skStream,const SkBitmap & src,const SkEncodedImageFormat & skFormat)460 uint32_t ExtEncoder::DoEncode(SkWStream* skStream, const SkBitmap& src, const SkEncodedImageFormat& skFormat)
461 {
462 ImageFuncTimer imageFuncTimer("%s:(%d, %d)", __func__, pixelmap_->GetWidth(), pixelmap_->GetHeight());
463 ImageInfo imageInfo;
464 pixelmap_->GetImageInfo(imageInfo);
465 if (!SkEncodeImage(skStream, src, skFormat, opts_.quality)) {
466 IMAGE_LOGE("Failed to encode image without exif data");
467 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, "Failed to encode image");
468 return ERR_IMAGE_ENCODE_FAILED;
469 }
470 return SUCCESS;
471 }
472
HardwareEncode(SkWStream & skStream,bool needExif)473 bool ExtEncoder::HardwareEncode(SkWStream &skStream, bool needExif)
474 {
475 uint32_t retCode = ERR_IMAGE_ENCODE_FAILED;
476 if (IsHardwareEncodeSupported(opts_, pixelmap_)) {
477 if (!needExif || pixelmap_->GetExifMetadata() == nullptr ||
478 pixelmap_->GetExifMetadata()->GetExifData() == nullptr) {
479 retCode = DoHardWareEncode(&skStream);
480 IMAGE_LOGD("HardwareEncode retCode:%{public}d", retCode);
481 return (retCode == SUCCESS);
482 }
483 MetadataWStream tStream;
484 retCode = DoHardWareEncode(&tStream);
485 if (retCode != SUCCESS) {
486 IMAGE_LOGD("HardwareEncode failed");
487 return false;
488 }
489 ImageInfo imageInfo;
490 pixelmap_->GetImageInfo(imageInfo);
491 retCode = CreateAndWriteBlob(tStream, pixelmap_, skStream, imageInfo, opts_);
492 IMAGE_LOGD("HardwareEncode retCode :%{public}d", retCode);
493 return (retCode == SUCCESS);
494 }
495 return (retCode == SUCCESS);
496 }
497
EncodeImageByBitmap(SkBitmap & bitmap,bool needExif,SkWStream & outStream)498 uint32_t ExtEncoder::EncodeImageByBitmap(SkBitmap& bitmap, bool needExif, SkWStream& outStream)
499 {
500 ImageInfo imageInfo;
501 pixelmap_->GetImageInfo(imageInfo);
502 if (!needExif || pixelmap_->GetExifMetadata() == nullptr ||
503 pixelmap_->GetExifMetadata()->GetExifData() == nullptr) {
504 return DoEncode(&outStream, bitmap, encodeFormat_);
505 }
506
507 MetadataWStream tStream;
508 uint32_t errCode = DoEncode(&tStream, bitmap, encodeFormat_);
509 if (errCode != SUCCESS) {
510 return errCode;
511 }
512 return CreateAndWriteBlob(tStream, pixelmap_, outStream, imageInfo, opts_);
513 }
514
EncodeImageByPixelMap(PixelMap * pixelMap,bool needExif,SkWStream & outputStream)515 uint32_t ExtEncoder::EncodeImageByPixelMap(PixelMap* pixelMap, bool needExif, SkWStream& outputStream)
516 {
517 if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
518 return EncodeHeifByPixelmap(pixelMap, opts_);
519 }
520 SkBitmap bitmap;
521 TmpBufferHolder holder;
522 SkImageInfo skInfo;
523 ImageData imageData;
524 pixelMap->GetImageInfo(imageData.info);
525 uint32_t width = static_cast<uint32_t>(imageData.info.size.width);
526 uint32_t height = static_cast<uint32_t>(imageData.info.size.height);
527
528 if (HardwareEncode(outputStream, needExif) == true) {
529 IMAGE_LOGD("HardwareEncode Success return");
530 return SUCCESS;
531 }
532 IMAGE_LOGD("HardwareEncode failed or not Supported");
533
534 std::unique_ptr<uint8_t[]> dstData;
535 uint64_t rowStride = 0;
536 if (IsYuvImage(imageData.info.pixelFormat)) {
537 IMAGE_LOGD("YUV format, convert to RGB");
538 dstData = std::make_unique<uint8_t[]>(width * height * NUM_4);
539 if (YuvToRgbaSkInfo(imageData.info, skInfo, dstData.get(), pixelMap) != SUCCESS) {
540 IMAGE_LOGD("YUV format, convert to RGB fail");
541 return ERR_IMAGE_ENCODE_FAILED;
542 }
543 imageData.pixels = dstData.get();
544 rowStride = skInfo.minRowBytes64();
545 } else {
546 dstData = std::make_unique<uint8_t[]>(width * height * NUM_3);
547 imageData.dst = dstData.get();
548 if (pixelToSkInfo(imageData, skInfo, pixelMap, holder, encodeFormat_) != SUCCESS) {
549 IMAGE_LOGE("ExtEncoder::EncodeImageByPixelMap pixel convert failed");
550 return ERR_IMAGE_ENCODE_FAILED;
551 }
552 rowStride = skInfo.minRowBytes64();
553 #if !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
554 if (pixelMap->GetAllocatorType() == AllocatorType::DMA_ALLOC) {
555 SurfaceBuffer* sbBuffer = reinterpret_cast<SurfaceBuffer*> (pixelMap->GetFd());
556 rowStride = static_cast<uint64_t>(sbBuffer->GetStride());
557 IMAGE_LOGD("rowStride DMA: %{public}llu", static_cast<unsigned long long>(rowStride));
558 }
559 #endif
560 }
561 if (!bitmap.installPixels(skInfo, imageData.pixels, rowStride)) {
562 IMAGE_LOGE("ExtEncoder::EncodeImageByPixelMap to SkBitmap failed");
563 return ERR_IMAGE_ENCODE_FAILED;
564 }
565 return EncodeImageByBitmap(bitmap, needExif, outputStream);
566 }
567
EncodeHeifByPixelmap(PixelMap * pixelmap,const PlEncodeOptions & opts)568 uint32_t ExtEncoder::EncodeHeifByPixelmap(PixelMap* pixelmap, const PlEncodeOptions& opts)
569 {
570 if (output_ == nullptr) {
571 return ERR_IMAGE_INVALID_PARAMETER;
572 }
573 Media::PixelFormat format = pixelmap->GetPixelFormat();
574 if (format != PixelFormat::RGBA_8888 && format != PixelFormat::NV12 && format != PixelFormat::NV21) {
575 IMAGE_LOGE("EncodeHeifByPixelmap, invalid format:%{public}d", format);
576 return ERR_IMAGE_INVALID_PARAMETER;
577 }
578 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM) && \
579 defined(HEIF_HW_ENCODE_ENABLE)
580 sptr<SurfaceBuffer> surfaceBuffer;
581 bool needConvertToSurfaceBuffer = pixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC;
582 if (needConvertToSurfaceBuffer) {
583 surfaceBuffer = ConvertToSurfaceBuffer(pixelmap);
584 if (surfaceBuffer == nullptr) {
585 IMAGE_LOGE("EncodeHeifByPixelmap ConvertToSurfaceBuffer failed");
586 return ERR_IMAGE_INVALID_PARAMETER;
587 }
588 } else {
589 if (pixelmap->GetFd() == nullptr) {
590 IMAGE_LOGE("EncodeHeifByPixelmap pixelmap get fd failed");
591 return ERR_IMAGE_INVALID_PARAMETER;
592 }
593 surfaceBuffer = sptr<SurfaceBuffer>(reinterpret_cast<SurfaceBuffer*>(pixelmap->GetFd()));
594 }
595 std::vector<ImageItem> inputImgs;
596 std::shared_ptr<ImageItem> primaryItem = AssemblePrimaryImageItem(surfaceBuffer, opts);
597 if (primaryItem == nullptr) {
598 if (needConvertToSurfaceBuffer) {
599 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
600 }
601 return ERR_IMAGE_INVALID_PARAMETER;
602 }
603 inputImgs.push_back(*primaryItem);
604 std::vector<MetaItem> inputMetas;
605 std::vector<ItemRef> refs;
606 if (AssembleExifMetaItem(inputMetas)) {
607 AssembleExifRefItem(refs);
608 }
609 uint32_t result = DoHeifEncode(inputImgs, inputMetas, refs);
610 if (needConvertToSurfaceBuffer) {
611 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
612 }
613 return result;
614 #endif
615 return ERR_IMAGE_INVALID_PARAMETER;
616 }
617
RecycleResources()618 void ExtEncoder::RecycleResources()
619 {
620 for (std::shared_ptr<AbsMemory> &memory: tmpMemoryList_) {
621 if (memory == nullptr) {
622 continue;
623 }
624 memory->Release();
625 }
626 tmpMemoryList_.clear();
627 }
628
629 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
AllocSurfaceBuffer(int32_t width,int32_t height,uint32_t format=GRAPHIC_PIXEL_FMT_RGBA_8888)630 static sptr<SurfaceBuffer> AllocSurfaceBuffer(int32_t width, int32_t height,
631 uint32_t format = GRAPHIC_PIXEL_FMT_RGBA_8888)
632 {
633 sptr<SurfaceBuffer> sb = SurfaceBuffer::Create();
634 BufferRequestConfig requestConfig = {
635 .width = width,
636 .height = height,
637 .strideAlignment = 0x8,
638 .format = format,
639 .usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA | BUFFER_USAGE_MEM_MMZ_CACHE,
640 .timeout = 0,
641 };
642 GSError ret = sb->Alloc(requestConfig);
643 if (ret != GSERROR_OK) {
644 return nullptr;
645 }
646 void* nativeBuffer = sb.GetRefPtr();
647 int32_t err = ImageUtils::SurfaceBuffer_Reference(nativeBuffer);
648 if (err != OHOS::GSERROR_OK) {
649 return nullptr;
650 }
651 return sb;
652 }
653
ConvertToSurfaceBuffer(PixelMap * pixelmap)654 sptr<SurfaceBuffer> ExtEncoder::ConvertToSurfaceBuffer(PixelMap* pixelmap)
655 {
656 if (pixelmap->GetHeight() <= 0 || pixelmap->GetWidth() <= 0) {
657 IMAGE_LOGE("pixelmap height:%{public}d or width:%{public}d error", pixelmap->GetHeight(), pixelmap->GetWidth());
658 return nullptr;
659 }
660 uint32_t height = static_cast<uint32_t>(pixelmap->GetHeight());
661 uint32_t width = static_cast<uint32_t>(pixelmap->GetWidth());
662 PixelFormat format = pixelmap->GetPixelFormat();
663 IMAGE_LOGD("ExtEncoder::ConvertToSurfaceBuffer format is %{public}d", format);
664 auto formatSearch = SURFACE_FORMAT_MAP.find(format);
665 if (formatSearch == SURFACE_FORMAT_MAP.end()) {
666 IMAGE_LOGE("format:[%{public}d] is not in SURFACE_FORMAT_MAP", format);
667 return nullptr;
668 }
669 uint32_t graphicFormat = formatSearch->second;
670 sptr<SurfaceBuffer> surfaceBuffer = AllocSurfaceBuffer(width, height, graphicFormat);
671 if (surfaceBuffer == nullptr || surfaceBuffer->GetStride() < 0) {
672 IMAGE_LOGE("ConvertToSurfaceBuffer surfaceBuffer is nullptr failed");
673 return nullptr;
674 }
675 uint32_t dstStride = static_cast<uint32_t>(surfaceBuffer->GetStride());
676 uint8_t* src = const_cast<uint8_t*>(pixelmap->GetPixels());
677 uint8_t* dst = static_cast<uint8_t*>(surfaceBuffer->GetVirAddr());
678 uint32_t dstSize = surfaceBuffer->GetSize();
679 uint32_t copyHeight = height;
680 uint64_t srcStride = width;
681 if (format == PixelFormat::NV12 || format == PixelFormat::NV21) {
682 const int32_t NUM_2 = 2;
683 copyHeight = height + height / NUM_2;
684 srcStride = width;
685 } else if (format == PixelFormat::RGBA_8888) {
686 copyHeight = height;
687 srcStride = static_cast<uint64_t>(width * NUM_4);
688 }
689 for (uint32_t i = 0; i < copyHeight; i++) {
690 if (memcpy_s(dst, dstSize, src, srcStride) != EOK) {
691 IMAGE_LOGE("ConvertToSurfaceBuffer memcpy failed");
692 ImageUtils::SurfaceBuffer_Unreference(surfaceBuffer.GetRefPtr());
693 return nullptr;
694 }
695 dst += dstStride;
696 dstSize -= dstStride;
697 src += srcStride;
698 }
699 return surfaceBuffer;
700 }
701
FreeBaseAndGainMapSurfaceBuffer(sptr<SurfaceBuffer> & base,sptr<SurfaceBuffer> & gainMap)702 static void FreeBaseAndGainMapSurfaceBuffer(sptr<SurfaceBuffer>& base, sptr<SurfaceBuffer>& gainMap)
703 {
704 ImageUtils::SurfaceBuffer_Unreference(base.GetRefPtr());
705 ImageUtils::SurfaceBuffer_Unreference(gainMap.GetRefPtr());
706 }
707
GetHdrMetadata(sptr<SurfaceBuffer> & hdr,sptr<SurfaceBuffer> & gainmap)708 static HdrMetadata GetHdrMetadata(sptr<SurfaceBuffer>& hdr, sptr<SurfaceBuffer>& gainmap)
709 {
710 std::vector<uint8_t> dynamicMetadata = {};
711 VpeUtils::GetSbDynamicMetadata(hdr, dynamicMetadata);
712 std::vector<uint8_t> staticMetadata = {};
713 VpeUtils::GetSbStaticMetadata(hdr, staticMetadata);
714 HdrMetadata metadata = {
715 .staticMetadata = staticMetadata,
716 .dynamicMetadata = dynamicMetadata
717 };
718 std::vector<uint8_t> extendMetadataVec = {};
719 VpeUtils::GetSbDynamicMetadata(gainmap, extendMetadataVec);
720 if (extendMetadataVec.size() != sizeof(HDRVividExtendMetadata)) {
721 metadata.extendMetaFlag = false;
722 return metadata;
723 }
724 if (memcpy_s(&metadata.extendMeta, sizeof(HDRVividExtendMetadata),
725 extendMetadataVec.data(), extendMetadataVec.size()) != EOK) {
726 metadata.extendMetaFlag = false;
727 } else {
728 metadata.extendMetaFlag = true;
729 }
730 return metadata;
731 }
732
EncodeImageBySurfaceBuffer(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo info,bool needExif,SkWStream & outputStream)733 uint32_t ExtEncoder::EncodeImageBySurfaceBuffer(sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo info,
734 bool needExif, SkWStream& outputStream)
735 {
736 SkBitmap bitmap;
737 ImageInfo imageInfo;
738 uint64_t rowStride = 0;
739 if (surfaceBuffer == nullptr) {
740 IMAGE_LOGE("EncodeImageBySurfaceBuffer failed, surfaceBuffer is nullptr");
741 return ERR_IMAGE_INVALID_PARAMETER;
742 }
743 auto pixels = surfaceBuffer->GetVirAddr();
744 if (pixels == nullptr) {
745 IMAGE_LOGE("EncodeImageBySurfaceBuffer failed, pixels is nullptr");
746 return ERR_IMAGE_INVALID_PARAMETER;
747 }
748 /* do hardwareEncode first, if fail then soft encode*/
749 if (HardwareEncode(outputStream, needExif)) {
750 IMAGE_LOGD("HardwareEncode Success return");
751 return SUCCESS;
752 }
753 IMAGE_LOGD("HardwareEncode failed or not Supported");
754
755 pixelmap_->GetImageInfo(imageInfo);
756 std::unique_ptr<uint8_t[]> dstData;
757 if (IsYuvImage(imageInfo.pixelFormat)) {
758 IMAGE_LOGD("EncodeImageBySurfaceBuffer: YUV format, convert to RGB first");
759 dstData = std::make_unique<uint8_t[]>(imageInfo.size.width * imageInfo.size.height * NUM_4);
760 if (dstData == nullptr) {
761 IMAGE_LOGE("Memory allocate fail");
762 return ERR_IMAGE_ENCODE_FAILED;
763 }
764 if (YuvToRgbaSkInfo(imageInfo, info, dstData.get(), pixelmap_) != SUCCESS) {
765 IMAGE_LOGD("YUV format, convert to RGB fail");
766 return ERR_IMAGE_ENCODE_FAILED;
767 }
768 pixels = dstData.get();
769 rowStride = info.minRowBytes64();
770 } else {
771 rowStride = surfaceBuffer->GetStride();
772 }
773
774 if (!bitmap.installPixels(info, pixels, rowStride)) {
775 IMAGE_LOGE("ExtEncoder::EncodeImageBySurfaceBuffer to SkBitmap failed");
776 return ERR_IMAGE_ENCODE_FAILED;
777 }
778 return EncodeImageByBitmap(bitmap, needExif, outputStream);
779 }
780
GetImageEncodeData(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo info,bool needExif)781 sk_sp<SkData> ExtEncoder::GetImageEncodeData(sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo info, bool needExif)
782 {
783 SkDynamicMemoryWStream stream;
784 if (EncodeImageBySurfaceBuffer(surfaceBuffer, info, needExif, stream) != SUCCESS) {
785 return nullptr;
786 }
787 return stream.detachAsData();
788 }
789
DecomposeImage(VpeSurfaceBuffers & buffers,HdrMetadata & metadata,bool onlySdr,bool sdrIsSRGB=false)790 static bool DecomposeImage(VpeSurfaceBuffers& buffers, HdrMetadata& metadata, bool onlySdr, bool sdrIsSRGB = false)
791 {
792 VpeUtils::SetSbMetadataType(buffers.sdr, CM_IMAGE_HDR_VIVID_DUAL);
793 VpeUtils::SetSbColorSpaceType(buffers.sdr, sdrIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
794 std::unique_ptr<VpeUtils> utils = std::make_unique<VpeUtils>();
795 int32_t res;
796 if (onlySdr) {
797 if (buffers.hdr == nullptr || buffers.sdr == nullptr) {
798 return false;
799 }
800 res = utils->ColorSpaceConverterImageProcess(buffers.hdr, buffers.sdr);
801 } else {
802 VpeUtils::SetSbMetadataType(buffers.gainmap, CM_METADATA_NONE);
803 VpeUtils::SetSbColorSpaceType(buffers.gainmap, sdrIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
804 res = utils->ColorSpaceConverterDecomposeImage(buffers);
805 }
806 if (res != VPE_ERROR_OK) {
807 IMAGE_LOGE("DecomposeImage [%{public}d] failed, res = %{public}d", onlySdr, res);
808 return false;
809 }
810 if (!onlySdr) {
811 metadata = GetHdrMetadata(buffers.hdr, buffers.gainmap);
812 }
813 return true;
814 }
815
GetSkInfo(PixelMap * pixelMap,bool isGainmap,bool isSRGB=false)816 static SkImageInfo GetSkInfo(PixelMap* pixelMap, bool isGainmap, bool isSRGB = false)
817 {
818 ImageInfo info;
819 pixelMap->GetImageInfo(info);
820 SkColorType colorType = kRGBA_8888_SkColorType;
821 SkAlphaType alphaType = ImageTypeConverter::ToSkAlphaType(info.alphaType);
822 sk_sp<SkColorSpace> colorSpace =
823 SkColorSpace::MakeRGB(SkNamedTransferFn::kSRGB, isSRGB ? SkNamedGamut::kSRGB : SkNamedGamut::kDisplayP3);
824 int32_t width = info.size.width;
825 int32_t height = info.size.height;
826 if (isGainmap) {
827 const int halfSizeDenominator = 2;
828 width = width / halfSizeDenominator;
829 height = height / halfSizeDenominator;
830 #ifdef IMAGE_COLORSPACE_FLAG
831 if (pixelMap->InnerGetGrColorSpacePtr() != nullptr &&
832 pixelMap->InnerGetGrColorSpace().GetColorSpaceName() != ColorManager::ColorSpaceName::NONE) {
833 colorSpace = pixelMap->InnerGetGrColorSpacePtr()->ToSkColorSpace();
834 }
835 skcms_CICP cicp;
836 ColorUtils::ColorSpaceGetCicp(pixelMap->InnerGetGrColorSpace().GetColorSpaceName(),
837 cicp.colour_primaries, cicp.transfer_characteristics, cicp.matrix_coefficients, cicp.full_range_flag);
838 colorSpace->SetIccCicp(cicp);
839 #endif
840 }
841 return SkImageInfo::Make(width, height, colorType, alphaType, colorSpace);
842 }
843
EncodeSingleVivid(ExtWStream & outputStream)844 uint32_t ExtEncoder::EncodeSingleVivid(ExtWStream& outputStream)
845 {
846 return ERR_IMAGE_INVALID_PARAMETER;
847 }
848
849 #ifdef HEIF_HW_ENCODE_ENABLE
850 // format: enum + struct
FillLitePropertyItem(std::vector<uint8_t> & properties,size_t & offset,PropertyType type,const void * payloadData,size_t payloadSize)851 static bool FillLitePropertyItem(std::vector<uint8_t>& properties, size_t& offset, PropertyType type,
852 const void* payloadData, size_t payloadSize)
853 {
854 uint8_t* memData = properties.data();
855 if (memData == nullptr) {
856 IMAGE_LOGE("FillLitePropertyItem memData is nullptr");
857 return false;
858 }
859 size_t memSize = properties.size();
860 size_t typeSize = sizeof(type);
861 size_t endOffset = offset + typeSize + payloadSize;
862 if (endOffset > memSize) {
863 IMAGE_LOGE("FillLitePropertyItem, endOffset[%{public}ld] over memSize[%{public}ld]", endOffset, memSize);
864 return false;
865 }
866 bool res = memcpy_s(memData + offset, memSize - offset, &type, typeSize) == EOK &&
867 memcpy_s(memData + offset + typeSize, memSize - offset - typeSize, payloadData, payloadSize) == EOK;
868 if (res) {
869 offset = endOffset;
870 }
871 return res;
872 }
873
874 // format: enum + strlen + str
FillLitePropertyItemByString(std::vector<uint8_t> & properties,size_t & offset,PropertyType type,std::string & payloadString)875 static bool FillLitePropertyItemByString(std::vector<uint8_t>& properties, size_t& offset, PropertyType type,
876 std::string& payloadString)
877 {
878 uint8_t* memData = properties.data();
879 if (memData == nullptr) {
880 IMAGE_LOGE("%{public}s memData is nullptr", __func__);
881 return false;
882 }
883 size_t memSize = properties.size();
884 if (payloadString == "") {
885 IMAGE_LOGE("%{public}s failed, payloadString is Null", __func__);
886 return false;
887 }
888 size_t lenSize = UINT32_BYTES_NUM;
889 size_t strLen = payloadString.length();
890 size_t typeSize = sizeof(type);
891 size_t endOffset = offset + typeSize + lenSize + strLen;
892 if (endOffset >= memSize) {
893 IMAGE_LOGE("%{public}s failed, offset[%{public}ld] over memSize[%{public}ld]", __func__, offset, memSize);
894 return false;
895 }
896 size_t capacitySize = memSize - offset;
897 const char* payloadData = payloadString.data();
898 if (memcpy_s(memData + offset, capacitySize, &type, typeSize) != EOK || memcpy_s(memData + offset + typeSize,
899 capacitySize - typeSize, &strLen, lenSize) != EOK || memcpy_s(memData + offset + typeSize + lenSize,
900 capacitySize - typeSize - lenSize, payloadData, strLen) != EOK) {
901 IMAGE_LOGE("%{public}s copy failed, offset[%{public}ld] over memSize[%{public}d]", __func__, offset, memSize);
902 return false;
903 }
904
905 properties[endOffset] = '\0';
906 offset = endOffset + PLACE_HOLDER_LENGTH;
907 return true;
908 }
909
AssembleHdrBaseImageItem(sptr<SurfaceBuffer> & surfaceBuffer,ColorManager::ColorSpaceName color,HdrMetadata & metadata,const PlEncodeOptions & opts)910 std::shared_ptr<ImageItem> ExtEncoder::AssembleHdrBaseImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
911 ColorManager::ColorSpaceName color, HdrMetadata& metadata, const PlEncodeOptions& opts)
912 {
913 if (surfaceBuffer == nullptr) {
914 IMAGE_LOGI("AssembleHdrBaseImageItem surfaceBuffer is nullptr");
915 return nullptr;
916 }
917 auto item = std::make_shared<ImageItem>();
918 item->id = PRIMARY_IMAGE_ITEM_ID;
919 item->isPrimary = true;
920 item->isHidden = false;
921 item->compressType = COMPRESS_TYPE_HEVC;
922 item->quality = opts.quality;
923 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
924 item->sharedProperties.fd = -1;
925 item->pixelSharedBuffer.fd = -1;
926 ColourInfo colorInfo;
927 GetColourInfo(color, colorInfo);
928 ContentLightLevel light;
929 MasteringDisplayColourVolume colour;
930 bool hasLight = GetStaticMetadata(metadata, colour, light);
931 uint32_t propertiesSize = 0;
932 propertiesSize +=
933 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
934 if (hasLight) {
935 propertiesSize += (sizeof(PropertyType::CONTENT_LIGHT_LEVEL) + sizeof(ContentLightLevel));
936 }
937 item->liteProperties.resize(propertiesSize);
938 size_t offset = 0;
939 if (!FillNclxColorProperty(item, offset, colorInfo)) {
940 return nullptr;
941 }
942 if (hasLight && (!FillLitePropertyItem(item->liteProperties, offset,
943 PropertyType::CONTENT_LIGHT_LEVEL, &colour, sizeof(ContentLightLevel)))) {
944 return nullptr;
945 }
946 return item;
947 }
948
AssembleGainmapImageItem(sptr<SurfaceBuffer> & surfaceBuffer,ColorManager::ColorSpaceName color,const PlEncodeOptions & opts)949 std::shared_ptr<ImageItem> ExtEncoder::AssembleGainmapImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
950 ColorManager::ColorSpaceName color, const PlEncodeOptions &opts)
951 {
952 if (surfaceBuffer == nullptr) {
953 IMAGE_LOGI("AssembleGainmapImageItem surfacebuffer is nullptr");
954 return nullptr;
955 }
956 auto item = std::make_shared<ImageItem>();
957 item->id = GAINMAP_IMAGE_ITEM_ID;
958 item->itemName = GAINMAP_IMAGE_ITEM_NAME;
959 item->isPrimary = false;
960 item->isHidden = true;
961 item->compressType = COMPRESS_TYPE_HEVC;
962 item->quality = opts.quality;
963 item->sharedProperties.fd = -1;
964 item->pixelSharedBuffer.fd = -1;
965 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
966 ColourInfo colorInfo;
967 GetColourInfo(color, colorInfo);
968 uint32_t propertiesSize = 0;
969 propertiesSize +=
970 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
971 item->liteProperties.resize(propertiesSize);
972 size_t offset = 0;
973 if (!FillNclxColorProperty(item, offset, colorInfo)) {
974 return nullptr;
975 }
976 return item;
977 }
978
AssembleHeifHdrPicture(sptr<SurfaceBuffer> & mainSptr,bool sdrIsSRGB,std::vector<ImageItem> & inputImgs)979 uint32_t ExtEncoder::AssembleHeifHdrPicture(
980 sptr<SurfaceBuffer>& mainSptr, bool sdrIsSRGB, std::vector<ImageItem>& inputImgs)
981 {
982 auto gainPixelMap = picture_->GetGainmapPixelMap();
983 if (gainPixelMap == nullptr || gainPixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
984 IMAGE_LOGE("%{public}s, the gainPixelMap is nullptr or gainPixelMap is nonDMA", __func__);
985 return ERR_IMAGE_INVALID_PARAMETER;
986 }
987 sptr<SurfaceBuffer> gainMapSptr(reinterpret_cast<SurfaceBuffer*>(gainPixelMap->GetFd()));
988 HdrMetadata metadata = GetHdrMetadata(mainSptr, gainMapSptr);
989
990 ColorManager::ColorSpaceName colorspaceName =
991 sdrIsSRGB ? ColorManager::ColorSpaceName::SRGB : ColorManager::ColorSpaceName::DISPLAY_P3;
992 std::shared_ptr<ImageItem> primaryItem = AssembleHdrBaseImageItem(mainSptr, colorspaceName, metadata, opts_);
993 if (primaryItem == nullptr) {
994 IMAGE_LOGE("%{public}s, get primary image failed", __func__);
995 return ERR_IMAGE_INVALID_PARAMETER;
996 }
997 inputImgs.push_back(*primaryItem);
998 std::shared_ptr<ImageItem> gainmapItem = AssembleGainmapImageItem(gainMapSptr, colorspaceName, opts_);
999 if (gainmapItem == nullptr) {
1000 IMAGE_LOGE("%{public}s, get gainmap image item failed", __func__);
1001 return ERR_IMAGE_INVALID_PARAMETER;
1002 }
1003 inputImgs.push_back(*gainmapItem);
1004 ColorManager::ColorSpaceName tmapColor = picture_->GetMainPixel()->InnerGetGrColorSpace().GetColorSpaceName();
1005 std::shared_ptr<ImageItem> tmapItem = AssembleTmapImageItem(tmapColor, metadata, opts_);
1006 if (tmapItem == nullptr) {
1007 IMAGE_LOGE("%{public}s, get tmap image item failed", __func__);
1008 return ERR_IMAGE_INVALID_PARAMETER;
1009 }
1010 inputImgs.push_back(*tmapItem);
1011 return SUCCESS;
1012 }
1013
AssembleSdrImageItem(sptr<SurfaceBuffer> & surfaceBuffer,SkImageInfo sdrInfo,std::vector<ImageItem> & inputImgs)1014 uint32_t ExtEncoder::AssembleSdrImageItem(
1015 sptr<SurfaceBuffer>& surfaceBuffer, SkImageInfo sdrInfo, std::vector<ImageItem>& inputImgs)
1016 {
1017 if (surfaceBuffer == nullptr) {
1018 IMAGE_LOGI("%{public}s surfaceBuffer is nullptr", __func__);
1019 return ERR_IMAGE_INVALID_PARAMETER;
1020 }
1021 ImageItem item;
1022 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo);
1023 if (!AssembleICCImageProperty(iccProfile, item.sharedProperties)) {
1024 IMAGE_LOGE("%{public}s AssembleICCImageProperty failed", __func__);
1025 return ERR_IMAGE_INVALID_PARAMETER;
1026 }
1027 item.id = PRIMARY_IMAGE_ITEM_ID;
1028 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
1029 item.isPrimary = true;
1030 item.isHidden = false;
1031 item.compressType = COMPRESS_TYPE_HEVC;
1032 item.quality = opts_.quality;
1033 uint32_t litePropertiesSize = (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1034 item.liteProperties.resize(litePropertiesSize);
1035 size_t offset = 0;
1036 ColorType colorType = ColorType::RICC;
1037 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
1038 IMAGE_LOGE("%{public}s Fill color type failed", __func__);
1039 return ERR_IMAGE_INVALID_PARAMETER;
1040 }
1041 inputImgs.push_back(item);
1042 return SUCCESS;
1043 }
1044
AssembleHeifAuxiliaryPicture(std::vector<ImageItem> & inputImgs,std::vector<ItemRef> & refs)1045 uint32_t ExtEncoder::AssembleHeifAuxiliaryPicture(std::vector<ImageItem>& inputImgs, std::vector<ItemRef>& refs)
1046 {
1047 if (!picture_) {
1048 IMAGE_LOGE("picture_ is nullptr");
1049 return ERR_IMAGE_INVALID_PARAMETER;
1050 }
1051 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::DEPTH_MAP)) {
1052 if (AssembleHeifAuxiliaryNoncodingMap(inputImgs, AuxiliaryPictureType::DEPTH_MAP) == SUCCESS) {
1053 AssembleAuxiliaryRefItem(AuxiliaryPictureType::DEPTH_MAP, refs);
1054 } else {
1055 IMAGE_LOGE("%{public}s depthMap assemble fail", __func__);
1056 }
1057 }
1058 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::UNREFOCUS_MAP)) {
1059 if (AssembleHeifUnrefocusMap(inputImgs) == SUCCESS) {
1060 AssembleAuxiliaryRefItem(AuxiliaryPictureType::UNREFOCUS_MAP, refs);
1061 } else {
1062 IMAGE_LOGE("%{public}s unrefocusMap assemble fail", __func__);
1063 }
1064 }
1065 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::LINEAR_MAP)) {
1066 if (AssembleHeifAuxiliaryNoncodingMap(inputImgs, AuxiliaryPictureType::LINEAR_MAP) == SUCCESS) {
1067 AssembleAuxiliaryRefItem(AuxiliaryPictureType::LINEAR_MAP, refs);
1068 } else {
1069 IMAGE_LOGE("%{public}s linearMap assemble fail", __func__);
1070 }
1071 }
1072 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP)) {
1073 if (AssembleHeifFragmentMap(inputImgs) == SUCCESS) {
1074 AssembleAuxiliaryRefItem(AuxiliaryPictureType::FRAGMENT_MAP, refs);
1075 } else {
1076 IMAGE_LOGE("%{public}s fragmentMap assemble fail", __func__);
1077 }
1078 }
1079 return SUCCESS;
1080 }
1081
InitAuxiliaryImageItem(uint32_t id,std::string itemName)1082 ImageItem ExtEncoder::InitAuxiliaryImageItem(uint32_t id, std::string itemName)
1083 {
1084 ImageItem item;
1085 item.id = id;
1086 item.itemName = itemName;
1087 item.isPrimary = false;
1088 item.isHidden = true;
1089 item.quality = opts_.quality;
1090 item.sharedProperties.fd = -1;
1091 item.pixelSharedBuffer.fd = -1;
1092 if (id == DEPTH_MAP_ITEM_ID || id == LINEAR_MAP_ITEM_ID) {
1093 item.compressType = COMPRESS_TYPE_NONE;
1094 } else {
1095 item.compressType = COMPRESS_TYPE_HEVC;
1096 }
1097 return item;
1098 }
1099
GetHeifEncodeItemInfo(AuxiliaryPictureType auxType)1100 HeifEncodeItemInfo GetHeifEncodeItemInfo(AuxiliaryPictureType auxType)
1101 {
1102 HeifEncodeItemInfo itemInfo;
1103 switch (auxType) {
1104 case AuxiliaryPictureType::GAINMAP:
1105 itemInfo.itemId = GAINMAP_IMAGE_ITEM_ID;
1106 itemInfo.itemName = GAINMAP_IMAGE_ITEM_NAME;
1107 itemInfo.itemType = HEIF_AUXTTYPE_ID_GAINMAP;
1108 break;
1109 case AuxiliaryPictureType::DEPTH_MAP:
1110 itemInfo.itemId = DEPTH_MAP_ITEM_ID;
1111 itemInfo.itemName = DEPTH_MAP_ITEM_NAME;
1112 itemInfo.itemType = HEIF_AUXTTYPE_ID_DEPTH_MAP;
1113 break;
1114 case AuxiliaryPictureType::UNREFOCUS_MAP:
1115 itemInfo.itemId = UNREFOCUS_MAP_ITEM_ID;
1116 itemInfo.itemName = UNREFOCUS_MAP_ITEM_NAME;
1117 itemInfo.itemType = HEIF_AUXTTYPE_ID_UNREFOCUS_MAP;
1118 break;
1119 case AuxiliaryPictureType::LINEAR_MAP:
1120 itemInfo.itemId = LINEAR_MAP_ITEM_ID;
1121 itemInfo.itemName = LINEAR_MAP_ITEM_NAME;
1122 itemInfo.itemType = HEIF_AUXTTYPE_ID_LINEAR_MAP;
1123 break;
1124 case AuxiliaryPictureType::FRAGMENT_MAP:
1125 itemInfo.itemId = FRAGMENT_MAP_ITEM_ID;
1126 itemInfo.itemName = FRAGMENT_MAP_ITEM_NAME;
1127 itemInfo.itemType = HEIF_AUXTTYPE_ID_FRAGMENT_MAP;
1128 break;
1129 default:
1130 break;
1131 }
1132 return itemInfo;
1133 }
1134
AssembleHeifAuxiliaryNoncodingMap(std::vector<ImageItem> & inputImgs,AuxiliaryPictureType auxType)1135 uint32_t ExtEncoder::AssembleHeifAuxiliaryNoncodingMap(std::vector<ImageItem>& inputImgs, AuxiliaryPictureType auxType)
1136 {
1137 auto auxMap = picture_->GetAuxiliaryPicture(auxType);
1138 if (!auxMap || !auxMap->GetContentPixel() ||
1139 auxMap->GetContentPixel()->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1140 IMAGE_LOGE("%{public}s The auxMap is nullptr or allocator type is not DMA_ALLOC", __func__);
1141 return ERR_IMAGE_INVALID_PARAMETER;
1142 }
1143 sptr<SurfaceBuffer> auxMapSptr(reinterpret_cast<SurfaceBuffer*>(auxMap->GetContentPixel()->GetFd()));
1144 if (!auxMapSptr) {
1145 IMAGE_LOGE("%{public}s auxMapSptr is nullptr", __func__);
1146 return ERR_IMAGE_INVALID_PARAMETER;
1147 }
1148
1149 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(auxType);
1150 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1151 std::string auxTypeStr = itemInfo.itemType;
1152 Resolution resolution = {
1153 auxMap->GetContentPixel()->GetWidth(),
1154 auxMap->GetContentPixel()->GetHeight()
1155 };
1156 uint32_t litePropertiesSize =
1157 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1158 litePropertiesSize += (sizeof(PropertyType::IMG_RESOLUTION) + sizeof(Resolution));
1159 size_t offset = 0;
1160 item.liteProperties.resize(litePropertiesSize);
1161 if (!FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr)) {
1162 IMAGE_LOGE("%{public}s Fill auxiliary type failed", __func__);
1163 return ERR_IMAGE_INVALID_PARAMETER;
1164 }
1165 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::IMG_RESOLUTION, &resolution,
1166 sizeof(Resolution))) {
1167 IMAGE_LOGE("%{public}s Fill color type failed", __func__);
1168 return ERR_IMAGE_INVALID_PARAMETER;
1169 }
1170 uint32_t capacity = auxMap->GetContentPixel()->GetCapacity();
1171 if (!FillPixelSharedBuffer(auxMapSptr, capacity, item.pixelSharedBuffer)) {
1172 IMAGE_LOGE("%{public}s Fill pixel shared buffer failed", __func__);
1173 return ERR_IMAGE_INVALID_PARAMETER;
1174 }
1175 inputImgs.push_back(item);
1176 return SUCCESS;
1177 }
1178
AssembleHeifUnrefocusMap(std::vector<ImageItem> & inputImgs)1179 uint32_t ExtEncoder::AssembleHeifUnrefocusMap(std::vector<ImageItem>& inputImgs)
1180 {
1181 auto unrefocusMap = picture_->GetAuxiliaryPicture(AuxiliaryPictureType::UNREFOCUS_MAP);
1182 if (!unrefocusMap || !unrefocusMap->GetContentPixel() ||
1183 unrefocusMap->GetContentPixel()->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1184 IMAGE_LOGE("%{public}s The unrefocusMap is nullptr or allocator type is not DMA_ALLOC", __func__);
1185 return ERR_IMAGE_INVALID_PARAMETER;
1186 }
1187 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(AuxiliaryPictureType::UNREFOCUS_MAP);
1188 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1189 bool sdrIsSRGB = unrefocusMap->GetContentPixel()->GetToSdrColorSpaceIsSRGB();
1190 SkImageInfo unrefocusInfo = GetSkInfo(unrefocusMap->GetContentPixel().get(), false, sdrIsSRGB);
1191 sk_sp<SkData> iccProfile = icc_from_color_space(unrefocusInfo);
1192 if (!AssembleICCImageProperty(iccProfile, item.sharedProperties)) {
1193 IMAGE_LOGE("%{public}s AssembleICCImageProperty failed", __func__);
1194 return ERR_IMAGE_INVALID_PARAMETER;
1195 }
1196 sptr<SurfaceBuffer> unrefocusMapSptr(reinterpret_cast<SurfaceBuffer*>(unrefocusMap->GetContentPixel()->GetFd()));
1197 if (!unrefocusMapSptr) {
1198 IMAGE_LOGE("%{public}s unrefocusMapSptr is nullptr", __func__);
1199 return ERR_IMAGE_INVALID_PARAMETER;
1200 }
1201 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(unrefocusMapSptr->GetBufferHandle());
1202 std::string auxTypeStr = itemInfo.itemType;
1203
1204 uint32_t litePropertiesSize =
1205 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1206 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1207 item.liteProperties.resize(litePropertiesSize);
1208 size_t offset = 0;
1209 if (!FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr)) {
1210 IMAGE_LOGE("%{public}s Fill auxiliary type failed", __func__);
1211 return ERR_IMAGE_INVALID_PARAMETER;
1212 }
1213 ColorType colorType = ColorType::RICC;
1214 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
1215 IMAGE_LOGE("%{public}s Fill color type failed", __func__);
1216 return ERR_IMAGE_INVALID_PARAMETER;
1217 }
1218 inputImgs.push_back(item);
1219 return SUCCESS;
1220 }
1221
GetFragmentRelLocation(std::shared_ptr<AuxiliaryPicture> & fragmentMap)1222 RelativeLocation GetFragmentRelLocation(std::shared_ptr<AuxiliaryPicture> &fragmentMap)
1223 {
1224 RelativeLocation loc;
1225 auto fragmentMeta = fragmentMap->GetMetadata(MetadataType::FRAGMENT);
1226 if (fragmentMeta == nullptr) {
1227 IMAGE_LOGE("The fragmentMap has not fragmentMap");
1228 return loc;
1229 }
1230 std::string loc_x = "";
1231 std::string loc_y = "";
1232 uint32_t horizontalOffset = 0;
1233 uint32_t verticalOffset = 0;
1234 fragmentMeta->GetValue(FRAGMENT_METADATA_KEY_X, loc_x);
1235 fragmentMeta->GetValue(FRAGMENT_METADATA_KEY_Y, loc_y);
1236 ImageUtils::StrToUint32(loc_x, horizontalOffset);
1237 ImageUtils::StrToUint32(loc_y, verticalOffset);
1238 loc.horizontalOffset = horizontalOffset;
1239 loc.verticalOffset = verticalOffset;
1240 return loc;
1241 }
1242
AssembleHeifFragmentMap(std::vector<ImageItem> & inputImgs)1243 uint32_t ExtEncoder::AssembleHeifFragmentMap(std::vector<ImageItem>& inputImgs)
1244 {
1245 auto fragmentMap = picture_->GetAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP);
1246 if (!fragmentMap || !fragmentMap->GetContentPixel() ||
1247 fragmentMap->GetContentPixel()->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1248 IMAGE_LOGE("%{public}s The fragmentMap is nullptr or allocator type is not DMA_ALLOC", __func__);
1249 return ERR_IMAGE_INVALID_PARAMETER;
1250 }
1251 sptr<SurfaceBuffer> fragmentMapSptr(reinterpret_cast<SurfaceBuffer*>(fragmentMap->GetContentPixel()->GetFd()));
1252 if (!fragmentMapSptr) {
1253 IMAGE_LOGE("%{public}s fragmentMapSptr is nullptr", __func__);
1254 return ERR_IMAGE_INVALID_PARAMETER;
1255 }
1256 HeifEncodeItemInfo itemInfo = GetHeifEncodeItemInfo(AuxiliaryPictureType::FRAGMENT_MAP);
1257 auto item = InitAuxiliaryImageItem(itemInfo.itemId, itemInfo.itemName);
1258 bool sdrIsSRGB = fragmentMap->GetContentPixel()->GetToSdrColorSpaceIsSRGB();
1259 SkImageInfo fragmentInfo = GetSkInfo(fragmentMap->GetContentPixel().get(), false, sdrIsSRGB);
1260 sk_sp<SkData> iccProfile = icc_from_color_space(fragmentInfo);
1261 if (!AssembleICCImageProperty(iccProfile, item.sharedProperties)) {
1262 IMAGE_LOGE("%{public}s AssembleICCImageProperty failed", __func__);
1263 return ERR_IMAGE_INVALID_PARAMETER;
1264 }
1265 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(fragmentMapSptr->GetBufferHandle());
1266 std::string auxTypeStr = itemInfo.itemType;
1267 uint32_t litePropertiesSize =
1268 sizeof(PropertyType::AUX_TYPE) + UINT32_BYTES_NUM + auxTypeStr.length() + PLACE_HOLDER_LENGTH;
1269 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1270 litePropertiesSize += (sizeof(PropertyType::RLOC_INFO) + sizeof(RelativeLocation));
1271 item.liteProperties.resize(litePropertiesSize);
1272 size_t offset = 0;
1273 ColorType colorType = ColorType::RICC;
1274 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
1275 IMAGE_LOGE("%{public}s Fill color type failed", __func__);
1276 return ERR_IMAGE_INVALID_PARAMETER;
1277 }
1278 if (!FillLitePropertyItemByString(item.liteProperties, offset, PropertyType::AUX_TYPE, auxTypeStr)) {
1279 IMAGE_LOGE("%{public}s Fill auxiliary type failed", __func__);
1280 return ERR_IMAGE_INVALID_PARAMETER;
1281 }
1282 RelativeLocation loc = GetFragmentRelLocation(fragmentMap);
1283 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::RLOC_INFO, &loc, sizeof(RelativeLocation))) {
1284 IMAGE_LOGE("%{public}s Fill auxiliary type failed", __func__);
1285 return ERR_IMAGE_INVALID_PARAMETER;
1286 }
1287 inputImgs.push_back(item);
1288 return SUCCESS;
1289 }
1290 #endif
1291
EncodeDualVivid(ExtWStream & outputStream)1292 uint32_t ExtEncoder::EncodeDualVivid(ExtWStream& outputStream)
1293 {
1294 IMAGE_LOGD("ExtEncoder::EncodeDualVivid");
1295 if (!pixelmap_->IsHdr() ||
1296 pixelmap_->GetAllocatorType() != AllocatorType::DMA_ALLOC ||
1297 (encodeFormat_ != SkEncodedImageFormat::kJPEG && encodeFormat_ != SkEncodedImageFormat::kHEIF)) {
1298 return ERR_IMAGE_INVALID_PARAMETER;
1299 }
1300 bool sdrIsSRGB = pixelmap_->GetToSdrColorSpaceIsSRGB();
1301 SkImageInfo baseInfo = GetSkInfo(pixelmap_, false, sdrIsSRGB);
1302 SkImageInfo gainmapInfo = GetSkInfo(pixelmap_, true, sdrIsSRGB);
1303 sptr<SurfaceBuffer> baseSptr = AllocSurfaceBuffer(baseInfo.width(), baseInfo.height());
1304 sptr<SurfaceBuffer> gainMapSptr = AllocSurfaceBuffer(gainmapInfo.width(), gainmapInfo.height());
1305 if (baseSptr == nullptr || gainMapSptr == nullptr) {
1306 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1307 }
1308 HdrMetadata metadata;
1309 sptr<SurfaceBuffer> hdrSurfaceBuffer(reinterpret_cast<SurfaceBuffer*> (pixelmap_->GetFd()));
1310 CM_HDR_Metadata_Type hdrMetadataType;
1311 VpeUtils::GetSbMetadataType(hdrSurfaceBuffer, hdrMetadataType);
1312 VpeUtils::SetSbMetadataType(hdrSurfaceBuffer, CM_IMAGE_HDR_VIVID_SINGLE);
1313 VpeSurfaceBuffers buffers = {
1314 .sdr = baseSptr,
1315 .gainmap = gainMapSptr,
1316 .hdr = hdrSurfaceBuffer,
1317 };
1318 if (!DecomposeImage(buffers, metadata, false, sdrIsSRGB)) {
1319 IMAGE_LOGE("EncodeDualVivid decomposeImage failed");
1320 FreeBaseAndGainMapSurfaceBuffer(baseSptr, gainMapSptr);
1321 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1322 }
1323 metadata.hdrMetadataType = static_cast<int32_t>(hdrMetadataType);
1324 uint32_t error;
1325 if (encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1326 sk_sp<SkData> baseImageData = GetImageEncodeData(baseSptr, baseInfo, opts_.needsPackProperties);
1327 sk_sp<SkData> gainMapImageData = GetImageEncodeData(gainMapSptr, gainmapInfo, false);
1328 error = HdrJpegPackerHelper::SpliceHdrStream(baseImageData, gainMapImageData, outputStream, metadata);
1329 } else if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1330 error = EncodeHeifDualHdrImage(baseSptr, gainMapSptr, metadata);
1331 } else {
1332 error = ERR_IMAGE_INVALID_PARAMETER;
1333 }
1334 FreeBaseAndGainMapSurfaceBuffer(baseSptr, gainMapSptr);
1335 return error;
1336 }
1337
EncodeSdrImage(ExtWStream & outputStream)1338 uint32_t ExtEncoder::EncodeSdrImage(ExtWStream& outputStream)
1339 {
1340 IMAGE_LOGD("ExtEncoder EncodeSdrImage");
1341 if (!pixelmap_->IsHdr()) {
1342 return EncodeImageByPixelMap(pixelmap_, opts_.needsPackProperties, outputStream);
1343 }
1344 if (pixelmap_->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1345 IMAGE_LOGE("pixelmap is 10bit, but not dma buffer");
1346 return ERR_IMAGE_INVALID_PARAMETER;
1347 }
1348 ImageInfo info;
1349 pixelmap_->GetImageInfo(info);
1350 bool sdrIsSRGB = pixelmap_->GetToSdrColorSpaceIsSRGB();
1351 SkImageInfo baseInfo = GetSkInfo(pixelmap_, false, sdrIsSRGB);
1352 sptr<SurfaceBuffer> baseSptr = AllocSurfaceBuffer(baseInfo.width(), baseInfo.height());
1353 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1354 VpeUtils::SetSbColorSpaceType(baseSptr, CM_SRGB_FULL);
1355 if (baseSptr == nullptr) {
1356 IMAGE_LOGE("EncodeSdrImage sdr buffer alloc failed");
1357 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1358 }
1359 sptr<SurfaceBuffer> hdrSurfaceBuffer(reinterpret_cast<SurfaceBuffer*>(pixelmap_->GetFd()));
1360 VpeUtils::SetSbMetadataType(hdrSurfaceBuffer, CM_IMAGE_HDR_VIVID_SINGLE);
1361 VpeSurfaceBuffers buffers = {
1362 .sdr = baseSptr,
1363 .hdr = hdrSurfaceBuffer,
1364 };
1365 HdrMetadata metadata;
1366 if (!DecomposeImage(buffers, metadata, true, sdrIsSRGB)) {
1367 IMAGE_LOGE("EncodeSdrImage decomposeImage failed");
1368 ImageUtils::SurfaceBuffer_Unreference(baseSptr.GetRefPtr());
1369 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1370 }
1371 uint32_t error;
1372 if (encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1373 error = EncodeHeifSdrImage(baseSptr, baseInfo);
1374 } else {
1375 error = EncodeImageBySurfaceBuffer(baseSptr, baseInfo, opts_.needsPackProperties, outputStream);
1376 }
1377 ImageUtils::SurfaceBuffer_Unreference(baseSptr.GetRefPtr());
1378 return error;
1379 }
1380
EncodeHeifDualHdrImage(sptr<SurfaceBuffer> & sdr,sptr<SurfaceBuffer> & gainmap,Media::HdrMetadata & metadata,bool sdrIsSRGB)1381 uint32_t ExtEncoder::EncodeHeifDualHdrImage(sptr<SurfaceBuffer>& sdr, sptr<SurfaceBuffer>& gainmap,
1382 Media::HdrMetadata& metadata, bool sdrIsSRGB)
1383 {
1384 #ifdef HEIF_HW_ENCODE_ENABLE
1385 std::vector<ImageItem> inputImgs;
1386 ColorSpaceManager colorspaceName =
1387 sdrIsSRGB ? ColorManager::ColorSpaceName::SRGB : ColorManager::ColorSpaceName::DISPLAY_P3;
1388 std::shared_ptr<ImageItem> primaryItem =
1389 AssembleHdrBaseImageItem(sdr, colorspaceName, metadata, opts_);
1390 if (primaryItem == nullptr) {
1391 IMAGE_LOGE("AssmbleHeifDualHdrImage, get primary image failed");
1392 return ERR_IMAGE_INVALID_PARAMETER;
1393 }
1394 inputImgs.push_back(*primaryItem);
1395 std::shared_ptr<ImageItem> gainmapItem =
1396 AssembleGainmapImageItem(gainmap, colorspaceName, opts_);
1397 if (gainmapItem == nullptr) {
1398 IMAGE_LOGE("AssembleDualHdrImage, get gainmap image item failed");
1399 return ERR_IMAGE_INVALID_PARAMETER;
1400 }
1401 inputImgs.push_back(*gainmapItem);
1402 ColorManager::ColorSpaceName tmapColor = pixelmap_->InnerGetGrColorSpace().GetColorSpaceName();
1403 std::shared_ptr<ImageItem> tmapItem = AssembleTmapImageItem(tmapColor, metadata, opts_);
1404 if (tmapItem == nullptr) {
1405 IMAGE_LOGE("AssembleDualHdrImage, get tmap image item failed");
1406 return ERR_IMAGE_INVALID_PARAMETER;
1407 }
1408 inputImgs.push_back(*tmapItem);
1409 std::vector<MetaItem> inputMetas;
1410 std::vector<ItemRef> refs;
1411 if (AssembleExifMetaItem(inputMetas)) {
1412 AssembleExifRefItem(refs);
1413 }
1414 AssembleDualHdrRefItem(refs);
1415 return DoHeifEncode(inputImgs, inputMetas, refs);
1416 #else
1417 return ERR_IMAGE_INVALID_PARAMETER;
1418 #endif
1419 }
1420
EncodeHeifSdrImage(sptr<SurfaceBuffer> & sdr,SkImageInfo sdrInfo)1421 uint32_t ExtEncoder::EncodeHeifSdrImage(sptr<SurfaceBuffer>& sdr, SkImageInfo sdrInfo)
1422 {
1423 #ifdef HEIF_HW_ENCODE_ENABLE
1424 if (sdr == nullptr) {
1425 IMAGE_LOGE("EncodeHeifSdrImage, sdr surfacebuffer is nullptr");
1426 return ERR_IMAGE_INVALID_PARAMETER;
1427 }
1428 ImageItem item;
1429 item.id = PRIMARY_IMAGE_ITEM_ID;
1430 item.pixelBuffer = sptr<NativeBuffer>::MakeSptr(sdr->GetBufferHandle());
1431 item.isPrimary = true;
1432 item.isHidden = false;
1433 item.compressType = COMPRESS_TYPE_HEVC;
1434 item.quality = opts_.quality;
1435 item.sharedProperties.fd = -1;
1436 item.pixelSharedBuffer.fd = -1;
1437 ImageInfo info;
1438 pixelmap_->GetImageInfo(info);
1439 sk_sp<SkData> iccProfile = icc_from_color_space(sdrInfo);
1440 bool tempRes = AssembleICCImageProperty(iccProfile, item.sharedProperties);
1441 if (!tempRes) {
1442 IMAGE_LOGE("EncodeSdrImage AssembleICCImageProperty failed");
1443 return ERR_IMAGE_INVALID_PARAMETER;
1444 }
1445 uint32_t litePropertiesSize = (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
1446 item.liteProperties.resize(litePropertiesSize);
1447 size_t offset = 0;
1448 ColorType colorType = ColorType::RICC;
1449 if (!FillLitePropertyItem(item.liteProperties, offset, PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
1450 IMAGE_LOGE("EncodeHeifSdrImage Fill color type failed");
1451 return ERR_IMAGE_INVALID_PARAMETER;
1452 }
1453 std::vector<ImageItem> inputImgs;
1454 inputImgs.push_back(item);
1455 std::vector<MetaItem> inputMetas;
1456 std::vector<ItemRef> refs;
1457 if (AssembleExifMetaItem(inputMetas)) {
1458 AssembleExifRefItem(refs);
1459 }
1460 return DoHeifEncode(inputImgs, inputMetas, refs);
1461 #else
1462 return ERR_IMAGE_INVALID_PARAMETER;
1463 #endif
1464 }
1465
EncodePicture()1466 uint32_t ExtEncoder::EncodePicture()
1467 {
1468 if ((encodeFormat_ != SkEncodedImageFormat::kJPEG && encodeFormat_ != SkEncodedImageFormat::kHEIF)) {
1469 IMAGE_LOGE("%{public}s: unsupported encode format: %{public}s", __func__, opts_.format.c_str());
1470 return ERR_IMAGE_INVALID_PARAMETER;
1471 }
1472 if (opts_.isEditScene && encodeFormat_ == SkEncodedImageFormat::kHEIF) {
1473 return EncodeEditScenePicture();
1474 }
1475 if (encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1476 CheckJpegAuxiliaryTagName();
1477 }
1478 ExtWStream wStream(output_);
1479 return EncodeCameraScenePicture(wStream);
1480 }
1481
EncodeCameraScenePicture(SkWStream & skStream)1482 uint32_t ExtEncoder::EncodeCameraScenePicture(SkWStream& skStream)
1483 {
1484 uint32_t retCode = ERR_IMAGE_ENCODE_FAILED;
1485 std::string errorMsg = "Load hardware encode library failed";
1486 static ImageFwkExtManager imageFwkExtManager;
1487 if (imageFwkExtManager.LoadImageFwkExtNativeSo() && imageFwkExtManager.doHardwareEncodePictureFunc_ != nullptr) {
1488 retCode = imageFwkExtManager.doHardwareEncodePictureFunc_(&skStream, opts_, picture_);
1489 errorMsg = "Hardware encode picture failed";
1490 }
1491 if (retCode != SUCCESS && encodeFormat_ == SkEncodedImageFormat::kJPEG) {
1492 IMAGE_LOGE("%{public}s, retCode is: %{public}d, try jpeg software encode", errorMsg.c_str(), retCode);
1493 retCode = EncodeJpegPicture(skStream);
1494 errorMsg = "Jpeg software encode picture failed";
1495 }
1496 if (retCode != SUCCESS) {
1497 ImageInfo imageInfo;
1498 picture_->GetMainPixel()->GetImageInfo(imageInfo);
1499 IMAGE_LOGE("%{public}s, retCode is: %{public}d", errorMsg.c_str(), retCode);
1500 ReportEncodeFault(imageInfo.size.width, imageInfo.size.height, opts_.format, errorMsg);
1501 }
1502 return retCode;
1503 }
1504
EncodeEditScenePicture()1505 uint32_t ExtEncoder::EncodeEditScenePicture()
1506 {
1507 if (!picture_) {
1508 IMAGE_LOGE("picture_ is nullptr");
1509 return ERR_IMAGE_DATA_ABNORMAL;
1510 }
1511 auto mainPixelMap = picture_->GetMainPixel();
1512 if (!mainPixelMap || mainPixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1513 IMAGE_LOGE("MainPixelMap is nullptr or mainPixelMap is not DMA buffer");
1514 return ERR_IMAGE_DATA_ABNORMAL;
1515 }
1516
1517 bool sdrIsSRGB = mainPixelMap->GetToSdrColorSpaceIsSRGB();
1518 SkImageInfo baseInfo = GetSkInfo(mainPixelMap.get(), false, false);
1519 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelMap->GetFd()));
1520 if (!baseSptr) {
1521 IMAGE_LOGE("creat main pixels surfaceBuffer error");
1522 return IMAGE_RESULT_CREATE_SURFAC_FAILED;
1523 }
1524 return EncodeHeifPicture(baseSptr, baseInfo, sdrIsSRGB);
1525 }
1526
EncodeHeifPicture(sptr<SurfaceBuffer> & mainSptr,SkImageInfo mainInfo,bool sdrIsSRGB)1527 uint32_t ExtEncoder::EncodeHeifPicture(sptr<SurfaceBuffer>& mainSptr, SkImageInfo mainInfo, bool sdrIsSRGB)
1528 {
1529 #ifdef HEIF_HW_ENCODE_ENABLE
1530 uint32_t error = SUCCESS;
1531 std::vector<ImageItem> inputImgs;
1532 std::vector<MetaItem> inputMetas;
1533 std::vector<ItemRef> refs;
1534 switch (opts_.desiredDynamicRange) {
1535 case EncodeDynamicRange::AUTO:
1536 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP) &&
1537 AssembleHeifHdrPicture(mainSptr, sdrIsSRGB, inputImgs) == SUCCESS) {
1538 AssembleDualHdrRefItem(refs);
1539 } else {
1540 inputImgs.clear();
1541 error = AssembleSdrImageItem(mainSptr, mainInfo, inputImgs);
1542 }
1543 break;
1544 case EncodeDynamicRange::SDR:
1545 error = AssembleSdrImageItem(mainSptr, mainInfo, inputImgs);
1546 break;
1547 case EncodeDynamicRange::HDR_VIVID_DUAL:
1548 if (picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP)) {
1549 error = AssembleHeifHdrPicture(mainSptr, sdrIsSRGB, inputImgs);
1550 AssembleDualHdrRefItem(refs);
1551 } else {
1552 IMAGE_LOGE("Picture don't has GAINMAP pixels");
1553 error = ERR_IMAGE_ENCODE_FAILED;
1554 }
1555 break;
1556 case EncodeDynamicRange::HDR_VIVID_SINGLE:
1557 IMAGE_LOGE("Heif picture not support HDR_VIVID_SINGLE");
1558 return ERR_IMAGE_ENCODE_FAILED;
1559 default:
1560 return ERR_IMAGE_INVALID_PARAMETER;
1561 }
1562 if (error != SUCCESS) {
1563 return error;
1564 }
1565 if (AssembleExifMetaItem(inputMetas)) {
1566 AssembleExifRefItem(refs);
1567 }
1568 error = AssembleHeifAuxiliaryPicture(inputImgs, refs);
1569 if (error != SUCCESS) {
1570 return error;
1571 }
1572 return DoHeifEncode(inputImgs, inputMetas, refs);
1573 #else
1574 return ERR_IMAGE_INVALID_PARAMETER;
1575 #endif
1576 }
1577
CheckJpegAuxiliaryTagName()1578 void ExtEncoder::CheckJpegAuxiliaryTagName()
1579 {
1580 if (picture_ == nullptr) {
1581 return;
1582 }
1583 auto auxTypes = ImageUtils::GetAllAuxiliaryPictureType();
1584 for (AuxiliaryPictureType auxType : auxTypes) {
1585 auto auxPicture = picture_->GetAuxiliaryPicture(auxType);
1586 if (auxPicture == nullptr) {
1587 continue;
1588 }
1589 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
1590 auto iter = DEFAULT_AUXILIARY_TAG_MAP.find(auxType);
1591 if (auxInfo.jpegTagName.size() == 0 && iter != DEFAULT_AUXILIARY_TAG_MAP.end()) {
1592 auxInfo.jpegTagName = iter->second;
1593 auxPicture->SetAuxiliaryPictureInfo(auxInfo);
1594 }
1595 }
1596 }
1597
EncodeJpegPicture(SkWStream & skStream)1598 uint32_t ExtEncoder::EncodeJpegPicture(SkWStream& skStream)
1599 {
1600 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1601 switch (opts_.desiredDynamicRange) {
1602 case EncodeDynamicRange::AUTO:
1603 error = EncodeJpegPictureDualVivid(skStream);
1604 if (error != SUCCESS) {
1605 IMAGE_LOGI("%{public}s jpeg picture encode dual vivid failed, try encode sdr", __func__);
1606 error = EncodeJpegPictureSdr(skStream);
1607 }
1608 break;
1609 case EncodeDynamicRange::SDR:
1610 error = EncodeJpegPictureSdr(skStream);
1611 break;
1612 case EncodeDynamicRange::HDR_VIVID_DUAL:
1613 error = EncodeJpegPictureDualVivid(skStream);
1614 break;
1615 case EncodeDynamicRange::HDR_VIVID_SINGLE:
1616 error = ERR_IMAGE_DECODE_FAILED;
1617 break;
1618 default:
1619 error = ERR_IMAGE_INVALID_PARAMETER;
1620 break;
1621 }
1622 return error;
1623 }
1624
EncodeJpegPictureDualVividInner(SkWStream & skStream,std::shared_ptr<PixelMap> & mainPixelmap,std::shared_ptr<PixelMap> & gainmapPixelmap)1625 uint32_t ExtEncoder::EncodeJpegPictureDualVividInner(SkWStream& skStream, std::shared_ptr<PixelMap>& mainPixelmap,
1626 std::shared_ptr<PixelMap>& gainmapPixelmap)
1627 {
1628 bool mainIsSRGB = mainPixelmap->GetToSdrColorSpaceIsSRGB();
1629 SkImageInfo baseInfo = GetSkInfo(mainPixelmap.get(), false, mainIsSRGB);
1630 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelmap->GetFd()));
1631 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1632 VpeUtils::SetSbColorSpaceType(baseSptr, mainIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1633 pixelmap_ = mainPixelmap.get();
1634 sk_sp<SkData> baseImageData = GetImageEncodeData(baseSptr, baseInfo, opts_.needsPackProperties);
1635
1636 bool gainmapIsSRGB = gainmapPixelmap->GetToSdrColorSpaceIsSRGB();
1637 SkImageInfo gainmapInfo = GetSkInfo(gainmapPixelmap.get(), true, gainmapIsSRGB);
1638 ImageInfo tempInfo;
1639 gainmapPixelmap->GetImageInfo(tempInfo);
1640 gainmapInfo = gainmapInfo.makeWH(tempInfo.size.width, tempInfo.size.height);
1641 sptr<SurfaceBuffer> gainMapSptr(reinterpret_cast<SurfaceBuffer*>(gainmapPixelmap->GetFd()));
1642 VpeUtils::SetSbMetadataType(gainMapSptr, CM_METADATA_NONE);
1643 VpeUtils::SetSbColorSpaceType(gainMapSptr, gainmapIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1644 pixelmap_ = gainmapPixelmap.get();
1645 sk_sp<SkData> gainMapImageData = GetImageEncodeData(gainMapSptr, gainmapInfo, false);
1646
1647 HdrMetadata hdrMetadata = GetHdrMetadata(baseSptr, gainMapSptr);
1648 SkDynamicMemoryWStream hdrStream;
1649 uint32_t error = HdrJpegPackerHelper::SpliceHdrStream(baseImageData, gainMapImageData, hdrStream, hdrMetadata);
1650 IMAGE_LOGD("%{public}s splice hdr stream result is: %{public}u", __func__, error);
1651 if (error == SUCCESS) {
1652 sk_sp<SkData> hdrSkData = hdrStream.detachAsData();
1653 skStream.write(hdrSkData->data(), hdrSkData->size());
1654 EncodeJpegAuxiliaryPictures(skStream);
1655 }
1656 return error;
1657 }
1658
EncodeJpegPictureDualVivid(SkWStream & skStream)1659 uint32_t ExtEncoder::EncodeJpegPictureDualVivid(SkWStream& skStream)
1660 {
1661 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1662 if (!picture_->HasAuxiliaryPicture(AuxiliaryPictureType::GAINMAP)) {
1663 IMAGE_LOGE("%{public}s no gainmap in picture", __func__);
1664 return ERR_IMAGE_INVALID_PARAMETER;
1665 }
1666 auto mainPixelmap = picture_->GetMainPixel();
1667 auto gainmapPixelmap = picture_->GetGainmapPixelMap();
1668 if (!mainPixelmap || !gainmapPixelmap) {
1669 IMAGE_LOGE("%{public}s mainPixelmap or gainmapPixelmap is null", __func__);
1670 return ERR_IMAGE_INVALID_PARAMETER;
1671 }
1672 AllocatorType mainAllocType = mainPixelmap->GetAllocatorType();
1673 AllocatorType gainmapAllocType = gainmapPixelmap->GetAllocatorType();
1674 if (mainAllocType != AllocatorType::DMA_ALLOC || gainmapAllocType != AllocatorType::DMA_ALLOC) {
1675 IMAGE_LOGE("%{public}s AllocatorType is not DMA, mainAllocType: %{public}d, gainmapAllocType: %{public}d",
1676 __func__, mainAllocType, gainmapAllocType);
1677 return ERR_IMAGE_ENCODE_FAILED;
1678 }
1679 return EncodeJpegPictureDualVividInner(skStream, mainPixelmap, gainmapPixelmap);
1680 }
1681
EncodeJpegPictureSdr(SkWStream & skStream)1682 uint32_t ExtEncoder::EncodeJpegPictureSdr(SkWStream& skStream)
1683 {
1684 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1685 auto mainPixelmap = picture_->GetMainPixel();
1686 if (!mainPixelmap) {
1687 IMAGE_LOGE("%{public}s mainPixelmap is null", __func__);
1688 return ERR_IMAGE_INVALID_PARAMETER;
1689 }
1690 pixelmap_ = mainPixelmap.get();
1691 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1692 if (!mainPixelmap->IsHdr()) {
1693 error = EncodeImageByPixelMap(mainPixelmap.get(), opts_.needsPackProperties, skStream);
1694 IMAGE_LOGD("%{public}s encode sdr picture result is: %{public}u", __func__, error);
1695 } else {
1696 if (mainPixelmap->GetAllocatorType() != AllocatorType::DMA_ALLOC) {
1697 IMAGE_LOGE("pixelmap is 10bit, but not dma buffer");
1698 return ERR_IMAGE_INVALID_PARAMETER;
1699 }
1700 bool mainIsSRGB = mainPixelmap->GetToSdrColorSpaceIsSRGB();
1701 SkImageInfo baseInfo = GetSkInfo(mainPixelmap.get(), false, mainIsSRGB);
1702 sptr<SurfaceBuffer> baseSptr(reinterpret_cast<SurfaceBuffer*>(mainPixelmap->GetFd()));
1703 VpeUtils::SetSbMetadataType(baseSptr, CM_IMAGE_HDR_VIVID_DUAL);
1704 VpeUtils::SetSbColorSpaceType(baseSptr, mainIsSRGB ? CM_SRGB_FULL : CM_P3_FULL);
1705 error = EncodeImageBySurfaceBuffer(baseSptr, baseInfo, opts_.needsPackProperties, skStream);
1706 IMAGE_LOGD("%{public}s encode hdr picture result is: %{public}u", __func__, error);
1707 }
1708 if (error == SUCCESS) {
1709 EncodeJpegAuxiliaryPictures(skStream);
1710 }
1711 return error;
1712 }
1713
GetMetadataValueInt32(const std::shared_ptr<ImageMetadata> & metadata,const std::string & key,int32_t & outVal)1714 static bool GetMetadataValueInt32(const std::shared_ptr<ImageMetadata>& metadata, const std::string& key,
1715 int32_t& outVal)
1716 {
1717 std::string strVal("");
1718 uint32_t u32Val = 0;
1719 if (metadata == nullptr) {
1720 IMAGE_LOGE("%{public}s: metadata is nullptr!", __func__);
1721 return false;
1722 }
1723 if (metadata->GetValue(key, strVal) != SUCCESS || !ImageUtils::StrToUint32(strVal, u32Val)) {
1724 IMAGE_LOGE("%{public}s: get metadata key[%{public}s]'s value failed!", __func__, key.c_str());
1725 return false;
1726 }
1727 outVal = static_cast<int32_t>(u32Val);
1728 return true;
1729 }
1730
CheckFragmentMetadata(Picture * picture,Media::Rect & outData)1731 static bool CheckFragmentMetadata(Picture* picture, Media::Rect& outData)
1732 {
1733 if (!picture || !picture->GetMainPixel() || !picture->HasAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP)) {
1734 IMAGE_LOGE("%{public}s: picture or mainPixelMap or fragment picture does not exist!", __func__);
1735 return false;
1736 }
1737 int32_t mainW = picture->GetMainPixel()->GetWidth();
1738 int32_t mainH = picture->GetMainPixel()->GetHeight();
1739
1740 auto fragmentPicture = picture->GetAuxiliaryPicture(AuxiliaryPictureType::FRAGMENT_MAP);
1741 auto fragmentMetadata = fragmentPicture->GetMetadata(MetadataType::FRAGMENT);
1742 if (fragmentMetadata == nullptr) {
1743 IMAGE_LOGE("%{public}s: fragmentMetadata is nullptr!", __func__);
1744 return false;
1745 }
1746
1747 if (!GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_X, outData.left) ||
1748 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_Y, outData.top) ||
1749 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_WIDTH, outData.width) ||
1750 !GetMetadataValueInt32(fragmentMetadata, FRAGMENT_METADATA_KEY_HEIGHT, outData.height)) {
1751 IMAGE_LOGE("%{public}s: GetMetadataValueInt32 failed!", __func__);
1752 return false;
1753 }
1754 if (!ImageUtils::IsInRange(outData.left, 0, mainW) || !ImageUtils::IsInRange(outData.top, 0, mainH) ||
1755 !ImageUtils::IsInRange(outData.width, 0, mainW) || !ImageUtils::IsInRange(outData.height, 0, mainH) ||
1756 !ImageUtils::IsInRange(outData.left + outData.width, 0, mainW) ||
1757 !ImageUtils::IsInRange(outData.top + outData.height, 0, mainH)) {
1758 IMAGE_LOGW("%{public}s: Fragment Rect is not in main Rect!", __func__);
1759 return false;
1760 }
1761 return true;
1762 }
1763
EncodeJpegAuxiliaryPictures(SkWStream & skStream)1764 void ExtEncoder::EncodeJpegAuxiliaryPictures(SkWStream& skStream)
1765 {
1766 ImageFuncTimer imageFuncTimer("%s enter", __func__);
1767 if (picture_ == nullptr) {
1768 return;
1769 }
1770 auto auxTypes = ImageUtils::GetAllAuxiliaryPictureType();
1771 for (AuxiliaryPictureType auxType : auxTypes) {
1772 auto auxPicture = picture_->GetAuxiliaryPicture(auxType);
1773 // Gainmap has been encoded before
1774 if (auxPicture == nullptr || auxType == AuxiliaryPictureType::GAINMAP) {
1775 continue;
1776 }
1777 IMAGE_LOGI("%{public}s try to encode auxiliary picture type: %{public}d", __func__, auxType);
1778 uint32_t error = ERR_IMAGE_ENCODE_FAILED;
1779 size_t writtenSize = skStream.bytesWritten();
1780 if (ImageUtils::IsAuxiliaryPictureEncoded(auxType)) {
1781 error = WriteJpegCodedData(auxPicture, skStream);
1782 } else {
1783 error = WriteJpegUncodedData(auxPicture, skStream);
1784 }
1785 if (error != SUCCESS) {
1786 IMAGE_LOGE("%{public}s encode auxiliary picture type [%{public}d] failed, error: %{public}u",
1787 __func__, auxType, error);
1788 } else {
1789 uint32_t currentDataSize = static_cast<uint32_t>(skStream.bytesWritten() - writtenSize);
1790 WriteJpegAuxiliarySizeAndTag(currentDataSize, auxPicture, skStream);
1791 }
1792 }
1793 }
1794
WriteJpegCodedData(std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)1795 uint32_t ExtEncoder::WriteJpegCodedData(std::shared_ptr<AuxiliaryPicture>& auxPicture, SkWStream& skStream)
1796 {
1797 if (auxPicture == nullptr || auxPicture->GetContentPixel() == nullptr) {
1798 return ERR_IMAGE_ENCODE_FAILED;
1799 }
1800 auto pixelMap = auxPicture->GetContentPixel();
1801 pixelmap_ = pixelMap.get();
1802 sk_sp<SkData> skData = nullptr;
1803 if (pixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC || pixelMap->GetFd() == nullptr) {
1804 SkDynamicMemoryWStream nonDMAStream;
1805 uint32_t error = EncodeImageByPixelMap(pixelmap_, false, nonDMAStream);
1806 IMAGE_LOGD("%{public}s EncodeImageByPixelMap result:%{public}u", __func__, error);
1807 skData = nonDMAStream.detachAsData();
1808 } else {
1809 sptr<SurfaceBuffer> auxSptr(reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd()));
1810 bool isSRGB = pixelMap->GetToSdrColorSpaceIsSRGB();
1811 SkImageInfo skInfo = GetSkInfo(pixelmap_, false, isSRGB);
1812 skData = GetImageEncodeData(auxSptr, skInfo, false);
1813 }
1814
1815 if (skData == nullptr) {
1816 return ERR_IMAGE_ENCODE_FAILED;
1817 }
1818 if (auxPicture->GetType() == AuxiliaryPictureType::FRAGMENT_MAP) {
1819 return SpliceFragmentStream(skStream, skData);
1820 }
1821 skStream.write(skData->data(), skData->size());
1822 return SUCCESS;
1823 }
1824
SpliceFragmentStream(SkWStream & skStream,sk_sp<SkData> & skData)1825 uint32_t ExtEncoder::SpliceFragmentStream(SkWStream& skStream, sk_sp<SkData>& skData)
1826 {
1827 Media::Rect fragmentMetadata;
1828 if (!CheckFragmentMetadata(picture_, fragmentMetadata)) {
1829 IMAGE_LOGE("%{public}s: CheckFragmentMetadata failed!", __func__);
1830 return ERR_IMAGE_PROPERTY_NOT_EXIST;
1831 }
1832 const uint8_t* dataBytes = reinterpret_cast<const uint8_t*>(skData->data());
1833 // write JPEG SOI(0xFFD8)
1834 skStream.write(dataBytes, JPEG_MARKER_TAG_SIZE);
1835 std::vector<uint8_t> packedFragmentMetadata = JpegMpfPacker::PackFragmentMetadata(fragmentMetadata);
1836 // write fragment metadata
1837 skStream.write(packedFragmentMetadata.data(), packedFragmentMetadata.size());
1838 // write fragment auxiliary image data
1839 skStream.write(dataBytes + JPEG_MARKER_TAG_SIZE, skData->size() - JPEG_MARKER_TAG_SIZE);
1840 return SUCCESS;
1841 }
1842
WriteJpegUncodedData(std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)1843 uint32_t ExtEncoder::WriteJpegUncodedData(std::shared_ptr<AuxiliaryPicture>& auxPicture, SkWStream& skStream)
1844 {
1845 auto pixelMap = auxPicture->GetContentPixel();
1846 if (pixelMap == nullptr) {
1847 return ERR_IMAGE_DATA_ABNORMAL;
1848 }
1849 void* bytes = nullptr;
1850 uint32_t size = 0;
1851 Size imageSize;
1852 if (pixelMap->GetAllocatorType() != AllocatorType::DMA_ALLOC || pixelMap->GetFd() == nullptr) {
1853 bytes = pixelMap->GetWritablePixels();
1854 size = pixelMap->GetCapacity();
1855 imageSize.width = pixelMap->GetWidth();
1856 imageSize.height = pixelMap->GetHeight();
1857 } else {
1858 auto surfaceBuffer = reinterpret_cast<SurfaceBuffer*>(pixelMap->GetFd());
1859 bytes = surfaceBuffer->GetVirAddr();
1860 size = surfaceBuffer->GetSize();
1861 imageSize.width = surfaceBuffer->GetWidth();
1862 imageSize.height = surfaceBuffer->GetHeight();
1863 }
1864 uint32_t writeSize = imageSize.width * imageSize.height;
1865 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
1866 if (auxInfo.auxiliaryPictureType == AuxiliaryPictureType::DEPTH_MAP) {
1867 writeSize *= DEPTH_MAP_BYTES;
1868 } else if (auxInfo.auxiliaryPictureType == AuxiliaryPictureType::LINEAR_MAP) {
1869 writeSize *= LINEAR_MAP_BYTES;
1870 }
1871 IMAGE_LOGD("%{public}s auxType: %{public}d, width: %{public}d, height: %{public}d, buffer size: %{public}u,"
1872 " write size: %{public}u", __func__, auxInfo.auxiliaryPictureType, imageSize.width, imageSize.height,
1873 size, writeSize);
1874 skStream.write(bytes, std::min(size, writeSize));
1875 return SUCCESS;
1876 }
1877
WriteJpegAuxiliarySizeAndTag(uint32_t size,std::shared_ptr<AuxiliaryPicture> & auxPicture,SkWStream & skStream)1878 void ExtEncoder::WriteJpegAuxiliarySizeAndTag(uint32_t size, std::shared_ptr<AuxiliaryPicture>& auxPicture,
1879 SkWStream& skStream)
1880 {
1881 // Write auxiliary image size(little endian)
1882 std::vector<uint8_t> auxSize = JpegMpfPacker::PackDataSize(size, false);
1883 skStream.write(auxSize.data(), auxSize.size());
1884
1885 // Write auxiliary tag name
1886 AuxiliaryPictureInfo auxInfo = auxPicture->GetAuxiliaryPictureInfo();
1887 std::vector<uint8_t> tagName = JpegMpfPacker::PackAuxiliaryTagName(auxInfo.jpegTagName);
1888 skStream.write(tagName.data(), tagName.size());
1889 }
1890 #endif
1891
1892 #ifdef HEIF_HW_ENCODE_ENABLE
GetBufferSize(size_t contentSize)1893 static size_t GetBufferSize(size_t contentSize)
1894 {
1895 return sizeof(PropertyType) + sizeof(int) + contentSize;
1896 }
1897
1898 // format: enum + int buffersize + buffer
FillImagePropertyItem(const std::shared_ptr<AbsMemory> & mem,const size_t offset,PropertyType type,const void * payloadData,size_t payloadSize)1899 static bool FillImagePropertyItem(const std::shared_ptr<AbsMemory> &mem, const size_t offset,
1900 PropertyType type, const void* payloadData, size_t payloadSize)
1901 {
1902 uint8_t* memData = reinterpret_cast<uint8_t*>(mem->data.data);
1903 size_t memSize = mem->data.size;
1904 IMAGE_LOGD("FileImagePropertyItem memSize is %{public}ld, payloadSize is %{public}ld", memSize, payloadSize);
1905 IMAGE_LOGD("FileImagePropertyItem sizeof(type) is %{public}ld, sizeof(payloadSize) %{public}ld",
1906 sizeof(type), sizeof(payloadSize));
1907 if (payloadSize > INT32_MAX) {
1908 IMAGE_LOGI("payloadSize is over INT32_MAX");
1909 return false;
1910 }
1911 int payloadIntSize = static_cast<int>(payloadSize);
1912 size_t typeSize = sizeof(type);
1913 size_t intSize = sizeof(int);
1914 bool res = (memcpy_s(memData + offset, memSize - offset, &type, typeSize) == EOK) &&
1915 (memcpy_s(memData + offset + typeSize, memSize - offset - typeSize, &payloadIntSize, intSize) == EOK) &&
1916 (memcpy_s(memData + offset + typeSize + intSize, memSize - offset - typeSize - intSize,
1917 payloadData, payloadSize) == EOK);
1918 return res;
1919 }
1920
AllocateNewSharedMem(size_t memorySize,std::string tag)1921 std::shared_ptr<AbsMemory> ExtEncoder::AllocateNewSharedMem(size_t memorySize, std::string tag)
1922 {
1923 MemoryData memoryData;
1924 memoryData.size = memorySize;
1925 memoryData.tag = tag.empty() ? DEFAULT_ASHMEM_TAG.c_str() : tag.c_str();
1926 std::unique_ptr<AbsMemory> memory =
1927 MemoryManager::CreateMemory(AllocatorType::SHARE_MEM_ALLOC, memoryData);
1928 std::shared_ptr<AbsMemory> res = std::move(memory);
1929 return res;
1930 }
1931
GetStaticMetadata(HdrMetadata & metadata,MasteringDisplayColourVolume & color,ContentLightLevel & light)1932 bool ExtEncoder::GetStaticMetadata(HdrMetadata& metadata, MasteringDisplayColourVolume& color, ContentLightLevel& light)
1933 {
1934 HdrStaticMetadata staticMetadata;
1935 size_t staticMetadataSize = sizeof(HdrStaticMetadata);
1936 if (metadata.staticMetadata.size() < staticMetadataSize) {
1937 IMAGE_LOGI("GetStaticMetadata failed");
1938 return false;
1939 }
1940 if (memcpy_s(&staticMetadata, staticMetadataSize,
1941 metadata.staticMetadata.data(), metadata.staticMetadata.size()) != EOK) {
1942 IMAGE_LOGI("GetStaticMetadata failed, memcpy_s failed");
1943 return false;
1944 }
1945 color.displayPrimariesRX =
1946 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryRed.x);
1947 color.displayPrimariesRY =
1948 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryRed.y);
1949 color.displayPrimariesGX =
1950 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryGreen.x);
1951 color.displayPrimariesGY =
1952 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryGreen.y);
1953 color.displayPrimariesBX =
1954 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryBlue.x);
1955 color.displayPrimariesBY =
1956 (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.displayPrimaryBlue.y);
1957 color.whitePointX = (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.whitePoint.x);
1958 color.whitePointY = (uint16_t)(STATIC_METADATA_COLOR_SCALE * staticMetadata.smpte2086.whitePoint.y);
1959 color.maxDisplayMasteringLuminance = (uint32_t)staticMetadata.smpte2086.maxLuminance;
1960 color.minDisplayMasteringLuminance = (uint32_t)(STATIC_METADATA_LUM_SCALE * staticMetadata.smpte2086.minLuminance);
1961 light.maxContentLightLevel = (uint16_t)staticMetadata.cta861.maxContentLightLevel;
1962 light.maxPicAverageLightLevel = (uint16_t)staticMetadata.cta861.maxFrameAverageLightLevel;
1963 return true;
1964 }
1965
GetToneMapChannel(ISOMetadata & metadata,ToneMapChannel & channel,uint8_t index)1966 bool ExtEncoder::GetToneMapChannel(ISOMetadata& metadata, ToneMapChannel& channel, uint8_t index)
1967 {
1968 if (index > GAINMAP_CHANNEL_MULTI - 1) {
1969 IMAGE_LOGE("GetToneMapChannel index:[%{public}d] unsupported", index);
1970 return false;
1971 }
1972 channel.gainMapMin.numerator = DEFAULT_DENOMINATOR * metadata.enhanceClippedThreholdMinGainmap[index];
1973 channel.gainMapMin.denominator = DEFAULT_DENOMINATOR;
1974 channel.gainMapMax.numerator = DEFAULT_DENOMINATOR * metadata.enhanceClippedThreholdMaxGainmap[index];
1975 channel.gainMapMax.denominator = DEFAULT_DENOMINATOR;
1976 if (metadata.enhanceMappingGamma[index] > 0.0f) {
1977 channel.gamma.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingGamma[index];
1978 } else {
1979 channel.gamma.numerator = 0;
1980 }
1981 channel.gamma.denominator = DEFAULT_DENOMINATOR;
1982 channel.baseOffset.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingBaselineOffset[index];
1983 channel.baseOffset.denominator = DEFAULT_DENOMINATOR;
1984 channel.alternateOffset.numerator = DEFAULT_DENOMINATOR * metadata.enhanceMappingAlternateOffset[index];
1985 channel.alternateOffset.denominator = DEFAULT_DENOMINATOR;
1986 IMAGE_LOGD("GetToneMapChannel [%{public}d],gainmapMin:%{public}f, gainmapMax:%{public}f,"
1987 "gamma:%{public}f,baseOffset:%{public}f, alternateOffset:%{public}f", index,
1988 metadata.enhanceClippedThreholdMinGainmap[index], metadata.enhanceClippedThreholdMaxGainmap[index],
1989 metadata.enhanceMappingGamma[index],
1990 metadata.enhanceMappingBaselineOffset[index], metadata.enhanceMappingAlternateOffset[index]);
1991 return true;
1992 }
1993
GetToneMapMetadata(HdrMetadata & metadata,ToneMapMetadata & toneMapMetadata)1994 bool ExtEncoder::GetToneMapMetadata(HdrMetadata& metadata, ToneMapMetadata& toneMapMetadata)
1995 {
1996 ISOMetadata isoMeta = metadata.extendMeta.metaISO;
1997 toneMapMetadata.useBaseColorSpace = (isoMeta.useBaseColorFlag == 1) ? true : false;
1998 toneMapMetadata.channelCnt = (isoMeta.gainmapChannelNum == 0) ? GAINMAP_CHANNEL_SINGLE : GAINMAP_CHANNEL_MULTI;
1999 toneMapMetadata.baseHdrHeadroom.denominator = DEFAULT_DENOMINATOR;
2000 if (isoMeta.baseHeadroom > 0.0f) {
2001 toneMapMetadata.baseHdrHeadroom.numerator = (uint32_t)(isoMeta.baseHeadroom * DEFAULT_DENOMINATOR);
2002 } else {
2003 toneMapMetadata.baseHdrHeadroom.numerator = 0;
2004 }
2005 toneMapMetadata.alternateHdrHeadroom.denominator = DEFAULT_DENOMINATOR;
2006 if (isoMeta.alternateHeadroom > 0.0f) {
2007 toneMapMetadata.alternateHdrHeadroom.numerator = (uint32_t)(isoMeta.alternateHeadroom * DEFAULT_DENOMINATOR);
2008 } else {
2009 toneMapMetadata.alternateHdrHeadroom.numerator = 0;
2010 }
2011 GetToneMapChannel(isoMeta, toneMapMetadata.channels1, INDEX_ZERO);
2012 if (toneMapMetadata.channelCnt == GAINMAP_CHANNEL_MULTI) {
2013 GetToneMapChannel(isoMeta, toneMapMetadata.channels2, INDEX_ONE);
2014 GetToneMapChannel(isoMeta, toneMapMetadata.channels3, INDEX_TWO);
2015 }
2016 IMAGE_LOGD("GetToneMapMetadata useBaseColorSpace:%{public}d, gainmapChannelNum:%{public}d,"
2017 "baseHeadroom:%{public}f,alternateHeadroom:%{public}f", isoMeta.useBaseColorFlag, isoMeta.gainmapChannelNum,
2018 isoMeta.baseHeadroom, isoMeta.alternateHeadroom);
2019 return true;
2020 }
2021
GetColourInfo(ColorManager::ColorSpaceName color,ColourInfo & info)2022 void ExtEncoder::GetColourInfo(ColorManager::ColorSpaceName color, ColourInfo& info)
2023 {
2024 uint8_t fullRangeFlag;
2025 ColorUtils::ColorSpaceGetCicp(color, info.colourPrimaries, info.transferCharacteristics, info.matrixCoefficients,
2026 fullRangeFlag);
2027 // 1 : full range ; 0 : limit range.
2028 info.fullRangeFlag = (fullRangeFlag == 1);
2029 }
2030
AssembleIT35SharedBuffer(HdrMetadata metadata,SharedBuffer & outBuffer)2031 bool ExtEncoder::AssembleIT35SharedBuffer(HdrMetadata metadata, SharedBuffer& outBuffer)
2032 {
2033 std::vector<uint8_t> it35Info;
2034 if (!HdrHeifPackerHelper::PackIT35Info(metadata, it35Info)) {
2035 IMAGE_LOGE("get it35 info failed");
2036 return false;
2037 }
2038 std::shared_ptr<AbsMemory> propertyAshmem =
2039 AllocateNewSharedMem(GetBufferSize(it35Info.size()), IT35_ASHMEM_TAG);
2040 if (propertyAshmem == nullptr) {
2041 IMAGE_LOGE("AssembleIT35SharedBuffer it35 alloc failed");
2042 return false;
2043 };
2044 tmpMemoryList_.push_back(propertyAshmem);
2045 if (!FillImagePropertyItem(propertyAshmem, 0, PropertyType::IT35_INFO, it35Info.data(), it35Info.size())) {
2046 IMAGE_LOGE("AssembleIT35SharedBuffer fill failed");
2047 return false;
2048 }
2049 outBuffer.fd = *static_cast<int *>(propertyAshmem->extend.data);
2050 outBuffer.capacity = propertyAshmem->data.size;
2051 outBuffer.filledLen = propertyAshmem->data.size;
2052 return true;
2053 }
2054
AssembleICCImageProperty(sk_sp<SkData> & iccProfile,SharedBuffer & outBuffer)2055 bool ExtEncoder::AssembleICCImageProperty(sk_sp<SkData>& iccProfile, SharedBuffer& outBuffer)
2056 {
2057 if (iccProfile == nullptr || iccProfile->size() == 0) {
2058 IMAGE_LOGI("AssembleICCImageProperty iccprofile is nullptr");
2059 return false;
2060 }
2061 std::shared_ptr<AbsMemory> propertyAshmem =
2062 AllocateNewSharedMem(GetBufferSize(iccProfile->size()), ICC_ASHMEM_TAG);
2063 if (propertyAshmem == nullptr) {
2064 IMAGE_LOGE("AssembleICCImageProperty alloc failed");
2065 return false;
2066 }
2067 tmpMemoryList_.push_back(propertyAshmem);
2068 bool fillRes = FillImagePropertyItem(propertyAshmem, 0, PropertyType::ICC_PROFILE,
2069 iccProfile->data(), iccProfile->size());
2070 if (fillRes) {
2071 outBuffer.fd = *static_cast<int *>(propertyAshmem->extend.data);
2072 outBuffer.capacity = propertyAshmem->data.size;
2073 outBuffer.filledLen = propertyAshmem->data.size;
2074 }
2075 return fillRes;
2076 }
2077
FillNclxColorProperty(std::shared_ptr<ImageItem> & item,size_t & offset,ColourInfo & colorInfo)2078 bool ExtEncoder::FillNclxColorProperty(std::shared_ptr<ImageItem>& item, size_t& offset, ColourInfo& colorInfo)
2079 {
2080 ColorType colorType = ColorType::NCLX;
2081 if (!FillLitePropertyItem(item->liteProperties, offset,
2082 PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
2083 IMAGE_LOGE("Fill colorType failed");
2084 return false;
2085 }
2086 if (!FillLitePropertyItem(item->liteProperties, offset,
2087 PropertyType::COLOR_INFO, &colorInfo, sizeof(ColourInfo))) {
2088 IMAGE_LOGI("Fill colorInfo failed");
2089 return false;
2090 }
2091 return true;
2092 }
2093
AssembleOutputSharedBuffer(SharedBuffer & outBuffer,std::shared_ptr<AbsMemory> & outMem)2094 bool ExtEncoder::AssembleOutputSharedBuffer(SharedBuffer& outBuffer, std::shared_ptr<AbsMemory>& outMem)
2095 {
2096 if (output_ == nullptr) {
2097 IMAGE_LOGE("AssembleOutputSharedBuffer output_ is nullptr");
2098 return false;
2099 }
2100 OutputStreamType outType = output_->GetType();
2101 size_t outputCapacity = DEFAULT_OUTPUT_SIZE;
2102 if (outType != OutputStreamType::FILE_PACKER) {
2103 output_->GetCapicity(outputCapacity);
2104 }
2105 std::shared_ptr<AbsMemory> mem = AllocateNewSharedMem(outputCapacity, OUTPUT_ASHMEM_TAG);
2106 if (mem == nullptr) {
2107 IMAGE_LOGE("AssembleOutputSharedBuffer alloc out sharemem failed");
2108 return false;
2109 }
2110 outMem = mem;
2111 tmpMemoryList_.push_back(mem);
2112 outBuffer.fd = *static_cast<int *>(mem->extend.data);
2113 outBuffer.capacity = outputCapacity;
2114 return true;
2115 }
2116
AssembleDualHdrRefItem(std::vector<ItemRef> & refs)2117 void ExtEncoder::AssembleDualHdrRefItem(std::vector<ItemRef>& refs)
2118 {
2119 auto item = std::make_shared<ItemRef>();
2120 item->type = ReferenceType::DIMG;
2121 item->from = TMAP_IMAGE_ITEM_ID;
2122 item->to.resize(INDEX_TWO);
2123 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2124 item->to[INDEX_ONE] = GAINMAP_IMAGE_ITEM_ID;
2125 refs.push_back(*item);
2126 }
2127
DoHeifEncode(std::vector<ImageItem> & inputImgs,std::vector<MetaItem> & inputMetas,std::vector<ItemRef> & refs)2128 uint32_t ExtEncoder::DoHeifEncode(std::vector<ImageItem>& inputImgs, std::vector<MetaItem>& inputMetas,
2129 std::vector<ItemRef>& refs)
2130 {
2131 SharedBuffer outputBuffer {};
2132 std::shared_ptr<AbsMemory> outputAshmem;
2133 bool tempRes = AssembleOutputSharedBuffer(outputBuffer, outputAshmem);
2134 if (!tempRes) {
2135 IMAGE_LOGE("ExtEncoder::DoHeifEncode alloc sharedbuffer failed");
2136 return ERR_IMAGE_INVALID_PARAMETER;
2137 }
2138
2139 sptr<ICodecImage> codec = GetCodecManager();
2140 if (codec == nullptr) {
2141 return ERR_IMAGE_ENCODE_FAILED;
2142 }
2143 uint32_t outSize = DEFAULT_OUTPUT_SIZE;
2144 int32_t encodeRes = codec->DoHeifEncode(inputImgs, inputMetas, refs, outputBuffer, outSize);
2145 if (encodeRes != HDF_SUCCESS) {
2146 IMAGE_LOGE("ExtEncoder::DoHeifEncode DoHeifEncode failed");
2147 return ERR_IMAGE_ENCODE_FAILED;
2148 }
2149 IMAGE_LOGI("ExtEncoder::DoHeifEncode output type is %{public}d", output_->GetType());
2150 bool writeRes = output_->Write(reinterpret_cast<uint8_t *>(outputAshmem->data.data), outSize);
2151 if (!writeRes) {
2152 IMAGE_LOGE("ExtEncoder::DoHeifEncode Write failed");
2153 return ERR_IMAGE_ENCODE_FAILED;
2154 }
2155 return SUCCESS;
2156 }
2157
AssembleTmapImageItem(ColorManager::ColorSpaceName color,HdrMetadata metadata,const PlEncodeOptions & opts)2158 std::shared_ptr<ImageItem> ExtEncoder::AssembleTmapImageItem(ColorManager::ColorSpaceName color,
2159 HdrMetadata metadata, const PlEncodeOptions &opts)
2160 {
2161 auto item = std::make_shared<ImageItem>();
2162 item->id = TMAP_IMAGE_ITEM_ID;
2163 item->itemName = "Tone-mapped representation";
2164 item->isPrimary = false;
2165 item->isHidden = false;
2166 item->compressType = COMPRESS_TYPE_TMAP;
2167 item->quality = opts.quality;
2168 item->sharedProperties.fd = -1;
2169 item->pixelSharedBuffer.fd = -1;
2170 ColourInfo colorInfo;
2171 GetColourInfo(color, colorInfo);
2172 ContentLightLevel light;
2173 MasteringDisplayColourVolume colour;
2174 bool hasLight = GetStaticMetadata(metadata, colour, light);
2175 uint32_t propertiesSize = 0;
2176 propertiesSize +=
2177 (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType) + sizeof(PropertyType::COLOR_INFO) + sizeof(ColourInfo));
2178 if (hasLight) {
2179 propertiesSize += (sizeof(PropertyType::CONTENT_LIGHT_LEVEL) + sizeof(ContentLightLevel));
2180 }
2181 ToneMapMetadata toneMap;
2182 bool hasToneMap = GetToneMapMetadata(metadata, toneMap);
2183 if (hasToneMap) {
2184 propertiesSize += (sizeof(PropertyType::TONE_MAP_METADATA) + sizeof(ToneMapMetadata));
2185 }
2186 item->liteProperties.resize(propertiesSize);
2187 size_t offset = 0;
2188 if (!FillNclxColorProperty(item, offset, colorInfo)) {
2189 return nullptr;
2190 }
2191 if (hasLight && (!FillLitePropertyItem(item->liteProperties, offset,
2192 PropertyType::CONTENT_LIGHT_LEVEL, &colour, sizeof(ContentLightLevel)))) {
2193 IMAGE_LOGE("AssembleTmapImageItem fill CONTENT_LIGHT_LEVEL failed");
2194 return nullptr;
2195 }
2196 if (hasToneMap && (!FillLitePropertyItem(item->liteProperties, offset,
2197 PropertyType::TONE_MAP_METADATA, &toneMap, sizeof(ToneMapMetadata)))) {
2198 IMAGE_LOGE("AssembleTmapImageItem fill toneMap failed");
2199 return nullptr;
2200 }
2201 if (!AssembleIT35SharedBuffer(metadata, item->sharedProperties)) {
2202 IMAGE_LOGE("AssembleTmapImageItem fill it35 failed");
2203 return nullptr;
2204 }
2205 return item;
2206 }
2207
2208 #if !defined(_WIN32) && !defined(_APPLE) && !defined(IOS_PLATFORM) && !defined(ANDROID_PLATFORM)
AssemblePrimaryImageItem(sptr<SurfaceBuffer> & surfaceBuffer,const PlEncodeOptions & opts)2209 std::shared_ptr<ImageItem> ExtEncoder::AssemblePrimaryImageItem(sptr<SurfaceBuffer>& surfaceBuffer,
2210 const PlEncodeOptions &opts)
2211 {
2212 if (pixelmap_ == nullptr || surfaceBuffer == nullptr) {
2213 IMAGE_LOGE("AssemblePrimaryImageItem surfaceBuffer is nullptr");
2214 return nullptr;
2215 }
2216 auto item = std::make_shared<ImageItem>();
2217 item->id = PRIMARY_IMAGE_ITEM_ID;
2218 item->pixelBuffer = sptr<NativeBuffer>::MakeSptr(surfaceBuffer->GetBufferHandle());
2219 item->isPrimary = true;
2220 item->isHidden = false;
2221 item->compressType = COMPRESS_TYPE_HEVC;
2222 item->quality = opts.quality;
2223 item->sharedProperties.fd = -1;
2224 item->pixelSharedBuffer.fd = -1;
2225 sk_sp<SkData> iccProfile = icc_from_color_space(ToSkInfo(pixelmap_));
2226 bool tempRes = AssembleICCImageProperty(iccProfile, item->sharedProperties);
2227 if (!tempRes) {
2228 return nullptr;
2229 }
2230 uint32_t litePropertiesSize = 0;
2231 litePropertiesSize += (sizeof(PropertyType::COLOR_TYPE) + sizeof(ColorType));
2232 item->liteProperties.resize(litePropertiesSize);
2233 size_t offset = 0;
2234 ColorType colorType = ColorType::RICC;
2235 if (!FillLitePropertyItem(item->liteProperties, offset,
2236 PropertyType::COLOR_TYPE, &colorType, sizeof(ColorType))) {
2237 IMAGE_LOGE("AssemblePrimaryImageItem Fill color type failed");
2238 return nullptr;
2239 }
2240 return item;
2241 }
2242 #endif
2243
AssembleExifRefItem(std::vector<ItemRef> & refs)2244 void ExtEncoder::AssembleExifRefItem(std::vector<ItemRef>& refs)
2245 {
2246 auto item = std::make_shared<ItemRef>();
2247 item->type = ReferenceType::CDSC;
2248 item->from = EXIF_META_ITEM_ID;
2249 item->to.resize(INDEX_ONE);
2250 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2251 refs.push_back(*item);
2252 }
2253
FreeExifBlob(uint8_t ** exifBlob)2254 void inline FreeExifBlob(uint8_t** exifBlob)
2255 {
2256 if (exifBlob != nullptr && *exifBlob != nullptr) {
2257 free(*exifBlob);
2258 *exifBlob = nullptr;
2259 }
2260 }
2261
AssembleExifMetaItem(std::vector<MetaItem> & metaItems)2262 bool ExtEncoder::AssembleExifMetaItem(std::vector<MetaItem>& metaItems)
2263 {
2264 if (!opts_.needsPackProperties) {
2265 IMAGE_LOGD("no need encode exif");
2266 return false;
2267 }
2268 ExifData* exifData = nullptr;
2269 if (picture_ != nullptr && picture_->GetExifMetadata() != nullptr &&
2270 picture_->GetExifMetadata()->GetExifData() != nullptr) {
2271 exifData = picture_->GetExifMetadata()->GetExifData();
2272 } else if (pixelmap_ != nullptr && pixelmap_->GetExifMetadata() != nullptr &&
2273 pixelmap_->GetExifMetadata()->GetExifData() != nullptr) {
2274 exifData = pixelmap_->GetExifMetadata()->GetExifData();
2275 } else {
2276 IMAGE_LOGD("no exif");
2277 return false;
2278 }
2279 uint8_t* exifBlob = nullptr;
2280 uint32_t exifSize = 0;
2281 TiffParser::Encode(&exifBlob, exifSize, exifData);
2282 if (exifBlob == nullptr) {
2283 IMAGE_LOGE("Encode exif data failed");
2284 return false;
2285 }
2286 auto item = std::make_shared<MetaItem>();
2287 item->id = EXIF_META_ITEM_ID;
2288 item->itemName = "exif";
2289 item->data.fd = -1;
2290 std::shared_ptr<AbsMemory> propertyAshmem = AllocateNewSharedMem(exifSize + EXIF_PRE_SIZE, EXIF_ASHMEM_TAG);
2291 if (propertyAshmem == nullptr) {
2292 FreeExifBlob(&exifBlob);
2293 IMAGE_LOGE("AssembleExifMetaItem alloc propertyAshmem failed");
2294 return false;
2295 }
2296 tmpMemoryList_.push_back(propertyAshmem);
2297 uint8_t* memData = reinterpret_cast<uint8_t*>(propertyAshmem->data.data);
2298 size_t memSize = propertyAshmem->data.size;
2299 bool fillRes = (memcpy_s(memData, memSize, EXIF_PRE_TAG, EXIF_PRE_SIZE) == EOK) &&
2300 (memcpy_s(memData + EXIF_PRE_SIZE, memSize - EXIF_PRE_SIZE, exifBlob, exifSize) == EOK);
2301 if (fillRes) {
2302 item->data.fd = *static_cast<int *>(propertyAshmem->extend.data);
2303 item->data.capacity = propertyAshmem->data.size;
2304 item->data.filledLen = propertyAshmem->data.size;
2305 metaItems.push_back(*item);
2306 }
2307 FreeExifBlob(&exifBlob);
2308 return fillRes;
2309 }
2310
FillPixelSharedBuffer(sptr<SurfaceBuffer> sbBuffer,uint32_t capacity,SharedBuffer & outBuffer)2311 bool ExtEncoder::FillPixelSharedBuffer(sptr<SurfaceBuffer> sbBuffer, uint32_t capacity, SharedBuffer& outBuffer)
2312 {
2313 if (sbBuffer == nullptr || capacity == 0) {
2314 IMAGE_LOGI("%{public}s iccprofile is nullptr", __func__);
2315 return false;
2316 }
2317 std::shared_ptr<AbsMemory> sharedMem = AllocateNewSharedMem(capacity, IMAGE_DATA_TAG);
2318 if (sharedMem == nullptr) {
2319 IMAGE_LOGE("%{public}s alloc failed", __func__);
2320 return false;
2321 }
2322 tmpMemoryList_.push_back(sharedMem);
2323 uint8_t* memData = reinterpret_cast<uint8_t*>(sharedMem->data.data);
2324 size_t memSize = sharedMem->data.size;
2325 if (capacity > memSize) {
2326 IMAGE_LOGE("%{public}s shared capacity[%{public}d] over memSize[%{public}ld]", __func__, capacity, memSize);
2327 return false;
2328 }
2329 if (sharedMem->extend.data == nullptr) {
2330 IMAGE_LOGE("%{public}s sharedMem's extend data is nullptr", __func__);
2331 return false;
2332 }
2333 if (memcpy_s(memData, memSize, sbBuffer->GetVirAddr(), capacity) == EOK) {
2334 outBuffer.fd = *static_cast<int *>(sharedMem->extend.data);
2335 outBuffer.capacity = memSize;
2336 outBuffer.filledLen = memSize;
2337 } else {
2338 IMAGE_LOGE("%{public}s memcpy failed", __func__);
2339 return false;
2340 }
2341 return true;
2342 }
2343
AssembleAuxiliaryRefItem(AuxiliaryPictureType type,std::vector<ItemRef> & refs)2344 void ExtEncoder::AssembleAuxiliaryRefItem(AuxiliaryPictureType type, std::vector<ItemRef>& refs)
2345 {
2346 auto item = std::make_shared<ItemRef>();
2347 item->type = ReferenceType::AUXL;
2348 item->to.resize(INDEX_ONE);
2349 item->to[INDEX_ZERO] = PRIMARY_IMAGE_ITEM_ID;
2350 switch (type) {
2351 case AuxiliaryPictureType::DEPTH_MAP:
2352 item->from = DEPTH_MAP_ITEM_ID;
2353 break;
2354 case AuxiliaryPictureType::UNREFOCUS_MAP:
2355 item->from = UNREFOCUS_MAP_ITEM_ID;
2356 break;
2357 case AuxiliaryPictureType::LINEAR_MAP:
2358 item->from = LINEAR_MAP_ITEM_ID;
2359 break;
2360 case AuxiliaryPictureType::FRAGMENT_MAP:
2361 item->from = FRAGMENT_MAP_ITEM_ID;
2362 break;
2363 default:
2364 break;
2365 }
2366 refs.push_back(*item);
2367 }
2368 #endif
2369 } // namespace ImagePlugin
2370 } // namespace OHOS
2371