/* * Copyright (C) 2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "iconsumer_surface.h" #include "openssl/crypto.h" #include "openssl/sha.h" #include "videodec_sample.h" using namespace OHOS; using namespace OHOS::Media; using namespace std; namespace { constexpr int64_t NANOS_IN_SECOND = 1000000000L; constexpr int64_t NANOS_IN_MICRO = 1000L; constexpr int32_t EIGHT = 8; constexpr int32_t SIXTEEN = 16; constexpr int32_t TWENTY_FOUR = 24; constexpr uint8_t SEI = 6; constexpr uint8_t SPS = 7; constexpr uint8_t PPS = 8; constexpr uint32_t START_CODE_SIZE = 4; constexpr uint8_t START_CODE[START_CODE_SIZE] = {0, 0, 0, 1}; constexpr uint32_t FRAME_INTERVAL = 16666; constexpr uint32_t EOS_COUNT = 10; constexpr uint32_t MAX_WIDTH = 4000; constexpr uint32_t MAX_HEIGHT = 3000; VDecNdkSample *dec_sample = nullptr; constexpr uint8_t H264_NALU_TYPE = 0x1f; SHA512_CTX c; sptr cs = nullptr; sptr ps = nullptr; unsigned char md[SHA512_DIGEST_LENGTH]; bool g_fuzzError = false; void clearIntqueue(std::queue &q) { std::queue empty; swap(empty, q); } void clearBufferqueue(std::queue &q) { std::queue empty; swap(empty, q); } } // namespace class TestConsumerListener : public IBufferConsumerListener { public: TestConsumerListener(sptr cs, std::string_view name) : cs(cs) { outFile_ = std::make_unique(); outFile_->open(name.data(), std::ios::out | std::ios::binary); }; ~TestConsumerListener() { if (outFile_ != nullptr) { outFile_->close(); } } void OnBufferAvailable() override { sptr buffer; int32_t flushFence; cs->AcquireBuffer(buffer, flushFence, timestamp, damage); cs->ReleaseBuffer(buffer, -1); } private: int64_t timestamp = 0; Rect damage = {}; sptr cs {nullptr}; std::unique_ptr outFile_; }; VDecNdkSample::~VDecNdkSample() { Release(); } void VdecError(OH_AVCodec *codec, int32_t errorCode, void *userData) { VDecSignal *signal = static_cast(userData); if (signal == nullptr) { return; } cout << "Error errorCode=" << errorCode << endl; g_fuzzError = true; signal->inCond_.notify_all(); } void VdecFormatChanged(OH_AVCodec *codec, OH_AVFormat *format, void *userData) { cout << "Format Changed" << endl; int32_t current_width = 0; int32_t current_height = 0; OH_AVFormat_GetIntValue(format, OH_MD_KEY_WIDTH, ¤t_width); OH_AVFormat_GetIntValue(format, OH_MD_KEY_HEIGHT, ¤t_height); dec_sample->DEFAULT_WIDTH = current_width; dec_sample->DEFAULT_HEIGHT = current_height; } void VdecInputDataReady(OH_AVCodec *codec, uint32_t index, OH_AVMemory *data, void *userData) { VDecSignal *signal = static_cast(userData); if (signal == nullptr) { return; } unique_lock lock(signal->inMutex_); signal->inIdxQueue_.push(index); signal->inBufferQueue_.push(data); signal->inCond_.notify_all(); } void VdecOutputDataReady(OH_AVCodec *codec, uint32_t index, OH_AVMemory *data, OH_AVCodecBufferAttr *attr, void *userData) { VDecSignal *signal = static_cast(userData); if (signal == nullptr) { return; } unique_lock lock(signal->outMutex_); signal->outIdxQueue_.push(index); signal->attrQueue_.push(*attr); signal->outBufferQueue_.push(data); signal->outCond_.notify_all(); } bool VDecNdkSample::MdCompare(unsigned char *buffer, int len, const char *source[]) { bool result = true; for (int i = 0; i < len; i++) { char std[SHA512_DIGEST_LENGTH] = {0}; int re = strcmp(source[i], std); if (re != 0) { result = false; break; } } return result; } int64_t VDecNdkSample::GetSystemTimeUs() { struct timespec now; (void)clock_gettime(CLOCK_BOOTTIME, &now); int64_t nanoTime = (int64_t)now.tv_sec * NANOS_IN_SECOND + now.tv_nsec; return nanoTime / NANOS_IN_MICRO; } int32_t VDecNdkSample::ConfigureVideoDecoder() { OH_AVFormat *format = OH_AVFormat_Create(); if (format == nullptr) { cout << "Fatal: Failed to create format" << endl; return AV_ERR_UNKNOWN; } if (maxInputSize > 0) { (void)OH_AVFormat_SetIntValue(format, OH_MD_KEY_MAX_INPUT_SIZE, maxInputSize); } (void)OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, DEFAULT_WIDTH); (void)OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, DEFAULT_HEIGHT); originalHeight = DEFAULT_HEIGHT; originalWidth = DEFAULT_WIDTH; (void)OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, DEFAULT_FRAME_RATE); (void)OH_AVFormat_SetIntValue(format, OH_MD_KEY_ROTATION, DEFAULT_ROTATION); (void)OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, DEFAULT_PIXEL_FORMAT); int ret = OH_VideoDecoder_Configure(vdec_, format); OH_AVFormat_Destroy(format); return ret; } void VDecNdkSample::CheckOutputDescription() { OH_AVFormat *newFormat = OH_VideoDecoder_GetOutputDescription(vdec_); if (newFormat != nullptr) { int32_t picWidth = 0; int32_t picHeight = 0; OH_AVFormat_GetIntValue(newFormat, OH_MD_KEY_VIDEO_PIC_WIDTH, &picWidth); OH_AVFormat_GetIntValue(newFormat, OH_MD_KEY_VIDEO_PIC_HEIGHT, &picHeight); if (picWidth != originalWidth || picHeight != originalHeight) { std::cout << "picWidth:" << picWidth << " picHeight:" << picHeight << std::endl; std::cout << "errCount !=:" << errCount << std::endl; errCount++; } } else { std::cout << "errCount newFormat == nullptr:" << errCount << std::endl; errCount++; } OH_AVFormat_Destroy(newFormat); } int32_t VDecNdkSample::RunVideoDec_Surface(string codeName) { SURFACE_OUTPUT = true; int err = AV_ERR_OK; cs = Surface::CreateSurfaceAsConsumer(); sptr listener = new TestConsumerListener(cs, OUT_DIR); cs->RegisterConsumerListener(listener); auto p = cs->GetProducer(); ps = Surface::CreateSurfaceAsProducer(p); OHNativeWindow *nativeWindow = CreateNativeWindowFromSurface(&ps); if (!nativeWindow) { cout << "Failed to create surface" << endl; return AV_ERR_UNKNOWN; } err = CreateVideoDecoder(codeName); if (err != AV_ERR_OK) { cout << "Failed to create video decoder" << endl; return err; } err = SetVideoDecoderCallback(); if (err != AV_ERR_OK) { cout << "Failed to setCallback" << endl; Release(); return err; } err = ConfigureVideoDecoder(); if (err != AV_ERR_OK) { cout << "Failed to configure video decoder" << endl; Release(); return err; } err = OH_VideoDecoder_SetSurface(vdec_, nativeWindow); if (err != AV_ERR_OK) { cout << "Failed to set surface" << endl; return err; } err = StartVideoDecoder(); if (err != AV_ERR_OK) { cout << "Failed to start video decoder" << endl; Release(); return err; } return err; } int32_t VDecNdkSample::RunVideoDec(string codeName) { SURFACE_OUTPUT = false; int err = CreateVideoDecoder(codeName); if (err != AV_ERR_OK) { cout << "Failed to create video decoder" << endl; return err; } err = ConfigureVideoDecoder(); if (err != AV_ERR_OK) { cout << "Failed to configure video decoder" << endl; Release(); return err; } err = SetVideoDecoderCallback(); if (err != AV_ERR_OK) { cout << "Failed to setCallback" << endl; Release(); return err; } err = StartVideoDecoder(); if (err != AV_ERR_OK) { cout << "Failed to start video decoder" << endl; Release(); return err; } return err; } int32_t VDecNdkSample::SetVideoDecoderCallback() { signal_ = new VDecSignal(); if (signal_ == nullptr) { cout << "Failed to new VDecSignal" << endl; return AV_ERR_UNKNOWN; } cb_.onError = VdecError; cb_.onStreamChanged = VdecFormatChanged; cb_.onNeedInputData = VdecInputDataReady; cb_.onNeedOutputData = VdecOutputDataReady; return OH_VideoDecoder_SetCallback(vdec_, cb_, static_cast(signal_)); } void VDecNdkSample::ReleaseInFile() { if (inFile_ != nullptr) { if (inFile_->is_open()) { inFile_->close(); } inFile_.reset(); inFile_ = nullptr; } } void VDecNdkSample::StopInloop() { if (inputLoop_ != nullptr && inputLoop_->joinable()) { unique_lock lock(signal_->inMutex_); clearIntqueue(signal_->inIdxQueue_); signal_->inCond_.notify_all(); lock.unlock(); inputLoop_->join(); inputLoop_.reset(); } } int32_t VDecNdkSample::StartVideoDecoder() { int ret = OH_VideoDecoder_Start(vdec_); if (ret != AV_ERR_OK) { cout << "Failed to start codec" << endl; return ret; } isRunning_.store(true); inFile_ = make_unique(); if (inFile_ == nullptr) { isRunning_.store(false); (void)OH_VideoDecoder_Stop(vdec_); return AV_ERR_UNKNOWN; } inFile_->open(INP_DIR, ios::in | ios::binary); if (!inFile_->is_open()) { cout << "open input file failed" << endl; isRunning_.store(false); (void)OH_VideoDecoder_Stop(vdec_); inFile_->close(); inFile_.reset(); inFile_ = nullptr; return AV_ERR_UNKNOWN; } inputLoop_ = make_unique(&VDecNdkSample::InputFunc_AVCC, this); if (inputLoop_ == nullptr) { cout << "Failed to create input loop" << endl; isRunning_.store(false); (void)OH_VideoDecoder_Stop(vdec_); ReleaseInFile(); return AV_ERR_UNKNOWN; } outputLoop_ = make_unique(&VDecNdkSample::OutputFunc, this); if (outputLoop_ == nullptr) { cout << "Failed to create output loop" << endl; isRunning_.store(false); (void)OH_VideoDecoder_Stop(vdec_); ReleaseInFile(); StopInloop(); Release(); return AV_ERR_UNKNOWN; } return AV_ERR_OK; } int32_t VDecNdkSample::CreateVideoDecoder(string codeName) { if (!codeName.empty()) { vdec_ = OH_VideoDecoder_CreateByName(codeName.c_str()); } else { vdec_ = OH_VideoDecoder_CreateByMime(OH_AVCODEC_MIMETYPE_VIDEO_AVC); } dec_sample = this; return vdec_ == nullptr ? AV_ERR_UNKNOWN : AV_ERR_OK; } void VDecNdkSample::WaitForEOS() { if (!AFTER_EOS_DESTORY_CODEC && inputLoop_ && inputLoop_->joinable()) { inputLoop_->join(); } if (outputLoop_ && outputLoop_->joinable()) { outputLoop_->join(); } } void VDecNdkSample::WriteOutputFrame(uint32_t index, OH_AVMemory *buffer, OH_AVCodecBufferAttr attr, FILE *outFile) { if (!SURFACE_OUTPUT) { uint8_t *tmpBuffer = new uint8_t[attr.size]; if (memcpy_s(tmpBuffer, attr.size, OH_AVMemory_GetAddr(buffer), attr.size) != EOK) { cout << "Fatal: memory copy failed" << endl; } fwrite(tmpBuffer, 1, attr.size, outFile); SHA512_Update(&c, tmpBuffer, attr.size); delete[] tmpBuffer; if (OH_VideoDecoder_FreeOutputData(vdec_, index) != AV_ERR_OK) { cout << "Fatal: ReleaseOutputBuffer fail" << endl; errCount = errCount + 1; } } else { if (OH_VideoDecoder_RenderOutputData(vdec_, index) != AV_ERR_OK) { cout << "Fatal: RenderOutputBuffer fail" << endl; errCount = errCount + 1; } } } void VDecNdkSample::OutputFunc() { SHA512_Init(&c); FILE *outFile = fopen(OUT_DIR, "wb"); if (outFile == nullptr) { return; } while (true) { if (!isRunning_.load()) { break; } unique_lock lock(signal_->outMutex_); signal_->outCond_.wait(lock, [this]() { if (!isRunning_.load()) { cout << "quit out signal" << endl; return true; } return signal_->outIdxQueue_.size() > 0; }); if (!isRunning_.load()) { break; } uint32_t index = signal_->outIdxQueue_.front(); OH_AVCodecBufferAttr attr = signal_->attrQueue_.front(); OH_AVMemory *buffer = signal_->outBufferQueue_.front(); signal_->outBufferQueue_.pop(); signal_->outIdxQueue_.pop(); signal_->attrQueue_.pop(); lock.unlock(); if (attr.flags == AVCODEC_BUFFER_FLAGS_EOS) { SHA512_Final(md, &c); OPENSSL_cleanse(&c, sizeof(c)); MdCompare(md, SHA512_DIGEST_LENGTH, fileSourcesha256); if (AFTER_EOS_DESTORY_CODEC) { (void)Stop(); Release(); } break; } if (dec_sample->checkOutPut) { CheckOutputDescription(); } WriteOutputFrame(index, buffer, attr, outFile); if (errCount > 0) { break; } } (void)fclose(outFile); } void VDecNdkSample::Flush_buffer() { unique_lock inLock(signal_->inMutex_); clearIntqueue(signal_->inIdxQueue_); std::queue empty; swap(empty, signal_->inBufferQueue_); signal_->inCond_.notify_all(); inLock.unlock(); unique_lock outLock(signal_->outMutex_); clearIntqueue(signal_->outIdxQueue_); clearBufferqueue(signal_->attrQueue_); signal_->outCond_.notify_all(); outLock.unlock(); } void VDecNdkSample::CopyStartCode(uint8_t *frameBuffer, uint32_t bufferSize, OH_AVCodecBufferAttr &attr) { switch (frameBuffer[START_CODE_SIZE] & H264_NALU_TYPE) { case SPS: case PPS: case SEI: if (memcpy_s(frameBuffer, bufferSize + START_CODE_SIZE, START_CODE, START_CODE_SIZE) != EOK) { cout << "Fatal: memory copy failed" << endl; } attr.pts = GetSystemTimeUs(); attr.size = bufferSize + START_CODE_SIZE; attr.offset = 0; attr.flags = AVCODEC_BUFFER_FLAGS_CODEC_DATA; break; default: { if (memcpy_s(frameBuffer, bufferSize + START_CODE_SIZE, START_CODE, START_CODE_SIZE) != EOK) { cout << "Fatal: memory copy failed" << endl; } attr.pts = GetSystemTimeUs(); attr.size = bufferSize + START_CODE_SIZE; attr.offset = 0; attr.flags = AVCODEC_BUFFER_FLAGS_NONE; } } } int32_t VDecNdkSample::ReadData(uint32_t index, OH_AVMemory *buffer) { OH_AVCodecBufferAttr attr; if (BEFORE_EOS_INPUT && frameCount_ > EOS_COUNT) { SetEOS(index); return 1; } if (BEFORE_EOS_INPUT_INPUT && frameCount_ > EOS_COUNT) { memset_s(&attr, sizeof(OH_AVCodecBufferAttr), 0, sizeof(OH_AVCodecBufferAttr)); attr.flags = AVCODEC_BUFFER_FLAGS_EOS; BEFORE_EOS_INPUT_INPUT = false; } uint8_t ch[4] = {}; (void)inFile_->read(reinterpret_cast(ch), START_CODE_SIZE); if (repeatRun && inFile_->eof()) { inFile_->clear(); inFile_->seekg(0, ios::beg); cout << "repeat" << endl; return 0; } else if (inFile_->eof()) { SetEOS(index); return 1; } uint32_t bufferSize = (uint32_t)(((ch[3] & 0xFF)) | ((ch[2] & 0xFF) << EIGHT) | ((ch[1] & 0xFF) << SIXTEEN) | ((ch[0] & 0xFF) << TWENTY_FOUR)); if (bufferSize > MAX_WIDTH * MAX_HEIGHT << 1) { return 1; } return SendData(bufferSize, index, buffer); } uint32_t VDecNdkSample::SendData(uint32_t bufferSize, uint32_t index, OH_AVMemory *buffer) { OH_AVCodecBufferAttr attr; uint8_t *frameBuffer = new uint8_t[bufferSize + START_CODE_SIZE]; (void)inFile_->read(reinterpret_cast(frameBuffer + START_CODE_SIZE), bufferSize); CopyStartCode(frameBuffer, bufferSize, attr); int32_t size = OH_AVMemory_GetSize(buffer); if (size < attr.size) { delete[] frameBuffer; cout << "ERROR:AVMemory not enough, buffer size" << attr.size << " AVMemory Size " << size << endl; isRunning_.store(false); StopOutloop(); return 1; } uint8_t *buffer_addr = OH_AVMemory_GetAddr(buffer); if (memcpy_s(buffer_addr, size, frameBuffer, attr.size) != EOK) { delete[] frameBuffer; cout << "Fatal: memcpy fail" << endl; isRunning_.store(false); return 1; } delete[] frameBuffer; int32_t ret = OH_VideoDecoder_PushInputData(vdec_, index, attr); if (ret != AV_ERR_OK) { errCount++; cout << "push input data failed, error:" << ret << endl; } frameCount_ = frameCount_ + 1; if (inFile_->eof()) { isRunning_.store(false); StopOutloop(); } return 0; } void VDecNdkSample::InputFunc_AVCC() { frameCount_ = 1; errCount = 0; while (true) { if (!isRunning_.load()) { break; } if (frameCount_ % (EOS_COUNT >> 1) == 0) { if (REPEAT_START_FLUSH_BEFORE_EOS > 0) { REPEAT_START_FLUSH_BEFORE_EOS--; OH_VideoDecoder_Flush(vdec_); Flush_buffer(); OH_VideoDecoder_Start(vdec_); } if (REPEAT_START_STOP_BEFORE_EOS > 0) { REPEAT_START_STOP_BEFORE_EOS--; OH_VideoDecoder_Stop(vdec_); Flush_buffer(); OH_VideoDecoder_Start(vdec_); } } unique_lock lock(signal_->inMutex_); signal_->inCond_.wait(lock, [this]() { if (!isRunning_.load()) { cout << "quit signal" << endl; return true; } return signal_->inIdxQueue_.size() > 0; }); if (!isRunning_.load()) { break; } uint32_t index = signal_->inIdxQueue_.front(); auto buffer = signal_->inBufferQueue_.front(); signal_->inIdxQueue_.pop(); signal_->inBufferQueue_.pop(); lock.unlock(); if (!inFile_->eof()) { int ret = ReadData(index, buffer); if (ret == 1) { break; } } if (sleepOnFPS) { usleep(FRAME_INTERVAL); } } } OH_AVErrCode VDecNdkSample::InputFunc_FUZZ(const uint8_t *data, size_t size) { uint32_t index; unique_lock lock(signal_->inMutex_); signal_->inCond_.wait(lock, [this]() { if (!isRunning_.load() && g_fuzzError) { return true; } return signal_->inIdxQueue_.size() > 0; }); if (g_fuzzError) return AV_ERR_TIMEOUT; index = signal_->inIdxQueue_.front(); auto buffer = signal_->inBufferQueue_.front(); lock.unlock(); int32_t buffer_size = OH_AVMemory_GetSize(buffer); uint8_t *buffer_addr = OH_AVMemory_GetAddr(buffer); if (memcpy_s(buffer_addr, buffer_size, data, size) != EOK) { cout << "Fatal: memcpy fail" << endl; return AV_ERR_NO_MEMORY; } OH_AVCodecBufferAttr attr; attr.pts = GetSystemTimeUs(); attr.size = buffer_size; attr.offset = 0; attr.flags = AVCODEC_BUFFER_FLAGS_NONE; OH_AVErrCode ret = OH_VideoDecoder_PushInputData(vdec_, index, attr); signal_->inIdxQueue_.pop(); signal_->inBufferQueue_.pop(); return ret; } void VDecNdkSample::SetEOS(uint32_t index) { OH_AVCodecBufferAttr attr; attr.pts = 0; attr.size = 0; attr.offset = 0; attr.flags = AVCODEC_BUFFER_FLAGS_EOS; int32_t res = OH_VideoDecoder_PushInputData(vdec_, index, attr); cout << "OH_VideoDecoder_PushInputData EOS res: " << res << endl; } int32_t VDecNdkSample::state_EOS() { unique_lock lock(signal_->inMutex_); signal_->inCond_.wait(lock, [this]() { if (!isRunning_.load()) { return true; } return signal_->inIdxQueue_.size() > 0; }); uint32_t index = signal_->inIdxQueue_.front(); signal_->inIdxQueue_.pop(); OH_AVCodecBufferAttr attr; attr.pts = 0; attr.size = 0; attr.offset = 0; attr.flags = AVCODEC_BUFFER_FLAGS_EOS; return OH_VideoDecoder_PushInputData(vdec_, index, attr); } int32_t VDecNdkSample::Flush() { unique_lock inLock(signal_->inMutex_); clearIntqueue(signal_->inIdxQueue_); signal_->inCond_.notify_all(); inLock.unlock(); unique_lock outLock(signal_->outMutex_); clearIntqueue(signal_->outIdxQueue_); clearBufferqueue(signal_->attrQueue_); signal_->outCond_.notify_all(); outLock.unlock(); return OH_VideoDecoder_Flush(vdec_); } int32_t VDecNdkSample::Reset() { isRunning_.store(false); StopInloop(); StopOutloop(); ReleaseInFile(); return OH_VideoDecoder_Reset(vdec_); } int32_t VDecNdkSample::Release() { int ret = 0; if (vdec_ != nullptr) { ret = OH_VideoDecoder_Destroy(vdec_); vdec_ = nullptr; } if (signal_ != nullptr) { delete signal_; signal_ = nullptr; } return ret; } int32_t VDecNdkSample::Stop() { StopInloop(); clearIntqueue(signal_->outIdxQueue_); clearBufferqueue(signal_->attrQueue_); ReleaseInFile(); return OH_VideoDecoder_Stop(vdec_); } int32_t VDecNdkSample::Start() { int32_t ret = OH_VideoDecoder_Start(vdec_); if (ret == AV_ERR_OK) { isRunning_.store(true); } return ret; } void VDecNdkSample::StopOutloop() { if (outputLoop_ != nullptr && outputLoop_->joinable()) { unique_lock lock(signal_->outMutex_); clearIntqueue(signal_->outIdxQueue_); clearBufferqueue(signal_->attrQueue_); signal_->outCond_.notify_all(); lock.unlock(); } } int32_t VDecNdkSample::SetParameter(OH_AVFormat *format) { return OH_VideoDecoder_SetParameter(vdec_, format); }