1 /*
2  * Copyright (c) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "ic_plugin.h"
17 
18 #include <algorithm>
19 #include <utility>
20 #include <vector>
21 
22 #include "aie_log.h"
23 #include "aie_retcode_inner.h"
24 #include "encdec_facade.h"
25 #include "ic_constants.h"
26 #include "plugin_helper.h"
27 #include "securec.h"
28 
29 #ifdef USE_NNIE
30 #include "nnie_adapter.h"
31 #endif
32 
33 using namespace std;
34 
35 namespace OHOS {
36 namespace AI {
37 namespace {
38     const std::string PLUGIN_MODEL_PATH = "/storage/data/image_classification.wk";
39     const std::string DEFAULT_INFER_MODE = "SYNC";
40     const std::string ALGORITHM_NAME_IC = "IC";
41     const int32_t OPTION_GET_INPUT_SIZE = 1001;
42     const int32_t OPTION_GET_OUTPUT_SIZE = 1002;
43     const int32_t OPTION_SET_OUTPUT_SIZE = 2002;
44     const uint16_t MODEL_INPUT_NODE_ID = 0;
45     const uint16_t MODEL_OUTPUT_NODE_ID = 0;
46     const uint16_t DEFAULT_OUTPUT_SIZE = 5;
47     const intptr_t EMPTY_UINTPTR = 0;
48     using Item = std::pair<int32_t, int32_t>;
49     using Items = std::vector<Item>;
GetTopK(const int32_t * data,size_t size,size_t topK,Items & result)50     int32_t GetTopK(const int32_t *data, size_t size, size_t topK, Items &result)
51     {
52         if (data == nullptr) {
53             HILOGE("[ICPlugin]Fail with null data pointer");
54             return RETCODE_FAILURE;
55         }
56         if (topK > size) {
57             topK = size;
58         }
59         size_t index = 0;
60         while (index < topK) {
61             result.emplace_back(index, data[index]);
62             index++;
63         }
64         const auto heapComparer = [](const Item &x, const Item &y) {
65             return (x.second > y.second);
66         };
67         std::make_heap(result.begin(), result.end(), heapComparer);
68         while (index < size) {
69             if (result.front().second < data[index]) {
70                 std::pop_heap(result.begin(), result.end(), heapComparer);
71                 result.pop_back();
72                 result.emplace_back(index, data[index]);
73                 std::push_heap(result.begin(), result.end(), heapComparer);
74             }
75             index++;
76         }
77         std::sort_heap(result.begin(), result.end(), heapComparer);
78         return RETCODE_SUCCESS;
79     }
80 }
81 
ICPlugin()82 ICPlugin::ICPlugin() : adapter_(nullptr)
83 {
84     HILOGD("[ICPlugin]Ctor");
85     handles_.clear();
86 }
87 
~ICPlugin()88 ICPlugin::~ICPlugin()
89 {
90     HILOGD("[ICPlugin]Dtor");
91     ReleaseAllHandles();
92 }
93 
Prepare(long long transactionId,const DataInfo & inputInfo,DataInfo & outputInfo)94 int32_t ICPlugin::Prepare(long long transactionId, const DataInfo &inputInfo, DataInfo &outputInfo)
95 {
96     HILOGI("[ICPlugin]Start to prepare, transactionId = %lld", transactionId);
97     if (adapter_ == nullptr) {
98 #ifdef USE_NNIE
99         adapter_ = std::make_shared<NNIEAdapter>();
100 #endif
101         if (adapter_ == nullptr) {
102             HILOGE("[ICPlugin]Fail to create engine adapter");
103             return RETCODE_FAILURE;
104         }
105     }
106     intptr_t handle = EMPTY_UINTPTR;
107     if (adapter_->Init(PLUGIN_MODEL_PATH.c_str(), handle) != RETCODE_SUCCESS) {
108         HILOGE("[ICPlugin]EngineAdapterInit failed");
109         return RETCODE_FAILURE;
110     }
111     std::lock_guard<std::mutex> lock(mutex_);
112     const auto iter = handles_.find(handle);
113     if (iter != handles_.end()) {
114         HILOGE("[ICPlugin]Handle=%lld has already existed", static_cast<long long>(handle));
115         return RETCODE_SUCCESS;
116     }
117     ICPluginConfig config;
118     if (BuildConfig(handle, config) != RETCODE_SUCCESS) {
119         HILOGE("[ICPlugin]BuildConfig failed");
120         return RETCODE_FAILURE;
121     }
122     handles_.emplace(handle, config);
123     return EncdecFacade::ProcessEncode(outputInfo, handle);
124 }
125 
Release(bool isFullUnload,long long transactionId,const DataInfo & inputInfo)126 int32_t ICPlugin::Release(bool isFullUnload, long long transactionId, const DataInfo &inputInfo)
127 {
128     if (adapter_ == nullptr) {
129         HILOGE("[ICPlugin]The engine adapter has not been created");
130         return RETCODE_FAILURE;
131     }
132     HILOGI("[ICPlugin]Begin to release, transactionId = %lld", transactionId);
133     intptr_t handle = EMPTY_UINTPTR;
134     int32_t retCode = EncdecFacade::ProcessDecode(inputInfo, handle);
135     if (retCode != RETCODE_SUCCESS) {
136         HILOGE("[ICPlugin]UnSerializeHandle Failed");
137         return RETCODE_FAILURE;
138     }
139     std::lock_guard<std::mutex> lock(mutex_);
140     const auto iter = handles_.find(handle);
141     if (iter == handles_.end()) {
142         HILOGE("[ICPlugin]Fail to find handle(%lld)", static_cast<long long>(handle));
143         return RETCODE_NULL_PARAM;
144     }
145     retCode = adapter_->ReleaseHandle(handle);
146     if (retCode != RETCODE_SUCCESS) {
147         HILOGE("[ICPlugin]ReleaseHandle failed");
148         return RETCODE_FAILURE;
149     }
150     handles_.erase(iter);
151     if (isFullUnload) {
152         retCode = adapter_->Deinit();
153         if (retCode != RETCODE_SUCCESS) {
154             HILOGE("[ICPlugin]Engine adapter deinit failed");
155             return RETCODE_FAILURE;
156         }
157     }
158     return RETCODE_SUCCESS;
159 }
160 
ReleaseAllHandles()161 void ICPlugin::ReleaseAllHandles()
162 {
163     for (auto iter = handles_.begin(); iter != handles_.end(); ++iter) {
164         (void)adapter_->ReleaseHandle(iter->first);
165     }
166     adapter_->Deinit();
167     handles_.clear();
168 }
169 
GetVersion() const170 const long long ICPlugin::GetVersion() const
171 {
172     return ALGOTYPE_VERSION_IC;
173 }
174 
GetName() const175 const char *ICPlugin::GetName() const
176 {
177     return ALGORITHM_NAME_IC.c_str();
178 }
179 
GetInferMode() const180 const char *ICPlugin::GetInferMode() const
181 {
182     return DEFAULT_INFER_MODE.c_str();
183 }
184 
SetOption(int32_t optionType,const DataInfo & inputInfo)185 int32_t ICPlugin::SetOption(int32_t optionType, const DataInfo &inputInfo)
186 {
187     if (inputInfo.data == nullptr || inputInfo.length <= 0) {
188         HILOGE("[ICPlugin]Fail to set option with empty input info");
189         return RETCODE_NULL_PARAM;
190     }
191     intptr_t handle = EMPTY_UINTPTR;
192     uint32_t tmpUInt32Val = 0;
193     int32_t retCode = RETCODE_SUCCESS;
194     ICPluginConfig newConfig;
195     newConfig.outputSize = 0;
196     switch (optionType) {
197         case OPTION_SET_OUTPUT_SIZE:
198             retCode = EncdecFacade::ProcessDecode(inputInfo, handle, tmpUInt32Val);
199             if (retCode != RETCODE_SUCCESS) {
200                 HILOGE("[ICPlugin]Fail to unserialize output size");
201                 return retCode;
202             }
203             newConfig.outputSize = tmpUInt32Val;
204             break;
205         default:
206             HILOGE("[ICPlugin]OptionType[%d] is not supported", optionType);
207             return RETCODE_FAILURE;
208     }
209     std::lock_guard<std::mutex> lock(mutex_);
210     const auto iter = handles_.find(handle);
211     if (iter == handles_.end()) {
212         HILOGE("[ICPlugin]No matched handle[%lld]", static_cast<long long>(handle));
213         return RETCODE_FAILURE;
214     }
215     switch (optionType) {
216         case OPTION_SET_OUTPUT_SIZE:
217             if (newConfig.outputSize > 0 && newConfig.outputSize <= iter->second.maxOutputSize) {
218                 iter->second.outputSize = newConfig.outputSize;
219             }
220             break;
221         default:
222             break;
223     }
224     return RETCODE_SUCCESS;
225 }
226 
GetOption(int32_t optionType,const DataInfo & inputInfo,DataInfo & outputInfo)227 int32_t ICPlugin::GetOption(int32_t optionType, const DataInfo &inputInfo, DataInfo &outputInfo)
228 {
229     if (inputInfo.data == nullptr || inputInfo.length <= 0) {
230         HILOGE("[ICPlugin]Fail to set option with empty input info");
231         return RETCODE_NULL_PARAM;
232     }
233     intptr_t handle = EMPTY_UINTPTR;
234     int32_t retCode = EncdecFacade::ProcessDecode(inputInfo, handle);
235     if (retCode != RETCODE_SUCCESS) {
236         HILOGE("[ICPlugin]Fail to get handle from input info");
237         return retCode;
238     }
239     std::lock_guard<std::mutex> lock(mutex_);
240     const auto iter = handles_.find(handle);
241     if (iter == handles_.end()) {
242         HILOGE("[ICPlugin]No matched handle [%lld]", static_cast<long long>(handle));
243         return RETCODE_FAILURE;
244     }
245     outputInfo.length = 0;
246     switch (optionType) {
247         case OPTION_GET_INPUT_SIZE:
248             return EncdecFacade::ProcessEncode(outputInfo, handle, iter->second.inputSize);
249         case OPTION_GET_OUTPUT_SIZE:
250             return EncdecFacade::ProcessEncode(outputInfo, handle, iter->second.outputSize);
251         default:
252             HILOGE("[ICPlugin]GetOption optionType[%d] undefined", optionType);
253             return RETCODE_FAILURE;
254     }
255     return RETCODE_SUCCESS;
256 }
257 
BuildConfig(intptr_t handle,ICPluginConfig & config)258 int32_t ICPlugin::BuildConfig(intptr_t handle, ICPluginConfig &config)
259 {
260     int32_t retCode = adapter_->GetInputAddr(handle, MODEL_INPUT_NODE_ID, config.inputAddr, config.inputSize);
261     if (retCode != RETCODE_SUCCESS) {
262         HILOGE("[ICPlugin]GetInputAddr failed with error code[%d]", retCode);
263         return RETCODE_FAILURE;
264     }
265     retCode = adapter_->GetOutputAddr(handle, MODEL_OUTPUT_NODE_ID, config.outputAddr, config.maxOutputSize);
266     if (retCode != RETCODE_SUCCESS) {
267         HILOGE("[ICPlugin]GetOutputAddr failed with error code[%d]", retCode);
268         return RETCODE_FAILURE;
269     }
270     config.outputSize = DEFAULT_OUTPUT_SIZE;
271     return RETCODE_SUCCESS;
272 }
273 
SyncProcess(IRequest * request,IResponse * & response)274 int32_t ICPlugin::SyncProcess(IRequest *request, IResponse *&response)
275 {
276     if (adapter_ == nullptr) {
277         HILOGE("[ICPlugin]The engine adapter has not been created");
278         return RETCODE_FAILURE;
279     }
280     if (request == nullptr) {
281         HILOGE("[ICPlugin]Fail to synchronously process with nullptr request");
282         return RETCODE_NULL_PARAM;
283     }
284     DataInfo inputInfo = request->GetMsg();
285     if (inputInfo.data == nullptr || inputInfo.length <= 0) {
286         HILOGE("[ICPlugin]Fail to synchronously process with empty input info");
287         return RETCODE_NULL_PARAM;
288     }
289     intptr_t handle = EMPTY_UINTPTR;
290     uint32_t slicedIndex = 0;
291     Array<uint8_t> slicedImage = {0};
292     int32_t retCode = EncdecFacade::ProcessDecode(inputInfo, handle, slicedIndex, slicedImage);
293     if (retCode != RETCODE_SUCCESS) {
294         HILOGE("[ICPlugin]Fail to unserialize input data");
295         return retCode;
296     }
297     std::lock_guard<std::mutex> lock(mutex_);
298     const auto iter = handles_.find(handle);
299     if (iter == handles_.end()) {
300         HILOGE("[ICPlugin]No matched handle [%lld]", static_cast<long long>(handle));
301         return RETCODE_FAILURE;
302     }
303     if (slicedIndex + slicedImage.size > iter->second.inputSize) {
304         HILOGE("[ICPlugin]Illegal slicedIndex");
305         return RETCODE_FAILURE;
306     }
307     auto imageAddr = reinterpret_cast<uint8_t *>(iter->second.inputAddr);
308     errno_t retCopy = memcpy_s(&imageAddr[slicedIndex], iter->second.inputSize - slicedIndex,
309         slicedImage.data, slicedImage.size);
310     if (retCopy != EOK) {
311         HILOGE("[ICPlugin]Fail to copy sliced data to model input space");
312         return RETCODE_FAILURE;
313     }
314     if (slicedIndex + slicedImage.size == iter->second.inputSize) {
315         retCode = DoProcess(handle, iter->second, request, response);
316         if (retCode != RETCODE_SUCCESS) {
317             HILOGE("[ICPlugin]Fail to do process");
318             return retCode;
319         }
320     }
321     return RETCODE_SUCCESS;
322 }
323 
DoProcess(intptr_t handle,const ICPluginConfig & config,IRequest * request,IResponse * & response)324 int32_t ICPlugin::DoProcess(intptr_t handle, const ICPluginConfig &config, IRequest *request, IResponse *&response)
325 {
326     DataInfo outputInfo = {0};
327     int32_t retCode = MakeInference(handle, config, outputInfo);
328     if (retCode != RETCODE_SUCCESS) {
329         HILOGE("[ICPlugin]Fail to make inference");
330         return retCode;
331     }
332     response = IResponse::Create(request);
333     response->SetResult(outputInfo);
334     return RETCODE_SUCCESS;
335 }
336 
AsyncProcess(IRequest * request,IPluginCallback * callback)337 int32_t ICPlugin::AsyncProcess(IRequest *request, IPluginCallback *callback)
338 {
339     return RETCODE_SUCCESS;
340 }
341 
MakeInference(intptr_t handle,const ICPluginConfig & config,DataInfo & outputInfo)342 int32_t ICPlugin::MakeInference(intptr_t handle, const ICPluginConfig &config, DataInfo &outputInfo)
343 {
344     HILOGI("[ICPlugin]Start with handle = %lld", static_cast<long long>(handle));
345     int32_t retCode = adapter_->Invoke(handle);
346     if (retCode != RETCODE_SUCCESS) {
347         HILOGE("[ICPlugin]MakeInference failed");
348         return RETCODE_FAILURE;
349     }
350     // Return top K
351     int32_t *outputData = reinterpret_cast<int32_t *>(config.outputAddr);
352     vector<pair<int32_t, int32_t>> result;
353     result.clear();
354     retCode = GetTopK(outputData, config.maxOutputSize, config.outputSize, result);
355     if (retCode != RETCODE_SUCCESS) {
356         HILOGE("[ICPlugin]Fail to get TopK");
357         return retCode;
358     }
359     retCode = EncdecFacade::ProcessEncode(outputInfo, handle, result);
360     if (retCode != RETCODE_SUCCESS) {
361         HILOGE("[ICPlugin]Fail to serialize output data");
362     }
363     return retCode;
364 }
365 
366 PLUGIN_INTERFACE_IMPL(ICPlugin);
367 } // namespace AI
368 } // namespace OHOS