1 /*
2  * Copyright (c) 2021 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "kws_sdk_impl.h"
17 
18 #include "aie_guard.h"
19 #include "aie_retcode_inner.h"
20 #include "encdec_facade.h"
21 #include "i_aie_client.inl"
22 #include "kws_retcode.h"
23 #include "mfcc_processor.h"
24 #include "plugin_helper.h"
25 
26 using namespace OHOS::AI::Feature;
27 
28 namespace OHOS {
29 namespace AI {
30 namespace {
31     const int16_t ONE_SECOND_MS = 1000;
32 }
33 
InitMFCCConfiguration(MFCCConfig & config)34 static void InitMFCCConfiguration(MFCCConfig &config)
35 {
36     config.dataType = UINT16;
37     config.windowSize = (DEFAULT_MFCC_WINDOW_SIZE_MS * DEFAULT_MFCC_SAMPLE_RATE) / ONE_SECOND_MS;
38     config.slideSize = (DEFAULT_MFCC_SLIDE_SIZE_MS * DEFAULT_MFCC_SAMPLE_RATE) / ONE_SECOND_MS;
39     config.sampleRate = DEFAULT_MFCC_SAMPLE_RATE;
40     config.featureSize = DEFAULT_MFCC_FEATURE_SIZE;
41     config.numChannels = DEFAULT_MFCC_NUM_CHANNELS;
42     config.filterbankLowerBandLimit = DEFAULT_FILTERBANK_LOWER_BAND_LIMIT;
43     config.filterbankUpperBandLimit = DEFAULT_FILTERBANK_UPPER_BAND_LIMIT;
44     config.noiseSmoothingBits = DEFAULT_NOISE_SMOOTHING_BITS;
45     config.noiseEvenSmoothing = DEFAULT_NOISE_EVEN_SMOOTHING;
46     config.noiseOddSmoothing = DEFAULT_NOISE_ODD_SMOOTHING;
47     config.noiseMinSignalRemaining = DEFAULT_NOISE_MIN_SIGNAL_REMAINING;
48     config.enablePcanGain = DEFAULT_ENABLE_PCAN_GAIN;
49     config.pcanGainStrength = DEFAULT_PCAN_GAIN_STRENGTH;
50     config.pcanGainOffset = DEFAULT_PCAN_GAIN_OFFSET;
51     config.pcanGainBits = DEFAULT_PCAN_GAIN_BITS;
52     config.enableLogScale = DEFAULT_ENABLE_LOG_SCALE;
53     config.logScaleShift = DEFAULT_LOG_SCALE_SHIFT;
54 }
55 
56 KWSSdk::KWSSdkImpl::KWSSdkImpl() = default;
57 
~KWSSdkImpl()58 KWSSdk::KWSSdkImpl::~KWSSdkImpl()
59 {
60     Destroy();
61 }
62 
Create()63 int32_t KWSSdk::KWSSdkImpl::Create()
64 {
65     if (kwsHandle_ != INVALID_KWS_HANDLE) {
66         HILOGE("[KWSSdkImpl]Fail to create kws sdk more than once");
67         return KWS_RETCODE_INIT_ERROR;
68     }
69     if (InitComponents() != KWS_RETCODE_SUCCESS) {
70         HILOGE("[KWSSdkImpl]Fail to init sdk components");
71         return KWS_RETCODE_INIT_ERROR;
72     }
73     int32_t retCode = AieClientInit(configInfo_, clientInfo_, algorithmInfo_, nullptr);
74     if (retCode != RETCODE_SUCCESS) {
75         HILOGE("[KWSSdkImpl]AieClientInit failed. Error code[%d]", retCode);
76         return KWS_RETCODE_INIT_ERROR;
77     }
78     if (clientInfo_.clientId == INVALID_CLIENT_ID) {
79         HILOGE("[KWSSdkImpl]Fail to allocate client id");
80         return KWS_RETCODE_INIT_ERROR;
81     }
82     DataInfo inputInfo = {0};
83     DataInfo outputInfo = {0};
84     retCode = AieClientPrepare(clientInfo_, algorithmInfo_, inputInfo, outputInfo, nullptr);
85     if (retCode != RETCODE_SUCCESS) {
86         HILOGE("[KWSSdkImpl]AieClientPrepare failed. Error code[%d]", retCode);
87         return KWS_RETCODE_INIT_ERROR;
88     }
89     if (outputInfo.data == nullptr || outputInfo.length <= 0) {
90         HILOGE("[KWSSdkImpl]The data or length of output info is invalid");
91         return KWS_RETCODE_INIT_ERROR;
92     }
93     MallocPointerGuard<unsigned char> pointerGuard(outputInfo.data);
94     retCode = EncdecFacade::ProcessDecode(outputInfo, kwsHandle_);
95     if (retCode != RETCODE_SUCCESS) {
96         HILOGE("[KWSSdkImpl]Fail to get handle from input info");
97         return KWS_RETCODE_UNSERIALIZATION_ERROR;
98     }
99     return KWS_RETCODE_SUCCESS;
100 }
101 
SyncExecute(const Array<int16_t> & input)102 int32_t KWSSdk::KWSSdkImpl::SyncExecute(const Array<int16_t> &input)
103 {
104     if (kwsHandle_ == INVALID_KWS_HANDLE) {
105         HILOGE("[KWSSdkImpl]The SDK has not been created");
106         return KWS_RETCODE_FAILURE;
107     }
108     if (callback_ == nullptr) {
109         HILOGE("[KWSSdkImpl]Fail to execute with nullptr callback");
110         return KWS_RETCODE_FAILURE;
111     }
112     Array<int16_t> pcmInput = {0};
113     Array<uint16_t> mfccInput = {0};
114     FeatureData pcmFeature = {
115         .dataType = INT16,
116         .data = nullptr,
117         .size = 0
118     };
119     FeatureData mfccFeature = {
120         .dataType = UINT16,
121         .data = nullptr,
122         .size = 0
123     };
124     int32_t retCode = pcmIterator_->SetInput(input);
125     if (retCode != RETCODE_SUCCESS) {
126         HILOGE("[KWSSdkImpl]Fail to set input to pcm iterator");
127         return KWS_RETCODE_FAILURE;
128     }
129     while (pcmIterator_->HasNext()) {
130         pcmInput = pcmIterator_->Next();
131         pcmFeature.data = pcmInput.data;
132         pcmFeature.size = pcmInput.size;
133         // Preprocess
134         if (mfccProcessor_->Process(pcmFeature, mfccFeature) != RETCODE_SUCCESS) {
135             HILOGE("[KWSSdkImpl]Fail to process pcm data");
136             return KWS_RETCODE_FAILURE;
137         }
138         mfccInput.data = static_cast<uint16_t *>(mfccFeature.data);
139         mfccInput.size = mfccFeature.size;
140         // Execute
141         retCode = Execute(mfccInput);
142         if (retCode != KWS_RETCODE_SUCCESS) {
143             HILOGE("[KWSSdkImpl]Fail to execute synchronously");
144             return retCode;
145         }
146         mfccFeature.data = nullptr;
147         mfccFeature.size = 0;
148     }
149     return KWS_RETCODE_SUCCESS;
150 }
151 
Execute(const Array<uint16_t> & input)152 int32_t KWSSdk::KWSSdkImpl::Execute(const Array<uint16_t> &input)
153 {
154     intptr_t receivedHandle = 0;
155     Array<int32_t> kwsResult = {0};
156     DataInfo inputInfo = {0};
157     DataInfo outputInfo = {0};
158     int32_t retCode = EncdecFacade::ProcessEncode(inputInfo, kwsHandle_, input);
159     if (retCode != RETCODE_SUCCESS) {
160         HILOGE("[KWSSdkImpl]Fail to serialize input data");
161         callback_->OnError(KWS_RETCODE_SERIALIZATION_ERROR);
162         return KWS_RETCODE_SERIALIZATION_ERROR;
163     }
164     retCode = AieClientSyncProcess(clientInfo_, algorithmInfo_, inputInfo, outputInfo);
165     if (retCode != RETCODE_SUCCESS) {
166         HILOGE("[KWSSdkImpl]AieClientSyncProcess failed. Error code[%d]", retCode);
167         callback_->OnError(KWS_RETCODE_PLUGIN_EXECUTION_ERROR);
168         return KWS_RETCODE_PLUGIN_EXECUTION_ERROR;
169     }
170     if (outputInfo.data == nullptr || outputInfo.length <= 0) {
171         HILOGE("[KWSSdkImpl]The data or length of output info is invalid. Error code[%d]", retCode);
172         callback_->OnError(KWS_RETCODE_NULL_PARAM);
173         return KWS_RETCODE_NULL_PARAM;
174     }
175     MallocPointerGuard<unsigned char> pointerGuard(outputInfo.data);
176     retCode = EncdecFacade::ProcessDecode(outputInfo, receivedHandle, kwsResult);
177     if (retCode != RETCODE_SUCCESS) {
178         HILOGE("[KWSSdkImpl]UnSerializeOutputData failed. Error code[%d]", retCode);
179         callback_->OnError(KWS_RETCODE_UNSERIALIZATION_ERROR);
180         return KWS_RETCODE_UNSERIALIZATION_ERROR;
181     }
182     if (kwsHandle_ != receivedHandle) {
183         HILOGE("[KWSSdkImpl]The handle[%lld] of output data is not equal to the current handle[%lld]",
184             static_cast<long long>(receivedHandle), static_cast<long long>(kwsHandle_));
185         callback_->OnError(KWS_RETCODE_PLUGIN_SESSION_ERROR);
186         return KWS_RETCODE_PLUGIN_SESSION_ERROR;
187     }
188     callback_->OnResult(kwsResult);
189     return KWS_RETCODE_SUCCESS;
190 }
191 
SetCallback(std::shared_ptr<KWSCallback> callback)192 int32_t KWSSdk::KWSSdkImpl::SetCallback(std::shared_ptr<KWSCallback> callback)
193 {
194     if (callback == nullptr) {
195         return KWS_RETCODE_NULL_PARAM;
196     }
197     callback_ = callback;
198     return KWS_RETCODE_SUCCESS;
199 }
200 
InitComponents()201 int32_t KWSSdk::KWSSdkImpl::InitComponents()
202 {
203     // Create MFCC Processor
204     MFCCConfig mfccConfig;
205     InitMFCCConfiguration(mfccConfig);
206     mfccProcessor_ = std::unique_ptr<FeatureProcessor>(new (std::nothrow) MFCCProcessor());
207     if (mfccProcessor_ == nullptr) {
208         HILOGE("[KWSSdkImpl]Fail to allocate memory for MFCCProcessor");
209         return KWS_RETCODE_FAILURE;
210     }
211     if (mfccProcessor_->Init(&mfccConfig) != RETCODE_SUCCESS) {
212         HILOGE("[KWSSdkImpl]Fail to init MFCCProcessor");
213         return KWS_RETCODE_FAILURE;
214     }
215     // Create PCM Iterator
216     pcmIterator_ = std::unique_ptr<PCMIterator>(new (std::nothrow) PCMIterator());
217     if (pcmIterator_ == nullptr) {
218         HILOGE("[KWSSdkImpl]Fail to allocate memory for PCMIterator");
219         return KWS_RETCODE_FAILURE;
220     }
221     size_t stepSize = (DEFAULT_SLIDE_STEP_SIZE / DEFAULT_MFCC_NUM_CHANNELS) * mfccConfig.slideSize;
222     size_t windowSize = stepSize + mfccConfig.windowSize - mfccConfig.slideSize;
223     if (pcmIterator_->Init(stepSize, windowSize) != RETCODE_SUCCESS) {
224         HILOGE("[KWSSdkImpl]Fail to init PCMIterator");
225         return KWS_RETCODE_FAILURE;
226     }
227     return KWS_RETCODE_SUCCESS;
228 }
229 
Destroy()230 int32_t KWSSdk::KWSSdkImpl::Destroy()
231 {
232     if (kwsHandle_ == INVALID_KWS_HANDLE) {
233         return KWS_RETCODE_SUCCESS;
234     }
235     DataInfo inputInfo = {0};
236     int32_t retCode = EncdecFacade::ProcessEncode(inputInfo, kwsHandle_);
237     if (retCode != RETCODE_SUCCESS) {
238         HILOGE("[KWSSdkImpl]SerializeHandle failed. Error code[%d]", retCode);
239         return KWS_RETCODE_SERIALIZATION_ERROR;
240     }
241     retCode = AieClientRelease(clientInfo_, algorithmInfo_, inputInfo);
242     if (retCode != RETCODE_SUCCESS) {
243         HILOGE("[KWSSdkImpl]AieClientRelease failed. Error code[%d]", retCode);
244         return KWS_RETCODE_FAILURE;
245     }
246     retCode = AieClientDestroy(clientInfo_);
247     if (retCode != RETCODE_SUCCESS) {
248         HILOGE("[KWSSdkImpl]AieClientDestroy failed. Error code[%d]", retCode);
249         return KWS_RETCODE_FAILURE;
250     }
251     mfccProcessor_ = nullptr;
252     pcmIterator_ = nullptr;
253     callback_ = nullptr;
254     kwsHandle_ = INVALID_KWS_HANDLE;
255     return KWS_RETCODE_SUCCESS;
256 }
257 } // namespace AI
258 } // namespace OHOS