1 /*
2  * Copyright (c) 2022 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef TENSORFLOW_LITE_DELEGATES_NNRT_OP_BUILDER_H
17 #define TENSORFLOW_LITE_DELEGATES_NNRT_OP_BUILDER_H
18 
19 #include <vector>
20 
21 #include "tensorflow/lite/builtin_ops.h"
22 #include "tensorflow/lite/c/builtin_op_data.h"
23 #include "tensorflow/lite/kernels/kernel_util.h"
24 #include "tensorflow/lite/minimal_logging.h"
25 
26 #include "../nnrt/nnrt_implementation.h"
27 #include "tensor_mapping.h"
28 
29 namespace tflite {
30 namespace delegate {
31 namespace nnrt {
32 constexpr int32_t PADDING_SAME = 0;
33 constexpr int32_t PADDING_VALID = 1;
34 
35 // NN API Operator Builder
36 class NnrtOpBuilder;
37 
38 // The kernel that represents the node sub set of TF Lite being run on NN API.
39 struct NnrtOpMappingArgs {
40     TfLiteContext* context {nullptr};
41     NnrtOpBuilder* builder {nullptr};
42     TfLiteNode* node {nullptr};
43     int32_t nodeIndex {-1};
44 };
45 
46 struct NnrtOpBuilderArgs {
47     TfLiteContext* context {nullptr};
48     OH_NNModel* nnModel {nullptr};
49     TfLiteIntArray* inputTensors {nullptr};
50     TensorMapping* pTensorMapping {nullptr};
51     NnrtDelegate::Options delegateOptions;
52 };
53 
54 // Abstract builder for building an op in the NN API graph. This handles
55 // the disparity between TFLite and NN API nnTensor types. NN API has singular
56 // nnTensors for both tensors and parameters, and TFLite separates the two.
57 class NnrtOpBuilder {
58 public:
59     NnrtOpBuilder(const NnrtApi* nnrt, NnrtOpBuilderArgs& opBuilderArgs);
60     ~NnrtOpBuilder() = default;
61 
62     // Add scalar nnTensor, the datatypes involved are bool, Int32, Int8, Int64, Float32
AddScalarBoolTensor(bool value,OH_NN_TensorType nnTensorType)63     TfLiteStatus AddScalarBoolTensor(bool value, OH_NN_TensorType nnTensorType)
64     {
65         return AddScalarTensor<bool>(value, OH_NN_BOOL, nnTensorType);
66     }
AddScalarInt32Tensor(int32_t value,OH_NN_TensorType nnTensorType)67     TfLiteStatus AddScalarInt32Tensor(int32_t value, OH_NN_TensorType nnTensorType)
68     {
69         return AddScalarTensor<int32_t>(value, OH_NN_INT32, nnTensorType);
70     }
AddScalarInt8Tensor(int32_t value,OH_NN_TensorType nnTensorType)71     TfLiteStatus AddScalarInt8Tensor(int32_t value, OH_NN_TensorType nnTensorType)
72     {
73         return AddScalarTensor<int8_t>(value, OH_NN_INT8, nnTensorType);
74     }
AddScalarInt64Tensor(int64_t value,OH_NN_TensorType nnTensorType)75     TfLiteStatus AddScalarInt64Tensor(int64_t value, OH_NN_TensorType nnTensorType)
76     {
77         return AddScalarTensor<int64_t>(value, OH_NN_INT64, nnTensorType);
78     }
AddScalarFloat32Tensor(float value,OH_NN_TensorType nnTensorType)79     TfLiteStatus AddScalarFloat32Tensor(float value, OH_NN_TensorType nnTensorType)
80     {
81         return AddScalarTensor<float>(value, OH_NN_FLOAT32, nnTensorType);
82     }
83 
84     // Add vector nnTensor, the datatypes involved are Int32, Int64, Int16, Int8, Float32
AddVectorInt32Tensor(const int32_t * values,uint32_t numValues,OH_NN_TensorType nnTensorType)85     TfLiteStatus AddVectorInt32Tensor(const int32_t* values, uint32_t numValues, OH_NN_TensorType nnTensorType)
86     {
87         return AddVectorTensor<int32_t>(values, numValues, OH_NN_UINT32, nnTensorType);
88     }
AddVectorInt64Tensor(const int64_t * values,uint32_t numValues,OH_NN_TensorType nnTensorType)89     TfLiteStatus AddVectorInt64Tensor(const int64_t* values, uint32_t numValues, OH_NN_TensorType nnTensorType)
90     {
91         return AddVectorTensor<int64_t>(values, numValues, OH_NN_INT64, nnTensorType);
92     }
AddVectorFloat32Tensor(const float * values,uint32_t numValues,OH_NN_TensorType nnTensorType)93     TfLiteStatus AddVectorFloat32Tensor(const float* values, uint32_t numValues, OH_NN_TensorType nnTensorType)
94     {
95         return AddVectorTensor<float>(values, numValues, OH_NN_FLOAT32, nnTensorType);
96     }
97 
98     // Add input tensor
99     TfLiteStatus AddTensorInput(int32_t tensorIndex, int32_t builtinCode, int32_t tensorFlags = 0)
100     {
101         return AddTensor(tensorIndex, builtinCode, m_augmentedInputs, tensorFlags);
102     }
103     // Add output tensor
104     TfLiteStatus AddTensorOutput(int32_t tensorIndex, int32_t builtinCode, int32_t tensorFlags = 0)
105     {
106         return AddTensor(tensorIndex, builtinCode, m_augmentedOutputs, tensorFlags);
107     }
108 
109     // Finish emitting the op (of type `type`) into the NN API.
110     TfLiteStatus FinalizeAddOperation(OH_NN_OperationType type, int32_t liteNodeIndex);
111 
ClearInputOuputLists()112     void ClearInputOuputLists()
113     {
114         m_augmentedInputs.clear();
115         m_augmentedOutputs.clear();
116         m_augmentedParams.clear();
117     }
118 
119     TfLiteStatus AddOpFuncParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
120 
121     TfLiteStatus MapBuiltinCodeToFunc();
122 
123 private:
124     template<typename T>
AddScalarTensor(T value,OH_NN_DataType nnType,OH_NN_TensorType nnTensorType)125     TfLiteStatus AddScalarTensor(T value, OH_NN_DataType nnType, OH_NN_TensorType nnTensorType)
126     {
127         OH_NN_Tensor tensor {
128             .dataType = nnType,
129             .dimensionCount = 0,
130             .dimensions = nullptr,
131             .quantParam = nullptr,
132             .type = nnTensorType,
133         };
134 
135         RETURN_TFLITE_ERROR_IF_NN_ERROR(
136             m_nnrt->OH_NNModel_AddTensor(m_nnModel, &tensor), "adding nnTensor");
137 
138         const int32_t nnIndex = m_pTensorMapping->AddNewNonTensorTensor();
139         RETURN_TFLITE_ERROR_IF_NN_ERROR(
140             m_nnrt->OH_NNModel_SetTensorData(m_nnModel, nnIndex, &value, sizeof(value)),
141             "setting new nnTensor value");
142         m_augmentedParams.emplace_back(nnIndex);
143 
144         return kTfLiteOk;
145     }
146 
147     template<typename T>
AddVectorTensor(const T * values,int32_t numValues,OH_NN_DataType nnType,OH_NN_TensorType nnTensorType)148     TfLiteStatus AddVectorTensor(const T* values, int32_t numValues, OH_NN_DataType nnType,
149         OH_NN_TensorType nnTensorType)
150     {
151         if (values == nullptr) {
152             TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
153                 "[NNRT-OPBUILDER] The variable of values is nullptr when adding vector to operator.");
154             return kTfLiteError;
155         }
156         uint32_t numBits = 8;
157         double doubleScale = 0.f;
158         int32_t zeroPoint = 0;
159         OH_NN_QuantParam quantParam = {
160             .quantCount = 1,
161             .numBits = &numBits,
162             .scale = &doubleScale,
163             .zeroPoint = &zeroPoint
164         };
165 
166         OH_NN_Tensor tensor {
167             .dataType = nnType,
168             .dimensionCount = 1, // For 1-dim vector, dimensionCount is one.
169             .dimensions = &numValues,
170             .quantParam = &quantParam,
171             .type = nnTensorType,
172         };
173 
174         RETURN_TFLITE_ERROR_IF_NN_ERROR(
175             m_nnrt->OH_NNModel_AddTensor(m_nnModel, &tensor), "adding nnTensor");
176         const int32_t nnIndex = m_pTensorMapping->AddNewNonTensorTensor();
177         RETURN_TFLITE_ERROR_IF_NN_ERROR(
178             m_nnrt->OH_NNModel_SetTensorData(m_nnModel, nnIndex, values, sizeof(*(values)) * numValues),
179             "settings new nnTensor value");
180         m_augmentedParams.emplace_back(nnIndex);
181 
182         return kTfLiteOk;
183     }
184 
185     template<typename T>
AddActivateParamsInOperator(const NnrtOpMappingArgs & mappingArgs,T * builtinParams,int32_t builtinCode,OH_NN_TensorType nnTensorType)186     TfLiteStatus AddActivateParamsInOperator(const NnrtOpMappingArgs& mappingArgs, T* builtinParams,
187         int32_t builtinCode, OH_NN_TensorType nnTensorType)
188     {
189         if (builtinParams == nullptr) {
190             TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
191                 "[NNRT-OPBUILDER] The builtin params is nullptr when adding activate params to operator.");
192             return kTfLiteError;
193         }
194 
195         if ((builtinParams->activation >= 0) &&
196             (builtinParams->activation < ACTIVATE_FUSE_TYPE_LIST.size()) &&
197             (ACTIVATE_FUSE_TYPE_LIST[builtinParams->activation] != OH_NN_FUSE_UNSUPPORTED)) {
198             mappingArgs.builder->AddScalarInt8Tensor(ACTIVATE_FUSE_TYPE_LIST[builtinParams->activation], nnTensorType);
199         } else {
200             TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
201                 "[NNRT-OPBUILDER] unsupportted fused activation type %d for OpType %d.",
202                 builtinParams->activation, builtinCode);
203             return kTfLiteError;
204         }
205 
206         return kTfLiteOk;
207     }
208 
209     template<typename T>
AddPadParamsInOperator(const NnrtOpMappingArgs & mappingArgs,T * builtinParams,int32_t builtinCode,OH_NN_TensorType nnTensorType)210     TfLiteStatus AddPadParamsInOperator(const NnrtOpMappingArgs& mappingArgs, T* builtinParams, int32_t builtinCode,
211         OH_NN_TensorType nnTensorType)
212     {
213         if (builtinParams == nullptr) {
214             TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
215                 "[NNRT-OPBUILDER] The builtin params is nullptr when adding pad params to operator.");
216             return kTfLiteError;
217         }
218 
219         int32_t padding = 0;
220         if (builtinParams->padding == kTfLitePaddingUnknown) {
221             TFLITE_LOG_PROD(TFLITE_LOG_ERROR, "[NNRT-OPBUILDER] unknown padding mode for OpType %d.", builtinCode);
222             return kTfLiteError;
223         } else {
224             padding = (builtinParams->padding == kTfLitePaddingSame) ? PADDING_SAME : PADDING_VALID;
225         }
226         mappingArgs.builder->AddScalarInt8Tensor(padding, nnTensorType);
227 
228         return kTfLiteOk;
229     }
230 
231     // NNRT requires a bias tensor, so we allocate a new tensor to fill it with zeroes.
232     // It is deleted with other tensors in the context during subgraph destructor call.
233     TfLiteStatus AddZerosBias(const NnrtOpMappingArgs& mappingArgs, int32_t inputId, int32_t filterId,
234         int32_t channelNum);
235 
236     TfLiteStatus AddBasicComputeParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
237 
238     TfLiteStatus AddAvgPoolingParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
239 
240     TfLiteStatus AddMaxPoolingParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
241 
242     TfLiteStatus AddFullConnectedParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
243 
244     TfLiteStatus AddConv2DParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
245 
246     TfLiteStatus AddConcatenationParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
247 
248     TfLiteStatus AddSoftmaxParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
249 
250     TfLiteStatus AddQuantizeParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
251 
252     TfLiteStatus AddPackParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
253 
254     TfLiteStatus AddPadParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
255 
256     TfLiteStatus AddReduceMeanParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
257 
258     TfLiteStatus AddReshapeParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
259 
260     TfLiteStatus AddStridedSliceParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
261 
262     TfLiteStatus AddDepthwiseConv2DParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
263 
AddDefaultOpParams(const NnrtOpMappingArgs & mappingArgs,int32_t builtinCode)264     TfLiteStatus AddDefaultOpParams(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode)
265     {
266         return kTfLiteOk;
267     }
268 
269     // Adds a new NN API tensor that shadows the TF Lite tensor `tensorIndex`.
270     // This restores the NN API tensor index corresponding to the created tensor.
271     // If another caller previously created a NN API tensor for `tensorIndex`
272     // then the existing one is restored.
273     TfLiteStatus AddTensor(int32_t tensorIndex, int32_t builtinCode, std::vector<uint32_t>& indices,
274         int32_t tensorFlags = 0);
275 
276     // Adds a new NN API nnTensor to NNModel.
277     // If the builtinCode is kTfLiteBuiltinDepthwiseConv2d, the weight tensor will be transposed to CHWN format.
278     TfLiteStatus AddTensor(int32_t tensorIndex, int32_t builtinCode, int32_t tensorFlags, int32_t& nnTensorIndex);
279 
280     // Transpose dimension for Depth-wise Convolution Operator.
281     TfLiteStatus TransposeDepthwiseTensor(int32_t tensorIndex, OH_NN_Tensor& nnTensor, std::vector<int32_t>& destDims,
282         std::vector<int8_t>& tensorData);
283 
284     // Get NN nnTensor from tensor
285     TfLiteStatus ConstructNNTensor(int32_t tensorIndex, int32_t builtinCode, int32_t tensorFlags,
286         OH_NN_QuantParam& nnQuantParam, OH_NN_Tensor& nnTensor);
287 
288 private:
289     // Access to NNRT.
290     const NnrtApi* const m_nnrt;
291 
292     // TfLiteContext for error handling.
293     TfLiteContext* const m_context;
294 
295     // Indices of all inputs of tflite subgraph.
296     std::vector<int32_t> m_inputs;
297 
298     // Tracks relationship between indices.
299     TensorMapping* const m_pTensorMapping;
300 
301     // The NNRT model.
302     OH_NNModel* const m_nnModel;
303 
304     // Inputs and outputs for the current op. These are augmented in the sense
305     // that NN API uses nnTensors for all arguments, not just tensors, unlike
306     // TensorFlow Lite.
307     std::vector<uint32_t> m_augmentedInputs;
308     std::vector<uint32_t> m_augmentedParams;
309     std::vector<uint32_t> m_augmentedOutputs;
310 
311     // Whether to allow dynamic batch size without re-compilation.
312     bool m_allowDynamicDimensions;
313 
314     // the dynamic dimension information.
315     std::vector<int32_t> m_dimsUnspecified;
316 
317     // key builtInCode to OpFunc Map
318     using OpFuncPtr = TfLiteStatus(NnrtOpBuilder::*)(const NnrtOpMappingArgs& mappingArgs, int32_t builtinCode);
319     std::map<int32_t, OpFuncPtr> m_keyToOpFunc;
320 };
321 } // namespace nnrt
322 } // namespace delegate
323 } // namespace tflite
324 
325 #endif // TENSORFLOW_LITE_DELEGATES_NNRT_OP_BUILDER_H