1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "render_staging.h"
17 
18 #include <algorithm>
19 #include <cinttypes>
20 #include <cstring>
21 
22 #include <render/datastore/intf_render_data_store_manager.h>
23 #include <render/device/intf_gpu_resource_manager.h>
24 #include <render/namespace.h>
25 #include <render/nodecontext/intf_node_context_pso_manager.h>
26 #include <render/nodecontext/intf_render_command_list.h>
27 #include <render/nodecontext/intf_render_node_context_manager.h>
28 
29 #include "datastore/render_data_store_default_staging.h"
30 #include "util/log.h"
31 
32 using namespace BASE_NS;
33 
34 RENDER_BEGIN_NAMESPACE()
35 namespace {
36 constexpr uint32_t MIP_MAP_TILING_FEATURES { CORE_FORMAT_FEATURE_BLIT_DST_BIT | CORE_FORMAT_FEATURE_BLIT_SRC_BIT };
37 
ExplicitBarrierUndefinedImageToTransferDst(IRenderCommandList & cmdList,const RenderHandle handle)38 void ExplicitBarrierUndefinedImageToTransferDst(IRenderCommandList& cmdList, const RenderHandle handle)
39 {
40     const ImageResourceBarrier src { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
41         PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
42         ImageLayout::CORE_IMAGE_LAYOUT_UNDEFINED }; // NOTE: undefined, because we do not care the previous data
43     const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
44         PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT, ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
45     const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0,
46         PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
47     cmdList.CustomImageBarrier(handle, src, dst, imageSubresourceRange);
48     cmdList.AddCustomBarrierPoint();
49 }
50 
ExplicitBarrierTransferDstImageToTransferSrc(IRenderCommandList & cmdList,const RenderHandle handle)51 void ExplicitBarrierTransferDstImageToTransferSrc(IRenderCommandList& cmdList, const RenderHandle handle)
52 {
53     const ImageResourceBarrier src { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
54         PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT, ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
55     const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
56         PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT, ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
57     const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0,
58         PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
59     cmdList.CustomImageBarrier(handle, src, dst, imageSubresourceRange);
60     cmdList.AddCustomBarrierPoint();
61 }
62 
BlitScalingImage(IRenderCommandList & cmdList,const BufferImageCopy & copy,const RenderHandle srcHandle,const RenderHandle dstHandle,const GpuImageDesc & dstImageDesc)63 void BlitScalingImage(IRenderCommandList& cmdList, const BufferImageCopy& copy, const RenderHandle srcHandle,
64     const RenderHandle dstHandle, const GpuImageDesc& dstImageDesc)
65 {
66     const ImageBlit imageBlit {
67         {
68             ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
69             0,
70             0,
71             1,
72         },
73         {
74             { 0, 0, 0 },
75             { copy.imageExtent.width, copy.imageExtent.height, 1 },
76         },
77         {
78             ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
79             0,
80             0,
81             1,
82         },
83         {
84             { 0, 0, 0 },
85             { dstImageDesc.width, dstImageDesc.height, 1 },
86         },
87     };
88     constexpr Filter filter = Filter::CORE_FILTER_LINEAR;
89     cmdList.BlitImage(srcHandle, dstHandle, imageBlit, filter);
90 }
91 
GetMipLevelSize(uint32_t originalSize,uint32_t mipLevel)92 uint32_t GetMipLevelSize(uint32_t originalSize, uint32_t mipLevel)
93 {
94     const uint32_t mipSize = originalSize >> mipLevel;
95     return mipSize >= 1u ? mipSize : 1u;
96 }
97 
GenerateMipmaps(IRenderCommandList & cmdList,const GpuImageDesc & imageDesc,const RenderHandle mipImageHandle,const uint32_t currentMipLevel)98 void GenerateMipmaps(IRenderCommandList& cmdList, const GpuImageDesc& imageDesc, const RenderHandle mipImageHandle,
99     const uint32_t currentMipLevel)
100 {
101     constexpr Filter filter = Filter::CORE_FILTER_LINEAR;
102     const uint32_t mipCount = imageDesc.mipCount;
103     const uint32_t layerCount = imageDesc.layerCount;
104 
105     const uint32_t iw = imageDesc.width;
106     const uint32_t ih = imageDesc.height;
107     for (uint32_t mipIdx = currentMipLevel + 1; mipIdx < mipCount; ++mipIdx) {
108         const uint32_t dstMipLevel = mipIdx;
109         const uint32_t srcMipLevel = dstMipLevel - 1;
110         // explicit transition for src mip
111         {
112             const ImageResourceBarrier src { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
113                 PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
114                 ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
115             const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
116                 PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
117                 ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
118             const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
119                 srcMipLevel, 1, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
120             cmdList.CustomImageBarrier(mipImageHandle, src, dst, imageSubresourceRange);
121             cmdList.AddCustomBarrierPoint();
122         }
123         {
124             const ImageBlit imageBlit {
125                 {
126                     ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
127                     srcMipLevel,
128                     0,
129                     layerCount,
130                 },
131                 {
132                     { 0, 0, 0 },
133                     { GetMipLevelSize(iw, srcMipLevel), GetMipLevelSize(ih, srcMipLevel), 1 },
134                 },
135                 {
136                     ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
137                     dstMipLevel,
138                     0,
139                     layerCount,
140                 },
141                 {
142                     { 0, 0, 0 },
143                     { GetMipLevelSize(iw, dstMipLevel), GetMipLevelSize(ih, dstMipLevel), 1 },
144                 },
145             };
146             cmdList.BlitImage(mipImageHandle, mipImageHandle, imageBlit, filter);
147 
148             // explicit "out" transition for src mip to dst to enable easy all mip transition in the end
149             {
150                 const ImageResourceBarrier src { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
151                     PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
152                     ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
153                 const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
154                     PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
155                     ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
156                 const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT,
157                     srcMipLevel, 1, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
158                 cmdList.CustomImageBarrier(mipImageHandle, src, dst, imageSubresourceRange);
159                 cmdList.AddCustomBarrierPoint();
160             }
161         }
162     }
163 }
164 
CopyBuffersToImages(const IRenderNodeGpuResourceManager & gpuResourceMgr,IRenderCommandList & cmdList,const vector<StagingCopyStruct> & bufferToImage,const vector<BufferImageCopy> & bufferImageCopies,const ScalingImageDataStruct & scalingImageData)165 void CopyBuffersToImages(const IRenderNodeGpuResourceManager& gpuResourceMgr, IRenderCommandList& cmdList,
166     const vector<StagingCopyStruct>& bufferToImage, const vector<BufferImageCopy>& bufferImageCopies,
167     const ScalingImageDataStruct& scalingImageData)
168 {
169     for (const auto& ref : bufferToImage) {
170         if (ref.invalidOperation) {
171             continue;
172         }
173         const uint32_t beginIndex = ref.beginIndex;
174         const uint32_t count = ref.count;
175         for (uint32_t idx = 0; idx < count; ++idx) {
176             const BufferImageCopy& copyRef = bufferImageCopies[beginIndex + idx];
177             // barriers are only done for dynamic resources automatically (e.g. when copying to same image multiple
178             // times)
179             // NOTE: add desc to data (so we do not need to lock gpu resource manager)
180             const GpuImageDesc imageDesc = gpuResourceMgr.GetImageDescriptor(ref.dstHandle.GetHandle());
181             RenderHandle bufferToImageDst = ref.dstHandle.GetHandle();
182             bool useScaling = false;
183             if (ref.format != Format::BASE_FORMAT_UNDEFINED) {
184                 if (const auto iter = scalingImageData.formatToScalingImages.find(ref.format);
185                     iter != scalingImageData.formatToScalingImages.cend()) {
186                     PLUGIN_ASSERT(iter->second < scalingImageData.scalingImages.size());
187                     bufferToImageDst = scalingImageData.scalingImages[iter->second].handle.GetHandle();
188                     ExplicitBarrierUndefinedImageToTransferDst(cmdList, bufferToImageDst);
189                     useScaling = true;
190                 }
191             }
192             cmdList.CopyBufferToImage(ref.srcHandle.GetHandle(), bufferToImageDst, copyRef);
193             if (useScaling) {
194                 ExplicitBarrierTransferDstImageToTransferSrc(cmdList, bufferToImageDst);
195                 BlitScalingImage(cmdList, copyRef, bufferToImageDst, ref.dstHandle.GetHandle(), imageDesc);
196             }
197 
198             // generate mips
199             if (imageDesc.engineCreationFlags & EngineImageCreationFlagBits::CORE_ENGINE_IMAGE_CREATION_GENERATE_MIPS) {
200                 const FormatProperties formatProperties = gpuResourceMgr.GetFormatProperties(imageDesc.format);
201                 if ((formatProperties.optimalTilingFeatures & MIP_MAP_TILING_FEATURES) == MIP_MAP_TILING_FEATURES) {
202                     const uint32_t mipCount = imageDesc.mipCount;
203                     const uint32_t availableMipCount = ref.count;
204                     const uint32_t currentMipLevel = copyRef.imageSubresource.mipLevel;
205 
206                     const bool isLastAvailableMipLevel = (currentMipLevel + 1) == availableMipCount;
207                     const bool isMissingMipLevels = availableMipCount < mipCount;
208 
209                     if (isMissingMipLevels && isLastAvailableMipLevel) {
210                         GenerateMipmaps(
211                             cmdList, imageDesc, ref.dstHandle.GetHandle(), copyRef.imageSubresource.mipLevel);
212                     }
213                 }
214 #if (RENDER_VALIDATION_ENABLED == 1)
215                 if ((formatProperties.optimalTilingFeatures & MIP_MAP_TILING_FEATURES) != MIP_MAP_TILING_FEATURES) {
216                     PLUGIN_LOG_ONCE_W("core_validation_auto_mipmapping",
217                         "RENDER_VALIDATION: requested automatic mip mapping not done for format: %u", imageDesc.format);
218                 }
219 #endif
220             }
221         }
222     }
223 }
224 
CopyImagesToBuffersImpl(const IRenderNodeGpuResourceManager & gpuResourceMgr,IRenderCommandList & cmdList,const vector<StagingCopyStruct> & imageToBuffer,const vector<BufferImageCopy> & bufferImageCopies)225 void CopyImagesToBuffersImpl(const IRenderNodeGpuResourceManager& gpuResourceMgr, IRenderCommandList& cmdList,
226     const vector<StagingCopyStruct>& imageToBuffer, const vector<BufferImageCopy>& bufferImageCopies)
227 {
228     for (const auto& ref : imageToBuffer) {
229         if (ref.invalidOperation) {
230             continue;
231         }
232         const uint32_t beginIndex = ref.beginIndex;
233         const uint32_t count = ref.count;
234         for (uint32_t idx = 0; idx < count; ++idx) {
235             const BufferImageCopy& copyRef = bufferImageCopies[beginIndex + idx];
236             cmdList.CopyImageToBuffer(ref.srcHandle.GetHandle(), ref.dstHandle.GetHandle(), copyRef);
237         }
238     }
239 }
240 
CopyImagesToImagesImpl(const IRenderNodeGpuResourceManager & gpuResourceMgr,IRenderCommandList & cmdList,const vector<StagingCopyStruct> & imageToImage,const vector<ImageCopy> & imageCopies)241 void CopyImagesToImagesImpl(const IRenderNodeGpuResourceManager& gpuResourceMgr, IRenderCommandList& cmdList,
242     const vector<StagingCopyStruct>& imageToImage, const vector<ImageCopy>& imageCopies)
243 {
244     for (const auto& ref : imageToImage) {
245         if (ref.invalidOperation) {
246             continue;
247         }
248         const uint32_t beginIndex = ref.beginIndex;
249         const uint32_t count = ref.count;
250         for (uint32_t idx = 0; idx < count; ++idx) {
251             const ImageCopy& copyRef = imageCopies[beginIndex + idx];
252             cmdList.CopyImageToImage(ref.srcHandle.GetHandle(), ref.dstHandle.GetHandle(), copyRef);
253         }
254     }
255 }
256 } // namespace
257 
Init(IRenderNodeContextManager & renderNodeContextMgr)258 void RenderStaging::Init(IRenderNodeContextManager& renderNodeContextMgr)
259 {
260     renderNodeContextMgr_ = &renderNodeContextMgr;
261 }
262 
PreExecuteFrame(const uint32_t clearByteSize)263 void RenderStaging::PreExecuteFrame(const uint32_t clearByteSize)
264 {
265     if (renderNodeContextMgr_->GetRenderNodeGraphData().renderingConfiguration.renderBackend ==
266         DeviceBackendType::OPENGLES) {
267         if (clearByteSize == 0) {
268             additionalCopyBuffer_ = {};
269         } else {
270             additionalCopyBuffer_.byteOffset = 0;
271             additionalCopyBuffer_.byteSize = clearByteSize;
272             const GpuBufferDesc desc {
273                 BufferUsageFlagBits::CORE_BUFFER_USAGE_TRANSFER_SRC_BIT, // usageFlags
274                 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT |
275                     MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT, // memoryPropertyFlags
276                 0,                                                                 // engineCreationFlags
277                 additionalCopyBuffer_.byteSize,                                    // byteSize
278                 BASE_NS::Format::BASE_FORMAT_UNDEFINED,                            // format
279             };
280 
281             additionalCopyBuffer_.handle =
282                 renderNodeContextMgr_->GetGpuResourceManager().Create(additionalCopyBuffer_.handle, desc);
283         }
284     }
285 }
286 
CopyHostToStaging(const IRenderNodeGpuResourceManager & gpuResourceMgr,const StagingConsumeStruct & stagingData)287 void RenderStaging::CopyHostToStaging(
288     const IRenderNodeGpuResourceManager& gpuResourceMgr, const StagingConsumeStruct& stagingData)
289 {
290     auto const copyUserDataToStagingBuffer = [](auto& gpuResourceMgr, auto const& ref) {
291         uint8_t* data = static_cast<uint8_t*>(gpuResourceMgr.MapBuffer(ref.srcHandle.GetHandle()));
292         if (!data) {
293             PLUGIN_LOG_E("staging: srcHandle %" PRIu64 " dstHandle %" PRIu64, ref.srcHandle.GetHandle().id,
294                 ref.dstHandle.GetHandle().id);
295             return;
296         }
297         auto const& bufferDesc = gpuResourceMgr.GetBufferDescriptor(ref.srcHandle.GetHandle());
298         const void* srcPtr = nullptr;
299         size_t srcSize = 0;
300         // should be removed already
301         PLUGIN_ASSERT(ref.dataType != StagingCopyStruct::DataType::DATA_TYPE_DIRECT_SRC_COPY);
302         if (ref.dataType == StagingCopyStruct::DataType::DATA_TYPE_VECTOR) {
303             srcPtr = ref.stagingData.data();
304             srcSize = ref.stagingData.size();
305         } else if (ref.dataType == StagingCopyStruct::DataType::DATA_TYPE_IMAGE_CONTAINER) {
306             PLUGIN_ASSERT(ref.imageContainerPtr);
307             if (ref.imageContainerPtr) {
308                 srcPtr = ref.imageContainerPtr->GetData().data();
309                 srcSize = ref.imageContainerPtr->GetData().size_bytes();
310             }
311         }
312         if ((srcPtr) && (srcSize > 0)) {
313             PLUGIN_ASSERT(bufferDesc.byteSize >= srcSize);
314             if (!CloneData(data, bufferDesc.byteSize, srcPtr, srcSize)) {
315                 PLUGIN_LOG_E("Copying of staging data failed");
316             }
317         }
318         gpuResourceMgr.UnmapBuffer(ref.srcHandle.GetHandle());
319     };
320 
321     for (const auto& ref : stagingData.bufferToImage) {
322         if ((!ref.invalidOperation) && (ref.dataType != StagingCopyStruct::DataType::DATA_TYPE_DIRECT_SRC_COPY)) {
323             copyUserDataToStagingBuffer(gpuResourceMgr, ref);
324         }
325     }
326     for (const auto& ref : stagingData.bufferToBuffer) {
327         if ((!ref.invalidOperation) && (ref.dataType != StagingCopyStruct::DataType::DATA_TYPE_DIRECT_SRC_COPY)) {
328             copyUserDataToStagingBuffer(gpuResourceMgr, ref);
329         }
330     }
331 }
332 
CopyStagingToImages(IRenderCommandList & cmdList,const IRenderNodeGpuResourceManager & gpuResourceMgr,const StagingConsumeStruct & stagingData,const StagingConsumeStruct & renderDataStoreStagingData)333 void RenderStaging::CopyStagingToImages(IRenderCommandList& cmdList,
334     const IRenderNodeGpuResourceManager& gpuResourceMgr, const StagingConsumeStruct& stagingData,
335     const StagingConsumeStruct& renderDataStoreStagingData)
336 {
337     // explicit input barriers
338     cmdList.BeginDisableAutomaticBarrierPoints();
339     {
340         ImageResourceBarrier src { 0, PipelineStageFlagBits::CORE_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT };
341         const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
342             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
343             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
344         const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
345             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
346 
347         for (const auto& ref : stagingData.bufferToImage) {
348             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
349                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), src, dst, imageSubresourceRange);
350             }
351         }
352         for (const auto& ref : renderDataStoreStagingData.bufferToImage) {
353             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
354                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), src, dst, imageSubresourceRange);
355             }
356         }
357     }
358     cmdList.EndDisableAutomaticBarrierPoints();
359     cmdList.AddCustomBarrierPoint();
360 
361     CopyBuffersToImages(gpuResourceMgr, cmdList, stagingData.bufferToImage, stagingData.bufferImageCopies,
362         stagingData.scalingImageData);
363     CopyBuffersToImages(gpuResourceMgr, cmdList, renderDataStoreStagingData.bufferToImage,
364         renderDataStoreStagingData.bufferImageCopies, {}); // scaling from render data store not supported ATM
365 
366     // explicit output barriers
367     cmdList.BeginDisableAutomaticBarrierPoints();
368     {
369         const ImageResourceBarrier src {
370             AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT, PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
371             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL // we expect all mips to be transferred to dst
372         };
373         const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_SHADER_READ_BIT,
374             PipelineStageFlagBits::CORE_PIPELINE_STAGE_VERTEX_SHADER_BIT, // some shader stage
375             ImageLayout::CORE_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL };
376         const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
377             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
378 
379         for (const auto& ref : stagingData.bufferToImage) {
380             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
381                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), src, dst, imageSubresourceRange);
382             }
383         }
384         for (const auto& ref : renderDataStoreStagingData.bufferToImage) {
385             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
386                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), src, dst, imageSubresourceRange);
387             }
388         }
389     }
390     cmdList.EndDisableAutomaticBarrierPoints();
391     cmdList.AddCustomBarrierPoint();
392 }
393 
CopyImagesToBuffers(IRenderCommandList & cmdList,const IRenderNodeGpuResourceManager & gpuResourceMgr,const StagingConsumeStruct & stagingData,const StagingConsumeStruct & renderDataStoreStagingData)394 void RenderStaging::CopyImagesToBuffers(IRenderCommandList& cmdList,
395     const IRenderNodeGpuResourceManager& gpuResourceMgr, const StagingConsumeStruct& stagingData,
396     const StagingConsumeStruct& renderDataStoreStagingData)
397 {
398     // explicit input barriers
399     cmdList.BeginDisableAutomaticBarrierPoints();
400     {
401         // we transfer all mip levels, but we only copy the one
402         ImageResourceBarrier src { 0, PipelineStageFlagBits::CORE_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT };
403         const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
404             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
405             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
406         const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
407             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
408 
409         for (const auto& ref : renderDataStoreStagingData.imageToBuffer) {
410             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.srcHandle.GetHandle()))) {
411                 cmdList.CustomImageBarrier(ref.srcHandle.GetHandle(), src, dst, imageSubresourceRange);
412             }
413         }
414     }
415     cmdList.EndDisableAutomaticBarrierPoints();
416     cmdList.AddCustomBarrierPoint();
417 
418     // Not supported through gpu resource manager staging
419     PLUGIN_ASSERT(stagingData.imageToBuffer.empty());
420     CopyImagesToBuffersImpl(gpuResourceMgr, cmdList, renderDataStoreStagingData.imageToBuffer,
421         renderDataStoreStagingData.bufferImageCopies);
422 
423     // explicit output barriers
424     cmdList.BeginDisableAutomaticBarrierPoints();
425     {
426         const ImageResourceBarrier src {
427             AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT, PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
428             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL // we expect all mips to be transferred to dst
429         };
430         // NOTE: should fetch usage flags
431         const ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_SHADER_READ_BIT,
432             PipelineStageFlagBits::CORE_PIPELINE_STAGE_VERTEX_SHADER_BIT, // some shader stage
433             ImageLayout::CORE_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL };
434         const ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
435             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
436 
437         for (const auto& ref : renderDataStoreStagingData.imageToBuffer) {
438             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.srcHandle.GetHandle()))) {
439                 cmdList.CustomImageBarrier(ref.srcHandle.GetHandle(), src, dst, imageSubresourceRange);
440             }
441         }
442     }
443     cmdList.EndDisableAutomaticBarrierPoints();
444     cmdList.AddCustomBarrierPoint();
445 }
446 
CopyImagesToImages(IRenderCommandList & cmdList,const IRenderNodeGpuResourceManager & gpuResourceMgr,const StagingConsumeStruct & stagingData,const StagingConsumeStruct & renderDataStoreStagingData)447 void RenderStaging::CopyImagesToImages(IRenderCommandList& cmdList, const IRenderNodeGpuResourceManager& gpuResourceMgr,
448     const StagingConsumeStruct& stagingData, const StagingConsumeStruct& renderDataStoreStagingData)
449 {
450     // explicit input barriers
451     cmdList.BeginDisableAutomaticBarrierPoints();
452     {
453         // we transfer all mip levels, but we only copy the one
454         constexpr ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
455             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
456 
457         constexpr ImageResourceBarrier src { 0, PipelineStageFlagBits::CORE_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT };
458         constexpr ImageResourceBarrier srcDst { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
459             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
460             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
461 
462         constexpr ImageResourceBarrier dstDst { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
463             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
464             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
465 
466         for (const auto& ref : renderDataStoreStagingData.imageToImage) {
467             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.srcHandle.GetHandle()))) {
468                 cmdList.CustomImageBarrier(ref.srcHandle.GetHandle(), src, srcDst, imageSubresourceRange);
469             }
470             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
471                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), src, dstDst, imageSubresourceRange);
472             }
473         }
474     }
475     cmdList.EndDisableAutomaticBarrierPoints();
476     cmdList.AddCustomBarrierPoint();
477 
478     // Not supported through gpu resource manager staging
479     PLUGIN_ASSERT(stagingData.imageToImage.empty());
480     CopyImagesToImagesImpl(
481         gpuResourceMgr, cmdList, renderDataStoreStagingData.imageToImage, renderDataStoreStagingData.imageCopies);
482 
483     // explicit output barriers
484     cmdList.BeginDisableAutomaticBarrierPoints();
485     {
486         constexpr ImageResourceBarrier srcSrc { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
487             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
488             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL };
489         constexpr ImageResourceBarrier dstSrc { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
490             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
491             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
492         // NOTE: should fetch usage flags
493         constexpr ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_SHADER_READ_BIT,
494             PipelineStageFlagBits::CORE_PIPELINE_STAGE_VERTEX_SHADER_BIT, // some shader stage
495             ImageLayout::CORE_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL };
496         constexpr ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
497             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
498 
499         for (const auto& ref : renderDataStoreStagingData.imageToImage) {
500             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.srcHandle.GetHandle()))) {
501                 cmdList.CustomImageBarrier(ref.srcHandle.GetHandle(), srcSrc, dst, imageSubresourceRange);
502             }
503             if ((!ref.invalidOperation) && (!RenderHandleUtil::IsDynamicResource(ref.dstHandle.GetHandle()))) {
504                 cmdList.CustomImageBarrier(ref.dstHandle.GetHandle(), dstSrc, dst, imageSubresourceRange);
505             }
506         }
507     }
508     cmdList.EndDisableAutomaticBarrierPoints();
509     cmdList.AddCustomBarrierPoint();
510 }
511 
CopyBuffersToBuffers(IRenderCommandList & cmdList,const StagingConsumeStruct & stagingData,const StagingConsumeStruct & renderDataStoreStagingData)512 void RenderStaging::CopyBuffersToBuffers(IRenderCommandList& cmdList, const StagingConsumeStruct& stagingData,
513     const StagingConsumeStruct& renderDataStoreStagingData)
514 {
515     const auto copyBuffersToBuffers = [](IRenderCommandList& cmdList, const vector<StagingCopyStruct>& bufferToBuffer,
516                                           const vector<BufferCopy>& bufferCopies) {
517         for (const auto& ref : bufferToBuffer) {
518             if (ref.invalidOperation) {
519                 continue;
520             }
521             const auto copies = array_view(bufferCopies.data() + ref.beginIndex, ref.count);
522             for (const BufferCopy& copyRef : copies) {
523                 cmdList.CopyBufferToBuffer(ref.srcHandle.GetHandle(), ref.dstHandle.GetHandle(), copyRef);
524             }
525         }
526     };
527 
528     // dynamic resources can create barriers if needed
529     copyBuffersToBuffers(cmdList, stagingData.bufferToBuffer, stagingData.bufferCopies);
530     copyBuffersToBuffers(cmdList, renderDataStoreStagingData.bufferToBuffer, renderDataStoreStagingData.bufferCopies);
531 
532     // explict output barriers
533     cmdList.BeginDisableAutomaticBarrierPoints();
534     {
535         const BufferResourceBarrier src { AccessFlagBits::CORE_ACCESS_TRANSFER_WRITE_BIT,
536             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT };
537         const BufferResourceBarrier dst { 0, // access flags are not currently known
538             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TOP_OF_PIPE_BIT };
539 
540         for (const auto& ref : stagingData.bufferToBuffer) {
541             if (!ref.invalidOperation) {
542                 cmdList.CustomBufferBarrier(
543                     ref.dstHandle.GetHandle(), src, dst, 0, PipelineStateConstants::GPU_BUFFER_WHOLE_SIZE);
544             }
545         }
546         for (const auto& ref : renderDataStoreStagingData.bufferToBuffer) {
547             if (!ref.invalidOperation) {
548                 cmdList.CustomBufferBarrier(
549                     ref.dstHandle.GetHandle(), src, dst, 0, PipelineStateConstants::GPU_BUFFER_WHOLE_SIZE);
550             }
551         }
552     }
553     cmdList.EndDisableAutomaticBarrierPoints();
554     cmdList.AddCustomBarrierPoint();
555 }
556 
ClearImages(IRenderCommandList & cmdList,const IRenderNodeGpuResourceManager & gpuResourceMgr,const StagingImageClearConsumeStruct & imageClearData)557 void RenderStaging::ClearImages(IRenderCommandList& cmdList, const IRenderNodeGpuResourceManager& gpuResourceMgr,
558     const StagingImageClearConsumeStruct& imageClearData)
559 {
560     // explicit input barriers for resources that are not dynamic trackable
561     // NOTE: one probably only needs to clear dynamic trackable resources anyhow
562     cmdList.BeginDisableAutomaticBarrierPoints();
563     {
564         // we transfer all mip levels
565         constexpr ImageSubresourceRange imageSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
566             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
567 
568         constexpr ImageResourceBarrier src { 0, PipelineStageFlagBits::CORE_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
569             ImageLayout::CORE_IMAGE_LAYOUT_UNDEFINED };
570         constexpr ImageResourceBarrier dst { AccessFlagBits::CORE_ACCESS_TRANSFER_READ_BIT,
571             PipelineStageFlagBits::CORE_PIPELINE_STAGE_TRANSFER_BIT,
572             ImageLayout::CORE_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL };
573 
574         for (const auto& ref : imageClearData.clears) {
575             if (!RenderHandleUtil::IsDynamicResource(ref.handle.GetHandle())) {
576                 cmdList.CustomImageBarrier(ref.handle.GetHandle(), src, dst, imageSubresourceRange);
577             }
578         }
579     }
580     cmdList.EndDisableAutomaticBarrierPoints();
581     cmdList.AddCustomBarrierPoint();
582 
583     // alternative path for GLES, ClearColorImage
584     if (renderNodeContextMgr_->GetRenderNodeGraphData().renderingConfiguration.renderBackend ==
585         DeviceBackendType::OPENGLES) {
586         if (additionalCopyBuffer_.handle && (!imageClearData.clears.empty())) {
587             if (auto dataPtr =
588                     reinterpret_cast<uint8_t*>(gpuResourceMgr.MapBuffer(additionalCopyBuffer_.handle.GetHandle()));
589                 dataPtr) {
590                 for (const auto& ref : imageClearData.clears) {
591                     const GpuImageDesc desc = gpuResourceMgr.GetImageDescriptor(ref.handle.GetHandle());
592                     const uint32_t bytesPerPixel =
593                         gpuResourceMgr.GetFormatProperties(ref.handle.GetHandle()).bytesPerPixel;
594                     const uint32_t imgByteSize = desc.width * desc.height * bytesPerPixel;
595                     ImageSubresourceLayers imageSubresourceLayers { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
596                         0, 1u };
597                     const BufferImageCopy bic {
598                         additionalCopyBuffer_.byteOffset, // bufferOffset
599                         0,                                // bufferRowLength
600                         0,                                // bufferImageHeight
601                         imageSubresourceLayers,           // imageSubresource
602                         { 0, 0, 0 },                      // imageOffset
603                         { desc.width, desc.height, 1u },  // imageExtent
604                     };
605 #if (RENDER_VALIDATION_ENABLED == 1)
606                     const bool hasDiffColorVals =
607                         ((ref.color.uint32[0] != ref.color.uint32[1]) || (ref.color.uint32[0] != ref.color.uint32[2]) ||
608                             (ref.color.uint32[0] != ref.color.uint32[3]));
609                     if ((bytesPerPixel > 4u) || hasDiffColorVals || (desc.depth > 1u)) {
610                         PLUGIN_LOG_ONCE_W("RenderStaging::ClearImages_gles",
611                             "RENDER_VALIDATION: only byte clears support with OpenGLES ClearColorImage");
612                     }
613 #endif
614                     if ((additionalCopyBuffer_.byteOffset + imgByteSize) <= additionalCopyBuffer_.byteSize) {
615                         ClearToValue(
616                             dataPtr, additionalCopyBuffer_.byteSize, uint8_t(0xff & ref.color.uint32[0]), imgByteSize);
617                         cmdList.CopyBufferToImage(
618                             additionalCopyBuffer_.handle.GetHandle(), ref.handle.GetHandle(), bic);
619                     }
620 
621                     // advance
622                     additionalCopyBuffer_.byteOffset += imgByteSize;
623                     dataPtr = dataPtr + additionalCopyBuffer_.byteOffset;
624                 }
625             }
626             gpuResourceMgr.UnmapBuffer(additionalCopyBuffer_.handle.GetHandle());
627         }
628     } else {
629         constexpr ImageSubresourceRange imgSubresourceRange { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0,
630             PipelineStateConstants::GPU_IMAGE_ALL_MIP_LEVELS, 0, PipelineStateConstants::GPU_IMAGE_ALL_LAYERS };
631         for (const auto& ref : imageClearData.clears) {
632             cmdList.ClearColorImage(ref.handle.GetHandle(), ref.color, { &imgSubresourceRange, 1 });
633         }
634         // NOTE: there's no way to guess the desired layout for non dynamic trackable clear resources
635     }
636 }
637 RENDER_END_NAMESPACE()
638