Vulkan:Migrate events and queries to secondary Cmd Buffer

Replace flushAndGetPrimaryCommandBuffer() function with
endRenderPassAndGetCommandBuffer() for events and queries. The end
result should be the same, but this allows a number of places that were
putting commands directly into the primary to put the commands into
ANGLE's custom SecondaryCommandBuffer (SCB) instead.

This also fixes a couple of minor bugs related to command buffer
ordering. flushAndBeginRenderPass() now flushes any outside
RenderPass (RP) commands first. Also, when insideRP commands are
flushed to the primary, set "mHasPrimaryCommands = true;"

Bug: b/153666475
Change-Id: I68413f25d27175afed0a20bc49f22f4c8d01e4fb
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2156932
Reviewed-by: Jamie Madill <[email protected]>
Reviewed-by: Charlie Lao <[email protected]>
Commit-Queue: Tobin Ehlis <[email protected]>

Author: tobine

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -4195,8 +4195,11 @@ angle::Result ContextVk::flushAndBeginRenderPass(
     const std::vector<VkClearValue> &clearValues,
     vk::CommandBuffer **commandBufferOut)
 {
-    vk::PrimaryCommandBuffer *primary;
-    ANGLE_TRY(flushAndGetPrimaryCommandBuffer(&primary));
+    // Flush any outside renderPass commands first
+    flushOutsideRenderPassCommands();
+    // Next end any currently outstanding renderPass
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
 
     gl::Rectangle rotatedRenderArea = renderArea;
     if (isRotatedAspectRatioForDrawFBO())
@@ -4237,6 +4240,7 @@ angle::Result ContextVk::endRenderPass()
         return angle::Result::Continue;
     }
 
+    ASSERT(mOutsideRenderPassCommands.empty());
     if (mActiveQueryAnySamples)
     {
         mActiveQueryAnySamples->getQueryHelper()->endOcclusionQuery(this, mRenderPassCommandBuffer);
@@ -4262,6 +4266,7 @@ angle::Result ContextVk::endRenderPass()
     mRenderPassCommands.pauseTransformFeedbackIfStarted();
 
     ANGLE_TRY(mRenderPassCommands.flushToPrimary(this, &mPrimaryCommands));
+    mHasPrimaryCommands = true;
 
     if (mGpuEventsEnabled)
     {
@@ -4608,8 +4613,7 @@ void RenderPassCommandBuffer::beginTransformFeedback(size_t validBufferCount,
 angle::Result RenderPassCommandBuffer::flushToPrimary(ContextVk *contextVk,
                                                       vk::PrimaryCommandBuffer *primary)
 {
-    if (empty())
-        return angle::Result::Continue;
+    ASSERT(!empty());
 
     if (kEnableCommandStreamDiagnostics)
     {

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -652,17 +652,6 @@ class ContextVk : public ContextImpl, public vk::Context
         return mRenderPassCommands;
     }
 
-    angle::Result flushAndGetPrimaryCommandBuffer(vk::PrimaryCommandBuffer **primaryCommands)
-    {
-        flushOutsideRenderPassCommands();
-        ANGLE_TRY(endRenderPass());
-        *primaryCommands = &mPrimaryCommands;
-
-        // We assume any calling code is going to record primary commands.
-        mHasPrimaryCommands = true;
-        return angle::Result::Continue;
-    }
-
     egl::ContextPriority getContextPriority() const override { return mContextPriority; }
     angle::Result startRenderPass(gl::Rectangle renderArea);
     angle::Result endRenderPass();
@@ -770,6 +759,17 @@ class ContextVk : public ContextImpl, public vk::Context
         double cpuTimestampS;
     };
 
+    angle::Result flushAndGetPrimaryCommandBuffer(vk::PrimaryCommandBuffer **primaryCommands)
+    {
+        flushOutsideRenderPassCommands();
+        ANGLE_TRY(endRenderPass());
+        *primaryCommands = &mPrimaryCommands;
+
+        // We assume any calling code is going to record primary commands.
+        mHasPrimaryCommands = true;
+        return angle::Result::Continue;
+    }
+
     angle::Result setupDraw(const gl::Context *context,
                             gl::PrimitiveMode mode,
                             GLint firstVertexOrInvalid,

src/libANGLE/renderer/vulkan/SyncVk.cpp

@@ -46,9 +46,10 @@ angle::Result SyncHelper::initialize(ContextVk *contextVk)
 
     mEvent = event.release();
 
-    vk::PrimaryCommandBuffer *primary;
-    ANGLE_TRY(contextVk->flushAndGetPrimaryCommandBuffer(&primary));
-    primary->setEvent(mEvent.getHandle(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(contextVk->endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
+    outsideRenderPassCommandBuffer->setEvent(mEvent.getHandle(),
+                                             VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
     retain(&contextVk->getResourceUseList());
 
     return angle::Result::Continue;
@@ -100,10 +101,11 @@ angle::Result SyncHelper::clientWait(Context *context,
 
 angle::Result SyncHelper::serverWait(ContextVk *contextVk)
 {
-    vk::PrimaryCommandBuffer *primary;
-    ANGLE_TRY(contextVk->flushAndGetPrimaryCommandBuffer(&primary));
-    primary->waitEvents(1, mEvent.ptr(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
-                        VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, nullptr, 0, nullptr, 0, nullptr);
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(contextVk->endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
+    outsideRenderPassCommandBuffer->waitEvents(
+        1, mEvent.ptr(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
+        0, nullptr, 0, nullptr, 0, nullptr);
     retain(&contextVk->getResourceUseList());
     return angle::Result::Continue;
 }

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -1172,20 +1172,20 @@ void QueryHelper::deinit()
 
 angle::Result QueryHelper::beginQuery(ContextVk *contextVk)
 {
-    vk::PrimaryCommandBuffer *primaryCommands;
-    ANGLE_TRY(contextVk->flushAndGetPrimaryCommandBuffer(&primaryCommands));
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(contextVk->endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
     const QueryPool &queryPool = getQueryPool();
-    primaryCommands->resetQueryPool(queryPool, mQuery, 1);
-    primaryCommands->beginQuery(queryPool, mQuery, 0);
+    outsideRenderPassCommandBuffer->resetQueryPool(queryPool.getHandle(), mQuery, 1);
+    outsideRenderPassCommandBuffer->beginQuery(queryPool.getHandle(), mQuery, 0);
     mMostRecentSerial = contextVk->getCurrentQueueSerial();
     return angle::Result::Continue;
 }
 
 angle::Result QueryHelper::endQuery(ContextVk *contextVk)
 {
-    vk::PrimaryCommandBuffer *primaryCommands;
-    ANGLE_TRY(contextVk->flushAndGetPrimaryCommandBuffer(&primaryCommands));
-    primaryCommands->endQuery(getQueryPool(), mQuery);
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(contextVk->endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
+    outsideRenderPassCommandBuffer->endQuery(getQueryPool().getHandle(), mQuery);
     mMostRecentSerial = contextVk->getCurrentQueueSerial();
     return angle::Result::Continue;
 }
@@ -1209,9 +1209,9 @@ void QueryHelper::endOcclusionQuery(ContextVk *contextVk, CommandBuffer *renderP
 
 angle::Result QueryHelper::flushAndWriteTimestamp(ContextVk *contextVk)
 {
-    vk::PrimaryCommandBuffer *primary;
-    ANGLE_TRY(contextVk->flushAndGetPrimaryCommandBuffer(&primary));
-    writeTimestamp(contextVk, primary);
+    vk::CommandBuffer *outsideRenderPassCommandBuffer;
+    ANGLE_TRY(contextVk->endRenderPassAndGetCommandBuffer(&outsideRenderPassCommandBuffer));
+    writeTimestamp(contextVk, outsideRenderPassCommandBuffer);
     return angle::Result::Continue;
 }
 
@@ -1223,6 +1223,15 @@ void QueryHelper::writeTimestamp(ContextVk *contextVk, PrimaryCommandBuffer *pri
     mMostRecentSerial = contextVk->getCurrentQueueSerial();
 }
 
+void QueryHelper::writeTimestamp(ContextVk *contextVk, CommandBuffer *commandBuffer)
+{
+    const QueryPool &queryPool = getQueryPool();
+    commandBuffer->resetQueryPool(queryPool.getHandle(), mQuery, 1);
+    commandBuffer->writeTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, queryPool.getHandle(),
+                                  mQuery);
+    mMostRecentSerial = contextVk->getCurrentQueueSerial();
+}
+
 bool QueryHelper::hasPendingWork(ContextVk *contextVk)
 {
     // If the renderer has a queue serial higher than the stored one, the command buffers that

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -394,7 +394,10 @@ class QueryHelper final
     void endOcclusionQuery(ContextVk *contextVk, CommandBuffer *renderPassCommandBuffer);
 
     angle::Result flushAndWriteTimestamp(ContextVk *contextVk);
+    // When syncing gpu/cpu time, main thread accesses primary directly
     void writeTimestamp(ContextVk *contextVk, PrimaryCommandBuffer *primary);
+    // All other timestamp accesses should be made on outsideRenderPassCommandBuffer
+    void writeTimestamp(ContextVk *contextVk, CommandBuffer *outsideRenderPassCommandBuffer);
 
     Serial getStoredQueueSerial() { return mMostRecentSerial; }
     bool hasPendingWork(ContextVk *contextVk);