Vulkan: Add RefCountedEvent class and VkCmdSetEvent call

This CL defines RefCountedEvent class that adds reference counting to
VkEvent. CommandBufferHelper and ImageHelper each holds one reference
count to the event. Every time an event is added to the command buffer,
the corresponding RefCountedEvent will be added to the garbage list
which tracks the GPU completion using ResourceUse. That event garbage's
reference count will not decremented until GPU is finished, thus ensures
we never destroy a VkEvent until GPU is completed.

For images used by RenderPassCommands, As
RenderPassCommandBufferHelper::imageRead and imageWrite get called, an
event with that layout gets created and added to the image. That event
is saved in RenderPassCommandBufferHelper::mRefCountedEvents and that
VkCmdSetEvents calls are issued from
RenderPassCommandBufferHelper::flushToPrimary(). For renderPass
attachments, the events are created and added to image when attachment
image gets finalized.

For images used in OutsideRenderPassCommands, The events are inserted as
needed as we generates commands that uses image. We do not wait until
commands gets flushed to issue VkCmdSetEvent calls. A convenient
function trackImageWithEvent() is added to create and setEvent and add
event to image all in one call. You can add this call after the image
operation whenever we think it benefits, which gives us better control.

(Note: Even if forgot to insert the trackImageWithEvent call, it is
still okay since every time barrier is inserted, the event gets
released. Next time when we inserts barrier again we will fallback to
pipelineBarrier since there is no event associated with it. But that is
next CL's content).

This CL only adds the VkCmdSetEvent call when feature flag is enabled.
The feature flag is still disabled and no VkCmdWaitEvent is used in this
CL (will be added in later CL).

Bug: b/336844257
Change-Id: Iae5c4d2553a80f0f74cd6065d72a9c592c79f075
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/5490203
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
Reviewed-by: Geoff Lang <geofflang@chromium.org>
Commit-Queue: Charlie Lao <cclao@google.com>

14 files changed, 503 insertions, 24 deletions

diff --git a/src/libANGLE/renderer/vulkan/ContextVk.cpp b/src/libANGLE/renderer/vulkan/ContextVk.cpp
index 3ef52a6366..44b937d2b9 100644
--- a/src/libANGLE/renderer/vulkan/ContextVk.cpp
+++ b/src/libANGLE/renderer/vulkan/ContextVk.cpp
@@ -6624,6 +6624,8 @@ angle::Result ContextVk::dispatchCompute(const gl::Context *context,
     ANGLE_TRY(setupDispatch(context));
 
     mOutsideRenderPassCommands->getCommandBuffer().dispatch(numGroupsX, numGroupsY, numGroupsZ);
+    // Track completion of compute.
+    mOutsideRenderPassCommands->flushSetEvents(this);
 
     return angle::Result::Continue;
 }
@@ -6651,6 +6653,9 @@ angle::Result ContextVk::dispatchComputeIndirect(const gl::Context *context, GLi
     mOutsideRenderPassCommands->getCommandBuffer().dispatchIndirect(buffer.getBuffer(),
                                                                     buffer.getOffset() + indirect);
 
+    // Track completion of compute.
+    mOutsideRenderPassCommands->flushSetEvents(this);
+
     return angle::Result::Continue;
 }
 
diff --git a/src/libANGLE/renderer/vulkan/ContextVk.h b/src/libANGLE/renderer/vulkan/ContextVk.h
index b11ffb46ae..ed473f791e 100644
--- a/src/libANGLE/renderer/vulkan/ContextVk.h
+++ b/src/libANGLE/renderer/vulkan/ContextVk.h
@@ -588,6 +588,28 @@ class ContextVk : public ContextImpl, public vk::Context, public MultisampleText
         return angle::Result::Continue;
     }
 
+    void trackImageWithOutsideRenderPassEvent(vk::ImageHelper *image)
+    {
+        if (mRenderer->getFeatures().useVkEventForImageBarrier.enabled)
+        {
+            mOutsideRenderPassCommands->trackImageWithEvent(this, image);
+        }
+    }
+    void trackImagesWithOutsideRenderPassEvent(vk::ImageHelper *srcImage, vk::ImageHelper *dstImage)
+    {
+        if (mRenderer->getFeatures().useVkEventForImageBarrier.enabled)
+        {
+            mOutsideRenderPassCommands->trackImagesWithEvent(this, srcImage, dstImage);
+        }
+    }
+    void trackImagesWithOutsideRenderPassEvent(const vk::ImageHelperPtr *images, size_t count)
+    {
+        if (mRenderer->getFeatures().useVkEventForImageBarrier.enabled)
+        {
+            mOutsideRenderPassCommands->trackImagesWithEvent(this, images, count);
+        }
+    }
+
     angle::Result submitStagedTextureUpdates()
     {
         // Staged updates are recorded in outside RP cammand buffer, submit them.
diff --git a/src/libANGLE/renderer/vulkan/FramebufferVk.cpp b/src/libANGLE/renderer/vulkan/FramebufferVk.cpp
index 8c77f510b1..07554f8a7e 100644
--- a/src/libANGLE/renderer/vulkan/FramebufferVk.cpp
+++ b/src/libANGLE/renderer/vulkan/FramebufferVk.cpp
@@ -1386,13 +1386,20 @@ angle::Result FramebufferVk::blit(const gl::Context *context,
 
         if (canBlitWithCommand && areChannelsBlitCompatible && !reinterpretsColorspace)
         {
+            // Stash all images that involved with blit so that we can track them all at once.
+            angle::FixedVector<vk::ImageHelper *, 1 + gl::IMPLEMENTATION_MAX_DRAW_BUFFERS>
+                accessedImages;
+            accessedImages.push_back(&readRenderTarget->getImageForCopy());
             for (size_t colorIndexGL : mState.getEnabledDrawBuffers())
             {
                 RenderTargetVk *drawRenderTarget = mRenderTargetCache.getColors()[colorIndexGL];
                 ANGLE_TRY(blitWithCommand(contextVk, sourceArea, destArea, readRenderTarget,
                                           drawRenderTarget, filter, true, false, false, flipX,
                                           flipY));
+                accessedImages.push_back(&drawRenderTarget->getImageForWrite());
             }
+            contextVk->trackImagesWithOutsideRenderPassEvent(accessedImages.data(),
+                                                             accessedImages.size());
         }
         // If we're not flipping or rotating, use Vulkan's builtin resolve.
         else if (isColorResolve && !flipX && !flipY && areChannelsBlitCompatible &&
@@ -1500,6 +1507,8 @@ angle::Result FramebufferVk::blit(const gl::Context *context,
             ANGLE_TRY(blitWithCommand(contextVk, sourceArea, destArea, readRenderTarget,
                                       drawRenderTarget, filter, false, blitDepthBuffer,
                                       blitStencilBuffer, flipX, flipY));
+            contextVk->trackImagesWithOutsideRenderPassEvent(&readRenderTarget->getImageForCopy(),
+                                                             &drawRenderTarget->getImageForWrite());
         }
         else
         {
@@ -1843,6 +1852,9 @@ angle::Result FramebufferVk::generateFragmentShadingRateWithCPU(
     dataUpload->copyBufferToImage(buffer->getBuffer().getHandle(),
                                   mFragmentShadingRateImage.getImage(),
                                   VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
+
+    contextVk->trackImageWithOutsideRenderPassEvent(&mFragmentShadingRateImage);
+
     return angle::Result::Continue;
 }
 
@@ -1997,6 +2009,8 @@ angle::Result FramebufferVk::resolveColorWithCommand(ContextVk *contextVk,
     resolveRegion.extent.depth                  = 1;
 
     angle::VulkanPerfCounters &perfCounters = contextVk->getPerfCounters();
+    angle::FixedVector<vk::ImageHelperPtr, 1 + gl::IMPLEMENTATION_MAX_DRAW_BUFFERS> accessedImages;
+    accessedImages.push_back(srcImage);
     for (size_t colorIndexGL : mState.getEnabledDrawBuffers())
     {
         RenderTargetVk *drawRenderTarget = mRenderTargetCache.getColors()[colorIndexGL];
@@ -2009,7 +2023,9 @@ angle::Result FramebufferVk::resolveColorWithCommand(ContextVk *contextVk,
         srcImage->resolve(&dstImage, resolveRegion, commandBuffer);
 
         perfCounters.resolveImageCommands++;
+        accessedImages.push_back(&dstImage);
     }
+    contextVk->trackImagesWithOutsideRenderPassEvent(accessedImages.data(), accessedImages.size());
 
     return angle::Result::Continue;
 }
diff --git a/src/libANGLE/renderer/vulkan/SurfaceVk.cpp b/src/libANGLE/renderer/vulkan/SurfaceVk.cpp
index 7bac2b9e5d..326b8c73fe 100644
--- a/src/libANGLE/renderer/vulkan/SurfaceVk.cpp
+++ b/src/libANGLE/renderer/vulkan/SurfaceVk.cpp
@@ -2212,6 +2212,8 @@ angle::Result WindowSurfaceVk::prePresentSubmit(ContextVk *contextVk,
 
         mColorImageMS.resolve(image.image.get(), resolveRegion,
                               &commandBufferHelper->getCommandBuffer());
+
+        contextVk->trackImagesWithOutsideRenderPassEvent(&mColorImageMS, image.image.get());
         contextVk->getPerfCounters().swapchainResolveOutsideSubpass++;
     }
 
diff --git a/src/libANGLE/renderer/vulkan/TextureVk.cpp b/src/libANGLE/renderer/vulkan/TextureVk.cpp
index baa66a1410..f49b975d43 100644
--- a/src/libANGLE/renderer/vulkan/TextureVk.cpp
+++ b/src/libANGLE/renderer/vulkan/TextureVk.cpp
@@ -1449,6 +1449,8 @@ angle::Result TextureVk::copySubImageImplWithTransfer(ContextVk *contextVk,
 
         vk::ImageHelper::Copy(contextVk, srcImage, mImage, srcOffset, dstOffsetModified, extents,
                               srcSubresource, destSubresource, commandBuffer);
+
+        contextVk->trackImagesWithOutsideRenderPassEvent(srcImage, mImage);
     }
     else
     {
@@ -1483,6 +1485,8 @@ angle::Result TextureVk::copySubImageImplWithTransfer(ContextVk *contextVk,
         vk::ImageHelper::Copy(contextVk, srcImage, &stagingImage->get(), srcOffset, gl::kOffsetZero,
                               extents, srcSubresource, destSubresource, commandBuffer);
 
+        contextVk->trackImagesWithOutsideRenderPassEvent(srcImage, &stagingImage->get());
+
         // Stage the copy for when the image storage is actually created.
         VkImageType imageType = gl_vk::GetImageType(mState.getType());
         const gl::ImageIndex stagingIndex =
@@ -2209,6 +2213,8 @@ angle::Result TextureVk::copyBufferDataToImage(ContextVk *contextVk,
     commandBuffer->copyBufferToImage(srcBuffer->getBuffer().getHandle(), mImage->getImage(),
                                      mImage->getCurrentLayout(contextVk), 1, &region);
 
+    contextVk->trackImageWithOutsideRenderPassEvent(mImage);
+
     return angle::Result::Continue;
 }
 
@@ -2318,6 +2324,8 @@ angle::Result TextureVk::generateMipmapsWithCompute(ContextVk *contextVk)
         }
     }
 
+    contextVk->trackImageWithOutsideRenderPassEvent(mImage);
+
     return angle::Result::Continue;
 }
 
@@ -2519,6 +2527,8 @@ angle::Result TextureVk::copyAndStageImageData(ContextVk *contextVk,
                                  stagingImage->get().getCurrentLayout(contextVk), 1, &copyRegion);
     }
 
+    contextVk->trackImagesWithOutsideRenderPassEvent(srcImage, &stagingImage->get());
+
     // Stage the staging image in the destination
     dstImage->stageSubresourceUpdatesFromAllImageLevels(stagingImage.release(),
                                                         previousFirstAllocateLevel);
diff --git a/src/libANGLE/renderer/vulkan/vk_helpers.cpp b/src/libANGLE/renderer/vulkan/vk_helpers.cpp
index 1ad847e359..109bb099f8 100644
--- a/src/libANGLE/renderer/vulkan/vk_helpers.cpp
+++ b/src/libANGLE/renderer/vulkan/vk_helpers.cpp
@@ -1507,10 +1507,18 @@ void CommandBufferHelperCommon::initializeImpl()
     mCommandAllocator.init();
 }
 
-void CommandBufferHelperCommon::resetImpl()
+void CommandBufferHelperCommon::resetImpl(Context *context)
 {
     ASSERT(!mAcquireNextImageSemaphore.valid());
     mCommandAllocator.resetAllocator();
+
+    // Clean up event garbage. Note that ImageHelper object may still holding reference count to it,
+    // so the event itself will not gets destroyed until the last refCount goes away.
+    if (!mRefCountedEventGarbage.empty())
+    {
+        context->getRenderer()->collectRefCountedEventGarbage(mQueueSerial,
+                                                              std::move(mRefCountedEventGarbage));
+    }
 }
 
 template <class DerivedT>
@@ -1693,6 +1701,46 @@ void CommandBufferHelperCommon::updateImageLayoutAndBarrier(Context *context,
     }
 }
 
+void CommandBufferHelperCommon::retainImage(Context *context, ImageHelper *image)
+{
+    image->setQueueSerial(mQueueSerial);
+
+    if (context->getRenderer()->getFeatures().useVkEventForImageBarrier.enabled)
+    {
+        image->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    }
+}
+
+template <typename CommandBufferT>
+void CommandBufferHelperCommon::flushSetEventsImpl(Context *context, CommandBufferT *commandBuffer)
+{
+    if (mRefCountedEvents.mask.none())
+    {
+        return;
+    }
+
+    // Add VkCmdSetEvent here to track the completion of this renderPass.
+    for (ImageLayout layout : mRefCountedEvents.mask)
+    {
+        RefCountedEvent &refCountedEvent = mRefCountedEvents.map[layout];
+        ASSERT(refCountedEvent.valid());
+        const ImageMemoryBarrierData &layoutData =
+            kImageMemoryBarrierData[refCountedEvent.getImageLayout()];
+        commandBuffer->setEvent(refCountedEvent.getEvent().getHandle(),
+                                GetImageLayoutDstStageMask(context, layoutData));
+        // We no longer need event, so garbage collect it.
+        mRefCountedEventGarbage.add(&refCountedEvent);
+    }
+    mRefCountedEvents.mask.reset();
+}
+
+template void CommandBufferHelperCommon::flushSetEventsImpl<priv::SecondaryCommandBuffer>(
+    Context *context,
+    priv::SecondaryCommandBuffer *commandBuffer);
+template void CommandBufferHelperCommon::flushSetEventsImpl<VulkanSecondaryCommandBuffer>(
+    Context *context,
+    VulkanSecondaryCommandBuffer *commandBuffer);
+
 void CommandBufferHelperCommon::addCommandDiagnosticsCommon(std::ostringstream *out)
 {
     mPipelineBarriers.addDiagnosticsString(*out);
@@ -1723,7 +1771,7 @@ angle::Result OutsideRenderPassCommandBufferHelper::reset(
     Context *context,
     SecondaryCommandBufferCollector *commandBufferCollector)
 {
-    resetImpl();
+    resetImpl(context);
 
     // Collect/Reset the command buffer
     commandBufferCollector->collectCommandBuffer(std::move(mCommandBuffer));
@@ -1760,13 +1808,17 @@ void OutsideRenderPassCommandBufferHelper::imageRead(ContextVk *contextVk,
 {
     imageReadImpl(contextVk, aspectFlags, imageLayout, image);
 
-    if (!contextVk->isRenderPassStartedAndUsesImage(*image))
+    if (contextVk->isRenderPassStartedAndUsesImage(*image))
     {
         // Usually an image can only used by a RenderPassCommands or OutsideRenderPassCommands
         // because the layout will be different, except with image sampled from compute shader. In
         // this case, the renderPassCommands' read will override the outsideRenderPassCommands'
-        // read, since its queueSerial must be greater than outsideRP.
-        image->setQueueSerial(mQueueSerial);
+        // read, since its queueSerial must be greater than outsideRP. So dont update queueSerial
+        // here.
+    }
+    else
+    {
+        retainImage(contextVk, image);
     }
 }
 
@@ -1779,7 +1831,33 @@ void OutsideRenderPassCommandBufferHelper::imageWrite(ContextVk *contextVk,
                                                       ImageHelper *image)
 {
     imageWriteImpl(contextVk, level, layerStart, layerCount, aspectFlags, imageLayout, image);
-    image->setQueueSerial(mQueueSerial);
+    retainImage(contextVk, image);
+}
+
+void OutsideRenderPassCommandBufferHelper::trackImageWithEvent(Context *context, ImageHelper *image)
+{
+    image->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    flushSetEventsImpl(context, &mCommandBuffer);
+}
+
+void OutsideRenderPassCommandBufferHelper::trackImagesWithEvent(Context *context,
+                                                                ImageHelper *srcImage,
+                                                                ImageHelper *dstImage)
+{
+    srcImage->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    dstImage->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    flushSetEventsImpl(context, &mCommandBuffer);
+}
+
+void OutsideRenderPassCommandBufferHelper::trackImagesWithEvent(Context *context,
+                                                                const ImageHelperPtr *images,
+                                                                size_t count)
+{
+    for (size_t i = 0; i < count; i++)
+    {
+        images[i]->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    }
+    flushSetEventsImpl(context, &mCommandBuffer);
 }
 
 angle::Result OutsideRenderPassCommandBufferHelper::flushToPrimary(Context *context,
@@ -1805,6 +1883,9 @@ angle::Result OutsideRenderPassCommandBufferHelper::flushToPrimary(Context *cont
     ASSERT(mIsCommandBufferEnded);
     mCommandBuffer.executeCommands(&commandsState->primaryCommands);
 
+    // Call VkCmdSetEvent to track the completion of this renderPass.
+    flushSetEventsImpl(context, &commandsState->primaryCommands);
+
     // Restart the command buffer.
     return reset(context, &commandsState->secondaryCommands);
 }
@@ -1986,7 +2067,7 @@ angle::Result RenderPassCommandBufferHelper::reset(
     Context *context,
     SecondaryCommandBufferCollector *commandBufferCollector)
 {
-    resetImpl();
+    resetImpl(context);
 
     for (PackedAttachmentIndex index = kAttachmentIndexZero; index < mColorAttachmentsCount;
          ++index)
@@ -2039,7 +2120,7 @@ void RenderPassCommandBufferHelper::imageRead(ContextVk *contextVk,
     imageReadImpl(contextVk, aspectFlags, imageLayout, image);
     // As noted in the header we don't support multiple read layouts for Images.
     // We allow duplicate uses in the RP to accommodate for normal GL sampler usage.
-    image->setQueueSerial(mQueueSerial);
+    retainImage(contextVk, image);
 }
 
 void RenderPassCommandBufferHelper::imageWrite(ContextVk *contextVk,
@@ -2051,7 +2132,7 @@ void RenderPassCommandBufferHelper::imageWrite(ContextVk *contextVk,
                                                ImageHelper *image)
 {
     imageWriteImpl(contextVk, level, layerStart, layerCount, aspectFlags, imageLayout, image);
-    image->setQueueSerial(mQueueSerial);
+    retainImage(contextVk, image);
 }
 
 void RenderPassCommandBufferHelper::colorImagesDraw(gl::LevelIndex level,
@@ -2571,6 +2652,16 @@ void RenderPassCommandBufferHelper::finalizeDepthStencilImageLayoutAndLoadStore(
     mDepthAttachment.getImage()->resetRenderPassUsageFlags();
 }
 
+void RenderPassCommandBufferHelper::trackImagesWithEvent(Context *context,
+                                                         const ImageHelperPtr *images,
+                                                         size_t count)
+{
+    for (size_t i = 0; i < count; i++)
+    {
+        images[i]->setCurrentRefCountedEvent(context, mRefCountedEvents);
+    }
+}
+
 angle::Result RenderPassCommandBufferHelper::beginRenderPass(
     ContextVk *contextVk,
     RenderPassFramebuffer &&framebuffer,
@@ -2615,40 +2706,55 @@ angle::Result RenderPassCommandBufferHelper::endRenderPass(ContextVk *contextVk)
 {
     ANGLE_TRY(endRenderPassCommandBuffer(contextVk));
 
+    // *2 for resolve attachments
+    angle::FixedVector<ImageHelperPtr, gl::IMPLEMENTATION_MAX_FRAMEBUFFER_ATTACHMENTS << 1>
+        accessedImages;
     for (PackedAttachmentIndex index = kAttachmentIndexZero; index < mColorAttachmentsCount;
          ++index)
     {
         if (mColorAttachments[index].getImage() != nullptr)
         {
             finalizeColorImageLayoutAndLoadStore(contextVk, index);
+            accessedImages.push_back(mColorAttachments[index].getImage());
         }
         if (mColorResolveAttachments[index].getImage() != nullptr)
         {
             finalizeColorImageLayout(contextVk, mColorResolveAttachments[index].getImage(), index,
                                      true);
+            accessedImages.push_back(mColorResolveAttachments[index].getImage());
         }
     }
 
     if (mFragmentShadingRateAtachment.getImage() != nullptr)
     {
         finalizeFragmentShadingRateImageLayout(contextVk);
+        accessedImages.push_back(mFragmentShadingRateAtachment.getImage());
     }
 
-    if (mDepthStencilAttachmentIndex == kAttachmentIndexInvalid)
+    if (mDepthStencilAttachmentIndex != kAttachmentIndexInvalid)
     {
-        return angle::Result::Continue;
+        // Do depth stencil layout change and load store optimization.
+        ASSERT(mDepthAttachment.getImage() == mStencilAttachment.getImage());
+        ASSERT(mDepthResolveAttachment.getImage() == mStencilResolveAttachment.getImage());
+        if (mDepthAttachment.getImage() != nullptr)
+        {
+            finalizeDepthStencilImageLayoutAndLoadStore(contextVk);
+            accessedImages.push_back(mDepthAttachment.getImage());
+        }
+        if (mDepthResolveAttachment.getImage() != nullptr)
+        {
+            finalizeDepthStencilResolveImageLayout(contextVk);
+            accessedImages.push_back(mDepthResolveAttachment.getImage());
+        }
     }
 
-    // Do depth stencil layout change and load store optimization.
-    ASSERT(mDepthAttachment.getImage() == mStencilAttachment.getImage());
-    ASSERT(mDepthResolveAttachment.getImage() == mStencilResolveAttachment.getImage());
-    if (mDepthAttachment.getImage() != nullptr)
-    {
-        finalizeDepthStencilImageLayoutAndLoadStore(contextVk);
-    }
-    if (mDepthResolveAttachment.getImage() != nullptr)
+    if (contextVk->getRenderer()->getFeatures().useVkEventForImageBarrier.enabled)
     {
-        finalizeDepthStencilResolveImageLayout(contextVk);
+        // Even if there is no layout change, we always have to update event. In case of feedback
+        // loop, the sampler code should already set the event, which means we will be set it twice
+        // here. But since they uses the same layout and event is refCounted, it should work just
+        // fine.
+        trackImagesWithEvent(contextVk, accessedImages.data(), accessedImages.size());
     }
 
     return angle::Result::Continue;
@@ -2819,6 +2925,9 @@ angle::Result RenderPassCommandBufferHelper::flushToPrimary(Context *context,
     }
     primary.endRenderPass();
 
+    // Call VkCmdSetEvent to track the completion of this renderPass.
+    flushSetEventsImpl(context, &primary);
+
     // Restart the command buffer.
     return reset(context, &commandsState->secondaryCommands);
 }
@@ -6002,6 +6111,7 @@ void ImageHelper::releaseImage(Renderer *renderer)
         renderer->onMemoryDealloc(mMemoryAllocationType, mAllocationSize, mMemoryTypeIndex,
                                   mVmaAllocation.getHandle());
     }
+    mCurrentEvent.release(renderer->getDevice());
 
     renderer->collectGarbage(mUse, &mImage, &mDeviceMemory, &mVmaAllocation);
     mViewFormats.clear();
@@ -6477,6 +6587,7 @@ void ImageHelper::destroy(Renderer *renderer)
                                   mVmaAllocation.getHandle());
     }
 
+    mCurrentEvent.release(device);
     mImage.destroy(device);
     mDeviceMemory.destroy(device);
     mVmaAllocation.destroy(renderer->getAllocator());
@@ -7213,6 +7324,26 @@ void ImageHelper::updateLayoutAndBarrier(Context *context,
     *semaphoreOut = mAcquireNextImageSemaphore.release();
 }
 
+void ImageHelper::setCurrentRefCountedEvent(Context *context, ImageLayoutEventMaps &layoutEventMaps)
+{
+    ASSERT(context->getRenderer()->getFeatures().useVkEventForImageBarrier.enabled);
+
+    // Create the event if we have not yet so. Otherwise just use the already created event. This
+    // means all images used in the same render pass that has the same layout will be tracked by the
+    // same event.
+    if (!layoutEventMaps.map[mCurrentLayout].valid())
+    {
+        layoutEventMaps.map[mCurrentLayout].init(context, mCurrentLayout);
+        layoutEventMaps.mask.set(mCurrentLayout);
+    }
+
+    // If there is already an event, release it first.
+    mCurrentEvent.release(context->getDevice());
+    // Copy the event to mCurrentEvent so that we can wait for it in future. This will add extra
+    // refcount to the underlying VkEvent.
+    mCurrentEvent = layoutEventMaps.map[mCurrentLayout];
+}
+
 void ImageHelper::clearColor(Context *context,
                              const VkClearColorValue &color,
                              LevelIndex baseMipLevelVk,
@@ -7476,6 +7607,8 @@ angle::Result ImageHelper::CopyImageSubData(const gl::Context *context,
         ANGLE_VK_CHECK(contextVk, false, VK_ERROR_FEATURE_NOT_PRESENT);
     }
 
+    contextVk->trackImagesWithOutsideRenderPassEvent(srcImage, dstImage);
+
     return angle::Result::Continue;
 }
 
@@ -7587,6 +7720,8 @@ angle::Result ImageHelper::generateMipmapsWithBlit(ContextVk *contextVk,
     mCurrentShaderReadStageMask  = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
     mCurrentLayout               = ImageLayout::FragmentShaderReadOnly;
 
+    contextVk->trackImageWithOutsideRenderPassEvent(this);
+
     return angle::Result::Continue;
 }
 
@@ -9364,6 +9499,12 @@ angle::Result ImageHelper::flushStagedUpdatesImpl(ContextVk *contextVk,
         *levelUpdates = std::move(updatesToKeep);
     }
 
+    if (commandBuffer != nullptr)
+    {
+        // Track completion of this copy operation.
+        contextVk->trackImageWithOutsideRenderPassEvent(this);
+    }
+
     return angle::Result::Continue;
 }
 
@@ -9828,6 +9969,8 @@ angle::Result ImageHelper::copyImageDataToBuffer(ContextVk *contextVk,
 
     commandBuffer->copyImageToBuffer(mImage, getCurrentLayout(contextVk), bufferHandle, regionCount,
                                      &regions);
+    // Track completion of this copy.
+    contextVk->trackImageWithOutsideRenderPassEvent(this);
 
     return angle::Result::Continue;
 }
@@ -10446,12 +10589,15 @@ angle::Result ImageHelper::readPixelsImpl(ContextVk *contextVk,
 
             copyCommandBuffer->copyImageToBuffer(src->getImage(), src->getCurrentLayout(contextVk),
                                                  packBuffer.getBuffer().getHandle(), 1, &region);
+            contextVk->trackImageWithOutsideRenderPassEvent(this);
             return angle::Result::Continue;
         }
         if (canCopyWithComputeForReadPixels(packPixelsParams, readFormat, pixelsOffset))
         {
-            return readPixelsWithCompute(contextVk, src, packPixelsParams, srcOffset, srcExtent,
-                                         pixelsOffset, srcSubresource);
+            ANGLE_TRY(readPixelsWithCompute(contextVk, src, packPixelsParams, srcOffset, srcExtent,
+                                            pixelsOffset, srcSubresource));
+            contextVk->trackImageWithOutsideRenderPassEvent(this);
+            return angle::Result::Continue;
         }
     }
 
diff --git a/src/libANGLE/renderer/vulkan/vk_helpers.h b/src/libANGLE/renderer/vulkan/vk_helpers.h
index 1d0a79b7f9..f93c40a1a3 100644
--- a/src/libANGLE/renderer/vulkan/vk_helpers.h
+++ b/src/libANGLE/renderer/vulkan/vk_helpers.h
@@ -14,6 +14,7 @@
 #include "libANGLE/renderer/vulkan/Suballocation.h"
 #include "libANGLE/renderer/vulkan/vk_cache_utils.h"
 #include "libANGLE/renderer/vulkan/vk_format_utils.h"
+#include "libANGLE/renderer/vulkan/vk_ref_counted_event.h"
 
 #include <functional>
 
@@ -167,6 +168,14 @@ GLenum ConvertImageLayoutToGLImageLayout(ImageLayout imageLayout);
 
 VkImageLayout ConvertImageLayoutToVkImageLayout(Context *context, ImageLayout imageLayout);
 
+struct ImageLayoutEventMaps
+{
+    // The list of RefCountedEvents that have been tracked. The mask is used to accelerate the
+    // loop of map
+    angle::PackedEnumMap<ImageLayout, RefCountedEvent> map;
+    angle::PackedEnumBitSet<ImageLayout, uint64_t> mask;
+};
+
 // A dynamic buffer is conceptually an infinitely long buffer. Each time you write to the buffer,
 // you will always write to a previously unused portion. After a series of writes, you must flush
 // the buffer data to the device. Buffer lifetime currently assumes that each new allocation will
@@ -1123,6 +1132,7 @@ class PackedClearValuesArray final
 };
 
 class ImageHelper;
+using ImageHelperPtr = ImageHelper *;
 
 // Reference to a render pass attachment (color or depth/stencil) alongside render-pass-related
 // tracking such as when the attachment is last written to or invalidated.  This is used to
@@ -1322,6 +1332,14 @@ class CommandBufferHelperCommon : angle::NonCopyable
         writeResource->setWriteQueueSerial(mQueueSerial);
     }
 
+    // Update image with this command buffer's queueSerial. If VkEvent is enabled, image's current
+    // event is also updated with this command's event.
+    void retainImage(Context *context, ImageHelper *image);
+
+    // Issue VkCmdSetEvent call for events in this command buffer.
+    template <typename CommandBufferT>
+    void flushSetEventsImpl(Context *context, CommandBufferT *commandBuffer);
+
     const QueueSerial &getQueueSerial() const { return mQueueSerial; }
 
     void setAcquireNextImageSemaphore(VkSemaphore semaphore)
@@ -1340,7 +1358,7 @@ class CommandBufferHelperCommon : angle::NonCopyable
 
     void initializeImpl();
 
-    void resetImpl();
+    void resetImpl(Context *context);
 
     template <class DerivedT>
     angle::Result attachCommandPoolImpl(Context *context, SecondaryCommandPool *commandPool);
@@ -1413,6 +1431,11 @@ class CommandBufferHelperCommon : angle::NonCopyable
 
     // Only used for swapChain images
     Semaphore mAcquireNextImageSemaphore;
+
+    // The list of RefCountedEvents that have be tracked
+    ImageLayoutEventMaps mRefCountedEvents;
+    // The list of RefCountedEvents that should be garbage collected when it gets reset.
+    RefCountedEventGarbageObjects mRefCountedEventGarbage;
 };
 
 class SecondaryCommandBufferCollector;
@@ -1480,6 +1503,14 @@ class OutsideRenderPassCommandBufferHelper final : public CommandBufferHelperCom
                     ImageLayout imageLayout,
                     ImageHelper *image);
 
+    // Call SetEvent and have image's current event pointing to it.
+    void trackImageWithEvent(Context *context, ImageHelper *image);
+    void trackImagesWithEvent(Context *context, ImageHelper *srcImage, ImageHelper *dstImage);
+    void trackImagesWithEvent(Context *context, const ImageHelperPtr *images, size_t count);
+
+    // Issues VkCmdSetEvent calls.
+    void flushSetEvents(Context *context) { flushSetEventsImpl(context, &mCommandBuffer); }
+
     angle::Result flushToPrimary(Context *context, CommandsState *commandsState);
 
     void setGLMemoryBarrierIssued()
@@ -1864,6 +1895,8 @@ class RenderPassCommandBufferHelper final : public CommandBufferHelperCommon
     void finalizeDepthStencilImageLayoutAndLoadStore(Context *context);
     void finalizeFragmentShadingRateImageLayout(Context *context);
 
+    void trackImagesWithEvent(Context *context, const ImageHelperPtr *images, size_t count);
+
     // When using Vulkan secondary command buffers, each subpass must be recorded in a separate
     // command buffer.  Currently ANGLE produces render passes with at most 2 subpasses.
     static constexpr size_t kMaxSubpassCount = 2;
@@ -2684,6 +2717,9 @@ class ImageHelper final : public Resource, public angle::Subject
 
     size_t getLevelUpdateCount(gl::LevelIndex level) const;
 
+    // Create event if needed and record the event in ImageHelper::mCurrentEvent.
+    void setCurrentRefCountedEvent(Context *context, ImageLayoutEventMaps &layoutEventMaps);
+
   private:
     ANGLE_ENABLE_STRUCT_PADDING_WARNINGS
     struct ClearUpdate
@@ -3028,6 +3064,10 @@ class ImageHelper final : public Resource, public angle::Subject
     // The QueueSerial that associated with the last barrier.
     QueueSerial mBarrierQueueSerial;
 
+    // The current refCounted event. When barrier or layout change is needed, we should wait for
+    // this event.
+    RefCountedEvent mCurrentEvent;
+
     // Whether ANGLE currently has ownership of this resource or it's released to external.
     bool mIsReleasedToExternal;
 
diff --git a/src/libANGLE/renderer/vulkan/vk_ref_counted_event.cpp b/src/libANGLE/renderer/vulkan/vk_ref_counted_event.cpp
new file mode 100644
index 0000000000..22b5a81ec7
--- /dev/null
+++ b/src/libANGLE/renderer/vulkan/vk_ref_counted_event.cpp
@@ -0,0 +1,58 @@
+//
+// Copyright 2024 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// RefCountedEvent:
+//    Manages reference count of VkEvent and its associated functions.
+//
+
+#include "libANGLE/renderer/vulkan/vk_ref_counted_event.h"
+#include "libANGLE/renderer/vulkan/vk_helpers.h"
+#include "libANGLE/renderer/vulkan/vk_renderer.h"
+
+namespace rx
+{
+namespace vk
+{
+
+void ReleaseRefcountedEvent(VkDevice device, RefCountedEventAndLayoutHandle atomicRefCountedEvent)
+{
+    const bool isLastReference = atomicRefCountedEvent->getAndReleaseRef() == 1;
+    if (isLastReference)
+    {
+        atomicRefCountedEvent->get().event.destroy(device);
+        SafeDelete(atomicRefCountedEvent);
+    }
+}
+
+void RefCountedEvent::init(Context *context, ImageLayout layout)
+{
+    ASSERT(mHandle == nullptr);
+    ASSERT(layout != ImageLayout::Undefined);
+
+    mHandle                      = new AtomicRefCounted<EventAndLayout>;
+    VkEventCreateInfo createInfo = {};
+    createInfo.sType             = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO;
+    createInfo.flags             = 0;
+    mHandle->get().event.init(context->getDevice(), createInfo);
+    mHandle->addRef();
+    mHandle->get().imageLayout = layout;
+}
+
+// RefCountedEventGarbageObjects implementation
+void RefCountedEventGarbageObjects::add(RefCountedEvent *event)
+{
+    mGarbageObjects.emplace_back(GetGarbage(event));
+}
+
+void RefCountedEventGarbageObjects::add(std::vector<RefCountedEvent> *events)
+{
+    while (!events->empty())
+    {
+        mGarbageObjects.emplace_back(GetGarbage(&events->back()));
+        events->pop_back();
+    }
+}
+}  // namespace vk
+}  // namespace rx
diff --git a/src/libANGLE/renderer/vulkan/vk_ref_counted_event.h b/src/libANGLE/renderer/vulkan/vk_ref_counted_event.h
new file mode 100644
index 0000000000..1667dad0b8
--- /dev/null
+++ b/src/libANGLE/renderer/vulkan/vk_ref_counted_event.h
@@ -0,0 +1,163 @@
+//
+// Copyright 2024 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// RefCountedEvent:
+//    Manages reference count of VkEvent and its associated functions.
+//
+
+#ifndef LIBANGLE_RENDERER_VULKAN_REFCOUNTED_EVENT_H_
+#define LIBANGLE_RENDERER_VULKAN_REFCOUNTED_EVENT_H_
+
+#include <atomic>
+#include <limits>
+#include <queue>
+
+#include "common/PackedEnums.h"
+#include "common/debug.h"
+#include "libANGLE/renderer/serial_utils.h"
+#include "libANGLE/renderer/vulkan/vk_utils.h"
+#include "libANGLE/renderer/vulkan/vk_wrapper.h"
+
+namespace rx
+{
+namespace vk
+{
+enum class ImageLayout;
+
+// There are two ways to implement a barrier: Using VkCmdPipelineBarrier or VkCmdWaitEvents. The
+// BarrierType enum will be passed around to indicate which barrier caller want to use.
+enum class BarrierType
+{
+    Pipeline,
+    Event,
+};
+
+// VkCmdWaitEvents requires srcStageMask must be the bitwise OR of the stageMask parameter used in
+// previous calls to vkCmdSetEvent (See VUID-vkCmdWaitEvents-srcStageMask-01158). This mean we must
+// keep the record of what stageMask each event has been used in VkCmdSetEvent call so that we can
+// retrieve that information when we need to wait for the event. Instead of keeping just stageMask
+// here, we keep the ImageLayout for now which gives us more information for debugging.
+struct EventAndLayout
+{
+    bool valid() const { return event.valid(); }
+    Event event;
+    ImageLayout imageLayout;
+};
+
+// The VkCmdSetEvent is called after VkCmdEndRenderPass and all images that used at the given
+// pipeline stage (i.e, they have the same stageMask) will be tracked by the same event. This means
+// there will be multiple objects pointing to the same event. Events are thus reference counted so
+// that we do not destroy it while other objects still referencing to it.
+using RefCountedEventAndLayoutHandle = AtomicRefCounted<EventAndLayout> *;
+
+void ReleaseRefcountedEvent(VkDevice device, RefCountedEventAndLayoutHandle atomicRefCountedEvent);
+
+// Wrapper for RefCountedEventAndLayoutHandle.
+class RefCountedEvent final : public WrappedObject<RefCountedEvent, RefCountedEventAndLayoutHandle>
+{
+  public:
+    RefCountedEvent() = default;
+
+    // Move constructor moves reference of the underline object from other to this.
+    RefCountedEvent(RefCountedEvent &&other)
+    {
+        mHandle       = other.mHandle;
+        other.mHandle = nullptr;
+    }
+
+    // Copy constructor adds reference to the underline object.
+    RefCountedEvent(const RefCountedEvent &other)
+    {
+        mHandle = other.mHandle;
+        if (mHandle != nullptr)
+        {
+            mHandle->addRef();
+        }
+    }
+
+    // Move assignment moves reference of the underline object from other to this.
+    RefCountedEvent &operator=(RefCountedEvent &&other)
+    {
+        ASSERT(!valid());
+        std::swap(mHandle, other.mHandle);
+        return *this;
+    }
+
+    // Copy assignment adds reference to the underline object.
+    RefCountedEvent &operator=(const RefCountedEvent &other)
+    {
+        ASSERT(!valid());
+        ASSERT(other.valid());
+        mHandle = other.mHandle;
+        mHandle->addRef();
+        return *this;
+    }
+
+    // Returns true if both points to the same underline object.
+    bool operator==(const RefCountedEvent &other) const { return mHandle == other.mHandle; }
+
+    // Create VkEvent and associated it with given layout
+    void init(Context *context, ImageLayout layout);
+
+    // Release one reference count to the underline Event object and destroy if this is the
+    // very last reference.
+    void release(VkDevice device)
+    {
+        if (!valid())
+        {
+            return;
+        }
+        ReleaseRefcountedEvent(device, mHandle);
+        mHandle = nullptr;
+    }
+
+    bool valid() const { return mHandle != nullptr; }
+
+    // Returns the underlying Event object
+    const Event &getEvent() const
+    {
+        ASSERT(valid());
+        return mHandle->get().event;
+    }
+
+    // Returns the ImageLayout associated with the event.
+    ImageLayout getImageLayout() const
+    {
+        ASSERT(valid());
+        return mHandle->get().imageLayout;
+    }
+};
+
+template <>
+struct HandleTypeHelper<RefCountedEvent>
+{
+    constexpr static HandleType kHandleType = HandleType::RefCountedEvent;
+};
+
+// This class tracks a vector of RefcountedEvent garbage. For performance reason, instead of
+// individually tracking each VkEvent garbage, we collect all events that are accessed in the
+// CommandBufferHelper into this class. After we submit the command buffer, we treat this vector of
+// events as one garbage object and add it to renderer's garbage list. The garbage clean up will
+// decrement the refCount and destroy event only when last refCount goes away. Basically all GPU
+// usage will use one refCount and that refCount ensures we never destroy event until GPU is
+// finished.
+class RefCountedEventGarbageObjects final
+{
+  public:
+    // Move event to the garbage list
+    void add(RefCountedEvent *event);
+    // Move the vector of events to the garbage list
+    void add(std::vector<RefCountedEvent> *events);
+
+    bool empty() const { return mGarbageObjects.empty(); }
+
+    GarbageObjects &&release() { return std::move(mGarbageObjects); }
+
+  private:
+    GarbageObjects mGarbageObjects;
+};
+}  // namespace vk
+}  // namespace rx
+#endif  // LIBANGLE_RENDERER_VULKAN_REFCOUNTED_EVENT_H_
diff --git a/src/libANGLE/renderer/vulkan/vk_renderer.h b/src/libANGLE/renderer/vulkan/vk_renderer.h
index 306482126c..dde956b2a5 100644
--- a/src/libANGLE/renderer/vulkan/vk_renderer.h
+++ b/src/libANGLE/renderer/vulkan/vk_renderer.h
@@ -338,6 +338,14 @@ class Renderer : angle::NonCopyable
         mSuballocationGarbageList.add(this, std::move(garbage));
     }
 
+    void collectRefCountedEventGarbage(const QueueSerial &queueSerial,
+                                       vk::RefCountedEventGarbageObjects &&garbageObjets)
+    {
+        ASSERT(!garbageObjets.empty());
+        vk::SharedGarbage garbage(vk::ResourceUse(queueSerial), std::move(garbageObjets.release()));
+        mSharedGarbageList.add(this, std::move(garbage));
+    }
+
     angle::Result getPipelineCache(vk::Context *context, vk::PipelineCacheAccess *pipelineCacheOut);
     angle::Result mergeIntoPipelineCache(vk::Context *context,
                                          const vk::PipelineCache &pipelineCache);
diff --git a/src/libANGLE/renderer/vulkan/vk_utils.cpp b/src/libANGLE/renderer/vulkan/vk_utils.cpp
index c7fd784619..d49bdccf81 100644
--- a/src/libANGLE/renderer/vulkan/vk_utils.cpp
+++ b/src/libANGLE/renderer/vulkan/vk_utils.cpp
@@ -16,6 +16,7 @@
 #include "libANGLE/renderer/vulkan/DisplayVk.h"
 #include "libANGLE/renderer/vulkan/android/vk_android_utils.h"
 #include "libANGLE/renderer/vulkan/vk_mem_alloc_wrapper.h"
+#include "libANGLE/renderer/vulkan/vk_ref_counted_event.h"
 #include "libANGLE/renderer/vulkan/vk_renderer.h"
 #include "libANGLE/renderer/vulkan/vk_resource.h"
 
@@ -714,6 +715,9 @@ void GarbageObject::destroy(Renderer *renderer)
         case HandleType::PipelineLayout:
             vkDestroyPipelineLayout(device, (VkPipelineLayout)mHandle, nullptr);
             break;
+        case HandleType::RefCountedEvent:
+            ReleaseRefcountedEvent(device, (RefCountedEventAndLayoutHandle)mHandle);
+            break;
         case HandleType::RenderPass:
             vkDestroyRenderPass(device, (VkRenderPass)mHandle, nullptr);
             break;
diff --git a/src/libANGLE/renderer/vulkan/vk_utils.h b/src/libANGLE/renderer/vulkan/vk_utils.h
index a4e53d3f1b..1eec37a180 100644
--- a/src/libANGLE/renderer/vulkan/vk_utils.h
+++ b/src/libANGLE/renderer/vulkan/vk_utils.h
@@ -701,7 +701,9 @@ class AtomicRefCounted : angle::NonCopyable
     unsigned int getAndReleaseRef()
     {
         ASSERT(isReferenced());
-        return mRefCount.fetch_sub(1, std::memory_order_relaxed);
+        // This is used by RefCountedEvent which will decrement in clean up thread, so
+        // memory_order_acq_rel is needed.
+        return mRefCount.fetch_sub(1, std::memory_order_acq_rel);
     }
 
     bool isReferenced() const { return mRefCount.load(std::memory_order_relaxed) != 0; }
diff --git a/src/libANGLE/renderer/vulkan/vk_wrapper.h b/src/libANGLE/renderer/vulkan/vk_wrapper.h
index c0ab78d908..098ac432dd 100644
--- a/src/libANGLE/renderer/vulkan/vk_wrapper.h
+++ b/src/libANGLE/renderer/vulkan/vk_wrapper.h
@@ -61,6 +61,7 @@ enum class HandleType
 {
     Invalid,
     CommandBuffer,
+    RefCountedEvent,
     ANGLE_HANDLE_TYPES_X(ANGLE_COMMA_SEP_FUNC) EnumCount
 };
 
diff --git a/src/libANGLE/renderer/vulkan/vulkan_backend.gni b/src/libANGLE/renderer/vulkan/vulkan_backend.gni
index 225d31c7e7..89b80cfad1 100644
--- a/src/libANGLE/renderer/vulkan/vulkan_backend.gni
+++ b/src/libANGLE/renderer/vulkan/vulkan_backend.gni
@@ -98,6 +98,8 @@ vulkan_backend_sources = [
   "vk_internal_shaders_autogen.cpp",
   "vk_internal_shaders_autogen.h",
   "vk_mandatory_format_support_table_autogen.cpp",
+  "vk_ref_counted_event.cpp",
+  "vk_ref_counted_event.h",
   "vk_renderer.cpp",
   "vk_renderer.h",
   "vk_resource.cpp",