[AArch64] Add SVE2 implementation for I444ToARGBRow

Being able to use SVE2 functionality for these kernels has a number of
performance wins compared to the existing Neon code:

* For the Y component calculation we are able to use UMULH, versus the
  existing UMULL x2 + UZP2 sequence in Neon.

* For the RGBTORGBA8 calculation we are able to take advantage of
  interleaving narrowing instructions, allowing us to use ST2 rather
  than ST4 for the store. This is a big performance win on some
  micro-architectures where ST4 is costly.

* The use of predication means we do not need to add "any" kernels, we
  can simply rerun the calculation with a not-full predicate for the
  final iteration.

To avoid the overhead of generating a predicate register on every
iteration we duplicate the loop body and only generate a predicate on
the final iteration of the loop. This costs a small amount on the final
iteration but should still be significantly quicker than the overhead of
a function call needed by the "any" cases. Duplicating the loop body to
reduce the use of the WHILELT instruction improves little core
performance by ~12% by itself but has negligable impact on other
micro-architectures.

Reduction in runtime for the new SVE2 implementation compared to the
existing Neon implementation on selected micro-architectures:

Cortex-A510: -36.5%
Cortex-A720: -17.3%
  Cortex-X2: -11.3%

Bug: libyuv:973
Change-Id: I2a485f0dfa077a56f96b80a667ad38bbea47b4b4
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/5424739
Reviewed-by: Frank Barchard <fbarchard@chromium.org>

3 files changed, 113 insertions, 1 deletions

diff --git a/include/libyuv/row.h b/include/libyuv/row.h
index 6519fd80..6b672bfa 100644
--- a/include/libyuv/row.h
+++ b/include/libyuv/row.h
@@ -558,6 +558,11 @@ extern "C" {
 #define HAS_SOBELYROW_NEON
 #endif
 
+// The following are available on AArch64 SVE platforms:
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+#define HAS_I444TOARGBROW_SVE2
+#endif
+
 // The following are available on AArch64 platforms:
 #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
 #define HAS_GAUSSCOL_F32_NEON
@@ -1026,6 +1031,12 @@ void I444ToARGBRow_NEON(const uint8_t* src_y,
                         uint8_t* dst_argb,
                         const struct YuvConstants* yuvconstants,
                         int width);
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
+                        const uint8_t* src_u,
+                        const uint8_t* src_v,
+                        uint8_t* dst_argb,
+                        const struct YuvConstants* yuvconstants,
+                        int width);
 void I444ToRGB24Row_NEON(const uint8_t* src_y,
                          const uint8_t* src_u,
                          const uint8_t* src_v,
diff --git a/source/convert_argb.cc b/source/convert_argb.cc
index 3655e30a..81b774b5 100644
--- a/source/convert_argb.cc
+++ b/source/convert_argb.cc
@@ -609,6 +609,11 @@ int I444ToARGBMatrix(const uint8_t* src_y,
     }
   }
 #endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+  if (TestCpuFlag(kCpuHasSVE2)) {
+    I444ToARGBRow = I444ToARGBRow_SVE2;
+  }
+#endif
 #if defined(HAS_I444TOARGBROW_MSA)
   if (TestCpuFlag(kCpuHasMSA)) {
     I444ToARGBRow = I444ToARGBRow_Any_MSA;
@@ -6212,6 +6217,11 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y,
     }
   }
 #endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+  if (TestCpuFlag(kCpuHasSVE2)) {
+    I444ToARGBRow = I444ToARGBRow_SVE2;
+  }
+#endif
 #if defined(HAS_I444TOARGBROW_MSA)
   if (TestCpuFlag(kCpuHasMSA)) {
     I444ToARGBRow = I444ToARGBRow_Any_MSA;
@@ -6359,6 +6369,11 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y,
     }
   }
 #endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+  if (TestCpuFlag(kCpuHasSVE2)) {
+    I444ToARGBRow = I444ToARGBRow_SVE2;
+  }
+#endif
 #if defined(HAS_I444TOARGBROW_MSA)
   if (TestCpuFlag(kCpuHasMSA)) {
     I444ToARGBRow = I444ToARGBRow_Any_MSA;
diff --git a/source/row_sve.cc b/source/row_sve.cc
index 670403c4..54968ba4 100644
--- a/source/row_sve.cc
+++ b/source/row_sve.cc
@@ -17,7 +17,93 @@ extern "C" {
 
 #if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
 
-// TODO(libyuv:973) Port kernels to SVE/SVE2.
+#define READYUV444_SVE                           \
+  "ld1b       {z0.h}, p1/z, [%[src_y]]       \n" \
+  "ld1b       {z1.h}, p1/z, [%[src_u]]       \n" \
+  "ld1b       {z2.h}, p1/z, [%[src_v]]       \n" \
+  "add        %[src_y], %[src_y], %[vl]      \n" \
+  "add        %[src_u], %[src_u], %[vl]      \n" \
+  "add        %[src_v], %[src_v], %[vl]      \n" \
+  "prfm       pldl1keep, [%[src_y], 448]     \n" \
+  "prfm       pldl1keep, [%[src_u], 448]     \n" \
+  "trn1       z0.b, z0.b, z0.b               \n" \
+  "prfm       pldl1keep, [%[src_v], 448]     \n"
+
+#define YUVTORGB_SVE_SETUP                          \
+  "ld1rb  {z28.h}, p0/z, [%[kUVCoeff], #0]      \n" \
+  "ld1rb  {z29.h}, p0/z, [%[kUVCoeff], #1]      \n" \
+  "ld1rb  {z30.h}, p0/z, [%[kUVCoeff], #2]      \n" \
+  "ld1rb  {z31.h}, p0/z, [%[kUVCoeff], #3]      \n" \
+  "ld1rh  {z24.h}, p0/z, [%[kRGBCoeffBias], #0] \n" \
+  "ld1rh  {z25.h}, p0/z, [%[kRGBCoeffBias], #2] \n" \
+  "ld1rh  {z26.h}, p0/z, [%[kRGBCoeffBias], #4] \n" \
+  "ld1rh  {z27.h}, p0/z, [%[kRGBCoeffBias], #6] \n"
+
+#define I4XXTORGB_SVE                                     \
+  "umulh      z0.h, z24.h, z0.h              \n" /* Y */  \
+  "mul        z6.h, z30.h, z1.h              \n"          \
+  "mul        z4.h, z28.h, z1.h              \n" /* DB */ \
+  "mul        z5.h, z29.h, z2.h              \n" /* DR */ \
+  "mla        z6.h, p0/m, z31.h, z2.h        \n" /* DG */ \
+  "add        z17.h, z0.h, z26.h             \n" /* G */  \
+  "add        z16.h, z0.h, z4.h              \n" /* B */  \
+  "add        z18.h, z0.h, z5.h              \n" /* R */  \
+  "uqsub      z17.h, z17.h, z6.h             \n" /* G */  \
+  "uqsub      z16.h, z16.h, z25.h            \n" /* B */  \
+  "uqsub      z18.h, z18.h, z27.h            \n" /* R */
+
+// Convert from 2.14 fixed point RGB to 8 bit RGBA, interleaving as BG and RA
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBTORGBA8_SVE                  \
+  "uqshrnb     z16.b, z16.h, #6     \n" \
+  "uqshrnb     z18.b, z18.h, #6     \n" \
+  "uqshrnt     z16.b, z17.h, #6     \n" \
+  "trn1        z17.b, z18.b, z19.b  \n"
+
+#define YUVTORGB_SVE_REGS                                                     \
+  "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", \
+      "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "p0", "p1"
+
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
+                        const uint8_t* src_u,
+                        const uint8_t* src_v,
+                        uint8_t* dst_argb,
+                        const struct YuvConstants* yuvconstants,
+                        int width) {
+  uint64_t vl;
+  asm volatile(
+      "cnth     %[vl]                                   \n"
+      "ptrue    p0.b                                    \n" YUVTORGB_SVE_SETUP
+      "dup      z19.b, #255                             \n" /* A */
+      "cmp      %w[width], %w[vl]                       \n"
+      "b.le     2f                                      \n"
+
+      // Run bulk of computation with an all-true predicate to avoid predicate
+      // generation overhead.
+      "ptrue    p1.h                                    \n"
+      "1:                                               \n" READYUV444_SVE
+          I4XXTORGB_SVE RGBTORGBA8_SVE
+      "sub      %w[width], %w[width], %w[vl]            \n"
+      "st2h     {z16.h, z17.h}, p1, [%[dst_argb]]       \n"
+      "add      %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+      "cmp      %w[width], %w[vl]                       \n"
+      "b.gt     1b                                      \n"
+
+      // Calculate a predicate for the final iteration to deal with the tail.
+      "2:                                               \n"
+      "whilelt  p1.h, wzr, %w[width]                    \n" READYUV444_SVE
+          I4XXTORGB_SVE RGBTORGBA8_SVE
+      "st2h     {z16.h, z17.h}, p1, [%[dst_argb]]       \n"
+      : [src_y] "+r"(src_y),                               // %[src_y]
+        [src_u] "+r"(src_u),                               // %[src_u]
+        [src_v] "+r"(src_v),                               // %[src_v]
+        [dst_argb] "+r"(dst_argb),                         // %[dst_argb]
+        [width] "+r"(width),                               // %[width]
+        [vl] "=&r"(vl)                                     // %[vl]
+      : [kUVCoeff] "r"(&yuvconstants->kUVCoeff),           // %[kUVCoeff]
+        [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias)  // %[kRGBCoeffBias]
+      : "cc", "memory", YUVTORGB_SVE_REGS);
+}
 
 #endif  // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)