Fix value class encoding in various corner cases (#2242)

- Value class is located at top-level, but wraps non-primitive and thus does not fall in 'primitive on top-level' branch
- Value class is a subclass in a polymorphic hierarchy, but either is primitive or explicitly recorded without type info

Note that type info is omitted in the latter case and 'can't add type info to primitive' error is not thrown deliberately, as
there seems to be use-cases for that.

Fixes #1774
Fixes #2159

6 files changed, 126 insertions, 9 deletions

diff --git a/core/api/kotlinx-serialization-core.api b/core/api/kotlinx-serialization-core.api
index 8948dbc6..e6d526ad 100644
--- a/core/api/kotlinx-serialization-core.api
+++ b/core/api/kotlinx-serialization-core.api
@@ -1067,7 +1067,7 @@ public abstract class kotlinx/serialization/internal/TaggedDecoder : kotlinx/ser
 	public final fun decodeEnum (Lkotlinx/serialization/descriptors/SerialDescriptor;)I
 	public final fun decodeFloat ()F
 	public final fun decodeFloatElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)F
-	public final fun decodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Decoder;
+	public fun decodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Decoder;
 	public final fun decodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Decoder;
 	public final fun decodeInt ()I
 	public final fun decodeIntElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)I
@@ -1123,7 +1123,7 @@ public abstract class kotlinx/serialization/internal/TaggedEncoder : kotlinx/ser
 	public final fun encodeEnum (Lkotlinx/serialization/descriptors/SerialDescriptor;I)V
 	public final fun encodeFloat (F)V
 	public final fun encodeFloatElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IF)V
-	public final fun encodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Encoder;
+	public fun encodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Encoder;
 	public final fun encodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Encoder;
 	public final fun encodeInt (I)V
 	public final fun encodeIntElement (Lkotlinx/serialization/descriptors/SerialDescriptor;II)V
diff --git a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt
index 8fc5f7cb..3af5d351 100644
--- a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt
+++ b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt
@@ -51,7 +51,7 @@ public abstract class TaggedEncoder<Tag : Any?> : Encoder, CompositeEncoder {
     protected open fun encodeTaggedInline(tag: Tag, inlineDescriptor: SerialDescriptor): Encoder =
         this.apply { pushTag(tag) }
 
-    final override fun encodeInline(descriptor: SerialDescriptor): Encoder =
+    override fun encodeInline(descriptor: SerialDescriptor): Encoder =
         encodeTaggedInline(popTag(), descriptor)
 
     // ---- Implementation of low-level API ----
@@ -209,7 +209,7 @@ public abstract class TaggedDecoder<Tag : Any?> : Decoder, CompositeDecoder {
 
     // ---- Implementation of low-level API ----
 
-    final override fun decodeInline(descriptor: SerialDescriptor): Decoder =
+    override fun decodeInline(descriptor: SerialDescriptor): Decoder =
         decodeTaggedInline(popTag(), descriptor)
 
     // TODO this method should be overridden by any sane format that supports top-level nulls
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/InlineClassesTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/InlineClassesTest.kt
index fdbb74af..0d30fc11 100644
--- a/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/InlineClassesTest.kt
+++ b/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/InlineClassesTest.kt
@@ -22,7 +22,7 @@ data class SimpleContainerForUInt(val i: UInt)
 @JvmInline
 value class MyUInt(val m: Int)
 
-object MyUIntSerializer: KSerializer<MyUInt> {
+object MyUIntSerializer : KSerializer<MyUInt> {
     override val descriptor = UInt.serializer().descriptor
     override fun serialize(encoder: Encoder, value: MyUInt) {
         encoder.encodeInline(descriptor).encodeInt(value.m)
@@ -73,13 +73,46 @@ value class ResourceKind(val kind: SampleEnum)
 @Serializable
 data class ResourceIdentifier(val id: ResourceId, val type: ResourceType, val type2: ValueWrapper)
 
-@Serializable @JvmInline
+@Serializable
+@JvmInline
 value class ValueWrapper(val wrapped: ResourceType)
 
+@Serializable
+@JvmInline
+value class Outer(val inner: Inner)
+
+@Serializable
+data class Inner(val n: Int)
+
+@Serializable
+data class OuterOuter(val outer: Outer)
+
+@Serializable
+@JvmInline
+value class WithList(val value: List<Int>)
+
 class InlineClassesTest : JsonTestBase() {
     private val precedent: UInt = Int.MAX_VALUE.toUInt() + 10.toUInt()
 
     @Test
+    fun withList() = noLegacyJs {
+        val withList = WithList(listOf(1, 2, 3))
+        assertJsonFormAndRestored(WithList.serializer(), withList, """[1,2,3]""")
+    }
+
+    @Test
+    fun testOuterInner() = noLegacyJs {
+        val o = Outer(Inner(10))
+        assertJsonFormAndRestored(Outer.serializer(), o, """{"n":10}""")
+    }
+
+    @Test
+    fun testOuterOuterInner() = noLegacyJs {
+        val o = OuterOuter(Outer(Inner(10)))
+        assertJsonFormAndRestored(OuterOuter.serializer(), o, """{"outer":{"n":10}}""")
+    }
+
+    @Test
     fun testTopLevel() = noLegacyJs {
         assertJsonFormAndRestored(
             ResourceType.serializer(),
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/ValueClassesInSealedHierarchyTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/ValueClassesInSealedHierarchyTest.kt
new file mode 100644
index 00000000..f3b482f1
--- /dev/null
+++ b/formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/ValueClassesInSealedHierarchyTest.kt
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.features.inline
+
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.test.*
+import kotlin.jvm.*
+import kotlin.test.*
+
+class ValueClassesInSealedHierarchyTest : JsonTestBase() {
+    @Test
+    fun testSingle() = noLegacyJs {
+        val single = "foo"
+        assertJsonFormAndRestored(
+            AnyValue.serializer(),
+            AnyValue.Single(single),
+            "\"$single\""
+        )
+    }
+
+    @Test
+    fun testComplex() = noLegacyJs {
+        val complexJson = """{"id":"1","name":"object"}"""
+        assertJsonFormAndRestored(
+            AnyValue.serializer(),
+            AnyValue.Complex(mapOf("id" to "1", "name" to "object")),
+            complexJson
+        )
+    }
+
+    @Test
+    fun testMulti() = noLegacyJs {
+        val multiJson = """["list","of","strings"]"""
+        assertJsonFormAndRestored(AnyValue.serializer(), AnyValue.Multi(listOf("list", "of", "strings")), multiJson)
+    }
+}
+
+
+// From https://github.com/Kotlin/kotlinx.serialization/issues/2159
+@Serializable(with = AnyValue.Companion.Serializer::class)
+sealed interface AnyValue {
+
+    @JvmInline
+    @Serializable
+    value class Single(val value: String) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Multi(val values: List<String>) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Complex(val values: Map<String, String>) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Unknown(val value: JsonElement) : AnyValue
+
+    companion object {
+        object Serializer : JsonContentPolymorphicSerializer<AnyValue>(AnyValue::class) {
+
+            override fun selectDeserializer(element: JsonElement): DeserializationStrategy<AnyValue> =
+                when {
+                    element is JsonArray && element.all { it is JsonPrimitive && it.isString } -> Multi.serializer()
+                    element is JsonObject && element.values.all { it is JsonPrimitive && it.isString } -> Complex.serializer()
+                    element is JsonPrimitive && element.isString -> Single.serializer()
+                    else -> Unknown.serializer()
+                }
+        }
+    }
+}
diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt
index 9e0cb2ad..2ffe8648 100644
--- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt
+++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt
@@ -165,9 +165,14 @@ private sealed class AbstractJsonTreeDecoder(
     override fun decodeTaggedInline(tag: String, inlineDescriptor: SerialDescriptor): Decoder =
         if (inlineDescriptor.isUnsignedNumber) JsonDecoderForUnsignedTypes(StringJsonLexer(getPrimitiveValue(tag).content), json)
         else super.decodeTaggedInline(tag, inlineDescriptor)
+
+    override fun decodeInline(descriptor: SerialDescriptor): Decoder {
+        return if (currentTagOrNull != null) super.decodeInline(descriptor)
+        else JsonPrimitiveDecoder(json, value).decodeInline(descriptor)
+    }
 }
 
-private class JsonPrimitiveDecoder(json: Json, override val value: JsonPrimitive) : AbstractJsonTreeDecoder(json, value) {
+private class JsonPrimitiveDecoder(json: Json, override val value: JsonElement) : AbstractJsonTreeDecoder(json, value) {
 
     init {
         pushTag(PRIMITIVE_TAG)
diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt
index d3c8a3d3..e42d4250 100644
--- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt
+++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt
@@ -25,7 +25,7 @@ public fun <T> Json.writeJson(value: T, serializer: SerializationStrategy<T>): J
 @ExperimentalSerializationApi
 private sealed class AbstractJsonTreeEncoder(
     final override val json: Json,
-    private val nodeConsumer: (JsonElement) -> Unit
+    protected val nodeConsumer: (JsonElement) -> Unit
 ) : NamedValueEncoder(), JsonEncoder {
 
     final override val serializersModule: SerializersModule
@@ -80,7 +80,6 @@ private sealed class AbstractJsonTreeEncoder(
             encodePolymorphically(serializer, value) { polymorphicDiscriminator = it }
         } else JsonPrimitiveEncoder(json, nodeConsumer).apply {
             encodeSerializableValue(serializer, value)
-            endEncode(serializer.descriptor)
         }
     }
 
@@ -112,6 +111,11 @@ private sealed class AbstractJsonTreeEncoder(
             else -> super.encodeTaggedInline(tag, inlineDescriptor)
         }
 
+    override fun encodeInline(descriptor: SerialDescriptor): Encoder {
+        return if (currentTagOrNull != null) super.encodeInline(descriptor)
+        else JsonPrimitiveEncoder(json, nodeConsumer).encodeInline(descriptor)
+    }
+
     @SuppressAnimalSniffer // Long(Integer).toUnsignedString(long)
     private fun inlineUnsignedNumberEncoder(tag: String) = object : AbstractEncoder() {
             override val serializersModule: SerializersModule = json.serializersModule
@@ -176,6 +180,7 @@ private class JsonPrimitiveEncoder(
         require(key === PRIMITIVE_TAG) { "This output can only consume primitives with '$PRIMITIVE_TAG' tag" }
         require(content == null) { "Primitive element was already recorded. Does call to .encodeXxx happen more than once?" }
         content = element
+        nodeConsumer(element)
     }
 
     override fun getCurrent(): JsonElement =