-
Notifications
You must be signed in to change notification settings - Fork 645
Support preserving unknown fields in ProtoBuf format #2860
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
xiaozhikang0916
wants to merge
6
commits into
Kotlin:dev
Choose a base branch
from
xiaozhikang0916:feature/proto-unknown-fields
base: dev
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+1,149
−250
Open
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
1785c51
introduce ProtoUnknownFields
xiaozhikang0916 7bdceee
add some test
xiaozhikang0916 cfcdddb
comments
xiaozhikang0916 310c9c4
docs
xiaozhikang0916 e95931b
api dump
xiaozhikang0916 06df0a5
add unit test for unknown one-of field
xiaozhikang0916 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
199 changes: 199 additions & 0 deletions
199
formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoMessage.kt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,199 @@ | ||
| /* | ||
| * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
| */ | ||
|
|
||
| package kotlinx.serialization.protobuf | ||
|
|
||
| import kotlinx.serialization.* | ||
| import kotlinx.serialization.protobuf.internal.* | ||
|
|
||
| /** | ||
| * Represents a protobuf message. | ||
| * | ||
| * Especially used as a holder of unknown proto fields in an arbitrary protobuf message. | ||
| */ | ||
| @Serializable(with = ProtoMessageSerializer::class) | ||
| public class ProtoMessage internal constructor( | ||
| internal val fields: List<ProtoField> | ||
| ) { | ||
| public companion object { | ||
| /** | ||
| * An empty [ProtoMessage] instance. | ||
| * | ||
| * Useful as a default value for [ProtoUnknownFields] properties. | ||
| */ | ||
| public val Empty: ProtoMessage = ProtoMessage(emptyList()) | ||
| } | ||
|
|
||
| /** | ||
| * Number of fields holding in the message. | ||
| */ | ||
| public val size: Int get() = fields.size | ||
|
|
||
| /** | ||
| * Returns a byte array representing of the message. | ||
| */ | ||
| public fun asByteArray(): ByteArray = | ||
| fields.fold(ByteArray(0)) { acc, protoField -> acc + protoField.asWireContent() } | ||
|
|
||
| internal constructor(vararg fields: ProtoField) : this(fields.toList()) | ||
|
|
||
| /** | ||
| * Merges two [ProtoMessage] instances. | ||
| */ | ||
| public operator fun plus(other: ProtoMessage): ProtoMessage = merge(other) | ||
|
|
||
| /** | ||
| * Merges two [ProtoMessage] instances. | ||
| */ | ||
| public fun merge(other: ProtoMessage): ProtoMessage { | ||
| return ProtoMessage(fields + other.fields) | ||
| } | ||
|
|
||
| /** | ||
| * Convenience method to merge multiple [ProtoField] with this message. | ||
| */ | ||
| internal fun merge(vararg field: ProtoField): ProtoMessage { | ||
| return ProtoMessage(fields + field) | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| return fields.hashCode() | ||
| } | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| if (this === other) return true | ||
| if (other == null || this::class != other::class) return false | ||
|
|
||
| other as ProtoMessage | ||
|
|
||
| return fields == other.fields | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Convenience method to merge two nullable [ProtoMessage] instances. | ||
| */ | ||
| public fun ProtoMessage?.merge(other: ProtoMessage?): ProtoMessage { | ||
| return when { | ||
| this == null -> other ?: ProtoMessage.Empty | ||
| other == null -> this | ||
| else -> this + other | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Convenience method to merge multiple [ProtoField] with a nullable [ProtoMessage]. | ||
| */ | ||
| internal fun ProtoMessage?.merge(vararg fields: ProtoField): ProtoMessage { | ||
| return when { | ||
| this == null -> ProtoMessage(fields.toList()) | ||
| else -> this.merge(ProtoMessage(fields.toList())) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Represents a single field in a protobuf message. | ||
| */ | ||
| @Serializable(with = ProtoFieldSerializer::class) | ||
| @ConsistentCopyVisibility | ||
| internal data class ProtoField internal constructor( | ||
| internal val id: Int, | ||
| internal val wireType: ProtoWireType, | ||
| internal val data: ProtoContentHolder | ||
| ) { | ||
| companion object { | ||
| val Empty: ProtoField = ProtoField(0, ProtoWireType.INVALID, ProtoContentHolder.ByteArrayContent(ByteArray(0))) | ||
| } | ||
|
|
||
| fun asWireContent(): ByteArray = byteArrayOf(((id shl 3) or wireType.typeId).toByte()) + data.byteArray | ||
|
|
||
| val contentLength: Int | ||
| get() = asWireContent().size | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| if (this === other) return true | ||
| if (other == null || this::class != other::class) return false | ||
|
|
||
| other as ProtoField | ||
|
|
||
| if (id != other.id) return false | ||
| if (wireType != other.wireType) return false | ||
| if (!data.contentEquals(other.data)) return false | ||
|
|
||
| return true | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| var result = id | ||
| result = 31 * result + wireType.hashCode() | ||
| result = 31 * result + data.contentHashCode() | ||
| return result | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * A data representation of a protobuf field in [ProtoField.data], without the field number and wire type. | ||
| */ | ||
| internal sealed interface ProtoContentHolder { | ||
|
|
||
| /** | ||
| * Returns a byte array representation of the content. | ||
| */ | ||
| val byteArray: ByteArray | ||
|
|
||
| /** | ||
| * Represents the content in raw byte array. | ||
| */ | ||
| data class ByteArrayContent(override val byteArray: ByteArray) : ProtoContentHolder { | ||
| override fun equals(other: Any?): Boolean { | ||
| return other is ProtoContentHolder && this.contentEquals(other) | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| return this.contentHashCode() | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Represents the content with a nested [ProtoMessage]. | ||
| */ | ||
| data class MessageContent(val content: ProtoMessage) : ProtoContentHolder { | ||
| override val byteArray: ByteArray | ||
| get() = content.asByteArray() | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| return other is ProtoContentHolder && this.contentEquals(other) | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| return this.contentHashCode() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Creates a [ProtoContentHolder] instance with a byte array content. | ||
| */ | ||
| internal fun ProtoContentHolder(content: ByteArray): ProtoContentHolder = ProtoContentHolder.ByteArrayContent(content) | ||
|
|
||
| /** | ||
| * Get the length in bytes of the content in the [ProtoContentHolder]. | ||
| */ | ||
| internal val ProtoContentHolder.contentLength: Int | ||
| get() = byteArray.size | ||
|
|
||
| /** | ||
| * Checks if the content of two [ProtoContentHolder] instances are equal. | ||
| */ | ||
| internal fun ProtoContentHolder.contentEquals(other: ProtoContentHolder): Boolean { | ||
| return byteArray.contentEquals(other.byteArray) | ||
| } | ||
|
|
||
| /** | ||
| * Calculates the hash code of the content in the [ProtoContentHolder]. | ||
| */ | ||
| internal fun ProtoContentHolder.contentHashCode(): Int { | ||
| return byteArray.contentHashCode() | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
90 changes: 90 additions & 0 deletions
90
...protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtoMessageSerializer.kt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| /* | ||
| * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
| */ | ||
|
|
||
| package kotlinx.serialization.protobuf.internal | ||
|
|
||
| import kotlinx.serialization.* | ||
| import kotlinx.serialization.builtins.* | ||
| import kotlinx.serialization.descriptors.* | ||
| import kotlinx.serialization.encoding.* | ||
| import kotlinx.serialization.protobuf.* | ||
|
|
||
| internal object ProtoMessageSerializer : KSerializer<ProtoMessage> { | ||
| internal val fieldsSerializer = ProtoFieldSerializer | ||
|
|
||
| override val descriptor: SerialDescriptor | ||
| get() = UnknownFieldsDescriptor(fieldsSerializer.descriptor) | ||
|
|
||
| override fun deserialize(decoder: Decoder): ProtoMessage { | ||
| if (decoder is ProtobufDecoder) { | ||
| return decoder.decodeStructure(descriptor) { | ||
| ProtoMessage(fieldsSerializer.deserializeComposite(this)) | ||
| } | ||
| } | ||
| return ProtoMessage.Empty | ||
| } | ||
|
|
||
| override fun serialize(encoder: Encoder, value: ProtoMessage) { | ||
| if (encoder is ProtobufEncoder) { | ||
| value.fields.forEach { | ||
| fieldsSerializer.serialize(encoder, it) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| internal object ProtoFieldSerializer : KSerializer<ProtoField> { | ||
| private val delegate = ByteArraySerializer() | ||
|
|
||
| override val descriptor: SerialDescriptor | ||
| get() = UnknownFieldsDescriptor(delegate.descriptor) | ||
|
|
||
| fun deserializeComposite(compositeDecoder: CompositeDecoder): ProtoField { | ||
| if (compositeDecoder is ProtobufDecoder) { | ||
| return deserialize(compositeDecoder) | ||
| } | ||
| return ProtoField.Empty | ||
| } | ||
|
|
||
| override fun deserialize(decoder: Decoder): ProtoField { | ||
| if (decoder is ProtobufDecoder) { | ||
| return deserialize(decoder, decoder.currentTag) | ||
| } | ||
| return ProtoField.Empty | ||
| } | ||
|
|
||
| internal fun deserialize(protobufDecoder: ProtobufDecoder, currentTag: ProtoDesc): ProtoField { | ||
| if (currentTag != MISSING_TAG) { | ||
| val id = currentTag.protoId | ||
| val type = protobufDecoder.currentType | ||
| val data = protobufDecoder.decodeRawElement() | ||
| val field = ProtoField( | ||
| id = id, | ||
| wireType = type, | ||
| data = ProtoContentHolder(data), | ||
| ) | ||
| return field | ||
| } | ||
| return ProtoField.Empty | ||
| } | ||
|
|
||
| override fun serialize(encoder: Encoder, value: ProtoField) { | ||
| if (encoder is ProtobufEncoder) { | ||
| encoder.encodeRawElement(value.id, value.wireType, value.data.byteArray) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| internal class UnknownFieldsDescriptor(private val original: SerialDescriptor) : SerialDescriptor by original { | ||
| override val serialName: String | ||
| get() = "UnknownProtoFieldsHolder" | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| return other is UnknownFieldsDescriptor && other.original == original | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| return original.hashCode() | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
362 changes: 362 additions & 0 deletions
362
formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,362 @@ | ||
| /* | ||
| * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. | ||
| */ | ||
|
|
||
| package kotlinx.serialization.protobuf | ||
|
|
||
| import kotlinx.serialization.* | ||
| import kotlinx.serialization.descriptors.* | ||
| import kotlinx.serialization.encoding.* | ||
| import kotlin.test.* | ||
|
|
||
| class ProtobufUnknownFieldsTest { | ||
| @Serializable | ||
| data class InnerData(val name: String, val b: Int, val c: List<String>) | ||
|
|
||
| @Serializable | ||
| data class BuildData(val a: Int, val b: String, val c: ByteArray, val d: List<Int>, val e: InnerData) { | ||
| override fun equals(other: Any?): Boolean { | ||
| if (this === other) return true | ||
| if (other == null || this::class != other::class) return false | ||
|
|
||
| other as BuildData | ||
|
|
||
| if (a != other.a) return false | ||
| if (b != other.b) return false | ||
| if (!c.contentEquals(other.c)) return false | ||
| if (d != other.d) return false | ||
| if (e != other.e) return false | ||
|
|
||
| return true | ||
| } | ||
|
|
||
| override fun hashCode(): Int { | ||
| var result = a | ||
| result = 31 * result + b.hashCode() | ||
| result = 31 * result + c.contentHashCode() | ||
| result = 31 * result + d.hashCode() | ||
| result = 31 * result + e.hashCode() | ||
| return result | ||
| } | ||
|
|
||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithUnknownFields( | ||
| val a: Int, | ||
| @ProtoUnknownFields val unknownFields: ProtoMessage | ||
| ) | ||
|
|
||
| @Test | ||
| fun testDecodeWithUnknownField() { | ||
| val data = BuildData( | ||
| 42, | ||
| "42", | ||
| byteArrayOf(42, 42, 42), | ||
| listOf(42, 42, 42), | ||
| InnerData("42", 42, listOf("42", "42", "42")) | ||
| ) | ||
|
|
||
| /** | ||
| * 1: 42 | ||
| * 2: {"42"} | ||
| * 3: {"***"} | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 5: { | ||
| * 1: {"42"} | ||
| * 2: 42 | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * } | ||
| */ | ||
| val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(DataWithUnknownFields.serializer(), encoded) | ||
| assertEquals(data.a, decoded.a) | ||
| assertEquals(6, decoded.unknownFields.size) | ||
|
|
||
| val encoded2 = ProtoBuf.encodeToHexString(DataWithUnknownFields.serializer(), decoded) | ||
| assertEquals(encoded, encoded2) | ||
| val data2 = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded2) | ||
| assertEquals(data, data2) | ||
| } | ||
|
|
||
| @Test | ||
| fun testCannotDecodeArbitraryMessage() { | ||
| assertFailsWith<IllegalArgumentException> { | ||
| ProtoBuf.decodeFromHexString(ProtoMessage.serializer(), "") | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| fun testCannotEncodeArbitraryMessage() { | ||
| assertFailsWith<IllegalArgumentException> { | ||
| ProtoBuf.encodeToHexString(ProtoMessage.serializer(), ProtoMessage.Empty) | ||
| } | ||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithMultipleUnknownFields( | ||
| val a: Int, | ||
| @ProtoUnknownFields val unknownFields: ProtoMessage, | ||
| @ProtoUnknownFields val unknownFields2: ProtoMessage | ||
| ) | ||
|
|
||
| @Test | ||
| fun testOnlyOneUnknownFieldAllowed() { | ||
| val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| assertFailsWith<IllegalArgumentException> { | ||
| ProtoBuf.decodeFromHexString(DataWithMultipleUnknownFields.serializer(), encoded) | ||
| } | ||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithStaggeredFields( | ||
| @ProtoNumber(2) | ||
| val b: String, | ||
| @ProtoUnknownFields val unknownFields: ProtoMessage, | ||
| @ProtoNumber(4) | ||
| val d: List<Int> | ||
| ) | ||
|
|
||
| @Test | ||
| fun testUnknownFieldBeforeKnownField() { | ||
| val data = BuildData( | ||
| 42, | ||
| "42", | ||
| byteArrayOf(42, 42, 42), | ||
| listOf(42, 42, 42), | ||
| InnerData("42", 42, listOf("42", "42", "42")) | ||
| ) | ||
|
|
||
| /** | ||
| * 1: 42 | ||
| * 2: {"42"} | ||
| * 3: {"***"} | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 5: { | ||
| * 1: {"42"} | ||
| * 2: 42 | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * } | ||
| * } | ||
| */ | ||
| val hex = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(DataWithStaggeredFields.serializer(), hex) | ||
| assertEquals(3, decoded.unknownFields.size) | ||
| assertEquals("42", decoded.b) | ||
| assertEquals(listOf(42, 42, 42), decoded.d) | ||
|
|
||
| val encoded = ProtoBuf.encodeToHexString(DataWithStaggeredFields.serializer(), decoded) | ||
| /** | ||
| * fields are re-ordered but acceptable in protobuf wire data | ||
| * | ||
| * 2: {"42"} | ||
| * 1: 42 | ||
| * 3: {"***"} | ||
| * 5: { | ||
| * 1: {"42"} | ||
| * 2: 42 | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * } | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 4: 42 | ||
| */ | ||
| assertEquals("12023432082a1a032a2a2a2a120a023432102a1a0234321a0234321a023432202a202a202a", encoded) | ||
| val decodeOrigin = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded) | ||
| assertEquals(data, decodeOrigin) | ||
| } | ||
|
|
||
| @Serializable | ||
| data class TotalKnownData(@ProtoUnknownFields val fields: ProtoMessage = ProtoMessage.Empty) | ||
|
|
||
| @Serializable | ||
| data class NestedUnknownData( | ||
| val a: Int, | ||
| @ProtoNumber(5) val inner: TotalKnownData, | ||
| @ProtoUnknownFields val unknown: ProtoMessage | ||
| ) | ||
|
|
||
| @Test | ||
| fun testDecodeNestedUnknownData() { | ||
| /** | ||
| * 1: 42 | ||
| * 2: {"42"} | ||
| * 3: {"***"} | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 5: { | ||
| * 1: {"42"} | ||
| * 2: 42 | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * } | ||
| */ | ||
| val hex = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(NestedUnknownData.serializer(), hex) | ||
| assertEquals(5, decoded.unknown.size) | ||
| } | ||
|
|
||
| object CustomSerializer : KSerializer<DataWithUnknownFields> { | ||
| override val descriptor: SerialDescriptor | ||
| get() = buildClassSerialDescriptor("CustomData") { | ||
| element<Int>("a", annotations = listOf(ProtoNumber(1))) | ||
| element<ProtoMessage>("unknownFields", annotations = listOf(ProtoUnknownFields())) | ||
| } | ||
|
|
||
| override fun deserialize(decoder: Decoder): DataWithUnknownFields { | ||
| var a = 0 | ||
| var unknownFields = ProtoMessage.Empty | ||
| decoder.decodeStructure(descriptor) { | ||
| loop@ while (true) { | ||
| when (val index = decodeElementIndex(descriptor)) { | ||
| CompositeDecoder.DECODE_DONE -> break@loop | ||
| 0 -> a = decodeIntElement(descriptor, index) | ||
| 1 -> unknownFields += decodeSerializableElement(descriptor, index, ProtoMessage.serializer()) | ||
| else -> error("Unexpected index: $index") | ||
| } | ||
| } | ||
| } | ||
| return DataWithUnknownFields(a, unknownFields) | ||
| } | ||
|
|
||
| override fun serialize(encoder: Encoder, value: DataWithUnknownFields) { | ||
| encoder.encodeStructure(descriptor) { | ||
| encodeIntElement(descriptor, 0, value.a) | ||
| encodeSerializableElement(descriptor, 1, ProtoMessage.serializer(), value.unknownFields) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| fun testCustomSerializer() { | ||
| val data = BuildData( | ||
| 42, | ||
| "42", | ||
| byteArrayOf(42, 42, 42), | ||
| listOf(42, 42, 42), | ||
| InnerData("42", 42, listOf("42", "42", "42")) | ||
| ) | ||
|
|
||
| /** | ||
| * 1: 42 | ||
| * 2: {"42"} | ||
| * 3: {"***"} | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 4: 42 | ||
| * 5: { | ||
| * 1: {"42"} | ||
| * 2: 42 | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * 3: {"42"} | ||
| * } | ||
| */ | ||
| val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(CustomSerializer, encoded) | ||
|
|
||
| assertEquals(data.a, decoded.a) | ||
| assertEquals(6, decoded.unknownFields.size) | ||
|
|
||
| val encoded2 = ProtoBuf.encodeToHexString(CustomSerializer, decoded) | ||
| assertEquals(encoded, encoded2) | ||
| val data2 = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded2) | ||
| assertEquals(data, data2) | ||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithWrongTypeUnknownFields( | ||
| val a: Int, | ||
| @ProtoUnknownFields val unknownFields: Map<Int, ByteArray>, | ||
| ) | ||
|
|
||
| @Test | ||
| fun testCannotDecodeWrongTypeUnknownFields() { | ||
| assertFailsWith<IllegalArgumentException> { | ||
| ProtoBuf.decodeFromHexString(DataWithWrongTypeUnknownFields.serializer(), "") | ||
| } | ||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithMissingUnknownFields( | ||
| val a: Int, | ||
| val unknownFields: ProtoMessage = ProtoMessage.Empty | ||
| ) | ||
|
|
||
| @Test | ||
| fun testCannotEncodeMissingAnnotationUnknownFields() { | ||
| val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(DataWithMissingUnknownFields.serializer(), encoded) | ||
| assertFailsWith<IllegalArgumentException> { | ||
| ProtoBuf.encodeToHexString(DataWithMissingUnknownFields.serializer(), decoded) | ||
| } | ||
| } | ||
|
|
||
| @Serializable | ||
| data class DataWithNullableUnknownFields( | ||
| @ProtoNumber(1) val a: Int, | ||
| @ProtoUnknownFields val unknownFields: ProtoMessage? = null | ||
| ) | ||
|
|
||
| @Test | ||
| fun testDataWithNullableUnknownFields() { | ||
| val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432" | ||
| val decoded = ProtoBuf.decodeFromHexString(DataWithNullableUnknownFields.serializer(), encoded) | ||
| assertEquals(42, decoded.a) | ||
| assertEquals(6, decoded.unknownFields?.size) | ||
|
|
||
| val encoded2 = "082a" | ||
| val decoded2 = ProtoBuf.decodeFromHexString(DataWithNullableUnknownFields.serializer(), encoded2) | ||
| assertEquals(42, decoded2.a) | ||
| assertNull(decoded2.unknownFields) | ||
| } | ||
|
|
||
| @Serializable | ||
| data class ToBuildOneOf(val a: String? = null, val b: Long? = null, val c: InnerData? = null) | ||
|
|
||
| @Serializable | ||
| data class TestFewerOneOf( | ||
| @ProtoOneOf val oneOf: OneOf? = null, | ||
| @ProtoUnknownFields val unknownFields: ProtoMessage = ProtoMessage.Empty | ||
| ) | ||
|
|
||
| @Serializable | ||
| sealed interface OneOf { | ||
| @Serializable | ||
| data class A( | ||
| @ProtoNumber(1) val a: String | ||
| ) : OneOf | ||
|
|
||
| @Serializable | ||
| data class B( | ||
| @ProtoNumber(2) val b: Long | ||
| ) : OneOf | ||
| } | ||
|
|
||
| @Test | ||
| fun testUnknownOneOfField() { | ||
| val present = ToBuildOneOf(a = "test") | ||
| val encoded = ProtoBuf.encodeToHexString(present) | ||
| val decoded = ProtoBuf.decodeFromHexString(TestFewerOneOf.serializer(), encoded) | ||
| assertEquals(OneOf.A("test"), decoded.oneOf) | ||
| assertEquals(0, decoded.unknownFields.size) | ||
|
|
||
| val absent = ToBuildOneOf(c = InnerData("test", 42, listOf("test"))) | ||
| val encoded2 = ProtoBuf.encodeToHexString(absent) | ||
| val decoded2 = ProtoBuf.decodeFromHexString(TestFewerOneOf.serializer(), encoded2) | ||
| assertNull(decoded2.oneOf) | ||
| assertEquals(1, decoded2.unknownFields.size) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| // This file was automatically generated from formats.md by Knit tool. Do not edit. | ||
| package example.exampleFormats18 | ||
|
|
||
| import kotlinx.serialization.* | ||
| import kotlinx.serialization.Serializable | ||
| import kotlinx.serialization.descriptors.* | ||
| import kotlinx.serialization.modules.* | ||
| import kotlinx.serialization.encoding.* | ||
| import java.io.* | ||
|
|
||
| private val byteArraySerializer = serializer<ByteArray>() | ||
| @ExperimentalSerializationApi | ||
| class DataOutputEncoder(val output: DataOutput) : AbstractEncoder() { | ||
| override val serializersModule: SerializersModule = EmptySerializersModule() | ||
| override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0) | ||
| override fun encodeByte(value: Byte) = output.writeByte(value.toInt()) | ||
| override fun encodeShort(value: Short) = output.writeShort(value.toInt()) | ||
| override fun encodeInt(value: Int) = output.writeInt(value) | ||
| override fun encodeLong(value: Long) = output.writeLong(value) | ||
| override fun encodeFloat(value: Float) = output.writeFloat(value) | ||
| override fun encodeDouble(value: Double) = output.writeDouble(value) | ||
| override fun encodeChar(value: Char) = output.writeChar(value.code) | ||
| override fun encodeString(value: String) = output.writeUTF(value) | ||
| override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeInt(index) | ||
|
|
||
| override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder { | ||
| encodeInt(collectionSize) | ||
| return this | ||
| } | ||
|
|
||
| override fun encodeNull() = encodeBoolean(false) | ||
| override fun encodeNotNullMark() = encodeBoolean(true) | ||
|
|
||
| override fun <T> encodeSerializableValue(serializer: SerializationStrategy<T>, value: T) { | ||
| if (serializer.descriptor == byteArraySerializer.descriptor) | ||
| encodeByteArray(value as ByteArray) | ||
| else | ||
| super.encodeSerializableValue(serializer, value) | ||
| } | ||
|
|
||
| private fun encodeByteArray(bytes: ByteArray) { | ||
| encodeCompactSize(bytes.size) | ||
| output.write(bytes) | ||
| } | ||
|
|
||
| private fun encodeCompactSize(value: Int) { | ||
| if (value < 0xff) { | ||
| output.writeByte(value) | ||
| } else { | ||
| output.writeByte(0xff) | ||
| output.writeInt(value) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @ExperimentalSerializationApi | ||
| fun <T> encodeTo(output: DataOutput, serializer: SerializationStrategy<T>, value: T) { | ||
| val encoder = DataOutputEncoder(output) | ||
| encoder.encodeSerializableValue(serializer, value) | ||
| } | ||
|
|
||
| @ExperimentalSerializationApi | ||
| inline fun <reified T> encodeTo(output: DataOutput, value: T) = encodeTo(output, serializer(), value) | ||
|
|
||
| @ExperimentalSerializationApi | ||
| class DataInputDecoder(val input: DataInput, var elementsCount: Int = 0) : AbstractDecoder() { | ||
| private var elementIndex = 0 | ||
| override val serializersModule: SerializersModule = EmptySerializersModule() | ||
| override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0 | ||
| override fun decodeByte(): Byte = input.readByte() | ||
| override fun decodeShort(): Short = input.readShort() | ||
| override fun decodeInt(): Int = input.readInt() | ||
| override fun decodeLong(): Long = input.readLong() | ||
| override fun decodeFloat(): Float = input.readFloat() | ||
| override fun decodeDouble(): Double = input.readDouble() | ||
| override fun decodeChar(): Char = input.readChar() | ||
| override fun decodeString(): String = input.readUTF() | ||
| override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readInt() | ||
|
|
||
| override fun decodeElementIndex(descriptor: SerialDescriptor): Int { | ||
| if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE | ||
| return elementIndex++ | ||
| } | ||
|
|
||
| override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder = | ||
| DataInputDecoder(input, descriptor.elementsCount) | ||
|
|
||
| override fun decodeSequentially(): Boolean = true | ||
|
|
||
| override fun decodeCollectionSize(descriptor: SerialDescriptor): Int = | ||
| decodeInt().also { elementsCount = it } | ||
|
|
||
| override fun decodeNotNullMark(): Boolean = decodeBoolean() | ||
|
|
||
| @Suppress("UNCHECKED_CAST") | ||
| override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>, previousValue: T?): T = | ||
| if (deserializer.descriptor == byteArraySerializer.descriptor) | ||
| decodeByteArray() as T | ||
| else | ||
| super.decodeSerializableValue(deserializer, previousValue) | ||
|
|
||
| private fun decodeByteArray(): ByteArray { | ||
| val bytes = ByteArray(decodeCompactSize()) | ||
| input.readFully(bytes) | ||
| return bytes | ||
| } | ||
|
|
||
| private fun decodeCompactSize(): Int { | ||
| val byte = input.readByte().toInt() and 0xff | ||
| if (byte < 0xff) return byte | ||
| return input.readInt() | ||
| } | ||
| } | ||
|
|
||
| @ExperimentalSerializationApi | ||
| fun <T> decodeFrom(input: DataInput, deserializer: DeserializationStrategy<T>): T { | ||
| val decoder = DataInputDecoder(input) | ||
| return decoder.decodeSerializableValue(deserializer) | ||
| } | ||
|
|
||
| @ExperimentalSerializationApi | ||
| inline fun <reified T> decodeFrom(input: DataInput): T = decodeFrom(input, serializer()) | ||
|
|
||
| fun ByteArray.toAsciiHexString() = joinToString("") { | ||
| if (it in 32..127) it.toInt().toChar().toString() else | ||
| "{${it.toUByte().toString(16).padStart(2, '0').uppercase()}}" | ||
| } | ||
|
|
||
| @Serializable | ||
| data class Project(val name: String, val attachment: ByteArray) | ||
|
|
||
| @OptIn(ExperimentalSerializationApi::class) | ||
| fun main() { | ||
| val data = Project("kotlinx.serialization", byteArrayOf(0x0A, 0x0B, 0x0C, 0x0D)) | ||
| val output = ByteArrayOutputStream() | ||
| encodeTo(DataOutputStream(output), data) | ||
| val bytes = output.toByteArray() | ||
| println(bytes.toAsciiHexString()) | ||
| val input = ByteArrayInputStream(bytes) | ||
| val obj = decodeFrom<Project>(DataInputStream(input)) | ||
| println(obj) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Given that protobuf is a recursive format, it would make sense for ProgoMessage and ProtoContentHolder to be the same. It may also make sense to have support for proto primitives (with fixed size) rather than storing that data as bytearray.