Skip to content

Commit 29384c9

Browse files
pjanczykpjanczyk
authored
Support default enums and records (#889)
1 parent 1500ddb commit 29384c9

11 files changed

Lines changed: 192 additions & 206 deletions

File tree

‎avro4s-core/src/main/scala/com/sksamuel/avro4s/CustomDefaults.scala‎

Lines changed: 48 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,50 @@
1-
//package com.sksamuel.avro4s
2-
//
3-
//import java.time.Instant
4-
//
1+
package com.sksamuel.avro4s
2+
53
//import magnolia.{SealedTrait, Subtype}
6-
//import org.json4s.native.JsonMethods.parse
7-
//import org.json4s.native.Serialization.write
8-
//import org.apache.avro.Schema
9-
//import org.apache.avro.Schema.Type
10-
//import org.json4s.DefaultFormats
11-
//
12-
//import scala.collection.JavaConverters._
13-
//
14-
//sealed trait CustomDefault
15-
//case class CustomUnionDefault(className: String, values: java.util.Map[String, Any]) extends CustomDefault
16-
//case class CustomUnionWithEnumDefault(parentName: String, default: String, value: String) extends CustomDefault
17-
//case class CustomEnumDefault(value: String) extends CustomDefault
18-
//
19-
//object CustomDefaults {
20-
//
21-
// implicit val formats = DefaultFormats
4+
import java.util.Map
5+
import org.apache.avro.Schema
6+
import org.apache.avro.Schema.Type
7+
import scala.collection.JavaConverters.*
8+
9+
/** Extends [[java.util.Map]] for JSON serialization in [[org.apache.avro.util.internal.JacksonUtils.toJson]] */
10+
case class CustomUnionDefault(className: String, value: java.util.Map[String, Any])
11+
extends java.util.AbstractMap[String, Any] {
12+
override def entrySet(): java.util.Set[java.util.Map.Entry[String, Any]] = value.entrySet()
13+
}
14+
15+
/** Extends [[CharSequence]] for JSON serialization in [[org.apache.avro.util.internal.JacksonUtils.toJson]] */
16+
case class CustomUnionWithEnumDefault(parentName: String, default: String, value: String)
17+
extends CharSequence {
18+
override def toString: String = value
19+
override def length: Int = value.length
20+
override def charAt(index: Int): Char = value.charAt(index)
21+
override def subSequence(start: Int, end: Int): CharSequence = value.subSequence(start, end)
22+
}
23+
24+
object CustomDefaults {
2225
//
2326
// def customScalaEnumDefault(value: Any) = CustomEnumDefault(value.toString)
2427
//
25-
// def customDefault(p: Product, schema: Schema): CustomDefault =
26-
// if(isEnum(p, schema.getType))
27-
// CustomEnumDefault(trimmedClassName(p))
28-
// else {
29-
// if(isUnionOfEnum(schema)) {
30-
// val enumType = schema.getTypes.asScala.filter(_.getType == Schema.Type.ENUM).head
31-
// CustomUnionWithEnumDefault(enumType.getName, trimmedClassName(p), p.toString)
32-
// } else
33-
// CustomUnionDefault(trimmedClassName(p), parse(write(p)).extract[Map[String, Any]].map {
28+
def customDefault(p: Product, schema: Schema): AnyRef =
29+
if (isEnum(p, schema.getType))
30+
trimmedClassName(p)
31+
else {
32+
if (isUnionOfEnum(schema)) {
33+
val enumType = schema.getTypes.asScala.filter(_.getType == Schema.Type.ENUM).head
34+
CustomUnionWithEnumDefault(enumType.getName, trimmedClassName(p), p.toString)
35+
} else
36+
CustomUnionDefault(trimmedClassName(p), p.productElementNames.zip(p.productIterator).map {
3437
// case (name, b: BigInt) if b.isValidInt => name -> b.intValue
3538
// case (name, b: BigInt) if b.isValidLong => name -> b.longValue
36-
// case (name, z) if schema.getType == Type.UNION => name ->
37-
// schema.getTypes.asScala.find(_.getName == trimmedClassName(p)).map(_.getField(name).schema())
38-
// .map(DefaultResolver(z, _)).getOrElse(z)
39-
// case (name, z) => name -> DefaultResolver(z, schema.getField(name).schema())
40-
//
41-
// }.asJava)
42-
// }
43-
//
44-
// def isUnionOfEnum(schema: Schema) = schema.getType == Schema.Type.UNION && schema.getTypes.asScala.map(_.getType).contains(Schema.Type.ENUM)
39+
case (name, z) if schema.getType == Type.UNION => name ->
40+
schema.getTypes.asScala.find(_.getName == trimmedClassName(p)).map(_.getField(name).schema())
41+
.map(DefaultResolver(z, _)).getOrElse(z)
42+
case (name, z) => name -> DefaultResolver(z, schema.getField(name).schema())
43+
44+
}.toMap.asJava)
45+
}
46+
47+
def isUnionOfEnum(schema: Schema) = schema.getType == Schema.Type.UNION && schema.getTypes.asScala.map(_.getType).contains(Schema.Type.ENUM)
4548
//
4649
// def sealedTraitEnumDefaultValue[T](ctx: SealedTrait[SchemaFor, T]) = {
4750
// val defaultExtractor = new AnnotationExtractors(ctx.annotations)
@@ -58,10 +61,10 @@
5861
// def isScalaEnumeration(value: Any) = value.getClass.getCanonicalName == "scala.Enumeration.Val"
5962
//
6063
//
61-
// private def isEnum(product: Product, schemaType: Schema.Type) =
62-
// product.productArity == 0 && schemaType == Schema.Type.ENUM
63-
//
64-
// private def trimmedClassName(p: Product) = trimDollar(p.getClass.getSimpleName)
65-
//
66-
// private def trimDollar(s: String) = if(s.endsWith("$")) s.dropRight(1) else s
67-
//}
64+
private def isEnum(product: Product, schemaType: Schema.Type) =
65+
product.productArity == 0 && schemaType == Schema.Type.ENUM
66+
67+
private def trimmedClassName(p: Product) = trimDollar(p.getClass.getSimpleName)
68+
69+
private def trimDollar(s: String) = if(s.endsWith("$")) s.dropRight(1) else s
70+
}

‎avro4s-core/src/main/scala/com/sksamuel/avro4s/DefaultResolver.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import org.apache.avro.LogicalTypes.Decimal
99
import org.apache.avro.generic.GenericFixed
1010
import org.apache.avro.util.Utf8
1111
import org.apache.avro.{Conversions, Schema}
12-
//import CustomDefaults._
12+
import CustomDefaults._
1313
import scala.collection.JavaConverters._
1414

1515
/**
@@ -46,7 +46,7 @@ object DefaultResolver {
4646
case x: Seq[_] => x.asJava
4747
case x: Set[_] => x.asJava
4848
// case shapeless.Inl(x) => apply(x, schema)
49-
// case p: Product => customDefault(p, schema)
49+
case p: Product => customDefault(p, schema)
5050
// case v if isScalaEnumeration(v) => customScalaEnumDefault(value)
5151
case _ => value.asInstanceOf[AnyRef]
5252
}

‎avro4s-core/src/main/scala/com/sksamuel/avro4s/avroutils/SchemaHelper.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.sksamuel.avro4s.avroutils
22

33
import com.sksamuel.avro4s.{Avro4sConfigurationException, FieldMapper}
4+
import com.sksamuel.avro4s.{CustomUnionDefault, CustomUnionWithEnumDefault}
45
import org.apache.avro.generic.GenericData
56
import org.apache.avro.util.Utf8
67
import org.apache.avro.{JsonProperties, Schema, SchemaBuilder}
@@ -119,15 +120,13 @@ object SchemaHelper {
119120
case _: java.util.Map[_, _] => Schema.Type.MAP
120121
case JsonProperties.NULL_VALUE => Schema.Type.NULL
121122
case Schema.Field.NULL_DEFAULT_VALUE => Schema.Type.NULL
122-
// case CustomEnumDefault(_) => Schema.Type.ENUM
123123
case other => other
124124
}
125125

126126
val (first, rest) = schema.getTypes.asScala.partition { t =>
127-
defaultType match {
128-
// case CustomUnionDefault(name, _) => name == t.getName
129-
// case CustomUnionWithEnumDefault(name, default, _) =>
130-
// name == t.getName
127+
default match {
128+
case CustomUnionDefault(name, _) => name == t.getName
129+
case CustomUnionWithEnumDefault(name, _, _) => name == t.getName
131130
case _ => t.getType == defaultType
132131
}
133132
}

‎avro4s-core/src/main/scala/com/sksamuel/avro4s/schemas/records.scala‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,4 @@ object Records:
9999
// val schemaWithPossibleNull = if (default.contains(null) && schema.getType != Schema.Type.UNION) {
100100
// SchemaBuilder.unionOf().`type`(schema).and().`type`(Schema.create(Schema.Type.NULL)).endUnion()
101101
// } else schema
102-
103-
// val field = encodedDefault match {
104-
// case null => new Schema.Field(name, schemaWithResolvedNamespace, doc)
105-
// case CustomUnionDefault(_, m) =>
106-
// new Schema.Field(name, schemaWithResolvedNamespace, doc, m)
107-
// case CustomEnumDefault(m) =>
108-
// new Schema.Field(name, schemaWithResolvedNamespace, doc, m)
109-
// case CustomUnionWithEnumDefault(_, _, m) => new Schema.Field(name, schemaWithResolvedNamespace, doc, m)
110-
// case _ => new Schema.Field(name, schemaWithResolvedNamespace, doc, encodedDefault)
111-
// }
112102
//

‎avro4s-core/src/test/scala/com/sksamuel/avro4s/github/Github411.scala‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
//package com.sksamuel.avro4s.github
2-
//
3-
//import com.sksamuel.avro4s.SchemaFor
4-
//import org.scalatest.funsuite.AnyFunSuite
5-
//import org.scalatest.matchers.should.Matchers
6-
//
7-
//class Github411 extends AnyFunSuite with Matchers {
8-
//
9-
// case class DefaultOfDefault(value: String)
10-
// case class DefaultValue(property: DefaultOfDefault = DefaultOfDefault("some-default"))
11-
// case class Github411Class(property: DefaultValue = DefaultValue())
12-
//
13-
// test("schema generation with defaults in defaults") {
14-
// SchemaFor[Github411Class].schema.toString(true)
15-
// }
16-
//}
1+
package com.sksamuel.avro4s.github
2+
3+
import com.sksamuel.avro4s.SchemaFor
4+
import org.scalatest.funsuite.AnyFunSuite
5+
import org.scalatest.matchers.should.Matchers
6+
7+
class Github411 extends AnyFunSuite with Matchers {
8+
9+
case class DefaultOfDefault(value: String)
10+
case class DefaultValue(property: DefaultOfDefault = DefaultOfDefault("some-default"))
11+
case class Github411Class(property: DefaultValue = DefaultValue())
12+
13+
test("schema generation with defaults in defaults") {
14+
SchemaFor[Github411Class].schema.toString(true)
15+
}
16+
}

‎avro4s-core/src/test/scala/com/sksamuel/avro4s/github/GithubIssue110.scala‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
//package com.sksamuel.avro4s.github
2-
//
3-
//import com.sksamuel.avro4s.RecordFormat
4-
//import org.scalatest.funsuite.AnyFunSuite
5-
//import org.scalatest.matchers.should.Matchers
6-
//
7-
//case class P1(name: String, age: Int = 18)
8-
//case class P2(name: String)
9-
//
10-
//class GithubIssue110 extends AnyFunSuite with Matchers {
11-
//
12-
// test("default value should be picked up") {
13-
// val f1 = RecordFormat[P1]
14-
// val f2 = RecordFormat[P2]
15-
// f1.from(f2.to(P2("foo"))) shouldBe P1("foo")
16-
// }
17-
//}
1+
package com.sksamuel.avro4s.github
2+
3+
import com.sksamuel.avro4s.{AvroSchema, FromRecord, ToRecord}
4+
import org.scalatest.funsuite.AnyFunSuite
5+
import org.scalatest.matchers.should.Matchers
6+
7+
case class P1(name: String, age: Int = 18)
8+
case class P2(name: String)
9+
10+
class GithubIssue110 extends AnyFunSuite with Matchers {
11+
12+
test("default value should be picked up") {
13+
val f1 = FromRecord[P1](AvroSchema[P1])
14+
val f2 = ToRecord[P2](AvroSchema[P2])
15+
f1.from(f2.to(P2("foo"))) shouldBe P1("foo")
16+
}
17+
}

‎avro4s-core/src/test/scala/com/sksamuel/avro4s/record/decoder/SchemaEvolutionTest.scala‎

Lines changed: 39 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
1-
//package com.sksamuel.avro4s.record.decoder
2-
//
1+
package com.sksamuel.avro4s.record.decoder
2+
33
//import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
4-
//
5-
//import com.sksamuel.avro4s._
6-
//import org.apache.avro.SchemaBuilder
7-
//import org.apache.avro.generic.GenericData
8-
//import org.apache.avro.util.Utf8
9-
//import org.scalatest.funsuite.AnyFunSuite
10-
//import org.scalatest.matchers.should.Matchers
11-
//
12-
//class SchemaEvolutionTest extends AnyFunSuite with Matchers {
13-
//
4+
5+
import com.sksamuel.avro4s._
6+
import org.apache.avro.SchemaBuilder
7+
import org.apache.avro.generic.GenericData
8+
import org.apache.avro.util.Utf8
9+
import org.scalatest.funsuite.AnyFunSuite
10+
import org.scalatest.matchers.should.Matchers
11+
12+
class SchemaEvolutionTest extends AnyFunSuite with Matchers {
13+
1414
// case class Version1(original: String)
1515
// case class Version2(@AvroAlias("original") renamed: String)
16-
//
17-
// case class P1(name: String, age: Int = 18)
18-
// case class P2(name: String)
19-
//
20-
// case class OptionalStringTest(a: String, b: Option[String])
21-
// case class DefaultStringTest(a: String, b: String = "foo")
22-
//
16+
17+
case class P1(name: String, age: Int = 18)
18+
case class P2(name: String)
19+
20+
case class OptionalStringTest(a: String, b: Option[String])
21+
case class DefaultStringTest(a: String, b: String = "foo")
22+
2323
// ignore("@AvroAlias should be used when a reader schema has a field missing from the write schema") {
2424
//
2525
// val v1schema = AvroSchema[Version1]
@@ -35,27 +35,23 @@
3535
//
3636
// v2.renamed shouldBe v1.original
3737
// }
38-
//
39-
// test("when decoding, if the record and schema are missing a field and the target has a scala default, use that") {
40-
//
41-
// val f1 = RecordFormat[P1]
42-
// val f2 = RecordFormat[P2]
43-
//
44-
// f1.from(f2.to(P2("foo"))) shouldBe P1("foo")
45-
// }
46-
//
47-
// test("when decoding, if the record is missing a field that is present in the schema with a default, use the default from the schema") {
48-
// val schema = SchemaBuilder.record("foo").fields().requiredString("a").endRecord()
49-
// val record = new GenericData.Record(schema)
50-
// record.put("a", new Utf8("hello"))
51-
// Decoder[DefaultStringTest].decode(record) shouldBe DefaultStringTest("hello")
52-
// }
53-
//
54-
// test("when decoding, if the record is missing a field that is present in the schema and the type is option, then set to None") {
55-
// val schema1 = SchemaBuilder.record("foo").fields().requiredString("a").endRecord()
56-
// val schema2 = SchemaBuilder.record("foo").fields().requiredString("a").optionalString("b").endRecord()
57-
// val record = new GenericData.Record(schema1)
58-
// record.put("a", new Utf8("hello"))
59-
// Decoder[OptionalStringTest].decode(record) shouldBe OptionalStringTest("hello", None)
60-
// }
61-
//}
38+
39+
test("when decoding, if the record and schema are missing a field and the target has a scala default, use that") {
40+
val record = ToRecord[P2](AvroSchema[P2]).to(P2("foo"))
41+
FromRecord[P1](AvroSchema[P1]).from(record) shouldBe P1("foo", 18)
42+
}
43+
44+
test("when decoding, if the record is missing a field that is present in the schema with a default, use the default from the schema") {
45+
val schema = SchemaBuilder.record("foo").fields().requiredString("a").endRecord()
46+
val record = new GenericData.Record(schema)
47+
record.put("a", new Utf8("hello"))
48+
FromRecord[DefaultStringTest](AvroSchema[DefaultStringTest]).from(record) shouldBe DefaultStringTest("hello")
49+
}
50+
51+
test("when decoding, if the record is missing a field that is present in the schema and the type is option, then set to None") {
52+
val schema1 = SchemaBuilder.record("foo").fields().requiredString("a").endRecord()
53+
val record = new GenericData.Record(schema1)
54+
record.put("a", new Utf8("hello"))
55+
FromRecord[OptionalStringTest](AvroSchema[OptionalStringTest]).from(record) shouldBe OptionalStringTest("hello", None)
56+
}
57+
}

0 commit comments

Comments
 (0)