Skip to content

Commit 06b3d56

Browse files
marcuswrgrouzen
marcuswr
and
grouzen
authored
Support encoding/decoding of case classes with > 22 fields. (#130)
* Support encoding/decoding of case classes with > 22 fields. TODO: determine where transform param should be used in SchemaEncoderDeriver.deriveTransformedRecord Ziverge zio-schema derivation reference: https://www.ziverge.com/post/type-class-derivation-with-zio-schema * Address PR feedback. Separate case classes with more than 22 fields. Naming loosely based on: https://github.com/zio/zio-http/pull/3065/files#diff-b4f1cb60f121214f58b86c32e2f1a633a6727796328ad1ffd1e66738d8e40d94 * Update modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/codec/ValueEncoderDeriver.scala Co-authored-by: Michael Nedokushev <[email protected]> * Update modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/codec/ValueDecoderDeriver.scala Co-authored-by: Michael Nedokushev <[email protected]> * Unused Code Removal * formatting sbt scalafmtAll sbt scalafixAll * Fix schema derivation error for scala 2.13 * Re-generate ci.yaml * Make linter happy --------- Co-authored-by: Michael Nedokushev <[email protected]>
1 parent 20e9999 commit 06b3d56

File tree

18 files changed

+234
-15
lines changed

18 files changed

+234
-15
lines changed

‎.github/workflows/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
strategy:
2323
matrix:
2424
os: [ubuntu-latest]
25-
scala: [2.13.16, 3.6.4]
25+
scala: [2.13.16, 3.3.5]
2626
java: [temurin@11, temurin@17]
2727
runs-on: ${{ matrix.os }}
2828
steps:

‎.scalafmt.conf

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
version = "3.9.5"
2+
runner.dialect = scala3
23
maxColumn = 120
34
align.preset = most
45
continuationIndent.defnSite = 2

‎build.sbt

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import BuildHelper._
2+
import org.typelevel.scalacoptions.ScalacOptions
23

34
inThisBuild(
45
List(
@@ -21,7 +22,7 @@ inThisBuild(
2122
)
2223
),
2324
crossScalaVersions := Seq(Scala213, Scala3),
24-
ThisBuild / scalaVersion := Scala3,
25+
scalaVersion := Scala3,
2526
githubWorkflowJavaVersions := Seq(JavaSpec.temurin("11"), JavaSpec.temurin("17")),
2627
githubWorkflowPublishTargetBranches := Seq(),
2728
githubWorkflowBuildPreamble := Seq(
@@ -50,6 +51,13 @@ lazy val core =
5051
.in(file("modules/core"))
5152
.settings(
5253
stdSettings("core"),
54+
tpolecatSettings,
5355
libraryDependencies ++= Dep.core,
5456
testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework")
5557
)
58+
59+
val tpolecatSettings =
60+
Seq(
61+
tpolecatScalacOptions += ScalacOptions.source3,
62+
tpolecatExcludeOptions += ScalacOptions.lintInferAny
63+
)

‎docs/scala-cli/Filtering.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import zio.*
55
import zio.schema.*

‎docs/scala-cli/ParquetIO.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import zio.schema.*
55
import me.mnedokushev.zio.apache.parquet.core.codec.*

‎docs/scala-cli/Schema.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
1+
//> using scala "3.6.4"
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import zio.schema.*
55
import me.mnedokushev.zio.apache.parquet.core.codec.*

‎docs/scala-cli/SchemaArity23.scala

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
//> using scala "2.13.16"
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.2.0
3+
4+
import zio.schema._
5+
import me.mnedokushev.zio.apache.parquet.core.codec._
6+
7+
object SchemaArity23 extends App {
8+
9+
final case class Arity23(
10+
a: Int,
11+
b: Option[String],
12+
c: Int,
13+
d: Int,
14+
e: Int,
15+
f: Int,
16+
g: Int,
17+
h: Int,
18+
i: Int,
19+
j: Int,
20+
k: Int,
21+
l: Int,
22+
m: Int,
23+
n: Int,
24+
o: Int,
25+
p: Int,
26+
q: Int,
27+
r: Int,
28+
s: Int,
29+
t: Int,
30+
u: Int,
31+
v: Int,
32+
w: Int
33+
)
34+
35+
object Arity23 {
36+
implicit val schema: Schema[Arity23] =
37+
DeriveSchema.gen[Arity23]
38+
implicit val schemaEncoder: SchemaEncoder[Arity23] =
39+
Derive.derive[SchemaEncoder, Arity23](SchemaEncoderDeriver.default)
40+
}
41+
42+
val arity23Schema = Arity23.schemaEncoder.encode(Arity23.schema, "arity23", optional = false)
43+
44+
println(arity23Schema)
45+
46+
}

‎docs/scala-cli/SchemaSummoned.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import me.mnedokushev.zio.apache.parquet.core.Schemas
55
import zio.schema.*

‎docs/scala-cli/Value.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import zio.schema.*
55
import me.mnedokushev.zio.apache.parquet.core.codec.*

‎docs/scala-cli/ValueSummoned.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//> using scala "3.5.0"
2-
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.11
2+
//> using dep me.mnedokushev::zio-apache-parquet-core:0.1.12
33

44
import me.mnedokushev.zio.apache.parquet.core.Value
55
import zio.schema.*

‎modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/codec/SchemaEncoderDeriver.scala

+13-1
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,20 @@ object SchemaEncoderDeriver {
155155
transform: Schema.Transform[A, B, ?],
156156
fields: => Chunk[Deriver.WrappedF[SchemaEncoder, ?]],
157157
summoned: => Option[SchemaEncoder[B]]
158-
): SchemaEncoder[B] = ???
158+
): SchemaEncoder[B] = summoned.getOrElse {
159+
new SchemaEncoder[B] {
160+
private def enc[A1](name0: String, schema0: Schema[A1], encoder: SchemaEncoder[?]) =
161+
encoder.asInstanceOf[SchemaEncoder[A1]].encode(schema0, name0, isSchemaOptional(schema0))
162+
163+
override def encode(schema: Schema[B], name: String, optional: Boolean): Type = {
164+
val fieldTypes = record.fields.zip(fields.map(_.unwrap)).map { case (field, encoder) =>
165+
enc(field.name, field.schema, encoder)
166+
}
159167

168+
Schemas.record(fieldTypes).optionality(optional).named(name)
169+
}
170+
}
171+
}
160172
}.cached
161173

162174
val summoned: Deriver[SchemaEncoder] = default.autoAcceptSummoned

‎modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/codec/ValueDecoderDeriver.scala

+26-1
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,33 @@ object ValueDecoderDeriver {
217217
transform: Schema.Transform[A, B, ?],
218218
fields: => Chunk[Deriver.WrappedF[ValueDecoder, ?]],
219219
summoned: => Option[ValueDecoder[B]]
220-
): ValueDecoder[B] = ???
220+
): ValueDecoder[B] = summoned.getOrElse {
221+
new ValueDecoder[B] {
222+
override def decode(value: Value): B =
223+
value match {
224+
case GroupValue.RecordValue(values) =>
225+
Unsafe.unsafe { implicit unsafe =>
226+
record
227+
.construct(
228+
Chunk
229+
.fromIterable(record.fields.map(f => values(f.name)))
230+
.zip(fields.map(_.unwrap))
231+
.map { case (v, decoder) =>
232+
decoder.decode(v)
233+
}
234+
)
235+
.flatMap(transform.f) match {
236+
case Right(v) => v
237+
case Left(reason) =>
238+
throw DecoderError(s"Couldn't decode $value: $reason")
239+
}
240+
}
221241

242+
case other =>
243+
throw DecoderError(s"Couldn't decode $other, it must be of type RecordValue")
244+
}
245+
}
246+
}
222247
}.cached
223248

224249
def summoned: Deriver[ValueDecoder] =

‎modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/codec/ValueEncoderDeriver.scala

+21-2
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,27 @@ object ValueEncoderDeriver {
180180
transform: Schema.Transform[A, B, ?],
181181
fields: => Chunk[Deriver.WrappedF[ValueEncoder, ?]],
182182
summoned: => Option[ValueEncoder[B]]
183-
): ValueEncoder[B] = ???
184-
183+
): ValueEncoder[B] = summoned.getOrElse {
184+
new ValueEncoder[B] {
185+
private def enc[A1](v: A, field: Schema.Field[A, A1], encoder: ValueEncoder[?]) =
186+
encoder.asInstanceOf[ValueEncoder[A1]].encode(field.get(v))
187+
188+
override def encode(value: B): Value =
189+
transform.g(value) match {
190+
case Right(v) =>
191+
Value.record(
192+
record.fields
193+
.zip(fields.map(_.unwrap))
194+
.map { case (field, encoder) =>
195+
field.name -> enc(v, field, encoder)
196+
}
197+
.toMap
198+
)
199+
case Left(reason) =>
200+
throw EncoderError(s"Failed to encode transformed record for value $value: $reason")
201+
}
202+
}
203+
}
185204
}.cached
186205

187206
val summoned: Deriver[ValueEncoder] = default.autoAcceptSummoned

‎modules/core/src/test/scala-2.13+/me/mnedokushev/zio/apache/parquet/core/Fixtures.scala

+34
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,40 @@ import java.util.{ Currency, UUID }
2020

2121
object Fixtures {
2222

23+
// unable to generate code for case classes with more than 120 int fields due to following error:
24+
// tested with jdk 11.0.23 64bit
25+
// Error while emitting me/mnedokushev/zio/apache/parquet/core/codec/SchemaEncoderDeriverSpec$MaxArityRecord$
26+
// Method too large: me/mnedokushev/zio/apache/parquet/core/codec/SchemaEncoderDeriverSpec$MaxArityRecord$.derivedSchema0$lzyINIT4$1$$anonfun$364 (Lscala/collection/immutable/ListMap;)Lscala/util/Either;
27+
case class Arity23(
28+
a: Int,
29+
b: Option[String],
30+
c: Int,
31+
d: Int,
32+
e: Int,
33+
f: Int,
34+
g: Int,
35+
h: Int,
36+
i: Int,
37+
j: Int,
38+
k: Int,
39+
l: Int,
40+
m: Int,
41+
n: Int,
42+
o: Int,
43+
p: Int,
44+
q: Int,
45+
r: Int,
46+
s: Int,
47+
t: Int,
48+
u: Int,
49+
v: Int,
50+
w: Int
51+
)
52+
object Arity23 {
53+
implicit lazy val schema: Schema[Arity23] =
54+
DeriveSchema.gen[Arity23]
55+
}
56+
2357
case class MyRecord(a: String, b: Int, child: MyRecord.Child, enm: MyRecord.Enum, opt: Option[Int])
2458

2559
object MyRecord {

‎modules/core/src/test/scala/me/mnedokushev/zio/apache/parquet/core/codec/SchemaEncoderDeriverSpec.scala

+39
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import zio.schema._
77
import zio.test._
88

99
import java.util.UUID
10+
import me.mnedokushev.zio.apache.parquet.core.Fixtures
1011
//import scala.annotation.nowarn
1112

1213
object SchemaEncoderDeriverSpec extends ZIOSpecDefault {
@@ -114,6 +115,44 @@ object SchemaEncoderDeriverSpec extends ZIOSpecDefault {
114115
tpeRequired == schemaDef.required.named(name)
115116
)
116117
},
118+
test("record arity > 22") {
119+
val name = "arity"
120+
val encoder = Derive.derive[SchemaEncoder, Fixtures.Arity23](SchemaEncoderDeriver.default)
121+
val tpeOptional = encoder.encode(Fixtures.Arity23.schema, name, optional = true)
122+
val tpeRequired = encoder.encode(Fixtures.Arity23.schema, name, optional = false)
123+
val schemaDef = Schemas.record(
124+
Chunk(
125+
Schemas.int.required.named("a"),
126+
Schemas.string.optional.named("b"),
127+
Schemas.int.required.named("c"),
128+
Schemas.int.required.named("d"),
129+
Schemas.int.required.named("e"),
130+
Schemas.int.required.named("f"),
131+
Schemas.int.required.named("g"),
132+
Schemas.int.required.named("h"),
133+
Schemas.int.required.named("i"),
134+
Schemas.int.required.named("j"),
135+
Schemas.int.required.named("k"),
136+
Schemas.int.required.named("l"),
137+
Schemas.int.required.named("m"),
138+
Schemas.int.required.named("n"),
139+
Schemas.int.required.named("o"),
140+
Schemas.int.required.named("p"),
141+
Schemas.int.required.named("q"),
142+
Schemas.int.required.named("r"),
143+
Schemas.int.required.named("s"),
144+
Schemas.int.required.named("t"),
145+
Schemas.int.required.named("u"),
146+
Schemas.int.required.named("v"),
147+
Schemas.int.required.named("w")
148+
)
149+
)
150+
151+
assertTrue(
152+
tpeOptional == schemaDef.optional.named(name),
153+
tpeRequired == schemaDef.required.named(name)
154+
)
155+
},
117156
test("sequence") {
118157
val name = "mylist"
119158
val encoders: List[SchemaEncoder[?]] =

‎modules/core/src/test/scala/me/mnedokushev/zio/apache/parquet/core/codec/ValueCodecDeriverSpec.scala

+35
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import java.time.{
2727
ZonedDateTime
2828
}
2929
import java.util.{ Currency, UUID }
30+
import me.mnedokushev.zio.apache.parquet.core.Fixtures
3031

3132
//import java.nio.ByteBuffer
3233
//import java.util.UUID
@@ -327,6 +328,40 @@ object ValueCodecDeriverSpec extends ZIOSpecDefault {
327328
result <- decoder.decodeZIO(value)
328329
} yield assertTrue(result == payload)
329330
},
331+
test("record arity > 22") {
332+
val encoder = Derive.derive[ValueEncoder, Fixtures.Arity23](ValueEncoderDeriver.default)
333+
val decoder = Derive.derive[ValueDecoder, Fixtures.Arity23](ValueDecoderDeriver.default)
334+
val payload = Fixtures.Arity23(
335+
a = 1,
336+
b = None,
337+
c = 3,
338+
d = 4,
339+
e = 5,
340+
f = 6,
341+
g = 7,
342+
h = 8,
343+
i = 9,
344+
j = 10,
345+
k = 11,
346+
l = 12,
347+
m = 13,
348+
n = 14,
349+
o = 15,
350+
p = 16,
351+
q = 17,
352+
r = 18,
353+
s = 19,
354+
t = 20,
355+
u = 21,
356+
v = 22,
357+
w = 23
358+
)
359+
360+
for {
361+
value <- encoder.encodeZIO(payload)
362+
result <- decoder.decodeZIO(value)
363+
} yield assertTrue(result == payload)
364+
},
330365
test("enum") {
331366
val encoder = Derive.derive[ValueEncoder, MyEnum](ValueEncoderDeriver.default)
332367
val decoder = Derive.derive[ValueDecoder, MyEnum](ValueDecoderDeriver.default)

‎project/BuildHelper.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ object BuildHelper {
2525

2626
val Scala212 = "2.12.20"
2727
val Scala213 = "2.13.16"
28-
val Scala3 = "3.6.4"
28+
val Scala3 = "3.3.5"
2929

3030
private def betterMonadicFor(scalaVersion: String) =
3131
CrossVersion.partialVersion(scalaVersion) match {

‎project/Dep.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ object Dep {
3333

3434
lazy val scalaCollectionCompat = O.scalaLangModules %% "scala-collection-compat" % V.scalaCollectionCompat
3535

36-
lazy val scalaReflect = Def.setting("org.scala-lang" % "scala-reflect" % scalaVersion.value)
36+
lazy val scalaReflect = Def.setting("org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided")
3737

3838
lazy val core = Seq(
3939
zio,

0 commit comments

Comments
 (0)