Merge branch 'release/0.2.3'

johnynek · johnynek · commit 0043573ddcae · 2013-05-16T16:43:34.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -28,6 +28,7 @@
 /src/intellij/*.iws
 /.cache
 /.idea
+/.idea_modules
 /.settings
 
 # bak files produced by ./cleanup-commit
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,15 @@
 # chill #
 
+### 0.2.3
+* Update to Kryo 2.21
+* Update to Bijection 0.4.0
+* Improve Traversable Serializer
+
+### 0.2.2
+
+* Custom RegexSerializer
+* ArrayBuffer serialization fix (#48)
+
 ### 0.2.1
 * Improve MeatLocker (safe to call get before serialization)
 * Fix a bug with serialization of mutable collections
diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ Chill provides support for singletons, scala Objects and the following types:
 
 ## Maven
 
-Current version is `0.2.1 `. groupid=`"com.twitter"` artifact=`"chill_2.9.2"` or artifact=`"chill_2.10"`.
+Current version is `0.2.3`. groupid=`"com.twitter"` artifact=`"chill_2.9.2"` or artifact=`"chill_2.10"`.
 
 ## Authors
 
diff --git a/chill-scala/src/main/scala/com/twitter/chill/InjectiveSerializer.scala b/chill-scala/src/main/scala/com/twitter/chill/InjectiveSerializer.scala
@@ -23,7 +23,7 @@ class InjectiveSerializer[T] private (injection: Injection[T, Array[Byte]]) exte
   def write(kser: Kryo, out: Output, obj: T) {
     val bytes = injection(obj)
     out.writeInt(bytes.length, true)
-    out.writeBytes(bytes);
+    out.writeBytes(bytes)
   }
 
   def read(kser: Kryo, in: Input, cls: Class[T]): T = {
diff --git a/chill-scala/src/main/scala/com/twitter/chill/KryoBase.scala b/chill-scala/src/main/scala/com/twitter/chill/KryoBase.scala
@@ -19,19 +19,19 @@ package com.twitter.chill
 import com.esotericsoftware.kryo.Kryo
 import com.esotericsoftware.kryo.KryoException
 import com.esotericsoftware.kryo.{ Serializer => KSerializer }
-import com.esotericsoftware.reflectasm.ConstructorAccess;
+import com.esotericsoftware.reflectasm.ConstructorAccess
 import com.esotericsoftware.kryo.serializers.FieldSerializer
 
-import org.objenesis.instantiator.ObjectInstantiator;
-import org.objenesis.strategy.InstantiatorStrategy;
+import org.objenesis.instantiator.ObjectInstantiator
+import org.objenesis.strategy.InstantiatorStrategy
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Modifier
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Modifier;
 /*
  * This is the base class of Kryo we use to fix specific scala
  * related issues discovered (ideally, this should be fixed in Kryo)
  */
-
 class KryoBase extends Kryo {
 
   lazy val objSer = new ObjectSerializer[AnyRef]
diff --git a/chill-scala/src/main/scala/com/twitter/chill/KryoBijection.scala b/chill-scala/src/main/scala/com/twitter/chill/KryoBijection.scala
@@ -56,6 +56,43 @@ trait KryoBijection extends Bijection[AnyRef, Array[Byte]] {
  * TODO: Delete KryoBijection, use KryoInjection everywhere.
  */
 object KryoInjection extends Injection[AnyRef, Array[Byte]] {
-  override def apply(obj: AnyRef) = KryoBijection(obj)
-  override def invert(bytes: Array[Byte]) = allCatch.opt(KryoBijection.invert(bytes))
+  // Create a default injection to use, 4KB init, max 16 MB
+  private val kinject = instance(init = 1 << 12, max = 1 << 24)
+
+  override def apply(obj: AnyRef) = kinject.synchronized { kinject(obj) }
+  override def invert(bytes: Array[Byte]) = kinject.synchronized { kinject.invert(bytes) }
+
+  /**
+   * Create a new KryoInjection instance that serializes items using
+   * the supplied Kryo instance. The buffer used for serialization is
+   * initialized, by default, to 1KB, with a max size of
+   * 16MB. Configure these limits by passing in new values for "init"
+   * and "max" respectively.
+   */
+  def instance(
+    kryo: Kryo = KryoBijection.getKryo,
+    init: Int = 1 << 10,
+    max: Int = 1 << 24
+  ): Injection[AnyRef, Array[Byte]] =
+    new KryoInjectionInstance(kryo, new Output(init, max))
+}
+
+/**
+ * Reuse the Output and Kryo, which is faster
+ * register any additional serializers you need before passing in the
+ * Kryo instance
+ */
+class KryoInjectionInstance(kryo: Kryo, output: Output) extends Injection[AnyRef, Array[Byte]] {
+  private val input: Input = new Input
+
+  def apply(obj: AnyRef): Array[Byte] = {
+    output.clear
+    kryo.writeClassAndObject(output, obj)
+    output.toBytes
+  }
+
+  def invert(b: Array[Byte]): Option[AnyRef] = {
+    input.setBuffer(b)
+    allCatch.opt(kryo.readClassAndObject(input))
+  }
 }
diff --git a/chill-scala/src/main/scala/com/twitter/chill/KryoSerializer.scala b/chill-scala/src/main/scala/com/twitter/chill/KryoSerializer.scala
@@ -20,9 +20,7 @@ import com.esotericsoftware.kryo.Kryo
 import com.esotericsoftware.kryo.{ Serializer => KSerializer }
 import com.esotericsoftware.kryo.io.{ Input, Output }
 
-import com.twitter.bijection.{ Base64String, Bufferable, ImplicitBijection, Injection }
-
-import org.objenesis.strategy.StdInstantiatorStrategy
+import com.twitter.bijection.{ Bufferable, ImplicitBijection }
 
 import scala.collection.immutable.{
   BitSet,
@@ -32,14 +30,14 @@ import scala.collection.immutable.{
 }
 
 import scala.collection.mutable.{
-  Builder,
   WrappedArray,
   Map => MMap,
   Set => MSet,
   ListBuffer,
   Queue => MQueue,
   Buffer
 }
+import scala.util.matching.Regex
 
 object KryoSerializer {
 
@@ -88,13 +86,17 @@ object KryoSerializer {
      * You should go from MOST specific, to least to specific when using
      * default serializers. The FIRST one found is the one used
      */
-    // wrapper array is abstract
-    newK.forSubclass[WrappedArray[Any]](new WrappedArraySerializer[Any])
+    newK
+      .forSubclass[Regex](new RegexSerializer)
+      // wrapper array is abstract
+      .forSubclass[WrappedArray[Any]](new WrappedArraySerializer[Any])
       .forSubclass[BitSet](new BitSetSerializer)
       .forSubclass[java.util.PriorityQueue[AnyRef]](new PriorityQueueSerializer[AnyRef])
       .forTraversableSubclass(Queue.newBuilder[Any])
       // List is a sealed class, so there are only two subclasses:
       .forTraversableSubclass(List.newBuilder[Any])
+      // add mutable Buffer before Vector, otherwise Vector is used
+      .forTraversableSubclass(Buffer.newBuilder[Any], isImmutable = false)
       //Vector is a final class
       .forTraversableClass(Vector.newBuilder[Any])
       .forTraversableSubclass(IndexedSeq.newBuilder[Any])
@@ -109,7 +111,6 @@ object KryoSerializer {
       .forTraversableSubclass(MMap.newBuilder[Any,Any], isImmutable = false)
       .forTraversableSubclass(MSet.newBuilder[Any], isImmutable = false)
       .forTraversableSubclass(ListBuffer.newBuilder[Any], isImmutable = false)
-      .forTraversableSubclass(Buffer.newBuilder[Any], isImmutable = false)
       // This should be last, lots of things are seq/iterable/traversable
       // These are questionable and might break things.
       // rarely will you only expect an iterable/traversable on the reverse
diff --git a/chill-scala/src/main/scala/com/twitter/chill/RegexSerializer.scala b/chill-scala/src/main/scala/com/twitter/chill/RegexSerializer.scala
@@ -0,0 +1,32 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.twitter.chill
+
+import com.esotericsoftware.kryo.Kryo
+import com.esotericsoftware.kryo.{ Serializer => KSerializer }
+import com.esotericsoftware.kryo.io.{ Input, Output }
+import scala.util.matching.Regex
+
+class RegexSerializer extends KSerializer[Regex] {
+  def write(kser: Kryo, out: Output, obj: Regex) {
+    out.writeString(obj.pattern.pattern)
+  }
+
+  def read(kser: Kryo, in: Input, cls: Class[Regex]): Regex = {
+    new Regex(in.readString)
+  }
+}
diff --git a/chill-scala/src/main/scala/com/twitter/chill/RichKryo.scala b/chill-scala/src/main/scala/com/twitter/chill/RichKryo.scala
@@ -20,7 +20,7 @@ import com.esotericsoftware.kryo.Kryo
 import com.esotericsoftware.kryo.{ Serializer => KSerializer }
 import com.esotericsoftware.kryo.io.{ Input, Output }
 
-import com.twitter.bijection.{ Bufferable, ImplicitBijection, Injection }
+import com.twitter.bijection.{ Bufferable, Bijection, ImplicitBijection, Injection }
 
 import scala.collection.mutable.Builder
 
@@ -100,6 +100,12 @@ class RichKryo(k: Kryo) {
     k
   }
 
+  /** Helpful override to alleviate rewriting types. */
+  def forClassViaBijection[A,B](bij: Bijection[A,B])(implicit acmf: ClassManifest[A], bcmf: ClassManifest[B]): Kryo = {
+    implicit def implicitBij = bij
+    this.forClassViaBijection[A, B]
+  }
+
   /** Use Java serialization, which is very slow.
    * avoid this if possible, but for very rare classes it is probably fine
    */
diff --git a/chill-scala/src/main/scala/com/twitter/chill/Traversable.scala b/chill-scala/src/main/scala/com/twitter/chill/Traversable.scala
@@ -26,6 +26,18 @@ class TraversableSerializer[T, C <: Traversable[T]](builder: Builder[T, C],
   override val isImmutable: Boolean = true)
   extends KSerializer[C] {
 
+  sealed trait Finisher {
+    def apply(in: C): C
+  }
+  case object Clone extends Finisher {
+    def apply(in: C) = in.asInstanceOf[Cloneable[C]].clone
+  }
+  case object Pass extends Finisher {
+    def apply(in: C) = in
+  }
+
+  var finisher: Option[Finisher] = None
+
   def write(kser: Kryo, out: Output, obj: C) {
     //Write the size:
     out.writeInt(obj.size, true)
@@ -52,11 +64,16 @@ class TraversableSerializer[T, C <: Traversable[T]](builder: Builder[T, C],
 
   // TODO remove this and use CanBuildFrom from rather than Builder
   // when we bump major versions
-  protected def copyIfMutable(c: C): C = c match {
-      case m: scala.Mutable => m match {
-          case toclone: Cloneable[C] => toclone.clone
-          case _ => throw new Exception("Cannot use TraversableSerializer with non-clonable mutables")
-        }
-      case _ => c
-    }
+  protected def copyIfMutable(c: C): C = finisher.getOrElse {
+      val fin = c match {
+        case m: scala.Mutable => m match {
+            case toclone: Cloneable[C] => Clone
+            case _ => throw new Exception("Cannot use TraversableSerializer with non-clonable mutables")
+          }
+        case _ => Pass
+      }
+      // Cache this for next time
+      finisher = Some(fin)
+      fin
+    }.apply(c)
 }
diff --git a/chill-scala/src/test/scala/com/twitter/chill/BaseProperties.scala b/chill-scala/src/test/scala/com/twitter/chill/BaseProperties.scala
@@ -17,16 +17,17 @@ limitations under the License.
 package com.twitter.chill
 
 import java.io._
+import com.twitter.bijection.Injection
 
 trait BaseProperties {
-  def serialize[T](t: T): Array[Byte] = KryoBijection(t.asInstanceOf[AnyRef])
+  def serialize[T](t: T): Array[Byte] = KryoInjection(t.asInstanceOf[AnyRef])
   def deserialize[T](bytes: Array[Byte]): T =
-    KryoBijection.invert(bytes).asInstanceOf[T]
+    KryoInjection.invert(bytes).get.asInstanceOf[T]
 
-  def rt[T](t: T): T = rt[T](KryoBijection, t)
-  def rt[T](k: KryoBijection, t: T): T = {
+  def rt[T](t: T): T = rt[T](KryoInjection, t)
+  def rt[T](k: Injection[AnyRef, Array[Byte]], t: T): T = {
     val bytes = k(t.asInstanceOf[AnyRef])
-    k.invert(bytes).asInstanceOf[T]
+    k.invert(bytes).get.asInstanceOf[T]
   }
 
   // using java serialization. TODO: remove when this is shipped in bijection
diff --git a/chill-scala/src/test/scala/com/twitter/chill/CustomSerializationSpec.scala b/chill-scala/src/test/scala/com/twitter/chill/CustomSerializationSpec.scala
@@ -0,0 +1,64 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.twitter.chill
+
+import org.specs._
+
+import com.twitter.bijection.Bijection
+
+class CustomSerializationSpec extends Specification with BaseProperties {
+  "Custom KryoSerializers and KryoDeserializers" should {
+    "serialize objects that have registered serialization" in {
+      import KryoImplicits.toRich
+
+      /* These classes can be inside CustomSerializationSpec since their
+       * serialization is precisely specified. */
+      case class Point(x: Int, y: Int)
+      case class Color(name: String)
+      case class ColoredPoint(color: Color, point: Point) {
+        override def toString = color + ":" + point
+      }
+
+      // write bijections
+      implicit val pointBijection = Bijection.build[Point, (Int,Int)](
+        Point.unapply(_).get)(
+        (Point.apply _).tupled)
+      implicit val colorBijection = Bijection.build[Color, String](
+        Color.unapply(_).get)(
+        Color.apply)
+      implicit val coloredPointBijection = Bijection.build[ColoredPoint, (Color, Point)](
+        ColoredPoint.unapply(_).get)(
+        (ColoredPoint.apply _).tupled)
+
+      val myKryoInjection = KryoInjection.instance {
+        KryoBijection.getKryo
+          // use the implicit bijection by specifying the type
+          .forClassViaBijection[Point, (Int,Int)]
+          // use an explicit bijection, avoiding specifying the type
+          .forClassViaBijection(pointBijection)
+          .forClassViaBijection(colorBijection)
+          .forClassViaBijection(coloredPointBijection)
+      }
+
+      val color = Color("blue")
+      val point = Point(5, 6)
+      val coloredPoint = ColoredPoint(color, point)
+
+      rt(myKryoInjection, coloredPoint) must_== coloredPoint
+    }
+  }
+}
diff --git a/chill-scala/src/test/scala/com/twitter/chill/KryoSpec.scala b/chill-scala/src/test/scala/com/twitter/chill/KryoSpec.scala
diff --git a/project/Build.scala b/project/Build.scala

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ class InjectiveSerializer[T] private (injection: Injection[T, Array[Byte]]) exte`
`23`	`23`	`def write(kser: Kryo, out: Output, obj: T) {`
`24`	`24`	`val bytes = injection(obj)`
`25`	`25`	`out.writeInt(bytes.length, true)`
`26`		`- out.writeBytes(bytes);`
	`26`	`+ out.writeBytes(bytes)`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`def read(kser: Kryo, in: Input, cls: Class[T]): T = {`