handle segment granularity changes

Author: pjain1

core/src/main/scala/com/metamx/tranquility/beam/Beam.scala

@@ -17,12 +17,13 @@
 package com.metamx.tranquility.beam
 
 import com.twitter.util.Future
+import org.joda.time.Interval
 
 /**
  * Beams can accept events and forward them along. The propagate method may throw a DefunctBeamException, which means
  * the beam should be discarded (after calling close()).
  */
-trait Beam[A]
+trait Beam[A] extends DiscoverableInterval
 {
   /**
    * Request propagation of events. The operation may fail in various ways, which tend to be specific to
@@ -40,3 +41,11 @@ class DefunctBeamException(s: String, t: Throwable) extends Exception(s, t)
 {
   def this(s: String) = this(s, null)
 }
+
+trait DiscoverableInterval
+{
+  /**
+   * Returns the interval handled by the Beam, can return None if there is no associated interval
+   * */
+  def getInterval(): Option[Interval]
+}

core/src/main/scala/com/metamx/tranquility/beam/ClusteredBeam.scala

@@ -58,6 +58,9 @@ class HttpBeam[A: Timestamper](
   emitter: ServiceEmitter
 ) extends Beam[A] with Logging
 {
+
+  def getInterval() = None
+
   private[this] implicit val timer: Timer = DefaultTimer.twitter
 
   private[this] val port = if (uri.port > 0) {

core/src/main/scala/com/metamx/tranquility/beam/HttpBeam.scala

@@ -28,6 +28,8 @@ class MemoryBeam[A](
   jsonWriter: JsonWriter[A]
 ) extends Beam[A]
 {
+  def getInterval() = None
+
   def propagate(events: Seq[A]) = {
     events.map(event => Jackson.parse[Dict](jsonWriter.asBytes(event))) foreach {
       d =>

core/src/main/scala/com/metamx/tranquility/beam/MemoryBeam.scala

@@ -44,5 +44,12 @@ class MergingPartitioningBeam[A](
     Future.collect(beams map (_.close())) map (_ => ())
   }
 
+  def getInterval() = {
+    beams.headOption match {
+      case Some(x) => x.getInterval()
+      case None => None
+    }
+  }
+
   override def toString = s"MergingPartitioningBeam(${beams.mkString(", ")})"
 }

core/src/main/scala/com/metamx/tranquility/beam/MergingPartitioningBeam.scala

@@ -17,6 +17,8 @@
 package com.metamx.tranquility.beam
 
 import com.twitter.util.Future
+import org.joda.time.Interval
+import org.joda.time.chrono.ISOChronology
 
 class NoopBeam[A] extends Beam[A]
 {
@@ -25,4 +27,6 @@ class NoopBeam[A] extends Beam[A]
   def close() = Future.Done
 
   override def toString = "NoopBeam()"
+
+  def getInterval() = Some(new Interval(0, 0, ISOChronology.getInstanceUTC))
 }

core/src/main/scala/com/metamx/tranquility/beam/NoopBeam.scala

@@ -38,4 +38,11 @@ class RoundRobinBeam[A](
   }
 
   override def toString = "RoundRobinBeam(%s)" format beams.mkString(", ")
+
+  def getInterval() = {
+    beams.headOption match {
+      case Some(x) => x.getInterval()
+      case None => None
+    }
+  }
 }

core/src/main/scala/com/metamx/tranquility/beam/RoundRobinBeam.scala

@@ -19,8 +19,7 @@ package com.metamx.tranquility.druid
 import com.metamx.common.scala.Logging
 import com.metamx.common.scala.Predef._
 import com.metamx.emitter.service.ServiceEmitter
-import com.metamx.tranquility.beam.Beam
-import com.metamx.tranquility.beam.DefunctBeamException
+import com.metamx.tranquility.beam.{Beam, DefunctBeamException}
 import com.metamx.tranquility.finagle._
 import com.metamx.tranquility.typeclass.ObjectWriter
 import com.twitter.util.Closable
@@ -44,6 +43,9 @@ class DruidBeam[A](
   objectWriter: ObjectWriter[A]
 ) extends Beam[A] with Logging with Closable
 {
+
+  def getInterval() = Some(interval)
+
   private[this] val clients = Map(
     tasks map {
       task =>

core/src/main/scala/com/metamx/tranquility/druid/DruidBeam.scala

@@ -138,10 +138,6 @@ class DruidBeamMaker[A: Timestamper](
   }
 
   override def newBeam(interval: Interval, partition: Int) = {
-    require(
-      beamTuning.segmentGranularity.widen(interval) == interval,
-      "Interval does not match segmentGranularity[%s]: %s" format(beamTuning.segmentGranularity, interval)
-    )
     val availabilityGroup = DruidBeamMaker.generateBaseFirehoseId(
       location.dataSource,
       beamTuning.segmentGranularity,
@@ -190,10 +186,7 @@ class DruidBeamMaker[A: Timestamper](
       // Backwards compatibility (see toDict).
       beamTuning.segmentBucket(new DateTime(d("timestamp"), ISOChronology.getInstanceUTC))
     }
-    require(
-      beamTuning.segmentGranularity.widen(interval) == interval,
-      "Interval does not match segmentGranularity[%s]: %s" format(beamTuning.segmentGranularity, interval)
-    )
+
     val partition = int(d("partition"))
     val tasks = if (d contains "tasks") {
       list(d("tasks")).map(dict(_)).map(d => TaskPointer(str(d("id")), str(d("firehoseId"))))

core/src/main/scala/com/metamx/tranquility/druid/DruidBeamMaker.scala

@@ -221,6 +221,9 @@ object DruidBeams
         def close() = clusteredBeam.close() map (_ => lifecycle.stop())
 
         override def toString = clusteredBeam.toString
+
+        def getInterval() = clusteredBeam.getInterval()
+
       }
     }