[WIP] Latis-1079-DODS

core/src/main/scala/latis/output/BinaryEncoder.scala

@@ -31,22 +31,6 @@ class BinaryEncoder(val dataCodec: Scalar => Codec[Data] = DataCodec.defaultData
   def sampleStreamEncoder(model: DataType): StreamEncoder[Sample] =
     StreamEncoder.many(sampleCodec(model))
 
-  private def codecOfList[A](cs: List[Codec[A]]): Codec[List[A]] = new Codec[List[A]] {
-
-    override def sizeBound: SizeBound =
-      cs.map(_.sizeBound).foldLeft(SizeBound.exact(0))(_ + _)
-
-    override def decode(bits: BitVector): Attempt[DecodeResult[List[A]]] =
-      cs.traverse(_.asDecoder).decode(bits)
-
-    override def encode(value: List[A]): Attempt[BitVector] =
-      if (cs.length == value.length) {
-        cs.zip(value).foldMapM { case (c, v) => c.encode(v) }
-      } else {
-        Attempt.failure(Err("wrong length"))
-      }
-  }
-
   def sampleCodec(model: DataType): Codec[Sample] = {
     val (domainScalars: List[Scalar], rangeScalars: List[Scalar]) = model match {
       case s: Scalar =>
@@ -59,10 +43,26 @@ class BinaryEncoder(val dataCodec: Scalar => Codec[Data] = DataCodec.defaultData
     }
     val domainList: List[Codec[Datum]] = domainScalars.map(s => dataCodec(s).downcast[Datum])
     val rangeList: List[Codec[Data]] = rangeScalars.map(s => dataCodec(s))
-    codecOfList(domainList) ~ codecOfList(rangeList)
+    BinaryEncoder.codecOfList(domainList) ~ BinaryEncoder.codecOfList(rangeList)
   }
 }
 
 object BinaryEncoder {
   def apply(dataCodec: Scalar => Codec[Data] = DataCodec.defaultDataCodec): BinaryEncoder = new BinaryEncoder(dataCodec)
+
+  def codecOfList[A](cs: List[Codec[A]]): Codec[List[A]] = new Codec[List[A]] {
+
+    override def sizeBound: SizeBound =
+      cs.map(_.sizeBound).foldLeft(SizeBound.exact(0))(_ + _)
+
+    override def decode(bits: BitVector): Attempt[DecodeResult[List[A]]] =
+      cs.traverse(_.asDecoder).decode(bits)
+
+    override def encode(value: List[A]): Attempt[BitVector] =
+      if (cs.length == value.length) {
+        cs.zip(value).foldMapM { case (c, v) => c.encode(v) }
+      } else {
+        Attempt.failure(Err("wrong length"))
+      }
+  }
 }

dap2-service/src/main/scala/latis/service/dap2/AtomicType.scala

@@ -1,42 +1,87 @@
 package latis.service.dap2
 
 import java.net.URL
+import java.nio.charset.StandardCharsets
 
-import latis.util.StringUtils._
-
-sealed trait AtomicType[F] {
-  private val fmtInt: AnyVal => String = i => i.toString
-  private val fmtFloat: Double => String = f => f"$f%.6g"
-  private val fmtString: String => String = ensureQuotedAndEscaped
-
-  def ofValue(value: F): F = value
-  def asDasString(value: F): String = this.ofValue(value) match {
-      case int: Byte => fmtInt(int)
-      case int: Short => fmtInt(int)
-      case int: Int => fmtInt(int)
-      case int: Long => fmtInt(int)
-      case float: Float => fmtFloat(float.toDouble)
-      case float: Double => fmtFloat(float)
-      case string: String => fmtString(string)
-      case string: URL => fmtString(string.toString)
-      case other => other.toString
-  }
+import scodec._
+import scodec.bits.BitVector
+import scodec.bits.ByteVector
+
+import latis.model._
+import latis.service.dap2.AtomicTypeValue._
+import latis.util.LatisException
+
+sealed trait AtomicType[F<:AtomicTypeValue[_]] {
+  protected val vcodec: Codec[F]
+  protected def removePad(paddedBytes: Array[Byte]): Array[Byte] = paddedBytes
+  private def addPad(pureBytes: Array[Byte]): Array[Byte] =
+    pureBytes.padTo(pureBytes.length + (4 - pureBytes.length % 4) % 4, 0x00.toByte)
+  private val padCodec = codecs.bytes.xmap[Array[Byte]](
+    bv => removePad(bv.toArray),
+    arr => ByteVector(addPad(arr))
+  )
+
+  final def codec: Codec[F] = padCodec.xmap[F](
+    bytes => vcodec.decode(BitVector(bytes)).getOrElse(
+      return codecs.fail(Err("Value codec decoding failed"))
+    ).value,
+    value => vcodec.encode(value).getOrElse(
+      return codecs.fail(Err("Value codec encoding failed"))
+    ).toByteArray
+  )
 }
 
 object AtomicType {
-  final case object Byte extends AtomicType[Byte]
-  final case object Int16 extends AtomicType[Short]
-  final case object UInt16 extends AtomicType[Int] {
-    val max: Int = Integer.parseInt("FFFF",16)
-    override def ofValue(value: Int): Int = value.min(max).max(0)
-  }
-  final case object Int32 extends AtomicType[Int]
-  final case object UInt32 extends AtomicType[Long] {
-    val max: Long = java.lang.Long.parseLong("FFFFFFFF",16)
-    override def ofValue(value: Long): Long = value.min(max).max(0)
-  }
-  final case object Float32 extends AtomicType[Float]
-  final case object Float64 extends AtomicType[Double]
-  final case object String extends AtomicType[String]
-  final case object Url extends AtomicType[URL]
+  final case object Byte extends AtomicType[ByteValue] {
+    override protected val vcodec: Codec[ByteValue] = codecs.byte.xmap(ByteValue, _.value)
+    override protected def removePad(paddedBytes: Array[Byte]): Array[Byte] = paddedBytes.slice(0, 1)
+  }
+  final case object Int16 extends AtomicType[Int16Value] {
+    override protected val vcodec: Codec[Int16Value] = codecs.short16.xmap(Int16Value, _.value)
+    override protected def removePad(paddedBytes: Array[Byte]): Array[Byte] = paddedBytes.slice(0, 2)
+  }
+  final case object UInt16 extends AtomicType[UInt16Value] {
+    override protected val vcodec: Codec[UInt16Value] = codecs.uint16.xmap(UInt16Value, _.value)
+  }
+  final case object Int32 extends AtomicType[Int32Value] {
+    override protected val vcodec: Codec[Int32Value] = codecs.int32.xmap(Int32Value, _.value)
+  }
+  final case object UInt32 extends AtomicType[UInt32Value] {
+    override protected val vcodec: Codec[UInt32Value] = codecs.uint32.xmap(UInt32Value, _.value)
+  }
+  final case object Float32 extends AtomicType[Float32Value] {
+    override protected val vcodec: Codec[Float32Value] = codecs.float.xmap(Float32Value, _.value)
+  }
+  final case object Float64 extends AtomicType[Float64Value] {
+    override protected val vcodec: Codec[Float64Value] = codecs.double.xmap(Float64Value, _.value)
+  }
+  final case object String extends AtomicType[StringValue] {
+    override protected val vcodec: Codec[StringValue] = codecs.string(StandardCharsets.UTF_8).xmap(StringValue, _.value)
+    override protected def removePad(paddedBytes: Array[Byte]): Array[Byte] = {
+      val len = paddedBytes.length
+      val suffix = paddedBytes.indexOf(0x00.toByte)
+      paddedBytes.slice(0, if (suffix > 0) suffix else len)
+    }
+  }
+  final case object Url extends AtomicType[UrlValue] {
+    override protected val vcodec: Codec[UrlValue] = codecs.string(StandardCharsets.UTF_8)
+      .xmap[URL](str => new URL(str), url => url.toString)
+      .xmap(UrlValue, _.value)
+
+    override protected def removePad(paddedBytes: Array[Byte]): Array[Byte] = {
+      val len = paddedBytes.length
+      val suffix = paddedBytes.indexOf(0x00.toByte)
+      paddedBytes.slice(0, if (suffix > 0) suffix else len)
+    }
+  }
+
+  def fromScalar[AT<:AtomicType[ATV],ATV<:AtomicTypeValue[_]](scalar: Scalar): Either[LatisException, AT] = scalar.valueType match {
+    case DoubleValueType => Right(Float64.asInstanceOf[AT])
+    case FloatValueType => Right(Float32.asInstanceOf[AT])
+    case IntValueType => Right(Int32.asInstanceOf[AT])
+    case ShortValueType => Right(Int16.asInstanceOf[AT])
+    case ByteValueType => Right(Byte.asInstanceOf[AT])
+    case StringValueType => Right(String.asInstanceOf[AT])
+    case _ => Left(LatisException("Scalar could not be parsed to a DDS atomic type."))
+  }
 }

dap2-service/src/main/scala/latis/service/dap2/AtomicTypeValue.scala

@@ -0,0 +1,62 @@
+package latis.service.dap2
+
+import java.net.URL
+
+import latis.data.Data
+import latis.data.Data._
+import latis.data.Datum
+import latis.util.LatisException
+import latis.util.StringUtils.ensureQuotedAndEscaped
+
+sealed trait AtomicTypeValue[+T] {
+  val value: T
+  val dasStr: String
+}
+object AtomicTypeValue {
+  private val fmtInt: AnyVal => String = i => i.toString
+  private val fmtFloat: Double => String = f => f"$f%.6g"
+  private val fmtString: String => String = ensureQuotedAndEscaped
+
+  final case class ByteValue(override val value: Byte) extends AtomicTypeValue[Byte] {
+    override val dasStr: String = fmtInt(value)
+  }
+  final case class Int16Value(override val value: Short) extends AtomicTypeValue[Short] {
+    override val dasStr: String = fmtInt(value)
+  }
+  final case class UInt16Value(_value: Int) extends AtomicTypeValue[Int] {
+    val max: Int = Integer.parseInt("FFFF", 16)
+    override val value: Int = _value.min(max).max(0)
+    override val dasStr: String = fmtInt(value)
+  }
+  final case class Int32Value(value: Int) extends AtomicTypeValue[Int] {
+    override val dasStr: String = fmtInt(value)
+  }
+  final case class UInt32Value(_value: Long) extends AtomicTypeValue[Long] {
+    val max: Long = java.lang.Long.parseLong("FFFFFFFF", 16)
+    override val value: Long = _value.min(max).max(0)
+    override val dasStr: String = fmtInt(value)
+  }
+  final case class Float32Value(override val value: Float) extends AtomicTypeValue[Float] {
+    override val dasStr: String = fmtFloat(value.toDouble)
+  }
+  final case class Float64Value(override val value: Double) extends AtomicTypeValue[Double] {
+    override val dasStr: String = fmtFloat(value)
+  }
+  final case class StringValue(override val value: String) extends AtomicTypeValue[String] {
+    override val dasStr: String = fmtString(value)
+  }
+
+  final case class UrlValue(override val value: URL) extends AtomicTypeValue[URL] {
+    override val dasStr: String = fmtString(value.toString)
+  }
+
+  def fromDatum(datum: Datum): Either[LatisException, AtomicTypeValue[Any]] = datum match {
+    case double: DoubleValue => Right(Float64Value(double.value))
+    case float: FloatValue => Right(Float32Value(float.value))
+    case int: IntValue => Right(Int32Value(int.value))
+    case short: ShortValue => Right(Int16Value(short.value))
+    case byte: Data.ByteValue => Right(ByteValue(byte.value))
+    case str: Data.StringValue => Right(StringValue(str.value))
+    case _ => Left(LatisException("Datum could not be parsed to a DDS atomic type."))
+  }
+}

dap2-service/src/main/scala/latis/service/dap2/Dap2Service.scala

@@ -159,6 +159,8 @@ class Dap2Service(catalog: Catalog) extends ServiceInterface(catalog) with Http4
       .map((_, Headers(Raw(ci"Content-Type", "text/plain"), Raw(ci"Content-Description", "dods-das"))))
     case "dds"   => new DdsEncoder().encode(ds).through(text.utf8.encode).asRight
       .map((_, Headers(Raw(ci"Content-Type", "text/plain"), Raw(ci"Content-Description", "dods-dds"))))
+    case "dods" => new DataDdsEncoder().encode(ds).asRight
+      .map((_, Headers(Raw(ci"Content-Type", "application/octet-stream"), Raw(ci"Content-Description", "dods-data"))))
     case "jsonl" => new JsonEncoder().encode(ds).map(_.noSpaces).intersperse("\n").through(text.utf8.encode).asRight
       .map((_, Headers(Raw(ci"Content-Type", "application/jsonl"))))
     case "meta"  => new MetadataEncoder().encode(ds).map(_.noSpaces).through(text.utf8.encode).asRight

dap2-service/src/main/scala/latis/service/dap2/Das.scala

@@ -5,6 +5,7 @@ import org.typelevel.paiges.Doc
 
 import latis.dataset.Dataset
 import latis.model._
+import latis.service.dap2.AtomicTypeValue.StringValue
 import latis.util.Identifier
 import latis.util.Identifier.IdentifierStringContext
 
@@ -47,9 +48,9 @@ object Das {
     }
   }
 
-  case class Attribute[A<:AtomicType[F],F](id: Identifier, ty: A, values: NonEmptyList[F]) {
+  case class Attribute[A<:AtomicType[F],F<:AtomicTypeValue[_]](id: Identifier, ty: A, values: NonEmptyList[F]) {
     val toDoc: Doc = Doc.str(ty) + Doc.space + Doc.text(id.asString) + Doc.space +
-      Doc.text(values.tail.foldLeft(ty.asDasString(values.head))((acc, v) => acc + ", " + ty.asDasString(v))) +
+      Doc.text(values.tail.foldLeft(values.head.dasStr)((acc, v) => acc + ", " + v.dasStr)) +
       Doc.char(';')
   }
 
@@ -60,7 +61,7 @@ object Das {
         Attribute(
           Identifier.fromString(prop._1).get, // Should be safe, since the Identifier is being constructed from a source with a valid Identifier
           AtomicType.String,
-          NonEmptyList.one(prop._2)
+          NonEmptyList.one(StringValue(prop._2))
         )
       }
     )