Design and implement scala model for ncbiTaxonomy

src/main/scala/com/bio4j/dynamograph/AnyDynamoEdge.scala

@@ -3,7 +3,7 @@ package com.bio4j.dynamograph
 import com.amazonaws.services.dynamodbv2.model.AttributeValue
 import ohnosequences.scarph._
 import com.bio4j.dynamograph.dao.go.{AnyDynamoDbDao}
-import com.bio4j.dynamograph.model.GeneralSchema._
+import com.bio4j.dynamograph.model.Properties._
 
 
 trait AnyDynamoEdge extends AnyEdge { dynamoEdge =>

src/main/scala/com/bio4j/dynamograph/AnyDynamoVertex.scala

@@ -2,7 +2,7 @@ package com.bio4j.dynamograph
 
 import ohnosequences.scarph._
 import com.bio4j.dynamograph.dao.go.{AnyDynamoDbDao, DynamoDbDao}
-import com.bio4j.dynamograph.model.GeneralSchema.id
+import com.bio4j.dynamograph.model.Properties.id
 import com.amazonaws.services.dynamodbv2.model.AttributeValue
 import scala.collection.JavaConverters._
 

src/main/scala/com/bio4j/dynamograph/mapper/GoMapper.scala

@@ -1,7 +1,7 @@
 package com.bio4j.dynamograph.mapper
 
 import com.bio4j.dynamograph.model.go.GoImplementation._
-import com.bio4j.dynamograph.model.GeneralSchema.{relationId, targetId, sourceId, id}
+import com.bio4j.dynamograph.model.Properties.{relationId, targetId, sourceId, id}
 import com.bio4j.dynamograph.parser.{ParsingContants, SingleElement}
 import com.amazonaws.services.dynamodbv2.model.{PutItemRequest, AttributeValue}
 import com.bio4j.dynamograph.model.go.GoSchema._

src/main/scala/com/bio4j/dynamograph/mapper/NcbiTaxonomyMapper.scala

@@ -0,0 +1,10 @@
+package com.bio4j.dynamograph.mapper
+
+import com.bio4j.dynamograph.parser.SingleElement
+import com.amazonaws.services.dynamodbv2.model.PutItemRequest
+
+
+class NcbiTaxonomyMapper extends AnyMapper {
+  override def map(element: SingleElement): List[PutItemRequest] = ???
+
+}

src/main/scala/com/bio4j/dynamograph/model/Properties.scala

@@ -0,0 +1,19 @@
+package com.bio4j.dynamograph.model
+
+import ohnosequences.tabula.Attribute
+
+object Properties {
+
+  case object id              extends Attribute[String]
+  case object nodeId          extends Attribute[String]
+  case object relationId      extends Attribute[String]
+  case object sourceId        extends Attribute[String]
+  case object targetId        extends Attribute[String]
+
+  case object name            extends Attribute[String]
+  case object definition      extends Attribute[String]
+  case object comment         extends Attribute[String]
+  case object number          extends Attribute[Int]
+  case object scientificName  extends Attribute[String]
+
+ }

src/main/scala/com/bio4j/dynamograph/model/go/GoSchema.scala

@@ -1,15 +1,12 @@
 package com.bio4j.dynamograph.model.go
 
 import ohnosequences.scarph._
-import ohnosequences.tabula.Attribute
-import com.bio4j.dynamograph.model.GeneralSchema.id
+import com.bio4j.dynamograph.model.Properties._
 
 object GoSchema {
 
   // Properties
-  case object name extends Attribute[String]
-  case object definition extends Attribute[String]
-  case object comment extends Attribute[String]
+
 
 
 

src/main/scala/com/bio4j/dynamograph/model/go/TableGoSchema.scala

@@ -2,7 +2,8 @@ package com.bio4j.dynamograph.model.go
 
 import ohnosequences.tabula._
 import com.bio4j.dynamograph.{AnyDynamoVertex, AnyDynamoEdge}
-import com.bio4j.dynamograph.model.GeneralSchema._
+ import com.bio4j.dynamograph.model.Properties._
+import shapeless._
 
 
 object TableGoSchema {
@@ -27,6 +28,8 @@ object TableGoSchema {
     case object edgeTable extends HashKeyTable(tablaName, relationId, region)
 
     type EdgeTpe = et.type
+
+    val tables = inTable :: outTable :: edgeTable :: HNil
   }
 
 }

src/main/scala/com/bio4j/dynamograph/model/taxonomy/ModelInitializer.scala

@@ -0,0 +1,41 @@
+package com.bio4j.dynamograph.model.taxonomy
+
+import com.bio4j.dynamograph.ServiceProvider
+import shapeless.Poly1
+import ohnosequences.tabula._
+import ohnosequences.tabula.InitialState
+import ohnosequences.tabula.InitialThroughput
+import ohnosequences.tabula.CreateTable
+
+//Temporal object - until refactoring/recomendations branch will be merged
+object ModelInitializer {
+
+  val service = ServiceProvider.service
+
+  object createTable extends Poly1{
+    implicit def caseAnyTable[T <: Singleton with AnyTable, E <: Executor.For[CreateTable[T]]]
+    (implicit exec: CreateTable[T] => E) =
+      at[T](t => service please CreateTable(t, InitialState(t, service.account, InitialThroughput(1,1))))
+  }
+
+  object deleteTable extends Poly1{
+    implicit def caseAnyTable[T <: Singleton with AnyTable, E <: Executor.For[DeleteTable[T]]]
+    (implicit exec: DeleteTable[T] => E) =
+      at[T](t => service please DeleteTable(t, Active(t, service.account, InitialThroughput(1,1))))
+  }
+
+  def initialize(): Unit = {
+    import ServiceProvider.executors._
+
+    TableSchema.vertexTables map createTable
+    TableSchema.edgeTables map createTable
+  }
+
+  def clear(): Unit = {
+    import ServiceProvider.executors._
+
+    TableSchema.vertexTables map deleteTable
+    TableSchema.edgeTables map deleteTable
+  }
+
+}

src/main/scala/com/bio4j/dynamograph/model/taxonomy/NcbiTaxonomyImplementation.scala

@@ -0,0 +1,16 @@
+package com.bio4j.dynamograph.model.taxonomy
+
+import com.bio4j.dynamograph.{DynamoEdge, DynamoVertex}
+import com.bio4j.dynamograph.model.taxonomy.NcbiTaxonomySchema._
+
+
+object NcbiTaxonomyImplementation {
+
+  case object NcbiTaxon     extends DynamoVertex(NcbiTaxonType)
+  case object NcbiRank      extends DynamoVertex(NcbiRankType)
+
+  case object Parent        extends DynamoEdge(NcbiTaxon, ParentType, NcbiTaxon)
+  case object AssignedRank  extends DynamoEdge(NcbiTaxon, AssignedRankType, NcbiRank)
+  case object SubRank       extends DynamoEdge(NcbiRank, SubrankType, NcbiRank)
+
+}

src/main/scala/com/bio4j/dynamograph/model/taxonomy/NcbiTaxonomySchema.scala

@@ -0,0 +1,24 @@
+package com.bio4j.dynamograph.model.taxonomy
+
+import ohnosequences.scarph.{ManyToOne, VertexType}
+import com.bio4j.dynamograph.model.Properties._
+
+
+object NcbiTaxonomySchema {
+
+  case object NcbiTaxonType     extends VertexType("NcbiTaxon")
+  implicit val NcbiTaxonType_id                = NcbiTaxonType has id
+  implicit val NcbiTaxonType_name              = NcbiTaxonType has name
+  implicit val NcbiTaxonType_comment           = NcbiTaxonType has comment
+  implicit val NcbiTaxonType_scientificName    = NcbiTaxonType has scientificName
+
+  case object NcbiRankType      extends VertexType("Rank")
+  implicit val RankType_name                   = NcbiRankType has name
+  implicit val RankType_number                 = NcbiRankType has number
+
+  // edges
+  case object ParentType         extends ManyToOne(NcbiTaxonType, "Parent",       NcbiTaxonType)
+  case object AssignedRankType   extends ManyToOne(NcbiTaxonType, "AssignedRank", NcbiRankType)
+  case object SubrankType        extends ManyToOne(NcbiRankType,  "Subrank",      NcbiRankType)
+
+ }

src/main/scala/com/bio4j/dynamograph/model/taxonomy/TableSchema.scala

@@ -0,0 +1,21 @@
+package com.bio4j.dynamograph.model.taxonomy
+
+import com.bio4j.dynamograph.model.go.TableGoSchema.{EdgeTables, VertexTable}
+import com.bio4j.dynamograph.model.taxonomy.NcbiTaxonomyImplementation._
+import ohnosequences.tabula.EU
+import shapeless._
+import javafx.scene.effect.BlendMode
+
+object TableSchema {
+  case object TaxonTable          extends VertexTable (NcbiTaxon,     "NcbiTaxon",   EU)
+  case object RankTable           extends VertexTable (NcbiRank,      "NcbiRank",    EU)
+
+  case object AssignedRankTables  extends EdgeTables  (AssignedRank, "AssignedRank", EU)
+  case object SubrankTables       extends EdgeTables  (SubRank,      "Subrank",      EU)
+  case object ParentTables        extends EdgeTables  (Parent,       "TaxonParent",  EU)
+
+
+  val vertexTables = TaxonTable :: RankTable :: HNil
+
+  val edgeTables = AssignedRankTables.tables ::: SubrankTables.tables ::: ParentTables.tables
+}

src/main/scala/com/bio4j/dynamograph/parser/PullGoParser.scala

@@ -2,7 +2,7 @@ package com.bio4j.dynamograph.parser
 
 import scala.xml.pull._
 import scala.io.Source
-import com.bio4j.dynamograph.model.GeneralSchema.{id, targetId}
+import com.bio4j.dynamograph.model.Properties._
 import com.bio4j.dynamograph.model.go.GoSchema._
 import scala.xml.MetaData
 
@@ -35,7 +35,7 @@ class PullGoParser(val src: Source) extends AnyGoParser {
         case _ => ()
       }
     }
-    SingleElement(vertex, edges)
+    SingleElement(GoTermType.label, vertex, edges)
   }
 
   private def skip(label: String,parser: XMLEventReader) = {

src/main/scala/com/bio4j/dynamograph/parser/SingleElement.scala

@@ -1,4 +1,4 @@
 package com.bio4j.dynamograph.parser
 
 
-case class SingleElement(val vertexAttributes: Map[String,String], val edges: List[Map[String,String]])
+case class SingleElement(val vertexType : String, val vertexAttributes: Map[String,String], val edges: List[Map[String,String]])

src/main/scala/com/bio4j/dynamograph/parser/TaxonomyParser.scala

@@ -0,0 +1,67 @@
+package com.bio4j.dynamograph.parser
+
+import scala.io.Source
+import scala.annotation.tailrec
+import com.bio4j.dynamograph.model.Properties._
+import com.bio4j.dynamograph.model.taxonomy.NcbiTaxonomySchema._
+import com.amazonaws.services.cloudfront.model.InvalidArgumentException
+
+class TaxonomyParser(val nodesSrc: Source,val namesSrc: Source, val rankSrc : Source) extends AnyGoParser {
+
+  val nameIt = namesSrc.getLines()
+
+  override def foreach[U](f: (SingleElement) => U): Unit = {
+
+    val ranks = parseRanks(rankSrc.getLines)
+    ranks.foreach(x => f(toSingleElement(ranks, x)))
+    for (line <- nodesSrc.getLines){
+        f(processLine(line))
+    }
+  }
+
+  private def parseRanks(lines : Iterator[String]) : Map[Int,String] = {
+
+    @tailrec
+    def parserRanksHelper(lines: Iterator[String], result : Map[Int,String]) : Map[Int,String] = {
+      if (!lines.hasNext) result
+      else {
+        val line = lines.next
+        parserRanksHelper(lines, result +  parseRankEntry(line))
+      }
+    }
+
+    parserRanksHelper(lines,Map())
+  }
+
+  def toSingleElement(ranks: Map[Int, String],rank: (Int, String)) : SingleElement = {
+    val vertex = Map(name.label -> rank._2, number.label -> rank._1.toString)
+    val relations = ranks.filterKeys(_ > rank._1).map(x =>
+      Map(targetId.label -> x._2.trim.toString, ParsingContants.relationType -> SubrankType.label)
+    ).toList
+    SingleElement(NcbiRankType.label, vertex, relations)
+  }
+
+  private def parseRankEntry(line: String) = {
+    val splitResult = line.split("\\|")
+    splitResult(0).trim.toInt -> splitResult(1).trim
+  }
+
+  private def processLine(line : String) : SingleElement = {
+    def scientific(id: String, it : Iterator[String]) : String = {
+      val tId = id.toInt
+      val nameColumns = it.next().split("\\|")
+      if (!nameColumns(0).trim().equals(id)) throw new InvalidArgumentException("")
+      nameColumns(1).trim
+    }
+    val columns = line.split("\\|");
+    val taxonId = columns(0).trim
+    val parentId = columns(1).trim
+    val rank = columns(2).trim
+    val taxComment = if (columns.length < 13) ""  else columns(12).trim
+    val sName = scientific(taxonId, nameIt)
+    val vertex = Map(id.label -> taxonId, comment.label -> taxComment, scientificName.label -> sName)
+    val relationships = List(Map(targetId.label -> parentId, ParsingContants.relationType -> ParentType.label),
+                             Map(targetId.label -> rank,     ParsingContants.relationType -> AssignedRankType.label))
+    SingleElement(NcbiTaxonType.label, vertex, relationships)
+  }
+}

src/main/scala/com/bio4j/dynamograph/reader/AnyReader.scala

@@ -1,7 +1,7 @@
 package com.bio4j.dynamograph.reader
 
 import com.bio4j.dynamograph.{AnyDynamoEdge, AnyDynamoVertex}
-import com.bio4j.dynamograph.model.GeneralSchema.id
+import com.bio4j.dynamograph.model.Properties.id
 
 trait AnyReader {
   type returnType

src/main/scala/com/bio4j/dynamograph/reader/EdgeReader.scala

@@ -4,7 +4,7 @@ import com.bio4j.dynamograph.model.go.TableGoSchema.EdgeTables
 import ohnosequences.tabula.AnyRegion
 import com.bio4j.dynamograph.AnyDynamoEdge
 import com.amazonaws.services.dynamodbv2.model._
-import com.bio4j.dynamograph.model.GeneralSchema.{nodeId, id, relationId}
+import com.bio4j.dynamograph.model.Properties.{nodeId, id, relationId}
 import com.bio4j.dynamograph.dynamodb.AnyDynamoDbExecutor
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._

src/main/scala/com/bio4j/dynamograph/reader/VertexReader.scala

@@ -4,7 +4,7 @@ import com.amazonaws.services.dynamodbv2.model.{AttributeValue, GetItemRequest}
 import com.bio4j.dynamograph.model.go.TableGoSchema.VertexTable
 import ohnosequences.tabula.AnyRegion
 import com.bio4j.dynamograph.AnyDynamoVertex
-import com.bio4j.dynamograph.model.GeneralSchema.id
+import com.bio4j.dynamograph.model.Properties.id
 import scala.collection.JavaConversions._
 import com.bio4j.dynamograph.dynamodb.AnyDynamoDbExecutor
 

src/main/scala/com/bio4j/dynamograph/writer/EdgeWriter.scala

@@ -5,7 +5,7 @@ import com.bio4j.dynamograph.model.go.TableGoSchema.EdgeTables
 import ohnosequences.tabula.AnyRegion
 import com.amazonaws.services.dynamodbv2.model.AttributeValue
 import scala.collection.JavaConversions._
-import com.bio4j.dynamograph.model.GeneralSchema._
+import com.bio4j.dynamograph.model.Properties._
 import com.bio4j.dynamograph.AnyDynamoEdge
 
 

src/main/scala/com/bio4j/dynamograph/writer/TaxonomyWriters.scala

@@ -0,0 +1,28 @@
+package com.bio4j.dynamograph.writer
+
+import com.bio4j.dynamograph.model.taxonomy.NcbiTaxonomyImplementation._
+import com.bio4j.dynamograph.model.taxonomy.TableSchema._
+import com.bio4j.dynamograph.model.taxonomy.NcbiTaxonomySchema._
+
+
+object TaxonomyWriters {
+
+  case object TaxonWriter             extends VertexWriter(NcbiTaxon, TaxonTable)
+  case object RankWriter              extends VertexWriter(NcbiRank, RankTable)
+
+  case object ParentEdgeWriter        extends EdgeWriter(Parent, ParentTables)
+  case object AssignedRankEdgeWriter  extends EdgeWriter(AssignedRank, AssignedRankTables)
+  case object SubrankEdgeWriter       extends EdgeWriter(SubRank, SubrankTables)
+
+  val vertexWritersMap = Map[String, AnyVertexWriter] (
+    NcbiTaxonType.label           -> TaxonWriter,
+    NcbiRankType.label            -> RankWriter
+  )
+
+  val edgeWritersMap = Map[String, AnyEdgeWriter] (
+    ParentType.label              -> ParentEdgeWriter,
+    AssignedRankType.label        -> AssignedRankEdgeWriter,
+    SubrankType.label             -> SubrankEdgeWriter
+  )
+
+}