Fixed two filtering bugs (#1991)

Author: stonesmi

toolbox/.gitignore

@@ -1,2 +1,4 @@
 target
-target/**
+target/**
+
+test-graphviz.txt

toolbox/src/main/scala/org/finos/toolbox/collection/ChunkSize.scala

@@ -0,0 +1,12 @@
+package org.finos.toolbox.collection
+
+object ChunkSize {
+
+  def from(hint: Int): Int = {
+    hint match
+      case h if h < 100_000 => 5_000
+      case h if h < 9_000_000 => 50_000
+      case _ => 100_000
+  }
+  
+}

toolbox/src/main/scala/org/finos/toolbox/collection/array/ChunkedImmutableArray.scala

@@ -1,39 +1,49 @@
 package org.finos.toolbox.collection.array
 
+import org.finos.toolbox.collection.{ChunkSize, array}
+
 import java.util
 import scala.reflect.ClassTag
 import scala.util.control.Breaks
 
-object ChunkedImmutableArray{
+object ChunkedImmutableArray {
+
+  def from[T <: Object : ClassTag](arr: Array[T]): ImmutableArray[T] = {
+    val chunkSize = ChunkSize.from(arr.length)
+    new ChunkedImmutableArray[T](chunks = Array(), chunkSize = chunkSize).fromArray(arr)
+  }
 
-  def fromArray[T <: Object:ClassTag](arr: Array[T], chunkSize: Int): ImmutableArray[T] = {
-    new ChunkedImmutableArray[T](Array(), chunkSize = chunkSize).fromArray(arr)
+  def from[T <: Object:ClassTag](arr: Array[T], chunkSize: Int): ImmutableArray[T] = {
+    new ChunkedImmutableArray[T](chunks = Array(), chunkSize = chunkSize).fromArray(arr)
   }
 
-  def empty[T <: Object:ClassTag](chunkSize: Int = 1000): ImmutableArray[T] = {
-    new ChunkedImmutableArray[T](Array(), chunkSize = chunkSize)
+  def empty[T <: Object:ClassTag](): ImmutableArray[T] = {
+    val chunkSize = ChunkSize.from(0)
+    new ChunkedImmutableArray[T](chunks = Array(), chunkSize = chunkSize)
   }
 
-  def from[T <: Object:ClassTag](chunkSize: Int = 1000): ImmutableArray[T] = {
-    new ChunkedImmutableArray[T](Array(), chunkSize = chunkSize)
+  def empty[T <: Object : ClassTag](chunkSize: Int): ImmutableArray[T] = {
+    new ChunkedImmutableArray[T](chunks = Array(), chunkSize = chunkSize)
   }
 
 }
 
 class ChunkedImmutableArray[T <: Object :ClassTag](private val chunks:Array[Array[T]], private val lastUsedIndex: Int = 0, val chunkSize: Int = 1000) extends ImmutableArray[T] with Iterable[T]{
 
-
-
   override def fromArray(arr: Array[T]): ImmutableArray[T] = {
     //https://www.cs.nott.ac.uk/~psarb2/G51MPC/slides/NumberLogic.pdf
     val chunkCount = (arr.length - 1) / chunkSize + 1
     val newChunks = new Array[Array[T]](chunkCount)
 
     (0 until chunkCount).foreach( i => {
       val start = i * chunkSize;
-      val end   = Math.min(start + chunkSize, arr.length -1);
+      val end   = Math.min(start + chunkSize, arr.length);
       val chunk = util.Arrays.copyOfRange[T](arr, start, end)
-      newChunks(i) = chunk
+      if(chunk.length < chunkSize){
+        newChunks(i) = Array.concat(chunk, new Array[T](chunkSize - chunk.length))
+      }else{
+        newChunks(i) = chunk
+      }
     })
 
     val lastUsedIndex = arr.length
@@ -95,15 +105,15 @@ class ChunkedImmutableArray[T <: Object :ClassTag](private val chunks:Array[Arra
   private def indexMinusOne(): Int = lastUsedIndex - 1
 
   private def setFullChunks(oldChunks:Array[Array[T]], newChunks: Array[Array[T]]): Unit = {
-    for (a <- 0 until oldChunks.length){
+    for (a <- oldChunks.indices){
       if(oldChunks(a).length == chunkSize) {
         newChunks(a) = oldChunks(a)
       }
     }
   }
 
   private def setChunks(oldChunks:Array[Array[T]], newChunks: Array[Array[T]]): Unit ={
-    for (a <- 0 until oldChunks.length){
+    for (a <- oldChunks.indices){
       newChunks(a) = oldChunks(a)
     }
   }
@@ -155,7 +165,7 @@ class ChunkedImmutableArray[T <: Object :ClassTag](private val chunks:Array[Arra
 
     //create any empty chunks required till we get to required chunks
     //create empty chunks
-    (currentChunks to requiredChunks - 1).foreach( i =>
+    (currentChunks until requiredChunks).foreach(i =>
       newChunks(i) = new Array[T](chunkSize)
     )
 
@@ -190,8 +200,8 @@ class ChunkedImmutableArray[T <: Object :ClassTag](private val chunks:Array[Arra
       activeChunk += 1
     }
 
-    //println(s"getIndex($index) -> ($activeChunk)($indexInChunk)")
-    chunks(activeChunk)(indexInChunk)
+    val chunk = chunks(activeChunk)
+    chunk(indexInChunk)
   }
 
   override def indexOf(element: T): Int = {
@@ -247,14 +257,26 @@ class ChunkedImmutableArray[T <: Object :ClassTag](private val chunks:Array[Arra
         setInPlace(a, getIndex(a), newChunks)
       }
 
-      for(a <- idxOf until this.length){
+      for(a <- idxOf until this.length - 1){
         setInPlace(a, getIndex(a+ 1), newChunks)
       }
 
       new ChunkedImmutableArray[T](newChunks, indexMinusOne(), chunkSize = this.chunkSize)
     }
   }
 
-  override def distinct: ImmutableArray[T] = ???
+  override def distinct: ImmutableArray[T] = fromArray(iterator.distinct.toArray)
+
+  private lazy val hash = iterator.foldLeft(0)((h, elem) => 31 * h + (if elem == null then 0 else elem.hashCode()))
+
+  override def hashCode(): Int = hash
+
+  override def equals(obj: Any): Boolean = {
+    obj match {
+      case value: ChunkedImmutableArray[_] =>
+        (this eq value) || (this.length == value.length && this.iterator.sameElements(value.iterator))
+      case _ => false
+    }
+  }
 
 }

toolbox/src/main/scala/org/finos/toolbox/collection/array/ImmutableArray.scala

@@ -1,35 +1,22 @@
 package org.finos.toolbox.collection.array
 
-import org.finos.toolbox.collection.set.ChunkedUniqueImmutableArraySet
-
 import scala.reflect.ClassTag
 
-object ImmutableArray{
+object ImmutableArray {
+
   def empty[T <: Object](implicit c: ClassTag[T]): ImmutableArray[T] = {
-    new ChunkedUniqueImmutableArraySet[T](Set(), Array(), chunkSize = 5000)
-  }
-  def from[T <: Object](array: Array[T])(implicit c: ClassTag[T]): ImmutableArray[T] = {
-    empty[T].addAll(new NaiveImmutableArray[T](array))
+    ChunkedImmutableArray.empty()
   }
 
-  def fromArray[T <: Object](array: Array[T])(implicit c: ClassTag[T]): ImmutableArray[T] = {
-    new ChunkedUniqueImmutableArraySet[T](Set(), Array(), chunkSize = chunkSize(array.length)).fromArray(array)
+  def from[T <: Object](array: Array[T])(implicit c: ClassTag[T]): ImmutableArray[T] = {
+    ChunkedImmutableArray.from(array)
   }
 
-  private def chunkSize(hint: Int): Int = {
-    if(hint < 100_000){
-        5000
-    }else if(hint < 9000_000){
-        50_000
-    }else{
-        100_000
-    }
-  }
 }
 
 object ImmutableArrays{
   def empty[T <: Object :ClassTag](i: Int): Array[ImmutableArray[T]] = {
-    (0 to (i - 1)).map( i=> new NaiveImmutableArray(Array[T]())).toArray
+    (0 until i).map(i => ImmutableArray.empty()).toArray
   }
 }