style: fix all credo strict warnings

Author: preciz

lib/chunx/chunker/semantic.ex

@@ -8,6 +8,7 @@ defmodule Chunx.Chunker.Semantic do
 
   @behaviour Chunx.Chunker
 
+  alias Chunx.Chunker.Semantic.Sentences
   alias Chunx.SentenceChunk
   alias Scholar.Metrics.Distance
 
@@ -53,7 +54,7 @@ defmodule Chunx.Chunker.Semantic do
       {:ok, []}
     else
       sentences =
-        Chunx.Chunker.Semantic.Sentences.prepare_sentences(text, tokenizer, embedding_fun, opts)
+        Sentences.prepare_sentences(text, tokenizer, embedding_fun, opts)
 
       if length(sentences) <= config.min_sentences do
         chunk = create_chunk(sentences)
@@ -78,26 +79,37 @@ defmodule Chunx.Chunker.Semantic do
     min_chunk_size = Keyword.fetch!(opts, :min_chunk_size)
     threshold_step = Keyword.fetch!(opts, :threshold_step)
 
+    validate_positive_integers!(chunk_size, min_sentences, min_chunk_size)
+    validate_threshold_step!(threshold_step)
+    validate_threshold!(threshold)
+
+    %{
+      chunk_size: chunk_size,
+      threshold: threshold,
+      min_sentences: min_sentences,
+      min_chunk_size: min_chunk_size,
+      threshold_step: threshold_step
+    }
+  end
+
+  defp validate_positive_integers!(chunk_size, min_sentences, min_chunk_size) do
     if chunk_size <= 0, do: raise(ArgumentError, "chunk_size must be positive")
     if min_sentences <= 0, do: raise(ArgumentError, "min_sentences must be positive")
     if min_chunk_size <= 0, do: raise(ArgumentError, "min_chunk_size must be positive")
+  end
 
-    if threshold_step <= 0 or threshold_step >= 1,
-      do: raise(ArgumentError, "threshold_step must be between 0 and 1")
+  defp validate_threshold_step!(step) do
+    if step <= 0 or step >= 1 do
+      raise(ArgumentError, "threshold_step must be between 0 and 1")
+    end
+  end
 
+  defp validate_threshold!(threshold) do
     case threshold do
       :auto -> :ok
       t when is_float(t) and t >= 0 and t <= 1 -> :ok
       _ -> raise(ArgumentError, "threshold must be :auto or a float between 0 and 1")
     end
-
-    %{
-      chunk_size: chunk_size,
-      threshold: threshold,
-      min_sentences: min_sentences,
-      min_chunk_size: min_chunk_size,
-      threshold_step: threshold_step
-    }
   end
 
   defp calculate_similarity_threshold(sentences, %{threshold: :auto} = config) do

lib/chunx/chunker/semantic/sentences.ex

@@ -89,19 +89,20 @@ defmodule Chunx.Chunker.Semantic.Sentences do
         if String.length(String.trim(split)) < min_chars do
           {sentences, current <> split}
         else
-          if current != "" do
-            {sentences ++ [current], split}
-          else
-            {sentences, split}
-          end
+          append_valid_sentence(sentences, current, split)
         end
       end)
 
-    if current != "" do
-      sentences ++ [current]
-    else
-      sentences
-    end
+    sentences = if current != "", do: [current | sentences], else: sentences
+    Enum.reverse(sentences)
+  end
+
+  defp append_valid_sentence(sentences, "", split) do
+    {sentences, split}
+  end
+
+  defp append_valid_sentence(sentences, current, split) do
+    {[current | sentences], split}
   end
 
   defp get_token_counts(sentences, tokenizer) do

lib/chunx/chunker/sentence.ex

@@ -206,22 +206,26 @@ defmodule Chunx.Chunker.Sentence do
 
   defp find_overlap_start(chunk_sentences, split_idx, total_len, config) do
     if config.chunk_overlap > 0 and split_idx < total_len do
-      {overlap_pos, _} =
-        chunk_sentences
-        |> Enum.reverse()
-        |> Enum.reduce_while({split_idx, 0}, fn sentence, {current_idx, total_tokens} ->
-          new_total = total_tokens + sentence.token_count
-
-          if new_total > config.chunk_overlap do
-            {:halt, {current_idx, new_total}}
-          else
-            {:cont, {current_idx - 1, new_total}}
-          end
-        end)
-
-      overlap_pos
+      calculate_sentence_overlap(chunk_sentences, split_idx, config.chunk_overlap)
     else
       split_idx
     end
   end
+
+  defp calculate_sentence_overlap(chunk_sentences, split_idx, chunk_overlap) do
+    {overlap_pos, _} =
+      chunk_sentences
+      |> Enum.reverse()
+      |> Enum.reduce_while({split_idx, 0}, fn sentence, {current_idx, total_tokens} ->
+        new_total = total_tokens + sentence.token_count
+
+        if new_total > chunk_overlap do
+          {:halt, {current_idx, new_total}}
+        else
+          {:cont, {current_idx - 1, new_total}}
+        end
+      end)
+
+    overlap_pos
+  end
 end

lib/chunx/chunker/word.ex

@@ -125,16 +125,7 @@ defmodule Chunx.Chunker.Word do
           chunk = create_chunk(words_in_chunk, text, current_length)
 
           {overlap_chunk_reversed, overlap_length} =
-            current_chunk
-            |> Enum.reduce_while({[], 0}, fn {_, l} = item, {acc, len} ->
-              if len + l <= config.chunk_overlap do
-                {:cont, {[item | acc], len + l}}
-              else
-                {:halt, {acc, len}}
-              end
-            end)
-
-          overlap_chunk_reversed = Enum.reverse(overlap_chunk_reversed)
+            calculate_overlap(current_chunk, config.chunk_overlap)
 
           new_chunk = [{word, length} | overlap_chunk_reversed]
           new_length = overlap_length + length
@@ -151,6 +142,20 @@ defmodule Chunx.Chunker.Word do
     |> Enum.reject(&is_nil/1)
   end
 
+  defp calculate_overlap(current_chunk, chunk_overlap) do
+    {overlap_chunk, overlap_length} =
+      current_chunk
+      |> Enum.reduce_while({[], 0}, fn {_, l} = item, {acc, len} ->
+        if len + l <= chunk_overlap do
+          {:cont, {[item | acc], len + l}}
+        else
+          {:halt, {acc, len}}
+        end
+      end)
+
+    {Enum.reverse(overlap_chunk), overlap_length}
+  end
+
   defp create_chunk(words, text, token_count) do
     chunk_text = Enum.join(words)
 

lib/chunx/helper.ex

@@ -1,4 +1,7 @@
 defmodule Chunx.Helper do
+  @moduledoc """
+  Provides math and statistic helper functions for array analysis.
+  """
   @spec median([number()]) :: number()
   def median(values) do
     sorted = Enum.sort(values)

mix.exs

@@ -27,7 +27,8 @@ defmodule Chunx.MixProject do
       {:exla, "~> 0.10", only: [:dev, :test]},
       {:bumblebee, "~> 0.6", only: [:dev, :test]},
       {:stream_data, "~> 1.1", only: [:dev, :test]},
-      {:benchee, "~> 1.3", only: [:dev, :test]}
+      {:benchee, "~> 1.3", only: [:dev, :test]},
+      {:credo, "~> 1.7", only: [:dev, :test], runtime: false}
     ]
   end
 end

mix.lock

@@ -2,12 +2,15 @@
   "axon": {:hex, :axon, "0.7.0", "2e2c6d93b4afcfa812566b8922204fa022b60081e86ebd411df4db7ea30f5457", [:mix], [{:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}, {:kino_vega_lite, "~> 0.1.7", [hex: :kino_vega_lite, repo: "hexpm", optional: true]}, {:nx, "~> 0.9", [hex: :nx, repo: "hexpm", optional: false]}, {:polaris, "~> 0.1", [hex: :polaris, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: true]}], "hexpm", "ee9857a143c9486597ceff434e6ca833dc1241be6158b01025b8217757ed1036"},
   "benchee": {:hex, :benchee, "1.5.0", "4d812c31d54b0ec0167e91278e7de3f596324a78a096fd3d0bea68bb0c513b10", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "5b075393aea81b8ae74eadd1c28b1d87e8a63696c649d8293db7c4df3eb67535"},
   "bumblebee": {:hex, :bumblebee, "0.6.3", "c0028643c92de93258a9804da1d4d48797eaf7911b702464b3b3dd2cc7f938f1", [:mix], [{:axon, "~> 0.7.0", [hex: :axon, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.0", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.9.0 or ~> 0.10.0", [hex: :nx, repo: "hexpm", optional: false]}, {:nx_image, "~> 0.1.0", [hex: :nx_image, repo: "hexpm", optional: false]}, {:nx_signal, "~> 0.2.0", [hex: :nx_signal, repo: "hexpm", optional: false]}, {:progress_bar, "~> 3.0", [hex: :progress_bar, repo: "hexpm", optional: false]}, {:safetensors, "~> 0.1.3", [hex: :safetensors, repo: "hexpm", optional: false]}, {:tokenizers, "~> 0.4", [hex: :tokenizers, repo: "hexpm", optional: false]}, {:unpickler, "~> 0.1.0", [hex: :unpickler, repo: "hexpm", optional: false]}, {:unzip, "~> 0.12.0", [hex: :unzip, repo: "hexpm", optional: false]}], "hexpm", "c619197787561f8e5fb2ffba269c341654accaec9d591999b7fddd55761dd079"},
+  "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.17", "4f9770d2d45fbd91dcf6bd404cf64e7e58fed04fadda0923dc32acca0badffa2", [:mix], [], "hexpm", "12d24b9d80b910dd3953e165636d68f147a31db945d2dcb9365e441f8b5351e5"},
   "complex": {:hex, :complex, "0.6.0", "b0130086a7a8c33574d293b2e0e250f4685580418eac52a5658a4bd148f3ccf1", [:mix], [], "hexpm", "0a5fa95580dcaf30fcd60fe1aaf24327c0fe401e98c24d892e172e79498269f9"},
+  "credo": {:hex, :credo, "1.7.17", "f92b6aa5b26301eaa5a35e4d48ebf5aa1e7094ac00ae38f87086c562caf8a22f", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1eb5645c835f0b6c9b5410f94b5a185057bcf6d62a9c2b476da971cde8749645"},
   "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"},
   "exla": {:hex, :exla, "0.10.0", "93e7d75a774fbc06ce05b96de20c4b01bda413b315238cb3c727c09a05d2bc3a", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:nx, "~> 0.10.0", [hex: :nx, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:xla, "~> 0.9.0", [hex: :xla, repo: "hexpm", optional: false]}], "hexpm", "16fffdb64667d7f0a3bc683fdcd2792b143a9b345e4b1f1d5cd50330c63d8119"},
+  "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
   "fine": {:hex, :fine, "0.1.4", "b19a89c1476c7c57afb5f9314aed5960b5bc95d5277de4cb5ee8e1d1616ce379", [:mix], [], "hexpm", "be3324cc454a42d80951cf6023b9954e9ff27c6daa255483b3e8d608670303f5"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},

test/chunx/chunker/property_test.exs

@@ -2,7 +2,7 @@ defmodule Chunx.Chunker.PropertyTest do
   use ExUnit.Case, async: true
   use ExUnitProperties
 
-  alias Chunx.Chunker.{Token, Word, Sentence}
+  alias Chunx.Chunker.{Sentence, Token, Word}
 
   setup_all do
     {:ok, tokenizer} = Tokenizers.Tokenizer.from_pretrained("gpt2")

test/chunx/chunker/semantic/sentences_test.exs

@@ -1,8 +1,8 @@
 defmodule Chunx.Chunker.Semantic.SentencesTest do
   use ExUnit.Case, async: true
 
-  alias Chunx.Chunker.Semantic.Sentences
   alias Chunx.Chunk
+  alias Chunx.Chunker.Semantic.Sentences
 
   @titanic """
   Titanic was a ship that sank in the North Atlantic Ocean in 1912 after hitting an iceberg.

test/chunx/chunker/semantic_test.exs

@@ -68,7 +68,7 @@ defmodule Chunx.Chunker.SemanticTest do
           serving_fun
         )
 
-      assert length(chunks) > 0
+      assert chunks != []
       assert Enum.all?(chunks, &match?(%SentenceChunk{}, &1))
     end
 
@@ -213,7 +213,7 @@ defmodule Chunx.Chunker.SemanticTest do
           serving_fun
         )
 
-      assert length(chunks) > 0
+      assert chunks != []
       assert Enum.all?(chunks, &match?(%SentenceChunk{}, &1))
     end