fix sorting to use neighbor distance (#212)

* fix sorting to use neighbor distance

* fix sql

---------

Co-authored-by: Vijay Swamidass <[email protected]>

Author: vswamidass-sfdc

app/controllers/base_documents_controller.rb

@@ -11,13 +11,16 @@ class BaseDocumentsController < ApplicationController
   def index
     @documents = Document.includes(:library, :user)
 
-    @documents = if params[:sort] == 'questions'
-                   @documents.order(questions_count: :desc)
-                 elsif params[:sort] == 'tokens'
-                   @documents.order(token_count: :desc)
-                 else
-                   @documents.order(updated_at: :desc)
-                 end
+    # Only apply default sorting if not doing similarity search
+    unless params[:similar_to].present?
+      @documents = if params[:sort] == 'questions'
+                     @documents.order(questions_count: :desc)
+                   elsif params[:sort] == 'tokens'
+                     @documents.order(token_count: :desc)
+                   else
+                     @documents.order(updated_at: :desc)
+                   end
+    end
 
     library_id = params[:library_id]
     if library_id.present?
@@ -57,9 +60,13 @@ def index
       embedding = get_embedding(params[:similar_to])
       # Get similar documents but preserve existing filters
       similar_docs = related_documents_from_embedding_by_libraries(embedding, library_id)
-      # Apply the similarity search as an additional filter by getting the IDs
-      similar_doc_ids = similar_docs.pluck(:id)
-      @documents = @documents.where(id: similar_doc_ids)
+
+      # Sort by neighbor_distance (ascending - closest first) and convert to array
+      # This maintains the similarity order without needing raw SQL
+      @documents = similar_docs.sort_by(&:neighbor_distance)
+
+      # Convert to Kaminari-compatible array for pagination
+      @documents = Kaminari.paginate_array(@documents)
     end
 
     @documents = @documents.search_by_title_and_document(params[:contains]) if params[:contains].present?