Fix document search (#223)

* date filter

* fix search to be more precise on multiple words

* ensure sorting by relevance works

---------

Co-authored-by: Vijay Swamidass <[email protected]>

Author: vswamidass-sfdc

app/controllers/base_documents_controller.rb

@@ -11,17 +11,7 @@ class BaseDocumentsController < ApplicationController
   def index
     @documents = Document.includes(:library, :user)
 
-    # Only apply default sorting if not doing similarity search
-    unless params[:similar_to].present?
-      @documents = if params[:sort] == 'questions'
-                     @documents.order(questions_count: :desc)
-                   elsif params[:sort] == 'tokens'
-                     @documents.order(token_count: :desc)
-                   else
-                     @documents.order(updated_at: :desc)
-                   end
-    end
-
+    # Apply cheap filters first for better performance
     library_id = params[:library_id]
     if library_id.present?
       @library = Library.find(params[:library_id])
@@ -56,17 +46,29 @@ def index
       end
     end
 
-    # Apply text search before similarity search to maintain ActiveRecord relation
-    @documents = @documents.search_by_title_and_document(params[:contains]) if params[:contains].present?
+    # Apply text search after basic filtering to work on smaller dataset
 
+    # Apply similarity search on already filtered dataset
     if params[:similar_to].present?
       embedding = get_embedding(params[:similar_to])
       # Get similar documents but preserve existing filters
       @documents = @documents.related_by_embedding(embedding)
 
       # Sort by neighbor_distance using SQL to maintain ActiveRecord relation
       @documents = @documents.order('neighbor_distance ASC')
+    elsif params[:contains].present?
+      @documents = @documents.smart_search(params[:contains])
+    else
+      # Only apply default sorting if not doing similarity search
+      @documents = if params[:sort] == 'questions'
+                     @documents.order(questions_count: :desc)
+                   elsif params[:sort] == 'tokens'
+                     @documents.order(token_count: :desc)
+                   else
+                     @documents.order(updated_at: :desc)
+                   end
     end
+
     @documents = @documents.page(params[:page]).per(params[:per_page] || 10)
   end
 

app/models/document.rb

@@ -7,16 +7,52 @@ class Document < ApplicationRecord
   has_many :documents_questions
   has_many :questions, through: :documents_questions
   has_many :comments, dependent: :destroy
+  has_neighbors :embedding
 
+  # Primary search scope using PostgreSQL full-text search
+  # Searches across both title and document content with strict word matching
+  # - prefix: true allows partial word matching (e.g., "test" matches "testing")
+  # - any_word: false requires ALL search terms to be present (AND logic)
+  # - Uses precomputed search_vector for performance
   pg_search_scope :search_by_title_and_document,
                   against: %i[title document],
                   using: {
-                    tsearch: { prefix: true, dictionary: 'english',
-                               tsvector_column: 'search_vector',
-                               any_word: true } # This enables OR search - matches any word
+                    tsearch: {
+                      prefix: true,
+                      dictionary: 'english',
+                      tsvector_column: 'search_vector',
+                      any_word: true
+                    }
                   }
 
-  has_neighbors :embedding
+  # Strict search scope with enhanced exact matching
+  # Similar to search_by_title_and_document but with stricter ranking:
+  # - normalization: 0 disables document length normalization in ranking
+  # - This prioritizes exact word matches over document length considerations
+  # - Better for finding specific content regardless of document size
+  pg_search_scope :strict_search,
+                  against: %i[title document],
+                  using: {
+                    tsearch: {
+                      prefix: true,
+                      dictionary: 'english',
+                      tsvector_column: 'search_vector',
+                      any_word: false,
+                      normalization: 0 # No normalization - prioritizes exact word matches over document length
+                    }
+                  }
+
+  # Smart search scope that handles partial word matching like "test12" -> "test"
+  # This is chainable and preserves existing filters (library_id, date ranges, etc.)
+  scope :smart_search, lambda { |query|
+    return all if query.blank?
+
+    strict_results = strict_search(query)
+    return strict_results if strict_results.count >= 1
+
+    # Return empty relation if no results found
+    none
+  }
 
   # Scope to find related documents by embedding with optional library filtering
   # _limit is the number of documents to return. Returning fewer is better since the most relevant documents are at the top.

app/views/documents/index.html.erb

@@ -10,11 +10,27 @@
     <% end %>
   </div>
   <!-- Search Form -->
-  <div class="flex-grow flex items-center mt-3">
+  <div class="mt-3">
     <% action_path = @library.present? ? library_documents_path(@library) : documents_path %>
-    <%= form_with(url: action_path, method: :get, local: true, class: "w-full flex") do %>
-      <input type="text" name="contains" id="contains" placeholder="Search documents..." class="flex-grow rounded-lg p-2 border border-stone-300">
-      <button type="submit" class="ml-2 rounded-lg p-2 bg-white text-sky-500 border-sky-500 border">Search</button>
+    <%= form_with(url: action_path, method: :get, local: true, class: "w-full") do %>
+      <div class="flex flex-col space-y-3">
+        <!-- Text Search -->
+        <div class="flex">
+          <input type="text" name="contains" id="contains" value="<%= params[:contains] %>" placeholder="Search documents..." class="flex-grow rounded-lg p-2 border border-stone-300">
+          <button type="submit" class="ml-2 rounded-lg p-2 bg-white text-sky-500 border-sky-500 border">Search</button>
+        </div>
+        <!-- Date Range Filters -->
+        <div class="flex space-x-4">
+          <div class="flex-1">
+            <label for="since" class="block text-sm font-medium text-stone-700 mb-1">Updated Since:</label>
+            <input type="datetime-local" name="since" id="since" value="<%= params[:since] %>" class="w-full rounded-lg p-2 border border-stone-300 text-sm">
+          </div>
+          <div class="flex-1">
+            <label for="until" class="block text-sm font-medium text-stone-700 mb-1">Updated Until:</label>
+            <input type="datetime-local" name="until" id="until" value="<%= params[:until] %>" class="w-full rounded-lg p-2 border border-stone-300 text-sm">
+          </div>
+        </div>
+      </div>
     <% end %>
   </div>
   <div class="flex-grow flex items-center mt-3">
@@ -28,6 +44,8 @@
       <%= link_to 'Sort by Date', request.path, class: 'rounded-lg p-2 bg-white text-sky-500 border-sky-500 border hover:bg-sky-50' %>
     <% end %>
   </div>
+  <div class="my-2">
+    <%= render 'shared/doc_list', docs: @documents %>
+  </div>
   <%= paginate @documents %>
-  <%= render 'shared/doc_list', docs: @documents %>
 </div>

app/views/shared/_doc_list.html.erb

@@ -1,5 +1,5 @@
 <ul class="space-y-4">
-  <% docs.each do |document| %>
+  <% docs.each_with_index do |document, index| %>
     <li class="bg-white p-4 rounded-lg border border-stone-300 hover:shadow-lg group">
       <details class="text-base">
         <summary class="text-lg cursor-pointer flex justify-between items-center">
@@ -10,7 +10,7 @@
                 <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-3 h-3 mr-1 text-stone-700">
                   <path stroke-linecap="round" stroke-linejoin="round" d="M12 6v6h4.5m4.5 0a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z" />
                 </svg>
-                <%= time_ago_in_words(document.created_at) %>
+                <%= time_ago_in_words(document.updated_at) %>
               </div>
               <div class="flex items-center">
                 <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-3 h-3 mr-1 text-stone-700">