Merge pull request #199 from inutano/add-fts-search

Add server-side full-text experiment search with SQLite FTS5

Author: inutano

app.rb

@@ -73,6 +73,7 @@ def self.download_json_with_fallback(remote_url, local_filename)
       set :bedsizes, PJ::Bedsize.dump
       set :experiment_list, download_json_with_fallback("https://chip-atlas.dbcls.jp/data/metadata/ExperimentList.json", "ExperimentList.json")
       set :experiment_list_adv, download_json_with_fallback("https://chip-atlas.dbcls.jp/data/metadata/ExperimentList_adv.json", "ExperimentList_adv.json")
+      PJ::ExperimentSearch.load_from_json(settings.experiment_list_adv)
       set :gsm_to_srx, Hash[settings.experiment_list["data"].map{|a| [a[2], a[0]] }]
       set :wabi_endpoint, "https://dtn1.ddbj.nig.ac.jp/wabi/chipatlas/"
     rescue ActiveRecord::StatementInvalid
@@ -178,6 +179,15 @@ def self.download_json_with_fallback(remote_url, local_filename)
     JSON(data)
   end
 
+  get '/data/search' do
+    query  = params[:q]
+    genome = params[:genome]
+    limit  = (params[:limit] || 20).to_i.clamp(1, 100)
+    data = PJ::ExperimentSearch.search(query, genome: genome, limit: limit)
+    content_type "application/json"
+    JSON(data)
+  end
+
   get "/health" do
     checks = {}
 

db/migrate/20260303000000_create_experiments_fts.rb

@@ -0,0 +1,14 @@
+class CreateExperimentsFts < ActiveRecord::Migration[4.2]
+  def up
+    execute <<-SQL
+      CREATE VIRTUAL TABLE IF NOT EXISTS experiments_fts USING fts5(
+        expid, sra_id, geo_id, genome, agClass, agSubClass,
+        clClass, clSubClass, title, attributes
+      )
+    SQL
+  end
+
+  def down
+    execute "DROP TABLE IF EXISTS experiments_fts"
+  end
+end

lib/pj.rb

@@ -2,6 +2,7 @@
 require 'pj/bedfile'
 require 'pj/bedsize'
 require 'pj/experiment'
+require 'pj/experiment_search'
 require 'pj/fastqc'
 require 'pj/location'
 require 'pj/metadata'

lib/pj/experiment_search.rb

@@ -0,0 +1,103 @@
+require 'sinatra/activerecord'
+
+module PJ
+  module ExperimentSearch
+    COLUMNS = %w[expid sra_id geo_id genome agClass agSubClass clClass clSubClass title attributes].freeze
+
+    class << self
+      def load_from_json(json_data)
+        rows = json_data["data"]
+        return if rows.nil? || rows.empty?
+
+        db = ActiveRecord::Base.connection
+
+        db.execute("DELETE FROM experiments_fts")
+
+        # Bulk insert in batches
+        rows.each_slice(500) do |batch|
+          placeholders = batch.map { "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" }.join(", ")
+          values = batch.flat_map do |row|
+            COLUMNS.each_with_index.map { |_, i| row[i] || "" }
+          end
+
+          db.exec_insert(
+            "INSERT INTO experiments_fts (#{COLUMNS.join(', ')}) VALUES #{placeholders}",
+            "FTS Insert",
+            values.each_with_index.map { |v, i| [nil, v] }
+          )
+        end
+
+        puts "ExperimentSearch: loaded #{rows.size} rows into FTS5 table"
+      end
+
+      def search(query, genome: nil, limit: 20)
+        return { total: 0, returned: 0, experiments: [] } if query.nil? || query.strip.empty?
+
+        db = ActiveRecord::Base.connection
+
+        # Escape special FTS5 characters and build match expression
+        sanitized = fts5_sanitize(query)
+
+        where_clause = "experiments_fts MATCH ?"
+        bind_values = [sanitized]
+
+        if genome && !genome.empty?
+          where_clause += " AND genome = ?"
+          bind_values << genome
+        end
+
+        # Count total matches
+        count_sql = "SELECT COUNT(*) FROM experiments_fts WHERE #{where_clause}"
+        total = db.select_value(count_sql, "FTS Count", bind_values.map { |v| [nil, v] }).to_i
+
+        # Fetch ranked results
+        select_sql = <<-SQL
+          SELECT expid, sra_id, geo_id, genome, agClass, agSubClass,
+                 clClass, clSubClass, title, attributes,
+                 rank
+          FROM experiments_fts
+          WHERE #{where_clause}
+          ORDER BY rank
+          LIMIT ?
+        SQL
+
+        rows = db.select_all(
+          select_sql,
+          "FTS Search",
+          (bind_values + [limit]).map { |v| [nil, v] }
+        )
+
+        experiments = rows.map do |row|
+          {
+            expid: row["expid"],
+            sra_id: row["sra_id"],
+            geo_id: row["geo_id"],
+            genome: row["genome"],
+            agClass: row["agClass"],
+            agSubClass: row["agSubClass"],
+            clClass: row["clClass"],
+            clSubClass: row["clSubClass"],
+            title: row["title"],
+            attributes: row["attributes"]
+          }
+        end
+
+        { total: total, returned: experiments.size, experiments: experiments }
+      end
+
+      private
+
+      def fts5_sanitize(query)
+        # Remove FTS5 special characters, then wrap each token in quotes for safety
+        tokens = query.strip.split(/\s+/).map do |token|
+          # Strip special FTS5 operators
+          cleaned = token.gsub(/["()*^{}:]/, "")
+          next nil if cleaned.empty?
+          %("#{cleaned}")
+        end.compact
+
+        tokens.join(" ")
+      end
+    end
+  end
+end

lib/tasks/metadata.rake

@@ -63,11 +63,25 @@ namespace :metadata do
     :load_bedfile,
     :load_analysis,
     :load_bedsize,
-    :load_run
+    :load_run,
+    :load_fts
   ] do
     puts "All metadata loading completed successfully!"
   end
 
+  task :load_fts do
+    puts "[6/6] Loading FTS5 search index..."
+    start_time = Time.now
+    json_path = File.join(PROJ_ROOT, "public", "ExperimentList_adv.json")
+    if File.exist?(json_path)
+      json_data = JSON.parse(File.read(json_path))
+      PJ::ExperimentSearch.load_from_json(json_data)
+      puts "   FTS5 index loaded (#{sprintf('%.2f', Time.now - start_time)}s)"
+    else
+      puts "   Skipping FTS5: ExperimentList_adv.json not found (will be populated at app startup)"
+    end
+  end
+
   task :load_experiment => experiment_table_fpath do |t|
     puts "[1/5] Loading experiments data..."
     start_time = Time.now

mcp/dist/client.d.ts

@@ -0,0 +1,45 @@
+export declare class ChipAtlasClient {
+    private baseUrl;
+    constructor(baseUrl?: string);
+    private fetchJson;
+    private postJson;
+    listGenomes(): Promise<string[]>;
+    listExperimentTypes(genome?: string, clClass?: string): Promise<{
+        id: string;
+        label: string;
+        count?: number;
+    }[]>;
+    listSampleTypes(genome: string, agClass: string): Promise<{
+        id: string;
+        label: string;
+        count: number;
+    }[]>;
+    listAntigens(genome: string, agClass: string, clClass?: string): Promise<{
+        id: string;
+        label: string;
+        count: number | null;
+    }[]>;
+    listCellTypes(genome: string, agClass: string, clClass?: string): Promise<{
+        id: string;
+        label: string;
+        count: number | null;
+    }[]>;
+    searchExperiments(query: string, limit?: number, genome?: string): Promise<{
+        total: number;
+        returned: number;
+        experiments: Record<string, string>[];
+    }>;
+    getExperiment(expid: string): Promise<Record<string, unknown>[]>;
+    getColocalization(genome: string): Promise<Record<string, unknown>>;
+    getTargetGenes(): Promise<Record<string, string[]>>;
+    getBedUrl(condition: {
+        genome: string;
+        agClass: string;
+        agSubClass?: string;
+        clClass?: string;
+        clSubClass?: string;
+        qval?: string;
+    }): Promise<{
+        url: string;
+    }>;
+}