Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions lib/classifier/lsi.rb
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def build_index(cutoff = 0.75)
tdm = Matrix.rows(tda).trans
ntdm = build_reduced_matrix(tdm, cutoff)

ntdm.row_size.times do |col|
ntdm.column_size.times do |col|
next unless doc_list[col]

column = ntdm.column(col)
Expand Down Expand Up @@ -332,7 +332,13 @@ def build_reduced_matrix(matrix, cutoff = 0.75)
s[ord] = 0.0 if s[ord] < s_cutoff
end
# Reconstruct the term document matrix, only with reduced rank
u * (self.class.gsl_available ? GSL::Matrix : ::Matrix).diag(s) * v.trans
result = u * (self.class.gsl_available ? GSL::Matrix : ::Matrix).diag(s) * v.trans

# Native Ruby SVD returns transposed dimensions when row_size < column_size
# Ensure result matches input dimensions
result = result.trans if !self.class.gsl_available && result.row_size != matrix.row_size

result
end

def node_for_content(item, &block)
Expand Down
20 changes: 20 additions & 0 deletions test/lsi/lsi_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -307,4 +307,24 @@ def test_empty_word_hash_handling

refute_predicate lsi, :needs_rebuild?
end

def test_large_similar_document_sets
# Regression test for issue #72
# When many similar documents create few unique terms (M < N),
# native Ruby SVD returns transposed dimensions causing ErrDimensionMismatch
lsi = Classifier::LSI.new auto_rebuild: false

10.times do |i|
lsi.add_item "This text deals with dogs. Dogs number #{i}.", 'Dog'
end
10.times do |i|
lsi.add_item "This text deals with cats. Cats number #{i}.", 'Cat'
end

lsi.build_index

result = lsi.classify('Dogs are great pets')

assert_equal 'Dog', result
end
end