RUBY-3521 Add binary vector support (#344)

Author: comandeo-mongo

.evergreen/config.yml

@@ -328,6 +328,14 @@ axes:
         display_name: 7.0
         variables:
           WITH_ACTIVE_SUPPORT: "~> 7.0.0"
+      - id: "7.1"
+        display_name: 7.1
+        variables:
+          WITH_ACTIVE_SUPPORT: "~> 7.1.0"
+      - id: "8.0"
+        display_name: 8.0
+        variables:
+          WITH_ACTIVE_SUPPORT: "~> 8.0.0"
 
   - id: "compact"
     display_name: GC.compact
@@ -365,9 +373,18 @@ buildvariants:
 
   - matrix_name: "activesupport-7"
     matrix_spec:
-      ruby: ["ruby-3.3", "ruby-2.7"]
+      ruby: ruby-3.3
+      all-os: ubuntu2004
+      as: [ '7.0', '7.1' ]
+    display_name: "AS ${as} ${ruby}, ${all-os}"
+    tasks:
+      - name: "test"
+
+  - matrix_name: "activesupport-8"
+    matrix_spec:
+      ruby: ruby-3.3
       all-os: ubuntu2004
-      as: '7.0'
+      as: '8.0'
     display_name: "AS ${as} ${ruby}, ${all-os}"
     tasks:
       - name: "test"

.evergreen/config/axes.yml.erb

@@ -60,6 +60,14 @@ axes:
         display_name: 7.0
         variables:
           WITH_ACTIVE_SUPPORT: "~> 7.0.0"
+      - id: "7.1"
+        display_name: 7.1
+        variables:
+          WITH_ACTIVE_SUPPORT: "~> 7.1.0"
+      - id: "8.0"
+        display_name: 8.0
+        variables:
+          WITH_ACTIVE_SUPPORT: "~> 8.0.0"
 
   - id: "compact"
     display_name: GC.compact

.evergreen/config/variants.yml.erb

@@ -22,9 +22,18 @@ buildvariants:
 
   - matrix_name: "activesupport-7"
     matrix_spec:
-      ruby: <%= sample_mri_rubies %>
+      ruby: <%= latest_mri_ruby %>
+      all-os: ubuntu2004
+      as: [ '7.0', '7.1' ]
+    display_name: "AS ${as} ${ruby}, ${all-os}"
+    tasks:
+      - name: "test"
+
+  - matrix_name: "activesupport-8"
+    matrix_spec:
+      ruby: <%= latest_mri_ruby %>
       all-os: ubuntu2004
-      as: '7.0'
+      as: '8.0'
     display_name: "AS ${as} ${ruby}, ${all-os}"
     tasks:
       - name: "test"

Gemfile

@@ -15,8 +15,9 @@ group :development, :test do
   if ENV['WITH_ACTIVE_SUPPORT'] =~ /[0-9]/ && ENV['WITH_ACTIVE_SUPPORT'] != '0'
     gem 'activesupport', ENV['WITH_ACTIVE_SUPPORT']
   else
-    gem 'activesupport', '<7.1'
+    gem 'activesupport', '<8.1'
   end
+  gem 'concurrent-ruby', '1.3.4'
   gem 'ruby-prof', platforms: :mri
 
   gem 'byebug', platforms: :mri

bson.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
   if RUBY_VERSION > '3.2.99'
     s.add_dependency 'base64'
     s.add_dependency 'bigdecimal'
+    s.add_dependency 'ostruct'
   end
 
   s.test_files = Dir.glob('spec/**/*')

lib/bson.rb

@@ -95,6 +95,7 @@ def self.ObjectId(string)
 require "bson/timestamp"
 require "bson/true_class"
 require "bson/undefined"
+require "bson/vector"
 require "bson/version"
 
 # If we are using JRuby, attempt to load the Java extensions, if we are using

lib/bson/binary.rb

@@ -50,6 +50,7 @@ class Binary
       ciphertext: 6.chr,
       column: 7.chr,
       sensitive: 8.chr,
+      vector: 9.chr,
       user: 128.chr,
     }.freeze
 
@@ -61,6 +62,16 @@ class Binary
     # @since 2.0.0
     TYPES = SUBTYPES.invert.freeze
 
+    # Types of vector data.
+    VECTOR_DATA_TYPES = {
+      int8: '0x03'.hex,
+      float32: '0x27'.hex,
+      packed_bit: '0x10'.hex
+    }.freeze
+
+    # @api private
+    VECTOR_DATA_TYPES_INVERSE = VECTOR_DATA_TYPES.invert.freeze
+
     # @return [ String ] The raw binary data.
     #
     # The string is always stored in BINARY encoding.
@@ -89,6 +100,7 @@ def ==(other)
 
       type == other.type && data == other.data
     end
+
     alias eql? ==
 
     # Compare this binary object to another object. The two objects must have
@@ -113,7 +125,7 @@ def <=>(other)
     #
     # @since 2.3.1
     def hash
-      [ data, type ].hash
+      [data, type].hash
     end
 
     # Return a representation of the object for use in
@@ -146,6 +158,26 @@ def as_extended_json(**options)
       end
     end
 
+    # Decode the binary data as a vector data type.
+    #
+    # @return [ BSON::Vector ] The decoded vector data.
+    def as_vector
+      raise BSON::Error, "Cannot decode subtype #{type} as vector" unless type == :vector
+
+      dtype_value, padding, = data[0..1].unpack('CC')
+      dtype = VECTOR_DATA_TYPES_INVERSE[dtype_value]
+      raise ArgumentError, "Unsupported vector type: #{dtype_value}" unless dtype
+
+      format = case dtype
+               when :int8 then 'c*'
+               when :float32 then 'f*'
+               when :packed_bit then 'C*'
+               else
+                 raise ArgumentError, "Unsupported type: #{dtype}"
+               end
+      BSON::Vector.new(data[2..-1].unpack(format), dtype, padding)
+    end
+
     # Instantiate the new binary object.
     #
     # This method accepts a string in any encoding; however, if a string is
@@ -368,8 +400,97 @@ def self.from_python_legacy_uuid(uuid_binary)
       new(uuid_binary, :uuid_old)
     end
 
+    # Constructs a new binary object from a binary vector.
+
+    # @param [ BSON::Vector | Array ] vector The vector data.
+    # @param [ Symbol | nil ] dtype The vector data type, must be nil if vector is a BSON::Vector.
+    # @param [ Integer ] padding The number of bits in the final byte that are to
+    # be ignored when a vector element's size is less than a byte. Must be 0 if vector is a BSON::Vector.
+    # @param [ Boolean ] validate_vector_data Whether to validate the vector data.
+    #
+    # @return [ BSON::Binary ] The binary object.
+    def self.from_vector(vector, dtype = nil, padding = 0, validate_vector_data: false)
+      data, dtype, padding = extract_args_for_vector(vector, dtype, padding)
+      validate_args_for_vector!(data, dtype, padding)
+
+      format = case dtype
+               when :int8 then 'c*'
+               when :float32 then 'f*'
+               when :packed_bit then 'C*'
+               else raise ArgumentError, "Unsupported type: #{dtype}"
+               end
+      if validate_vector_data
+        validate_vector_data!(data, dtype)
+      end
+      metadata = [ VECTOR_DATA_TYPES[dtype], padding ].pack('CC')
+      data = data.pack(format)
+      new(metadata.concat(data), :vector)
+    end
+
     private
 
+    # Extracts the arguments for a binary vector.
+    #
+    # @param [ BSON::Vector | Array ] vector The vector data.
+    # @param [ ::Symbol | nil ] dtype The vector data type, must be nil if vector is a BSON::Vector.
+    # @param [ Integer ] padding The padding. Must be 0 if vector is a BSON::Vector.
+    #
+    # @return [ Array ] The extracted data, dtype, and padding.
+    def self.extract_args_for_vector(vector, dtype, padding)
+      if vector.is_a?(BSON::Vector)
+        if dtype || padding != 0
+          raise ArgumentError, 'Do not specify dtype and padding if the first argument is BSON::Vector'
+        end
+
+        data = vector.data
+        dtype = vector.dtype
+        padding = vector.padding
+      else
+        data = vector
+      end
+      [ data, dtype, padding ]
+    end
+    private_class_method :extract_args_for_vector
+
+    # Validate the arguments for a binary vector.
+    # @param [ Array ] data The vector data.
+    # @param [ ::Symbol ] dtype The vector data type.
+    # @param [ Integer | nil ] padding The padding. Must be 0 if vector is a BSON::Vector.
+    # @raise [ ArgumentError ] If the arguments are invalid.
+    def self.validate_args_for_vector!(data, dtype, padding)
+      raise ArgumentError, "Unknown dtype #{dtype}" unless VECTOR_DATA_TYPES.key?(dtype)
+
+      if %i[int8 float32].include?(dtype)
+        raise ArgumentError, 'Padding applies only to packed_bit' if padding != 0
+      elsif padding.positive? && data.empty?
+        raise ArgumentError, 'Padding must be zero when the vector is empty for PACKED_BIT'
+      elsif padding.negative? || padding > 7
+        raise ArgumentError, "Padding must be between 0 and 7, got #{padding}"
+      end
+    end
+    private_class_method :validate_args_for_vector!
+
+    # Validate that all the values in the vector data are valid for the given dtype.
+    #
+    # @param [ Array ] data The vector data.
+    # @param [ ::Symbol ] dtype The vector data type.
+    def self.validate_vector_data!(data, dtype)
+      validator = case dtype
+                  when :int8
+                    ->(v) { v.is_a?(Integer) && v.between?(-128, 127) }
+                  when :float32
+                    ->(v) { v.is_a?(Float) }
+                  when :packed_bit
+                    ->(v) { v.is_a?(Integer) && v.between?(0, 255) }
+                  else
+                    raise ArgumentError, "Unsupported type: #{dtype}"
+                  end
+      data.each do |v|
+        raise ArgumentError, "Invalid value #{v} for type #{dtype}" unless validator.call(v)
+      end
+    end
+    private_class_method :validate_vector_data!
+
     # initializes an instance of BSON::Binary.
     #
     # @param [ String ] data the data to initialize the object with
@@ -398,7 +519,7 @@ def from_uuid_to_uuid(representation)
       if representation != :standard
         raise ArgumentError,
               'Binary of type :uuid can only be stringified to :standard representation, ' \
-              "requested: #{representation.inspect}"
+                "requested: #{representation.inspect}"
       end
 
       data
@@ -490,7 +611,8 @@ def validate_type!(type)
           validate_integer_type!(type.bytes.first)
         end
       when Symbol then validate_symbol_type!(type)
-      else raise BSON::Error::InvalidBinaryType, type
+      else
+        raise BSON::Error::InvalidBinaryType, type
       end
     end
 

lib/bson/vector.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025-present MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module BSON
+  # Vector of numbers along with metadata for binary interoperability.
+  class Vector < ::Array
+
+    # @return [ Integer ] The data type stored in the vector.
+    attr_reader :dtype
+
+    # @return [ Integer ]  The number of bits in the final byte that are to
+    # be ignored when a vector element's size is less than a byte
+    # and the length of the vector is not a multiple of 8.
+    attr_reader :padding
+
+    # @return [ BSON::ByteBuffer ] The data in the vector.
+    def data
+      self
+    end
+
+    # @param [ ::Array ] data The data to initialize the vector with.
+    # @param [ Integer ] dtype The data type of the vector.
+    # @param [ Integer ] padding The number of bits in the final byte that are to
+    # be ignored when a vector element's size is less than a byte
+    # and the length of the vector is not a multiple of 8.
+    def initialize(data, dtype, padding)
+      @dtype = dtype
+      @padding = padding
+      super(data.dup)
+    end
+  end
+end