added statistics methods to DataFrame including #describe, #cov, #corr, #mean, #std etc. Also monkey patched Matrix to include #elementwise_division

Author: v0dro

History.txt

@@ -69,4 +69,9 @@
 * Convert Daru::Vector to horizontal or vertical Ruby Matrix with #to_matrix.
 * Added shortcut to DataFrame to allow access of vectors by using only #[] instead of calling #vector or *[vector_names, :vector]*.
 * Added DSL for Vector and DataFrame plotting with nyaplot. Can now grab the underlying Nyaplot::Plot and Nyaplot::Diagram object for performing different operations. Only need to supply parameters for the initial creation of the diagram.
-* Added #pivot\_table to DataFrame for reducing and aggregating data to generate a quick summary.
+* Added #pivot\_table to DataFrame for reducing and aggregating data to generate a quick summary.
+* Added #shape to DataFrame for knowing the numbers of rows and columns in a DataFrame.
+* Added statistics methods #mean, #std, #max, #min, #count, #product, #sum to DataFrame.
+* Added #describe to DataFrame for producing multiple statistics data of numerical vectors in one shot.
+* Monkey patched Ruby Matrix to include #elementwise_division.
+* Added #covariance to calculate the covariance between numbers of a DataFrame and #correlation to calculate correlation. 

lib/daru/dataframe.rb

@@ -393,6 +393,21 @@ def filter_vectors &block
       df
     end
 
+    # Return the number of rows and columns of the DataFrame in an Array.
+    def shape
+      [@index.size, @vectors.size]
+    end
+
+    # The number of rows
+    def rows
+      shape[0]
+    end
+
+    # The number of vectors
+    def cols
+      shape[1]
+    end
+
     # Check if a vector is present
     def has_vector? vector
       !!@vectors[*vector]

lib/daru/maths/statistics/dataframe.rb

@@ -4,53 +4,107 @@ module Statistics
       module DataFrame
         # Calculate mean of numeric vectors.
         def mean
-                  
+          compute_stats :mean
         end
 
-        # Calculate standard deviation of numeric vectors.
+        # Calculate sample standard deviation of numeric vectors.
         def std
-          
+          compute_stats :std
         end
 
         # Calculate sum of numeric vectors
         def sum
-          
+          compute_stats :sum
         end
 
         # Count the number of non-nil values in each vector.
         def count
-          
+          compute_stats :count
         end
 
         # Calculate the maximum value of each numeric vector.
         def max
-          
+          compute_stats :max
         end
 
         # Calculate the minimmum value of each numeric vector.
         def min
-          
+          compute_stats :min
+        end
+
+        # Compute the product of each numeric vector.
+        def product
+          compute_stats :product
         end
 
         # Create a summary of mean, standard deviation, count, max and min of 
         # each numeric vector in the dataframe in one shot.
-        def describe
-          
+        # 
+        # == Arguments
+        # 
+        # +methods+ - An array with aggregation methods specified as symbols to 
+        # be applied to numeric vectors. Default is [:count, :mean, :std, :max, 
+        # :min]. Methods will be applied in the specified order.
+        def describe methods=nil
+          methods ||= [:count, :mean, :std, :min, :max]
+
+          description_hash = {}
+          numeric_vectors.each do |vec|
+            description_hash[vec] = methods.map { |m| self[vec].send(m) }
+          end
+          Daru::DataFrame.new(description_hash, index: methods)
         end
 
         # Calculate variance-covariance between the numeric vectors.
-        def covariance
-          
+        # 
+        # == Arguments
+        # 
+        # +for_sample_data+ - If set to false, will calculate the population 
+        # covariance (denominator N), otherwise calculates the sample covariance
+        # matrix. Default to true.
+        def covariance for_sample_data=true
+          cov_arry = 
+          if defined? NMatrix and NMatrix.respond_to?(:cov)
+            to_nmatrix.cov(for_sample_data).to_a
+          else
+            df_as_matrix = to_matrix
+            denominator  = for_sample_data ? rows - 1 : rows
+            ones         = Matrix.column_vector [1]*rows
+            deviation_scores = df_as_matrix - (ones * ones.transpose * df_as_matrix) / rows
+            ((deviation_scores.transpose * deviation_scores) / denominator).to_a
+          end
+
+          Daru::DataFrame.rows(cov_arry, index: numeric_vectors, order: numeric_vectors)
         end
 
         alias :cov :covariance
 
         # Calculate the correlation between the numeric vectors.
         def correlation
-          
+          corr_arry = 
+          if defined? NMatrix and NMatrix.respond_to?(:corr)
+            to_nmatrix.corr.to_a
+          else
+            standard_deviation = std.to_matrix
+            (cov.to_matrix.elementwise_division(standard_deviation.transpose * 
+              standard_deviation)).to_a
+          end
+
+          Daru::DataFrame.rows(corr_arry, index: numeric_vectors, order: numeric_vectors)
         end
 
         alias :corr :correlation
+
+       private
+
+        def compute_stats method
+          Daru::Vector.new(
+            numeric_vectors.inject({}) do |hash, vec|
+              hash[vec] = self[vec].send(method)
+              hash
+            end
+          )
+        end
       end
     end
   end

lib/daru/maths/statistics/vector.rb

@@ -97,17 +97,17 @@ def coefficient_of_variation
         end
 
         # Retrieves number of cases which comply condition. If block given, 
-        #   retrieves number of instances where block returns true. If other 
-        #   values given, retrieves the frequency for this value. If no value
-        #   given, counts the number of elements in the Vector.
+        # retrieves number of instances where block returns true. If other 
+        # values given, retrieves the frequency for this value. If no value
+        # given, counts the number of non-nil elements in the Vector.
         def count value=false
           if block_given?
             @data.inject(0){ |memo, val| memo += 1 if yield val; memo}
           elsif value
             val = frequencies[value]
             val.nil? ? 0 : val
           else
-            size
+            size - @nil_positions.size
           end
         end
 
@@ -175,6 +175,7 @@ def percentile percent
 
         alias :sdp :standard_deviation_population
         alias :sds :standard_deviation_sample
+        alias :std :sds
         alias :adp :average_deviation_population
         alias :cov :coefficient_of_variation
         alias :variance :variance_sample    

lib/daru/monkeys.rb

@@ -50,4 +50,12 @@ class Numeric
   def square
     self * self
   end
+end
+
+class Matrix
+  def elementwise_division other
+    self.map.with_index do |e, index|
+      e / other.to_a.flatten[index]
+    end
+  end
 end

spec/dataframe_spec.rb

@@ -1369,6 +1369,7 @@
 
   context "#shape" do
     it "returns an array containing number of rows and columns" do
+      expect(@data_frame.shape).to eq([5,3])
     end
   end
 end if mri?

spec/math/statistics/dataframe_spec.rb

@@ -14,13 +14,13 @@
 
   context "#mean" do
     it "calculates mean of single level numeric only vectors and returns values in a Vector" do
-      expect(@df.mean).to eq(Daru::Vector.new([3.66, 7.33, 36.66], 
+      expect(@df.mean.round(2)).to eq(Daru::Vector.new([3.67, 7.33, 36.67], 
         index: [:d, :e, :f]
       ))
     end
 
     it "calculates mean of multi level numeric only vectors and returns values in a DataFrame" do
-      pending "next release"
+      # TODO - pending
     end
   end
 
@@ -32,7 +32,7 @@
 
   context "#sum" do
     it "calculates sum of single level numeric only vectors and returns values in a Vector" do
-      # TODO
+      # TODO - write tests
     end
   end
 
@@ -62,10 +62,10 @@
 
   context "#describe" do
     it "generates mean, std, max, min and count of numeric vectors in one shot" do
-      expect(@df.describe).to eq(Daru::DataFrame.new({
-        d: [9.00, 3.66 ,2.00 , 1.00,  7.00],
+      expect(@df.describe.round(2)).to eq(Daru::DataFrame.new({
+        d: [9.00, 3.67 ,2.00 , 1.00,  7.00],
         e: [9.00, 7.33 ,4.00 , 2.00, 14.00],
-        f: [9.00, 36.66,20.00,10.00, 70.00]
+        f: [9.00, 36.67,20.00,10.00, 70.00]
         }, index: [:count, :mean, :std, :min, :max]
       ))
     end
@@ -82,7 +82,7 @@
     end
   end
 
-  context "#corr" do
+  context "#corr", focus: true do
     it "calculates the correlation between the numeric vectors of DataFrame" do
       expect(@df.corr).to eq(Daru::DataFrame.new({
         d: [1,1,1],

spec/monkeys_spec.rb

@@ -1,6 +1,15 @@
 require 'spec_helper.rb'
 
 describe "Monkeys" do
-  context "Array" do
+  context Array do
+  end
+
+  context Matrix do
+    it "performs elementwise division" do
+      left  = Matrix[[3,6,9],[4,8,12],[2,4,6]]
+      right = Matrix[[3,6,9],[4,8,12],[2,4,6]]
+
+      expect(left.elementwise_division(right)).to eq(Matrix[[1,1,1],[1,1,1],[1,1,1]])
+    end
   end
 end