Merge pull request #108 from RTICWDT/dev-long

Handle Long Integers

Author: brownpl

Gemfile.lock

@@ -30,7 +30,7 @@ GEM
     concurrent-ruby (1.1.7)
     crass (1.0.6)
     diff-lcs (1.4.4)
-    docile (1.3.2)
+    docile (1.3.4)
     dotenv (2.7.6)
     elasticsearch (5.0.5)
       elasticsearch-api (= 5.0.5)
@@ -44,13 +44,15 @@ GEM
     erubis (2.7.0)
     ethon (0.12.0)
       ffi (>= 1.3.0)
-    excon (0.78.0)
-    faraday (1.1.0)
+    excon (0.78.1)
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords
-    ffi (1.13.1)
+    faraday-net_http (1.0.0)
+    ffi (1.14.2)
     hashie (4.1.0)
-    i18n (1.8.5)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     liquid (3.0.3)
@@ -66,18 +68,19 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.1104)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.2)
+    mini_portile2 (2.5.0)
+    minitest (5.14.3)
     moneta (1.1.1)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     newrelic_rpm (6.14.0)
     nio4r (2.5.4)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    oj (3.10.16)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    oj (3.10.18)
     padrino (0.15.0)
       padrino-admin (= 0.15.0)
       padrino-cache (= 0.15.0)
@@ -119,6 +122,7 @@ GEM
       pry (~> 0.13.0)
     puma (4.3.7)
       nio4r (~> 2.0)
+    racc (1.5.2)
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
@@ -129,23 +133,23 @@ GEM
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    rake (13.0.1)
+    rake (13.0.3)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
       rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
+    rspec-core (3.10.1)
       rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+    rspec-mocks (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
+    rspec-support (3.10.1)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     ruby-prof (1.4.2)
@@ -156,7 +160,7 @@ GEM
     sass-listen (4.0.0)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
-    simplecov (0.20.0)
+    simplecov (0.21.1)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -175,7 +179,7 @@ GEM
     tilt (2.0.10)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.8)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
     zeitwerk (2.4.2)
 

lib/data_magic/index.rb

@@ -153,7 +153,7 @@ def self.import_with_delta(options = {})
 
 private
   def self.valid_types
-    %w[integer float string literal name autocomplete boolean]
+    %w[integer float string literal name autocomplete boolean long]
   end
 
 end # module DataMagic

lib/data_magic/index/document_builder.rb

@@ -132,6 +132,8 @@ def fix_field_type(type, value, key=nil)
                         value.to_s.downcase
                       when "boolean"
                         parse_boolean(value)
+                      when "long"
+                        value.to_i
                       else # "string"
                         value.to_s
           end

sample-data/data.yaml

@@ -7,10 +7,10 @@ version: cities100-2010
 # then convertes to csv and removed " city" from after each city name
 index: city-data
 api: cities
-unique: ['name']
+unique: ["name"]
 
 options:
-  search: dictionary_only   # API provides error when requesting fields not in dictionary
+  search: dictionary_only # API provides error when requesting fields not in dictionary
 
 dictionary:
   id:
@@ -51,7 +51,6 @@ dictionary:
   land_area:
     source: ALAND_SQMI
     description: Land Area (square miles)
-    source: ALAND_SQMI
     type: float
   area.water:
     description: Water Area (square miles)

tasks/delta.rake

@@ -1,4 +1,4 @@
-require 'data_magic'
+require_relative '../lib/data_magic.rb'
 
 desc 'delta update current ES index with delta data file. ARG 1 must be title as specified in data.yaml, e.g., Most-Recent-Cohorts-All-Data-Elements.csv. ARG 2 is the filename of the new delta file nested in a "<DATA_PATH>/delta/<DELTA_CSV_FILE>" subdirectory. USAGE delta[ORIGINAL_FILENAME.csv,DELTA_FILENAME.csv]'
 task :delta, [:original, :update] => :environment do |t, args|

tasks/es.rake

@@ -1,4 +1,4 @@
-require 'data_magic'
+require_relative '../lib/data_magic.rb'
 
 namespace :es do
   desc "delete elasticsearch index (_all for all)"

tasks/import.rake

@@ -1,4 +1,4 @@
-require 'data_magic'
+require_relative '../lib/data_magic.rb'
 require 'ruby-prof'
 
 desc "import files from DATA_PATH via rake import, optionally rake import[filename.csv] to restart import from a specific file, also rake import[nil,profile=true] for profile output."