Merge pull request #21 from pakozm/devel

Version 0.4.0

Author: pakozm

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "external/luamongo"]
+	path = external/luamongo
+	url = https://github.com/moai/luamongo.git

.travis.yml

@@ -3,16 +3,15 @@ services:
 env: LUA_PATH="./?/init.lua;./?.lua"
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libssl-dev
-  - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-0.9.0.tar.gz
-  - tar zxvf legacy-0.9.0.tar.gz
-  - cd mongo-cxx-driver-legacy-0.9.0 && sudo scons --prefix=/usr install-mongoclient && cd -
-  - wget https://github.com/moai/luamongo/archive/v0.4.3.tar.gz
-  - tar zxvf v0.4.3.tar.gz
-  - cd luamongo-0.4.3 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd -
+  - sudo apt-get install -y mongodb mongodb-server screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libssl-dev
+  - git submodule update --init --recursive
+  - cd external/luamongo/external/mongo-cxx-driver/ && sudo scons --sharedclient=yes --prefix=/usr install && cd -
+  - cd external/luamongo/ && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd -
   - ssh-keygen -b 2048 -f /home/travis/.ssh/id_rsa -t rsa -q -N ""
   - ssh-keyscan -t rsa localhost > ~/.ssh/known_hosts
   - ssh-keyscan -t rsa $(hostname) >> ~/.ssh/known_hosts
   - cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
+before_script:
+  - sleep 15
 language: lua
 script: ./test.sh

external/luamongo

@@ -44,8 +44,15 @@ function cnn:gridfs()
   return gridfs
 end
 
+-- this weak table allow to index gridfs objects using weak grid_file_builder
+-- keys
+local gridfs_instances = setmetatable({}, { __mode="k" })
+
 function cnn:grid_file_builder()
-  return mongo.GridFileBuilder.New(self:connect(), self.gridfs_dbname)
+  local gridfs = self:gridfs()
+  local grid_file_builder = mongo.GridFileBuilder.New(gridfs)
+  gridfs_instances[grid_file_builder] = gridfs
+  return grid_file_builder
 end
 
 function cnn:get_dbname()

mapreduce/cnn.lua

@@ -23,13 +23,13 @@ util.omp_set_num_threads(1)
 
 local serialize_to_gridfs = function(gridfs, filename, obj)
   gridfs:remove_file(filename)
-  local builder = mongo.GridFileBuilder.New(db, dbname)
+  local builder = mongo.GridFileBuilder.New(gridfs)
   builder:append(util.serialize(obj))
   builder:build(filename)
 end
 
 local deserialize_from_gridfs = function(gridfs, filename)
-  local file = assert( gridfs:find_file(filename) )
+  local file = assert( gridfs:find_file_by_name(filename) )
   local str_tbl = {}
   for i=1,file:num_chunks() do
     local chunk = file:chunk(i-1)

mapreduce/examples/APRIL-ANN/common.lua

@@ -23,7 +23,7 @@ local tuple  = require "mapreduce.tuple"
 local persistent_table = require "mapreduce.persistent_table"
 
 local mapreduce = {
-  _VERSION = "0.3.7",
+  _VERSION = "0.4.0",
   _NAME    = "mapreduce",
   worker   = worker,
   server   = server,

mapreduce/init.lua

@@ -333,7 +333,9 @@ local function server_drop_collections(self)
   local dbname = self.cnn:get_dbname()
   -- drop all the collections
   for _,name in ipairs(db:get_collections(dbname)) do
-    db:drop_collection(name)
+    if name ~= "system_indexes" then
+      db:drop_collection(dbname .. "." .. name)
+    end
   end
   local gridfs = self.cnn:gridfs()
   local list = gridfs:list()

mapreduce/server.lua

@@ -19,7 +19,7 @@
 local mongo = require "mongo"
 local heap  = require "mapreduce.heap"
 
-assert(mongo._VERSION == "0.4" or tonumber(mongo._VERSION > 0.4))
+assert(mongo._VERSION == "0.5" or tonumber(mongo._VERSION) > 0.5)
 
 local utils = {
   _VERSION = "0.3",
@@ -131,7 +131,7 @@ end
 -- and last chunks where the line is contained, and the first and last position
 -- inside the corresponding chunks
 local function gridfs_lines_iterator(gridfs, filename)
-  local gridfile      = gridfs:find_file(filename)
+  local gridfile      = assert( gridfs:find_file_by_name(filename) )
   local size          = #gridfile
   local current_chunk = 0
   local current_pos   = 1

mapreduce/utils.lua

@@ -6,7 +6,7 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 for storage in gridfs shared sshfs; do
-    ## COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER
+    echo COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER
     screen -d -m ./execute_example_worker.sh
     diff <(./execute_example_server.sh $storage | awk '{ print $1,$2 }' | sort) \
         <(cat mapreduce/server.lua \
@@ -17,7 +17,7 @@ for storage in gridfs shared sshfs; do
         echo "ERROR"
         exit 1
     fi
-    ## NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER
+    echo NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER
     screen -d -m ./execute_example_worker.sh
     diff <(lua execute_server.lua localhost wordcount \
         mapreduce.examples.WordCount.taskfn \
@@ -34,7 +34,7 @@ for storage in gridfs shared sshfs; do
         echo "ERROR"
         exit 1
     fi
-    ## NO COMBINER + GENERAL REDUCER
+    echo NO COMBINER + GENERAL REDUCER
     screen -d -m ./execute_example_worker.sh
     diff <(lua execute_server.lua localhost wordcount \
         mapreduce.examples.WordCount.taskfn \
@@ -51,7 +51,7 @@ for storage in gridfs shared sshfs; do
         echo "ERROR"
         exit 1
     fi
-    ## INIT SCRIPT
+    echo INIT SCRIPT
     screen -d -m ./execute_example_worker.sh
     diff <(lua execute_server.lua localhost wordcount \
         mapreduce.examples.WordCount \