release v1.0.0

* add garbage collector.

Author: egg-

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.0.0
+
+2015-11-12
+
+- add garbage collector.
+- improve multiple request processing logic.
+
 # 0.9.1
 
 2015-11-09

README.md

@@ -1,5 +1,11 @@
 # redmoon
 
+[![version](https://img.shields.io/npm/v/redmoon.svg) ![download](https://img.shields.io/npm/dm/redmoon.svg)](https://www.npmjs.com/package/redmoon)
+
+Asynchronous broker for multi-source data collection using redis.
+
+If redmoon.load is called, and it returned to find the data in the cache, if cache data does not exist, call the specified function(subscriber) to subscribe.
+
 [![js-standard-style](https://cdn.rawgit.com/feross/standard/master/badge.svg)](https://github.com/feross/standard)
 
 ## Usage
@@ -9,12 +15,14 @@
 // setting
 var redmoon = require('redmoon').create(config)
 
-redmoon.on('timeout', function(keyword) {
-	console.error('redmoon timeout', keyword)
-});
-redmoon.on('error', function(err) {
-	console.error('redmoon error', err)
-});
+// event handle
+redmoon
+	.on('timeout', function(keyword) {
+		console.error('redmoon timeout', keyword)
+	})
+	.on('error', function(err) {
+		console.error('redmoon error', err)
+	});
 
 // express
 app.get('/search/:key/:page/:limit?', function (req, res) {
@@ -28,72 +36,122 @@ app.get('/search/:key/:page/:limit?', function (req, res) {
 
 // collector
 // The first event is no meta data in moon.
-// Thus, the logic required for implementation so as to avoid unnecessary requests using totalcount.
-redmoon.subscribe(function (moon) {
-  var provider = ['youtube', 'dailymotion']
-  for (var i = 0; i < provider.length; i++) {
-    search(function (err, result) {
-      var meta = {
-        provider: result.provider,
-        totalcount: result.totalcount
-      }
-
-      redmoon.add(function (err) {
-        redmoon.trigger(param.topic)
-      }, moon, meta, result.items)
-    }, provider[i], moon)
-  }
-})
+redmoon
+	.subscribe(subscriber)
+	.garbage.start(3600)	// It deletes the data collected before to 60 minutes.
+
+var subscriber = function (moon) {
+	redmoon.atomic(moon.uuid, function(cb) {
+		process(cb, moon)
+	}, function(err, data) {
+		complete(moon, data)
+	})
+}
+
+var process = function (cb, moon) {
+	// call cb when complete load data
+	cb(err, {
+		meta: {},
+		items: {}
+	})
+}
+
+var complete = function(moon, data) {
+	redmoon.add(function(err) {
+		redmoon.trigger(moon.topic)
+	}, moon, data.meta, data.items)
+}
+
+
+// moon
+// {
+// 	uuid: uuid,
+// 	key: key,
+// 	topic: topic,
+// 	meta: meta	// if first event then it is undefined
+// }
 
 ```
 
 ## Methods
 
 ### redmoon.connect(config)
 
-connect redis.
+Connect redis.
 
 ### redmoon.events()
 
-initialize redmoon events.
+Initialize redmoon events.
 * error
 * ready
 * connect
 * end
 
-### redmoon.load(cb, key, page, limit)
+### redmoon.load(cb(err, items, meta), key, page, limit)
+
+Load collected data.
+If not exist data then trigger event for collection.
+
+### redmoon.loadMeta(cb(err, metas), uuid, count)
 
-load collected data.
-if not exist data then trigger event for collection.
+Load meta data.
 
 ### redmoon.add(cb, moon, meta, items)
 
-add collected data to cache.
+Add collected data to cache.
 
 ### redmoon.subscribe(cb(err, moon))
 
-set callback function for the collection event.
+Set callback function for the collection event.
 
 ### redmoon.unsubscribe(cb())
 
-unset callback function for the collection event.
+Unset callback function for the collection event.
 
 ### redmoon.trigger(topic, data)
 
-trigger event for the specified topic.
+Trigger event for the specified topic.
+
+### redmoon.atomic(uuid, process(cb), complete(err, data))
+
+The function to prevent the duplication proccess.
+
+If it has available resources, given `process` calls and then release the resources used by `cb` call in process.
+After the release, and it executes the transfer of data in `proccess` to `complete`.
 
 ### redmoon.end()
 
-close the connection to the redis server.
+Close the connection to the redis server.
 
-### Redmoon.create(config)
+### redmoon.truncate(unix)
+
+Deleting cache data collected before a specified time.
 
-create Redmoon instance with config.
+### redmmon.toTopic(uuid)
 
+It converts uuid to the topic.
+
+### redmoon.garbage.start(offset)
+
+Start garbage collection.
+Deleting cache data collected before a offset time.
+
+### redmoon.garbage.stop()
+
+Stop garbage collection.
+
+### Redmoon.uuid(key)
+
+Make unique string.
+
+### Redmmon.uuid()
+
+Return unixtimestamp.
+
+### Redmoon.create(config)
 
-## Todo
+Create Redmoon instance with config.
 
-* Garbage collector.
 
 ## Release History
 

lib/redmoon.js

@@ -9,20 +9,35 @@ function Redmoon (config) {
   this._client = null
   this._nrp = null
 
+  this._timer = null
+
   this.config = {
     scope: 'redmoon',
     name: 'default',
     host: '127.0.0.1',
     port: 6379,
     timeout: 5000, // 5 sec
-    buffer: 100,
-    ttl: 5
+    buffer: 100, // buffer count for preloading
+    ttl: 5, // max time for atomic
+    interval: 60 // interval value for garbage collection
   }
   this.key = {
     atomic: [this.config.scope, 'atomic'].join(':'), // hash
     meta: [this.config.scope, 'meta'].join(':'), // hash
     cycle: [this.config.scope, 'cycle'].join(':') // set (sorted)
   }
+
+  this.garbage = {
+    start: function (offset) {
+      this._timer = setInterval(function () {
+        this.truncate(Redmoon.unix() - offset)
+      }.bind(this), this.config.interval * 1000)
+    }.bind(this),
+    stop: function () {
+      clearInterval(this._timer)
+      this._timer = null
+    }.bind(this)
+  }
 }
 
 util.inherits(Redmoon, events.EventEmitter)
@@ -37,6 +52,9 @@ Redmoon.prototype.connect = function (config) {
   typeof config.host !== 'undefined' && (this.config.host = config.host)
   typeof config.port !== 'undefined' && (this.config.port = config.port)
   typeof config.timeout !== 'undefined' && (this.config.timeout = config.timeout)
+  typeof config.buffer !== 'undefined' && (this.config.buffer = config.buffer)
+  typeof config.ttl !== 'undefined' && (this.config.ttl = config.ttl)
+  typeof config.interval !== 'undefined' && (this.config.interval = config.interval)
 
   this._client = require('redis').createClient(this.config.port, this.config.host)
   this._nrp = new NRP(this.config)
@@ -72,14 +90,11 @@ Redmoon.prototype.load = function (cb, key, page, limit) {
   limit = limit || 10
 
   // check exist search result on meta
-  this._client.send_command('HSCAN', [this.key.meta, 0, 'MATCH', uuid + ':*', 'COUNT', 1000], function (err, metas) {
+  this.loadMeta(function (err, metas) {
     if (err) {
       return cb(err)
     }
 
-    // migrate
-    metas = metas[1]
-
     if (metas.length > 0) {
       var start = (page - 1) * limit
       var end = start + limit - 1
@@ -132,12 +147,20 @@ Redmoon.prototype.load = function (cb, key, page, limit) {
         self._nrp.off(topic)
         self.load(cb, key, page, limit)
       })
-      self._nrp.emit([self.config.name, uuid].join(':'), {
+      self._nrp.emit([self.config.scope, self.config.name, 'request', uuid].join(':'), {
         uuid: uuid,
         key: key,
         topic: topic
       })
     }
+  }, uuid)
+
+  return this
+}
+
+Redmoon.prototype.loadMeta = function (cb, uuid, count) {
+  this._client.send_command('HSCAN', [this.key.meta, 0, 'MATCH', uuid + ':*', 'COUNT', count || 1000], function (err, metas) {
+    cb(err, metas ? metas[1] : [])
   })
 
   return this
@@ -161,26 +184,29 @@ Redmoon.prototype.add = function (cb, moon, meta, items) {
 }
 
 Redmoon.prototype.subscribe = function (cb) {
-  this._nrp.on(this.config.name + ':*', cb)
+  this._nrp.on([this.config.scope, this.config.name, 'request', '*'].join(':'), cb)
+  return this
 }
 
 Redmoon.prototype.unsubscribe = function (cb) {
-  this._nrp.off(this.config.name + ':*', cb)
+  this._nrp.off([this.config.scope, this.config.name, 'request', '*'].join(':'), cb)
+  return this
 }
 
 Redmoon.prototype.trigger = function (topic, data) {
   this._nrp.emit(topic, data || {})
+  return this
 }
 
-// 중복 이벤트 처리를 방지 목적.
-Redmoon.prototype.atomic = function (cb, uuid) {
+// atomic processing
+Redmoon.prototype.atomic = function (uuid, process, complete) {
   // atomic
   var self = this
   var key = [this.key.atomic, uuid].join(':')
 
   this._client.get(key, function (err, item) {
     if (err) {
-      return console.error(err)
+      return self.emit('error', err)
     }
 
     if (item) {
@@ -191,13 +217,19 @@ Redmoon.prototype.atomic = function (cb, uuid) {
     self._client.multi()
       .set(key, uuid)
       .expire(key, self.config.ttl)
-      .exec(cb)
-  })
-}
+      .exec(function (err) {
+        if (err) {
+          return self.emit('error', err)
+        }
 
-// 중복 이벤트 처리를 방지 목적.
-Redmoon.prototype.release = function (cb, uuid) {
-  this._client.del([this.key.atomic, uuid].join(':'), cb)
+        process(function (err, data) {
+          self._client.del([self.key.atomic, uuid].join(':'), function () {
+            complete(err, data)
+          })
+        })
+      })
+  })
+  return this
 }
 
 Redmoon.prototype.end = function () {
@@ -210,10 +242,47 @@ Redmoon.prototype.end = function () {
     this._nrp.end()
     this._nrp = null
   }
+  return this
+}
+
+Redmoon.prototype.truncate = function (unix) {
+  var self = this
+  unix = unix || Redmoon.unix() - 3600 // default 1 hour
+
+  this._client.zrangebyscore(this.key.cycle, 0, unix, function (err, items) {
+    if (err) {
+      return self.emit('error', err)
+    }
+
+    for (var i = 0; i < items.length; i++) {
+      self.loadMeta((function (uuid) {
+        return function (err, metas) {
+          if (err) {
+            return self.emit('error', err)
+          }
+
+          var multi = self._client.multi()
+
+          for (var j = 0; j < metas.length; j += 2) {
+            multi.hdel(self.key.meta, metas[j])
+          }
+
+          multi.del([self.config.scope, self.config.name, uuid].join(':'))
+          multi.zrem(self.key.cycle, uuid)
+          multi.exec(function (err, replies) {
+            if (err) {
+              return self.emit('error', err)
+            }
+          })
+        }
+      })(items[i]), items[i])
+    }
+  })
+  return this
 }
 
 Redmoon.prototype.toTopic = function (uuid) {
-  return [shortid.generate(), this.config.name, 'result', uuid].join(':')
+  return [this.config.name, 'result', uuid].join(':')
 }
 
 Redmoon.uuid = function (key) {