From 926948d8ec6563de83181e486d9af6ab1bf4c7f0 Mon Sep 17 00:00:00 2001 From: Raj Sahae Date: Thu, 27 Feb 2014 11:14:37 -0800 Subject: [PATCH 1/2] Added a gzip file watcher. Initially, I started refactoring everything but as this isn't my gem I ended up simply inheriting FileWatch::Tail as to maintain backwards compatibility as much as possible. Unfortunately, if Tail ever changes, Gzip will need to be retested. --- filewatch.gemspec | 2 +- lib/filewatch/gzip.rb | 63 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 lib/filewatch/gzip.rb diff --git a/filewatch.gemspec b/filewatch.gemspec index 7a59a8b..1becd8c 100644 --- a/filewatch.gemspec +++ b/filewatch.gemspec @@ -6,7 +6,7 @@ Gem::Specification.new do |spec| end spec.name = "filewatch" - spec.version = "0.5.0" + spec.version = "0.6.0" spec.summary = "filewatch - file watching for ruby" spec.description = "Watch files and directories in ruby. Also supports tailing and glob file patterns." spec.files = files diff --git a/lib/filewatch/gzip.rb b/lib/filewatch/gzip.rb new file mode 100644 index 0000000..6106f45 --- /dev/null +++ b/lib/filewatch/gzip.rb @@ -0,0 +1,63 @@ +#!/usr/bin/env ruby +# encoding: UTF-8 + +require "filewatch/tail" +require 'zlib' + +module FileWatch + class Gzip < Tail + + def initialize(opts = {}) + opts[:start_new_files_at] = :beginning + super(opts) + end + + public + ## + # yields |path, line| to block + def subscribe(&block) + @watch.subscribe(@opts[:stat_interval], + @opts[:discover_interval]) do |event, path| + @logger.debug("#{self.class}#subscribe"){"Event: #{event} Path: #{path}"} + + case event + when :create, :create_initial + if @files.member?(path) + @logger.debug("#{self.class}#subscribe"){"#{event} for #{path}: already exists in @files"} + next + end + if _open_file(path, event) + _read_file(path, &block) + end + when :modify + if !@files.member?(path) + @logger.debug(":modify for #{path}, does not exist in @files") + end + when :delete + @logger.debug(":delete for #{path}, deleted from @files") + @files[path].close + @files.delete(path) + @statcache.delete(path) + else + @logger.warn("unknown event type #{event} for #{path}") + end + + end # @watch.subscribe + end # def subscribe + + private + def _read_file(path, &block) + begin + @logger.debug("#{self.class}#_read_file"){"GzipReader on #{path}"} + gz = Zlib::GzipReader.new(@files[path]) + gz.each_line{|line| yield(path, line)} + rescue Zlib::Error, Zlib::GzipFile::Error, Zlib::GzipFile::NoFooter, Zlib::GzipFile::CRCError, Zlib::GzipFile::LengthError => e + @logger.debug("#{self.class}#_read_file"){"#{e.class}:#{e.message} path:#{path}"} + ensure + gz.close + end + + end # _read_file + + end # class Gzip +end # module FileWatch From ec10ccf743b8e22048b0a4f8204860d37346b158 Mon Sep 17 00:00:00 2001 From: Raj Sahae Date: Thu, 27 Feb 2014 11:48:18 -0800 Subject: [PATCH 2/2] Added comments --- lib/filewatch/gzip.rb | 62 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/lib/filewatch/gzip.rb b/lib/filewatch/gzip.rb index 6106f45..7d18423 100644 --- a/lib/filewatch/gzip.rb +++ b/lib/filewatch/gzip.rb @@ -7,14 +7,52 @@ module FileWatch class Gzip < Tail + # Public: Initialize a new FileWatch::Gzip object + # + # opts - An options hash (default: {}) + # :sincedb_write_interval - The Integer in seconds write to database (default: 10) + # :stat_interval - The Integer in seconds to sleep inbetween stat checks (default: 1) + # :discover_interval - The Integer in seconds to wait between globs (default: 5) + # :exclude - The array of Strings to exclude from glob check (default: []) + # + # Examples + # + # FileWatch::Gzip.new({:discover_interval => 10, :exclude => ['/tmp/*']}) + # def initialize(opts = {}) opts[:start_new_files_at] = :beginning super(opts) end + + # Public: Watch a path for new gzipped files + # + # path - The String glob expression to watch for gzipped files + # + # Returns true if successfull, false otherwise + # + # Examples + # + # gz = FileWatch::Gzip.new + # gz.watch('/tmp/*gz') # => true + # + alias_method :watch, :tail public - ## - # yields |path, line| to block + # Public: Initiates the loop that watches for files. Takes a block. + # + # block - Mandatory block. It will be sent the path to a file and a line + # from that file. + # + # Yields - The path of a file, and a line within that file + # + # Examples + # + # gz = FileWatch::Gzip.new + # gz.watch('/tmp/*gz') + # gz.subscribe{|path, line| puts "#{path}:#{line}"} + # + # Returns nothing. Runs infinitely. + # def subscribe(&block) @watch.subscribe(@opts[:stat_interval], @opts[:discover_interval]) do |event, path| @@ -48,15 +86,29 @@ def subscribe(&block) private def _read_file(path, &block) begin + changed = false @logger.debug("#{self.class}#_read_file"){"GzipReader on #{path}"} gz = Zlib::GzipReader.new(@files[path]) - gz.each_line{|line| yield(path, line)} + gz.each_line do |line| + changed = true + yield(path, line) + end + @sincedb[@statcache[path]] = @files[path].pos + + if changed + now = Time.now.to_i + delta = now - @sincedb_last_write + if delta >= @opts[:sincedb_write_interval] + @logger.debug("writing sincedb (delta since last write = #{delta})") + _sincedb_write + @sincedb_last_write = now + end + end rescue Zlib::Error, Zlib::GzipFile::Error, Zlib::GzipFile::NoFooter, Zlib::GzipFile::CRCError, Zlib::GzipFile::LengthError => e @logger.debug("#{self.class}#_read_file"){"#{e.class}:#{e.message} path:#{path}"} ensure - gz.close + gz.close unless gz.nil? end - end # _read_file end # class Gzip