diff --git a/.gitignore b/.gitignore index 5d18e3d..81b3083 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ rdoc spec/reports tmp bundler_bin +.idea \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index d2d1279..9424b56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,6 +6,7 @@ PATH colorful execjs json_pure + nokogiri rake slop @@ -15,14 +16,17 @@ GEM coffee-script-source (1.6.3) colorful (0.0.3) diff-lcs (1.2.1) - execjs (2.0.1) + execjs (2.0.2) ffi (1.0.11) guard (1.0.1) ffi (>= 0.5.0) thor (~> 0.14.6) guard-rspec (0.6.0) guard (>= 0.10.0) - json_pure (1.8.0) + json_pure (1.8.1) + mini_portile (0.5.1) + nokogiri (1.6.0) + mini_portile (~> 0.5.0) rake (10.1.0) rspec (2.13.0) rspec-core (~> 2.13.0) @@ -41,4 +45,5 @@ PLATFORMS DEPENDENCIES bwoken! guard-rspec + nokogiri rspec diff --git a/bwoken.gemspec b/bwoken.gemspec index c5f9672..d733730 100644 --- a/bwoken.gemspec +++ b/bwoken.gemspec @@ -23,7 +23,9 @@ Gem::Specification.new do |gem| gem.add_dependency 'json_pure' gem.add_dependency 'rake' gem.add_dependency 'slop' + gem.add_dependency 'nokogiri' gem.add_development_dependency 'rspec' gem.add_development_dependency 'guard-rspec' + gem.add_development_dependency 'nokogiri' end diff --git a/lib/bwoken/cli/test.rb b/lib/bwoken/cli/test.rb index b9a5c7f..5b019b9 100644 --- a/lib/bwoken/cli/test.rb +++ b/lib/bwoken/cli/test.rb @@ -10,6 +10,7 @@ require 'bwoken/formatter' require 'bwoken/formatters/passthru_formatter' require 'bwoken/formatters/colorful_formatter' +require 'bwoken/formatters/junit_formatter' require 'bwoken/script_runner' module Bwoken @@ -115,6 +116,7 @@ def clean def select_formatter formatter_name case formatter_name when 'passthru' then Bwoken::PassthruFormatter.new + when 'junit' then Bwoken::JUnitFormatter.new else Bwoken::ColorfulFormatter.new end end diff --git a/lib/bwoken/formatter.rb b/lib/bwoken/formatter.rb index ff3a231..5daee5a 100644 --- a/lib/bwoken/formatter.rb +++ b/lib/bwoken/formatter.rb @@ -10,9 +10,9 @@ def format_build stdout new.format_build stdout end - def on name, &block + def on(name, formatter=nil, &block) define_method "_on_#{name}_callback" do |*line| - block.call(*line) + block.call(*line, formatter) end end @@ -28,26 +28,26 @@ def method_missing(method_name, *args, &block) def line_demuxer line, exit_status if line =~ /Instruments Trace Error/ exit_status = 1 - _on_fail_callback(line) + _on_fail_callback(line,self) elsif line =~ /^\d{4}/ tokens = line.split(' ') if tokens[3] =~ /Pass/ - _on_pass_callback(line) + _on_pass_callback(line,self) elsif tokens[3] =~ /Start/ - _on_start_callback(line) + _on_start_callback(line,self) elsif tokens[3] =~ /Fail/ || line =~ /Script threw an uncaught JavaScript error/ exit_status = 1 - _on_fail_callback(line) + _on_fail_callback(line,self) elsif tokens[3] =~ /Error/ - _on_error_callback(line) + _on_error_callback(line,self) else - _on_debug_callback(line) + _on_debug_callback(line, self) end elsif line =~ /Instruments Trace Complete/ - _on_complete_callback(line) + _on_complete_callback(line,self) else - _on_other_callback(line) + _on_other_callback(line,self) end exit_status end @@ -73,7 +73,7 @@ def format_build stdout stdout.each_line do |line| out_string << line if line.length > 1 - _on_build_line_callback(line) + _on_build_line_callback(line,self) end end out_string diff --git a/lib/bwoken/formatters/junit_formatter.rb b/lib/bwoken/formatters/junit_formatter.rb new file mode 100644 index 0000000..9da81d7 --- /dev/null +++ b/lib/bwoken/formatters/junit_formatter.rb @@ -0,0 +1,247 @@ +require 'nokogiri' +require 'bwoken/formatter' + +module Bwoken + + class JUnitTestSuite + attr_accessor :id + attr_accessor :package + attr_accessor :host_name + attr_accessor :name + attr_accessor :tests + attr_accessor :failures + attr_accessor :errors + attr_accessor :time + attr_accessor :timestamp + + attr_accessor :test_cases + + def initialize + self.test_cases = [] + self.tests = 0 + self.failures = 0 + self.errors = 0 + end + + def complete + self.time = Time.now - self.timestamp + end + + end + + class JUnitTestCase + attr_accessor :name + attr_accessor :classname + attr_accessor :time + attr_accessor :error + attr_accessor :logs + + attr_accessor :start_time + + def initialize + self.logs = String.new + self.error = nil + end + + def complete + self.time = Time.now - self.start_time + end + + end + + class JUnitFormatter < Formatter + attr_accessor :test_suites + + def initialize + self.test_suites = [] + end + + def format stdout + exit_status = super stdout + generate_report + exit_status + end + + + on :complete do |line,formatter| + tokens = line.split(' ') + test_suite = formatter.test_suites.last + test_suite.time = tokens[5].sub(';', '') + end + + on :debug do |line, formatter| + filtered_line = line.sub(/(target\.frontMostApp.+)\.tap\(\)/, "#{'tap'} \\1") + filtered_line = filtered_line.gsub(/\[("[^\]]*")\]/, "[" + '\1' + "]") + filtered_line = filtered_line.gsub('()', '') + filtered_line = filtered_line.sub(/target.frontMostApp.(?:mainWindow.)?/, '') + tokens = filtered_line.split(' ') + + test_suite = formatter.test_suites.last + test_case = test_suite.test_cases.last + + if test_case + test_case.logs << "\n#{tokens[3].cyan}\t#{tokens[4..-1].join(' ')}" + end + end + + on :error do |line,formatter| + @failed = true + tokens = line.split(' ') + + test_suite = formatter.test_suites.last + test_case = test_suite.test_cases.last + if test_case + test_case.complete + test_case.error = tokens[4..-1].join(' ') + end + + test_suite.errors += 1 + + end + + on :fail do |line,formatter| + @failed = true + tokens = line.split(' ') + + test_suite = formatter.test_suites.last + test_case = test_suite.test_cases.last + if test_case + test_case.complete + test_case.error = tokens[4..-1].join(' ') + end + + test_suite.failures += 1 + + end + + on :start do |line,formatter| + tokens = line.split(' ') + + suite = formatter.test_suites.last + if suite + test_case = Bwoken::JUnitTestCase.new + test_case.name = tokens[4..-1].join(' ') + test_case.classname = test_case.name + test_case.start_time = Time.now + + suite.tests+=1 + suite.test_cases << test_case + end + end + + on :pass do |line,formatter| + tokens = line.split(' ') + + test_case = formatter.test_suites.last.test_cases.last + if test_case + test_case.complete + test_case.error = nil + end + end + + on :before_script_run do |path, formatter| + tokens = path.split('/') + + new_suite = Bwoken::JUnitTestSuite.new + new_suite.timestamp = Time.now + new_suite.host_name = tokens[-2] + new_suite.name = tokens[-1] + new_suite.package = new_suite.name + new_suite.id = formatter.test_suites.count + 1 + + formatter.test_suites << new_suite + + @failed = false + end + + on :before_build_start do + print "Building" + end + + on :build_line do |line,formatter| + print '.' + end + + on :build_successful do |line,formatter| + puts + puts 'Build Successful!' + end + + on :build_failed do |build_log, error_log| + puts build_log + puts 'Standard Error:' + puts error_log + puts 'Build failed!' + end + + on :other do |line,formatter| + nil + end + + + def generate_report + doc = Nokogiri::XML::Document.new() + root = Nokogiri::XML::Element.new('testsuites', doc) + doc.add_child(root) + + result_name = 'unknown' + + self.test_suites.each do |suite| + result_name = suite.name.gsub /\.js$/, '' + + suite_elm = Nokogiri::XML::Element.new('testsuite', doc) + suite_elm['id'] = suite.id + suite_elm['package'] = suite.package + suite_elm['hostname'] = suite.host_name + suite_elm['name'] = suite.name + suite_elm['tests'] = suite.tests + suite_elm['failures'] = suite.failures + suite_elm['errors'] = suite.errors + suite_elm['time'] = suite.time + suite_elm['timestamp'] = suite.timestamp.to_s + + system_out = '' + system_err = '' + + suite.test_cases.each do |test_case| + test_case_elm = Nokogiri::XML::Element.new('testcase', doc) + test_case_elm['name'] = test_case.name + test_case_elm['classname'] = test_case.classname + test_case_elm['time'] = test_case.time + + if test_case.error + error = Nokogiri::XML::Element.new('error', doc) + error['type'] = test_case.error + test_case_elm.add_child(error) + system_err << "\n\n#{test_case.logs}" + else + system_out << "\n\n#{test_case.logs}" + end + + suite_elm.add_child(test_case_elm) + end + + suite_elm.add_child("#{doc.create_cdata(system_out)}") + suite_elm.add_child("#{doc.create_cdata(system_err)}") + + root.add_child(suite_elm) + + end + + out_xml = doc.to_xml + + write_results(out_xml, result_name) + end + + def write_results(xml, suite_name) + output_path = File.join(Bwoken.results_path, "#{suite_name}_results.xml") + File.open(output_path, 'w+') do |io| + io.write(xml) + end + + puts "\nJUnit report generated to #{output_path}\n\n" + end + + + end +end diff --git a/lib/bwoken/script.rb b/lib/bwoken/script.rb index 606fe4d..2fd2f65 100644 --- a/lib/bwoken/script.rb +++ b/lib/bwoken/script.rb @@ -48,7 +48,7 @@ def device_flag end def run - formatter.before_script_run path + formatter.before_script_run path, formatter Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr| exit_status = formatter.format stdout diff --git a/spec/lib/bwoken/formatter_spec.rb b/spec/lib/bwoken/formatter_spec.rb index ee13c19..20aaaa1 100644 --- a/spec/lib/bwoken/formatter_spec.rb +++ b/spec/lib/bwoken/formatter_spec.rb @@ -40,7 +40,7 @@ context 'for a passing line' do it 'calls _on_pass_callback' do - subject.should_receive(:_on_pass_callback).with('1234 a a Pass') + subject.should_receive(:_on_pass_callback).with('1234 a a Pass', subject) subject.line_demuxer('1234 a a Pass', 0) end it 'returns 0' do @@ -55,7 +55,7 @@ context 'for a failing line' do context 'Fail error' do it 'calls _on_fail_callback' do - subject.should_receive(:_on_fail_callback).with('1234 a a Fail') + subject.should_receive(:_on_fail_callback).with('1234 a a Fail', subject) subject.line_demuxer('1234 a a Fail', 0) end end @@ -63,7 +63,7 @@ context 'Instruments Trace Error message' do it 'calls _on_fail_callback' do msg = 'Instruments Trace Error foo' - subject.should_receive(:_on_fail_callback).with(msg) + subject.should_receive(:_on_fail_callback).with(msg, subject) subject.line_demuxer(msg, 0) end end @@ -79,14 +79,14 @@ context 'for a debug line' do it 'calls _on_debug_callback' do - subject.should_receive(:_on_debug_callback).with('1234 a a feh') + subject.should_receive(:_on_debug_callback).with('1234 a a feh', subject) subject.line_demuxer('1234 a a feh', 0) end end context 'for any other line' do it 'calls _on_other_callback' do - subject.should_receive(:_on_other_callback).with('blah blah blah') + subject.should_receive(:_on_other_callback).with('blah blah blah', subject) subject.line_demuxer('blah blah blah', 0) end end diff --git a/spec/lib/bwoken/formatters/junit_formatter_spec.rb b/spec/lib/bwoken/formatters/junit_formatter_spec.rb new file mode 100644 index 0000000..dbedd76 --- /dev/null +++ b/spec/lib/bwoken/formatters/junit_formatter_spec.rb @@ -0,0 +1,162 @@ +require 'spec_helper' +require 'bwoken/formatters/junit_formatter' + +describe Bwoken::JUnitTestSuite do + describe '#initialize' do + it 'sets initial state for an instance' do + expect(subject.test_cases).to be_kind_of Array + expect(subject.test_cases).to have(0).items + expect(subject.tests).to eq(0) + expect(subject.failures).to eq(0) + expect(subject.errors).to eq(0) + end + end + + describe '#complete' do + it 'calculates the correct elapsed time for a test' do + subject.timestamp = Time.now + sleep 0.1 + subject.complete + expect(subject.time.round(1)).to eq(0.1) + end + end + +end + +describe Bwoken::JUnitTestCase do + describe '#initialize' do + it 'sets initial state for an instance' do + expect(subject.logs).to be_kind_of String + expect(subject.error).to be_nil + end + + end + + describe '#complete' do + it 'calculates the correct elapsed time for a test case' do + subject.start_time = Time.now + sleep 0.1 + subject.complete + expect(subject.time.round(1)).to eq(0.1) + end + end +end + +describe Bwoken::JUnitFormatter do + describe '.on' do + it 'increments tests counter when a test is run' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + formatter._on_start_callback('2013-10-25 16:10:01 +0000 Start: test one', formatter) + expect(formatter.test_suites[0].tests).to eq(1) + end + + it 'increments failure counter when a test fails' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + test_case = Bwoken::JUnitTestCase.new + test_case.start_time = Time.now + formatter.test_suites[0].test_cases = [test_case] + formatter._on_fail_callback('2013-10-25 16:10:01 +0000 Fail: login', formatter) + expect(formatter.test_suites[0].failures).to eq(1) + end + + it 'increments error counter when a test error occurs' do + formatter = Bwoken::JUnitFormatter.new + formatter.test_suites = [Bwoken::JUnitTestSuite.new] + test_case = Bwoken::JUnitTestCase.new + test_case.start_time = Time.now + formatter.test_suites[0].test_cases = [test_case] + formatter._on_error_callback('2013-10-25 16:10:01 +0000 Error: login', formatter) + expect(formatter.test_suites[0].errors).to eq(1) + end + end + + + describe '#generate_report' do + it 'outputs a valid XML report for test suites' do + # Setup + #=================================================================================================================== + now = Time.new(2013, 10, 25, 10, 34, 51, '-05:00') + + test_suite = Bwoken::JUnitTestSuite.new + test_suite.id = 'suite id' + test_suite.package = 'suite package' + test_suite.host_name = 'suite host_name' + test_suite.name = 'suite_name.js' + test_suite.tests = 2 + test_suite.failures = 1 + test_suite.errors = 1 + test_suite.timestamp = now + test_suite.time = 10.0 + + test_case_passed = Bwoken::JUnitTestCase.new + test_case_passed.name = 'test one' + test_case_passed.classname = 'TestOne' + test_case_passed.time = 3.0 + test_case_passed.logs = 'test one logs' + + test_case_failed = Bwoken::JUnitTestCase.new + test_case_failed.name = 'test two' + test_case_failed.classname = 'TestTwo' + test_case_failed.time = 5.0 + test_case_failed.logs = 'test two logs' + test_case_failed.error = 'case 2 error' + + test_suite.test_cases << test_case_passed + test_suite.test_cases << test_case_failed + + subject.test_suites = [test_suite] + + + # Assert + #=================================================================================================================== + subject.stub(:write_results) do |xml, suite_name| + + expect(xml).to be_kind_of(String) + + # Check the test suite + expect(xml.scan(/testsuite\sid="([^"]+)"/)[0]).to include('suite id') + expect(xml.scan(/hostname="([^"]+)"/)[0]).to include('suite host_name') + expect(xml.scan(/testsuite.*name="([^"]+)"/)[0]).to include('suite_name.js') + expect(xml.scan(/testsuite.*tests="([^"]+)"/)[0]).to include('2') + expect(xml.scan(/testsuite.*failures="([^"]+)"/)[0]).to include('1') + expect(xml.scan(/testsuite.*errors="([^"]+)"/)[0]).to include('1') + expect(xml.scan(/testsuite.*time="([^"]+)"/)[0]).to include('10.0') + expect(xml.scan(/testsuite.*timestamp="([^"]+)"/)[0]).to include('2013-10-25 10:34:51 -0500') + + # Check the test cases + expect(xml.scan(/testcase.*\sname="([^"]+)"/)[0]).to include('test one') + expect(xml.scan(/testcase.*\sclassname="([^"]+)"/)[0]).to include('TestOne') + expect(xml.scan(/testcase.*\stime="([^"]+)"/)[0]).to include('3.0') + + expect(xml.scan(/testcase.*\sname="([^"]+)"/)[1]).to include('test two') + expect(xml.scan(/testcase.*\sclassname="([^"]+)"/)[1]).to include('TestTwo') + expect(xml.scan(/testcase.*\stime="([^"]+)"/)[1]).to include('5.0') + + # Check stdout for logs + expect(xml.scan(/system-out.*\n.*\n.*test one logs/)).to have(1).items + expect(xml.scan(/system-err.*\n.*\n.*test two logs/)).to have(1).items + + # Ensure that the resultant document passes XSD validation + xsd = Nokogiri::XML::Schema(File.read(File.expand_path("#{File.dirname(__FILE__)}/../../../support/junit-4.xsd"))) + doc = Nokogiri::XML(xml) + + errors = [] + xsd.validate(doc).each do |error| + puts "Error: #{error}" + errors << error + end + + expect(errors).to have(0).items + + end + + # Test + #=================================================================================================================== + + subject.generate_report + + end + end +end diff --git a/spec/lib/bwoken/script_spec.rb b/spec/lib/bwoken/script_spec.rb index c347a71..9071bbc 100644 --- a/spec/lib/bwoken/script_spec.rb +++ b/spec/lib/bwoken/script_spec.rb @@ -21,7 +21,7 @@ class Simulator; end it 'outputs that a script is about to run' do subject.path = 'path' - subject.formatter.should_receive(:before_script_run).with('path') + subject.formatter.should_receive(:before_script_run).with('path', subject.formatter) Open3.stub(:popen3) subject.stub(:cmd) subject.run diff --git a/spec/support/junit-4.xsd b/spec/support/junit-4.xsd new file mode 100644 index 0000000..dd515df --- /dev/null +++ b/spec/support/junit-4.xsd @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file