diff --git a/lib/omnibus/health_check.rb b/lib/omnibus/health_check.rb index 6eb60b973..62c2e2f0b 100644 --- a/lib/omnibus/health_check.rb +++ b/lib/omnibus/health_check.rb @@ -1,5 +1,5 @@ -# Copyright 2012-2018 Chef Software, Inc. +# Copyright:: Copyright (c) Chef Software Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -66,20 +66,25 @@ def initialize(project) def run! measure("Health check time") do log.info(log_key) { "Running health on #{project.name}" } - bad_libs = case Ohai["platform"] - when "mac_os_x" - health_check_otool - when "aix" - health_check_aix - when "windows" - # TODO: objdump -p will provided a very limited check of - # explicit dependencies on windows. Most dependencies are - # implicit and hence not detected. - log.warn(log_key) { "Skipping dependency health checks on Windows." } - {} - else - health_check_ldd - end + bad_libs, good_libs = + case Ohai["platform"] + when "mac_os_x" + health_check_otool + when "aix" + health_check_aix + when "windows" + # TODO: objdump -p will provided a very limited check of + # explicit dependencies on windows. Most dependencies are + # implicit and hence not detected. + log.warn(log_key) { "Skipping dependency health checks on Windows." } + [{}, {}] + when "solaris2" + health_check_solaris + when "freebsd", "openbsd", "netbsd" + health_check_freebsd + else + health_check_linux + end unresolved = [] unreliable = [] @@ -167,6 +172,10 @@ def run! raise HealthCheckFailed end + if good_libs.keys.length == 0 && !windows? + raise "Internal error: no good libraries were found" + end + conflict_map = {} conflict_map = relocation_check if relocation_checkable? @@ -280,19 +289,20 @@ def relocation_check def health_check_otool current_library = nil bad_libs = {} + good_libs = {} - read_shared_libs("find #{project.install_dir}/ -type f | egrep '\.(dylib|bundle)$' | xargs otool -L") do |line| + read_shared_libs("find #{project.install_dir}/ -type f | egrep '\.(dylib|bundle)$'", "xargs otool -L") do |line| case line when /^(.+):$/ current_library = Regexp.last_match[1] when /^\s+(.+) \(.+\)$/ linked = Regexp.last_match[1] name = File.basename(linked) - bad_libs = check_for_bad_library(bad_libs, current_library, name, linked) + bad_libs, good_libs = check_for_bad_library(bad_libs, good_libs, current_library, name, linked) end end - bad_libs + [bad_libs, good_libs] end # @@ -304,8 +314,9 @@ def health_check_otool def health_check_aix current_library = nil bad_libs = {} + good_libs = {} - read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"RISC System\" | awk -F: '{print $1}' | xargs -n 1 ldd") do |line| + read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"XCOFF\" | awk -F: '{print $1}'", "xargs -n 1 ldd") do |line| case line when /^(.+) needs:$/ current_library = Regexp.last_match[1] @@ -313,31 +324,63 @@ def health_check_aix when /^\s+(.+)$/ name = Regexp.last_match[1] linked = Regexp.last_match[1] - bad_libs = check_for_bad_library(bad_libs, current_library, name, linked) + ( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked) when /File is not an executable XCOFF file/ # ignore non-executable files else log.warn(log_key) { "Line did not match for #{current_library}\n#{line}" } end end - bad_libs + [bad_libs, good_libs] end # - # Run healthchecks against ldd. + # Run healthchecks on Solaris. # # @return [Hash>>] # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count) # - def health_check_ldd - regexp_ends = ".*(" + IGNORED_ENDINGS.map { |e| e.gsub(/\./, '\.') }.join("|") + ")$" - regexp_patterns = IGNORED_PATTERNS.map { |e| ".*" + e.gsub(%r{/}, '\/') + ".*" }.join("|") - regexp = regexp_ends + "|" + regexp_patterns + def health_check_solaris + current_library = nil + bad_libs = {} + good_libs = {} + read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"ELF\" | awk -F: '{print $1}' | sed -e 's/:$//'", "xargs -n 1 ldd") do |line| + case line + when /^(.+):$/ + current_library = Regexp.last_match[1] + log.debug(log_key) { "Analyzing dependencies for #{current_library}" } + when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/ + name = Regexp.last_match[1] + linked = Regexp.last_match[2] + ( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked) + when /^\s+(.+) \(.+\)$/ + next + when /^\s+statically linked$/ + next + when /^\s+not a dynamic executable$/ # ignore non-executable files + else + log.warn(log_key) do + "Line did not match for #{current_library}\n#{line}" + end + end + end + + [bad_libs, good_libs] + end + + # + # Run healthchecks on FreeBSD + # + # @return [Hash>>] + # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count) + # + def health_check_freebsd current_library = nil bad_libs = {} + good_libs = {} - read_shared_libs("find #{project.install_dir}/ -type f -regextype posix-extended ! -regex '#{regexp}' | xargs ldd") do |line| + read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"ELF\" | awk -F: '{print $1}' | sed -e 's/:$//'", "xargs ldd") do |line| case line when /^(.+):$/ current_library = Regexp.last_match[1] @@ -345,16 +388,51 @@ def health_check_ldd when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/ name = Regexp.last_match[1] linked = Regexp.last_match[2] - bad_libs = check_for_bad_library(bad_libs, current_library, name, linked) + ( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked) when /^\s+(.+) \(.+\)$/ next when /^\s+statically linked$/ next - when /^\s+libjvm.so/ + when /^\s+not a dynamic executable$/ # ignore non-executable files + else + log.warn(log_key) do + "Line did not match for #{current_library}\n#{line}" + end + end + end + + [bad_libs, good_libs] + end + + # + # Run healthchecks against ldd. + # + # @return [Hash>>] + # the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count) + # + def health_check_linux + current_library = nil + bad_libs = {} + good_libs = {} + + read_shared_libs("find #{project.install_dir}/ -type f", "xargs ldd") do |line| + case line + when /^(.+):$/ + current_library = Regexp.last_match[1] + log.debug(log_key) { "Analyzing dependencies for #{current_library}" } + when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/ + name = Regexp.last_match[1] + linked = Regexp.last_match[2] + ( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked) + when /^\s+(.+) \(.+\)$/ next - when /^\s+libjava.so/ + when /^\s+statically linked$/ next - when /^\s+libmawt.so/ + when /^\s+libjvm.so/ # FIXME: should remove if it doesn't blow up server + next + when /^\s+libjava.so/ # FIXME: should remove if it doesn't blow up server + next + when /^\s+libmawt.so/ # FIXME: should remove if it doesn't blow up server next when /^\s+not a dynamic executable$/ # ignore non-executable files else @@ -364,7 +442,7 @@ def health_check_ldd end end - bad_libs + [bad_libs, good_libs] end private @@ -399,10 +477,40 @@ def whitelist_files # @yield [String] # each line # - def read_shared_libs(command) - cmd = shellout(command) - cmd.stdout.each_line do |line| - yield line + def read_shared_libs(find_command, ldd_command, &output_proc) + # + # construct the list of files to check + # + + find_output = shellout!(find_command).stdout.lines + + find_output.reject! { |file| IGNORED_ENDINGS.any? { |ending| file.end_with?("#{ending}\n") } } + + find_output.reject! { |file| IGNORED_SUBSTRINGS.any? { |substr| file.include?(substr) } } + + if find_output.empty? + # probably the find_command is busted, it should never be empty or why are you using omnibus? + raise "Internal Error: Health Check found no lines" + end + + if find_output.any? { |file| file !~ Regexp.new(project.install_dir) } + # every file in the find output should be within the install_dir + raise "Internal Error: Health Check lines not matching the install_dir" + end + + # + # feed the list of files to the "ldd" command + # + + # this command will typically fail if the last file isn't a valid lib/binary which happens often + ldd_output = shellout(ldd_command, input: find_output.join).stdout + + # + # do the output process to determine if the files are good or bad + # + + ldd_output.each_line do |line| + output_proc.call(line) end end @@ -420,7 +528,7 @@ def read_shared_libs(command) # # @return the modified bad_library hash # - def check_for_bad_library(bad_libs, current_library, name, linked) + def check_for_bad_library(bad_libs, good_libs, current_library, name, linked) safe = nil whitelist_libs = case Ohai["platform"] @@ -463,10 +571,11 @@ def check_for_bad_library(bad_libs, current_library, name, linked) bad_libs[current_library][name][linked] = 1 end else + good_libs[current_library] = true log.debug(log_key) { " -> PASSED: #{name} is either whitelisted or safely provided." } end - bad_libs + [bad_libs, good_libs] end end end diff --git a/lib/omnibus/whitelist.rb b/lib/omnibus/whitelist.rb index e7e33f8c1..b4fb1cc39 100644 --- a/lib/omnibus/whitelist.rb +++ b/lib/omnibus/whitelist.rb @@ -1,5 +1,5 @@ -# Copyright 2012-2020, Chef Software Inc. +# Copyright:: Copyright (c) Chef Software Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -177,28 +177,34 @@ /libkvm\.so/, /libprocstat\.so/, /libmd\.so/, + /libdl\.so/, ].freeze IGNORED_ENDINGS = %w{ .TXT - .[ch] - .[ch]pp - .[eh]rl .app .appup .bat .beam + .c .cc .cmake .conf + .cpp .css - .e*rb + .erb + .erl .feature .gemspec .gif .gitignore .gitkeep - .h*h + .h + .h + .hh + .hpp + .hrl + .html .jar .java .jpg @@ -210,6 +216,7 @@ .lua .md .mkd + .mo .npmignore .out .packlist @@ -219,21 +226,28 @@ .png .pod .properties - .py[oc]* - .r*html + .py + .pyc + .pyo .rake + .rb + .rbs .rdoc + .rhtml .ri + .rpm .rst .scss .sh .sql .svg .toml + .tt .ttf .txt .xml .yml + COPYING Gemfile LICENSE Makefile @@ -243,7 +257,7 @@ license }.freeze -IGNORED_PATTERNS = %w{ +IGNORED_SUBSTRINGS = %w{ /build_info/ /licenses/ /LICENSES/ diff --git a/spec/unit/health_check_spec.rb b/spec/unit/health_check_spec.rb index 8c59faf9c..5a2405959 100644 --- a/spec/unit/health_check_spec.rb +++ b/spec/unit/health_check_spec.rb @@ -99,9 +99,47 @@ def mkdump(base, size, x64 = false) context "on linux" do before { stub_ohai(platform: "ubuntu", version: "16.04") } + # file_list just needs to have one file which is inside of the install_dir + let(:file_list) do + double("Mixlib::Shellout", + error!: false, + stdout: <<~EOH + /opt/chefdk/shouldnt/matter + EOH + ) + end + + let(:empty_list) do + double("Mixlib::Shellout", + error!: false, + stdout: <<~EOH + EOH + ) + end + + let(:failed_list) do + failed_list = double("Mixlib::Shellout", + stdout: <<~EOH + /opt/chefdk/shouldnt/matter + EOH + ) + allow(failed_list).to receive(:error!).and_raise("Mixlib::Shellout::ShellCommandFailed") + failed_list + end + + let(:bad_list) do + double("Mixlib::Shellout", + error!: false, + stdout: <<~EOH + /somewhere/other/than/install/dir + EOH + ) + end + let(:bad_healthcheck) do double("Mixlib::Shellout", - stdout: <<-EOH.gsub(/^ {12}/, "") + error!: false, + stdout: <<~EOH /bin/ls: linux-vdso.so.1 => (0x00007fff583ff000) libselinux.so.1 => /lib/x86_64-linux-gnu/libselinux.so.1 (0x00007fad8592a000) @@ -122,7 +160,8 @@ def mkdump(base, size, x64 = false) let(:good_healthcheck) do double("Mixlib::Shellout", - stdout: <<-EOH.gsub(/^ {12}/, "") + error!: false, + stdout: <<~EOH /bin/echo: linux-vdso.so.1 => (0x00007fff8a6ee000) libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f70f58c0000) @@ -135,11 +174,13 @@ def mkdump(base, size, x64 = false) ) end - let(:regexp) { ".*(\\.TXT|\\.[ch]|\\.[ch]pp|\\.[eh]rl|\\.app|\\.appup|\\.bat|\\.beam|\\.cc|\\.cmake|\\.conf|\\.css|\\.e*rb|\\.feature|\\.gemspec|\\.gif|\\.gitignore|\\.gitkeep|\\.h*h|\\.jar|\\.java|\\.jpg|\\.js|\\.jsm|\\.json|\\.lock|\\.log|\\.lua|\\.md|\\.mkd|\\.npmignore|\\.out|\\.packlist|\\.perl|\\.pl|\\.pm|\\.png|\\.pod|\\.properties|\\.py[oc]*|\\.r*html|\\.rake|\\.rdoc|\\.ri|\\.rst|\\.scss|\\.sh|\\.sql|\\.svg|\\.toml|\\.ttf|\\.txt|\\.xml|\\.yml|Gemfile|LICENSE|Makefile|README|Rakefile|VERSION|license)$|.*\\/build_info\\/.*|.*\\/licenses\\/.*|.*\\/LICENSES\\/.*|.*\\/man\\/.*|.*\\/share\\/doc\\/.*|.*\\/share\\/info\\/.*|.*\\/share\\/postgresql\\/.*|.*\\/share\\/terminfo\\/.*|.*\\/share\\/timezone\\/.*|.*\\/terminfo\\/.*" } - it "raises an exception when there are external dependencies" do allow(subject).to receive(:shellout) - .with("find #{project.install_dir}/ -type f -regextype posix-extended ! -regex '#{regexp}' | xargs ldd") + .with("find /opt/chefdk/ -type f") + .and_return(file_list) + + allow(subject).to receive(:shellout) + .with("xargs ldd", { input: "/opt/chefdk/shouldnt/matter\n" }) .and_return(bad_healthcheck) expect { subject.run! }.to raise_error(HealthCheckFailed) @@ -147,7 +188,11 @@ def mkdump(base, size, x64 = false) it "does not raise an exception when the healthcheck passes" do allow(subject).to receive(:shellout) - .with("find #{project.install_dir}/ -type f -regextype posix-extended ! -regex '#{regexp}' | xargs ldd") + .with("find /opt/chefdk/ -type f") + .and_return(file_list) + + allow(subject).to receive(:shellout) + .with("xargs ldd", { input: "/opt/chefdk/shouldnt/matter\n" }) .and_return(good_healthcheck) expect { subject.run! }.to_not raise_error @@ -156,6 +201,30 @@ def mkdump(base, size, x64 = false) it "will not perform dll base relocation checks" do expect(subject.relocation_checkable?).to be false end + + it "raises an exception if there's nothing in the file list" do + allow(subject).to receive(:shellout) + .with("find /opt/chefdk/ -type f") + .and_return(empty_list) + + expect { subject.run! }.to raise_error(RuntimeError, "Internal Error: Health Check found no lines") + end + + it "raises an exception if the file list command raises" do + allow(subject).to receive(:shellout) + .with("find /opt/chefdk/ -type f") + .and_return(failed_list) + + expect { subject.run! }.to raise_error(RuntimeError, "Mixlib::Shellout::ShellCommandFailed") + end + + it "raises an exception if the file list command has no entries in the install_dir" do + allow(subject).to receive(:shellout) + .with("find /opt/chefdk/ -type f") + .and_return(bad_list) + + expect { subject.run! }.to raise_error(RuntimeError, "Internal Error: Health Check lines not matching the install_dir") + end end end end