Skip to content

Commit 22689c9

Browse files
jeremiahsnappjeremiahsnapp
authored
Merge pull request #1056 from chef/lcg/better-healthcheck-correctness-checking
Better healthcheck correctness
2 parents f077b40 + 41e598a commit 22689c9

File tree

3 files changed

+245
-53
lines changed

3 files changed

+245
-53
lines changed

‎lib/omnibus/health_check.rb‎

Lines changed: 147 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
# Copyright 2012-2018 Chef Software, Inc.
2+
# Copyright:: Copyright (c) Chef Software Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -66,20 +66,25 @@ def initialize(project)
6666
def run!
6767
measure("Health check time") do
6868
log.info(log_key) { "Running health on #{project.name}" }
69-
bad_libs = case Ohai["platform"]
70-
when "mac_os_x"
71-
health_check_otool
72-
when "aix"
73-
health_check_aix
74-
when "windows"
75-
# TODO: objdump -p will provided a very limited check of
76-
# explicit dependencies on windows. Most dependencies are
77-
# implicit and hence not detected.
78-
log.warn(log_key) { "Skipping dependency health checks on Windows." }
79-
{}
80-
else
81-
health_check_ldd
82-
end
69+
bad_libs, good_libs =
70+
case Ohai["platform"]
71+
when "mac_os_x"
72+
health_check_otool
73+
when "aix"
74+
health_check_aix
75+
when "windows"
76+
# TODO: objdump -p will provided a very limited check of
77+
# explicit dependencies on windows. Most dependencies are
78+
# implicit and hence not detected.
79+
log.warn(log_key) { "Skipping dependency health checks on Windows." }
80+
[{}, {}]
81+
when "solaris2"
82+
health_check_solaris
83+
when "freebsd", "openbsd", "netbsd"
84+
health_check_freebsd
85+
else
86+
health_check_linux
87+
end
8388

8489
unresolved = []
8590
unreliable = []
@@ -167,6 +172,10 @@ def run!
167172
raise HealthCheckFailed
168173
end
169174

175+
if good_libs.keys.length == 0 && !windows?
176+
raise "Internal error: no good libraries were found"
177+
end
178+
170179
conflict_map = {}
171180

172181
conflict_map = relocation_check if relocation_checkable?
@@ -280,19 +289,20 @@ def relocation_check
280289
def health_check_otool
281290
current_library = nil
282291
bad_libs = {}
292+
good_libs = {}
283293

284-
read_shared_libs("find #{project.install_dir}/ -type f | egrep '\.(dylib|bundle)$' | xargs otool -L") do |line|
294+
read_shared_libs("find #{project.install_dir}/ -type f | egrep '\.(dylib|bundle)$'", "xargs otool -L") do |line|
285295
case line
286296
when /^(.+):$/
287297
current_library = Regexp.last_match[1]
288298
when /^\s+(.+) \(.+\)$/
289299
linked = Regexp.last_match[1]
290300
name = File.basename(linked)
291-
bad_libs = check_for_bad_library(bad_libs, current_library, name, linked)
301+
bad_libs, good_libs = check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
292302
end
293303
end
294304

295-
bad_libs
305+
[bad_libs, good_libs]
296306
end
297307

298308
#
@@ -304,57 +314,125 @@ def health_check_otool
304314
def health_check_aix
305315
current_library = nil
306316
bad_libs = {}
317+
good_libs = {}
307318

308-
read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"RISC System\" | awk -F: '{print $1}' | xargs -n 1 ldd") do |line|
319+
read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"XCOFF\" | awk -F: '{print $1}'", "xargs -n 1 ldd") do |line|
309320
case line
310321
when /^(.+) needs:$/
311322
current_library = Regexp.last_match[1]
312323
log.debug(log_key) { "Analyzing dependencies for #{current_library}" }
313324
when /^\s+(.+)$/
314325
name = Regexp.last_match[1]
315326
linked = Regexp.last_match[1]
316-
bad_libs = check_for_bad_library(bad_libs, current_library, name, linked)
327+
( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
317328
when /File is not an executable XCOFF file/ # ignore non-executable files
318329
else
319330
log.warn(log_key) { "Line did not match for #{current_library}\n#{line}" }
320331
end
321332
end
322333

323-
bad_libs
334+
[bad_libs, good_libs]
324335
end
325336

326337
#
327-
# Run healthchecks against ldd.
338+
# Run healthchecks on Solaris.
328339
#
329340
# @return [Hash<String, Hash<String, Hash<String, Int>>>]
330341
# the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
331342
#
332-
def health_check_ldd
333-
regexp_ends = ".*(" + IGNORED_ENDINGS.map { |e| e.gsub(/\./, '\.') }.join("|") + ")$"
334-
regexp_patterns = IGNORED_PATTERNS.map { |e| ".*" + e.gsub(%r{/}, '\/') + ".*" }.join("|")
335-
regexp = regexp_ends + "|" + regexp_patterns
343+
def health_check_solaris
344+
current_library = nil
345+
bad_libs = {}
346+
good_libs = {}
336347

348+
read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"ELF\" | awk -F: '{print $1}' | sed -e 's/:$//'", "xargs -n 1 ldd") do |line|
349+
case line
350+
when /^(.+):$/
351+
current_library = Regexp.last_match[1]
352+
log.debug(log_key) { "Analyzing dependencies for #{current_library}" }
353+
when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/
354+
name = Regexp.last_match[1]
355+
linked = Regexp.last_match[2]
356+
( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
357+
when /^\s+(.+) \(.+\)$/
358+
next
359+
when /^\s+statically linked$/
360+
next
361+
when /^\s+not a dynamic executable$/ # ignore non-executable files
362+
else
363+
log.warn(log_key) do
364+
"Line did not match for #{current_library}\n#{line}"
365+
end
366+
end
367+
end
368+
369+
[bad_libs, good_libs]
370+
end
371+
372+
#
373+
# Run healthchecks on FreeBSD
374+
#
375+
# @return [Hash<String, Hash<String, Hash<String, Int>>>]
376+
# the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
377+
#
378+
def health_check_freebsd
337379
current_library = nil
338380
bad_libs = {}
381+
good_libs = {}
339382

340-
read_shared_libs("find #{project.install_dir}/ -type f -regextype posix-extended ! -regex '#{regexp}' | xargs ldd") do |line|
383+
read_shared_libs("find #{project.install_dir}/ -type f | xargs file | grep \"ELF\" | awk -F: '{print $1}' | sed -e 's/:$//'", "xargs ldd") do |line|
341384
case line
342385
when /^(.+):$/
343386
current_library = Regexp.last_match[1]
344387
log.debug(log_key) { "Analyzing dependencies for #{current_library}" }
345388
when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/
346389
name = Regexp.last_match[1]
347390
linked = Regexp.last_match[2]
348-
bad_libs = check_for_bad_library(bad_libs, current_library, name, linked)
391+
( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
349392
when /^\s+(.+) \(.+\)$/
350393
next
351394
when /^\s+statically linked$/
352395
next
353-
when /^\s+libjvm.so/
396+
when /^\s+not a dynamic executable$/ # ignore non-executable files
397+
else
398+
log.warn(log_key) do
399+
"Line did not match for #{current_library}\n#{line}"
400+
end
401+
end
402+
end
403+
404+
[bad_libs, good_libs]
405+
end
406+
407+
#
408+
# Run healthchecks against ldd.
409+
#
410+
# @return [Hash<String, Hash<String, Hash<String, Int>>>]
411+
# the bad libraries (library_name -> dependency_name -> satisfied_lib_path -> count)
412+
#
413+
def health_check_linux
414+
current_library = nil
415+
bad_libs = {}
416+
good_libs = {}
417+
418+
read_shared_libs("find #{project.install_dir}/ -type f", "xargs ldd") do |line|
419+
case line
420+
when /^(.+):$/
421+
current_library = Regexp.last_match[1]
422+
log.debug(log_key) { "Analyzing dependencies for #{current_library}" }
423+
when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/
424+
name = Regexp.last_match[1]
425+
linked = Regexp.last_match[2]
426+
( bad_libs, good_libs ) = check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
427+
when /^\s+(.+) \(.+\)$/
354428
next
355-
when /^\s+libjava.so/
429+
when /^\s+statically linked$/
356430
next
357-
when /^\s+libmawt.so/
431+
when /^\s+libjvm.so/ # FIXME: should remove if it doesn't blow up server
432+
next
433+
when /^\s+libjava.so/ # FIXME: should remove if it doesn't blow up server
434+
next
435+
when /^\s+libmawt.so/ # FIXME: should remove if it doesn't blow up server
358436
next
359437
when /^\s+not a dynamic executable$/ # ignore non-executable files
360438
else
@@ -364,7 +442,7 @@ def health_check_ldd
364442
end
365443
end
366444

367-
bad_libs
445+
[bad_libs, good_libs]
368446
end
369447

370448
private
@@ -399,10 +477,40 @@ def whitelist_files
399477
# @yield [String]
400478
# each line
401479
#
402-
def read_shared_libs(command)
403-
cmd = shellout(command)
404-
cmd.stdout.each_line do |line|
405-
yield line
480+
def read_shared_libs(find_command, ldd_command, &output_proc)
481+
#
482+
# construct the list of files to check
483+
#
484+
485+
find_output = shellout!(find_command).stdout.lines
486+
487+
find_output.reject! { |file| IGNORED_ENDINGS.any? { |ending| file.end_with?("#{ending}\n") } }
488+
489+
find_output.reject! { |file| IGNORED_SUBSTRINGS.any? { |substr| file.include?(substr) } }
490+
491+
if find_output.empty?
492+
# probably the find_command is busted, it should never be empty or why are you using omnibus?
493+
raise "Internal Error: Health Check found no lines"
494+
end
495+
496+
if find_output.any? { |file| file !~ Regexp.new(project.install_dir) }
497+
# every file in the find output should be within the install_dir
498+
raise "Internal Error: Health Check lines not matching the install_dir"
499+
end
500+
501+
#
502+
# feed the list of files to the "ldd" command
503+
#
504+
505+
# this command will typically fail if the last file isn't a valid lib/binary which happens often
506+
ldd_output = shellout(ldd_command, input: find_output.join).stdout
507+
508+
#
509+
# do the output process to determine if the files are good or bad
510+
#
511+
512+
ldd_output.each_line do |line|
513+
output_proc.call(line)
406514
end
407515
end
408516

@@ -420,7 +528,7 @@ def read_shared_libs(command)
420528
#
421529
# @return the modified bad_library hash
422530
#
423-
def check_for_bad_library(bad_libs, current_library, name, linked)
531+
def check_for_bad_library(bad_libs, good_libs, current_library, name, linked)
424532
safe = nil
425533

426534
whitelist_libs = case Ohai["platform"]
@@ -463,10 +571,11 @@ def check_for_bad_library(bad_libs, current_library, name, linked)
463571
bad_libs[current_library][name][linked] = 1
464572
end
465573
else
574+
good_libs[current_library] = true
466575
log.debug(log_key) { " -> PASSED: #{name} is either whitelisted or safely provided." }
467576
end
468577

469-
bad_libs
578+
[bad_libs, good_libs]
470579
end
471580
end
472581
end

‎lib/omnibus/whitelist.rb‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
# Copyright 2012-2020, Chef Software Inc.
2+
# Copyright:: Copyright (c) Chef Software Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -177,28 +177,34 @@
177177
/libkvm\.so/,
178178
/libprocstat\.so/,
179179
/libmd\.so/,
180+
/libdl\.so/,
180181
].freeze
181182

182183
IGNORED_ENDINGS = %w{
183184
.TXT
184-
.[ch]
185-
.[ch]pp
186-
.[eh]rl
187185
.app
188186
.appup
189187
.bat
190188
.beam
189+
.c
191190
.cc
192191
.cmake
193192
.conf
193+
.cpp
194194
.css
195-
.e*rb
195+
.erb
196+
.erl
196197
.feature
197198
.gemspec
198199
.gif
199200
.gitignore
200201
.gitkeep
201-
.h*h
202+
.h
203+
.h
204+
.hh
205+
.hpp
206+
.hrl
207+
.html
202208
.jar
203209
.java
204210
.jpg
@@ -210,6 +216,7 @@
210216
.lua
211217
.md
212218
.mkd
219+
.mo
213220
.npmignore
214221
.out
215222
.packlist
@@ -219,21 +226,28 @@
219226
.png
220227
.pod
221228
.properties
222-
.py[oc]*
223-
.r*html
229+
.py
230+
.pyc
231+
.pyo
224232
.rake
233+
.rb
234+
.rbs
225235
.rdoc
236+
.rhtml
226237
.ri
238+
.rpm
227239
.rst
228240
.scss
229241
.sh
230242
.sql
231243
.svg
232244
.toml
245+
.tt
233246
.ttf
234247
.txt
235248
.xml
236249
.yml
250+
COPYING
237251
Gemfile
238252
LICENSE
239253
Makefile
@@ -243,7 +257,7 @@
243257
license
244258
}.freeze
245259

246-
IGNORED_PATTERNS = %w{
260+
IGNORED_SUBSTRINGS = %w{
247261
/build_info/
248262
/licenses/
249263
/LICENSES/

0 commit comments

Comments
 (0)