aws · jaxesn · Jan 12, 2023 · Jan 9, 2023 · jonathanmeier5 · Jan 11, 2023
diff --git a/Common.mk b/Common.mk
@@ -374,6 +374,9 @@ GATHER_LICENSES_TARGETS?=$(call pairmap,LICENSE_TARGET_FROM_BINARY_GO_MOD,$(BINA
 LICENSES_OUTPUT_DIR?=$(OUTPUT_DIR)
 LICENSES_TARGETS_FOR_PREREQ=$(if $(filter true,$(HAS_LICENSES)),$(GATHER_LICENSES_TARGETS) \
 	$(foreach target,$(ATTRIBUTION_TARGETS),_output/$(target)),)
+# .9 is the default if nothing is passed to go-licenses
+# allow override on a per project basis for super specific cases
+LICENSE_THRESHOLD?=.9
 ####################################################
 
 #################### TARBALLS ######################
@@ -559,7 +562,7 @@ $(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,
 $(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS)))
 $(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER)
 $(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH))	
-	$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION)
+	$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD)
 
 .PHONY: gather-licenses
 gather-licenses: $(GATHER_LICENSES_TARGETS)

diff --git a/build/lib/common.sh b/build/lib/common.sh
@@ -94,7 +94,8 @@ function build::gather_licenses() {
   local -r outputdir=$1
   local -r patterns=$2
   local -r golang_version=$3
-
+  local -r threshold=$4
+
   # Force deps to only be pulled form vendor directories
   # this is important in a couple cases where license files
   # have to be manually created
@@ -131,11 +132,11 @@ function build::gather_licenses() {
 
   # go-licenses can be a bit noisy with its output and lot of it can be confusing 
   # the following messages are safe to ignore since we do not need the license url for our process
-  NOISY_MESSAGES="cannot determine URL for|Error discovering license URL|unsupported package host|contains non-Go code|has empty version|vendor.*\.s$"
-
-  go-licenses save --force $patterns --save_path="${outputdir}/LICENSES" 2>  >(grep -vE "$NOISY_MESSAGES" >&2)
+  NOISY_MESSAGES="cannot determine URL for|Error discovering license URL|unsupported package host|contains non-Go code|has empty version|vendor.*\.(h|s)$"
+ 
+  go-licenses save --confidence_threshold $threshold  --force $patterns --save_path="${outputdir}/LICENSES" 2>  >(grep -vE "$NOISY_MESSAGES" >&2)
 
-  go-licenses csv $patterns > "${outputdir}/attribution/go-license.csv" 2>  >(grep -vE "$NOISY_MESSAGES" >&2)
+  go-licenses csv --confidence_threshold $threshold $patterns > "${outputdir}/attribution/go-license.csv" 2>  >(grep -vE "$NOISY_MESSAGES" >&2)
 
   if cat "${outputdir}/attribution/go-license.csv" | grep -q "^vendor\/golang.org\/x"; then
       echo " go-licenses created a file with a std golang package (golang.org/x/*)"

diff --git a/build/lib/gather_licenses.sh b/build/lib/gather_licenses.sh
@@ -22,9 +22,10 @@ OUTPUT_DIR="$2"
 PACKAGE_FILTER="$3"
 REPO_SUBPATH="${4:-}"
 GOLANG_VERSION="${5:-}"
+LICENSE_THRESHOLD="${6:-}"
 
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 source "${SCRIPT_ROOT}/common.sh"
 
 cd $REPO/$REPO_SUBPATH
-build::gather_licenses $OUTPUT_DIR "$PACKAGE_FILTER" "$GOLANG_VERSION"
+build::gather_licenses $OUTPUT_DIR "$PACKAGE_FILTER" "$GOLANG_VERSION" "$LICENSE_THRESHOLD"
diff --git a/build/lib/install_go_versions.sh b/build/lib/install_go_versions.sh
@@ -46,16 +46,16 @@ setupgo "${GOLANG119_VERSION:-1.19.4}"
 # go-licenses needs to be installed by the same version of go that is being used
 # to generate the deps list during the attribution generation process
 build::common::use_go_version "1.16"
-GOBIN=${GOPATH}/go1.16/bin go install github.com/google/go-licenses@v1.2.1
+GOBIN=${GOPATH}/go1.16/bin go install github.com/jaxesn/go-licenses@4497a2a38565e4e6ad095ea8117c25ecd622d0cc
 
 build::common::use_go_version "1.17"
-GOBIN=${GOPATH}/go1.17/bin go install github.com/google/go-licenses@v1.2.1
+GOBIN=${GOPATH}/go1.17/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
 
 build::common::use_go_version "1.18"
-GOBIN=${GOPATH}/go1.18/bin go install github.com/google/go-licenses@v1.2.1
+GOBIN=${GOPATH}/go1.18/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
 
 build::common::use_go_version "1.19"
-GOBIN=${GOPATH}/go1.19/bin go install github.com/google/go-licenses@v1.2.1
+GOBIN=${GOPATH}/go1.19/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
 
 # 1.16 is the default so symlink it to /go/bin
 ln -sf ${GOPATH}/go1.16/bin/go-licenses ${GOPATH}/bin
diff --git a/projects/aquasecurity/trivy/ATTRIBUTION.txt b/projects/aquasecurity/trivy/ATTRIBUTION.txt
@@ -464,6 +464,13 @@ Copyright 2019-2020 Aqua Security Software Ltd.
 This product includes software developed by Aqua Security (https://aquasec.com).
 
 
+* For github.com/aquasecurity/trivy see also this required NOTICE:
+Trivy
+Copyright 2019-2020 Aqua Security Software Ltd. 
+
+This product includes software developed by Aqua Security (https://aquasec.com).
+
+
 * For github.com/aquasecurity/trivy-db/pkg see also this required NOTICE:
 Trivy-db
 Copyright 2019-2020 Aqua Security Software Ltd.
@@ -793,6 +800,105 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ------
 
+** modernc.org/libc; version v1.14.1 --
+https://gitlab.com/cznic/libc
+
+Copyright (c) 2017 The Libc Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------
+
+** modernc.org/mathutil; version v1.4.1 --
+https://gitlab.com/cznic/mathutil
+
+Copyright (c) 2014 The mathutil Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------
+
+** modernc.org/memory; version v1.0.5 --
+https://gitlab.com/cznic/memory
+
+Copyright (c) 2017 The Memory Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------
+
 ** github.com/aws/aws-sdk-go/internal/sync/singleflight; version v1.44.5 --
 https://github.com/aws/aws-sdk-go
 
@@ -841,9 +947,6 @@ https://github.com/kubernetes/apimachinery
 ** k8s.io/client-go/third_party/forked/golang/template; version v0.23.6 --
 https://github.com/kubernetes/client-go
 
-** modernc.org/libc; version v1.14.1 --
-https://gitlab.com/cznic/libc
-
 Copyright (c) 2009 The Go Authors. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -1558,37 +1661,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-------
-
-** modernc.org/memory; version v1.0.5 --
-https://gitlab.com/cznic/memory
-
-Copyright (c) 2011, Evan Shaw <[email protected]>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the copyright holder nor the
-      names of its contributors may be used to endorse or promote products
-      derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 ------
 
 ** modernc.org/sqlite; version v1.14.5 --

diff --git a/projects/aquasecurity/trivy/Makefile b/projects/aquasecurity/trivy/Makefile
@@ -15,6 +15,16 @@ EXTRA_GO_LDFLAGS=-X main.version=$(VERSION)
 HAS_S3_ARTIFACTS=true
 IMAGE_NAMES=
 
+# modernc.org/mathutil's license is actually mismatched
+# the current version of go-license finds it to match BSD-2 where its
+# really BSD-3.  The library go-liceses uses to determine the type
+# has a newer version which more accurately matches between the two BSD types
+# the confidence to BSD-2 is > .85 but under the .9 default whereas
+# the newer version matching BSD-3 ends up being > .99 confidence
+# We could remove this in the future when go-licenses upgrades
+# in the meantime we have confirmed the license type and that it is acceptable
+LICENSE_THRESHOLD=.85
+
 EXCLUDE_FROM_STAGING_BUILDSPEC=true
 
 include $(BASE_DIRECTORY)/Common.mk

diff --git a/projects/kubernetes/autoscaler/Makefile b/projects/kubernetes/autoscaler/Makefile
@@ -13,6 +13,7 @@ SOURCE_PATTERNS=.
 GO_MOD_PATHS=cluster-autoscaler
 
 FIX_LICENSES=$(REPO)/cluster-autoscaler/LICENSE
+FIX_LICENSES_AZURE_TARGET=$(REPO)/cluster-autoscaler/vendor/github.com/Azure/azure-sdk-for-go/LICENSE.txt
 
 HAS_RELEASE_BRANCHES=true
 EXCLUDE_FROM_STAGING_BUILDSPEC=true
@@ -27,16 +28,21 @@ HELM_DIRECTORY=charts/cluster-autoscaler
 HELM_DESTINATION_REPOSITORY=cluster-autoscaler/charts/cluster-autoscaler
 HELM_IMAGE_LIST=kubernetes/autoscaler
 
-
 include $(BASE_DIRECTORY)/Common.mk
 
 
-$(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES)
+$(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES) $(FIX_LICENSES_AZURE_TARGET) 
 
 $(FIX_LICENSES): | $(GO_MOD_DOWNLOAD_TARGETS)
 #go-licenses requires a LICENSE file in each folder with the go.mod
 	cp $(REPO)/LICENSE $@
 
+$(FIX_LICENSES_AZURE_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS)
+# The azure sdk dependency github repo has a LICENSE however it does not have a go.mod
+# checked in to the repo. Hence we need to manually download LICENSE from Github
+# and place them in the vendor directory so that they is available for go-licenses to pick up
+	wget -q --retry-connrefused https://raw.githubusercontent.com/Azure/azure-sdk-for-go/main/LICENSE.txt -O $@
+
 
 ########### DO NOT EDIT #############################
 # To update call: make add-generated-help-block

diff --git a/projects/prometheus/node_exporter/ATTRIBUTION.txt b/projects/prometheus/node_exporter/ATTRIBUTION.txt
@@ -296,6 +296,26 @@ This product includes software developed at
 SoundCloud Ltd. (http://soundcloud.com/).
 
 
+* For github.com/prometheus/node_exporter see also this required NOTICE:
+Configurable modular Prometheus exporter for various node metrics.
+Copyright 2013-2015 The Prometheus Authors
+
+This product includes software developed at
+SoundCloud Ltd. (http://soundcloud.com/).
+
+The following components are included in this product:
+
+wifi
+https://github.com/mdlayher/wifi
+Copyright 2016-2017 Matt Layher
+Licensed under the MIT License
+
+netlink
+https://github.com/mdlayher/netlink
+Copyright 2016-2017 Matt Layher
+Licensed under the MIT License
+
+
 * For github.com/prometheus/procfs see also this required NOTICE:
 procfs provides functions to retrieve system, kernel and process
 metrics from the pseudo-filesystem proc.
@@ -612,6 +632,10 @@ Copyright (C) 2013 Blake Mizerany
 https://github.com/cespare/xxhash/v2
 Copyright (c) 2016 Caleb Spare
 
+** github.com/dennwc/ioctl; version v1.0.0 --
+https://github.com/dennwc/ioctl
+Copyright (c) 2018 Denys Smirnov
+
 ** github.com/go-kit/log; version v0.2.1 --
 https://github.com/go-kit/log
 Copyright (c) 2021 Go kit

diff --git a/projects/prometheus/node_exporter/Makefile b/projects/prometheus/node_exporter/Makefile
@@ -11,17 +11,24 @@ EXTRA_GOBUILD_FLAGS=-tags netgo,osusergo,static_build
 
 NODE_EXPORTER_IMAGE_COMPONENT=prometheus/node-exporter
 
+FIX_LICENSES_DENNWC_TARGETS=$(REPO)/vendor/github.com/dennwc/btrfs/LICENSE $(REPO)/vendor/github.com/dennwc/ioctl/LICENSE
+
 EXCLUDE_FROM_STAGING_BUILDSPEC=true
 
 include $(BASE_DIRECTORY)/Common.mk
 
-# Fix licenses that have incorrect module names.
-# Module "github.com" should be "github.com/prometheus/node_exporter"
-$(ATTRIBUTION_TARGETS): fix-go-licenses
 
-.phony: fix-go-licenses
-fix-go-licenses: $(GATHER_LICENSES_TARGETS)
-	build/sed_replace.sh "github.com,https" "github.com\/prometheus\/node_exporter,https" $(OUTPUT_DIR)/attribution/go-license.csv
+$(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_DENNWC_TARGETS)
+
+$(FIX_LICENSES_DENNWC_TARGETS): | $(GO_MOD_DOWNLOAD_TARGETS)
+  # The modules from the github org dennwc are properly licensed but for some
+  # unknown reason are not included in the module download via go mod down
+  # Manually downloading from github and placing in each of the packages
+  # under vendor to make go-licenses happy
+	for package in btrfs ioctl ; do \
+		wget --retry-connrefused -q https://raw.githubusercontent.com/dennwc/$$package/master/LICENSE -O \
+			$(REPO)/vendor/github.com/dennwc/$$package/LICENSE; \
+	done;
 
 ########### DO NOT EDIT #############################
 # To update call: make add-generated-help-block

diff --git a/projects/prometheus/node_exporter/build/sed_replace.sh b/projects/prometheus/node_exporter/build/sed_replace.sh