Add nodejs-function-invoker buildpack (#47)

edmorley · Malax · web-flow · commit 0e7e2d166241 · 2021-05-06T12:10:57.000+01:00
This replaces the various Riff-based invoker buildpacks, now that Riff has been sunset. This buildpack vendors the new Node.js invoker: https://github.com/forcedotcom/sf-fx-runtime-nodejs For Riff invoker buildpack prior art, see: https://github.com/forcedotcom/nodejs-sf-fx-buildpack https://github.com/projectriff/node-function-buildpack Takes inspiration from the JVM invoker buildpack: https://github.com/heroku/buildpacks-jvm/tree/main/buildpacks/jvm-function-invoker ...though has a number of changes since this repo has different buildpack utility functions (eg for logging, downloads), and uses shpec rather than Rapier for testing. This initial version of the buildpack doesn't implement caching, since whilst the invoker is under active development, the release assets will be regularly changing, so using unversioned URLs and frequently changing archive SHA. This buildpack also currently doesn't pass `$PORT` to the entrypoint, since the invoker currently doesn't support it (and may read directly from the env anyway, when there is support for custom ports). The shellcheck version has been bumped to pick up a fix that reduces the number of `source=` annotations required to prevent `SC1090` errors. After this PR is merged and the buildpack published, the meta-buildpack `nodejs-function` will be updated in another PR, to reference this buildpack instead of the Riff-based buildpacks. Closes GUS-W-9115013. Co-authored-by: Manuel Fuchs <manuel.fuchs@salesforce.com>
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -4,11 +4,8 @@ orbs:
   pack: buildpacks/pack@0.2.2
   heroku-buildpacks:
     commands:
-      install-build-dependencies:
+      install-yq:
         steps:
-          - run:
-              name: "Install rsync, jq via apt"
-              command: sudo apt-get update && sudo apt-get install -y rsync jq
           - run:
               name: "Install yj 5.0.0"
               command: |
@@ -17,6 +14,12 @@ orbs:
                 chmod +x "${bin_dir}/yj"
 
                 echo "export PATH=\"${bin_dir}:\${PATH}\"" >> $BASH_ENV
+      install-build-dependencies:
+        steps:
+          - run:
+              name: "Install rsync, jq via apt"
+              command: sudo apt-get update && sudo apt-get install -y rsync jq
+          - install-yq
 
 jobs:
   package-buildpack:
@@ -49,7 +52,7 @@ jobs:
 
   shell-linting:
     docker:
-      - image: koalaman/shellcheck-alpine:v0.7.1
+      - image: koalaman/shellcheck-alpine:v0.7.2
     steps:
       - run: "apk add git"
       # shfmt is currently (Jan 2021) only available in the edge community repository.
@@ -75,6 +78,8 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
+      # Required so that yq is available for buildpacks that depend upon the nodejs-engine's copy of yq.
+      - heroku-buildpacks/install-yq
       - run:
           name: Shpec unit tests on <<parameters.stack>>
           command: shpec << parameters.buildpack-dir >>/shpec/*_shpec.sh
@@ -89,6 +94,7 @@ workflows:
             parameters:
               buildpack-dir:
                 - "buildpacks/nodejs"
+                - "buildpacks/nodejs-function-invoker"
                 - "buildpacks/npm"
                 - "buildpacks/typescript"
               stack:
@@ -101,6 +107,7 @@ workflows:
             parameters:
               buildpack-dir:
                 - "buildpacks/nodejs"
+                - "buildpacks/nodejs-function-invoker"
                 - "buildpacks/npm"
                 - "buildpacks/typescript"
                 - "meta-buildpacks/nodejs"
diff --git a/buildpacks/nodejs-function-invoker/CHANGELOG.md b/buildpacks/nodejs-function-invoker/CHANGELOG.md
@@ -0,0 +1,7 @@
+# Changelog
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+- Initial implementation ([#47](https://github.com/heroku/buildpacks-node/pull/47))
diff --git a/buildpacks/nodejs-function-invoker/README.md b/buildpacks/nodejs-function-invoker/README.md
@@ -0,0 +1 @@
+# Node.js Function Invoker Cloud Native Buildpack
diff --git a/buildpacks/nodejs-function-invoker/bin/build b/buildpacks/nodejs-function-invoker/bin/build
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# shellcheck source-path=SCRIPTDIR/..
+
+set -euo pipefail
+
+layers_dir="${1:?}"
+
+source "${CNB_BUILDPACK_DIR}/lib/build.sh"
+
+install_runtime "${layers_dir}"
+write_launch_toml "${layers_dir}" "$(pwd)"
diff --git a/buildpacks/nodejs-function-invoker/bin/detect b/buildpacks/nodejs-function-invoker/bin/detect
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# shellcheck source-path=SCRIPTDIR/..
+
+set -euo pipefail
+
+app_dir=$(pwd)
+build_plan="${2:?}"
+
+source "${CNB_BUILDPACK_DIR}/lib/detect.sh"
+
+if detect_function_app "${app_dir}"; then
+	write_to_build_plan "${build_plan}"
+	exit 0
+fi
+
+exit 100
diff --git a/buildpacks/nodejs-function-invoker/buildpack.toml b/buildpacks/nodejs-function-invoker/buildpack.toml
@@ -0,0 +1,31 @@
+api = "0.6"
+
+[buildpack]
+id = "heroku/nodejs-function-invoker"
+version = "0.1.0"
+name = "Node.js Function Invoker"
+clear-env = true
+homepage = "https://github.com/heroku/buildpacks-nodejs"
+keywords = ["nodejs", "node", "javascript", "function"]
+
+[[licenses]]
+type = "MIT"
+
+[[stacks]]
+id = "heroku-18"
+
+[[stacks]]
+id = "heroku-20"
+
+[[stacks]]
+id = "io.buildpacks.stacks.bionic"
+
+[metadata]
+
+[metadata.runtime]
+url = "https://sf-fx-nodejs-internal-early-access.s3.us-east-2.amazonaws.com/sf-fx-runtime-nodejs-0.1.0-ea.tgz"
+
+[metadata.release]
+
+[metadata.release.docker]
+repository = "public.ecr.aws/heroku-buildpacks/heroku-nodejs-function-invoker-buildpack"
diff --git a/buildpacks/nodejs-function-invoker/fixtures/function-toml/function.toml b/buildpacks/nodejs-function-invoker/fixtures/function-toml/function.toml
diff --git a/buildpacks/nodejs-function-invoker/fixtures/function-toml/package.json b/buildpacks/nodejs-function-invoker/fixtures/function-toml/package.json
@@ -0,0 +1 @@
+{}
diff --git a/buildpacks/nodejs-function-invoker/fixtures/package-json-only/package.json b/buildpacks/nodejs-function-invoker/fixtures/package-json-only/package.json
diff --git a/buildpacks/nodejs-function-invoker/fixtures/project-toml/package.json b/buildpacks/nodejs-function-invoker/fixtures/project-toml/package.json
@@ -0,0 +1 @@
+{}
diff --git a/buildpacks/nodejs-function-invoker/fixtures/project-toml/project.toml b/buildpacks/nodejs-function-invoker/fixtures/project-toml/project.toml
diff --git a/buildpacks/nodejs-function-invoker/lib/build.sh b/buildpacks/nodejs-function-invoker/lib/build.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# shellcheck source-path=SCRIPTDIR/..
+
+source "${CNB_BUILDPACK_DIR}/lib/utils/download.sh"
+source "${CNB_BUILDPACK_DIR}/lib/utils/log.sh"
+source "${CNB_BUILDPACK_DIR}/lib/utils/toml.sh"
+
+install_runtime() {
+	local layers_dir="${1:?}"
+	local runtime_layer_dir="${layers_dir}/sf-fx-runtime-nodejs"
+	local runtime_layer_toml="${layers_dir}/sf-fx-runtime-nodejs.toml"
+
+	status "Installing Node.js function runtime"
+
+	mkdir -p "${runtime_layer_dir}"
+	cat >"${runtime_layer_toml}" <<-EOF
+		[types]
+		launch = true
+		build = false
+		cache = false
+	EOF
+
+	runtime_tarball_url=$(toml_get_key_from_metadata "${CNB_BUILDPACK_DIR}/buildpack.toml" "runtime.url")
+
+	info "Starting download of function runtime"
+
+	if ! download_file "${runtime_tarball_url}" "${runtime_layer_dir}/sf-fx-runtime-nodejs.tar.gz"; then
+		# Ideally this would be a multi-line error message explaining that the cause is
+		# likely network issues and to retry, but error() doesn't support multi-line.
+		error "Unable to download the function runtime from ${runtime_tarball_url}"
+		exit 1
+	fi
+
+	info "Download of function runtime successful"
+
+	npm install -g --prefix "${runtime_layer_dir}" "${runtime_layer_dir}/sf-fx-runtime-nodejs.tar.gz"
+
+	info "Function runtime installation successful"
+}
+
+write_launch_toml() {
+	local layers_dir="${1:?}"
+	local app_dir="${2:?}"
+
+	cat >"${layers_dir}/launch.toml" <<-EOF
+		[[processes]]
+		type = "web"
+		command = "sf-fx-runtime-nodejs ${app_dir}"
+	EOF
+}
diff --git a/buildpacks/nodejs-function-invoker/lib/detect.sh b/buildpacks/nodejs-function-invoker/lib/detect.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+detect_function_app() {
+	local app_dir="${1:?}"
+	[[ -f "${app_dir}/function.toml" || -f "${app_dir}/project.toml" ]]
+}
+
+write_to_build_plan() {
+	local build_plan="${1:?}"
+	cat >"${build_plan}" <<-EOF
+		[[provides]]
+		name = "nodejs-function-runtime"
+
+		[[requires]]
+		name = "nodejs-function-runtime"
+
+		[[requires]]
+		name = "node"
+	EOF
+}
diff --git a/buildpacks/nodejs-function-invoker/lib/utils/download.sh b/buildpacks/nodejs-function-invoker/lib/utils/download.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+download_file() {
+	local -r url="${1:?}"
+	local -r target_path="${2:?}"
+	curl --retry 3 --silent --fail --max-time 10 --location "${url}" --output "${target_path}"
+}
diff --git a/buildpacks/nodejs-function-invoker/lib/utils/log.sh b/buildpacks/nodejs-function-invoker/lib/utils/log.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+status() {
+	local color="\033[1;35m"
+	local no_color="\033[0m"
+	echo -e "\n${color}[${1:-""}]${no_color}"
+}
+
+info() {
+	echo -e "[INFO] ${1:-""}"
+}
+
+error() {
+	local color="\033[1;31m"
+	local no_color="\033[0m"
+
+	echo -e "\n${color}[Error: ${1:-""}]${no_color}\n"
+}
+
+warning() {
+	local color="\033[1;33m"
+	local no_color="\033[0m"
+
+	echo -e "\n${color}[Warning: ${1:-""}]${no_color}\n"
+}
+
+notice() {
+	local color="\033[1;34m"
+	local no_color="\033[0m"
+
+	echo -e "\n${color}[Notice: ${1:-""}]${no_color}\n"
+}
diff --git a/buildpacks/nodejs-function-invoker/lib/utils/toml.sh b/buildpacks/nodejs-function-invoker/lib/utils/toml.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+toml_get_key_from_metadata() {
+	local file="$1"
+	local key="$2"
+
+	if test -f "$file"; then
+		yj -t <"${file}" | jq -r ".metadata.${key}"
+	else
+		echo ""
+	fi
+}
diff --git a/buildpacks/nodejs-function-invoker/package.toml b/buildpacks/nodejs-function-invoker/package.toml
@@ -0,0 +1,2 @@
+[buildpack]
+uri = "."
diff --git a/buildpacks/nodejs-function-invoker/shpec/build_shpec.sh b/buildpacks/nodejs-function-invoker/shpec/build_shpec.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# shellcheck source-path=SCRIPTDIR/..
+
+# Not using -u due to:
+# https://github.com/rylnd/shpec/issues/126
+set -eo pipefail
+
+bp_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export CNB_BUILDPACK_DIR="${bp_dir}"
+
+source "${bp_dir}/lib/build.sh"
+
+create_temp_layers_dir() {
+	mktemp -d -t layers_shpec_XXXXX
+}
+
+create_temp_app_dir() {
+	mktemp -d -t layers_shpec_XXXXX
+}
+
+describe "lib/build.sh"
+	describe "install_runtime"
+		it "creates a runtime layer containing the runtime"
+			stub_command "npm"
+			layers_dir=$(create_temp_layers_dir)
+
+			output=$(install_runtime "${layers_dir}")
+			result="${?}"
+
+			assert equal "${result}" 0
+
+			assert file_present "${layers_dir}/sf-fx-runtime-nodejs.toml"
+			actual_layer_manifest=$(cat "${layers_dir}/sf-fx-runtime-nodejs.toml")
+			expected_layer_manifest=$(cat <<-EOF
+					[types]
+					launch = true
+					build = false
+					cache = false
+				EOF
+			)
+			assert equal "${actual_layer_manifest}" "${expected_layer_manifest}"
+			rm -rf "${layers_dir}"
+		end
+
+		it "handles download failures gracefully"
+			stub_command "download_file" "return 1"
+			layers_dir=$(create_temp_layers_dir)
+
+			set +e
+			output=$(install_runtime "${layers_dir}" 2>&1)
+			result="${?}"
+			set -e
+
+			assert equal "${result}" 1
+			assert grep "${output}" "Error: Unable to download the function runtime"
+			rm -rf "${layers_dir}"
+			unstub_command "download_file"
+		end
+	end
+
+	describe "write_launch_toml"
+		it "configures sf-fx-runtime-nodejs as the web process"
+			layers_dir=$(create_temp_layers_dir)
+			launch_toml="${layers_dir}/launch.toml"
+			app_dir=$(create_temp_app_dir)
+
+			write_launch_toml "${layers_dir}" "${app_dir}"
+			result="${?}"
+
+			assert equal "${result}" 0
+			assert file_present "${launch_toml}"
+			actual=$(cat "${launch_toml}")
+			expected=$(cat <<-EOF
+					[[processes]]
+					type = "web"
+					command = "sf-fx-runtime-nodejs ${app_dir}"
+				EOF
+			)
+			assert equal "${actual}" "${expected}"
+
+			rm -rf "${layers_dir}"
+			rm -rf "${app_dir}"
+		end
+	end
+end
diff --git a/buildpacks/nodejs-function-invoker/shpec/detect_shpec.sh b/buildpacks/nodejs-function-invoker/shpec/detect_shpec.sh

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+# Node.js Function Invoker Cloud Native Buildpack`