Persist pip-source layer; add layer path to PIP_FIND_LINKS env var

Author: thitch97

build.go

@@ -2,7 +2,6 @@ package pip
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -94,13 +93,20 @@ func Build(
 			return packit.BuildResult{}, err
 		}
 
+		pipSrcLayer, err := context.Layers.Get(PipSrc)
+		if err != nil {
+			return packit.BuildResult{}, err
+		}
+
 		cachedChecksum, ok := pipLayer.Metadata[DependencyChecksumKey].(string)
 		if ok && cargo.Checksum(cachedChecksum).Match(cargo.Checksum(dependency.Checksum)) {
 			logger.Process("Reusing cached layer %s", pipLayer.Path)
+			logger.Process("Reusing cached layer %s", pipSrcLayer.Path)
 			pipLayer.Launch, pipLayer.Build, pipLayer.Cache = launch, build, build
+			pipSrcLayer.Launch, pipSrcLayer.Build, pipSrcLayer.Cache = false, build, build
 
 			return packit.BuildResult{
-				Layers: []packit.Layer{pipLayer},
+				Layers: []packit.Layer{pipLayer, pipSrcLayer},
 				Build:  buildMetadata,
 				Launch: launchMetadata,
 			}, nil
@@ -111,25 +117,25 @@ func Build(
 			return packit.BuildResult{}, err
 		}
 
-		pipLayer.Launch, pipLayer.Build, pipLayer.Cache = launch, build, build
-
-		// Install the pip source to a temporary dir, since we only need access to
-		// it as an intermediate step when installing pip.
-		// It doesn't need to go into a layer, since we won't need it in future builds.
-		pipSrcDir, err := os.MkdirTemp("", "pip-source")
+		pipSrcLayer, err = pipSrcLayer.Reset()
 		if err != nil {
-			return packit.BuildResult{}, fmt.Errorf("failed to create temp pip-source dir: %w", err)
+			return packit.BuildResult{}, err
 		}
 
+		pipLayer.Launch, pipLayer.Build, pipLayer.Cache = launch, build, build
+		//Pip-source layer flags should mirror the Pip layer, but should never be
+		//available at launch.
+		pipSrcLayer.Launch, pipSrcLayer.Build, pipSrcLayer.Cache = false, build, build
+
 		logger.Process("Executing build process")
 		logger.Subprocess(fmt.Sprintf("Installing Pip %s", dependency.Version))
 
 		duration, err := clock.Measure(func() error {
-			err = dependencies.Deliver(dependency, context.CNBPath, pipSrcDir, context.Platform.Path)
+			err = dependencies.Deliver(dependency, context.CNBPath, pipSrcLayer.Path, context.Platform.Path)
 			if err != nil {
 				return err
 			}
-			return installProcess.Execute(pipSrcDir, pipLayer.Path)
+			return installProcess.Execute(pipSrcLayer.Path, pipLayer.Path)
 		})
 		if err != nil {
 			return packit.BuildResult{}, err
@@ -167,14 +173,21 @@ func Build(
 		}
 		pipLayer.SharedEnv.Prepend("PYTHONPATH", strings.TrimRight(sitePackagesPath, "\n"), ":")
 
+		// Append the pip source layer path to PIP_FIND_LINKS so that invocations
+		// of pip in downstream buildpacks have access to the packages bundled with
+		// the pip dependency (setuptools, wheel, etc.).
+
+		pipSrcLayer.BuildEnv.Append("PIP_FIND_LINKS", strings.TrimRight(pipSrcLayer.Path, "\n"), " ")
+
+		logger.EnvironmentVariables(pipSrcLayer)
 		logger.EnvironmentVariables(pipLayer)
 
 		pipLayer.Metadata = map[string]interface{}{
 			DependencyChecksumKey: dependency.Checksum,
 		}
 
 		return packit.BuildResult{
-			Layers: []packit.Layer{pipLayer},
+			Layers: []packit.Layer{pipLayer, pipSrcLayer},
 			Build:  buildMetadata,
 			Launch: launchMetadata,
 		}, nil

build_test.go

@@ -133,35 +133,52 @@ func testBuild(t *testing.T, context spec.G, it spec.S) {
 		result, err := build(buildContext)
 		Expect(err).NotTo(HaveOccurred())
 
-		Expect(result.Layers).To(HaveLen(1))
-		layer := result.Layers[0]
+		Expect(result.Layers).To(HaveLen(2))
+		pipLayer := result.Layers[0]
 
-		Expect(layer.Name).To(Equal("pip"))
+		Expect(pipLayer.Name).To(Equal("pip"))
 
-		Expect(layer.Path).To(Equal(filepath.Join(layersDir, "pip")))
+		Expect(pipLayer.Path).To(Equal(filepath.Join(layersDir, "pip")))
 
-		Expect(layer.SharedEnv).To(HaveLen(2))
-		Expect(layer.SharedEnv["PYTHONPATH.delim"]).To(Equal(":"))
-		Expect(layer.SharedEnv["PYTHONPATH.prepend"]).To(Equal(filepath.Join(layersDir, "pip", "lib/python1.23/site-packages")))
+		Expect(pipLayer.BuildEnv).To(BeEmpty())
+		Expect(pipLayer.LaunchEnv).To(BeEmpty())
+		Expect(pipLayer.ProcessLaunchEnv).To(BeEmpty())
 
-		Expect(layer.BuildEnv).To(BeEmpty())
-		Expect(layer.LaunchEnv).To(BeEmpty())
-		Expect(layer.ProcessLaunchEnv).To(BeEmpty())
+		Expect(pipLayer.Build).To(BeFalse())
+		Expect(pipLayer.Launch).To(BeFalse())
+		Expect(pipLayer.Cache).To(BeFalse())
 
-		Expect(layer.Build).To(BeFalse())
-		Expect(layer.Launch).To(BeFalse())
-		Expect(layer.Cache).To(BeFalse())
+		Expect(pipLayer.Metadata).To(HaveLen(1))
+		Expect(pipLayer.Metadata["dependency_checksum"]).To(Equal("some-sha"))
 
-		Expect(layer.Metadata).To(HaveLen(1))
-		Expect(layer.Metadata["dependency_checksum"]).To(Equal("some-sha"))
+		Expect(pipLayer.SharedEnv).To(HaveLen(2))
+		Expect(pipLayer.SharedEnv["PYTHONPATH.delim"]).To(Equal(":"))
+		Expect(pipLayer.SharedEnv["PYTHONPATH.prepend"]).To(Equal(filepath.Join(layersDir, "pip", "lib/python1.23/site-packages")))
 
-		Expect(layer.SBOM.Formats()).To(HaveLen(2))
+		Expect(pipLayer.SBOM.Formats()).To(HaveLen(2))
 		var actualExtensions []string
-		for _, format := range layer.SBOM.Formats() {
+		for _, format := range pipLayer.SBOM.Formats() {
 			actualExtensions = append(actualExtensions, format.Extension)
 		}
 		Expect(actualExtensions).To(ConsistOf("cdx.json", "spdx.json"))
 
+		pipSrcLayer := result.Layers[1]
+
+		Expect(pipSrcLayer.Name).To(Equal("pip-source"))
+
+		Expect(pipSrcLayer.Path).To(Equal(filepath.Join(layersDir, "pip-source")))
+
+		Expect(pipSrcLayer.LaunchEnv).To(BeEmpty())
+		Expect(pipSrcLayer.ProcessLaunchEnv).To(BeEmpty())
+
+		Expect(pipSrcLayer.Build).To(BeFalse())
+		Expect(pipSrcLayer.Launch).To(BeFalse())
+		Expect(pipSrcLayer.Cache).To(BeFalse())
+
+		Expect(pipSrcLayer.BuildEnv).To(HaveLen(2))
+		Expect(pipSrcLayer.BuildEnv["PIP_FIND_LINKS.delim"]).To(Equal(" "))
+		Expect(pipSrcLayer.BuildEnv["PIP_FIND_LINKS.append"]).To(Equal(filepath.Join(layersDir, "pip-source")))
+
 		Expect(dependencyManager.ResolveCall.Receives.Path).To(Equal(filepath.Join(cnbDir, "buildpack.toml")))
 		Expect(dependencyManager.ResolveCall.Receives.Id).To(Equal("pip"))
 		Expect(dependencyManager.ResolveCall.Receives.Version).To(Equal(""))
@@ -201,14 +218,22 @@ func testBuild(t *testing.T, context spec.G, it spec.S) {
 			result, err := build(buildContext)
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(result.Layers).To(HaveLen(1))
-			layer := result.Layers[0]
+			Expect(result.Layers).To(HaveLen(2))
+			pipLayer := result.Layers[0]
 
-			Expect(layer.Name).To(Equal("pip"))
+			Expect(pipLayer.Name).To(Equal("pip"))
 
-			Expect(layer.Build).To(BeTrue())
-			Expect(layer.Launch).To(BeTrue())
-			Expect(layer.Cache).To(BeTrue())
+			Expect(pipLayer.Build).To(BeTrue())
+			Expect(pipLayer.Launch).To(BeTrue())
+			Expect(pipLayer.Cache).To(BeTrue())
+
+			pipSrcLayer := result.Layers[1]
+
+			Expect(pipSrcLayer.Name).To(Equal("pip-source"))
+
+			Expect(pipSrcLayer.Build).To(BeTrue())
+			Expect(pipSrcLayer.Launch).To(BeFalse())
+			Expect(pipSrcLayer.Cache).To(BeTrue())
 
 			Expect(result.Build.BOM).To(Equal(
 				[]packit.BOMEntry{
@@ -261,14 +286,22 @@ func testBuild(t *testing.T, context spec.G, it spec.S) {
 			result, err := build(buildContext)
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(result.Layers).To(HaveLen(1))
-			layer := result.Layers[0]
+			Expect(result.Layers).To(HaveLen(2))
+			pipLayer := result.Layers[0]
+
+			Expect(pipLayer.Name).To(Equal("pip"))
+
+			Expect(pipLayer.Build).To(BeTrue())
+			Expect(pipLayer.Launch).To(BeFalse())
+			Expect(pipLayer.Cache).To(BeTrue())
+
+			pipSrcLayer := result.Layers[1]
 
-			Expect(layer.Name).To(Equal("pip"))
+			Expect(pipSrcLayer.Name).To(Equal("pip-source"))
 
-			Expect(layer.Build).To(BeTrue())
-			Expect(layer.Launch).To(BeFalse())
-			Expect(layer.Cache).To(BeTrue())
+			Expect(pipSrcLayer.Build).To(BeTrue())
+			Expect(pipSrcLayer.Launch).To(BeFalse())
+			Expect(pipSrcLayer.Cache).To(BeTrue())
 
 			Expect(buffer.String()).ToNot(ContainSubstring("Executing build process"))
 			Expect(buffer.String()).To(ContainSubstring("Reusing cached layer"))

constants.go

@@ -3,6 +3,8 @@ package pip
 // Pip is the name of the layer into which pip dependency is installed.
 const Pip = "pip"
 
+const PipSrc = "pip-source"
+
 // CPython is the name of the python runtime dependency provided by the CPython buildpack: https://github.com/paketo-buildpacks/cpython
 const CPython = "cpython"
 

integration/default_test.go

@@ -21,11 +21,20 @@ func testDefault(t *testing.T, context spec.G, it spec.S) {
 
 		pack   occam.Pack
 		docker occam.Docker
+		source string
 	)
 
 	it.Before(func() {
 		pack = occam.NewPack().WithVerbose()
 		docker = occam.NewDocker()
+
+		var err error
+		source, err = occam.Source(filepath.Join("testdata", "default_app"))
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	it.After(func() {
+		Expect(os.RemoveAll(source)).To(Succeed())
 	})
 
 	context("when the buildpack is run with pack build", func() {
@@ -49,6 +58,7 @@ func testDefault(t *testing.T, context spec.G, it spec.S) {
 
 		it("builds with the defaults", func() {
 			var err error
+
 			var logs fmt.Stringer
 			image, logs, err = pack.WithNoColor().Build.
 				WithPullPolicy("never").
@@ -57,7 +67,7 @@ func testDefault(t *testing.T, context spec.G, it spec.S) {
 					settings.Buildpacks.Pip.Online,
 					settings.Buildpacks.BuildPlan.Online,
 				).
-				Execute(name, filepath.Join("testdata", "default_app"))
+				Execute(name, source)
 			Expect(err).ToNot(HaveOccurred(), logs.String)
 
 			Expect(logs).To(ContainLines(
@@ -75,6 +85,9 @@ func testDefault(t *testing.T, context spec.G, it spec.S) {
 				MatchRegexp(`      Completed in \d+\.\d+`),
 			))
 			Expect(logs).To(ContainLines(
+				"  Configuring build environment",
+				MatchRegexp(fmt.Sprintf(`    PIP_FIND_LINKS -> "\$PIP_FIND_LINKS \/layers\/%s\/pip-source"`, strings.ReplaceAll(buildpackInfo.Buildpack.ID, "/", "_"))),
+				"",
 				"  Configuring build environment",
 				MatchRegexp(fmt.Sprintf(`    PYTHONPATH -> "\/layers\/%s\/pip\/lib\/python\d+\.\d+\/site-packages:\$PYTHONPATH"`, strings.ReplaceAll(buildpackInfo.Buildpack.ID, "/", "_"))),
 				"",
@@ -127,7 +140,7 @@ func testDefault(t *testing.T, context spec.G, it spec.S) {
 						"BP_LOG_LEVEL": "DEBUG",
 					}).
 					WithSBOMOutputDir(sbomDir).
-					Execute(name, filepath.Join("testdata", "default_app"))
+					Execute(name, source)
 				Expect(err).ToNot(HaveOccurred(), logs.String)
 
 				container, err = docker.Container.Run.