Skip to content

Use pre-build binary packages for cpython buildpack #2799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 22, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 75 additions & 4 deletions hack/cmd/update-builder/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -1306,7 +1306,7 @@ func fixupPythonBuildpackARM64(ctx context.Context, config *builder.Config) erro
// these buildpacks need rebuild since they are only amd64 in paketo upstream
needsRebuild := map[string]patchSourceFn{
"cpython": func(srcDir string) error {
return fixupCPythonDistPkgRefs(filepath.Join(srcDir, "buildpack.toml"))
return fixupCPythonDistPkgRefs(ctx, filepath.Join(srcDir, "buildpack.toml"))
},
"miniconda": func(srcDir string) error {
return fixupMinicondaDistPkgRefs(filepath.Join(srcDir, "buildpack.toml"))
Expand Down Expand Up @@ -1382,7 +1382,8 @@ func fixupPythonBuildpackARM64(ctx context.Context, config *builder.Config) erro
return nil
}

func fixupCPythonDistPkgRefs(buildpackToml string) error {
func fixupCPythonDistPkgRefs(ctx context.Context, buildpackToml string) error {

tomlBytes, err := os.ReadFile(buildpackToml)
if err != nil {
return err
Expand All @@ -1396,14 +1397,84 @@ func fixupCPythonDistPkgRefs(buildpackToml string) error {

deps := config.(map[string]any)["metadata"].(map[string]any)["dependencies"].([]map[string]any)

// Since there are no cpython arm64 packages we set uri to source.
r := regexp.MustCompile(`[_-](\d+\.\d+\.\d+)[._]`)
parseVersion := func(url string) (string, error) {
ms := r.FindStringSubmatch(url)
if len(ms) != 2 {
return "", fmt.Errorf("missing version match in %q", url)
}
return ms[1], nil
}

getPackageInfo := func(version string) (url string, checksum string, err error) {
url = fmt.Sprintf(`https://github.com/matejvasek/cpython-dist/`+
`releases/download/v0.0.0/python_%s_linux_arm64_jammy.tgz`, version)
req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
if err != nil {
return "", "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", "", err
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", "", fmt.Errorf("non 200 code")
}

req, err = http.NewRequestWithContext(ctx, "GET", url+".checksum", nil)
if err != nil {
return "", "", err
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
return "", "", err
}
bs, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
return "", "", err
}
if resp.StatusCode != http.StatusOK {
return "", "", fmt.Errorf("non 200 code")
}

checksum = strings.TrimSpace(string(bs))

return
}

// If there are no cpython arm64 packages we set uri to source.
// This will cause cpython compilation to be done during the build process.
// This takes a while, but it's done only once per project.
for _, dep := range deps {
uriToSource := func(dep map[string]any) {
dep["checksum"] = dep["source-checksum"]
dep["uri"] = dep["source"]
}

for _, dep := range deps {
if !slices.Equal(dep["stacks"].([]any), []any{"io.buildpacks.stacks.jammy"}) {
// we have binary packages only for jammy stack, fallback to source build
uriToSource(dep)
continue
}

var ver, uri, checksum string
ver, err = parseVersion(dep["uri"].(string))
if err != nil {
return err
}

uri, checksum, err = getPackageInfo(ver)
if err != nil {
// binary package not found for the version, fallback to source build
uriToSource(dep)
continue
}
dep["uri"] = uri
dep["checksum"] = checksum
}

bs, err := toml.Marshal(config)
if err != nil {
return err
Expand Down
Loading