feat: add support for amazon linux 2023

Signed-off-by: Massimiliano Giovagnoli <me@maxgio.it>

Author: Massimiliano Giovagnoli

.github/workflows/publish.yaml

@@ -70,6 +70,26 @@ jobs:
           version: "2"
       - name: Publish results to S3
         run: make publish/amazonlinux2022
+  amazonlinux-2023:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - run: echo "STABLE=`echo $(git describe --tags --abbrev=0)`" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.STABLE }}
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+      - name: Install AWS CLI
+        id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@master
+        with:
+          version: "2"
+      - name: Publish results to S3
+        run: make publish/amazonlinux2023
   centos:
     runs-on: ubuntu-latest
     steps:

README.md

@@ -42,6 +42,7 @@ Available distributions:
 - *amazonlinux*
 - *amazonlinux2*
 - *amazonlinux2022*
+- *amazonlinux2023*
 - *centos*
 - *debian*
 - *ubuntu*

cmd/list_amazonlinux2023.go

@@ -0,0 +1,47 @@
+/*
+Copyright © 2022 maxgio92 <me@maxgio.it>
+
+Licensed under the Apache License, Version v2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/maxgio92/krawler/internal/format"
+	v2023 "github.com/maxgio92/krawler/pkg/distro/amazonlinux/v2023"
+)
+
+// amazonLinux2Cmd represents the centos command.
+var amazonLinux2023Cmd = &cobra.Command{
+	Use:   "amazonlinux2023",
+	Short: "List Amazon Linux 2023 kernel releases",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		kernelReleases, err := getKernelReleases(&v2023.AmazonLinux{}, RPMKernelHeadersPackageName)
+		cobra.CheckErr(err)
+
+		if len(kernelReleases) > 0 {
+			Output, err = format.Encode(Output, kernelReleases, format.Type(outputFormat))
+			cobra.CheckErr(err)
+		} else {
+			//nolint:errcheck
+			Output.WriteString("No releases found.\n")
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	listCmd.AddCommand(amazonLinux2023Cmd)
+}

pkg/distro/amazonlinux/v2023/amazonlinux.go

@@ -0,0 +1,127 @@
+package v2023
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/maxgio92/krawler/pkg/distro"
+	common "github.com/maxgio92/krawler/pkg/distro/amazonlinux"
+	packages "github.com/maxgio92/krawler/pkg/packages"
+	"github.com/maxgio92/krawler/pkg/packages/rpm"
+)
+
+type AmazonLinux struct {
+	common.AmazonLinux
+}
+
+func (a *AmazonLinux) Configure(config distro.Config) error {
+	return a.ConfigureCommon(DefaultConfig, config)
+}
+
+// SearchPackages scrapes each mirror, for each distro version, for each repository,
+// for each architecture, and returns slice of Package and optionally an error.
+func (a *AmazonLinux) SearchPackages(options packages.SearchOptions) ([]packages.Package, error) {
+	a.Config.Output.Logger = options.Log()
+
+	// Build distribution version-specific mirror root URLs.
+	perVersionMirrorURLs, err := a.BuildMirrorURLs(a.Config.Mirrors, a.Config.Versions)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build available repository URLs based on provided configuration,
+	// for each distribution version.
+	repositoriesURLrefs, err := common.BuildRepositoriesURLs(perVersionMirrorURLs, a.Config.Repositories)
+	if err != nil {
+		return nil, err
+	}
+
+	// Dereference repository URLs.
+	repositoryURLs, err := a.dereferenceRepositoryURLs(repositoriesURLrefs, a.Config.Archs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get RPM packages from each repository.
+	rss := []string{}
+	for _, ru := range repositoryURLs {
+		rss = append(rss, ru.String())
+	}
+
+	searchOptions := rpm.NewSearchOptions(&options, a.Config.Archs, rss)
+	rpmPackages, err := rpm.SearchPackages(searchOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return rpmPackages, nil
+}
+
+func (a *AmazonLinux) dereferenceRepositoryURLs(repoURLs []*url.URL, archs []packages.Architecture) ([]*url.URL, error) {
+	var urls []*url.URL
+
+	for _, ar := range archs {
+		for _, v := range repoURLs {
+			r, err := a.dereferenceRepositoryURL(v, ar)
+			if err != nil {
+				return nil, err
+			}
+
+			if r != nil {
+				urls = append(urls, r)
+			}
+		}
+	}
+
+	return urls, nil
+}
+
+func (a *AmazonLinux) dereferenceRepositoryURL(src *url.URL, arch packages.Architecture) (*url.URL, error) {
+	var dest *url.URL
+
+	mirrorListURL, err := url.JoinPath(src.String(), string(arch), "mirror.list")
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, mirrorListURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		a.Config.Output.Logger.Error("Amazon Linux v2023 repository URL not valid to be dereferenced")
+		//nolint:nilnil
+		return nil, nil
+	}
+
+	if resp.Body == nil {
+		a.Config.Output.Logger.Error("empty response from Amazon Linux v2023 repository reference URL")
+		//nolint:nilnil
+		return nil, nil
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get first repository URL available, no matter what the geolocation.
+	s := strings.Split(string(b), "\n")[0]
+
+	dest, err = url.Parse(s)
+	if err != nil {
+		return nil, err
+	}
+
+	return dest, nil
+}

pkg/distro/amazonlinux/v2023/constants.go

@@ -0,0 +1,26 @@
+package v2023
+
+import (
+	"github.com/maxgio92/krawler/pkg/distro"
+	"github.com/maxgio92/krawler/pkg/packages"
+)
+
+// DefaultConfig is the default configuration for scrape Amazon Linux (RPM) packages.
+// As of now URI templating depends on distro's viper.AllSettings() data.
+var DefaultConfig = distro.Config{
+	Mirrors: []packages.Mirror{
+		{
+			Name: "AL2023",
+			URL:  "https://cdn.amazonlinux.com/al2023/core/mirrors/",
+		},
+	},
+	Repositories: []packages.Repository{
+		{Name: "", URI: "latest"},
+	},
+	Archs: []packages.Architecture{
+		"aarch64",
+		"x86_64",
+		"ppc64le",
+	},
+	Versions: []distro.Version{""},
+}