feat: add docker tag-level metrics for granular download tracking

- Add new metrics for individual docker tags:
  - ghcr_tag_downloads: download count per tag
  - ghcr_tag_last_published_timestamp: when each tag was published
  - ghcr_package_tags: total number of tags per package
- Add getTagDownloadStats function to scrape tag-specific download data
- Add updateTagMetrics function to process all tags for each package
- This enables tracking which specific versions get downloaded more
- Allows monitoring of older versions fading out of use over time

Author: d0ugal

internal/collectors/ghcr_collector.go

@@ -385,13 +385,59 @@ func (gc *GHCRCollector) updatePackageMetrics(ctx context.Context, pkg config.Pa
 		gc.metrics.PackageLastPublishedGauge.WithLabelValues(pkg.Owner, pkg.Repo).Set(float64(lastPublished.Unix()))
 	}
 
+	// Update tag-level metrics
+	gc.updateTagMetrics(ctx, pkg, versions)
+
 	slog.Info("Updated package metrics",
 		"package", pkg.Repo,
 		"version_count", packageInfo.VersionCount,
 		"download_count", downloadCount,
 		"last_published", lastPublished.Format(time.RFC3339))
 }
 
+// updateTagMetrics updates metrics for individual docker tags
+func (gc *GHCRCollector) updateTagMetrics(ctx context.Context, pkg config.PackageGroup, versions []GHCRVersionResponse) {
+	totalTags := 0
+	tagCount := 0
+
+	for _, version := range versions {
+		// Parse the created_at timestamp
+		created, err := time.Parse(time.RFC3339, version.CreatedAt)
+		if err != nil {
+			slog.Warn("Failed to parse version timestamp", "version", version.Name, "error", err)
+			continue
+		}
+
+		// Process each tag for this version
+		for _, tag := range version.Metadata.Container.Tags {
+			totalTags++
+
+			// Update tag last published timestamp
+			gc.metrics.TagLastPublishedGauge.WithLabelValues(pkg.Owner, pkg.Repo, tag).Set(float64(created.Unix()))
+
+			// Try to get download statistics for this specific tag
+			tagDownloads, err := gc.getTagDownloadStats(ctx, pkg.Owner, pkg.Repo, tag)
+			if err != nil {
+				slog.Debug("Failed to get tag download statistics", "owner", pkg.Owner, "repo", pkg.Repo, "tag", tag, "error", err)
+				// Set to -1 to indicate no data available
+				gc.metrics.TagDownloadsGauge.WithLabelValues(pkg.Owner, pkg.Repo, tag).Set(-1)
+			} else {
+				gc.metrics.TagDownloadsGauge.WithLabelValues(pkg.Owner, pkg.Repo, tag).Set(float64(tagDownloads))
+
+				tagCount++
+			}
+		}
+	}
+
+	// Update total tag count for this package
+	gc.metrics.TagCountGauge.WithLabelValues(pkg.Owner, pkg.Repo).Set(float64(totalTags))
+
+	slog.Info("Updated tag metrics",
+		"package", pkg.Repo,
+		"total_tags", totalTags,
+		"tags_with_download_data", tagCount)
+}
+
 func (gc *GHCRCollector) retryWithBackoff(operation func() error, maxRetries int, initialDelay time.Duration) error {
 	var lastErr error
 
@@ -581,6 +627,136 @@ func (gc *GHCRCollector) getPackageDownloadStats(ctx context.Context, owner, pac
 	return downloadCount, nil
 }
 
+// getTagDownloadStats scrapes the tag page to get download statistics for a specific tag
+func (gc *GHCRCollector) getTagDownloadStats(ctx context.Context, owner, packageName, tag string) (int64, error) {
+	slog.Debug("Starting tag download statistics collection", "owner", owner, "package", packageName, "tag", tag)
+
+	// Construct the tag page URL
+	tagURL := fmt.Sprintf("https://github.com/%s/%s/pkgs/container/%s?tag=%s", owner, packageName, packageName, tag)
+	slog.Debug("Constructed tag URL", "url", tagURL)
+
+	// Create request to the tag page
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tagURL, nil)
+	if err != nil {
+		slog.Error("Failed to create HTTP request for tag", "owner", owner, "package", packageName, "tag", tag, "error", err)
+		return 0, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Set headers to mimic a browser request
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+	req.Header.Set("DNT", "1")
+	req.Header.Set("Connection", "keep-alive")
+	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	req.Header.Set("Sec-Fetch-Dest", "document")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+	req.Header.Set("Sec-Fetch-Site", "none")
+	req.Header.Set("Sec-Fetch-User", "?1")
+	req.Header.Set("Cache-Control", "max-age=0")
+
+	// Make the request
+	resp, err := gc.client.Do(req)
+	if err != nil {
+		slog.Error("Failed to fetch tag page", "owner", owner, "package", packageName, "tag", tag, "url", tagURL, "error", err)
+		return 0, fmt.Errorf("failed to fetch tag page: %w", err)
+	}
+
+	defer func() {
+		if closeErr := resp.Body.Close(); closeErr != nil {
+			slog.Warn("Failed to close response body", "error", closeErr)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		slog.Error("Tag page returned non-OK status", "owner", owner, "package", packageName, "tag", tag, "status_code", resp.StatusCode, "url", tagURL)
+		return 0, fmt.Errorf("tag page returned status %d", resp.StatusCode)
+	}
+
+	// Read the response body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		slog.Error("Failed to read response body", "owner", owner, "package", packageName, "tag", tag, "error", err)
+		return 0, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	// Handle gzip decompression if needed
+	if resp.Header.Get("Content-Encoding") == "gzip" {
+		gzReader, err := gzip.NewReader(strings.NewReader(string(body)))
+		if err != nil {
+			slog.Error("Failed to create gzip reader", "owner", owner, "package", packageName, "tag", tag, "error", err)
+			return 0, fmt.Errorf("failed to create gzip reader: %w", err)
+		}
+
+		defer func() {
+			if closeErr := gzReader.Close(); closeErr != nil {
+				slog.Warn("Failed to close gzip reader", "error", closeErr)
+			}
+		}()
+
+		// Read the decompressed content
+		decompressedBody, err := io.ReadAll(gzReader)
+		if err != nil {
+			slog.Error("Failed to read decompressed body", "owner", owner, "package", packageName, "tag", tag, "error", err)
+			return 0, fmt.Errorf("failed to read decompressed body: %w", err)
+		}
+
+		body = decompressedBody
+	}
+
+	// Parse the HTML document to find download statistics
+	htmlContent := string(body)
+	lines := strings.Split(htmlContent, "\n")
+
+	var downloadLine string
+
+	// Look for download statistics in the tag page
+	for i, line := range lines {
+		if strings.Contains(line, "Total downloads") {
+			if i+1 < len(lines) {
+				downloadLine = strings.TrimSpace(lines[i+1])
+				slog.Debug("Found download line after 'Total downloads'", "line", downloadLine)
+
+				break
+			}
+		}
+	}
+
+	if downloadLine == "" {
+		slog.Debug("Download statistics not found for tag", "owner", owner, "package", packageName, "tag", tag)
+		return 0, fmt.Errorf("download statistics not found in tag page")
+	}
+
+	// Extract the title attribute which contains the full number
+	titleStart := strings.Index(downloadLine, `title="`)
+	if titleStart == -1 {
+		slog.Debug("Download count title attribute not found for tag", "owner", owner, "package", packageName, "tag", tag, "line", downloadLine)
+		return 0, fmt.Errorf("download count title attribute not found")
+	}
+
+	titleStart += 7 // Skip 'title="'
+
+	titleEnd := strings.Index(downloadLine[titleStart:], `"`)
+	if titleEnd == -1 {
+		slog.Debug("Download count title attribute malformed for tag", "owner", owner, "package", packageName, "tag", tag, "line", downloadLine)
+		return 0, fmt.Errorf("download count title attribute malformed")
+	}
+
+	title := downloadLine[titleStart : titleStart+titleEnd]
+
+	// Parse the download count from the title attribute
+	downloadCount, err := strconv.ParseInt(title, 10, 64)
+	if err != nil {
+		slog.Debug("Failed to parse download count for tag", "owner", owner, "package", packageName, "tag", tag, "title", title, "error", err)
+		return 0, fmt.Errorf("failed to parse download count %s: %w", title, err)
+	}
+
+	slog.Debug("Successfully extracted tag download statistics", "owner", owner, "package", packageName, "tag", tag, "download_count", downloadCount, "raw_title", title)
+
+	return downloadCount, nil
+}
+
 // getOwnerPackages retrieves all packages for a given owner
 func (gc *GHCRCollector) getOwnerPackages(ctx context.Context, owner string) ([]GHCRPackageResponse, error) {
 	slog.Info("Getting packages for owner", "owner", owner)

internal/collectors/ghcr_collector_test.go

@@ -12,7 +12,6 @@ import (
 
 	"ghcr-exporter/internal/config"
 	"ghcr-exporter/internal/metrics"
-
 	"github.com/prometheus/client_golang/prometheus"
 )
 

internal/config/config.go

@@ -106,6 +106,7 @@ func (p PackageGroup) GetName() string {
 	if p.Repo == "" {
 		return p.Owner + "-all"
 	}
+
 	return p.Owner + "-" + p.Repo
 }
 

internal/metrics/registry.go

@@ -15,6 +15,11 @@ type Registry struct {
 	PackageLastPublishedGauge *prometheus.GaugeVec
 	PackageDownloadStatsGauge *prometheus.GaugeVec
 
+	// Tag-level metrics
+	TagDownloadsGauge     *prometheus.GaugeVec
+	TagLastPublishedGauge *prometheus.GaugeVec
+	TagCountGauge         *prometheus.GaugeVec
+
 	// Collection statistics
 	CollectionFailedCounter  *prometheus.CounterVec
 	CollectionSuccessCounter *prometheus.CounterVec
@@ -84,6 +89,34 @@ func NewRegistry() *Registry {
 	)
 	r.addMetricInfo("ghcr_package_last_published_timestamp", "Timestamp of the last published version for a GHCR package", []string{"owner", "repo"})
 
+	// Tag-level metrics
+	r.TagDownloadsGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "ghcr_tag_downloads",
+			Help: "Total number of downloads for a specific docker tag (scraped from package page)",
+		},
+		[]string{"owner", "repo", "tag"},
+	)
+	r.addMetricInfo("ghcr_tag_downloads", "Total downloads for a specific docker tag", []string{"owner", "repo", "tag"})
+
+	r.TagLastPublishedGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "ghcr_tag_last_published_timestamp",
+			Help: "Timestamp when a specific docker tag was last published",
+		},
+		[]string{"owner", "repo", "tag"},
+	)
+	r.addMetricInfo("ghcr_tag_last_published_timestamp", "Timestamp when a specific docker tag was last published", []string{"owner", "repo", "tag"})
+
+	r.TagCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "ghcr_package_tags",
+			Help: "Total number of tags for a GHCR package",
+		},
+		[]string{"owner", "repo"},
+	)
+	r.addMetricInfo("ghcr_package_tags", "Total number of tags for a GHCR package", []string{"owner", "repo"})
+
 	r.CollectionFailedCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "ghcr_collection_failed_total",