Merge pull request #145 from zdtsw/cherrypick-upstream-prs

[Cherrypick] upstream PR from 0.5.1 to 0.6.0

Author: openshift-merge-bot[bot]

.github/actions/trivy-scan/action.yml

@@ -3,13 +3,14 @@ description: Scan container image with Trivy
 inputs:
   image:
     required: true
+    description: Image to scan
 runs:
   using: "composite"
   steps:
     - name: Install Trivy
       run: |
-        wget https://github.com/aquasecurity/trivy/releases/download/v0.44.1/trivy_0.44.1_Linux-64bit.deb
-        sudo dpkg -i trivy_0.44.1_Linux-64bit.deb
+        wget https://github.com/aquasecurity/trivy/releases/download/v0.69.2/trivy_0.69.2_Linux-64bit.deb
+        sudo dpkg -i trivy_0.69.2_Linux-64bit.deb
       shell: bash
 
 

.github/workflows/ci-build-images.yaml

@@ -0,0 +1,62 @@
+name: Build and Push Container Images
+
+on:
+  workflow_call:
+    inputs:
+      epp-image-name:
+        required: true
+        type: string
+      sidecar-image-name:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      prerelease:
+        required: true
+        type: string
+    secrets:
+      GHCR_TOKEN:
+        required: true
+
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+
+      - name: Checkout source
+        uses: actions/checkout@v6
+
+      - name: Build and push EPP image
+        uses: ./.github/actions/docker-build-and-push
+        with:
+          docker-file: Dockerfile.epp
+          tag: ${{ inputs.tag }}
+          image-name: ${{ inputs.epp-image-name }}
+          registry: ghcr.io/llm-d
+          github-token: ${{ secrets.GHCR_TOKEN }}
+          prerelease: ${{ inputs.prerelease }}
+
+      - name: Build and push sidecar image
+        uses: ./.github/actions/docker-build-and-push
+        with:
+          docker-file: Dockerfile.sidecar
+          tag: ${{ inputs.tag }}
+          image-name: ${{ inputs.sidecar-image-name }}
+          registry: ghcr.io/llm-d
+          github-token: ${{ secrets.GHCR_TOKEN }}
+          prerelease: ${{ inputs.prerelease }}
+
+      - name: Run Trivy scan on EPP image
+        uses: ./.github/actions/trivy-scan
+        with:
+          image: ghcr.io/llm-d/${{ inputs.epp-image-name }}:${{ inputs.tag }}
+
+      - name: Run Trivy scan on sidecar image
+        uses: ./.github/actions/trivy-scan
+        with:
+          image: ghcr.io/llm-d/${{ inputs.sidecar-image-name }}:${{ inputs.tag }}

.github/workflows/ci-dev.yaml

@@ -0,0 +1,38 @@
+name: CI - Dev - Docker Container Image
+
+on:
+  push:
+    branches:
+      - main
+      - 'release-*'
+
+jobs:
+  set-params:
+    runs-on: ubuntu-latest
+    outputs:
+      project_name: ${{ steps.version.outputs.project_name }}
+      sidecar_name: ${{ steps.version.outputs.sidecar_name }}
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+      - name: Set image names
+        id: version
+        run: |
+          repo="${GITHUB_REPOSITORY##*/}"
+          echo "project_name=${repo}-dev" >> "$GITHUB_OUTPUT"
+          echo "sidecar_name=llm-d-routing-sidecar-dev" >> "$GITHUB_OUTPUT"
+
+      - name: Set branch name as tag
+        id: tag
+        run: |
+          echo "tag=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
+
+  build-and-push:
+    needs: set-params
+    uses: ./.github/workflows/ci-build-images.yaml
+    with:
+      epp-image-name: ${{ needs.set-params.outputs.project_name }}
+      sidecar-image-name: ${{ needs.set-params.outputs.sidecar_name }}
+      tag: ${{ needs.set-params.outputs.tag }}
+      prerelease: "true"
+    secrets:
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}

.github/workflows/ci-release.yaml

@@ -8,27 +8,21 @@ on:
     types: [published]  # Also runs when a GitHub release is published
 
 jobs:
-  docker-build-and-push:
+  set-params:
     runs-on: ubuntu-latest
+    outputs:
+      project_name: ${{ steps.version.outputs.project_name }}
+      sidecar_name: ${{ steps.version.outputs.sidecar_name }}
+      tag: ${{ steps.tag.outputs.tag }}
+      prerelease: ${{ steps.tag.outputs.prerelease }}
     steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          tool-cache: false
-
-      - name: Checkout source
-        uses: actions/checkout@v6
-
       - name: Set image names
         id: version
         run: |
           repo="${GITHUB_REPOSITORY##*/}"
           echo "project_name=$repo" >> "$GITHUB_OUTPUT"
           echo "sidecar_name=llm-d-routing-sidecar" >> "$GITHUB_OUTPUT"
 
-      - name: Print project name
-        run: echo "Project is ${{ steps.version.outputs.project_name }}"
-
       - name: Determine tag name
         id: tag
         run: |
@@ -43,32 +37,13 @@ jobs:
           echo "prerelease=${PRE_RELEASE}" >> "$GITHUB_OUTPUT"
         shell: bash
 
-      - name: Build and push EPP image
-        uses: ./.github/actions/docker-build-and-push
-        with:
-          docker-file: Dockerfile.epp
-          tag: ${{ steps.tag.outputs.tag }}
-          image-name: ${{ steps.version.outputs.project_name }}
-          registry: ghcr.io/llm-d
-          github-token: ${{ secrets.GHCR_TOKEN }}
-          prerelease: ${{ steps.tag.outputs.prerelease }}
-
-      - name: Build and push sidecar image
-        uses: ./.github/actions/docker-build-and-push
-        with:
-          docker-file: Dockerfile.sidecar
-          tag: ${{ steps.tag.outputs.tag }}
-          image-name: ${{ steps.version.outputs.sidecar_name }}
-          registry: ghcr.io/llm-d
-          github-token: ${{ secrets.GHCR_TOKEN }}
-          prerelease: ${{ steps.tag.outputs.prerelease }}
-
-      - name: Run Trivy scan on EPP image
-        uses: ./.github/actions/trivy-scan
-        with:
-          image: ghcr.io/llm-d/${{ steps.version.outputs.project_name }}:${{ steps.tag.outputs.tag }}
-
-      - name: Run Trivy scan on sidecar image
-        uses: ./.github/actions/trivy-scan
-        with:
-          image: ghcr.io/llm-d/${{ steps.version.outputs.sidecar_name }}:${{ steps.tag.outputs.tag }}
+  build-and-push:
+    needs: set-params
+    uses: ./.github/workflows/ci-build-images.yaml
+    with:
+      epp-image-name: ${{ needs.set-params.outputs.project_name }}
+      sidecar-image-name: ${{ needs.set-params.outputs.sidecar_name }}
+      tag: ${{ needs.set-params.outputs.tag }}
+      prerelease: ${{ needs.set-params.outputs.prerelease }}
+    secrets:
+      GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}

DEVELOPMENT.md

@@ -8,15 +8,15 @@ Documentation for developing the inference scheduler.
 - [Golang] `v1.24`+
 - [Docker] (or [Podman])
 - [Kubernetes in Docker (KIND)]
-- [Kustomize]
+- [Kubectl] `v1.14`+
 - [ZeroMQ]
 
 [Make]:https://www.gnu.org/software/make/
 [Golang]:https://go.dev/
 [Docker]:https://www.docker.com/
 [Podman]:https://podman.io/
 [Kubernetes in Docker (KIND)]:https://github.com/kubernetes-sigs/kind
-[Kustomize]:https://kubectl.docs.kubernetes.io/installation/kustomize/
+[Kubectl]:https://kubectl.docs.kubernetes.io/installation/kubectl/
 [ZeroMQ]:https://zeromq.org/
 
 > [!NOTE]
@@ -59,30 +59,30 @@ There are several ways to access the gateway:
 **Port forward**:
 
 ```bash
-$ kubectl --context llm-d-inference-scheduler-dev port-forward service/inference-gateway 8080:80
+kubectl --context kind-llm-d-inference-scheduler-dev port-forward service/inference-gateway-istio 8080:80
 ```
 
 **NodePort**
 
 ```bash
 # Determine the k8s node address
-$ kubectl --context llm-d-inference-scheduler-dev get node -o yaml | grep address
+kubectl --context kind-llm-d-inference-scheduler-dev get node -o yaml | grep address
 # The service is accessible over port 80 of the worker IP address.
 ```
 
 **LoadBalancer**
 
 ```bash
 # Install and run cloud-provider-kind:
-$ go install sigs.k8s.io/cloud-provider-kind@latest && cloud-provider-kind &
-$ kubectl --context llm-d-inference-scheduler-dev get service inference-gateway
+go install sigs.k8s.io/cloud-provider-kind@latest && cloud-provider-kind &
+kubectl --context kind-llm-d-inference-scheduler-dev get service inference-gateway-istio
 # Wait for the LoadBalancer External-IP to become available. The service is accessible over port 80.
 ```
 
 You can now make requests matching the IP:port of one of the access mode above:
 
 ```bash
-$ curl -s -w '\n' http://<IP:port>/v1/completions -H 'Content-Type: application/json' -d '{"model":"food-review","prompt":"hi","max_tokens":10,"temperature":0}' | jq
+curl -s -w '\n' http://<IP:port>/v1/completions -H 'Content-Type: application/json' -d '{"model":"food-review","prompt":"hi","max_tokens":10,"temperature":0}' | jq
 ```
 
 By default the created inference gateway, can be accessed on port 30080. This can

Dockerfile.sidecar

@@ -17,6 +17,7 @@ RUN go mod download
 COPY cmd/pd-sidecar/main.go cmd/cmd.go
 COPY pkg/sidecar pkg/sidecar
 COPY pkg/common pkg/common
+COPY pkg/telemetry pkg/telemetry
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

Makefile

@@ -190,12 +190,12 @@ image-build-uds-tokenizer: check-container-tool ## Build UDS tokenizer image fro
 	if [ -z "$$KV_CACHE_PATH_CHECK" ]; then \
 		echo "Error: Could not find kv-cache module even after download."; \
 		exit 1; \
-	fi
+	fi; \
 	$(CONTAINER_RUNTIME) build \
 		--platform linux/$(TARGETARCH) \
 		-t $(UDS_TOKENIZER_IMAGE) \
-		-f $(KV_CACHE_PATH)/services/uds_tokenizer/Dockerfile \
-		$(KV_CACHE_PATH)/services/uds_tokenizer
+		-f $$KV_CACHE_PATH_CHECK/services/uds_tokenizer/Dockerfile \
+		$$KV_CACHE_PATH_CHECK/services/uds_tokenizer
 
 .PHONY: image-build-%
 image-build-%: check-container-tool ## Build Container image using $(CONTAINER_RUNTIME)

cmd/epp/main.go

@@ -32,15 +32,40 @@ import (
 
 	"github.com/llm-d/llm-d-inference-scheduler/pkg/metrics"
 	"github.com/llm-d/llm-d-inference-scheduler/pkg/plugins"
+	"github.com/llm-d/llm-d-inference-scheduler/pkg/telemetry"
 )
 
 func main() {
+	os.Exit(run())
+}
+
+func run() int {
+	ctx := ctrl.SetupSignalHandler()
+
+	// Initialize tracing before creating any spans
+	shutdownTracing, err := telemetry.InitTracing(ctx)
+	if err != nil {
+		// Log error but don't fail - tracing is optional
+		ctrl.Log.Error(err, "Failed to initialize tracing")
+	}
+	if shutdownTracing != nil {
+		defer func() {
+			if err := shutdownTracing(ctx); err != nil {
+				ctrl.Log.Error(err, "Failed to shutdown tracing")
+			}
+		}()
+	}
+
 	// Register llm-d-inference-scheduler plugins
 	plugins.RegisterAllPlugins()
 
+	// Note: GIE built-in plugins are automatically registered by the runner
+	// when it processes configuration in runner.parsePluginsConfiguration()
+
 	if err := runner.NewRunner().
 		WithCustomCollectors(metrics.GetCollectors()...).
-		Run(ctrl.SetupSignalHandler()); err != nil {
-		os.Exit(1)
+		Run(ctx); err != nil {
+		return 1
 	}
+	return 0
 }

cmd/pd-sidecar/main.go

@@ -16,7 +16,6 @@ limitations under the License.
 package main
 
 import (
-	"crypto/tls"
 	"flag"
 	"net/url"
 	"os"
@@ -29,6 +28,7 @@ import (
 
 	"github.com/llm-d/llm-d-inference-scheduler/pkg/sidecar/proxy"
 	"github.com/llm-d/llm-d-inference-scheduler/pkg/sidecar/version"
+	"github.com/llm-d/llm-d-inference-scheduler/pkg/telemetry"
 )
 
 var (
@@ -70,6 +70,20 @@ func main() {
 	ctx := ctrl.SetupSignalHandler()
 	log.IntoContext(ctx, logger)
 
+	// Initialize tracing before creating any spans
+	shutdownTracing, err := telemetry.InitTracing(ctx)
+	if err != nil {
+		// Log error but don't fail - tracing is optional
+		logger.Error(err, "Failed to initialize tracing")
+	}
+	if shutdownTracing != nil {
+		defer func() {
+			if err := shutdownTracing(ctx); err != nil {
+				logger.Error(err, "Failed to shutdown tracing")
+			}
+		}()
+	}
+
 	logger.Info("Proxy starting", "Built on", version.BuildRef, "From Git SHA", version.CommitSHA)
 
 	// Validate connector
@@ -111,28 +125,15 @@ func main() {
 		return
 	}
 
-	var cert *tls.Certificate
-	if *secureProxy {
-		var tempCert tls.Certificate
-		if *certPath != "" {
-			tempCert, err = tls.LoadX509KeyPair(*certPath+"/tls.crt", *certPath+"/tls.key")
-		} else {
-			tempCert, err = proxy.CreateSelfSignedTLSCertificate()
-		}
-		if err != nil {
-			logger.Error(err, "failed to create TLS certificate")
-			return
-		}
-		cert = &tempCert
-	}
-
 	config := proxy.Config{
 		Connector:                   *connector,
 		PrefillerUseTLS:             *prefillerUseTLS,
 		PrefillerInsecureSkipVerify: *prefillerInsecureSkipVerify,
 		DecoderInsecureSkipVerify:   *decoderInsecureSkipVerify,
 		DataParallelSize:            *vLLMDataParallelSize,
 		EnablePrefillerSampling:     *enablePrefillerSampling,
+		SecureServing:               *secureProxy,
+		CertPath:                    *certPath,
 	}
 
 	// Create SSRF protection validator
@@ -144,7 +145,7 @@ func main() {
 
 	proxyServer := proxy.NewProxy(*port, targetURL, config)
 
-	if err := proxyServer.Start(ctx, cert, validator); err != nil {
+	if err := proxyServer.Start(ctx, validator); err != nil {
 		logger.Error(err, "failed to start proxy server")
 	}
 }