Skip to content

Feat/async job info query use informer #4484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fscnick wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Feat/async job info query use informer #4484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
21257f1
[RayJob] AsyncJobInfoQuery uses informer
fscnick Feb 2, 2026
6ae635e
[RayJob] add WatchErrorHandler for informer
fscnick Feb 2, 2026
23e44dc
[RayJob] dedup before sending to channel
fscnick Feb 3, 2026
712c43b
[RayJob] replace lru cache with normal cache
fscnick Feb 4, 2026
c140f4c
[RayJob] rename function and check url before enqueue
fscnick Feb 5, 2026
b11e6a9
Merge remote-tracking branch 'upstream/master' into feat/async-job-in…
fscnick Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apiserver/pkg/server/ray_job_submission_service_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type RayJobSubmissionServiceServer struct {
// Create RayJobSubmissionServiceServer
func NewRayJobSubmissionServiceServer(clusterServer *ClusterServer, options *RayJobSubmissionServiceServerOptions) *RayJobSubmissionServiceServer {
zl := zerolog.New(os.Stdout).Level(zerolog.DebugLevel)
return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(context.Background(), nil, false)}
return &RayJobSubmissionServiceServer{clusterServer: clusterServer, options: options, log: zerologr.New(&zl).WithName("jobsubmissionservice"), dashboardClientFunc: utils.GetRayDashboardClientFunc(nil, false)}
}

// Submit Ray job
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,12 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/maypok86/otter/v2 v2.3.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/moby/spdystream v0.5.0 // indirect
github.com/moby/term v0.5.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions ray-operator/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ COPY controllers/ controllers/
COPY pkg/features pkg/features
COPY pkg/utils pkg/utils
COPY pkg/webhooks pkg/webhooks
COPY pkg/client pkg/client
COPY rayjob-submitter/ rayjob-submitter/

# Build
Expand Down
6 changes: 2 additions & 4 deletions ray-operator/apis/config/v1alpha1/configuration_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package v1alpha1

import (
"context"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/manager"
Expand Down Expand Up @@ -90,8 +88,8 @@ type Configuration struct {
EnableMetrics bool `json:"enableMetrics,omitempty"`
}

func (config Configuration) GetDashboardClient(ctx context.Context, mgr manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) {
return utils.GetRayDashboardClientFunc(ctx, mgr, config.UseKubernetesProxy)
func (config Configuration) GetDashboardClient(mgr manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) {
return utils.GetRayDashboardClientFunc(mgr, config.UseKubernetesProxy)
}

func (config Configuration) GetHttpProxyClient(mgr manager.Manager) func(hostIp, podNamespace, podName string, port int) utils.RayHttpProxyClientInterface {
Expand Down
7 changes: 5 additions & 2 deletions ray-operator/controllers/ray/rayjob_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,11 @@ type RayJobReconcilerOptions struct {
}

// NewRayJobReconciler returns a new reconcile.Reconciler
func NewRayJobReconciler(ctx context.Context, mgr manager.Manager, options RayJobReconcilerOptions, provider utils.ClientProvider) *RayJobReconciler {
dashboardClientFunc := provider.GetDashboardClient(ctx, mgr)
func NewRayJobReconciler(mgr manager.Manager, options RayJobReconcilerOptions, provider utils.ClientProvider) *RayJobReconciler {
dashboardClientFunc := provider.GetDashboardClient(mgr)
if features.Enabled(features.AsyncJobInfoQuery) {
dashboardClientFunc = dashboardclient.GetCachedDashboardClientFunc()
}
Comment on lines +61 to +63
Copy link
Collaborator Author

@fscnick fscnick Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By leveraging global variable, it could avoid modifying the existed function signature. Or, should we pass GetCachedDashboardClientFunc() as an argument or somewhat from the upper layer?

return &RayJobReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Expand Down
4 changes: 2 additions & 2 deletions ray-operator/controllers/ray/rayservice_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ type RayServiceReconciler struct {
}

// NewRayServiceReconciler returns a new reconcile.Reconciler
func NewRayServiceReconciler(ctx context.Context, mgr manager.Manager, provider utils.ClientProvider) *RayServiceReconciler {
dashboardClientFunc := provider.GetDashboardClient(ctx, mgr)
func NewRayServiceReconciler(mgr manager.Manager, provider utils.ClientProvider) *RayServiceReconciler {
dashboardClientFunc := provider.GetDashboardClient(mgr)
httpProxyClientFunc := provider.GetHttpProxyClient(mgr)
return &RayServiceReconciler{
Client: mgr.GetClient(),
Expand Down
7 changes: 3 additions & 4 deletions ray-operator/controllers/ray/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ limitations under the License.
package ray

import (
"context"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -53,7 +52,7 @@ var (

type TestClientProvider struct{}

func (testProvider TestClientProvider) GetDashboardClient(_ context.Context, _ manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) {
func (testProvider TestClientProvider) GetDashboardClient(_ manager.Manager) func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error) {
return func(_ *rayv1.RayCluster, _ string) (dashboardclient.RayDashboardClientInterface, error) {
return fakeRayDashboardClient, nil
}
Expand Down Expand Up @@ -122,11 +121,11 @@ var _ = BeforeSuite(func(ctx SpecContext) {
Expect(err).NotTo(HaveOccurred(), "failed to setup RayCluster controller")

testClientProvider := TestClientProvider{}
err = NewRayServiceReconciler(ctx, mgr, testClientProvider).SetupWithManager(mgr, 1)
err = NewRayServiceReconciler(mgr, testClientProvider).SetupWithManager(mgr, 1)
Expect(err).NotTo(HaveOccurred(), "failed to setup RayService controller")

rayJobOptions := RayJobReconcilerOptions{}
err = NewRayJobReconciler(ctx, mgr, rayJobOptions, testClientProvider).SetupWithManager(mgr, 1)
err = NewRayJobReconciler(mgr, rayJobOptions, testClientProvider).SetupWithManager(mgr, 1)
Expect(err).NotTo(HaveOccurred(), "failed to setup RayJob controller")

go func() {
Expand Down
10 changes: 10 additions & 0 deletions ray-operator/controllers/ray/utils/constant.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,16 @@ const (
// If set to true, we will use deterministic name for head pod. Otherwise, the non-deterministic name is used.
ENABLE_DETERMINISTIC_HEAD_POD_NAME = "ENABLE_DETERMINISTIC_HEAD_POD_NAME"

// Async job info query
ASYNC_JOB_INFO_QUERY_INTERVAL = "ASYNC_JOB_INFO_QUERY_INTERVAL"
DEFAULT_ASYNC_JOB_INFO_QUERY_INTERVAL = "3s"

ASYNC_JOB_INFO_QUERY_WORKER_SIZE = "ASYNC_JOB_INFO_QUERY_WORKER_SIZE"
DEFAULT_ASYNC_JOB_INFO_QUERY_WORKER_SIZE = "8"

ASYNC_JOB_INFO_QUERY_CACHE_EXPIRY = "ASYNC_JOB_INFO_QUERY_CACHE_EXPIRY"
DEFAULT_ASYNC_JOB_INFO_QUERY_CACHE_EXPIRY = "10m"

// Ray core default configurations
DefaultWorkerRayGcsReconnectTimeoutS = "600"

Expand Down
Loading
Loading