Fabric8 7.3.0 Informer rsync functionality intermittently fails, no event re-entry, returns to normal after restart, no error logs

[CrazyTomatoOo]

Describe the bug

    @PostConstruct
    public void init() {
        kubernetesClient = new KubernetesClientBuilder().build();
        modelLiteClient = new DefaultModelLiteClient();
        volcanoClient = new DefaultVolcanoClient(kubernetesClient);

        deploymentInformer =
            kubernetesClient.apps().deployments().inNamespace(namespace).runnableInformer(resyncPeriod * 5);
        serviceInformer = kubernetesClient.services().inNamespace(namespace).runnableInformer(0);
        statefulSetInformer =
            kubernetesClient.apps().statefulSets().inNamespace(namespace).runnableInformer(resyncPeriod * 5);
        modelInferenceInformer =
            kubernetesClient.resources(ModelInference.class).inNamespace(namespace).runnableInformer(resyncPeriod);

        deploymentLister = new Lister<>(deploymentInformer.getIndexer(), namespace);
        serviceLister = new Lister<>(serviceInformer.getIndexer(), namespace);
        statefulSetLister = new Lister<>(statefulSetInformer.getIndexer(), namespace);
        modelInferenceLister = new Lister<>(modelInferenceInformer.getIndexer(), namespace);
    }

    private void registerModelInferenceEventHandler() {
        helper.getModelInferenceInformer().addEventHandler(new ResourceEventHandler<>() {
            @Override
            public void onAdd(ModelInference obj) {
                enqueue(obj);
            }

            @Override
            public void onUpdate(ModelInference oldObj, ModelInference newObj) {
                enqueue(newObj);
            }

            @Override
            public void onDelete(ModelInference obj, boolean deletedFinalStateUnknown) {
                enqueue(obj);
            }
        });
    }

    private void enqueue(ModelInference obj) {
        log.info("Enqueue[{}]", obj.getMetadata().getName());
        String key = Cache.metaNamespaceKeyFunc(obj);
        log.info("Going to enqueue key {}", key);
        if (key != null && !key.isEmpty()) {
            log.info("Adding modelInference[{}] to workqueue", obj.getMetadata().getName());
            workQueue.add(key);
        }
    }

[2026/02/13 08:51:56.635][][INFO][com.....v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.641][][INFO][com.....v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.641][][INFO][com.....v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.642][][INFO][com.....v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.645][][INFO][com.....v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.674][][INFO][com.....v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.694][][INFO][com.....v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.694][][INFO][com.....v1.modelinference.ModelInferenceController][runController,266][-1207935560-pool-2-thread-1][Work Queue is empty]
[2026/02/13 08:52:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:52:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:10.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:53:25.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:54:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:10.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:55:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:56:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:57:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:55.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:58:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:58:17.869][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]

Fabric8 Kubernetes Client version

SNAPSHOT

Steps to reproduce

Cannot be consistently reproduced

Expected behavior

Event periodically re-enters

Runtime

Kubernetes (vanilla)

Kubernetes API Server version

other (please specify in additional context)

Environment

Linux

Fabric8 Kubernetes Client Logs

Additional context

Client Version: v1.31.1
Kustomize Version: v5.4.2
Server Version: v1.31.1

[shawkins]

Could you refine what the issue is showing?

The code sample shows an event handler, but not that it's actually getting used.

The log doesn't show any messages that appear in the code sample - presumably this is trying to show what the failed state looks like.

Can you show what normal operation looks like? And include a thread dump when you believe it is not functioning properly?

[CrazyTomatoOo]

Sorry, because the issue cannot be reproduced, it might be difficult to understand.
When I created the informer, I specified the rsync time, expecting it to trigger the handler at fixed intervals. There should be logs like 'Enqueue **' in the logs (which are from the enqueue method), but in reality, there are none. The log entry 'Inner url for health probe: /actuator/health]' appears when using the informer's isWatching() method, once per minute, which can prove that rsync was not triggered for a long time. After I restarted the Pod, everything started working normally.
I have also provided our reconciliation loop code below.

/**
 * ModelInferenceController
 *
 * @since 2025-09-02
 */
@Service
@Slf4j
@ConditionalOnExpression("'model-inference-controller'.equals('${controller.name}')")
public class ModelInferenceController implements ApplicationRunner, HealthIndicator {
    /**
     * 控制器名称
     */
    private final String controllerName = "model-inference-controller";

    /**
     * Kubernetes相关Client、informer、lister集合
     */
    @Resource
    private ModelInferenceHelper helper;

    /**
     * 推理服务处理器
     */
    @Resource
    private ModelInferenceReconciler reconciler;

    /**
     * 工作队列大小
     */
    @Value("${controller.work-queue-size:2048}")
    private int workQueueSize;

    /**
     * 租期
     */
    @Value("${controller.leader-election.leader-lease-duration:8}")
    private int leaseDuration;

    /**
     * 领导者重选周期
     */
    @Value("${controller.leader-election.leader-renew-deadline:6}")
    private int leaderRenew;

    /**
     * 重试周期
     */
    @Value("${controller.leader-election.leader-retry-period:2}")
    private int retryPeriod;

    /**
     * 工作队列
     */
    private BlockingQueue<String> workQueue;

    /**
     * 初始化方法，在对象创建后立即执行。
     * 初始化工作队列，并初始化事件处理器，最后运行工作队列。
     */
    @PostConstruct
    public void init() {
        workQueue = new ArrayBlockingQueue<>(workQueueSize);
    }

    @Override
    public void run(final ApplicationArguments args) throws Exception {
        log.info("ModelInference Starting controller {}", controllerName);
        for (int i = 0; i < Integer.MAX_VALUE; i++) {
            log.info("ModelInference register leader election begin");
            LeaderElector leaderElector = registerLeaderElection();
            log.info("ModelInference register leader election complete");
            try {
                leaderElector.run();
                log.warn("ModelInference Leader elector exited normally, restarting...");
                Thread.sleep(10000);
            } catch (Exception e) {
                log.error("ModelInference Leader election failed, retrying...", e);
            }
        }
        log.info("ModelInferenceController exit");
    }

    @Override
    public Health health() {
        if (helper.getModelInferenceInformer().isWatching() && helper.getDeploymentInformer().isWatching()) {
            return Health.up().withDetail("message", "ModelInference controller is running").build();
        }
        return Health.down().withDetail("message", "ModelInference controller is not running").build();
    }

    private LeaderElector registerLeaderElection() {
        // renewDeadline > retryPeriod+(retryPeriod×JITTER_FACTOR), JITTER_FACTOR around 1.2
        // leaseDuration >= renewDeadLine
        LeaderElectionConfig config = new LeaderElectionConfigBuilder().withReleaseOnCancel()
            .withLeaseDuration(Duration.ofSeconds(leaseDuration))
            .withRenewDeadline(Duration.ofSeconds(leaderRenew)) // 续租截止时间
            .withRetryPeriod(Duration.ofSeconds(retryPeriod)) // 重试周期
            .withReleaseOnCancel(true) // 取消时释放领导权
            .withName(controllerName + "-leader-election") // 选举名称
            .withLock(new LeaseLock(helper.getNamespace(), controllerName + "-lock", System.getenv("POD_NAME")))
            .withLeaderCallbacks(new LeaderCallbacks(() -> {
                log.info("Started leadership.");
                initInformerEventHandlers();
                helper.startAllRegisteredInformers();
                log.info("Starting modelInference controller");
                this.runController();
            }, () -> log.info("Stopped leadership."), newLeader -> log.info("New leader elected {}", newLeader)))
            .build();
        return helper.getKubernetesClient().leaderElector().withConfig(config).build();
    }

    private void initInformerEventHandlers() {
        registerModelInferenceEventHandler();
        registerCommonEventHandler(helper.getDeploymentInformer());
        registerCommonEventHandler(helper.getServiceInformer());
        registerCommonEventHandler(helper.getStatefulSetInformer());
    }

    private <T extends HasMetadata> void registerCommonEventHandler(SharedIndexInformer<T> informer) {
        informer.addEventHandler(new ResourceEventHandler<>() {
            @Override
            public void onAdd(T obj) {
                ownerReferenceFilter(obj);
            }

            @Override
            public void onUpdate(T oldObj, T newObj) {
                if (oldObj.getMetadata().getResourceVersion().equals(newObj.getMetadata().getResourceVersion())) {
                    return;
                }
                ownerReferenceFilter(newObj);
            }

            @Override
            public void onDelete(T obj, boolean deletedFinalStateUnknown) {
                ownerReferenceFilter(obj);
            }
        });
    }

    /**
     * 处理对象的所有者引用。
     * 如果对象的所有者引用为空，或者所有者的类型不是ModelInference，或者ModelInference不存在，则忽略该对象。
     * 否则，将ModelInference对象加入队列。
     *
     * @param obj 需要处理的对象
     */
    private void ownerReferenceFilter(HasMetadata obj) {
        log.info("Handle object[{}/{}/{}]", obj.getMetadata().getNamespace(), obj.getKind(),
            obj.getMetadata().getName());
        Optional<OwnerReference> ownerReferenceOption = getControllerOwnerReferenceOf(obj);
        if (ownerReferenceOption.isEmpty()) {
            return;
        }

        OwnerReference ownerReference = ownerReferenceOption.get();
        if (!ownerReference.getKind().equalsIgnoreCase(ModelInference.class.getSimpleName())) {
            return;
        }

        ModelInference modelInference = helper.getModelInferenceLister().get(ownerReference.getName());
        if (modelInference == null) {
            log.info("Ignoring orphaned object '{}' of modelInference '{}'", obj.getMetadata().getSelfLink(),
                ownerReference.getName());
            return;
        }

        log.info("Received resource[{}/{}/{}] event", obj.getMetadata().getNamespace(), obj.getKind(),
            obj.getMetadata().getName());
        enqueue(modelInference);
    }

    private Optional<OwnerReference> getControllerOwnerReferenceOf(HasMetadata obj) {
        List<OwnerReference> ownerReferences = obj.getMetadata().getOwnerReferences();
        for (OwnerReference ownerReference : ownerReferences) {
            if (ownerReference != null && Boolean.TRUE.equals(ownerReference.getController())) {
                return Optional.of(ownerReference);
            }
        }
        return Optional.empty();
    }

    private void registerModelInferenceEventHandler() {
        helper.getModelInferenceInformer().addEventHandler(new ResourceEventHandler<>() {
            @Override
            public void onAdd(ModelInference obj) {
                enqueue(obj);
            }

            @Override
            public void onUpdate(ModelInference oldObj, ModelInference newObj) {
                enqueue(newObj);
            }

            @Override
            public void onDelete(ModelInference obj, boolean deletedFinalStateUnknown) {
                enqueue(obj);
            }
        });
    }

    private void enqueue(ModelInference obj) {
        log.info("Enqueue[{}]", obj.getMetadata().getName());
        String key = Cache.metaNamespaceKeyFunc(obj);
        log.info("Going to enqueue key {}", key);
        if (key != null && !key.isEmpty()) {
            log.info("Adding modelInference[{}] to workqueue", obj.getMetadata().getName());
            workQueue.add(key);
        }
    }

    private void runController() {
        log.info("Starting {} controller", controllerName);
        while (!helper.hasSynced()) {
            helper.startAllRegisteredInformers();
            log.info("Waiting for informer sync.");
            ThreadUtil.safeSleep(5000);
        }

        String key = "";
        while (!Thread.currentThread().isInterrupted()) {
            try {
                log.info("trying to fetch item from workqueue.");
                if (workQueue.isEmpty()) {
                    log.info("Work Queue is empty");
                }

                key = workQueue.take();
                requireNonNull(key, "key can't be null");
                log.info("Got {}", key);
                if (!key.contains("/")) {
                    log.warn("Invalid resource key: {}", key);
                }

                // Get the ModelInference resource's name from key which is in format namespace/name
                String name = key.split("/")[1];
                ModelInference modelInference = helper.getModelLiteClient()
                    .v1()
                    .modelInference()
                    .inNamespace(helper.getNamespace())
                    .withName(name)
                    .get();
                if (modelInference == null) {
                    log.error("ModelInference {} in workqueue no longer exists", name);
                    continue;
                }
                reconcile(modelInference);
            } catch (ServiceException e) {
                log.error("Handle model inference reconcile meet err, code: {}, msg: {}", e.getErrorCode(),
                    e.getMessage());
            } catch (InterruptedException e) {
                log.error("Controller interrupted. ", e);
            } catch (Exception e) {
                log.error("Unexpected error occurs err: {}", e.getMessage());
            }
        }
        log.error("The thread was interrupted...................");
    }

    private void reconcile(ModelInference modelInference) throws ServiceException {
        try {
            reconciler.reconcile(modelInference);
        } catch (KubernetesClientException e) {
            log.error("Meet kubernetes client exception while reconcile: ", e);
        }
    }
}



/**
 * ModelInferenceReconciler
 *
 * @since 2025-09-02
 */
@Component
@Slf4j
@ConditionalOnExpression("'model-inference-controller'.equals('${controller.name}')")
public class ModelInferenceReconciler {
    @Resource
    private StatusHandler availableHandler;

    @Resource
    private StatusHandler deletingHandler;

    @Resource
    private StatusHandler insufficientResourceHandler;

    @Resource
    private StatusHandler waitingHandler;

    @Resource
    private StatusHandler startingHandler;

    @Resource
    private StatusHandler stoppedHandler;

    @Resource
    private StatusHandler stoppingHandler;

    @Resource
    private StatusHandler unavailableHandler;

    @Resource
    private StatusHandler unschedulableHandler;

    @Resource
    private StatusHandler imagePullFailureHandler;

    @Resource
    private StatusHandler unimplementedHandler;

    @Resource
    private ModelInferenceHelper helper;

    /**
     * reconcile
     *
     * @param modelInference modelInference
     * @throws ServiceException ServiceException
     */
    public void reconcile(ModelInference modelInference) throws ServiceException {
        String name = modelInference.getMetadata().getName();
        InferenceFramework framework = InferenceFrameworkFactory.byName(modelInference.getSpec().getFramework());
        log.info("Start to reconcile CR ModelInference[{}]", name);
        if (Objects.isNull(modelInference.getStatus())) {
            syncModelInferenceStatus(modelInference);
            return;
        }
        if (modelInference.isMarkedForDeletion()) {
            if (DELETING.equals(modelInference.getStatus().getStatus())) {
                deletingHandler.handle(modelInference, framework);
            } else {
                log.info("ModelInference[{}] is marked for deletion, change status to {}.", name, DELETING);
                helper.updateInferenceStatus(modelInference, DELETING, null);
            }
            return;
        }

        if (!modelInference.hasFinalizer(MODEL_INFERENCE_FINALIZER)) {
            log.info("Add finalizer for ModelInference[{}].", name);
            modelInference.addFinalizer(MODEL_INFERENCE_FINALIZER);
            helper.updateModelInference(modelInference);
            return;
        }
        if (!modelInference.getSpec().getEnable() && !STOPPING.equals(modelInference.getStatus().getStatus()) &&
                !STOPPED.equals(modelInference.getStatus().getStatus())) {
            log.info("ModelInference[{}] is disabled, change status to {}.", name, STOPPING);
            helper.updateInferenceStatus(modelInference, STOPPING, null);
            return;
        }
        if (modelInference.getSpec().getRestart() && !WAITING.equals(modelInference.getStatus().getStatus())) {
            restartModelInference(modelInference);
            return;
        }

        handle(modelInference, framework);
    }

    private void restartModelInference(final ModelInference modelInference) {
        log.info("Restart ModelInference[{}].", modelInference.getMetadata().getName());
        syncModelInferenceStatus(modelInference);
    }

    private void syncModelInferenceStatus(ModelInference modelInference) {
        log.info("Init ModelInference[{}] status.", modelInference.getMetadata().getName());
        ModelInference build = modelInference.edit()
            .editOrNewStatus()
            .withStatus(WAITING).withReason(WAITING.toString())
            .withFramework(modelInference.getSpec().getFramework())
            .withDeployInfos(modelInference.getSpec().getDeployInfos())
            .withModelMeta(modelInference.getSpec().getModelMeta())
            .withInstanceName(modelInference.getSpec().getInstanceName())
            .withFrameworkConfig(modelInference.getSpec().getFrameworkConfig())
            .withPodDetails(new ArrayList<>())
            .withPodResources(modelInference.getSpec().getPodResources())
            .withPvcParams(modelInference.getSpec().getPvcParams())
            .withAdditionalParams(modelInference.getSpec().getAdditionalParams())
            .endStatus()
            .build();
        helper.updateModelInferenceStatus(build);
    }

    private void handle(ModelInference modelInference, InferenceFramework framework) throws ServiceException {
        InferenceStatus status = modelInference.getStatus().getStatus();
        log.info("Handle model inference instance [{}] in status [{}]", modelInference.getMetadata().getName(),
            status);
        switch (status) {
            case WAITING -> waitingHandler.handle(modelInference, framework);
            case STARTING -> startingHandler.handle(modelInference, framework);
            case STOPPED -> stoppedHandler.handle(modelInference, framework);
            case STOPPING -> stoppingHandler.handle(modelInference, framework);
            case AVAILABLE -> availableHandler.handle(modelInference, framework);
            case UNAVAILABLE -> unavailableHandler.handle(modelInference, framework);
            case DELETING -> deletingHandler.handle(modelInference, framework);
            case INSUFFICIENT_RESOURCE -> insufficientResourceHandler.handle(modelInference, framework);
            case UNSCHEDULABLE -> unschedulableHandler.handle(modelInference, framework);
            case IMAGE_PULL_FAILURE -> imagePullFailureHandler.handle(modelInference, framework);
            default -> unimplementedHandler.handle(modelInference, framework);
        }
    }
}

[shawkins]

There should be logs like 'Enqueue **' in the logs (which are from the enqueue method), but in reality, there are none.

The resync mechanism is for items in the cache. You haven't shown log snippets with the enqueue method being called, but assuming those were previously appearing in the logs, then stop, you should take a thread dump. In particular the CachedSingleThreadScheduler thread is responsible for scheduling the work onto the main client executor. If you see something has caused the scheduler thread to hang, or you have exhausted all of the client threads, that would cause something like this behavior.

[CrazyTomatoOo]

Here is more log information. In addition, since I cannot reproduce this issue, I may not be able to obtain Thread Dump information or logs. Do you have any good suggestions for handling this? Or perhaps there is a way to proactively detect when this issue occurs and then restart the service.

[2026/02/13 08:51:42.136][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:42.136][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:42.142][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:42.142][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [WAITING]]
[2026/02/13 08:51:42.145][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Deployment[ai43ek-inference].]
[2026/02/13 08:51:42.171][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Service[ai43ek-inference].]
[2026/02/13 08:51:42.178][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource ConfigMap[ai43ek-inference].]
[2026/02/13 08:51:42.207][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:42.208][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:42.213][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:42.214][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [WAITING]]
[2026/02/13 08:51:42.216][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Deployment[ai43ek-inference].]
[2026/02/13 08:51:42.243][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Service[ai43ek-inference].]
[2026/02/13 08:51:42.254][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource ConfigMap[ai43ek-inference].]
[2026/02/13 08:51:42.291][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:42.291][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:42.298][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:42.298][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [WAITING]]
[2026/02/13 08:51:42.301][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Deployment[ai43ek-inference].]
[2026/02/13 08:51:42.326][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource Service[ai43ek-inference].]
[2026/02/13 08:51:42.334][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deployOrReplace,484][-1207935560-pool-2-thread-1][Create or replace resource ConfigMap[ai43ek-inference].]
[2026/02/13 08:51:42.368][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:42.369][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,266][-1207935560-pool-2-thread-1][Work Queue is empty]
[2026/02/13 08:51:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:51:56.106][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-900][Enqueue[ai43ek]]
[2026/02/13 08:51:56.107][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-900][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.107][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-900][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.108][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.115][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.115][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,106][-1207935560-pool-2-thread-1][ModelInference[ai43ek] is disabled, change status to STOPPING.]
[2026/02/13 08:51:56.116][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][updateInferenceStatus,147][-1207935560-pool-2-thread-1][ModelInference[ai43ek] status from [WAITING] to [STOPPING].]
[2026/02/13 08:51:56.132][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.133][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,266][-1207935560-pool-2-thread-1][Work Queue is empty]
[2026/02/13 08:51:56.135][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-899][Enqueue[ai43ek]]
[2026/02/13 08:51:56.136][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-899][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.136][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-899][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.136][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.142][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.142][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.142][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.146][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.167][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-900][Handle object[model-engine-dyy/Deployment/ai43ek-inference]]
[2026/02/13 08:51:56.168][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-900][Received resource[model-engine-dyy/Deployment/ai43ek-inference] event]
[2026/02/13 08:51:56.168][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-900][Enqueue[ai43ek]]
[2026/02/13 08:51:56.169][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-900][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.169][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-900][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.179][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-899][Handle object[model-engine-dyy/Service/ai43ek-inference]]
[2026/02/13 08:51:56.180][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-899][Received resource[model-engine-dyy/Service/ai43ek-inference] event]
[2026/02/13 08:51:56.181][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-899][Enqueue[ai43ek]]
[2026/02/13 08:51:56.181][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-899][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.181][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-899][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.191][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-900][Handle object[model-engine-dyy/Service/ai43ek-inference]]
[2026/02/13 08:51:56.192][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-900][Received resource[model-engine-dyy/Service/ai43ek-inference] event]
[2026/02/13 08:51:56.192][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-900][Enqueue[ai43ek]]
[2026/02/13 08:51:56.193][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-900][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.193][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-900][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.204][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][deleteServiceIfExist,379][-1207935560-pool-2-thread-1][Delete Service for inference instance [ai43ek], count: 1]
[2026/02/13 08:51:56.209][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-899][Handle object[model-engine-dyy/Deployment/ai43ek-inference]]
[2026/02/13 08:51:56.210][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-899][Received resource[model-engine-dyy/Deployment/ai43ek-inference] event]
[2026/02/13 08:51:56.210][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-899][Enqueue[ai43ek]]
[2026/02/13 08:51:56.212][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-899][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.212][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-899][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.215][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.234][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-900][Handle object[model-engine-dyy/Deployment/ai43ek-inference]]
[2026/02/13 08:51:56.235][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-900][Received resource[model-engine-dyy/Deployment/ai43ek-inference] event]
[2026/02/13 08:51:56.235][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-900][Enqueue[ai43ek]]
[2026/02/13 08:51:56.236][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-900][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.236][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-900][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.236][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-901][Enqueue[ai43ek]]
[2026/02/13 08:51:56.237][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-901][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.237][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-901][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.240][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.241][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.246][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.246][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.247][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.251][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.299][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.320][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.321][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.327][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.327][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.328][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.331][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.362][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.389][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,190][-1207935560-pool-2-thread-900][Handle object[model-engine-dyy/Deployment/ai43ek-inference]]
[2026/02/13 08:51:56.390][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][ownerReferenceFilter,209][-1207935560-pool-2-thread-900][Received resource[model-engine-dyy/Deployment/ai43ek-inference] event]
[2026/02/13 08:51:56.391][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,244][-1207935560-pool-2-thread-900][Enqueue[ai43ek]]
[2026/02/13 08:51:56.391][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,246][-1207935560-pool-2-thread-900][Going to enqueue key model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.392][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][enqueue,248][-1207935560-pool-2-thread-900][Adding modelInference[ai43ek] to workqueue]
[2026/02/13 08:51:56.392][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.393][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.399][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.399][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.400][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.403][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.433][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.453][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.454][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.459][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.460][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.460][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.464][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.497][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.517][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.517][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.523][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.523][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.524][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.527][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.555][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.575][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.575][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.581][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.581][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.582][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.585][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.614][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.635][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.635][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,271][-1207935560-pool-2-thread-1][Got model-engine-dyy/ai43ek]
[2026/02/13 08:51:56.641][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][reconcile,83][-1207935560-pool-2-thread-1][Start to reconcile CR ModelInference[ai43ek]]
[2026/02/13 08:51:56.641][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceReconciler][handle,144][-1207935560-pool-2-thread-1][Handle model inference instance [ai43ek] in status [STOPPING]]
[2026/02/13 08:51:56.642][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.StoppingHandler][handle,41][-1207935560-pool-2-thread-1][Handle ModelInference [model-engine-dyy/ai43ek] in Stopping status.]
[2026/02/13 08:51:56.645][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceHelper][scaleDeployment,501][-1207935560-pool-2-thread-1][Scale Deployment[ai43ek-inference] replicas to 0]
[2026/02/13 08:51:56.674][][INFO][com.....acs.modelengine.modellite.v1.modelinference.handler.Common][handlePodsDeletionInLimitTime,108][-1207935560-pool-2-thread-1][Waiting for model inference pods [ai43ek] to be deleted]
[2026/02/13 08:51:56.694][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,264][-1207935560-pool-2-thread-1][trying to fetch item from workqueue.]
[2026/02/13 08:51:56.694][][INFO][com.....acs.modelengine.modellite.v1.modelinference.ModelInferenceController][runController,266][-1207935560-pool-2-thread-1][Work Queue is empty]
[2026/02/13 08:52:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:52:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:52:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:10.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:53:25.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:53:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:54:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:54:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:10.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:55:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:55:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:56:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-7][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-8][Inner url for health probe: /actuator/health]
[2026/02/13 08:56:55.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-9][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-10][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:17.868][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-1][Inner url for health probe: /actuator/health/readiness]
[2026/02/13 08:57:25.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-2][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:40.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-3][Inner url for health probe: /actuator/health]
[2026/02/13 08:57:55.135][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-4][Inner url for health probe: /actuator/health]
[2026/02/13 08:58:10.136][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-5][Inner url for health probe: /actuator/health]
[2026/02/13 08:58:17.869][][INFO][com.....common.logupload.filter.UserContextFilter][doFilter,61][http-nio-8082-exec-6][Inner url for health probe: /actuator/health/readiness]