From 9e73bd5785c648cff8510d275bc37dcd51aa8e5b Mon Sep 17 00:00:00 2001 From: ymodlin Date: Thu, 19 Feb 2026 20:57:58 +0200 Subject: [PATCH 1/4] [FIX] Improve D4XX streaming resilience and I2C failure handling Address failures observed during 13+ hour streaming stress tests where repeated start/stop cycles eventually cause catastrophic I2C bus failure (errno -121) and unrecoverable camera state requiring power cycle. Key changes: - Add I2C circuit breaker: track consecutive errors via atomic counter and declare camera dead after 50 failures, short-circuiting all further I2C operations with -ENODEV to prevent bus thrashing - Rate-limit I2C error messages: demote per-retry warnings to dev_dbg, use dev_warn_ratelimited for final failures to prevent kernel log flooding (3,430+ identical messages observed in stress test logs) - Fix DFU release: skip I2C verification when camera is dead, reduce retries from 10 to 3, always return 0 to prevent file descriptor leaks - Verify GMSL link after HW reset recovery: replace blind 300ms sleep with active I2C polling (up to 1.2s) to confirm link is alive before declaring reset complete - Reset all sensor streaming state in HW reset recovery path to ensure ds5_configure() performs fresh setup on next stream-on instead of using stale cached state Co-Authored-By: Claude Opus 4.6 --- kernel/realsense/d4xx.c | 160 +++++++++++++++++++++++++++++++--------- 1 file changed, 126 insertions(+), 34 deletions(-) diff --git a/kernel/realsense/d4xx.c b/kernel/realsense/d4xx.c index a4d30fce..154669bd 100644 --- a/kernel/realsense/d4xx.c +++ b/kernel/realsense/d4xx.c @@ -194,6 +194,7 @@ enum ds5_mux_pad { /* I2C retry configuration */ #define DS5_I2C_RETRY_COUNT 5 #define DS5_I2C_RETRY_DELAY_US 5000 +#define DS5_I2C_DEAD_THRESHOLD 50 /* consecutive errors before declaring dead */ /* DFU definition section */ #define DFU_MAGIC_NUMBER "/0x01/0x02/0x03/0x04" @@ -481,6 +482,9 @@ struct ds5 { struct i2c_client *dser_i2c; const struct dser_interface *dser_ops; #endif + /* I2C health tracking */ + atomic_t i2c_consec_errors; + bool camera_dead; }; struct ds5_counters { @@ -545,6 +549,9 @@ static int ds5_write(struct ds5 *state, u16 reg, u16 val) int retry; u8 value[2]; + if (state->camera_dead) + return -ENODEV; + value[1] = val >> 8; value[0] = val & 0x00FF; @@ -557,20 +564,31 @@ static int ds5_write(struct ds5 *state, u16 reg, u16 val) if (ret == 0) break; if (retry < DS5_I2C_RETRY_COUNT - 1) { - dev_warn(&state->client->dev, + dev_dbg(&state->client->dev, "%s(): i2c write retry %d, 0x%04x = 0x%x, err %d\n", __func__, retry + 1, reg, val, ret); usleep_range(DS5_I2C_RETRY_DELAY_US, DS5_I2C_RETRY_DELAY_US + 500); } } - if (ret < 0) - dev_err(&state->client->dev, + if (ret < 0) { + dev_warn_ratelimited(&state->client->dev, "%s(): i2c write failed after %d retries, 0x%04x = 0x%x, err %d\n", __func__, DS5_I2C_RETRY_COUNT, reg, val, ret); - else if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) - dev_dbg(&state->client->dev, "%s(): i2c write 0x%04x: 0x%x\n", - __func__, reg, val); + if (atomic_inc_return(&state->i2c_consec_errors) >= + DS5_I2C_DEAD_THRESHOLD) { + state->camera_dead = true; + dev_err(&state->client->dev, + "camera unreachable after %d consecutive I2C errors\n", + DS5_I2C_DEAD_THRESHOLD); + } + } else { + atomic_set(&state->i2c_consec_errors, 0); + if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) + dev_dbg(&state->client->dev, + "%s(): i2c write 0x%04x: 0x%x\n", + __func__, reg, val); + } return ret; } @@ -581,26 +599,39 @@ static int ds5_raw_write(struct ds5 *state, u16 reg, int ret; int retry; + if (state->camera_dead) + return -ENODEV; + for (retry = 0; retry < DS5_I2C_RETRY_COUNT; retry++) { ret = regmap_raw_write(state->regmap, reg, val, val_len); if (ret == 0) break; if (retry < DS5_I2C_RETRY_COUNT - 1) { - dev_warn(&state->client->dev, + dev_dbg(&state->client->dev, "%s(): i2c raw write retry %d, 0x%04x size(%d), err %d\n", __func__, retry + 1, reg, (int)val_len, ret); usleep_range(DS5_I2C_RETRY_DELAY_US, DS5_I2C_RETRY_DELAY_US + 500); } } - if (ret < 0) - dev_err(&state->client->dev, + if (ret < 0) { + dev_warn_ratelimited(&state->client->dev, "%s(): i2c raw write failed after %d retries, 0x%04x size(%d), err %d\n", __func__, DS5_I2C_RETRY_COUNT, reg, (int)val_len, ret); - else if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) - dev_dbg(&state->client->dev, - "%s(): i2c raw write 0x%04x: %d bytes\n", - __func__, reg, (int)val_len); + if (atomic_inc_return(&state->i2c_consec_errors) >= + DS5_I2C_DEAD_THRESHOLD) { + state->camera_dead = true; + dev_err(&state->client->dev, + "camera unreachable after %d consecutive I2C errors\n", + DS5_I2C_DEAD_THRESHOLD); + } + } else { + atomic_set(&state->i2c_consec_errors, 0); + if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) + dev_dbg(&state->client->dev, + "%s(): i2c raw write 0x%04x: %d bytes\n", + __func__, reg, (int)val_len); + } return ret; } @@ -610,25 +641,39 @@ static int ds5_read(struct ds5 *state, u16 reg, u16 *val) int ret; int retry; + if (state->camera_dead) + return -ENODEV; + for (retry = 0; retry < DS5_I2C_RETRY_COUNT; retry++) { ret = regmap_raw_read(state->regmap, reg, val, 2); if (ret == 0) break; if (retry < DS5_I2C_RETRY_COUNT - 1) { - dev_warn(&state->client->dev, + dev_dbg(&state->client->dev, "%s(): i2c read retry %d, 0x%04x, err %d\n", __func__, retry + 1, reg, ret); usleep_range(DS5_I2C_RETRY_DELAY_US, DS5_I2C_RETRY_DELAY_US + 500); } } - if (ret < 0) - dev_err(&state->client->dev, + if (ret < 0) { + dev_warn_ratelimited(&state->client->dev, "%s(): i2c read failed after %d retries, 0x%04x, err %d\n", __func__, DS5_I2C_RETRY_COUNT, reg, ret); - else if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) - dev_dbg(&state->client->dev, "%s(): i2c read 0x%04x: 0x%x\n", - __func__, reg, *val); + if (atomic_inc_return(&state->i2c_consec_errors) >= + DS5_I2C_DEAD_THRESHOLD) { + state->camera_dead = true; + dev_err(&state->client->dev, + "camera unreachable after %d consecutive I2C errors\n", + DS5_I2C_DEAD_THRESHOLD); + } + } else { + atomic_set(&state->i2c_consec_errors, 0); + if (state->dfu_dev.dfu_state_flag == DS5_DFU_IDLE) + dev_dbg(&state->client->dev, + "%s(): i2c read 0x%04x: 0x%x\n", + __func__, reg, *val); + } return ret; } @@ -638,22 +683,35 @@ static int ds5_raw_read(struct ds5 *state, u16 reg, void *val, size_t val_len) int ret; int retry; + if (state->camera_dead) + return -ENODEV; + for (retry = 0; retry < DS5_I2C_RETRY_COUNT; retry++) { ret = regmap_raw_read(state->regmap, reg, val, val_len); if (ret == 0) break; if (retry < DS5_I2C_RETRY_COUNT - 1) { - dev_warn(&state->client->dev, + dev_dbg(&state->client->dev, "%s(): i2c raw read retry %d, 0x%04x size(%d), err %d\n", __func__, retry + 1, reg, (int)val_len, ret); usleep_range(DS5_I2C_RETRY_DELAY_US, DS5_I2C_RETRY_DELAY_US + 500); } } - if (ret < 0) - dev_err(&state->client->dev, + if (ret < 0) { + dev_warn_ratelimited(&state->client->dev, "%s(): i2c raw read failed after %d retries, 0x%04x size(%d), err %d\n", __func__, DS5_I2C_RETRY_COUNT, reg, (int)val_len, ret); + if (atomic_inc_return(&state->i2c_consec_errors) >= + DS5_I2C_DEAD_THRESHOLD) { + state->camera_dead = true; + dev_err(&state->client->dev, + "camera unreachable after %d consecutive I2C errors\n", + DS5_I2C_DEAD_THRESHOLD); + } + } else { + atomic_set(&state->i2c_consec_errors, 0); + } return ret; } @@ -2384,12 +2442,43 @@ static int ds5_hw_reset_with_recovery(struct ds5 *state) sensor->pipe_configured = false; sensor->pipe_id = -1; } + + /* Ensure all sensor streaming state is fully reset so + * ds5_configure() does a fresh setup on next stream-on */ + for (i = 0; i < ARRAY_SIZE(sensors); i++) + sensors[i]->streaming = false; + mutex_unlock(&serdes_lock__); dev_info(&state->client->dev, "%s(): Re-initializing SERDES link\n", __func__); state->dser_ops->reset_oneshot(state->dser_dev); - msleep(300); + + /* + * Verify GMSL link is re-established by polling I2C. + * reset_oneshot itself sleeps 100ms internally; we add + * settling time and verify the link is actually alive + * rather than blindly sleeping a fixed 300ms. + */ + msleep(200); + for (i = 0; i < 10; i++) { + ret = ds5_read(state, DS5_FW_VERSION, &status); + if (ret == 0) + break; + dev_dbg(&state->client->dev, + "%s(): GMSL link not ready, retry %d\n", + __func__, i); + msleep(100); + } + if (ret < 0) { + dev_err(&state->client->dev, + "%s(): GMSL link failed to recover after reset_oneshot (%d)\n", + __func__, ret); + return -EIO; + } + dev_info(&state->client->dev, + "%s(): GMSL link verified after %d ms\n", + __func__, 200 + i * 100); } #endif @@ -5560,20 +5649,21 @@ static int ds5_dfu_device_release(struct inode *inode, struct file *file) struct i2c_adapter *parent = i2c_parent_is_i2c_adapter( state->client->adapter); #endif - int ret = 0, retry = 10; + int ret = 0, retry = 3; mutex_lock(&state->lock); state->dfu_dev.device_open_count--; if (state->dfu_dev.dfu_state_flag != DS5_DFU_RECOVERY) state->dfu_dev.dfu_state_flag = DS5_DFU_IDLE; - /* We disable this section as it has no effect when device in operational - mode and has not enough effect when device in recovery mode */ - // if (state->dfu_dev.dfu_state_flag == DS5_DFU_DONE - // && state->dfu_dev.init_v4l_f) - // ds5_v4l_init(state->client, state); - // state->dfu_dev.init_v4l_f = 0; if (state->dfu_dev.dfu_msg) devm_kfree(&state->client->dev, state->dfu_dev.dfu_msg); state->dfu_dev.dfu_msg = NULL; + + /* Skip I2C verification if camera is unreachable */ + if (state->camera_dead) { + mutex_unlock(&state->lock); + return 0; + } + #ifdef CONFIG_TEGRA_CAMERA_PLATFORM /* get i2c controller and restore bus clock rate */ while (parent && i2c_parent_is_i2c_adapter(parent)) @@ -5594,16 +5684,16 @@ static int ds5_dfu_device_release(struct inode *inode, struct file *file) ret = ds5_read(state, DS5_FW_VERSION, &state->fw_version); if (ret) msleep_range(10); - } while (retry-- && ret != 0 ); + } while (retry-- && ret != 0); if (ret) { - dev_warn(&state->client->dev, + dev_warn_ratelimited(&state->client->dev, "%s(): no communication with d4xx\n", __func__); mutex_unlock(&state->lock); - return ret; + return 0; /* release must succeed to avoid FD leak */ } ret = ds5_read(state, DS5_FW_BUILD, &state->fw_build); mutex_unlock(&state->lock); - return ret; + return 0; }; static const struct file_operations ds5_device_file_ops = { @@ -5829,6 +5919,8 @@ static int ds5_probe(struct i2c_client *c, const struct i2c_device_id *id) return -ENOMEM; mutex_init(&state->lock); + atomic_set(&state->i2c_consec_errors, 0); + state->camera_dead = false; state->client = c; dev_warn(&c->dev, "Probing driver for D4xx\n"); From 47995ff4094aef60edcff8ae6ada3391c72280ca Mon Sep 17 00:00:00 2001 From: Yogev Modlin <117653169+ymodlin@users.noreply.github.com> Date: Thu, 19 Feb 2026 21:00:26 +0200 Subject: [PATCH 2/4] Bump version to 1.0.2.11 --- kernel/realsense/d4xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/realsense/d4xx.c b/kernel/realsense/d4xx.c index 154669bd..c5af3a4c 100644 --- a/kernel/realsense/d4xx.c +++ b/kernel/realsense/d4xx.c @@ -6165,4 +6165,4 @@ MODULE_AUTHOR("Guennadi Liakhovetski ,\n\ Shikun Ding ,\n\ Dmitry Perchanov "); MODULE_LICENSE("GPL v2"); -MODULE_VERSION("1.0.2.10"); +MODULE_VERSION("1.0.2.11"); From b17850a7c4e2a4c1d08d43dfc0bcc7c92da04a46 Mon Sep 17 00:00:00 2001 From: ymodlin Date: Thu, 19 Feb 2026 22:23:19 +0200 Subject: [PATCH 3/4] [FIX] Reset sensor pipe configuration on release --- kernel/realsense/d4xx.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/realsense/d4xx.c b/kernel/realsense/d4xx.c index c5af3a4c..86c9064a 100644 --- a/kernel/realsense/d4xx.c +++ b/kernel/realsense/d4xx.c @@ -4770,6 +4770,7 @@ static int ds5_mux_s_stream(struct v4l2_subdev *sd, int on) if (state->dser_ops->release_pipe(state->dser_dev, sensor->pipe_id) < 0) dev_warn(&state->client->dev, "release pipe failed\n"); sensor->pipe_id = -1; + sensor->pipe_configured = false; mutex_unlock(&serdes_lock__); } #endif From b99153812e5516a82dd05f677b587baa95ae89b0 Mon Sep 17 00:00:00 2001 From: Yogev Modlin <117653169+ymodlin@users.noreply.github.com> Date: Thu, 19 Feb 2026 22:27:34 +0200 Subject: [PATCH 4/4] Bump version to 1.0.2.12 --- kernel/realsense/d4xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/realsense/d4xx.c b/kernel/realsense/d4xx.c index 86c9064a..aa618a94 100644 --- a/kernel/realsense/d4xx.c +++ b/kernel/realsense/d4xx.c @@ -6166,4 +6166,4 @@ MODULE_AUTHOR("Guennadi Liakhovetski ,\n\ Shikun Ding ,\n\ Dmitry Perchanov "); MODULE_LICENSE("GPL v2"); -MODULE_VERSION("1.0.2.11"); +MODULE_VERSION("1.0.2.12");