Incorrect smartctl exit status

[fernandolcx]

Intentionally plugged some very bad SAS hard drives in a 24-bay enclosure.
Noticed some discrepancy regarding smartctl_device_smartctl_exit_status, which is an important indicator of drive failure:

# smartctl -a -T permissive /dev/sdc -d scsi; echo $?
4

smartctl -a -T permissive /dev/sdd -d scsi; echo $?
0

smartctl -a -T permissive /dev/sde -d scsi; echo $?
0

# smartctl -a -T permissive /dev/sdk -d scsi; echo $?
128

# smartctl -a -T permissive /dev/sdp -d scsi; echo $?
8

# smartctl -a -T permissive /dev/sdq -d scsi; echo $?
8

however:

smartctl_device_smartctl_exit_status{device="sdc"} 0
smartctl_device_smartctl_exit_status{device="sdd"} 0
smartctl_device_smartctl_exit_status{device="sde"} 0
smartctl_device_smartctl_exit_status{device="sdk"} 0
smartctl_device_smartctl_exit_status{device="sdp"} 8
smartctl_device_smartctl_exit_status{device="sdq"} 8

JSON data for/dev/sdc(drive defect: takes around 60s to respond)

{
  "json_format_version": [
    1,
    0
  ],
  "smartctl": {
    "version": [
      7,
      3
    ],
    "svn_revision": "5338",
    "platform_info": "x86_64-linux-6.8.12-4-pve",
    "build_info": "(local build)",
    "argv": [
      "smartctl",
      "-a",
      "-T",
      "permissive",
      "-d",
      "scsi",
      "/dev/sdc",
      "--json"
    ],
    "exit_status": 4
  },
  "local_time": {
    "time_t": 1737324399,
    "asctime": "Sun Jan 19 19:06:39 2025 -03"
  },
  "device": {
    "name": "/dev/sdc",
    "info_name": "/dev/sdc",
    "type": "scsi",
    "protocol": "SCSI"
  },
  "scsi_vendor": "SEAGATE",
  "scsi_product": "ST3600057SS",
  "scsi_model_name": "SEAGATE ST3600057SS",
  "scsi_revision": "EN03",
  "scsi_version": "SPC-3",
  "rotation_rate": 15000,
  "form_factor": {
    "scsi_value": 2,
    "name": "3.5 inches"
  },
  "logical_unit_id": "0x5000c50053c26987",
  "serial_number": "6SL458D5",
  "device_type": {
    "scsi_terminology": "Peripheral Device Type [PDT]",
    "scsi_value": 0,
    "name": "disk"
  },
  "scsi_transport_protocol": {
    "name": "SAS (SPL-4)",
    "value": 6
  },
  "smart_support": {
    "available": false
  },
  "temperature": {
    "current": 0
  },
  "scsi_temperature": {
    "drive_trip": 0
  }
}

JSON data for /dev/sdk (drive defect: several bad blocks, DST failing)

{
  "json_format_version": [
    1,
    0
  ],
  "smartctl": {
    "version": [
      7,
      3
    ],
    "svn_revision": "5338",
    "platform_info": "x86_64-linux-6.8.12-4-pve",
    "build_info": "(local build)",
    "argv": [
      "smartctl",
      "-a",
      "-T",
      "permissive",
      "-d",
      "scsi",
      "/dev/sdk",
      "--json"
    ],
    "exit_status": 128
  },
  "local_time": {
    "time_t": 1737324706,
    "asctime": "Sun Jan 19 19:11:46 2025 -03"
  },
  "device": {
    "name": "/dev/sdk",
    "info_name": "/dev/sdk",
    "type": "scsi",
    "protocol": "SCSI"
  },
  "scsi_vendor": "IBM-SSG",
  "scsi_product": "S7AQ3P0",
  "scsi_model_name": "IBM-SSG S7AQ3P0",
  "scsi_revision": "A058",
  "scsi_version": "SPC-4",
  "user_capacity": {
    "blocks": 5860533168,
    "bytes": 3000592982016
  },
  "logical_block_size": 512,
  "rotation_rate": 7200,
  "form_factor": {
    "scsi_value": 2,
    "name": "3.5 inches"
  },
  "logical_unit_id": "0x5000c500349f40f7",
  "serial_number": "Z290Q54400009145WCJ4",
  "device_type": {
    "scsi_terminology": "Peripheral Device Type [PDT]",
    "scsi_value": 0,
    "name": "disk"
  },
  "scsi_transport_protocol": {
    "name": "SAS (SPL-4)",
    "value": 6
  },
  "smart_support": {
    "available": true,
    "enabled": true
  },
  "temperature_warning": {
    "enabled": true
  },
  "smart_status": {
    "passed": true
  },
  "temperature": {
    "current": 39,
    "drive_trip": 65
  },
  "power_on_time": {
    "hours": 51871,
    "minutes": 57
  },
  "scsi_grown_defect_list": 1239,
  "scsi_error_counter_log": {
    "read": {
      "errors_corrected_by_eccfast": 2545895999,
      "errors_corrected_by_eccdelayed": 0,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 2545895999,
      "correction_algorithm_invocations": 1,
      "gigabytes_processed": "529664.798",
      "total_uncorrected_errors": 1
    },
    "write": {
      "errors_corrected_by_eccfast": 0,
      "errors_corrected_by_eccdelayed": 0,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 0,
      "correction_algorithm_invocations": 0,
      "gigabytes_processed": "104245.228",
      "total_uncorrected_errors": 0
    },
    "verify": {
      "errors_corrected_by_eccfast": 836386951,
      "errors_corrected_by_eccdelayed": 1,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 836386952,
      "correction_algorithm_invocations": 1,
      "gigabytes_processed": "4497341.609",
      "total_uncorrected_errors": 0
    }
  },
  "scsi_self_test_0": {
    "code": {
      "value": 2,
      "string": "Background long"
    },
    "result": {
      "value": 7,
      "string": "Failed in segment"
    },
    "power_on_time": {
      "hours": 51543,
      "aka": "accumulated_power_on_hours"
    },
    "lba_first_failure": {
      "value": 5165202415,
      "aka": "address_of_first_failure"
    },
    "sense_key": {
      "value": 3,
      "string": "Medium Error"
    },
    "asc": 17,
    "ascq": 0,
    "vendor_specific": 129
  },
  "scsi_self_test_1": {
    "code": {
      "value": 2,
      "string": "Background long"
    },
    "result": {
      "value": 7,
      "string": "Failed in segment"
    },
    "power_on_time": {
      "hours": 51288,
      "aka": "accumulated_power_on_hours"
    },
    "lba_first_failure": {
      "value": 5165202415,
      "aka": "address_of_first_failure"
    },
    "sense_key": {
      "value": 3,
      "string": "Medium Error"
    },
    "asc": 17,
    "ascq": 0,
    "vendor_specific": 129
  },
  "scsi_self_test_2": {
    "code": {
      "value": 2,
      "string": "Background long"
    },
    "result": {
      "value": 2,
      "string": "Aborted (device reset ?)"
    },
    "power_on_time": {
      "hours": 50516,
      "aka": "accumulated_power_on_hours"
    }
  },
  "scsi_extended_self_test_seconds": 27600
}

JSON data for /dev/sdp (drive defect: I don't know, available spare sectors depletion maybe?)

{
  "json_format_version": [
    1,
    0
  ],
  "smartctl": {
    "version": [
      7,
      3
    ],
    "svn_revision": "5338",
    "platform_info": "x86_64-linux-6.8.12-4-pve",
    "build_info": "(local build)",
    "argv": [
      "smartctl",
      "-a",
      "-T",
      "permissive",
      "-d",
      "scsi",
      "/dev/sdp",
      "--json"
    ],
    "exit_status": 8
  },
  "local_time": {
    "time_t": 1737324877,
    "asctime": "Sun Jan 19 19:14:37 2025 -03"
  },
  "device": {
    "name": "/dev/sdp",
    "info_name": "/dev/sdp",
    "type": "scsi",
    "protocol": "SCSI"
  },
  "scsi_vendor": "IBM-SSG",
  "scsi_product": "H0YH600",
  "scsi_model_name": "IBM-SSG H0YH600",
  "scsi_revision": "CF08",
  "scsi_version": "SPC-4",
  "user_capacity": {
    "blocks": 1172123568,
    "bytes": 600127266816
  },
  "logical_block_size": 512,
  "scsi_lb_provisioning": {
    "name": "fully provisioned",
    "value": 0,
    "management_enabled": {
      "name": "LBPME",
      "value": 0
    },
    "read_zeros": {
      "name": "LBPRZ",
      "value": 0
    }
  },
  "rotation_rate": 10000,
  "form_factor": {
    "scsi_value": 3,
    "name": "2.5 inches"
  },
  "logical_unit_id": "0x5000cca07d4e1980",
  "serial_number": "0BHBYHXH",
  "device_type": {
    "scsi_terminology": "Peripheral Device Type [PDT]",
    "scsi_value": 0,
    "name": "disk"
  },
  "scsi_transport_protocol": {
    "name": "SAS (SPL-4)",
    "value": 6
  },
  "smart_support": {
    "available": true,
    "enabled": true
  },
  "temperature_warning": {
    "enabled": true
  },
  "smart_status": {
    "passed": false,
    "scsi": {
      "asc": 93,
      "ascq": 100,
      "ie_string": "FIRMWARE IMPENDING FAILURE TOO MANY BLOCK REASSIGNS"
    }
  },
  "temperature": {
    "current": 32,
    "drive_trip": 65
  },
  "power_on_time": {
    "hours": 21975,
    "minutes": 22
  },
  "scsi_start_stop_cycle_counter": {
    "year_of_manufacture": "2018",
    "week_of_manufacture": "09",
    "specified_cycle_count_over_device_lifetime": 50000,
    "accumulated_start_stop_cycles": 88,
    "specified_load_unload_count_over_device_lifetime": 600000,
    "accumulated_load_unload_cycles": 998
  },
  "scsi_grown_defect_list": 0,
  "scsi_error_counter_log": {
    "read": {
      "errors_corrected_by_eccfast": 0,
      "errors_corrected_by_eccdelayed": 221387,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 221387,
      "correction_algorithm_invocations": 32499236,
      "gigabytes_processed": "5575.751",
      "total_uncorrected_errors": 40
    },
    "write": {
      "errors_corrected_by_eccfast": 0,
      "errors_corrected_by_eccdelayed": 0,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 0,
      "correction_algorithm_invocations": 87410,
      "gigabytes_processed": "19893.007",
      "total_uncorrected_errors": 0
    },
    "verify": {
      "errors_corrected_by_eccfast": 0,
      "errors_corrected_by_eccdelayed": 3795,
      "errors_corrected_by_rereads_rewrites": 0,
      "total_errors_corrected": 3795,
      "correction_algorithm_invocations": 389822872,
      "gigabytes_processed": "115152.801",
      "total_uncorrected_errors": 0
    }
  }
}

[k0ste]

It's not obvious what is smartctl_exporter version you try. Also better take a look to smartctl_exporter debug log, why your Z290Q54400009145WCJ4 don't take a metric

[fernandolcx]

smartctl_exporter --version:

smartctl_exporter, version 0.13.0 (branch: HEAD, revision: 2424bcba0dc3e2a326d3937304f4ff63efc36a2c)
  build user:       root@57b5cad14fb8
  build date:       20241220-06:37:30
  go version:       go1.23.4
  platform:         linux/amd64
  tags:             unknown

[fernandolcx]

why your Z290Q54400009145WCJ4 don't take a metric

Isn't /dev/sdc (6SL458D5)?
If yes, this drive is defective, it doesn't spin-up anymore, and fails to retrieve SMART data.

Causes a generalized slow-down in smartctl (almost 2min to respond), of course smartctl_exporter is also affected, but metrics are rendered in the end, differently to Zabbix Agent 2 which timeouts and doesn't even discover the drives - it's hardened to 30s maximum.

My intention is to put these monitoring tools to test, by using trash drives I'm validating it's reliability to know if they behaves good.

[k0ste]

My intention is to put these monitoring tools to test, by using trash drives I'm validating it's reliability to know if they behaves good.

As I suggested, you should look to smartctl_exporter log, to determine what exactly performed on your system. The example how this should look

time=2025-01-20T06:11:53.507Z level=INFO source=main.go:187 msg="Starting smartctl_exporter" version="(version=0.13.0, branch=HEAD, revision=2424bcba0dc3e2a326d3937304f4ff63efc36a2c)"
time=2025-01-20T06:11:53.507Z level=INFO source=main.go:188 msg="Build context" build_context="(go=go1.23.4, platform=linux/amd64, user=root@57b5cad14fb8, date=20241220-06:37:30, tags=unknown)"
time=2025-01-20T06:11:53.507Z level=INFO source=main.go:202 msg="Devices specified" devices=/dev/sdm
time=2025-01-20T06:11:53.507Z level=INFO source=main.go:204 msg="Devices filtered" count=1
time=2025-01-20T06:11:53.507Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --format=brief --log=error --device=auto /dev/sdm"
time=2025-01-20T06:11:53.538Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 64" device="/dev/sdm;auto (sdm)"
time=2025-01-20T06:11:53.538Z level=WARN source=readjson.go:151 msg="The device error log contains records of errors" device="/dev/sdm;auto (sdm)"
time=2025-01-20T06:11:53.538Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdm;auto (sdm)" duration=30.899579ms
time=2025-01-20T06:11:53.538Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdm family="Toshiba MG06ACA... Enterprise Capacity HDD" model="TOSHIBA MG06ACA10TE"

[fernandolcx]

./smartctl_exporter --smartctl.interval=3m --log.level=debug
time=2025-01-20T12:25:59.797Z level=INFO source=main.go:187 msg="Starting smartctl_exporter" version="(version=0.13.0, branch=HEAD, revision=2424bcba0dc3e2a326d3937304f4ff63efc36a2c)"
time=2025-01-20T12:25:59.797Z level=INFO source=main.go:188 msg="Build context" build_context="(go=go1.23.4, platform=linux/amd64, user=root@57b5cad14fb8, date=20241220-06:37:30, tags=unkn
time=2025-01-20T12:25:59.797Z level=DEBUG source=readjson.go:83 msg="Scanning for devices"
time=2025-01-20T12:25:59.824Z level=INFO source=main.go:145 msg="Found device" name=sda
time=2025-01-20T12:25:59.824Z level=INFO source=main.go:145 msg="Found device" name=sdc
time=2025-01-20T12:25:59.824Z level=INFO source=main.go:145 msg="Found device" name=sdd
time=2025-01-20T12:25:59.824Z level=INFO source=main.go:145 msg="Found device" name=sde
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdf
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdg
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdh
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdi
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdj
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdk
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdl
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdm
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdn
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdo
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdp
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdq
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdr
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sds
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdt
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdu
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdv
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdw
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdx
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=sdy
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=bus_0_megaraid_0
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:145 msg="Found device" name=bus_0_megaraid_1
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:198 msg="Number of devices found" count=26
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:213 msg="Start background scan process"
time=2025-01-20T12:25:59.825Z level=INFO source=main.go:214 msg="Rescanning for devices every" rescanInterval=10m0s
time=2025-01-20T12:25:59.825Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sda"
time=2025-01-20T12:25:59.846Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 2" device="/dev/sda;auto (sda)"
time=2025-01-20T12:25:59.846Z level=ERROR source=readjson.go:135 msg="Device open failed, device did not return an IDENTIFY DEVICE structure, or device is in a low-power mode" device="/devsda)"
time=2025-01-20T12:25:59.846Z level=ERROR source=readjson.go:167 msg="Smartctl open device: /dev/sda failed: DELL or MegaRaid controller, please try adding '-d megaraid,N'"
time=2025-01-20T12:25:59.846Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sda;auto (sda)" duration=20.318425ms
time=2025-01-20T12:25:59.846Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdc"
time=2025-01-20T12:25:59.869Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdc;auto (sdc)" duration=23.442494ms
time=2025-01-20T12:25:59.869Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdc family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:25:59.870Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdd"
time=2025-01-20T12:26:03.530Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdd;auto (sdd)" duration=3.660512961s
time=2025-01-20T12:26:03.530Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdd family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:03.531Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sde"
time=2025-01-20T12:26:07.159Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sde;auto (sde)" duration=3.628559541s
time=2025-01-20T12:26:07.160Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sde family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:07.160Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdf"
time=2025-01-20T12:26:07.265Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdf;auto (sdf)" duration=104.796333ms
time=2025-01-20T12:26:07.265Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdf family=unknown model="IBM-SSG H0YH600"
time=2025-01-20T12:26:07.265Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdg"
time=2025-01-20T12:26:08.833Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdg;auto (sdg)" duration=1.567199597s
time=2025-01-20T12:26:08.833Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdg family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:08.833Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdh"
time=2025-01-20T12:26:09.350Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdh;auto (sdh)" duration=517.198911ms
time=2025-01-20T12:26:09.350Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdh family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:09.351Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdi"
time=2025-01-20T12:26:10.707Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdi;auto (sdi)" duration=1.355858517s
time=2025-01-20T12:26:10.707Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdi family=unknown model="SEAGATE ST3450857SS"
time=2025-01-20T12:26:10.707Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdj"
time=2025-01-20T12:26:14.319Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdj;auto (sdj)" duration=3.611921869s
time=2025-01-20T12:26:14.319Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdj family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:14.320Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdk"
time=2025-01-20T12:26:18.226Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdk;auto (sdk)" duration=3.906183846s
time=2025-01-20T12:26:18.226Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdk family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:18.226Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdl"
time=2025-01-20T12:26:19.794Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdl;auto (sdl)" duration=1.568122421s
time=2025-01-20T12:26:19.795Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdl family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:19.795Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdm"
time=2025-01-20T12:26:20.323Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdm;auto (sdm)" duration=527.902176ms
time=2025-01-20T12:26:20.323Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdm family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:20.323Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdn"
time=2025-01-20T12:26:23.885Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdn;auto (sdn)" duration=3.561230792s
time=2025-01-20T12:26:23.885Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdn family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:23.885Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdo"
time=2025-01-20T12:26:27.391Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdo;auto (sdo)" duration=3.505625759s
time=2025-01-20T12:26:27.391Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdo family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:27.392Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdp"
time=2025-01-20T12:26:27.562Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 8" device="/dev/sdp;auto (sdp)"
time=2025-01-20T12:26:27.562Z level=WARN source=readjson.go:142 msg="SMART status check returned 'DISK FAILING'" device="/dev/sdp;auto (sdp)"
time=2025-01-20T12:26:27.562Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdp;auto (sdp)" duration=170.995973ms
time=2025-01-20T12:26:27.563Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdp family=unknown model="IBM-SSG H0YH600"
time=2025-01-20T12:26:27.563Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdq"
time=2025-01-20T12:26:31.114Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 8" device="/dev/sdq;auto (sdq)"
time=2025-01-20T12:26:31.114Z level=WARN source=readjson.go:142 msg="SMART status check returned 'DISK FAILING'" device="/dev/sdq;auto (sdq)"
time=2025-01-20T12:26:31.114Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdq;auto (sdq)" duration=3.551606206s
time=2025-01-20T12:26:31.115Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdq family=unknown model="SEAGATE ST32000444SS"
time=2025-01-20T12:26:31.115Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdr"
time=2025-01-20T12:26:31.594Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdr;auto (sdr)" duration=478.727026ms
time=2025-01-20T12:26:31.594Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdr family=unknown model="IBM-SSG H7AQ3P0"
time=2025-01-20T12:26:31.594Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sds"
time=2025-01-20T12:26:32.106Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sds;auto (sds)" duration=511.282608ms
time=2025-01-20T12:26:32.106Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sds family=unknown model="IBM-SSG H7AQ3P0"
time=2025-01-20T12:26:32.106Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdt"
time=2025-01-20T12:26:32.151Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdt;auto (sdt)" duration=44.775733ms
time=2025-01-20T12:26:32.151Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdt family=unknown model="IBM-SSG HSNF300"
time=2025-01-20T12:26:32.151Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdu"
time=2025-01-20T12:26:33.606Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdu;auto (sdu)" duration=1.454484338s
time=2025-01-20T12:26:33.606Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdu family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:33.606Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdv"
time=2025-01-20T12:26:33.818Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdv;auto (sdv)" duration=211.814354ms
time=2025-01-20T12:26:33.818Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdv family=unknown model="TOSHIBA MK2001TRKB"
time=2025-01-20T12:26:33.819Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdw"
time=2025-01-20T12:26:34.334Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdw;auto (sdw)" duration=515.755867ms
time=2025-01-20T12:26:34.335Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdw family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:34.335Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdx"
time=2025-01-20T12:26:35.790Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdx;auto (sdx)" duration=1.454544824s
time=2025-01-20T12:26:35.790Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdx family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:35.790Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sdy"
time=2025-01-20T12:26:35.832Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sdy;auto (sdy)" duration=42.287164ms
time=2025-01-20T12:26:35.833Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdy family=unknown model="SATA SSD"
time=2025-01-20T12:26:35.834Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=megaraid,0 /dev/bus/0"
time=2025-01-20T12:26:36.426Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/bus/0;megaraid,0 (bus_0_megaraid_0)" duration=592.126268ms
time=2025-01-20T12:26:36.427Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=bus_0_megaraid_0 family=unknown model="SEAGATE ST300MP0005"
time=2025-01-20T12:26:36.427Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=megaraid,1 /dev/bus/0"
time=2025-01-20T12:26:37.021Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/bus/0;megaraid,1 (bus_0_megaraid_1)" duration=594.567544ms
time=2025-01-20T12:26:37.022Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=bus_0_megaraid_1 family=unknown model="SEAGATE ST300MP0005"
time=2025-01-20T12:26:37.023Z level=INFO source=tls_config.go:347 msg="Listening on" address=[::]:9633
time=2025-01-20T12:26:37.023Z level=INFO source=tls_config.go:350 msg="TLS is disabled." http2=false address=[::]:9633
time=2025-01-20T12:26:47.442Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sda"
time=2025-01-20T12:26:47.464Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 2" device="/dev/sda;auto (sda)"
time=2025-01-20T12:26:47.464Z level=ERROR source=readjson.go:135 msg="Device open failed, device did not return an IDENTIFY DEVICE structure, or device is in a low-power mode" device="/devsda)"
time=2025-01-20T12:26:47.464Z level=ERROR source=readjson.go:167 msg="Smartctl open device: /dev/sda failed: DELL or MegaRaid controller, please try adding '-d megaraid,N'"
time=2025-01-20T12:26:47.464Z level=DEBUG source=readjson.go:78 msg="Collected S.M.A.R.T. json data" device="/dev/sda;auto (sda)" duration=21.872736ms
time=2025-01-20T12:26:47.464Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdc family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:47.464Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdd family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.464Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sde family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.465Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdf family=unknown model="IBM-SSG H0YH600"
time=2025-01-20T12:26:47.465Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdg family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:47.465Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdh family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:47.466Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdi family=unknown model="SEAGATE ST3450857SS"
time=2025-01-20T12:26:47.466Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdj family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.466Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdk family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.466Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdl family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:47.467Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdm family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:47.467Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdn family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.467Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdo family=unknown model="IBM-SSG S7AQ3P0"
time=2025-01-20T12:26:47.467Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdp family=unknown model="IBM-SSG H0YH600"
time=2025-01-20T12:26:47.468Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdq family=unknown model="SEAGATE ST32000444SS"
time=2025-01-20T12:26:47.468Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdr family=unknown model="IBM-SSG H7AQ3P0"
time=2025-01-20T12:26:47.468Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sds family=unknown model="IBM-SSG H7AQ3P0"
time=2025-01-20T12:26:47.468Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdt family=unknown model="IBM-SSG HSNF300"
time=2025-01-20T12:26:47.469Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdu family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:47.469Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdv family=unknown model="TOSHIBA MK2001TRKB"
time=2025-01-20T12:26:47.469Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdw family=unknown model="SEAGATE ST600MP0005"
time=2025-01-20T12:26:47.470Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdx family=unknown model="SEAGATE ST3600057SS"
time=2025-01-20T12:26:47.470Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=sdy family=unknown model="SATA SSD"
time=2025-01-20T12:26:47.471Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=bus_0_megaraid_0 family=unknown model="SEAGATE ST300MP0005"
time=2025-01-20T12:26:47.471Z level=DEBUG source=smartctl.go:87 msg="Collecting metrics from" device=bus_0_megaraid_1 family=unknown model="SEAGATE ST300MP0005"
time=2025-01-20T12:26:54.267Z level=DEBUG source=readjson.go:68 msg="Calling smartctl with args" args="--json --info --health --attributes --tolerance=verypermissive --nocheck=standby --fo--log=error --device=auto /dev/sda"

[k0ste]

As you can see, the smartctl_exporter added metric when value in field provided in JSON

time=2025-01-20T12:26:31.114Z level=WARN source=readjson.go:71 msg="S.M.A.R.T. output reading" err="exit status 8" device="/dev/sdq;auto (sdq)"

You can run to see it

[root@host:/]# smartctl --json --info --health --attributes --tolerance=verypermissive --nocheck=standby --format=brief --log=error --device=auto /dev/sdq | jq '.smartctl.exit_status'

[fernandolcx]

I think I found the real problem.

My drive /dev/sdk is failing Drive Self Test (DST), let's check the logs:

root@server-r630:~# smartctl --log selftest /dev/sdk
smartctl 7.3 2022-02-28 r5338 [x86_64-linux-6.8.12-4-pve] (local build)
Copyright (C) 2002-22, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART Self-test log
Num  Test              Status                 segment  LifeTime  LBA_first_err [SK ASC ASQ]
     Description                              number   (hours)
# 1  Background long   Failed in segment -->       -   51543        5165202415 [0x3 0x11 0x0]
# 2  Background long   Failed in segment -->       -   51288        5165202415 [0x3 0x11 0x0]
# 3  Background long   Aborted (device reset ?)    -   50516                 - [-   -    -]

Long (extended) Self-test duration: 27600 seconds [7.7 hours]

which returns:

root@server-r630:~# smartctl --log selftest /dev/sdk --json | jq '.smartctl.exit_status'
128

Using the same args the exporter uses:

root@server-r630:~# smartctl --json --info --health --attributes --tolerance=verypermissive --nocheck=standby --format=brief --log=error --device=auto /dev/sdk | jq '.smartctl.exit_status'
0

[fernandolcx]

Another example with /dev/sdc which doesn't spins anymore:

root@server-r630:~/smartctl_exporter-0.13.0.linux-amd64# smartctl --info --health --attributes --tolerance=verypermissive --nocheck=standby --format=brief --log=error --device=auto /dev/sdc
smartctl 7.3 2022-02-28 r5338 [x86_64-linux-6.8.12-4-pve] (local build)
Copyright (C) 2002-22, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===
Vendor:               SEAGATE
Product:              ST3600057SS
Revision:             EN03
Compliance:           SPC-3
Rotation Rate:        15000 rpm
Form Factor:          3.5 inches
Logical Unit id:      0x5000c50053c26987
Serial number:        6SL458D5
Device type:          disk
Transport protocol:   SAS (SPL-4)
Local Time is:        Tue Jan 21 19:34:17 2025 -03
device is NOT READY (e.g. spun down, busy)
SMART support is:     Unavailable - device lacks SMART capability.
Power mode is:        ACTIVE

=== START OF READ SMART DATA SECTION ===
Current Drive Temperature:     0 C
Drive Trip Temperature:        0 C

Error Counter logging not supported

# smartctl --json --info --health --attributes --tolerance=verypermissive --nocheck=standby --format=brief --log=error --device=auto /dev/sdc | jq '.smartctl.exit_status'
0

# smartctl --all /dev/sdc --json | jq '.smartctl.exit_status'
2

# smartctl -a /dev/sdc --json --tolerance=verypermissive | jq '.smartctl.exit_status'
4

[k0ste]

I think I found the real problem.

My drive /dev/sdk is failing Drive Self Test (DST), let's check the logs

This is a known issue #92