Skip to content

Commit f2605f6

Browse files
Chuck SilversChuck Silvers
Chuck Silvers
authored and
Chuck Silvers
committed
mpi3mr: configure larger max I/O size if the HBA firmware supports it
The max I/O size that an mpi3mr HBA supports is reported in the IOCFacts structure (with 0 representing the legacy max I/O size of 1 MB). By default, set the max I/O size of devices attached to mpi3mr controllers to the smaller of the HBA's max I/O size and the kernel's maxphys. Allow this default to be overriden by a global tunable "hw.mpi3mr.max_sgl_entries" or by a per-controller tunable "dev.mpi3mr.N.max_sgl_entries". Sponsored by: Netflix Reviewed by: imp Differential Revision: https://reviews.freebsd.org/D49090
1 parent 5e9af2b commit f2605f6

4 files changed

Lines changed: 41 additions & 10 deletions

File tree

‎sys/dev/mpi3mr/mpi3mr.c‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1617,6 +1617,7 @@ static int mpi3mr_process_factsdata(struct mpi3mr_softc *sc,
16171617
(facts_data->MaxPCIeSwitches);
16181618
sc->facts.max_sasexpanders =
16191619
(facts_data->MaxSASExpanders);
1620+
sc->facts.max_data_length = facts_data->MaxDataLength;
16201621
sc->facts.max_sasinitiators =
16211622
(facts_data->MaxSASInitiators);
16221623
sc->facts.max_enclosures = (facts_data->MaxEnclosures);
@@ -1651,6 +1652,10 @@ static int mpi3mr_process_factsdata(struct mpi3mr_softc *sc,
16511652
sc->facts.io_throttle_low = facts_data->IOThrottleLow;
16521653
sc->facts.io_throttle_high = facts_data->IOThrottleHigh;
16531654

1655+
if (sc->facts.max_data_length == MPI3_IOCFACTS_MAX_DATA_LENGTH_NOT_REPORTED)
1656+
sc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;
1657+
else
1658+
sc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;
16541659
/*Store in 512b block count*/
16551660
if (sc->facts.io_throttle_data_length)
16561661
sc->io_throttle_data_length =
@@ -2511,7 +2516,9 @@ static int mpi3mr_alloc_chain_bufs(struct mpi3mr_softc *sc)
25112516
goto out_failed;
25122517
}
25132518

2514-
sz = MPI3MR_CHAINSGE_SIZE;
2519+
if (sc->max_sgl_entries > sc->facts.max_data_length / PAGE_SIZE)
2520+
sc->max_sgl_entries = sc->facts.max_data_length / PAGE_SIZE;
2521+
sz = sc->max_sgl_entries * sizeof(Mpi3SGESimple_t);
25152522

25162523
if (bus_dma_tag_create(sc->mpi3mr_parent_dmat, /* parent */
25172524
4096, 0, /* algnmnt, boundary */
@@ -4961,7 +4968,7 @@ mpi3mr_alloc_requests(struct mpi3mr_softc *sc)
49614968
struct mpi3mr_cmd *cmd;
49624969
int i, j, nsegs, ret;
49634970

4964-
nsegs = MPI3MR_SG_DEPTH;
4971+
nsegs = sc->max_sgl_entries;
49654972
ret = bus_dma_tag_create( sc->mpi3mr_parent_dmat, /* parent */
49664973
1, 0, /* algnmnt, boundary */
49674974
sc->dma_loaddr, /* lowaddr */

‎sys/dev/mpi3mr/mpi3mr.h‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,11 @@
9595
#define MPI3MR_NAME_LENGTH 32
9696
#define IOCNAME "%s: "
9797

98+
#define MPI3MR_DEFAULT_MAX_IO_SIZE (1 * 1024 * 1024)
99+
98100
#define SAS4116_CHIP_REV_A0 0
99101
#define SAS4116_CHIP_REV_B0 1
100102

101-
#define MPI3MR_SG_DEPTH (MPI3MR_4K_PGSZ/sizeof(Mpi3SGESimple_t))
102103
#define MPI3MR_MAX_SECTORS 2048
103104
#define MPI3MR_MAX_CMDS_LUN 7
104105
#define MPI3MR_MAX_CDB_LENGTH 16
@@ -109,7 +110,12 @@
109110
#define MPI3MR_RAID_QDEPTH 128
110111
#define MPI3MR_NVME_QDEPTH 128
111112

113+
/* Definitions for internal SGL and Chain SGL buffers */
112114
#define MPI3MR_4K_PGSZ 4096
115+
#define MPI3MR_PAGE_SIZE_4K 4096
116+
#define MPI3MR_DEFAULT_SGL_ENTRIES 256
117+
#define MPI3MR_MAX_SGL_ENTRIES 2048
118+
113119
#define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ)
114120
#define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
115121
#define MPI3MR_AREQ_FRAME_SZ 128
@@ -125,8 +131,6 @@
125131

126132
#define MPI3MR_THRESHOLD_REPLY_COUNT 100
127133

128-
#define MPI3MR_CHAINSGE_SIZE MPI3MR_4K_PGSZ
129-
130134
#define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \
131135
(MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \
132136
MPI3_SGE_FLAGS_END_OF_LIST)
@@ -335,6 +339,7 @@ struct mpi3mr_ioc_facts
335339
U16 max_perids;
336340
U16 max_pds;
337341
U16 max_sasexpanders;
342+
U32 max_data_length;
338343
U16 max_sasinitiators;
339344
U16 max_enclosures;
340345
U16 max_pcieswitches;
@@ -671,6 +676,7 @@ struct mpi3mr_softc {
671676
struct mtx target_lock;
672677

673678
U16 max_host_ios;
679+
U32 max_sgl_entries;
674680
bus_dma_tag_t chain_sgl_list_tag;
675681
struct mpi3mr_chain *chain_sgl_list;
676682
U16 chain_bitmap_sz;

‎sys/dev/mpi3mr/mpi3mr_cam.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ static void mpi3mr_prepare_sgls(void *arg,
176176
bus_dmamap_sync(sc->buffer_dmat, cm->dmamap,
177177
BUS_DMASYNC_PREWRITE);
178178

179-
KASSERT(nsegs <= MPI3MR_SG_DEPTH && nsegs > 0,
179+
KASSERT(nsegs <= sc->max_sgl_entries && nsegs > 0,
180180
("%s: bad SGE count: %d\n", device_get_nameunit(sc->mpi3mr_dev), nsegs));
181181
KASSERT(scsiio_req->DataLength != 0,
182182
("%s: Data segments (%d), but DataLength == 0\n",
@@ -218,7 +218,7 @@ static void mpi3mr_prepare_sgls(void *arg,
218218

219219
chain = chain_req->buf;
220220
chain_dma = chain_req->buf_phys;
221-
memset(chain_req->buf, 0, PAGE_SIZE);
221+
memset(chain_req->buf, 0, sc->max_sgl_entries * sizeof(Mpi3SGESimple_t));
222222
sges_in_segment = sges_left;
223223
chain_length = sges_in_segment * sizeof(Mpi3SGESimple_t);
224224

@@ -1154,7 +1154,7 @@ mpi3mr_action_scsiio(struct mpi3mr_cam_softc *cam_sc, union ccb *ccb)
11541154
return;
11551155
case CAM_DATA_VADDR:
11561156
case CAM_DATA_BIO:
1157-
if (csio->dxfer_len > (MPI3MR_SG_DEPTH * MPI3MR_4K_PGSZ)) {
1157+
if (csio->dxfer_len > (sc->max_sgl_entries * PAGE_SIZE)) {
11581158
mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG);
11591159
mpi3mr_release_command(cm);
11601160
xpt_done(ccb);
@@ -1305,8 +1305,10 @@ mpi3mr_cam_action(struct cam_sim *sim, union ccb *ccb)
13051305
{
13061306
struct mpi3mr_cam_softc *cam_sc;
13071307
struct mpi3mr_target *targ;
1308+
struct mpi3mr_softc *sc;
13081309

13091310
cam_sc = cam_sim_softc(sim);
1311+
sc = cam_sc->sc;
13101312

13111313
mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "ccb func_code 0x%x target id: 0x%x\n",
13121314
ccb->ccb_h.func_code, ccb->ccb_h.target_id);
@@ -1357,7 +1359,7 @@ mpi3mr_cam_action(struct cam_sim *sim, union ccb *ccb)
13571359
"PCI device target_id: %u max io size: %u\n",
13581360
ccb->ccb_h.target_id, cpi->maxio);
13591361
} else {
1360-
cpi->maxio = PAGE_SIZE * (MPI3MR_SG_DEPTH - 1);
1362+
cpi->maxio = PAGE_SIZE * (sc->max_sgl_entries - 1);
13611363
}
13621364
mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP);
13631365
break;

‎sys/dev/mpi3mr/mpi3mr_pci.c‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,12 +178,15 @@ mpi3mr_get_tunables(struct mpi3mr_softc *sc)
178178
sc->reset_in_progress = 0;
179179
sc->reset.type = 0;
180180
sc->iot_enable = 1;
181+
sc->max_sgl_entries = maxphys / PAGE_SIZE;
182+
181183
/*
182184
* Grab the global variables.
183185
*/
184186
TUNABLE_INT_FETCH("hw.mpi3mr.debug_level", &sc->mpi3mr_debug);
185187
TUNABLE_INT_FETCH("hw.mpi3mr.ctrl_reset", &sc->reset.type);
186188
TUNABLE_INT_FETCH("hw.mpi3mr.iot_enable", &sc->iot_enable);
189+
TUNABLE_INT_FETCH("hw.mpi3mr.max_sgl_entries", &sc->max_sgl_entries);
187190

188191
/* Grab the unit-instance variables */
189192
snprintf(tmpstr, sizeof(tmpstr), "dev.mpi3mr.%d.debug_level",
@@ -197,6 +200,10 @@ mpi3mr_get_tunables(struct mpi3mr_softc *sc)
197200
snprintf(tmpstr, sizeof(tmpstr), "dev.mpi3mr.%d.iot_enable",
198201
device_get_unit(sc->mpi3mr_dev));
199202
TUNABLE_INT_FETCH(tmpstr, &sc->iot_enable);
203+
204+
snprintf(tmpstr, sizeof(tmpstr), "dev.mpi3mr.%d.max_sgl_entries",
205+
device_get_unit(sc->mpi3mr_dev));
206+
TUNABLE_INT_FETCH(tmpstr, &sc->max_sgl_entries);
200207
}
201208

202209
static struct mpi3mr_ident *
@@ -443,7 +450,16 @@ mpi3mr_pci_attach(device_t dev)
443450

444451
sc->mpi3mr_dev = dev;
445452
mpi3mr_get_tunables(sc);
446-
453+
454+
if (sc->max_sgl_entries > MPI3MR_MAX_SGL_ENTRIES)
455+
sc->max_sgl_entries = MPI3MR_MAX_SGL_ENTRIES;
456+
else if (sc->max_sgl_entries < MPI3MR_DEFAULT_SGL_ENTRIES)
457+
sc->max_sgl_entries = MPI3MR_DEFAULT_SGL_ENTRIES;
458+
else {
459+
sc->max_sgl_entries /= MPI3MR_DEFAULT_SGL_ENTRIES;
460+
sc->max_sgl_entries *= MPI3MR_DEFAULT_SGL_ENTRIES;
461+
}
462+
447463
if ((error = mpi3mr_initialize_ioc(sc, MPI3MR_INIT_TYPE_INIT)) != 0) {
448464
mpi3mr_dprint(sc, MPI3MR_ERROR, "FW initialization failed\n");
449465
goto load_failed;

0 commit comments

Comments
 (0)