diff options
| author | scottl <scottl@FreeBSD.org> | 2014-01-07 01:51:48 +0000 |
|---|---|---|
| committer | scottl <scottl@FreeBSD.org> | 2014-01-07 01:51:48 +0000 |
| commit | cd4455d63815797dee4bd5d2fbe903ace2888a8f (patch) | |
| tree | e4a932aed55b2e611c4a7cbe18560be7eb0cd1f1 /sys/cam | |
| parent | 0a34594b9cd7c8b87f719ed058da6be2b756a8e5 (diff) | |
| download | FreeBSD-src-cd4455d63815797dee4bd5d2fbe903ace2888a8f.zip FreeBSD-src-cd4455d63815797dee4bd5d2fbe903ace2888a8f.tar.gz | |
MFC Alexander Motin's direct dispatch, multi-queue, and finer-grained
locking support for CAM
r256826:
Fix several target mode SIMs to not blindly clear ccb_h.flags field of
ATIO CCBs. Not all CCB flags there belong to them.
r256836:
Remove hard limit on number of BIOs handled with one ATA TRIM request.
r256843:
Merge CAM locking changes from the projects/camlock branch to radically
reduce lock congestion and improve SMP scalability of the SCSI/ATA stack,
preparing the ground for the coming next GEOM direct dispatch support.
r256888:
Unconditionally acquire periph reference on CCB allocation failure.
r256895:
Fix memory and references leak due to unfreed path.
r256960:
Move CAM_UNQUEUED_INDEX setting to the last moment and under the periph lock.
This fixes race condition with cam_periph_ccbwait(), causing use-after-free.
r256975:
Minor (mostly cosmetical) addition to r256960.
r257054:
Some microoptimizations for da and ada drivers:
- Replace ordered_tag_count counter with single flag;
- From da remove outstanding_cmds counter, duplicating pending_ccbs list;
- From da_softc remove unused links field.
r257482:
Fix lock recursion, triggered by `smartctl -a /dev/adaX`.
r257501:
Make getenv_*() functions and respectively TUNABLE_*_FETCH() macros not
allocate memory and so not require sleepable environment. getenv() has
already used on-stack temporary storage, so just use it more rationally.
getenv_string() receives buffer as argument, so don't need another one.
r257914:
Some CAM locks polishing:
- Fix LOR and possible lock recursion when handling high-power commands.
Introduce new lock to protect left power quota and list of frozen devices.
- Correct locking around xpt periph creation.
- Remove seems never used XPT_FLAG_OPEN xpt periph flag.
Again, Netflix assisted with testing the merge, but all of the credit goes
to Alexander and iX Systems.
Submitted by: mav
Sponsored by: iX Systems
Diffstat (limited to 'sys/cam')
30 files changed, 1517 insertions, 1598 deletions
diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index e476a3e..989f0aa 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -80,7 +80,7 @@ typedef enum { ADA_FLAG_CAN_NCQ = 0x0008, ADA_FLAG_CAN_DMA = 0x0010, ADA_FLAG_NEED_OTAG = 0x0020, - ADA_FLAG_WENT_IDLE = 0x0040, + ADA_FLAG_WAS_OTAG = 0x0040, ADA_FLAG_CAN_TRIM = 0x0080, ADA_FLAG_OPEN = 0x0100, ADA_FLAG_SCTX_INIT = 0x0200, @@ -103,7 +103,6 @@ typedef enum { ADA_CCB_RAHEAD = 0x01, ADA_CCB_WCACHE = 0x02, ADA_CCB_BUFFER_IO = 0x03, - ADA_CCB_WAITING = 0x04, ADA_CCB_DUMP = 0x05, ADA_CCB_TRIM = 0x06, ADA_CCB_TYPE_MASK = 0x0F, @@ -123,21 +122,20 @@ struct disk_params { #define TRIM_MAX_BLOCKS 8 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES) -#define TRIM_MAX_BIOS (TRIM_MAX_RANGES * 4) struct trim_request { uint8_t data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE]; - struct bio *bps[TRIM_MAX_BIOS]; + TAILQ_HEAD(, bio) bps; }; struct ada_softc { struct bio_queue_head bio_queue; struct bio_queue_head trim_queue; + int outstanding_cmds; /* Number of active commands */ + int refcount; /* Active xpt_action() calls */ ada_state state; - ada_flags flags; + ada_flags flags; ada_quirks quirks; int sort_io_queue; - int ordered_tag_count; - int outstanding_cmds; int trim_max_ranges; int trim_running; int read_ahead; @@ -630,14 +628,8 @@ adaclose(struct disk *dp) int error; periph = (struct cam_periph *)dp->d_drv1; - cam_periph_lock(periph); - if (cam_periph_hold(periph, PRIBIO) != 0) { - cam_periph_unlock(periph); - cam_periph_release(periph); - return (0); - } - softc = (struct ada_softc *)periph->softc; + cam_periph_lock(periph); CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, ("adaclose\n")); @@ -645,7 +637,8 @@ adaclose(struct disk *dp) /* We only sync the cache if the drive is capable of it. */ if ((softc->flags & ADA_FLAG_DIRTY) != 0 && (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 && - (periph->flags & CAM_PERIPH_INVALID) == 0) { + (periph->flags & CAM_PERIPH_INVALID) == 0 && + cam_periph_hold(periph, PRIBIO) == 0) { ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); cam_fill_ataio(&ccb->ataio, @@ -669,10 +662,13 @@ adaclose(struct disk *dp) else softc->flags &= ~ADA_FLAG_DIRTY; xpt_release_ccb(ccb); + cam_periph_unhold(periph); } softc->flags &= ~ADA_FLAG_OPEN; - cam_periph_unhold(periph); + + while (softc->refcount != 0) + cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1); cam_periph_unlock(periph); cam_periph_release(periph); return (0); @@ -682,23 +678,15 @@ static void adaschedule(struct cam_periph *periph) { struct ada_softc *softc = (struct ada_softc *)periph->softc; - uint32_t prio; if (softc->state != ADA_STATE_NORMAL) return; - /* Check if cam_periph_getccb() was called. */ - prio = periph->immediate_priority; - /* Check if we have more work to do. */ if (bioq_first(&softc->bio_queue) || (!softc->trim_running && bioq_first(&softc->trim_queue))) { - prio = CAM_PRIORITY_NORMAL; + xpt_schedule(periph, CAM_PRIORITY_NORMAL); } - - /* Schedule CCB if any of above is true. */ - if (prio != CAM_PRIORITY_NONE) - xpt_schedule(periph, prio); } /* @@ -962,7 +950,7 @@ adaasync(void *callback_arg, u_int32_t code, status = cam_periph_alloc(adaregister, adaoninvalidate, adacleanup, adastart, "ada", CAM_PERIPH_BIO, - cgd->ccb_h.path, adaasync, + path, adaasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -1038,8 +1026,10 @@ adaasync(void *callback_arg, u_int32_t code, softc->state = ADA_STATE_WCACHE; else break; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); + if (cam_periph_acquire(periph) != CAM_REQ_CMP) + softc->state = ADA_STATE_NORMAL; + else + xpt_schedule(periph, CAM_PRIORITY_DEV); } default: cam_periph_async(periph, code, path, arg); @@ -1346,8 +1336,8 @@ adaregister(struct cam_periph *periph, void *arg) * Create our sysctl variables, now that we know * we have successfully attached. */ - cam_periph_acquire(periph); - taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); + if (cam_periph_acquire(periph) == CAM_REQ_CMP) + taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); /* * Add async callbacks for bus reset and @@ -1365,7 +1355,7 @@ adaregister(struct cam_periph *periph, void *arg) * Schedule a periodic event to occasionally send an * ordered tag to a device. */ - callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0); + callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0); callout_reset(&softc->sendordered_c, (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL, adasendorderedtag, softc); @@ -1373,16 +1363,17 @@ adaregister(struct cam_periph *periph, void *arg) if (ADA_RA >= 0 && cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) { softc->state = ADA_STATE_RAHEAD; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); } else if (ADA_WC >= 0 && cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) { softc->state = ADA_STATE_WCACHE; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); - } else + } else { softc->state = ADA_STATE_NORMAL; - + return(CAM_REQ_CMP); + } + if (cam_periph_acquire(periph) != CAM_REQ_CMP) + softc->state = ADA_STATE_NORMAL; + else + xpt_schedule(periph, CAM_PRIORITY_DEV); return(CAM_REQ_CMP); } @@ -1400,29 +1391,17 @@ adastart(struct cam_periph *periph, union ccb *start_ccb) struct bio *bp; u_int8_t tag_code; - /* Execute immediate CCB if waiting. */ - if (periph->immediate_priority <= periph->pinfo.priority) { - CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, - ("queuing for immediate ccb\n")); - start_ccb->ccb_h.ccb_state = ADA_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - /* Have more work to do, so ensure we stay scheduled */ - adaschedule(periph); - break; - } /* Run TRIM if not running yet. */ if (!softc->trim_running && (bp = bioq_first(&softc->trim_queue)) != 0) { struct trim_request *req = &softc->trim_req; struct bio *bp1; uint64_t lastlba = (uint64_t)-1; - int bps = 0, c, lastcount = 0, off, ranges = 0; + int c, lastcount = 0, off, ranges = 0; softc->trim_running = 1; bzero(req, sizeof(*req)); + TAILQ_INIT(&req->bps); bp1 = bp; do { uint64_t lba = bp1->bio_pblkno; @@ -1465,10 +1444,9 @@ adastart(struct cam_periph *periph, union ccb *start_ccb) */ } lastlba = lba; - req->bps[bps++] = bp1; + TAILQ_INSERT_TAIL(&req->bps, bp1, bio_queue); bp1 = bioq_first(&softc->trim_queue); - if (bps >= TRIM_MAX_BIOS || - bp1 == NULL || + if (bp1 == NULL || bp1->bio_bcount / softc->params.secsize > (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) @@ -1487,6 +1465,7 @@ adastart(struct cam_periph *periph, union ccb *start_ccb) ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES - 1) / ATA_DSM_BLK_RANGES); start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; goto out; } /* Run regular command. */ @@ -1500,7 +1479,7 @@ adastart(struct cam_periph *periph, union ccb *start_ccb) if ((bp->bio_flags & BIO_ORDERED) != 0 || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) { softc->flags &= ~ADA_FLAG_NEED_OTAG; - softc->ordered_tag_count++; + softc->flags |= ADA_FLAG_WAS_OTAG; tag_code = 0; } else { tag_code = 1; @@ -1655,10 +1634,15 @@ adastart(struct cam_periph *periph, union ccb *start_ccb) break; } start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; out: start_ccb->ccb_h.ccb_bp = bp; softc->outstanding_cmds++; + softc->refcount++; + cam_periph_unlock(periph); xpt_action(start_ccb); + cam_periph_lock(periph); + softc->refcount--; /* May have more work to do, so ensure we stay scheduled */ adaschedule(periph); @@ -1667,13 +1651,6 @@ out: case ADA_STATE_RAHEAD: case ADA_STATE_WCACHE: { - if ((periph->flags & CAM_PERIPH_INVALID) != 0) { - softc->state = ADA_STATE_NORMAL; - xpt_release_ccb(start_ccb); - cam_periph_release_locked(periph); - return; - } - cam_fill_ataio(ataio, 1, adadone, @@ -1722,10 +1699,12 @@ adadone(struct cam_periph *periph, union ccb *done_ccb) struct bio *bp; int error; + cam_periph_lock(periph); if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { error = adaerror(done_ccb, 0, 0); if (error == ERESTART) { /* A retry was scheduled, so just return. */ + cam_periph_unlock(periph); return; } if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) @@ -1754,29 +1733,32 @@ adadone(struct cam_periph *periph, union ccb *done_ccb) } softc->outstanding_cmds--; if (softc->outstanding_cmds == 0) - softc->flags |= ADA_FLAG_WENT_IDLE; + softc->flags |= ADA_FLAG_WAS_OTAG; + xpt_release_ccb(done_ccb); if (state == ADA_CCB_TRIM) { - struct trim_request *req = - (struct trim_request *)ataio->data_ptr; - int i; - - for (i = 1; i < TRIM_MAX_BIOS && req->bps[i]; i++) { - struct bio *bp1 = req->bps[i]; + TAILQ_HEAD(, bio) queue; + struct bio *bp1; - bp1->bio_error = bp->bio_error; - if (bp->bio_flags & BIO_ERROR) { + TAILQ_INIT(&queue); + TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue); + softc->trim_running = 0; + adaschedule(periph); + cam_periph_unlock(periph); + while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { + TAILQ_REMOVE(&queue, bp1, bio_queue); + bp1->bio_error = error; + if (error != 0) { bp1->bio_flags |= BIO_ERROR; bp1->bio_resid = bp1->bio_bcount; } else bp1->bio_resid = 0; biodone(bp1); } - softc->trim_running = 0; - biodone(bp); - adaschedule(periph); - } else + } else { + cam_periph_unlock(periph); biodone(bp); - break; + } + return; } case ADA_CCB_RAHEAD: { @@ -1852,12 +1834,6 @@ out: cam_periph_release_locked(periph); return; } - case ADA_CCB_WAITING: - { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } case ADA_CCB_DUMP: /* No-op. We're polling */ return; @@ -1919,14 +1895,11 @@ adasendorderedtag(void *arg) struct ada_softc *softc = arg; if (ada_send_ordered) { - if ((softc->ordered_tag_count == 0) - && ((softc->flags & ADA_FLAG_WENT_IDLE) == 0)) { - softc->flags |= ADA_FLAG_NEED_OTAG; + if (softc->outstanding_cmds > 0) { + if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0) + softc->flags |= ADA_FLAG_NEED_OTAG; + softc->flags &= ~ADA_FLAG_WAS_OTAG; } - if (softc->outstanding_cmds > 0) - softc->flags &= ~ADA_FLAG_WENT_IDLE; - - softc->ordered_tag_count = 0; } /* Queue us up again */ callout_reset(&softc->sendordered_c, diff --git a/sys/cam/ata/ata_pmp.c b/sys/cam/ata/ata_pmp.c index d220591..bd4d25a 100644 --- a/sys/cam/ata/ata_pmp.c +++ b/sys/cam/ata/ata_pmp.c @@ -293,7 +293,7 @@ pmpasync(void *callback_arg, u_int32_t code, status = cam_periph_alloc(pmpregister, pmponinvalidate, pmpcleanup, pmpstart, "pmp", CAM_PERIPH_BIO, - cgd->ccb_h.path, pmpasync, + path, pmpasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -318,13 +318,17 @@ pmpasync(void *callback_arg, u_int32_t code, if (code == AC_SENT_BDR || code == AC_BUS_RESET) softc->found = 0; /* We have to reset everything. */ if (softc->state == PMP_STATE_NORMAL) { - if (softc->pm_pid == 0x37261095 || - softc->pm_pid == 0x38261095) - softc->state = PMP_STATE_PM_QUIRKS_1; - else - softc->state = PMP_STATE_PRECONFIG; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); + if (cam_periph_acquire(periph) == CAM_REQ_CMP) { + if (softc->pm_pid == 0x37261095 || + softc->pm_pid == 0x38261095) + softc->state = PMP_STATE_PM_QUIRKS_1; + else + softc->state = PMP_STATE_PRECONFIG; + xpt_schedule(periph, CAM_PRIORITY_DEV); + } else { + pmprelease(periph, softc->found); + xpt_release_boot(); + } } else softc->restart = 1; break; diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c index 2d9b05d..867b2fe 100644 --- a/sys/cam/ata/ata_xpt.c +++ b/sys/cam/ata/ata_xpt.c @@ -182,7 +182,7 @@ static struct cam_ed * static void ata_device_transport(struct cam_path *path); static void ata_get_transfer_settings(struct ccb_trans_settings *cts); static void ata_set_transfer_settings(struct ccb_trans_settings *cts, - struct cam_ed *device, + struct cam_path *path, int async_update); static void ata_dev_async(u_int32_t async_code, struct cam_eb *bus, @@ -249,6 +249,7 @@ proberegister(struct cam_periph *periph, void *arg) return (status); } CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n")); + ata_device_transport(periph->path); probeschedule(periph); return(CAM_REQ_CMP); } @@ -1320,6 +1321,7 @@ ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb) struct cam_path *path; ata_scan_bus_info *scan_info; union ccb *work_ccb, *reset_ccb; + struct mtx *mtx; cam_status status; CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE, @@ -1395,11 +1397,14 @@ ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb) xpt_done(request_ccb); break; } + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); goto scan_next; case XPT_SCAN_LUN: work_ccb = request_ccb; /* Reuse the same CCB to query if a device was really found */ scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0; + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); + mtx_lock(mtx); /* If there is PMP... */ if ((scan_info->cpi->hba_inquiry & PI_SATAPM) && (scan_info->counter == scan_info->cpi->max_target)) { @@ -1428,6 +1433,7 @@ ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb) ((scan_info->cpi->hba_inquiry & PI_SATAPM) ? 0 : scan_info->cpi->max_target)) { done: + mtx_unlock(mtx); xpt_free_ccb(work_ccb); xpt_free_ccb((union ccb *)scan_info->cpi); request_ccb = scan_info->request_ccb; @@ -1444,6 +1450,8 @@ scan_next: scan_info->request_ccb->ccb_h.path_id, scan_info->counter, 0); if (status != CAM_REQ_CMP) { + if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) + mtx_unlock(mtx); printf("xpt_scan_bus: xpt_create_path failed" " with status %#x, bus scan halted\n", status); @@ -1459,9 +1467,15 @@ scan_next: scan_info->request_ccb->ccb_h.pinfo.priority); work_ccb->ccb_h.func_code = XPT_SCAN_LUN; work_ccb->ccb_h.cbfcnp = ata_scan_bus; + work_ccb->ccb_h.flags |= CAM_UNLOCKED; work_ccb->ccb_h.ppriv_ptr0 = scan_info; work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags; + mtx_unlock(mtx); + if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) + mtx = NULL; xpt_action(work_ccb); + if (mtx != NULL) + mtx_lock(mtx); break; default: break; @@ -1476,6 +1490,7 @@ ata_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_status status; struct cam_path *new_path; struct cam_periph *old_periph; + int lock; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n")); @@ -1510,10 +1525,14 @@ ata_scan_lun(struct cam_periph *periph, struct cam_path *path, } xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT); request_ccb->ccb_h.cbfcnp = xptscandone; + request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->ccb_h.func_code = XPT_SCAN_LUN; request_ccb->crcn.flags = flags; } + lock = (xpt_path_owned(path) == 0); + if (lock) + xpt_path_lock(path); if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) { if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) { probe_softc *softc; @@ -1540,6 +1559,8 @@ ata_scan_lun(struct cam_periph *periph, struct cam_path *path, xpt_done(request_ccb); } } + if (lock) + xpt_path_unlock(path); } static void @@ -1553,7 +1574,6 @@ xptscandone(struct cam_periph *periph, union ccb *done_ccb) static struct cam_ed * ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { - struct cam_path path; struct ata_quirk_entry *quirk; struct cam_ed *device; @@ -1574,22 +1594,6 @@ ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) device->queue_flags = 0; device->serial_num = NULL; device->serial_num_len = 0; - - /* - * XXX should be limited by number of CCBs this bus can - * do. - */ - bus->sim->max_ccbs += device->ccbq.devq_openings; - if (lun_id != CAM_LUN_WILDCARD) { - xpt_compile_path(&path, - NULL, - bus->path_id, - target->target_id, - lun_id); - ata_device_transport(&path); - xpt_release_path(&path); - } - return (device); } @@ -1712,15 +1716,8 @@ ata_dev_advinfo(union ccb *start_ccb) start_ccb->ccb_h.status = CAM_REQ_CMP; if (cdai->flags & CDAI_FLAG_STORE) { - int owned; - - owned = mtx_owned(start_ccb->ccb_h.path->bus->sim->mtx); - if (owned == 0) - mtx_lock(start_ccb->ccb_h.path->bus->sim->mtx); xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path, (void *)(uintptr_t)cdai->buftype); - if (owned == 0) - mtx_unlock(start_ccb->ccb_h.path->bus->sim->mtx); } } @@ -1732,7 +1729,7 @@ ata_action(union ccb *start_ccb) case XPT_SET_TRAN_SETTINGS: { ata_set_transfer_settings(&start_ccb->cts, - start_ccb->ccb_h.path->device, + start_ccb->ccb_h.path, /*async_update*/FALSE); break; } @@ -1791,11 +1788,9 @@ ata_get_transfer_settings(struct ccb_trans_settings *cts) struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; struct cam_ed *device; - struct cam_sim *sim; device = cts->ccb_h.path->device; - sim = cts->ccb_h.path->bus->sim; - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); if (cts->protocol == PROTO_UNKNOWN || cts->protocol == PROTO_UNSPECIFIED) { @@ -1832,17 +1827,17 @@ ata_get_transfer_settings(struct ccb_trans_settings *cts) } static void -ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, +ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path, int async_update) { struct ccb_pathinq cpi; struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; - struct cam_sim *sim; struct ata_params *ident_data; struct scsi_inquiry_data *inq_data; + struct cam_ed *device; - if (device == NULL) { + if (path == NULL || (device = path->device) == NULL) { cts->ccb_h.status = CAM_PATH_INVALID; xpt_done((union ccb *)cts); return; @@ -1859,14 +1854,14 @@ ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, cts->protocol_version = device->protocol_version; if (cts->protocol != device->protocol) { - xpt_print(cts->ccb_h.path, "Uninitialized Protocol %x:%x?\n", + xpt_print(path, "Uninitialized Protocol %x:%x?\n", cts->protocol, device->protocol); cts->protocol = device->protocol; } if (cts->protocol_version > device->protocol_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Protocol " + xpt_print(path, "Down reving Protocol " "Version from %d to %d?\n", cts->protocol_version, device->protocol_version); } @@ -1884,21 +1879,20 @@ ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, cts->transport_version = device->transport_version; if (cts->transport != device->transport) { - xpt_print(cts->ccb_h.path, "Uninitialized Transport %x:%x?\n", + xpt_print(path, "Uninitialized Transport %x:%x?\n", cts->transport, device->transport); cts->transport = device->transport; } if (cts->transport_version > device->transport_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Transport " + xpt_print(path, "Down reving Transport " "Version from %d to %d?\n", cts->transport_version, device->transport_version); } cts->transport_version = device->transport_version; } - sim = cts->ccb_h.path->bus->sim; ident_data = &device->ident_data; inq_data = &device->inq_data; if (cts->protocol == PROTO_ATA) @@ -1909,7 +1903,7 @@ ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, scsi = &cts->proto_specific.scsi; else scsi = NULL; - xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); @@ -1953,11 +1947,11 @@ ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, device->tag_delay_count = CAM_TAG_DELAY_COUNT; device->flags |= CAM_DEV_TAG_AFTER_COUNT; } else if (nowt && !newt) - xpt_stop_tags(cts->ccb_h.path); + xpt_stop_tags(path); } if (async_update == FALSE) - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); } /* @@ -2014,10 +2008,14 @@ ata_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, xpt_release_device(device); } else if (async_code == AC_TRANSFER_NEG) { struct ccb_trans_settings *settings; + struct cam_path path; settings = (struct ccb_trans_settings *)async_arg; - ata_set_transfer_settings(settings, device, + xpt_compile_path(&path, NULL, bus->path_id, target->target_id, + device->lun_id); + ata_set_transfer_settings(settings, &path, /*async_update*/TRUE); + xpt_release_path(&path); } } @@ -2030,7 +2028,7 @@ ata_announce_periph(struct cam_periph *periph) u_int speed; u_int mb; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h index 127246b..139bd50 100644 --- a/sys/cam/cam_ccb.h +++ b/sys/cam/cam_ccb.h @@ -104,7 +104,9 @@ typedef enum { CAM_SEND_SENSE = 0x08000000,/* Send sense data with status */ CAM_TERM_IO = 0x10000000,/* Terminate I/O Message sup. */ CAM_DISCONNECT = 0x20000000,/* Disconnects are mandatory */ - CAM_SEND_STATUS = 0x40000000 /* Send status after data phase */ + CAM_SEND_STATUS = 0x40000000,/* Send status after data phase */ + + CAM_UNLOCKED = 0x80000000 /* Call callback without lock. */ } ccb_flags; typedef enum { @@ -151,6 +153,9 @@ typedef enum { /* Device statistics (error counts, etc.) */ XPT_DEV_ADVINFO = 0x0e, /* Get/Set Device advanced information */ + XPT_ASYNC = 0x0f | XPT_FC_QUEUED | XPT_FC_USER_CCB + | XPT_FC_XPT_ONLY, + /* Asynchronous event */ /* SCSI Control Functions: 0x10->0x1F */ XPT_ABORT = 0x10, /* Abort the specified CCB */ @@ -1154,6 +1159,16 @@ struct ccb_dev_advinfo { }; /* + * CCB for sending async events + */ +struct ccb_async { + struct ccb_hdr ccb_h; + uint32_t async_code; + off_t async_arg_size; + void *async_arg_ptr; +}; + +/* * Union of all CCB types for kernel space allocation. This union should * never be used for manipulating CCBs - its only use is for the allocation * and deallocation of raw CCB space and is the return type of xpt_ccb_alloc @@ -1192,6 +1207,7 @@ union ccb { struct ccb_debug cdbg; struct ccb_ataio ataio; struct ccb_dev_advinfo cdai; + struct ccb_async casync; }; __BEGIN_DECLS diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c index 4e9138f..1c636cb 100644 --- a/sys/cam/cam_periph.c +++ b/sys/cam/cam_periph.c @@ -196,12 +196,12 @@ cam_periph_alloc(periph_ctor_t *periph_ctor, path_id = xpt_path_path_id(path); target_id = xpt_path_target_id(path); lun_id = xpt_path_lun_id(path); - cam_init_pinfo(&periph->pinfo); periph->periph_start = periph_start; periph->periph_dtor = periph_dtor; periph->periph_oninval = periph_oninvalidate; periph->type = type; periph->periph_name = name; + periph->scheduled_priority = CAM_PRIORITY_NONE; periph->immediate_priority = CAM_PRIORITY_NONE; periph->refcount = 1; /* Dropped by invalidation. */ periph->sim = sim; @@ -298,7 +298,7 @@ cam_periph_find(struct cam_path *path, char *name) TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { if (xpt_path_comp(periph->path, path) == 0) { xpt_unlock_buses(); - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); return(periph); } } @@ -379,7 +379,7 @@ void cam_periph_release_locked_buses(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); KASSERT(periph->refcount >= 1, ("periph->refcount >= 1")); if (--periph->refcount == 0) camperiphfree(periph); @@ -400,16 +400,16 @@ cam_periph_release_locked(struct cam_periph *periph) void cam_periph_release(struct cam_periph *periph) { - struct cam_sim *sim; + struct mtx *mtx; if (periph == NULL) return; - sim = periph->sim; - mtx_assert(sim->mtx, MA_NOTOWNED); - mtx_lock(sim->mtx); + cam_periph_assert(periph, MA_NOTOWNED); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); cam_periph_release_locked(periph); - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } int @@ -427,10 +427,10 @@ cam_periph_hold(struct cam_periph *periph, int priority) if (cam_periph_acquire(periph) != CAM_REQ_CMP) return (ENXIO); - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); while ((periph->flags & CAM_PERIPH_LOCKED) != 0) { periph->flags |= CAM_PERIPH_LOCK_WANTED; - if ((error = mtx_sleep(periph, periph->sim->mtx, priority, + if ((error = cam_periph_sleep(periph, periph, priority, "caplck", 0)) != 0) { cam_periph_release_locked(periph); return (error); @@ -449,7 +449,7 @@ void cam_periph_unhold(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); periph->flags &= ~CAM_PERIPH_LOCKED; if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) { @@ -577,7 +577,7 @@ void cam_periph_invalidate(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); /* * We only call this routine the first time a peripheral is * invalidated. @@ -600,7 +600,9 @@ camperiphfree(struct cam_periph *periph) { struct periph_driver **p_drv; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); + KASSERT(periph->periph_allocating == 0, ("%s%d: freed while allocating", + periph->periph_name, periph->unit_number)); for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { if (strcmp((*p_drv)->driver_name, periph->periph_name) == 0) break; @@ -947,40 +949,14 @@ cam_periph_unmapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo) PRELE(curproc); } -union ccb * -cam_periph_getccb(struct cam_periph *periph, u_int32_t priority) -{ - struct ccb_hdr *ccb_h; - - mtx_assert(periph->sim->mtx, MA_OWNED); - CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdgetccb\n")); - - while (SLIST_FIRST(&periph->ccb_list) == NULL) { - if (periph->immediate_priority > priority) - periph->immediate_priority = priority; - xpt_schedule(periph, priority); - if ((SLIST_FIRST(&periph->ccb_list) != NULL) - && (SLIST_FIRST(&periph->ccb_list)->pinfo.priority == priority)) - break; - mtx_assert(periph->sim->mtx, MA_OWNED); - mtx_sleep(&periph->ccb_list, periph->sim->mtx, PRIBIO, "cgticb", - 0); - } - - ccb_h = SLIST_FIRST(&periph->ccb_list); - SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle); - return ((union ccb *)ccb_h); -} - void cam_periph_ccbwait(union ccb *ccb) { - struct cam_sim *sim; - sim = xpt_path_sim(ccb->ccb_h.path); if ((ccb->ccb_h.pinfo.index != CAM_UNQUEUED_INDEX) || ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG)) - mtx_sleep(&ccb->ccb_h.cbfcnp, sim->mtx, PRIBIO, "cbwait", 0); + xpt_path_sleep(ccb->ccb_h.path, &ccb->ccb_h.cbfcnp, PRIBIO, + "cbwait", 0); } int @@ -1045,6 +1021,14 @@ cam_periph_ioctl(struct cam_periph *periph, u_long cmd, caddr_t addr, return(error); } +static void +cam_periph_done(struct cam_periph *periph, union ccb *done_ccb) +{ + + /* Caller will release the CCB */ + wakeup(&done_ccb->ccb_h.cbfcnp); +} + int cam_periph_runccb(union ccb *ccb, int (*error_routine)(union ccb *ccb, @@ -1053,12 +1037,9 @@ cam_periph_runccb(union ccb *ccb, cam_flags camflags, u_int32_t sense_flags, struct devstat *ds) { - struct cam_sim *sim; int error; - error = 0; - sim = xpt_path_sim(ccb->ccb_h.path); - mtx_assert(sim->mtx, MA_OWNED); + xpt_path_assert(ccb->ccb_h.path, MA_OWNED); /* * If the user has supplied a stats structure, and if we understand @@ -1068,6 +1049,7 @@ cam_periph_runccb(union ccb *ccb, ccb->ccb_h.func_code == XPT_ATA_IO)) devstat_start_transaction(ds, NULL); + ccb->ccb_h.cbfcnp = cam_periph_done; xpt_action(ccb); do { @@ -1786,9 +1768,11 @@ cam_periph_error(union ccb *ccb, cam_flags camflags, scan_ccb->ccb_h.func_code = XPT_SCAN_TGT; scan_ccb->crcn.flags = 0; xpt_rescan(scan_ccb); - } else + } else { xpt_print(newpath, "Can't allocate CCB to rescan target\n"); + xpt_free_path(newpath); + } } } diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h index 66b9d91..b9a04d9 100644 --- a/sys/cam/cam_periph.h +++ b/sys/cam/cam_periph.h @@ -35,6 +35,7 @@ #include <cam/cam_sim.h> #ifdef _KERNEL +#include <sys/taskqueue.h> #include <cam/cam_xpt.h> @@ -103,7 +104,6 @@ typedef cam_status periph_ctor_t (struct cam_periph *periph, typedef void periph_oninv_t (struct cam_periph *periph); typedef void periph_dtor_t (struct cam_periph *periph); struct cam_periph { - cam_pinfo pinfo; periph_start_t *periph_start; periph_oninv_t *periph_oninval; periph_dtor_t *periph_dtor; @@ -120,15 +120,20 @@ struct cam_periph { #define CAM_PERIPH_INVALID 0x08 #define CAM_PERIPH_NEW_DEV_FOUND 0x10 #define CAM_PERIPH_RECOVERY_INPROG 0x20 +#define CAM_PERIPH_RUN_TASK 0x40 #define CAM_PERIPH_FREE 0x80 #define CAM_PERIPH_ANNOUNCED 0x100 - u_int32_t immediate_priority; + uint32_t scheduled_priority; + uint32_t immediate_priority; + int periph_allocating; + int periph_allocated; u_int32_t refcount; SLIST_HEAD(, ccb_hdr) ccb_list; /* For "immediate" requests */ SLIST_ENTRY(cam_periph) periph_links; TAILQ_ENTRY(cam_periph) unit_links; ac_callback_t *deferred_callback; ac_code deferred_ac; + struct task periph_run_task; }; #define CAM_PERIPH_MAXMAPS 2 @@ -185,30 +190,26 @@ void cam_periph_freeze_after_event(struct cam_periph *periph, int cam_periph_error(union ccb *ccb, cam_flags camflags, u_int32_t sense_flags, union ccb *save_ccb); -static __inline void -cam_periph_lock(struct cam_periph *periph) +static __inline struct mtx * +cam_periph_mtx(struct cam_periph *periph) { - mtx_lock(periph->sim->mtx); + return (xpt_path_mtx(periph->path)); } -static __inline void -cam_periph_unlock(struct cam_periph *periph) -{ - mtx_unlock(periph->sim->mtx); -} +#define cam_periph_owned(periph) \ + mtx_owned(xpt_path_mtx((periph)->path)) -static __inline int -cam_periph_owned(struct cam_periph *periph) -{ - return (mtx_owned(periph->sim->mtx)); -} +#define cam_periph_lock(periph) \ + mtx_lock(xpt_path_mtx((periph)->path)) -static __inline int -cam_periph_sleep(struct cam_periph *periph, void *chan, int priority, - const char *wmesg, int timo) -{ - return (msleep(chan, periph->sim->mtx, priority, wmesg, timo)); -} +#define cam_periph_unlock(periph) \ + mtx_unlock(xpt_path_mtx((periph)->path)) + +#define cam_periph_assert(periph, what) \ + mtx_assert(xpt_path_mtx((periph)->path), (what)) + +#define cam_periph_sleep(periph, chan, priority, wmesg, timo) \ + xpt_path_sleep((periph)->path, (chan), (priority), (wmesg), (timo)) static inline struct cam_periph * cam_periph_acquire_first(struct periph_driver *driver) @@ -230,7 +231,7 @@ cam_periph_acquire_next(struct cam_periph *pperiph) { struct cam_periph *periph = pperiph; - mtx_assert(pperiph->sim->mtx, MA_NOTOWNED); + cam_periph_assert(pperiph, MA_NOTOWNED); xpt_lock_buses(); do { periph = TAILQ_NEXT(periph, unit_links); diff --git a/sys/cam/cam_queue.c b/sys/cam/cam_queue.c index daaa9b3..f6624f3 100644 --- a/sys/cam/cam_queue.c +++ b/sys/cam/cam_queue.c @@ -220,27 +220,30 @@ cam_devq_alloc(int devices, int openings) } if (cam_devq_init(devq, devices, openings) != 0) { free(devq, M_CAMDEVQ); - return (NULL); + return (NULL); } - return (devq); } int cam_devq_init(struct cam_devq *devq, int devices, int openings) { + bzero(devq, sizeof(*devq)); + mtx_init(&devq->send_mtx, "CAM queue lock", NULL, MTX_DEF); if (camq_init(&devq->send_queue, devices) != 0) return (1); devq->send_openings = openings; - devq->send_active = 0; - return (0); + devq->send_active = 0; + return (0); } void cam_devq_free(struct cam_devq *devq) { + camq_fini(&devq->send_queue); + mtx_destroy(&devq->send_mtx); free(devq, M_CAMDEVQ); } @@ -286,6 +289,7 @@ cam_ccbq_resize(struct cam_ccbq *ccbq, int new_size) int delta; delta = new_size - (ccbq->dev_active + ccbq->dev_openings); + ccbq->total_openings += delta; ccbq->devq_openings += delta; ccbq->dev_openings += delta; @@ -303,6 +307,7 @@ cam_ccbq_init(struct cam_ccbq *ccbq, int openings) if (camq_init(&ccbq->queue, imax(64, 1 << fls(openings + openings / 2))) != 0) return (1); + ccbq->total_openings = openings; ccbq->devq_openings = openings; ccbq->dev_openings = openings; return (0); diff --git a/sys/cam/cam_queue.h b/sys/cam/cam_queue.h index 0bb4491..0f74e82 100644 --- a/sys/cam/cam_queue.h +++ b/sys/cam/cam_queue.h @@ -33,6 +33,8 @@ #ifdef _KERNEL +#include <sys/lock.h> +#include <sys/mutex.h> #include <sys/queue.h> #include <cam/cam.h> @@ -59,8 +61,8 @@ struct cam_ccbq { struct camq queue; struct ccb_hdr_tailq queue_extra_head; int queue_extra_entries; + int total_openings; int devq_openings; - int devq_allocating; int dev_openings; int dev_active; int held; @@ -69,9 +71,10 @@ struct cam_ccbq { struct cam_ed; struct cam_devq { - struct camq send_queue; - int send_openings; - int send_active; + struct mtx send_mtx; + struct camq send_queue; + int send_openings; + int send_active; }; diff --git a/sys/cam/cam_sim.c b/sys/cam/cam_sim.c index 530e160..9a8c666 100644 --- a/sys/cam/cam_sim.c +++ b/sys/cam/cam_sim.c @@ -87,7 +87,6 @@ cam_sim_alloc(sim_action_func sim_action, sim_poll_func sim_poll, sim->flags = 0; sim->refcount = 1; sim->devq = queue; - sim->max_ccbs = 8; /* Reserve for management purposes. */ sim->mtx = mtx; if (mtx == &Giant) { sim->flags |= 0; @@ -96,17 +95,12 @@ cam_sim_alloc(sim_action_func sim_action, sim_poll_func sim_poll, sim->flags |= CAM_SIM_MPSAFE; callout_init(&sim->callout, 1); } - - SLIST_INIT(&sim->ccb_freeq); - TAILQ_INIT(&sim->sim_doneq); - return (sim); } void cam_sim_free(struct cam_sim *sim, int free_devq) { - union ccb *ccb; int error; mtx_assert(sim->mtx, MA_OWNED); @@ -118,10 +112,6 @@ cam_sim_free(struct cam_sim *sim, int free_devq) KASSERT(sim->refcount == 0, ("sim->refcount == 0")); - while ((ccb = (union ccb *)SLIST_FIRST(&sim->ccb_freeq)) != NULL) { - SLIST_REMOVE_HEAD(&sim->ccb_freeq, xpt_links.sle); - xpt_free_ccb(ccb); - } if (free_devq) cam_simq_free(sim->devq); free(sim, M_CAMSIM); @@ -130,21 +120,31 @@ cam_sim_free(struct cam_sim *sim, int free_devq) void cam_sim_release(struct cam_sim *sim) { - KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); - mtx_assert(sim->mtx, MA_OWNED); + int lock; + lock = (mtx_owned(sim->mtx) == 0); + if (lock) + CAM_SIM_LOCK(sim); + KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); sim->refcount--; if (sim->refcount == 0) wakeup(sim); + if (lock) + CAM_SIM_UNLOCK(sim); } void cam_sim_hold(struct cam_sim *sim) { - KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); - mtx_assert(sim->mtx, MA_OWNED); + int lock; + lock = (mtx_owned(sim->mtx) == 0); + if (lock) + CAM_SIM_LOCK(sim); + KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); sim->refcount++; + if (lock) + CAM_SIM_UNLOCK(sim); } void diff --git a/sys/cam/cam_sim.h b/sys/cam/cam_sim.h index 58127ba..4232650 100644 --- a/sys/cam/cam_sim.h +++ b/sys/cam/cam_sim.h @@ -104,23 +104,9 @@ struct cam_sim { u_int32_t flags; #define CAM_SIM_REL_TIMEOUT_PENDING 0x01 #define CAM_SIM_MPSAFE 0x02 -#define CAM_SIM_ON_DONEQ 0x04 -#define CAM_SIM_POLLED 0x08 -#define CAM_SIM_BATCH 0x10 struct callout callout; struct cam_devq *devq; /* Device Queue to use for this SIM */ int refcount; /* References to the SIM. */ - - /* "Pool" of inactive ccbs managed by xpt_get_ccb and xpt_release_ccb */ - SLIST_HEAD(,ccb_hdr) ccb_freeq; - /* - * Maximum size of ccb pool. Modified as devices are added/removed - * or have their * opening counts changed. - */ - u_int max_ccbs; - /* Current count of allocated ccbs */ - u_int ccb_count; - }; #define CAM_SIM_LOCK(sim) mtx_lock((sim)->mtx) diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c index 54c4acd..86e2f5e 100644 --- a/sys/cam/cam_xpt.c +++ b/sys/cam/cam_xpt.c @@ -40,7 +40,9 @@ __FBSDID("$FreeBSD$"); #include <sys/conf.h> #include <sys/fcntl.h> #include <sys/interrupt.h> +#include <sys/proc.h> #include <sys/sbuf.h> +#include <sys/smp.h> #include <sys/taskqueue.h> #include <sys/lock.h> @@ -90,14 +92,9 @@ struct xpt_task { uintptr_t data2; }; -typedef enum { - XPT_FLAG_OPEN = 0x01 -} xpt_flags; - struct xpt_softc { - xpt_flags flags; - /* number of high powered commands that can go through right now */ + struct mtx xpt_highpower_lock; STAILQ_HEAD(highpowerlist, cam_ed) highpowerq; int num_highpower; @@ -117,6 +114,7 @@ struct xpt_softc { struct mtx xpt_topo_lock; struct mtx xpt_lock; + struct taskqueue *xpt_taskq; }; typedef enum { @@ -155,14 +153,19 @@ TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay); SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN, &xsoftc.boot_delay, 0, "Bus registration wait time"); -/* Queues for our software interrupt handler */ -typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t; -typedef TAILQ_HEAD(cam_simq, cam_sim) cam_simq_t; -static cam_simq_t cam_simq; -static struct mtx cam_simq_lock; +struct cam_doneq { + struct mtx_padalign cam_doneq_mtx; + STAILQ_HEAD(, ccb_hdr) cam_doneq; + int cam_doneq_sleep; +}; + +static struct cam_doneq cam_doneqs[MAXCPU]; +static int cam_num_doneqs; +static struct proc *cam_proc; -/* Pointers to software interrupt handlers */ -static void *cambio_ih; +TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs); +SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN, + &cam_num_doneqs, 0, "Number of completion queues/threads"); struct cam_periph *xpt_periph; @@ -223,16 +226,21 @@ static void xpt_async_bcast(struct async_list *async_head, void *async_arg); static path_id_t xptnextfreepathid(void); static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus); -static union ccb *xpt_get_ccb(struct cam_ed *device); -static void xpt_run_dev_allocq(struct cam_ed *device); +static union ccb *xpt_get_ccb(struct cam_periph *periph); +static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph); +static void xpt_run_allocq(struct cam_periph *periph, int sleep); +static void xpt_run_allocq_task(void *context, int pending); static void xpt_run_devq(struct cam_devq *devq); static timeout_t xpt_release_devq_timeout; static void xpt_release_simq_timeout(void *arg) __unused; +static void xpt_acquire_bus(struct cam_eb *bus); static void xpt_release_bus(struct cam_eb *bus); -static void xpt_release_devq_device(struct cam_ed *dev, u_int count, +static uint32_t xpt_freeze_devq_device(struct cam_ed *dev, u_int count); +static int xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue); static struct cam_et* xpt_alloc_target(struct cam_eb *bus, target_id_t target_id); +static void xpt_acquire_target(struct cam_et *target); static void xpt_release_target(struct cam_et *target); static struct cam_eb* xpt_find_bus(path_id_t path_id); @@ -241,11 +249,14 @@ static struct cam_et* static struct cam_ed* xpt_find_device(struct cam_et *target, lun_id_t lun_id); static void xpt_config(void *arg); +static int xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo, + u_int32_t new_priority); static xpt_devicefunc_t xptpassannouncefunc; static void xptaction(struct cam_sim *sim, union ccb *work_ccb); static void xptpoll(struct cam_sim *sim); -static void camisr(void *); -static void camisr_runqueue(struct cam_sim *); +static void camisr_runqueue(void); +static void xpt_done_process(struct ccb_hdr *ccb_h); +static void xpt_done_td(void *); static dev_match_ret xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns, struct cam_eb *bus); static dev_match_ret xptdevicematch(struct dev_match_pattern *patterns, @@ -296,7 +307,6 @@ static xpt_devicefunc_t xptsetasyncfunc; static xpt_busfunc_t xptsetasyncbusfunc; static cam_status xptregister(struct cam_periph *periph, void *arg); -static __inline int periph_is_queued(struct cam_periph *periph); static __inline int device_is_queued(struct cam_ed *device); static __inline int @@ -304,6 +314,7 @@ xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev) { int retval; + mtx_assert(&devq->send_mtx, MA_OWNED); if ((dev->ccbq.queue.entries > 0) && (dev->ccbq.dev_openings > 0) && (dev->ccbq.queue.qfrozen_cnt == 0)) { @@ -314,7 +325,7 @@ xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev) */ retval = xpt_schedule_dev(&devq->send_queue, - &dev->devq_entry.pinfo, + &dev->devq_entry, CAMQ_GET_PRIO(&dev->ccbq.queue)); } else { retval = 0; @@ -323,15 +334,9 @@ xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev) } static __inline int -periph_is_queued(struct cam_periph *periph) -{ - return (periph->pinfo.index != CAM_UNQUEUED_INDEX); -} - -static __inline int device_is_queued(struct cam_ed *device) { - return (device->devq_entry.pinfo.index != CAM_UNQUEUED_INDEX); + return (device->devq_entry.index != CAM_UNQUEUED_INDEX); } static void @@ -340,13 +345,6 @@ xpt_periph_init() make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0"); } -static void -xptdone(struct cam_periph *periph, union ccb *done_ccb) -{ - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); -} - static int xptopen(struct cdev *dev, int flags, int fmt, struct thread *td) { @@ -365,11 +363,6 @@ xptopen(struct cdev *dev, int flags, int fmt, struct thread *td) return(ENODEV); } - /* Mark ourselves open */ - mtx_lock(&xsoftc.xpt_lock); - xsoftc.flags |= XPT_FLAG_OPEN; - mtx_unlock(&xsoftc.xpt_lock); - return(0); } @@ -377,11 +370,6 @@ static int xptclose(struct cdev *dev, int flag, int fmt, struct thread *td) { - /* Mark ourselves closed */ - mtx_lock(&xsoftc.xpt_lock); - xsoftc.flags &= ~XPT_FLAG_OPEN; - mtx_unlock(&xsoftc.xpt_lock); - return(0); } @@ -457,8 +445,6 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * ccb = xpt_alloc_ccb(); - CAM_SIM_LOCK(bus->sim); - /* * Create a path using the bus, target, and lun the * user passed in. @@ -469,7 +455,6 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * inccb->ccb_h.target_lun) != CAM_REQ_CMP){ error = EINVAL; - CAM_SIM_UNLOCK(bus->sim); xpt_free_ccb(ccb); break; } @@ -477,12 +462,12 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, inccb->ccb_h.pinfo.priority); xpt_merge_ccb(ccb, inccb); - ccb->ccb_h.cbfcnp = xptdone; + xpt_path_lock(ccb->ccb_h.path); cam_periph_runccb(ccb, NULL, 0, 0, NULL); + xpt_path_unlock(ccb->ccb_h.path); bcopy(ccb, inccb, sizeof(union ccb)); xpt_free_path(ccb->ccb_h.path); xpt_free_ccb(ccb); - CAM_SIM_UNLOCK(bus->sim); break; case XPT_DEBUG: { @@ -493,8 +478,6 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * * allocate it on the stack. */ - CAM_SIM_LOCK(bus->sim); - /* * Create a path using the bus, target, and lun the * user passed in. @@ -505,18 +488,15 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * inccb->ccb_h.target_lun) != CAM_REQ_CMP){ error = EINVAL; - CAM_SIM_UNLOCK(bus->sim); break; } /* Ensure all of our fields are correct */ xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path, inccb->ccb_h.pinfo.priority); xpt_merge_ccb(&ccb, inccb); - ccb.ccb_h.cbfcnp = xptdone; xpt_action(&ccb); bcopy(&ccb, inccb, sizeof(union ccb)); xpt_free_path(ccb.ccb_h.path); - CAM_SIM_UNLOCK(bus->sim); break; } @@ -564,9 +544,7 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * /* * This is an immediate CCB, we can send it on directly. */ - CAM_SIM_LOCK(xpt_path_sim(xpt_periph->path)); xpt_action(inccb); - CAM_SIM_UNLOCK(xpt_path_sim(xpt_periph->path)); /* * Map the buffers back into user space. @@ -784,7 +762,7 @@ static void xpt_scanner_thread(void *dummy) { union ccb *ccb; - struct cam_sim *sim; + struct cam_path path; xpt_lock_buses(); for (;;) { @@ -795,10 +773,16 @@ xpt_scanner_thread(void *dummy) TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe); xpt_unlock_buses(); - sim = ccb->ccb_h.path->bus->sim; - CAM_SIM_LOCK(sim); + /* + * Since lock can be dropped inside and path freed + * by completion callback even before return here, + * take our own path copy for reference. + */ + xpt_copy_path(&path, ccb->ccb_h.path); + xpt_path_lock(&path); xpt_action(ccb); - CAM_SIM_UNLOCK(sim); + xpt_path_unlock(&path); + xpt_release_path(&path); xpt_lock_buses(); } @@ -857,16 +841,18 @@ xpt_init(void *dummy) struct cam_path *path; struct cam_devq *devq; cam_status status; + int error, i; TAILQ_INIT(&xsoftc.xpt_busses); - TAILQ_INIT(&cam_simq); TAILQ_INIT(&xsoftc.ccb_scanq); STAILQ_INIT(&xsoftc.highpowerq); xsoftc.num_highpower = CAM_MAX_HIGHPOWER; - mtx_init(&cam_simq_lock, "CAM SIMQ lock", NULL, MTX_DEF); mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF); + mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF); mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF); + xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK, + taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq); #ifdef CAM_BOOT_DELAY /* @@ -901,6 +887,7 @@ xpt_init(void *dummy) " failing attach\n", status); return (EINVAL); } + mtx_unlock(&xsoftc.xpt_lock); /* * Looking at the XPT from the SIM layer, the XPT is @@ -915,13 +902,32 @@ xpt_init(void *dummy) " failing attach\n", status); return (EINVAL); } - + xpt_path_lock(path); cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO, path, NULL, 0, xpt_sim); + xpt_path_unlock(path); xpt_free_path(path); - mtx_unlock(&xsoftc.xpt_lock); - /* Install our software interrupt handlers */ - swi_add(NULL, "cambio", camisr, NULL, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih); + + if (cam_num_doneqs < 1) + cam_num_doneqs = 1 + mp_ncpus / 6; + else if (cam_num_doneqs > MAXCPU) + cam_num_doneqs = MAXCPU; + for (i = 0; i < cam_num_doneqs; i++) { + mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL, + MTX_DEF); + STAILQ_INIT(&cam_doneqs[i].cam_doneq); + error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i], + &cam_proc, NULL, 0, 0, "cam", "doneq%d", i); + if (error != 0) { + cam_num_doneqs = i; + break; + } + } + if (cam_num_doneqs < 1) { + printf("xpt_init: Cannot init completion queues " + "- failing attach\n"); + return (ENOMEM); + } /* * Register a callback for when interrupts are enabled. */ @@ -966,28 +972,15 @@ xpt_add_periph(struct cam_periph *periph) { struct cam_ed *device; int32_t status; - struct periph_list *periph_head; - - mtx_assert(periph->sim->mtx, MA_OWNED); + TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph); device = periph->path->device; - - periph_head = &device->periphs; - status = CAM_REQ_CMP; - if (device != NULL) { - /* - * Make room for this peripheral - * so it will fit in the queue - * when it's scheduled to run - */ - status = camq_resize(&device->drvq, - device->drvq.array_size + 1); - + mtx_lock(&device->target->bus->eb_mtx); device->generation++; - - SLIST_INSERT_HEAD(periph_head, periph, periph_links); + SLIST_INSERT_HEAD(&device->periphs, periph, periph_links); + mtx_unlock(&device->target->bus->eb_mtx); } return (status); @@ -998,21 +991,12 @@ xpt_remove_periph(struct cam_periph *periph) { struct cam_ed *device; - mtx_assert(periph->sim->mtx, MA_OWNED); - device = periph->path->device; - if (device != NULL) { - struct periph_list *periph_head; - - periph_head = &device->periphs; - - /* Release the slot for this peripheral */ - camq_resize(&device->drvq, device->drvq.array_size - 1); - + mtx_lock(&device->target->bus->eb_mtx); device->generation++; - - SLIST_REMOVE(periph_head, periph, cam_periph, periph_links); + SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links); + mtx_unlock(&device->target->bus->eb_mtx); } } @@ -1022,7 +1006,7 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string) { struct cam_path *path = periph->path; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); periph->flags |= CAM_PERIPH_ANNOUNCED; printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n", @@ -1077,7 +1061,7 @@ xpt_denounce_periph(struct cam_periph *periph) { struct cam_path *path = periph->path; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n", periph->periph_name, periph->unit_number, path->bus->sim->sim_name, @@ -1110,7 +1094,7 @@ xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path) struct ccb_dev_advinfo cdai; struct scsi_vpd_id_descriptor *idd; - mtx_assert(path->bus->sim->mtx, MA_OWNED); + xpt_path_assert(path, MA_OWNED); memset(&cdai, 0, sizeof(cdai)); xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL); @@ -1531,6 +1515,7 @@ static int xptedtbusfunc(struct cam_eb *bus, void *arg) { struct ccb_dev_match *cdm; + struct cam_et *target; dev_match_ret retval; cdm = (struct ccb_dev_match *)arg; @@ -1602,71 +1587,72 @@ xptedtbusfunc(struct cam_eb *bus, void *arg) * If there is a target generation recorded, check it to * make sure the target list hasn't changed. */ - if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (bus == cdm->pos.cookie.bus) - && (cdm->pos.position_type & CAM_DEV_POS_TARGET) - && (cdm->pos.generations[CAM_TARGET_GENERATION] != 0) - && (cdm->pos.generations[CAM_TARGET_GENERATION] != - bus->generation)) { - cdm->status = CAM_DEV_MATCH_LIST_CHANGED; - return(0); - } - + mtx_lock(&bus->eb_mtx); if ((cdm->pos.position_type & CAM_DEV_POS_BUS) && (cdm->pos.cookie.bus == bus) && (cdm->pos.position_type & CAM_DEV_POS_TARGET) - && (cdm->pos.cookie.target != NULL)) - return(xpttargettraverse(bus, - (struct cam_et *)cdm->pos.cookie.target, - xptedttargetfunc, arg)); - else - return(xpttargettraverse(bus, NULL, xptedttargetfunc, arg)); + && (cdm->pos.cookie.target != NULL)) { + if ((cdm->pos.generations[CAM_TARGET_GENERATION] != + bus->generation)) { + mtx_unlock(&bus->eb_mtx); + cdm->status = CAM_DEV_MATCH_LIST_CHANGED; + return (0); + } + target = (struct cam_et *)cdm->pos.cookie.target; + target->refcount++; + } else + target = NULL; + mtx_unlock(&bus->eb_mtx); + + return (xpttargettraverse(bus, target, xptedttargetfunc, arg)); } static int xptedttargetfunc(struct cam_et *target, void *arg) { struct ccb_dev_match *cdm; + struct cam_eb *bus; + struct cam_ed *device; cdm = (struct ccb_dev_match *)arg; + bus = target->bus; /* * If there is a device list generation recorded, check it to * make sure the device list hasn't changed. */ + mtx_lock(&bus->eb_mtx); if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (cdm->pos.cookie.bus == target->bus) + && (cdm->pos.cookie.bus == bus) && (cdm->pos.position_type & CAM_DEV_POS_TARGET) && (cdm->pos.cookie.target == target) && (cdm->pos.position_type & CAM_DEV_POS_DEVICE) - && (cdm->pos.generations[CAM_DEV_GENERATION] != 0) - && (cdm->pos.generations[CAM_DEV_GENERATION] != - target->generation)) { - cdm->status = CAM_DEV_MATCH_LIST_CHANGED; - return(0); - } + && (cdm->pos.cookie.device != NULL)) { + if (cdm->pos.generations[CAM_DEV_GENERATION] != + target->generation) { + mtx_unlock(&bus->eb_mtx); + cdm->status = CAM_DEV_MATCH_LIST_CHANGED; + return(0); + } + device = (struct cam_ed *)cdm->pos.cookie.device; + device->refcount++; + } else + device = NULL; + mtx_unlock(&bus->eb_mtx); - if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (cdm->pos.cookie.bus == target->bus) - && (cdm->pos.position_type & CAM_DEV_POS_TARGET) - && (cdm->pos.cookie.target == target) - && (cdm->pos.position_type & CAM_DEV_POS_DEVICE) - && (cdm->pos.cookie.device != NULL)) - return(xptdevicetraverse(target, - (struct cam_ed *)cdm->pos.cookie.device, - xptedtdevicefunc, arg)); - else - return(xptdevicetraverse(target, NULL, xptedtdevicefunc, arg)); + return (xptdevicetraverse(target, device, xptedtdevicefunc, arg)); } static int xptedtdevicefunc(struct cam_ed *device, void *arg) { - + struct cam_eb *bus; + struct cam_periph *periph; struct ccb_dev_match *cdm; dev_match_ret retval; cdm = (struct ccb_dev_match *)arg; + bus = device->target->bus; /* * If our position is for something deeper in the tree, that means @@ -1756,33 +1742,31 @@ xptedtdevicefunc(struct cam_ed *device, void *arg) * If there is a peripheral list generation recorded, make sure * it hasn't changed. */ + xpt_lock_buses(); + mtx_lock(&bus->eb_mtx); if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (device->target->bus == cdm->pos.cookie.bus) - && (cdm->pos.position_type & CAM_DEV_POS_TARGET) - && (device->target == cdm->pos.cookie.target) - && (cdm->pos.position_type & CAM_DEV_POS_DEVICE) - && (device == cdm->pos.cookie.device) - && (cdm->pos.position_type & CAM_DEV_POS_PERIPH) - && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0) - && (cdm->pos.generations[CAM_PERIPH_GENERATION] != - device->generation)){ - cdm->status = CAM_DEV_MATCH_LIST_CHANGED; - return(0); - } - - if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (cdm->pos.cookie.bus == device->target->bus) + && (cdm->pos.cookie.bus == bus) && (cdm->pos.position_type & CAM_DEV_POS_TARGET) && (cdm->pos.cookie.target == device->target) && (cdm->pos.position_type & CAM_DEV_POS_DEVICE) && (cdm->pos.cookie.device == device) && (cdm->pos.position_type & CAM_DEV_POS_PERIPH) - && (cdm->pos.cookie.periph != NULL)) - return(xptperiphtraverse(device, - (struct cam_periph *)cdm->pos.cookie.periph, - xptedtperiphfunc, arg)); - else - return(xptperiphtraverse(device, NULL, xptedtperiphfunc, arg)); + && (cdm->pos.cookie.periph != NULL)) { + if (cdm->pos.generations[CAM_PERIPH_GENERATION] != + device->generation) { + mtx_unlock(&bus->eb_mtx); + xpt_unlock_buses(); + cdm->status = CAM_DEV_MATCH_LIST_CHANGED; + return(0); + } + periph = (struct cam_periph *)cdm->pos.cookie.periph; + periph->refcount++; + } else + periph = NULL; + mtx_unlock(&bus->eb_mtx); + xpt_unlock_buses(); + + return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg)); } static int @@ -1858,6 +1842,7 @@ xptedtperiphfunc(struct cam_periph *periph, void *arg) static int xptedtmatch(struct ccb_dev_match *cdm) { + struct cam_eb *bus; int ret; cdm->num_matches = 0; @@ -1866,19 +1851,22 @@ xptedtmatch(struct ccb_dev_match *cdm) * Check the bus list generation. If it has changed, the user * needs to reset everything and start over. */ + xpt_lock_buses(); if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (cdm->pos.generations[CAM_BUS_GENERATION] != 0) - && (cdm->pos.generations[CAM_BUS_GENERATION] != xsoftc.bus_generation)) { - cdm->status = CAM_DEV_MATCH_LIST_CHANGED; - return(0); - } + && (cdm->pos.cookie.bus != NULL)) { + if (cdm->pos.generations[CAM_BUS_GENERATION] != + xsoftc.bus_generation) { + xpt_unlock_buses(); + cdm->status = CAM_DEV_MATCH_LIST_CHANGED; + return(0); + } + bus = (struct cam_eb *)cdm->pos.cookie.bus; + bus->refcount++; + } else + bus = NULL; + xpt_unlock_buses(); - if ((cdm->pos.position_type & CAM_DEV_POS_BUS) - && (cdm->pos.cookie.bus != NULL)) - ret = xptbustraverse((struct cam_eb *)cdm->pos.cookie.bus, - xptedtbusfunc, cdm); - else - ret = xptbustraverse(NULL, xptedtbusfunc, cdm); + ret = xptbustraverse(bus, xptedtbusfunc, cdm); /* * If we get back 0, that means that we had to stop before fully @@ -1895,29 +1883,29 @@ xptedtmatch(struct ccb_dev_match *cdm) static int xptplistpdrvfunc(struct periph_driver **pdrv, void *arg) { + struct cam_periph *periph; struct ccb_dev_match *cdm; cdm = (struct ccb_dev_match *)arg; + xpt_lock_buses(); if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR) && (cdm->pos.cookie.pdrv == pdrv) && (cdm->pos.position_type & CAM_DEV_POS_PERIPH) - && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0) - && (cdm->pos.generations[CAM_PERIPH_GENERATION] != - (*pdrv)->generation)) { - cdm->status = CAM_DEV_MATCH_LIST_CHANGED; - return(0); - } + && (cdm->pos.cookie.periph != NULL)) { + if (cdm->pos.generations[CAM_PERIPH_GENERATION] != + (*pdrv)->generation) { + xpt_unlock_buses(); + cdm->status = CAM_DEV_MATCH_LIST_CHANGED; + return(0); + } + periph = (struct cam_periph *)cdm->pos.cookie.periph; + periph->refcount++; + } else + periph = NULL; + xpt_unlock_buses(); - if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR) - && (cdm->pos.cookie.pdrv == pdrv) - && (cdm->pos.position_type & CAM_DEV_POS_PERIPH) - && (cdm->pos.cookie.periph != NULL)) - return(xptpdperiphtraverse(pdrv, - (struct cam_periph *)cdm->pos.cookie.periph, - xptplistperiphfunc, arg)); - else - return(xptpdperiphtraverse(pdrv, NULL,xptplistperiphfunc, arg)); + return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg)); } static int @@ -2066,35 +2054,31 @@ xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg) int retval; retval = 1; - - xpt_lock_buses(); - for (bus = (start_bus ? start_bus : TAILQ_FIRST(&xsoftc.xpt_busses)); - bus != NULL; - bus = next_bus) { - + if (start_bus) + bus = start_bus; + else { + xpt_lock_buses(); + bus = TAILQ_FIRST(&xsoftc.xpt_busses); + if (bus == NULL) { + xpt_unlock_buses(); + return (retval); + } bus->refcount++; - - /* - * XXX The locking here is obviously very complex. We - * should work to simplify it. - */ xpt_unlock_buses(); - CAM_SIM_LOCK(bus->sim); + } + for (; bus != NULL; bus = next_bus) { retval = tr_func(bus, arg); - CAM_SIM_UNLOCK(bus->sim); - + if (retval == 0) { + xpt_release_bus(bus); + break; + } xpt_lock_buses(); next_bus = TAILQ_NEXT(bus, links); + if (next_bus) + next_bus->refcount++; xpt_unlock_buses(); - xpt_release_bus(bus); - - if (retval == 0) - return(retval); - xpt_lock_buses(); } - xpt_unlock_buses(); - return(retval); } @@ -2105,24 +2089,32 @@ xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target, struct cam_et *target, *next_target; int retval; - mtx_assert(bus->sim->mtx, MA_OWNED); retval = 1; - for (target = (start_target ? start_target : - TAILQ_FIRST(&bus->et_entries)); - target != NULL; target = next_target) { - + if (start_target) + target = start_target; + else { + mtx_lock(&bus->eb_mtx); + target = TAILQ_FIRST(&bus->et_entries); + if (target == NULL) { + mtx_unlock(&bus->eb_mtx); + return (retval); + } target->refcount++; - + mtx_unlock(&bus->eb_mtx); + } + for (; target != NULL; target = next_target) { retval = tr_func(target, arg); - + if (retval == 0) { + xpt_release_target(target); + break; + } + mtx_lock(&bus->eb_mtx); next_target = TAILQ_NEXT(target, links); - + if (next_target) + next_target->refcount++; + mtx_unlock(&bus->eb_mtx); xpt_release_target(target); - - if (retval == 0) - return(retval); } - return(retval); } @@ -2130,36 +2122,39 @@ static int xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device, xpt_devicefunc_t *tr_func, void *arg) { + struct cam_eb *bus; struct cam_ed *device, *next_device; int retval; - mtx_assert(target->bus->sim->mtx, MA_OWNED); retval = 1; - for (device = (start_device ? start_device : - TAILQ_FIRST(&target->ed_entries)); - device != NULL; - device = next_device) { - - /* - * Hold a reference so the current device does not go away - * on us. - */ + bus = target->bus; + if (start_device) + device = start_device; + else { + mtx_lock(&bus->eb_mtx); + device = TAILQ_FIRST(&target->ed_entries); + if (device == NULL) { + mtx_unlock(&bus->eb_mtx); + return (retval); + } device->refcount++; - + mtx_unlock(&bus->eb_mtx); + } + for (; device != NULL; device = next_device) { + mtx_lock(&device->device_mtx); retval = tr_func(device, arg); - - /* - * Grab our next pointer before we release the current - * device. - */ + mtx_unlock(&device->device_mtx); + if (retval == 0) { + xpt_release_device(device); + break; + } + mtx_lock(&bus->eb_mtx); next_device = TAILQ_NEXT(device, links); - + if (next_device) + next_device->refcount++; + mtx_unlock(&bus->eb_mtx); xpt_release_device(device); - - if (retval == 0) - return(retval); } - return(retval); } @@ -2167,56 +2162,48 @@ static int xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph, xpt_periphfunc_t *tr_func, void *arg) { + struct cam_eb *bus; struct cam_periph *periph, *next_periph; int retval; retval = 1; - mtx_assert(device->sim->mtx, MA_OWNED); - xpt_lock_buses(); - for (periph = (start_periph ? start_periph : - SLIST_FIRST(&device->periphs)); - periph != NULL; - periph = next_periph) { - - - /* - * In this case, we want to show peripherals that have been - * invalidated, but not peripherals that are scheduled to - * be freed. So instead of calling cam_periph_acquire(), - * which will fail if the periph has been invalidated, we - * just check for the free flag here. If it is in the - * process of being freed, we skip to the next periph. - */ - if (periph->flags & CAM_PERIPH_FREE) { - next_periph = SLIST_NEXT(periph, periph_links); - continue; + bus = device->target->bus; + if (start_periph) + periph = start_periph; + else { + xpt_lock_buses(); + mtx_lock(&bus->eb_mtx); + periph = SLIST_FIRST(&device->periphs); + while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0) + periph = SLIST_NEXT(periph, periph_links); + if (periph == NULL) { + mtx_unlock(&bus->eb_mtx); + xpt_unlock_buses(); + return (retval); } - - /* - * Acquire a reference to this periph while we call the - * traversal function, so it can't go away. - */ periph->refcount++; - + mtx_unlock(&bus->eb_mtx); + xpt_unlock_buses(); + } + for (; periph != NULL; periph = next_periph) { retval = tr_func(periph, arg); - - /* - * Grab the next peripheral before we release this one, so - * our next pointer is still valid. - */ + if (retval == 0) { + cam_periph_release_locked(periph); + break; + } + xpt_lock_buses(); + mtx_lock(&bus->eb_mtx); next_periph = SLIST_NEXT(periph, periph_links); - - cam_periph_release_locked_buses(periph); - - if (retval == 0) - goto bailout_done; + while (next_periph != NULL && + (next_periph->flags & CAM_PERIPH_FREE) != 0) + next_periph = SLIST_NEXT(periph, periph_links); + if (next_periph) + next_periph->refcount++; + mtx_unlock(&bus->eb_mtx); + xpt_unlock_buses(); + cam_periph_release_locked(periph); } - -bailout_done: - - xpt_unlock_buses(); - return(retval); } @@ -2254,57 +2241,42 @@ xptpdperiphtraverse(struct periph_driver **pdrv, xpt_periphfunc_t *tr_func, void *arg) { struct cam_periph *periph, *next_periph; - struct cam_sim *sim; int retval; retval = 1; - xpt_lock_buses(); - for (periph = (start_periph ? start_periph : - TAILQ_FIRST(&(*pdrv)->units)); periph != NULL; - periph = next_periph) { - - - /* - * In this case, we want to show peripherals that have been - * invalidated, but not peripherals that are scheduled to - * be freed. So instead of calling cam_periph_acquire(), - * which will fail if the periph has been invalidated, we - * just check for the free flag here. If it is free, we - * skip to the next periph. - */ - if (periph->flags & CAM_PERIPH_FREE) { - next_periph = TAILQ_NEXT(periph, unit_links); - continue; + if (start_periph) + periph = start_periph; + else { + xpt_lock_buses(); + periph = TAILQ_FIRST(&(*pdrv)->units); + while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0) + periph = TAILQ_NEXT(periph, unit_links); + if (periph == NULL) { + xpt_unlock_buses(); + return (retval); } - - /* - * Acquire a reference to this periph while we call the - * traversal function, so it can't go away. - */ periph->refcount++; - sim = periph->sim; xpt_unlock_buses(); - CAM_SIM_LOCK(sim); - xpt_lock_buses(); + } + for (; periph != NULL; periph = next_periph) { + cam_periph_lock(periph); retval = tr_func(periph, arg); - - /* - * Grab the next peripheral before we release this one, so - * our next pointer is still valid. - */ + cam_periph_unlock(periph); + if (retval == 0) { + cam_periph_release(periph); + break; + } + xpt_lock_buses(); next_periph = TAILQ_NEXT(periph, unit_links); - - cam_periph_release_locked_buses(periph); - CAM_SIM_UNLOCK(sim); - - if (retval == 0) - goto bailout_done; + while (next_periph != NULL && + (next_periph->flags & CAM_PERIPH_FREE) != 0) + next_periph = TAILQ_NEXT(periph, unit_links); + if (next_periph) + next_periph->refcount++; + xpt_unlock_buses(); + cam_periph_release(periph); } -bailout_done: - - xpt_unlock_buses(); - return(retval); } @@ -2450,12 +2422,14 @@ xptsetasyncbusfunc(struct cam_eb *bus, void *arg) bus->path_id, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); + xpt_path_lock(&path); xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); csa->callback(csa->callback_arg, AC_PATH_REGISTERED, &path, &cpi); + xpt_path_unlock(&path); xpt_release_path(&path); return(1); @@ -2475,6 +2449,8 @@ void xpt_action_default(union ccb *start_ccb) { struct cam_path *path; + struct cam_sim *sim; + int lock; path = start_ccb->ccb_h.path; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n")); @@ -2523,14 +2499,18 @@ xpt_action_default(union ccb *start_ccb) case XPT_RESET_DEV: case XPT_ENG_EXEC: case XPT_SMP_IO: + { + struct cam_devq *devq; + + devq = path->bus->sim->devq; + mtx_lock(&devq->send_mtx); cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb); - if (xpt_schedule_devq(path->bus->sim->devq, path->device)) - xpt_run_devq(path->bus->sim->devq); + if (xpt_schedule_devq(devq, path->device) != 0) + xpt_run_devq(devq); + mtx_unlock(&devq->send_mtx); break; + } case XPT_CALC_GEOMETRY: - { - struct cam_sim *sim; - /* Filter out garbage */ if (start_ccb->ccg.block_size == 0 || start_ccb->ccg.volume_size == 0) { @@ -2558,10 +2538,7 @@ xpt_action_default(union ccb *start_ccb) break; } #endif - sim = path->bus->sim; - (*(sim->sim_action))(sim, start_ccb); - break; - } + goto call_sim; case XPT_ABORT: { union ccb* abort_ccb; @@ -2622,21 +2599,18 @@ xpt_action_default(union ccb *start_ccb) case XPT_NOTIFY_ACKNOWLEDGE: case XPT_GET_SIM_KNOB: case XPT_SET_SIM_KNOB: - { - struct cam_sim *sim; - - sim = path->bus->sim; - (*(sim->sim_action))(sim, start_ccb); - break; - } + case XPT_GET_TRAN_SETTINGS: + case XPT_SET_TRAN_SETTINGS: case XPT_PATH_INQ: - { - struct cam_sim *sim; - +call_sim: sim = path->bus->sim; + lock = (mtx_owned(sim->mtx) == 0); + if (lock) + CAM_SIM_LOCK(sim); (*(sim->sim_action))(sim, start_ccb); + if (lock) + CAM_SIM_UNLOCK(sim); break; - } case XPT_PATH_STATS: start_ccb->cpis.last_reset = path->bus->last_reset; start_ccb->ccb_h.status = CAM_REQ_CMP; @@ -2798,11 +2772,6 @@ xpt_action_default(union ccb *start_ccb) position_type = CAM_DEV_POS_PDRV; } - /* - * Note that we drop the SIM lock here, because the EDT - * traversal code needs to do its own locking. - */ - CAM_SIM_UNLOCK(xpt_path_sim(cdm->ccb_h.path)); switch(position_type & CAM_DEV_POS_TYPEMASK) { case CAM_DEV_POS_EDT: xptedtmatch(cdm); @@ -2814,7 +2783,6 @@ xpt_action_default(union ccb *start_ccb) cdm->status = CAM_DEV_MATCH_ERROR; break; } - CAM_SIM_LOCK(xpt_path_sim(cdm->ccb_h.path)); if (cdm->status == CAM_DEV_MATCH_ERROR) start_ccb->ccb_h.status = CAM_REQ_CMP_ERR; @@ -2869,6 +2837,8 @@ xpt_action_default(union ccb *start_ccb) break; } cur_entry->event_enable = csa->event_enable; + cur_entry->event_lock = + mtx_owned(path->bus->sim->mtx) ? 1 : 0; cur_entry->callback_arg = csa->callback_arg; cur_entry->callback = csa->callback; SLIST_INSERT_HEAD(async_head, cur_entry, links); @@ -2903,6 +2873,7 @@ xpt_action_default(union ccb *start_ccb) } } + mtx_lock(&dev->sim->devq->send_mtx); if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) { if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) { @@ -2955,6 +2926,7 @@ xpt_action_default(union ccb *start_ccb) start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE; } } + mtx_unlock(&dev->sim->devq->send_mtx); if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0) xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE); @@ -2964,7 +2936,6 @@ xpt_action_default(union ccb *start_ccb) } case XPT_DEBUG: { struct cam_path *oldpath; - struct cam_sim *oldsim; /* Check that all request bits are supported. */ if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) { @@ -2974,15 +2945,9 @@ xpt_action_default(union ccb *start_ccb) cam_dflags = CAM_DEBUG_NONE; if (cam_dpath != NULL) { - /* To release the old path we must hold proper lock. */ oldpath = cam_dpath; cam_dpath = NULL; - oldsim = xpt_path_sim(oldpath); - CAM_SIM_UNLOCK(xpt_path_sim(start_ccb->ccb_h.path)); - CAM_SIM_LOCK(oldsim); xpt_free_path(oldpath); - CAM_SIM_UNLOCK(oldsim); - CAM_SIM_LOCK(xpt_path_sim(start_ccb->ccb_h.path)); } if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) { if (xpt_create_path(&cam_dpath, NULL, @@ -3029,39 +2994,41 @@ xpt_polled_action(union ccb *start_ccb) struct cam_devq *devq; struct cam_ed *dev; - timeout = start_ccb->ccb_h.timeout * 10; sim = start_ccb->ccb_h.path->bus->sim; devq = sim->devq; dev = start_ccb->ccb_h.path->device; - mtx_assert(sim->mtx, MA_OWNED); - - /* Don't use ISR for this SIM while polling. */ - sim->flags |= CAM_SIM_POLLED; + mtx_unlock(&dev->device_mtx); /* * Steal an opening so that no other queued requests * can get it before us while we simulate interrupts. */ + mtx_lock(&devq->send_mtx); dev->ccbq.devq_openings--; dev->ccbq.dev_openings--; - - while(((devq != NULL && devq->send_openings <= 0) || - dev->ccbq.dev_openings < 0) && (--timeout > 0)) { + while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) && + (--timeout > 0)) { + mtx_unlock(&devq->send_mtx); DELAY(100); + CAM_SIM_LOCK(sim); (*(sim->sim_poll))(sim); - camisr_runqueue(sim); + CAM_SIM_UNLOCK(sim); + camisr_runqueue(); + mtx_lock(&devq->send_mtx); } - dev->ccbq.devq_openings++; dev->ccbq.dev_openings++; + mtx_unlock(&devq->send_mtx); if (timeout != 0) { xpt_action(start_ccb); while(--timeout > 0) { + CAM_SIM_LOCK(sim); (*(sim->sim_poll))(sim); - camisr_runqueue(sim); + CAM_SIM_UNLOCK(sim); + camisr_runqueue(); if ((start_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG) break; @@ -3080,8 +3047,7 @@ xpt_polled_action(union ccb *start_ccb) start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; } - /* We will use CAM ISR for this SIM again. */ - sim->flags &= ~CAM_SIM_POLLED; + mtx_lock(&dev->device_mtx); } /* @@ -3089,38 +3055,14 @@ xpt_polled_action(union ccb *start_ccb) * target device has space for more transactions. */ void -xpt_schedule(struct cam_periph *perph, u_int32_t new_priority) +xpt_schedule(struct cam_periph *periph, u_int32_t new_priority) { - struct cam_ed *device; - int runq = 0; - - mtx_assert(perph->sim->mtx, MA_OWNED); - CAM_DEBUG(perph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n")); - device = perph->path->device; - if (periph_is_queued(perph)) { - /* Simply reorder based on new priority */ - CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE, - (" change priority to %d\n", new_priority)); - if (new_priority < perph->pinfo.priority) { - camq_change_priority(&device->drvq, - perph->pinfo.index, - new_priority); - runq = 1; - } - } else { - /* New entry on the queue */ - CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE, - (" added periph to queue\n")); - perph->pinfo.priority = new_priority; - perph->pinfo.generation = ++device->drvq.generation; - camq_insert(&device->drvq, &perph->pinfo); - runq = 1; - } - if (runq != 0) { - CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE, - (" calling xpt_run_dev_allocq\n")); - xpt_run_dev_allocq(device); + CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n")); + cam_periph_assert(periph, MA_OWNED); + if (new_priority < periph->scheduled_priority) { + periph->scheduled_priority = new_priority; + xpt_run_allocq(periph, 0); } } @@ -3133,7 +3075,7 @@ xpt_schedule(struct cam_periph *perph, u_int32_t new_priority) * started the queue, return 0 so the caller doesn't attempt * to run the queue. */ -int +static int xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo, u_int32_t new_priority) { @@ -3173,51 +3115,79 @@ xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo, } static void -xpt_run_dev_allocq(struct cam_ed *device) +xpt_run_allocq_task(void *context, int pending) { - struct camq *drvq; + struct cam_periph *periph = context; - if (device->ccbq.devq_allocating) + cam_periph_lock(periph); + periph->flags &= ~CAM_PERIPH_RUN_TASK; + xpt_run_allocq(periph, 1); + cam_periph_unlock(periph); + cam_periph_release(periph); +} + +static void +xpt_run_allocq(struct cam_periph *periph, int sleep) +{ + struct cam_ed *device; + union ccb *ccb; + uint32_t prio; + + cam_periph_assert(periph, MA_OWNED); + if (periph->periph_allocating) return; - device->ccbq.devq_allocating = 1; - CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_dev_allocq(%p)\n", device)); - drvq = &device->drvq; - while ((drvq->entries > 0) && - (device->ccbq.devq_openings > 0 || - CAMQ_GET_PRIO(drvq) <= CAM_PRIORITY_OOB) && - (device->ccbq.queue.qfrozen_cnt == 0)) { - union ccb *work_ccb; - struct cam_periph *drv; - - KASSERT(drvq->entries > 0, ("xpt_run_dev_allocq: " - "Device on queue without any work to do")); - if ((work_ccb = xpt_get_ccb(device)) != NULL) { - drv = (struct cam_periph*)camq_remove(drvq, CAMQ_HEAD); - xpt_setup_ccb(&work_ccb->ccb_h, drv->path, - drv->pinfo.priority); + periph->periph_allocating = 1; + CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph)); + device = periph->path->device; + ccb = NULL; +restart: + while ((prio = min(periph->scheduled_priority, + periph->immediate_priority)) != CAM_PRIORITY_NONE && + (periph->periph_allocated - (ccb != NULL ? 1 : 0) < + device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) { + + if (ccb == NULL && + (ccb = xpt_get_ccb_nowait(periph)) == NULL) { + if (sleep) { + ccb = xpt_get_ccb(periph); + goto restart; + } + if (periph->flags & CAM_PERIPH_RUN_TASK) + break; + xpt_lock_buses(); + periph->refcount++; /* Unconditionally acquire */ + xpt_unlock_buses(); + periph->flags |= CAM_PERIPH_RUN_TASK; + taskqueue_enqueue(xsoftc.xpt_taskq, + &periph->periph_run_task); + break; + } + xpt_setup_ccb(&ccb->ccb_h, periph->path, prio); + if (prio == periph->immediate_priority) { + periph->immediate_priority = CAM_PRIORITY_NONE; CAM_DEBUG_PRINT(CAM_DEBUG_XPT, - ("calling periph start\n")); - drv->periph_start(drv, work_ccb); + ("waking cam_periph_getccb()\n")); + SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h, + periph_links.sle); + wakeup(&periph->ccb_list); } else { - /* - * Malloc failure in alloc_ccb - */ - /* - * XXX add us to a list to be run from free_ccb - * if we don't have any ccbs active on this - * device queue otherwise we may never get run - * again. - */ - break; + periph->scheduled_priority = CAM_PRIORITY_NONE; + CAM_DEBUG_PRINT(CAM_DEBUG_XPT, + ("calling periph_start()\n")); + periph->periph_start(periph, ccb); } + ccb = NULL; } - device->ccbq.devq_allocating = 0; + if (ccb != NULL) + xpt_release_ccb(ccb); + periph->periph_allocating = 0; } static void xpt_run_devq(struct cam_devq *devq) { char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1]; + int lock; CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n")); @@ -3225,14 +3195,12 @@ xpt_run_devq(struct cam_devq *devq) while ((devq->send_queue.entries > 0) && (devq->send_openings > 0) && (devq->send_queue.qfrozen_cnt <= 1)) { - struct cam_ed_qinfo *qinfo; struct cam_ed *device; union ccb *work_ccb; struct cam_sim *sim; - qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->send_queue, + device = (struct cam_ed *)camq_remove(&devq->send_queue, CAMQ_HEAD); - device = qinfo->device; CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("running device %p\n", device)); @@ -3244,7 +3212,7 @@ xpt_run_devq(struct cam_devq *devq) if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) { - mtx_lock(&xsoftc.xpt_lock); + mtx_lock(&xsoftc.xpt_highpower_lock); if (xsoftc.num_highpower <= 0) { /* * We got a high power command, but we @@ -3252,12 +3220,11 @@ xpt_run_devq(struct cam_devq *devq) * the device queue until we have a slot * available. */ - xpt_freeze_devq(work_ccb->ccb_h.path, 1); - STAILQ_INSERT_TAIL(&xsoftc.highpowerq, - work_ccb->ccb_h.path->device, + xpt_freeze_devq_device(device, 1); + STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device, highpowerq_entry); - mtx_unlock(&xsoftc.xpt_lock); + mtx_unlock(&xsoftc.xpt_highpower_lock); continue; } else { /* @@ -3266,15 +3233,14 @@ xpt_run_devq(struct cam_devq *devq) */ xsoftc.num_highpower--; } - mtx_unlock(&xsoftc.xpt_lock); + mtx_unlock(&xsoftc.xpt_highpower_lock); } cam_ccbq_remove_ccb(&device->ccbq, work_ccb); cam_ccbq_send_ccb(&device->ccbq, work_ccb); - devq->send_openings--; devq->send_active++; - xpt_schedule_devq(devq, device); + mtx_unlock(&devq->send_mtx); if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) { /* @@ -3318,13 +3284,18 @@ xpt_run_devq(struct cam_devq *devq) } /* - * Device queues can be shared among multiple sim instances - * that reside on different busses. Use the SIM in the queue - * CCB's path, rather than the one in the bus that was passed - * into this function. + * Device queues can be shared among multiple SIM instances + * that reside on different busses. Use the SIM from the + * queued device, rather than the one from the calling bus. */ - sim = work_ccb->ccb_h.path->bus->sim; + sim = device->sim; + lock = (mtx_owned(sim->mtx) == 0); + if (lock) + CAM_SIM_LOCK(sim); (*(sim->sim_action))(sim, work_ccb); + if (lock) + CAM_SIM_UNLOCK(sim); + mtx_lock(&devq->send_mtx); } devq->send_queue.qfrozen_cnt--; } @@ -3400,26 +3371,9 @@ xpt_create_path_unlocked(struct cam_path **new_path_ptr, struct cam_periph *periph, path_id_t path_id, target_id_t target_id, lun_id_t lun_id) { - struct cam_path *path; - struct cam_eb *bus = NULL; - cam_status status; - - path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_WAITOK); - bus = xpt_find_bus(path_id); - if (bus != NULL) - CAM_SIM_LOCK(bus->sim); - status = xpt_compile_path(path, periph, path_id, target_id, lun_id); - if (bus != NULL) { - CAM_SIM_UNLOCK(bus->sim); - xpt_release_bus(bus); - } - if (status != CAM_REQ_CMP) { - free(path, M_CAMPATH); - path = NULL; - } - *new_path_ptr = path; - return (status); + return (xpt_create_path(new_path_ptr, periph, path_id, target_id, + lun_id)); } cam_status @@ -3443,6 +3397,8 @@ xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph, if (bus == NULL) { status = CAM_PATH_INVALID; } else { + xpt_lock_buses(); + mtx_lock(&bus->eb_mtx); target = xpt_find_target(bus, target_id); if (target == NULL) { /* Create one */ @@ -3455,6 +3411,7 @@ xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph, target = new_target; } } + xpt_unlock_buses(); if (target != NULL) { device = xpt_find_device(target, lun_id); if (device == NULL) { @@ -3472,6 +3429,7 @@ xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph, } } } + mtx_unlock(&bus->eb_mtx); } /* @@ -3494,6 +3452,32 @@ xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph, return (status); } +cam_status +xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path) +{ + struct cam_path *new_path; + + new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT); + if (new_path == NULL) + return(CAM_RESRC_UNAVAIL); + xpt_copy_path(new_path, path); + *new_path_ptr = new_path; + return (CAM_REQ_CMP); +} + +void +xpt_copy_path(struct cam_path *new_path, struct cam_path *path) +{ + + *new_path = *path; + if (path->bus != NULL) + xpt_acquire_bus(path->bus); + if (path->target != NULL) + xpt_acquire_target(path->target); + if (path->device != NULL) + xpt_acquire_device(path->device); +} + void xpt_release_path(struct cam_path *path) { @@ -3688,11 +3672,6 @@ xpt_path_string(struct cam_path *path, char *str, size_t str_len) { struct sbuf sb; -#ifdef INVARIANTS - if (path != NULL && path->bus != NULL) - mtx_assert(path->bus->sim->mtx, MA_OWNED); -#endif - sbuf_new(&sb, str, str_len, 0); if (path == NULL) @@ -3761,7 +3740,6 @@ xpt_path_sim(struct cam_path *path) struct cam_periph* xpt_path_periph(struct cam_path *path) { - mtx_assert(path->bus->sim->mtx, MA_OWNED); return (path->periph); } @@ -3814,28 +3792,18 @@ xpt_path_legacy_ata_id(struct cam_path *path) void xpt_release_ccb(union ccb *free_ccb) { - struct cam_path *path; struct cam_ed *device; - struct cam_eb *bus; - struct cam_sim *sim; + struct cam_periph *periph; CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n")); - path = free_ccb->ccb_h.path; - device = path->device; - bus = path->bus; - sim = bus->sim; - - mtx_assert(sim->mtx, MA_OWNED); + xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED); + device = free_ccb->ccb_h.path->device; + periph = free_ccb->ccb_h.path->periph; + xpt_free_ccb(free_ccb); + periph->periph_allocated--; cam_ccbq_release_opening(&device->ccbq); - if (sim->ccb_count > sim->max_ccbs) { - xpt_free_ccb(free_ccb); - sim->ccb_count--; - } else { - SLIST_INSERT_HEAD(&sim->ccb_freeq, &free_ccb->ccb_h, - xpt_links.sle); - } - xpt_run_dev_allocq(device); + xpt_run_allocq(periph, 0); } /* Functions accessed by SIM drivers */ @@ -3868,12 +3836,13 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus) sim->bus_id = bus; new_bus = (struct cam_eb *)malloc(sizeof(*new_bus), - M_CAMXPT, M_NOWAIT); + M_CAMXPT, M_NOWAIT|M_ZERO); if (new_bus == NULL) { /* Couldn't satisfy request */ return (CAM_RESRC_UNAVAIL); } + mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF); TAILQ_INIT(&new_bus->et_entries); cam_sim_hold(sim); new_bus->sim = sim; @@ -4079,80 +4048,138 @@ xpt_async_string(u_int32_t async_code) return ("AC_UNKNOWN"); } -void -xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg) +static int +xpt_async_size(u_int32_t async_code) { - struct cam_eb *bus; - struct cam_et *target, *next_target; - struct cam_ed *device, *next_device; - mtx_assert(path->bus->sim->mtx, MA_OWNED); - CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO, - ("xpt_async(%s)\n", xpt_async_string(async_code))); + switch (async_code) { + case AC_BUS_RESET: return (0); + case AC_UNSOL_RESEL: return (0); + case AC_SCSI_AEN: return (0); + case AC_SENT_BDR: return (0); + case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq)); + case AC_PATH_DEREGISTERED: return (0); + case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev)); + case AC_LOST_DEVICE: return (0); + case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings)); + case AC_INQ_CHANGED: return (0); + case AC_GETDEV_CHANGED: return (0); + case AC_CONTRACT: return (sizeof(struct ac_contract)); + case AC_ADVINFO_CHANGED: return (-1); + case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio)); + } + return (0); +} + +static int +xpt_async_process_dev(struct cam_ed *device, void *arg) +{ + union ccb *ccb = arg; + struct cam_path *path = ccb->ccb_h.path; + void *async_arg = ccb->casync.async_arg_ptr; + u_int32_t async_code = ccb->casync.async_code; + int relock; + + if (path->device != device + && path->device->lun_id != CAM_LUN_WILDCARD + && device->lun_id != CAM_LUN_WILDCARD) + return (1); /* - * Most async events come from a CAM interrupt context. In - * a few cases, the error recovery code at the peripheral layer, - * which may run from our SWI or a process context, may signal - * deferred events with a call to xpt_async. + * The async callback could free the device. + * If it is a broadcast async, it doesn't hold + * device reference, so take our own reference. */ + xpt_acquire_device(device); - bus = path->bus; + /* + * If async for specific device is to be delivered to + * the wildcard client, take the specific device lock. + * XXX: We may need a way for client to specify it. + */ + if ((device->lun_id == CAM_LUN_WILDCARD && + path->device->lun_id != CAM_LUN_WILDCARD) || + (device->target->target_id == CAM_TARGET_WILDCARD && + path->target->target_id != CAM_TARGET_WILDCARD) || + (device->target->bus->path_id == CAM_BUS_WILDCARD && + path->target->bus->path_id != CAM_BUS_WILDCARD)) { + mtx_unlock(&device->device_mtx); + xpt_path_lock(path); + relock = 1; + } else + relock = 0; - if (async_code == AC_BUS_RESET) { - /* Update our notion of when the last reset occurred */ - microtime(&bus->last_reset); - } + (*(device->target->bus->xport->async))(async_code, + device->target->bus, device->target, device, async_arg); + xpt_async_bcast(&device->asyncs, async_code, path, async_arg); - for (target = TAILQ_FIRST(&bus->et_entries); - target != NULL; - target = next_target) { + if (relock) { + xpt_path_unlock(path); + mtx_lock(&device->device_mtx); + } + xpt_release_device(device); + return (1); +} - next_target = TAILQ_NEXT(target, links); +static int +xpt_async_process_tgt(struct cam_et *target, void *arg) +{ + union ccb *ccb = arg; + struct cam_path *path = ccb->ccb_h.path; - if (path->target != target - && path->target->target_id != CAM_TARGET_WILDCARD - && target->target_id != CAM_TARGET_WILDCARD) - continue; + if (path->target != target + && path->target->target_id != CAM_TARGET_WILDCARD + && target->target_id != CAM_TARGET_WILDCARD) + return (1); - if (async_code == AC_SENT_BDR) { - /* Update our notion of when the last reset occurred */ - microtime(&path->target->last_reset); - } + if (ccb->casync.async_code == AC_SENT_BDR) { + /* Update our notion of when the last reset occurred */ + microtime(&target->last_reset); + } - for (device = TAILQ_FIRST(&target->ed_entries); - device != NULL; - device = next_device) { + return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb)); +} - next_device = TAILQ_NEXT(device, links); +static void +xpt_async_process(struct cam_periph *periph, union ccb *ccb) +{ + struct cam_eb *bus; + struct cam_path *path; + void *async_arg; + u_int32_t async_code; - if (path->device != device - && path->device->lun_id != CAM_LUN_WILDCARD - && device->lun_id != CAM_LUN_WILDCARD) - continue; - /* - * The async callback could free the device. - * If it is a broadcast async, it doesn't hold - * device reference, so take our own reference. - */ - xpt_acquire_device(device); - (*(bus->xport->async))(async_code, bus, - target, device, - async_arg); + path = ccb->ccb_h.path; + async_code = ccb->casync.async_code; + async_arg = ccb->casync.async_arg_ptr; + CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO, + ("xpt_async(%s)\n", xpt_async_string(async_code))); + bus = path->bus; - xpt_async_bcast(&device->asyncs, async_code, - path, async_arg); - xpt_release_device(device); - } + if (async_code == AC_BUS_RESET) { + /* Update our notion of when the last reset occurred */ + microtime(&bus->last_reset); } + xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb); + /* * If this wasn't a fully wildcarded async, tell all * clients that want all async events. */ - if (bus != xpt_periph->path->bus) - xpt_async_bcast(&xpt_periph->path->device->asyncs, async_code, - path, async_arg); + if (bus != xpt_periph->path->bus) { + xpt_path_lock(xpt_periph->path); + xpt_async_process_dev(xpt_periph->path->device, ccb); + xpt_path_unlock(xpt_periph->path); + } + + if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD) + xpt_release_devq(path, 1, TRUE); + else + xpt_release_simq(path->bus->sim, TRUE); + if (ccb->casync.async_arg_size > 0) + free(async_arg, M_CAMXPT); + xpt_free_path(path); + xpt_free_ccb(ccb); } static void @@ -4161,6 +4188,7 @@ xpt_async_bcast(struct async_list *async_head, struct cam_path *path, void *async_arg) { struct async_node *cur_entry; + int lock; cur_entry = SLIST_FIRST(async_head); while (cur_entry != NULL) { @@ -4171,72 +4199,163 @@ xpt_async_bcast(struct async_list *async_head, * can delete its async callback entry. */ next_entry = SLIST_NEXT(cur_entry, links); - if ((cur_entry->event_enable & async_code) != 0) + if ((cur_entry->event_enable & async_code) != 0) { + lock = cur_entry->event_lock; + if (lock) + CAM_SIM_LOCK(path->device->sim); cur_entry->callback(cur_entry->callback_arg, async_code, path, async_arg); + if (lock) + CAM_SIM_UNLOCK(path->device->sim); + } cur_entry = next_entry; } } +void +xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg) +{ + union ccb *ccb; + int size; + + ccb = xpt_alloc_ccb_nowait(); + if (ccb == NULL) { + xpt_print(path, "Can't allocate CCB to send %s\n", + xpt_async_string(async_code)); + return; + } + + if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) { + xpt_print(path, "Can't allocate path to send %s\n", + xpt_async_string(async_code)); + xpt_free_ccb(ccb); + return; + } + ccb->ccb_h.path->periph = NULL; + ccb->ccb_h.func_code = XPT_ASYNC; + ccb->ccb_h.cbfcnp = xpt_async_process; + ccb->ccb_h.flags |= CAM_UNLOCKED; + ccb->casync.async_code = async_code; + ccb->casync.async_arg_size = 0; + size = xpt_async_size(async_code); + if (size > 0 && async_arg != NULL) { + ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT); + if (ccb->casync.async_arg_ptr == NULL) { + xpt_print(path, "Can't allocate argument to send %s\n", + xpt_async_string(async_code)); + xpt_free_path(ccb->ccb_h.path); + xpt_free_ccb(ccb); + return; + } + memcpy(ccb->casync.async_arg_ptr, async_arg, size); + ccb->casync.async_arg_size = size; + } else if (size < 0) + ccb->casync.async_arg_size = size; + if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD) + xpt_freeze_devq(path, 1); + else + xpt_freeze_simq(path->bus->sim, 1); + xpt_done(ccb); +} + static void xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, struct cam_ed *device, void *async_arg) { + + /* + * We only need to handle events for real devices. + */ + if (target->target_id == CAM_TARGET_WILDCARD + || device->lun_id == CAM_LUN_WILDCARD) + return; + printf("%s called\n", __func__); } -u_int32_t -xpt_freeze_devq(struct cam_path *path, u_int count) +static uint32_t +xpt_freeze_devq_device(struct cam_ed *dev, u_int count) { - struct cam_ed *dev = path->device; + struct cam_devq *devq; + uint32_t freeze; - mtx_assert(path->bus->sim->mtx, MA_OWNED); - CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq() %u->%u\n", + devq = dev->sim->devq; + mtx_assert(&devq->send_mtx, MA_OWNED); + CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, + ("xpt_freeze_devq_device(%d) %u->%u\n", count, dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count)); - dev->ccbq.queue.qfrozen_cnt += count; + freeze = (dev->ccbq.queue.qfrozen_cnt += count); /* Remove frozen device from sendq. */ - if (device_is_queued(dev)) { - camq_remove(&dev->sim->devq->send_queue, - dev->devq_entry.pinfo.index); - } - return (dev->ccbq.queue.qfrozen_cnt); + if (device_is_queued(dev)) + camq_remove(&devq->send_queue, dev->devq_entry.index); + return (freeze); +} + +u_int32_t +xpt_freeze_devq(struct cam_path *path, u_int count) +{ + struct cam_ed *dev = path->device; + struct cam_devq *devq; + uint32_t freeze; + + devq = dev->sim->devq; + mtx_lock(&devq->send_mtx); + CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count)); + freeze = xpt_freeze_devq_device(dev, count); + mtx_unlock(&devq->send_mtx); + return (freeze); } u_int32_t xpt_freeze_simq(struct cam_sim *sim, u_int count) { + struct cam_devq *devq; + uint32_t freeze; - mtx_assert(sim->mtx, MA_OWNED); - sim->devq->send_queue.qfrozen_cnt += count; - return (sim->devq->send_queue.qfrozen_cnt); + devq = sim->devq; + mtx_lock(&devq->send_mtx); + freeze = (devq->send_queue.qfrozen_cnt += count); + mtx_unlock(&devq->send_mtx); + return (freeze); } static void xpt_release_devq_timeout(void *arg) { - struct cam_ed *device; + struct cam_ed *dev; + struct cam_devq *devq; - device = (struct cam_ed *)arg; - CAM_DEBUG_DEV(device, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n")); - xpt_release_devq_device(device, /*count*/1, /*run_queue*/TRUE); + dev = (struct cam_ed *)arg; + CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n")); + devq = dev->sim->devq; + mtx_assert(&devq->send_mtx, MA_OWNED); + if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE)) + xpt_run_devq(devq); } void xpt_release_devq(struct cam_path *path, u_int count, int run_queue) { + struct cam_ed *dev; + struct cam_devq *devq; - mtx_assert(path->bus->sim->mtx, MA_OWNED); CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n", count, run_queue)); - xpt_release_devq_device(path->device, count, run_queue); + dev = path->device; + devq = dev->sim->devq; + mtx_lock(&devq->send_mtx); + if (xpt_release_devq_device(dev, count, run_queue)) + xpt_run_devq(dev->sim->devq); + mtx_unlock(&devq->send_mtx); } -void +static int xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue) { + mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED); CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue, dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count)); @@ -4262,34 +4381,32 @@ xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue) callout_stop(&dev->callout); dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING; } - xpt_run_dev_allocq(dev); - if (run_queue == 0) - return; /* * Now that we are unfrozen schedule the * device so any pending transactions are * run. */ - if (xpt_schedule_devq(dev->sim->devq, dev)) - xpt_run_devq(dev->sim->devq); - } + xpt_schedule_devq(dev->sim->devq, dev); + } else + run_queue = 0; + return (run_queue); } void xpt_release_simq(struct cam_sim *sim, int run_queue) { - struct camq *sendq; + struct cam_devq *devq; - mtx_assert(sim->mtx, MA_OWNED); - sendq = &(sim->devq->send_queue); - if (sendq->qfrozen_cnt <= 0) { + devq = sim->devq; + mtx_lock(&devq->send_mtx); + if (devq->send_queue.qfrozen_cnt <= 0) { #ifdef INVARIANTS printf("xpt_release_simq: requested 1 > present %u\n", - sendq->qfrozen_cnt); + devq->send_queue.qfrozen_cnt); #endif } else - sendq->qfrozen_cnt--; - if (sendq->qfrozen_cnt == 0) { + devq->send_queue.qfrozen_cnt--; + if (devq->send_queue.qfrozen_cnt == 0) { /* * If there is a timeout scheduled to release this * sim queue, remove it. The queue frozen count is @@ -4306,6 +4423,7 @@ xpt_release_simq(struct cam_sim *sim, int run_queue) xpt_run_devq(sim->devq); } } + mtx_unlock(&devq->send_mtx); } /* @@ -4323,49 +4441,34 @@ xpt_release_simq_timeout(void *arg) void xpt_done(union ccb *done_ccb) { - struct cam_sim *sim; - int first; + struct cam_doneq *queue; + int run, hash; CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n")); - if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) { - /* - * Queue up the request for handling by our SWI handler - * any of the "non-immediate" type of ccbs. - */ - sim = done_ccb->ccb_h.path->bus->sim; - TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h, - sim_links.tqe); - done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX; - if ((sim->flags & (CAM_SIM_ON_DONEQ | CAM_SIM_POLLED | - CAM_SIM_BATCH)) == 0) { - mtx_lock(&cam_simq_lock); - first = TAILQ_EMPTY(&cam_simq); - TAILQ_INSERT_TAIL(&cam_simq, sim, links); - mtx_unlock(&cam_simq_lock); - sim->flags |= CAM_SIM_ON_DONEQ; - if (first) - swi_sched(cambio_ih, 0); - } - } -} - -void -xpt_batch_start(struct cam_sim *sim) -{ + if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0) + return; - KASSERT((sim->flags & CAM_SIM_BATCH) == 0, ("Batch flag already set")); - sim->flags |= CAM_SIM_BATCH; + hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id + + done_ccb->ccb_h.target_lun) % cam_num_doneqs; + queue = &cam_doneqs[hash]; + mtx_lock(&queue->cam_doneq_mtx); + run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq)); + STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe); + done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX; + mtx_unlock(&queue->cam_doneq_mtx); + if (run) + wakeup(&queue->cam_doneq); } void -xpt_batch_done(struct cam_sim *sim) +xpt_done_direct(union ccb *done_ccb) { - KASSERT((sim->flags & CAM_SIM_BATCH) != 0, ("Batch flag was not set")); - sim->flags &= ~CAM_SIM_BATCH; - if (!TAILQ_EMPTY(&sim->sim_doneq) && - (sim->flags & CAM_SIM_ON_DONEQ) == 0) - camisr_runqueue(sim); + CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n")); + if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0) + return; + + xpt_done_process(&done_ccb->ccb_h); } union ccb * @@ -4398,32 +4501,64 @@ xpt_free_ccb(union ccb *free_ccb) /* * Get a CAM control block for the caller. Charge the structure to the device - * referenced by the path. If the this device has no 'credits' then the - * device already has the maximum number of outstanding operations under way - * and we return NULL. If we don't have sufficient resources to allocate more - * ccbs, we also return NULL. + * referenced by the path. If we don't have sufficient resources to allocate + * more ccbs, we return NULL. */ static union ccb * -xpt_get_ccb(struct cam_ed *device) +xpt_get_ccb_nowait(struct cam_periph *periph) { union ccb *new_ccb; - struct cam_sim *sim; - sim = device->sim; - if ((new_ccb = (union ccb *)SLIST_FIRST(&sim->ccb_freeq)) == NULL) { - new_ccb = xpt_alloc_ccb_nowait(); - if (new_ccb == NULL) { - return (NULL); - } - SLIST_INSERT_HEAD(&sim->ccb_freeq, &new_ccb->ccb_h, - xpt_links.sle); - sim->ccb_count++; - } - cam_ccbq_take_opening(&device->ccbq); - SLIST_REMOVE_HEAD(&sim->ccb_freeq, xpt_links.sle); + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_NOWAIT); + if (new_ccb == NULL) + return (NULL); + periph->periph_allocated++; + cam_ccbq_take_opening(&periph->path->device->ccbq); + return (new_ccb); +} + +static union ccb * +xpt_get_ccb(struct cam_periph *periph) +{ + union ccb *new_ccb; + + cam_periph_unlock(periph); + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_WAITOK); + cam_periph_lock(periph); + periph->periph_allocated++; + cam_ccbq_take_opening(&periph->path->device->ccbq); return (new_ccb); } +union ccb * +cam_periph_getccb(struct cam_periph *periph, u_int32_t priority) +{ + struct ccb_hdr *ccb_h; + + CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n")); + cam_periph_assert(periph, MA_OWNED); + while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL || + ccb_h->pinfo.priority != priority) { + if (priority < periph->immediate_priority) { + periph->immediate_priority = priority; + xpt_run_allocq(periph, 0); + } else + cam_periph_sleep(periph, &periph->ccb_list, PRIBIO, + "cgticb", 0); + } + SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle); + return ((union ccb *)ccb_h); +} + +static void +xpt_acquire_bus(struct cam_eb *bus) +{ + + xpt_lock_buses(); + bus->refcount++; + xpt_unlock_buses(); +} + static void xpt_release_bus(struct cam_eb *bus) { @@ -4434,12 +4569,13 @@ xpt_release_bus(struct cam_eb *bus) xpt_unlock_buses(); return; } - KASSERT(TAILQ_EMPTY(&bus->et_entries), - ("refcount is zero, but target list is not empty")); TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links); xsoftc.bus_generation++; xpt_unlock_buses(); + KASSERT(TAILQ_EMPTY(&bus->et_entries), + ("destroying bus, but target list is not empty")); cam_sim_release(bus->sim); + mtx_destroy(&bus->eb_mtx); free(bus, M_CAMXPT); } @@ -4448,7 +4584,8 @@ xpt_alloc_target(struct cam_eb *bus, target_id_t target_id) { struct cam_et *cur_target, *target; - mtx_assert(bus->sim->mtx, MA_OWNED); + mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED); + mtx_assert(&bus->eb_mtx, MA_OWNED); target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT, M_NOWAIT|M_ZERO); if (target == NULL) @@ -4460,14 +4597,13 @@ xpt_alloc_target(struct cam_eb *bus, target_id_t target_id) target->refcount = 1; target->generation = 0; target->luns = NULL; + mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF); timevalclear(&target->last_reset); /* * Hold a reference to our parent bus so it * will not go away before we do. */ - xpt_lock_buses(); bus->refcount++; - xpt_unlock_buses(); /* Insertion sort into our bus's target list */ cur_target = TAILQ_FIRST(&bus->et_entries); @@ -4483,17 +4619,32 @@ xpt_alloc_target(struct cam_eb *bus, target_id_t target_id) } static void +xpt_acquire_target(struct cam_et *target) +{ + struct cam_eb *bus = target->bus; + + mtx_lock(&bus->eb_mtx); + target->refcount++; + mtx_unlock(&bus->eb_mtx); +} + +static void xpt_release_target(struct cam_et *target) { + struct cam_eb *bus = target->bus; - mtx_assert(target->bus->sim->mtx, MA_OWNED); - if (--target->refcount > 0) + mtx_lock(&bus->eb_mtx); + if (--target->refcount > 0) { + mtx_unlock(&bus->eb_mtx); return; + } + TAILQ_REMOVE(&bus->et_entries, target, links); + bus->generation++; + mtx_unlock(&bus->eb_mtx); KASSERT(TAILQ_EMPTY(&target->ed_entries), - ("refcount is zero, but device list is not empty")); - TAILQ_REMOVE(&target->bus->et_entries, target, links); - target->bus->generation++; - xpt_release_bus(target->bus); + ("destroying target, but device list is not empty")); + xpt_release_bus(bus); + mtx_destroy(&target->luns_mtx); if (target->luns) free(target->luns, M_CAMXPT); free(target, M_CAMXPT); @@ -4511,10 +4662,19 @@ xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target, device->mintags = 1; device->maxtags = 1; - bus->sim->max_ccbs += device->ccbq.devq_openings; return (device); } +static void +xpt_destroy_device(void *context, int pending) +{ + struct cam_ed *device = context; + + mtx_lock(&device->device_mtx); + mtx_destroy(&device->device_mtx); + free(device, M_CAMDEV); +} + struct cam_ed * xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { @@ -4522,10 +4682,12 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) struct cam_devq *devq; cam_status status; - mtx_assert(target->bus->sim->mtx, MA_OWNED); + mtx_assert(&bus->eb_mtx, MA_OWNED); /* Make space for us in the device queue on our bus */ devq = bus->sim->devq; + mtx_lock(&devq->send_mtx); status = cam_devq_resize(devq, devq->send_queue.array_size + 1); + mtx_unlock(&devq->send_mtx); if (status != CAM_REQ_CMP) return (NULL); @@ -4534,19 +4696,12 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) if (device == NULL) return (NULL); - cam_init_pinfo(&device->devq_entry.pinfo); - device->devq_entry.device = device; + cam_init_pinfo(&device->devq_entry); device->target = target; device->lun_id = lun_id; device->sim = bus->sim; - /* Initialize our queues */ - if (camq_init(&device->drvq, 0) != 0) { - free(device, M_CAMDEV); - return (NULL); - } if (cam_ccbq_init(&device->ccbq, bus->sim->max_dev_openings) != 0) { - camq_fini(&device->drvq); free(device, M_CAMDEV); return (NULL); } @@ -4557,7 +4712,14 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) device->tag_delay_count = 0; device->tag_saved_openings = 0; device->refcount = 1; - callout_init_mtx(&device->callout, bus->sim->mtx, 0); + mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF); + callout_init_mtx(&device->callout, &devq->send_mtx, 0); + TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device); + /* + * Hold a reference to our parent bus so it + * will not go away before we do. + */ + target->refcount++; cur_device = TAILQ_FIRST(&target->ed_entries); while (cur_device != NULL && cur_device->lun_id < lun_id) @@ -4566,7 +4728,6 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) TAILQ_INSERT_BEFORE(cur_device, device, links); else TAILQ_INSERT_TAIL(&target->ed_entries, device, links); - target->refcount++; target->generation++; return (device); } @@ -4574,35 +4735,45 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) void xpt_acquire_device(struct cam_ed *device) { + struct cam_eb *bus = device->target->bus; - mtx_assert(device->sim->mtx, MA_OWNED); + mtx_lock(&bus->eb_mtx); device->refcount++; + mtx_unlock(&bus->eb_mtx); } void xpt_release_device(struct cam_ed *device) { + struct cam_eb *bus = device->target->bus; struct cam_devq *devq; - mtx_assert(device->sim->mtx, MA_OWNED); - if (--device->refcount > 0) + mtx_lock(&bus->eb_mtx); + if (--device->refcount > 0) { + mtx_unlock(&bus->eb_mtx); return; + } + + TAILQ_REMOVE(&device->target->ed_entries, device,links); + device->target->generation++; + mtx_unlock(&bus->eb_mtx); + + /* Release our slot in the devq */ + devq = bus->sim->devq; + mtx_lock(&devq->send_mtx); + cam_devq_resize(devq, devq->send_queue.array_size - 1); + mtx_unlock(&devq->send_mtx); KASSERT(SLIST_EMPTY(&device->periphs), - ("refcount is zero, but periphs list is not empty")); - if (device->devq_entry.pinfo.index != CAM_UNQUEUED_INDEX) - panic("Removing device while still queued for ccbs"); + ("destroying device, but periphs list is not empty")); + KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX, + ("destroying device while still queued for ccbs")); if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) callout_stop(&device->callout); - TAILQ_REMOVE(&device->target->ed_entries, device,links); - device->target->generation++; - device->target->bus->sim->max_ccbs -= device->ccbq.devq_openings; - /* Release our slot in the devq */ - devq = device->target->bus->sim->devq; - cam_devq_resize(devq, devq->send_queue.array_size - 1); - camq_fini(&device->drvq); + xpt_release_target(device->target); + cam_ccbq_fini(&device->ccbq); /* * Free allocated memory. free(9) does nothing if the @@ -4614,27 +4785,22 @@ xpt_release_device(struct cam_ed *device) free(device->physpath, M_CAMXPT); free(device->rcap_buf, M_CAMXPT); free(device->serial_num, M_CAMXPT); - - xpt_release_target(device->target); - free(device, M_CAMDEV); + taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task); } u_int32_t xpt_dev_ccbq_resize(struct cam_path *path, int newopenings) { - int diff; int result; struct cam_ed *dev; dev = path->device; - - diff = newopenings - (dev->ccbq.dev_active + dev->ccbq.dev_openings); + mtx_lock(&dev->sim->devq->send_mtx); result = cam_ccbq_resize(&dev->ccbq, newopenings); + mtx_unlock(&dev->sim->devq->send_mtx); if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 || (dev->inq_flags & SID_CmdQue) != 0) dev->tag_saved_openings = newopenings; - /* Adjust the global limit */ - dev->sim->max_ccbs += diff; return (result); } @@ -4661,7 +4827,7 @@ xpt_find_target(struct cam_eb *bus, target_id_t target_id) { struct cam_et *target; - mtx_assert(bus->sim->mtx, MA_OWNED); + mtx_assert(&bus->eb_mtx, MA_OWNED); for (target = TAILQ_FIRST(&bus->et_entries); target != NULL; target = TAILQ_NEXT(target, links)) { @@ -4678,7 +4844,7 @@ xpt_find_device(struct cam_et *target, lun_id_t lun_id) { struct cam_ed *device; - mtx_assert(target->bus->sim->mtx, MA_OWNED); + mtx_assert(&target->bus->eb_mtx, MA_OWNED); for (device = TAILQ_FIRST(&target->ed_entries); device != NULL; device = TAILQ_NEXT(device, links)) { @@ -4758,10 +4924,12 @@ xpt_config(void *arg) /* * Now that interrupts are enabled, go find our devices */ + if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq")) + printf("xpt_config: failed to create taskqueue thread.\n"); /* Setup debugging path */ if (cam_dflags != CAM_DEBUG_NONE) { - if (xpt_create_path_unlocked(&cam_dpath, NULL, + if (xpt_create_path(&cam_dpath, NULL, CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN) != CAM_REQ_CMP) { printf("xpt_config: xpt_create_path() failed for debug" @@ -4778,7 +4946,8 @@ xpt_config(void *arg) callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000, xpt_boot_delay, NULL); /* Fire up rescan thread. */ - if (kproc_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) { + if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0, + "cam", "scanner")) { printf("xpt_config: failed to create rescan thread.\n"); } } @@ -4863,13 +5032,11 @@ xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg, int xptpath = 0; if (path == NULL) { - mtx_lock(&xsoftc.xpt_lock); status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); - if (status != CAM_REQ_CMP) { - mtx_unlock(&xsoftc.xpt_lock); + if (status != CAM_REQ_CMP) return (status); - } + xpt_path_lock(path); xptpath = 1; } @@ -4882,8 +5049,8 @@ xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg, status = csa.ccb_h.status; if (xptpath) { + xpt_path_unlock(path); xpt_free_path(path); - mtx_unlock(&xsoftc.xpt_lock); } if ((status == CAM_REQ_CMP) && @@ -4968,122 +5135,171 @@ xpt_unlock_buses(void) mtx_unlock(&xsoftc.xpt_topo_lock); } -static void -camisr(void *dummy) +struct mtx * +xpt_path_mtx(struct cam_path *path) { - cam_simq_t queue; - struct cam_sim *sim; - - mtx_lock(&cam_simq_lock); - TAILQ_INIT(&queue); - while (!TAILQ_EMPTY(&cam_simq)) { - TAILQ_CONCAT(&queue, &cam_simq, links); - mtx_unlock(&cam_simq_lock); - while ((sim = TAILQ_FIRST(&queue)) != NULL) { - TAILQ_REMOVE(&queue, sim, links); - CAM_SIM_LOCK(sim); - camisr_runqueue(sim); - sim->flags &= ~CAM_SIM_ON_DONEQ; - CAM_SIM_UNLOCK(sim); - } - mtx_lock(&cam_simq_lock); - } - mtx_unlock(&cam_simq_lock); + return (&path->device->device_mtx); } static void -camisr_runqueue(struct cam_sim *sim) +xpt_done_process(struct ccb_hdr *ccb_h) { - struct ccb_hdr *ccb_h; + struct cam_sim *sim; + struct cam_devq *devq; + struct mtx *mtx = NULL; - while ((ccb_h = TAILQ_FIRST(&sim->sim_doneq)) != NULL) { - int runq; + if (ccb_h->flags & CAM_HIGH_POWER) { + struct highpowerlist *hphead; + struct cam_ed *device; - TAILQ_REMOVE(&sim->sim_doneq, ccb_h, sim_links.tqe); - ccb_h->pinfo.index = CAM_UNQUEUED_INDEX; + mtx_lock(&xsoftc.xpt_highpower_lock); + hphead = &xsoftc.highpowerq; - CAM_DEBUG(ccb_h->path, CAM_DEBUG_TRACE, - ("camisr\n")); + device = STAILQ_FIRST(hphead); - runq = FALSE; + /* + * Increment the count since this command is done. + */ + xsoftc.num_highpower++; - if (ccb_h->flags & CAM_HIGH_POWER) { - struct highpowerlist *hphead; - struct cam_ed *device; + /* + * Any high powered commands queued up? + */ + if (device != NULL) { - mtx_lock(&xsoftc.xpt_lock); - hphead = &xsoftc.highpowerq; + STAILQ_REMOVE_HEAD(hphead, highpowerq_entry); + mtx_unlock(&xsoftc.xpt_highpower_lock); - device = STAILQ_FIRST(hphead); + mtx_lock(&device->sim->devq->send_mtx); + xpt_release_devq_device(device, + /*count*/1, /*runqueue*/TRUE); + mtx_unlock(&device->sim->devq->send_mtx); + } else + mtx_unlock(&xsoftc.xpt_highpower_lock); + } - /* - * Increment the count since this command is done. - */ - xsoftc.num_highpower++; + sim = ccb_h->path->bus->sim; - /* - * Any high powered commands queued up? - */ - if (device != NULL) { + if (ccb_h->status & CAM_RELEASE_SIMQ) { + xpt_release_simq(sim, /*run_queue*/FALSE); + ccb_h->status &= ~CAM_RELEASE_SIMQ; + } - STAILQ_REMOVE_HEAD(hphead, highpowerq_entry); - mtx_unlock(&xsoftc.xpt_lock); + if ((ccb_h->flags & CAM_DEV_QFRZDIS) + && (ccb_h->status & CAM_DEV_QFRZN)) { + xpt_release_devq(ccb_h->path, /*count*/1, + /*run_queue*/FALSE); + ccb_h->status &= ~CAM_DEV_QFRZN; + } - xpt_release_devq_device(device, - /*count*/1, /*runqueue*/TRUE); - } else - mtx_unlock(&xsoftc.xpt_lock); + devq = sim->devq; + if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) { + struct cam_ed *dev = ccb_h->path->device; + + mtx_lock(&devq->send_mtx); + devq->send_active--; + devq->send_openings++; + cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h); + + if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0 + && (dev->ccbq.dev_active == 0))) { + dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY; + xpt_release_devq_device(dev, /*count*/1, + /*run_queue*/FALSE); } - if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) { - struct cam_ed *dev; - - dev = ccb_h->path->device; + if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0 + && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) { + dev->flags &= ~CAM_DEV_REL_ON_COMPLETE; + xpt_release_devq_device(dev, /*count*/1, + /*run_queue*/FALSE); + } - cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h); - sim->devq->send_active--; - sim->devq->send_openings++; - runq = TRUE; + if (!device_is_queued(dev)) + (void)xpt_schedule_devq(devq, dev); + mtx_unlock(&devq->send_mtx); - if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0 - && (dev->ccbq.dev_active == 0))) { - dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY; - xpt_release_devq(ccb_h->path, /*count*/1, - /*run_queue*/FALSE); - } - - if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0 - && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) { - dev->flags &= ~CAM_DEV_REL_ON_COMPLETE; - xpt_release_devq(ccb_h->path, /*count*/1, - /*run_queue*/FALSE); - } + if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) { + mtx = xpt_path_mtx(ccb_h->path); + mtx_lock(mtx); if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 && (--dev->tag_delay_count == 0)) xpt_start_tags(ccb_h->path); - if (!device_is_queued(dev)) { - (void)xpt_schedule_devq(sim->devq, dev); - } } + } - if (ccb_h->status & CAM_RELEASE_SIMQ) { - xpt_release_simq(sim, /*run_queue*/TRUE); - ccb_h->status &= ~CAM_RELEASE_SIMQ; - runq = FALSE; + if ((ccb_h->flags & CAM_UNLOCKED) == 0) { + if (mtx == NULL) { + mtx = xpt_path_mtx(ccb_h->path); + mtx_lock(mtx); + } + } else { + if (mtx != NULL) { + mtx_unlock(mtx); + mtx = NULL; } + } - if ((ccb_h->flags & CAM_DEV_QFRZDIS) - && (ccb_h->status & CAM_DEV_QFRZN)) { - xpt_release_devq(ccb_h->path, /*count*/1, - /*run_queue*/TRUE); - ccb_h->status &= ~CAM_DEV_QFRZN; - } else if (runq) { - xpt_run_devq(sim->devq); + /* Call the peripheral driver's callback */ + ccb_h->pinfo.index = CAM_UNQUEUED_INDEX; + (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h); + if (mtx != NULL) + mtx_unlock(mtx); + + mtx_lock(&devq->send_mtx); + xpt_run_devq(devq); + mtx_unlock(&devq->send_mtx); +} + +void +xpt_done_td(void *arg) +{ + struct cam_doneq *queue = arg; + struct ccb_hdr *ccb_h; + STAILQ_HEAD(, ccb_hdr) doneq; + + STAILQ_INIT(&doneq); + mtx_lock(&queue->cam_doneq_mtx); + while (1) { + while (STAILQ_EMPTY(&queue->cam_doneq)) { + queue->cam_doneq_sleep = 1; + msleep(&queue->cam_doneq, &queue->cam_doneq_mtx, + PRIBIO, "-", 0); + queue->cam_doneq_sleep = 0; + } + STAILQ_CONCAT(&doneq, &queue->cam_doneq); + mtx_unlock(&queue->cam_doneq_mtx); + + THREAD_NO_SLEEPING(); + while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) { + STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe); + xpt_done_process(ccb_h); } + THREAD_SLEEPING_OK(); - /* Call the peripheral driver's callback */ - (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h); + mtx_lock(&queue->cam_doneq_mtx); + } +} + +static void +camisr_runqueue(void) +{ + struct ccb_hdr *ccb_h; + struct cam_doneq *queue; + int i; + + /* Process global queues. */ + for (i = 0; i < cam_num_doneqs; i++) { + queue = &cam_doneqs[i]; + mtx_lock(&queue->cam_doneq_mtx); + while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) { + STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe); + mtx_unlock(&queue->cam_doneq_mtx); + xpt_done_process(ccb_h); + mtx_lock(&queue->cam_doneq_mtx); + } + mtx_unlock(&queue->cam_doneq_mtx); } } diff --git a/sys/cam/cam_xpt.h b/sys/cam/cam_xpt.h index 97933b9..1d983c9 100644 --- a/sys/cam/cam_xpt.h +++ b/sys/cam/cam_xpt.h @@ -56,6 +56,7 @@ struct cam_path; struct async_node { SLIST_ENTRY(async_node) links; u_int32_t event_enable; /* Async Event enables */ + u_int32_t event_lock; /* Take SIM lock for handlers. */ void (*callback)(void *arg, u_int32_t code, struct cam_path *path, void *args); void *callback_arg; @@ -110,6 +111,13 @@ void xpt_hold_boot(void); void xpt_release_boot(void); void xpt_lock_buses(void); void xpt_unlock_buses(void); +struct mtx * xpt_path_mtx(struct cam_path *path); +#define xpt_path_lock(path) mtx_lock(xpt_path_mtx(path)) +#define xpt_path_unlock(path) mtx_unlock(xpt_path_mtx(path)) +#define xpt_path_assert(path, what) mtx_assert(xpt_path_mtx(path), (what)) +#define xpt_path_owned(path) mtx_owned(xpt_path_mtx(path)) +#define xpt_path_sleep(path, chan, priority, wmesg, timo) \ + msleep((chan), xpt_path_mtx(path), (priority), (wmesg), (timo)) cam_status xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg, struct cam_path *path); cam_status xpt_compile_path(struct cam_path *new_path, @@ -117,6 +125,10 @@ cam_status xpt_compile_path(struct cam_path *new_path, path_id_t path_id, target_id_t target_id, lun_id_t lun_id); +cam_status xpt_clone_path(struct cam_path **new_path, + struct cam_path *path); +void xpt_copy_path(struct cam_path *new_path, + struct cam_path *path); void xpt_release_path(struct cam_path *path); diff --git a/sys/cam/cam_xpt_internal.h b/sys/cam/cam_xpt_internal.h index 70808ba..f8c6498 100644 --- a/sys/cam/cam_xpt_internal.h +++ b/sys/cam/cam_xpt_internal.h @@ -29,6 +29,8 @@ #ifndef _CAM_CAM_XPT_INTERNAL_H #define _CAM_CAM_XPT_INTERNAL_H 1 +#include <sys/taskqueue.h> + /* Forward Declarations */ struct cam_eb; struct cam_et; @@ -55,30 +57,16 @@ struct xpt_xport { }; /* - * Structure for queueing a device in a run queue. - * There is one run queue for allocating new ccbs, - * and another for sending ccbs to the controller. - */ -struct cam_ed_qinfo { - cam_pinfo pinfo; - struct cam_ed *device; -}; - -/* * The CAM EDT (Existing Device Table) contains the device information for * all devices for all busses in the system. The table contains a * cam_ed structure for each device on the bus. */ struct cam_ed { + cam_pinfo devq_entry; TAILQ_ENTRY(cam_ed) links; - struct cam_ed_qinfo devq_entry; struct cam_et *target; struct cam_sim *sim; lun_id_t lun_id; - struct camq drvq; /* - * Queue of type drivers wanting to do - * work on this device. - */ struct cam_ccbq ccbq; /* Queue of pending ccbs */ struct async_list asyncs; /* Async callback info for this B/T/L */ struct periph_list periphs; /* All attached devices */ @@ -125,6 +113,8 @@ struct cam_ed { u_int32_t refcount; struct callout callout; STAILQ_ENTRY(cam_ed) highpowerq_entry; + struct mtx device_mtx; + struct task device_destroy_task; }; /* @@ -143,6 +133,7 @@ struct cam_et { struct timeval last_reset; u_int rpl_size; struct scsi_report_luns_data *luns; + struct mtx luns_mtx; /* Protection for luns field. */ }; /* @@ -162,6 +153,7 @@ struct cam_eb { u_int generation; device_t parent_dev; struct xpt_xport *xport; + struct mtx eb_mtx; /* Bus topology mutex. */ }; struct cam_path { @@ -179,8 +171,6 @@ struct cam_ed * xpt_alloc_device(struct cam_eb *bus, lun_id_t lun_id); void xpt_acquire_device(struct cam_ed *device); void xpt_release_device(struct cam_ed *device); -int xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo, - u_int32_t new_priority); u_int32_t xpt_dev_ccbq_resize(struct cam_path *path, int newopenings); void xpt_start_tags(struct cam_path *path); void xpt_stop_tags(struct cam_path *path); diff --git a/sys/cam/cam_xpt_sim.h b/sys/cam/cam_xpt_sim.h index 09126cf..1543645 100644 --- a/sys/cam/cam_xpt_sim.h +++ b/sys/cam/cam_xpt_sim.h @@ -46,8 +46,7 @@ u_int32_t xpt_freeze_devq(struct cam_path *path, u_int count); void xpt_release_devq(struct cam_path *path, u_int count, int run_queue); void xpt_done(union ccb *done_ccb); -void xpt_batch_start(struct cam_sim *sim); -void xpt_batch_done(struct cam_sim *sim); +void xpt_done_direct(union ccb *done_ccb); #endif #endif /* _CAM_CAM_XPT_SIM_H */ diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c index 2e60b64..f381a6b 100644 --- a/sys/cam/ctl/ctl_frontend_cam_sim.c +++ b/sys/cam/ctl/ctl_frontend_cam_sim.c @@ -504,14 +504,9 @@ static void cfcs_done(union ctl_io *io) { union ccb *ccb; - struct cfcs_softc *softc; - struct cam_sim *sim; ccb = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; - sim = xpt_path_sim(ccb->ccb_h.path); - softc = (struct cfcs_softc *)cam_sim_softc(sim); - /* * At this point we should have status. If we don't, that's a bug. */ @@ -550,10 +545,7 @@ cfcs_done(union ctl_io *io) break; } - mtx_lock(sim->mtx); xpt_done(ccb); - mtx_unlock(sim->mtx); - ctl_free_io(io); } diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c index 45acdba..3f50269 100644 --- a/sys/cam/ctl/scsi_ctl.c +++ b/sys/cam/ctl/scsi_ctl.c @@ -73,8 +73,7 @@ __FBSDID("$FreeBSD$"); #include <cam/ctl/ctl_error.h> typedef enum { - CTLFE_CCB_DEFAULT = 0x00, - CTLFE_CCB_WAITING = 0x01 + CTLFE_CCB_DEFAULT = 0x00 } ctlfe_ccb_types; struct ctlfe_softc { @@ -82,6 +81,7 @@ struct ctlfe_softc { path_id_t path_id; struct cam_sim *sim; char port_name[DEV_IDLEN]; + struct mtx lun_softc_mtx; STAILQ_HEAD(, ctlfe_lun_softc) lun_softc_list; STAILQ_ENTRY(ctlfe_softc) links; }; @@ -320,7 +320,6 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) #ifdef CTLFE_INIT_ENABLE if (ctlfe_num_targets >= ctlfe_max_targets) { union ccb *ccb; - struct cam_sim *sim; ccb = (union ccb *)malloc(sizeof(*ccb), M_TEMP, M_NOWAIT | M_ZERO); @@ -328,18 +327,12 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) printf("%s: unable to malloc CCB!\n", __func__); return; } - xpt_setup_ccb(&ccb->ccb_h, cpi->ccb_h.path, - CAM_PRIORITY_NONE); - - sim = xpt_path_sim(cpi->ccb_h.path); + xpt_setup_ccb(&ccb->ccb_h, path, CAM_PRIORITY_NONE); ccb->ccb_h.func_code = XPT_SET_SIM_KNOB; ccb->knob.xport_specific.valid = KNOB_VALID_ROLE; ccb->knob.xport_specific.fc.role = KNOB_ROLE_INITIATOR; - /* We should hold the SIM lock here */ - mtx_assert(sim->mtx, MA_OWNED); - xpt_action(ccb); if ((ccb->ccb_h.status & CAM_STATUS_MASK) != @@ -381,7 +374,9 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) } bus_softc->path_id = cpi->ccb_h.path_id; - bus_softc->sim = xpt_path_sim(cpi->ccb_h.path); + bus_softc->sim = xpt_path_sim(path); + mtx_init(&bus_softc->lun_softc_mtx, "LUN softc mtx", NULL, + MTX_DEF); STAILQ_INIT(&bus_softc->lun_softc_list); fe = &bus_softc->fe; @@ -435,6 +430,7 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) if (retval != 0) { printf("%s: ctl_frontend_register() failed with " "error %d!\n", __func__, retval); + mtx_destroy(&bus_softc->lun_softc_mtx); free(bus_softc, M_CTLFE); break; } else { @@ -464,6 +460,7 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) * are no outstanding commands for this frontend? */ ctl_frontend_deregister(&softc->fe); + mtx_destroy(&softc->lun_softc_mtx); free(softc, M_CTLFE); } break; @@ -538,19 +535,18 @@ ctlferegister(struct cam_periph *periph, void *arg) { struct ctlfe_softc *bus_softc; struct ctlfe_lun_softc *softc; - struct cam_sim *sim; union ccb en_lun_ccb; cam_status status; int i; softc = (struct ctlfe_lun_softc *)arg; bus_softc = softc->parent_softc; - sim = xpt_path_sim(periph->path); TAILQ_INIT(&softc->work_queue); softc->periph = periph; - callout_init_mtx(&softc->dma_callout, sim->mtx, /*flags*/ 0); + callout_init_mtx(&softc->dma_callout, xpt_path_mtx(periph->path), + /*flags*/ 0); periph->softc = softc; xpt_setup_ccb(&en_lun_ccb.ccb_h, periph->path, CAM_PRIORITY_NONE); @@ -580,6 +576,7 @@ ctlferegister(struct cam_periph *periph, void *arg) xpt_setup_ccb(&new_ccb->ccb_h, periph->path, /*priority*/ 1); new_ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO; new_ccb->ccb_h.cbfcnp = ctlfedone; + new_ccb->ccb_h.flags |= CAM_UNLOCKED; xpt_action(new_ccb); softc->atios_sent++; status = new_ccb->ccb_h.status; @@ -615,6 +612,7 @@ ctlferegister(struct cam_periph *periph, void *arg) xpt_setup_ccb(&new_ccb->ccb_h, periph->path, /*priority*/ 1); new_ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY; new_ccb->ccb_h.cbfcnp = ctlfedone; + new_ccb->ccb_h.flags |= CAM_UNLOCKED; xpt_action(new_ccb); softc->inots_sent++; status = new_ccb->ccb_h.status; @@ -646,6 +644,7 @@ ctlfeoninvalidate(struct cam_periph *periph) { union ccb en_lun_ccb; cam_status status; + struct ctlfe_softc *bus_softc; struct ctlfe_lun_softc *softc; softc = (struct ctlfe_lun_softc *)periph->softc; @@ -668,21 +667,22 @@ ctlfeoninvalidate(struct cam_periph *periph) "INOTs outstanding, %d refs\n", softc->atios_sent - softc->atios_returned, softc->inots_sent - softc->inots_returned, periph->refcount); + + bus_softc = softc->parent_softc; + mtx_lock(&bus_softc->lun_softc_mtx); + STAILQ_REMOVE(&bus_softc->lun_softc_list, softc, ctlfe_lun_softc, links); + mtx_unlock(&bus_softc->lun_softc_mtx); } static void ctlfecleanup(struct cam_periph *periph) { struct ctlfe_lun_softc *softc; - struct ctlfe_softc *bus_softc; xpt_print(periph->path, "%s: Called\n", __func__); softc = (struct ctlfe_lun_softc *)periph->softc; - bus_softc = softc->parent_softc; - STAILQ_REMOVE(&bus_softc->lun_softc_list, softc, ctlfe_lun_softc, links); - /* * XXX KDM is there anything else that needs to be done here? */ @@ -705,14 +705,7 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb) start_ccb->ccb_h.ccb_type = CTLFE_CCB_DEFAULT; ccb_h = TAILQ_FIRST(&softc->work_queue); - if (periph->immediate_priority <= periph->pinfo.priority) { - panic("shouldn't get to the CCB waiting case!"); - start_ccb->ccb_h.ccb_type = CTLFE_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } else if (ccb_h == NULL) { + if (ccb_h == NULL) { softc->ccbs_freed++; xpt_release_ccb(start_ccb); } else { @@ -791,7 +784,6 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb) } start_ccb->ccb_h.func_code = XPT_ABORT; start_ccb->cab.abort_ccb = (union ccb *)atio; - start_ccb->ccb_h.cbfcnp = ctlfedone; /* Tell the SIM that we've aborted this ATIO */ xpt_action(start_ccb); @@ -1004,6 +996,7 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb) /*data_ptr*/ data_ptr, /*dxfer_len*/ dxfer_len, /*timeout*/ 5 * 1000); + start_ccb->ccb_h.flags |= CAM_UNLOCKED; start_ccb->ccb_h.ccb_atio = atio; if (((flags & CAM_SEND_STATUS) == 0) && (io != NULL)) @@ -1011,7 +1004,9 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb) softc->ctios_sent++; + cam_periph_unlock(periph); xpt_action(start_ccb); + cam_periph_lock(periph); if ((atio->ccb_h.status & CAM_DEV_QFRZN) != 0) { cam_release_devq(periph->path, @@ -1148,7 +1143,10 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) struct ctlfe_softc *bus_softc; struct ccb_accept_tio *atio = NULL; union ctl_io *io = NULL; + struct mtx *mtx; + KASSERT((done_ccb->ccb_h.flags & CAM_UNLOCKED) != 0, + ("CCB in ctlfedone() without CAM_UNLOCKED flag")); #ifdef CTLFE_DEBUG printf("%s: entered, func_code = %#x, type = %#lx\n", __func__, done_ccb->ccb_h.func_code, done_ccb->ccb_h.ccb_type); @@ -1156,12 +1154,8 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) softc = (struct ctlfe_lun_softc *)periph->softc; bus_softc = softc->parent_softc; - - if (done_ccb->ccb_h.ccb_type == CTLFE_CCB_WAITING) { - panic("shouldn't get to the CCB waiting case!"); - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); /* * If the peripheral is invalid, ATIOs and immediate notify CCBs @@ -1177,7 +1171,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) case XPT_IMMEDIATE_NOTIFY: case XPT_NOTIFY_ACKNOWLEDGE: ctlfe_free_ccb(periph, done_ccb); - return; + goto out; default: break; } @@ -1215,6 +1209,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) xpt_schedule(periph, /*priority*/ 1); break; } + mtx_unlock(mtx); ctl_zero_io(io); /* Save pointers on both sides */ @@ -1271,7 +1266,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) #endif ctl_queue(io); - break; + return; } case XPT_CONT_TARGET_IO: { int srr = 0; @@ -1333,7 +1328,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) TAILQ_INSERT_HEAD(&softc->work_queue, &atio->ccb_h, periph_links.tqe); xpt_schedule(periph, /*priority*/ 1); - return; + break; } /* @@ -1359,10 +1354,11 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) } if (periph->flags & CAM_PERIPH_INVALID) { ctlfe_free_ccb(periph, (union ccb *)atio); - return; } else { - xpt_action((union ccb *)atio); softc->atios_sent++; + mtx_unlock(mtx); + xpt_action((union ccb *)atio); + return; } } else { struct ctlfe_lun_cmd_info *cmd_info; @@ -1478,10 +1474,12 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) /*dxfer_len*/ dxfer_len, /*timeout*/ 5 * 1000); + csio->ccb_h.flags |= CAM_UNLOCKED; csio->resid = 0; csio->ccb_h.ccb_atio = atio; io->io_hdr.flags |= CTL_FLAG_DMA_INPROG; softc->ctios_sent++; + mtx_unlock(mtx); xpt_action((union ccb *)csio); } else { /* @@ -1490,10 +1488,12 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) */ softc->ccbs_freed++; xpt_release_ccb(done_ccb); + mtx_unlock(mtx); /* Call the backend move done callback */ io->scsiio.be_move_done(io); } + return; } break; } @@ -1614,7 +1614,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) ctl_free_io(io); ctlfe_free_ccb(periph, done_ccb); - return; + goto out; } if (send_ctl_io != 0) { ctl_queue(io); @@ -1651,12 +1651,6 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) xpt_action(done_ccb); softc->inots_sent++; break; - case XPT_ABORT: - /* - * XPT_ABORT is an immediate CCB, we shouldn't get here. - */ - panic("%s: XPT_ABORT CCB returned!", __func__); - break; case XPT_SET_SIM_KNOB: case XPT_GET_SIM_KNOB: break; @@ -1665,6 +1659,9 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) done_ccb->ccb_h.func_code); break; } + +out: + mtx_unlock(mtx); } static void @@ -1674,17 +1671,12 @@ ctlfe_onoffline(void *arg, int online) union ccb *ccb; cam_status status; struct cam_path *path; - struct cam_sim *sim; int set_wwnn; bus_softc = (struct ctlfe_softc *)arg; set_wwnn = 0; - sim = bus_softc->sim; - - mtx_assert(sim->mtx, MA_OWNED); - status = xpt_create_path(&path, /*periph*/ NULL, bus_softc->path_id, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) { @@ -1844,12 +1836,8 @@ ctlfe_online(void *arg) struct cam_path *path; cam_status status; struct ctlfe_lun_softc *lun_softc; - struct cam_sim *sim; bus_softc = (struct ctlfe_softc *)arg; - sim = bus_softc->sim; - - CAM_SIM_LOCK(sim); /* * Create the wildcard LUN before bringing the port online. @@ -1860,7 +1848,6 @@ ctlfe_online(void *arg) if (status != CAM_REQ_CMP) { printf("%s: unable to create path for wildcard periph\n", __func__); - CAM_SIM_UNLOCK(sim); return; } @@ -1870,15 +1857,16 @@ ctlfe_online(void *arg) xpt_print(path, "%s: unable to allocate softc for " "wildcard periph\n", __func__); xpt_free_path(path); - CAM_SIM_UNLOCK(sim); return; } + xpt_path_lock(path); lun_softc->parent_softc = bus_softc; lun_softc->flags |= CTLFE_LUN_WILDCARD; + mtx_lock(&bus_softc->lun_softc_mtx); STAILQ_INSERT_TAIL(&bus_softc->lun_softc_list, lun_softc, links); - + mtx_unlock(&bus_softc->lun_softc_mtx); status = cam_periph_alloc(ctlferegister, ctlfeoninvalidate, @@ -1901,11 +1889,10 @@ ctlfe_online(void *arg) entry->status_text : "Unknown", status); } - xpt_free_path(path); - ctlfe_onoffline(arg, /*online*/ 1); - CAM_SIM_UNLOCK(sim); + xpt_path_unlock(path); + xpt_free_path(path); } static void @@ -1915,14 +1902,8 @@ ctlfe_offline(void *arg) struct cam_path *path; cam_status status; struct cam_periph *periph; - struct cam_sim *sim; bus_softc = (struct ctlfe_softc *)arg; - sim = bus_softc->sim; - - CAM_SIM_LOCK(sim); - - ctlfe_onoffline(arg, /*online*/ 0); /* * Disable the wildcard LUN for this port now that we have taken @@ -1932,19 +1913,20 @@ ctlfe_offline(void *arg) bus_softc->path_id, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) { - CAM_SIM_UNLOCK(sim); printf("%s: unable to create path for wildcard periph\n", __func__); return; } + xpt_path_lock(path); + + ctlfe_onoffline(arg, /*online*/ 0); if ((periph = cam_periph_find(path, "ctl")) != NULL) cam_periph_invalidate(periph); + xpt_path_unlock(path); xpt_free_path(path); - - CAM_SIM_UNLOCK(sim); } static int @@ -1970,15 +1952,13 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id) struct ctlfe_lun_softc *softc; struct cam_path *path; struct cam_periph *periph; - struct cam_sim *sim; cam_status status; bus_softc = (struct ctlfe_softc *)arg; - sim = bus_softc->sim; - status = xpt_create_path_unlocked(&path, /*periph*/ NULL, - bus_softc->path_id, - targ_id.id, lun_id); + status = xpt_create_path(&path, /*periph*/ NULL, + bus_softc->path_id, + targ_id.id, lun_id); /* XXX KDM need some way to return status to CTL here? */ if (status != CAM_REQ_CMP) { printf("%s: could not create path, status %#x\n", __func__, @@ -1987,18 +1967,20 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id) } softc = malloc(sizeof(*softc), M_CTLFE, M_WAITOK | M_ZERO); - CAM_SIM_LOCK(sim); + xpt_path_lock(path); periph = cam_periph_find(path, "ctl"); if (periph != NULL) { /* We've already got a periph, no need to alloc a new one. */ + xpt_path_unlock(path); xpt_free_path(path); free(softc, M_CTLFE); - CAM_SIM_UNLOCK(sim); return (0); } softc->parent_softc = bus_softc; + mtx_lock(&bus_softc->lun_softc_mtx); STAILQ_INSERT_TAIL(&bus_softc->lun_softc_list, softc, links); + mtx_unlock(&bus_softc->lun_softc_mtx); status = cam_periph_alloc(ctlferegister, ctlfeoninvalidate, @@ -2011,10 +1993,8 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id) 0, softc); + xpt_path_unlock(path); xpt_free_path(path); - - CAM_SIM_UNLOCK(sim); - return (0); } @@ -2027,12 +2007,10 @@ ctlfe_lun_disable(void *arg, struct ctl_id targ_id, int lun_id) { struct ctlfe_softc *softc; struct ctlfe_lun_softc *lun_softc; - struct cam_sim *sim; softc = (struct ctlfe_softc *)arg; - sim = softc->sim; - CAM_SIM_LOCK(sim); + mtx_lock(&softc->lun_softc_mtx); STAILQ_FOREACH(lun_softc, &softc->lun_softc_list, links) { struct cam_path *path; @@ -2044,16 +2022,18 @@ ctlfe_lun_disable(void *arg, struct ctl_id targ_id, int lun_id) } } if (lun_softc == NULL) { - CAM_SIM_UNLOCK(sim); + mtx_unlock(&softc->lun_softc_mtx); printf("%s: can't find target %d lun %d\n", __func__, targ_id.id, lun_id); return (1); } + cam_periph_acquire(lun_softc->periph); + mtx_unlock(&softc->lun_softc_mtx); + cam_periph_lock(lun_softc->periph); cam_periph_invalidate(lun_softc->periph); - - CAM_SIM_UNLOCK(sim); - + cam_periph_unlock(lun_softc->periph); + cam_periph_release(lun_softc->periph); return (0); } @@ -2064,12 +2044,6 @@ ctlfe_dump_sim(struct cam_sim *sim) printf("%s%d: max tagged openings: %d, max dev openings: %d\n", sim->sim_name, sim->unit_number, sim->max_tagged_dev_openings, sim->max_dev_openings); - printf("%s%d: max_ccbs: %u, ccb_count: %u\n", - sim->sim_name, sim->unit_number, - sim->max_ccbs, sim->ccb_count); - printf("%s%d: ccb_freeq is %sempty\n", - sim->sim_name, sim->unit_number, - (SLIST_FIRST(&sim->ccb_freeq) == NULL) ? "" : "NOT "); printf("\n"); } @@ -2200,17 +2174,13 @@ static void ctlfe_datamove_done(union ctl_io *io) { union ccb *ccb; - struct cam_sim *sim; struct cam_periph *periph; struct ctlfe_lun_softc *softc; ccb = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr; - sim = xpt_path_sim(ccb->ccb_h.path); - - CAM_SIM_LOCK(sim); - periph = xpt_path_periph(ccb->ccb_h.path); + cam_periph_lock(periph); softc = (struct ctlfe_lun_softc *)periph->softc; @@ -2255,7 +2225,7 @@ ctlfe_datamove_done(union ctl_io *io) xpt_schedule(periph, /*priority*/ 1); } - CAM_SIM_UNLOCK(sim); + cam_periph_unlock(periph); } static void diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c index 1b59eca..354fd5c 100644 --- a/sys/cam/scsi/scsi_cd.c +++ b/sys/cam/scsi/scsi_cd.c @@ -118,7 +118,6 @@ typedef enum { typedef enum { CD_CCB_PROBE = 0x01, CD_CCB_BUFFER_IO = 0x02, - CD_CCB_WAITING = 0x03, CD_CCB_TUR = 0x04, CD_CCB_TYPE_MASK = 0x0F, CD_CCB_RETRY_UA = 0x10 @@ -549,7 +548,7 @@ cdasync(void *callback_arg, u_int32_t code, status = cam_periph_alloc(cdregister, cdoninvalidate, cdcleanup, cdstart, "cd", CAM_PERIPH_BIO, - cgd->ccb_h.path, cdasync, + path, cdasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -980,9 +979,9 @@ cdregister(struct cam_periph *periph, void *arg) STAILQ_INIT(&nchanger->chluns); callout_init_mtx(&nchanger->long_handle, - periph->sim->mtx, 0); + cam_periph_mtx(periph), 0); callout_init_mtx(&nchanger->short_handle, - periph->sim->mtx, 0); + cam_periph_mtx(periph), 0); mtx_lock(&changerq_mtx); num_changers++; @@ -1051,7 +1050,7 @@ cdregister(struct cam_periph *periph, void *arg) /* * Schedule a periodic media polling events. */ - callout_init_mtx(&softc->mediapoll_c, periph->sim->mtx, 0); + callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0); if ((softc->flags & CD_FLAG_DISC_REMOVABLE) && (softc->flags & CD_FLAG_CHANGER) == 0 && (cgd->inq_flags & SID_AEN) == 0 && @@ -1535,14 +1534,7 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb) case CD_STATE_NORMAL: { bp = bioq_first(&softc->bio_queue); - if (periph->immediate_priority <= periph->pinfo.priority) { - start_ccb->ccb_h.ccb_state = CD_CCB_WAITING; - - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } else if (bp == NULL) { + if (bp == NULL) { if (softc->tur) { softc->tur = 0; csio = &start_ccb->csio; @@ -1606,11 +1598,9 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb) xpt_action(start_ccb); } - if (bp != NULL || softc->tur || - periph->immediate_priority != CAM_PRIORITY_NONE) { + if (bp != NULL || softc->tur) { /* Have more work to do, so ensure we stay scheduled */ - xpt_schedule(periph, min(CAM_PRIORITY_NORMAL, - periph->immediate_priority)); + xpt_schedule(periph, CAM_PRIORITY_NORMAL); } break; } @@ -1895,15 +1885,6 @@ cddone(struct cam_periph *periph, union ccb *done_ccb) cam_periph_unhold(periph); return; } - case CD_CCB_WAITING: - { - /* Caller will release the CCB */ - CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, - ("trying to wakeup ccbwait\n")); - - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } case CD_CCB_TUR: { if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { diff --git a/sys/cam/scsi/scsi_ch.c b/sys/cam/scsi/scsi_ch.c index fb208ed..b83bc53 100644 --- a/sys/cam/scsi/scsi_ch.c +++ b/sys/cam/scsi/scsi_ch.c @@ -116,8 +116,7 @@ typedef enum { } ch_state; typedef enum { - CH_CCB_PROBE, - CH_CCB_WAITING + CH_CCB_PROBE } ch_ccb_types; typedef enum { @@ -248,20 +247,19 @@ chinit(void) static void chdevgonecb(void *arg) { - struct cam_sim *sim; struct ch_softc *softc; struct cam_periph *periph; + struct mtx *mtx; int i; periph = (struct cam_periph *)arg; - sim = periph->sim; - softc = (struct ch_softc *)periph->softc; + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); + softc = (struct ch_softc *)periph->softc; KASSERT(softc->open_count >= 0, ("Negative open count %d", softc->open_count)); - mtx_lock(sim->mtx); - /* * When we get this callback, we will get no more close calls from * devfs. So if we have any dangling opens, we need to release the @@ -278,13 +276,13 @@ chdevgonecb(void *arg) cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the final call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph * with a cam_periph_unlock() call would cause a page fault. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } static void @@ -350,7 +348,7 @@ chasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg) */ status = cam_periph_alloc(chregister, choninvalidate, chcleanup, chstart, "ch", - CAM_PERIPH_BIO, cgd->ccb_h.path, + CAM_PERIPH_BIO, path, chasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -503,25 +501,23 @@ chopen(struct cdev *dev, int flags, int fmt, struct thread *td) static int chclose(struct cdev *dev, int flag, int fmt, struct thread *td) { - struct cam_sim *sim; struct cam_periph *periph; struct ch_softc *softc; + struct mtx *mtx; periph = (struct cam_periph *)dev->si_drv1; if (periph == NULL) return(ENXIO); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); - sim = periph->sim; softc = (struct ch_softc *)periph->softc; - - mtx_lock(sim->mtx); - softc->open_count--; cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph @@ -532,7 +528,7 @@ chclose(struct cdev *dev, int flag, int fmt, struct thread *td) * protect the open count and avoid another lock acquisition and * release. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); return(0); } @@ -547,14 +543,7 @@ chstart(struct cam_periph *periph, union ccb *start_ccb) switch (softc->state) { case CH_STATE_NORMAL: { - if (periph->immediate_priority <= periph->pinfo.priority){ - start_ccb->ccb_h.ccb_state = CH_CCB_WAITING; - - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } + xpt_release_ccb(start_ccb); break; } case CH_STATE_PROBE: @@ -734,12 +723,6 @@ chdone(struct cam_periph *periph, union ccb *done_ccb) cam_periph_unhold(periph); return; } - case CH_CCB_WAITING: - { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } default: break; } @@ -1724,10 +1707,8 @@ chscsiversion(struct cam_periph *periph) struct scsi_inquiry_data *inq_data; struct ccb_getdev *cgd; int dev_scsi_version; - struct cam_sim *sim; - sim = xpt_path_sim(periph->path); - mtx_assert(sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); if ((cgd = (struct ccb_getdev *)xpt_alloc_ccb_nowait()) == NULL) return (-1); /* diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c index 59a332c..f1ea06a 100644 --- a/sys/cam/scsi/scsi_da.c +++ b/sys/cam/scsi/scsi_da.c @@ -84,7 +84,7 @@ typedef enum { DA_FLAG_PACK_LOCKED = 0x004, DA_FLAG_PACK_REMOVABLE = 0x008, DA_FLAG_NEED_OTAG = 0x020, - DA_FLAG_WENT_IDLE = 0x040, + DA_FLAG_WAS_OTAG = 0x040, DA_FLAG_RETRY_UA = 0x080, DA_FLAG_OPEN = 0x100, DA_FLAG_SCTX_INIT = 0x200, @@ -118,7 +118,6 @@ typedef enum { DA_CCB_PROBE_BDC = 0x05, DA_CCB_PROBE_ATA = 0x06, DA_CCB_BUFFER_IO = 0x07, - DA_CCB_WAITING = 0x08, DA_CCB_DUMP = 0x0A, DA_CCB_DELETE = 0x0B, DA_CCB_TUR = 0x0C, @@ -199,19 +198,17 @@ struct da_softc { struct bio_queue_head bio_queue; struct bio_queue_head delete_queue; struct bio_queue_head delete_run_queue; - SLIST_ENTRY(da_softc) links; LIST_HEAD(, ccb_hdr) pending_ccbs; + int tur; /* TEST UNIT READY should be sent */ + int refcount; /* Active xpt_action() calls */ da_state state; da_flags flags; da_quirks quirks; int sort_io_queue; int minimum_cmd_size; int error_inject; - int ordered_tag_count; - int outstanding_cmds; int trim_max_ranges; int delete_running; - int tur; int delete_available; /* Delete methods possibly available */ uint32_t unmap_max_ranges; uint32_t unmap_max_lba; @@ -1269,86 +1266,72 @@ daclose(struct disk *dp) { struct cam_periph *periph; struct da_softc *softc; + union ccb *ccb; int error; periph = (struct cam_periph *)dp->d_drv1; - cam_periph_lock(periph); - if (cam_periph_hold(periph, PRIBIO) != 0) { - cam_periph_unlock(periph); - cam_periph_release(periph); - return (0); - } - softc = (struct da_softc *)periph->softc; - + cam_periph_lock(periph); CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, ("daclose\n")); - if ((softc->flags & DA_FLAG_DIRTY) != 0 && - (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 && - (softc->flags & DA_FLAG_PACK_INVALID) == 0) { - union ccb *ccb; - - ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); - - scsi_synchronize_cache(&ccb->csio, - /*retries*/1, - /*cbfcnp*/dadone, - MSG_SIMPLE_Q_TAG, - /*begin_lba*/0,/* Cover the whole disk */ - /*lb_count*/0, - SSD_FULL_SIZE, - 5 * 60 * 1000); + if (cam_periph_hold(periph, PRIBIO) == 0) { + + /* Flush disk cache. */ + if ((softc->flags & DA_FLAG_DIRTY) != 0 && + (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 && + (softc->flags & DA_FLAG_PACK_INVALID) == 0) { + ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); + scsi_synchronize_cache(&ccb->csio, /*retries*/1, + /*cbfcnp*/dadone, MSG_SIMPLE_Q_TAG, + /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE, + 5 * 60 * 1000); + error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0, + /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR, + softc->disk->d_devstat); + if (error == 0) + softc->flags &= ~DA_FLAG_DIRTY; + xpt_release_ccb(ccb); + } - error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0, - /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR, - softc->disk->d_devstat); - if (error == 0) - softc->flags &= ~DA_FLAG_DIRTY; - xpt_release_ccb(ccb); + /* Allow medium removal. */ + if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 && + (softc->quirks & DA_Q_NO_PREVENT) == 0) + daprevent(periph, PR_ALLOW); + cam_periph_unhold(periph); } - if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0) { - if ((softc->quirks & DA_Q_NO_PREVENT) == 0) - daprevent(periph, PR_ALLOW); - /* - * If we've got removeable media, mark the blocksize as - * unavailable, since it could change when new media is - * inserted. - */ + /* + * If we've got removeable media, mark the blocksize as + * unavailable, since it could change when new media is + * inserted. + */ + if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0) softc->disk->d_devstat->flags |= DEVSTAT_BS_UNAVAILABLE; - } softc->flags &= ~DA_FLAG_OPEN; - cam_periph_unhold(periph); + while (softc->refcount != 0) + cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1); cam_periph_unlock(periph); cam_periph_release(periph); - return (0); + return (0); } static void daschedule(struct cam_periph *periph) { struct da_softc *softc = (struct da_softc *)periph->softc; - uint32_t prio; if (softc->state != DA_STATE_NORMAL) return; - /* Check if cam_periph_getccb() was called. */ - prio = periph->immediate_priority; - /* Check if we have more work to do. */ if (bioq_first(&softc->bio_queue) || (!softc->delete_running && bioq_first(&softc->delete_queue)) || softc->tur) { - prio = CAM_PRIORITY_NORMAL; + xpt_schedule(periph, CAM_PRIORITY_NORMAL); } - - /* Schedule CCB if any of above is true. */ - if (prio != CAM_PRIORITY_NONE) - xpt_schedule(periph, prio); } /* @@ -1382,9 +1365,7 @@ dastrategy(struct bio *bp) * Place it in the queue of disk activities for this disk */ if (bp->bio_cmd == BIO_DELETE) { - if (bp->bio_bcount == 0) - biodone(bp); - else if (DA_SIO) + if (DA_SIO) bioq_disksort(&softc->delete_queue, bp); else bioq_insert_tail(&softc->delete_queue, bp); @@ -1621,7 +1602,7 @@ daasync(void *callback_arg, u_int32_t code, status = cam_periph_alloc(daregister, daoninvalidate, dacleanup, dastart, "da", CAM_PERIPH_BIO, - cgd->ccb_h.path, daasync, + path, daasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -2066,7 +2047,7 @@ daregister(struct cam_periph *periph, void *arg) * Schedule a periodic event to occasionally send an * ordered tag to a device. */ - callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0); + callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0); callout_reset(&softc->sendordered_c, (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, dasendorderedtag, softc); @@ -2186,7 +2167,7 @@ daregister(struct cam_periph *periph, void *arg) /* * Schedule a periodic media polling events. */ - callout_init_mtx(&softc->mediapoll_c, periph->sim->mtx, 0); + callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0); if ((softc->flags & DA_FLAG_PACK_REMOVABLE) && (cgd->inq_flags & SID_AEN) == 0 && da_poll_period != 0) @@ -2214,20 +2195,6 @@ skipstate: struct bio *bp; uint8_t tag_code; - /* Execute immediate CCB if waiting. */ - if (periph->immediate_priority <= periph->pinfo.priority) { - CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, - ("queuing for immediate ccb\n")); - start_ccb->ccb_h.ccb_state = DA_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - /* May have more work to do, so ensure we stay scheduled */ - daschedule(periph); - break; - } - /* Run BIO_DELETE if not running yet. */ if (!softc->delete_running && (bp = bioq_first(&softc->delete_queue)) != NULL) { @@ -2266,7 +2233,7 @@ skipstate: if ((bp->bio_flags & BIO_ORDERED) != 0 || (softc->flags & DA_FLAG_NEED_OTAG) != 0) { softc->flags &= ~DA_FLAG_NEED_OTAG; - softc->ordered_tag_count++; + softc->flags |= DA_FLAG_WAS_OTAG; tag_code = MSG_ORDERED_Q_TAG; } else { tag_code = MSG_SIMPLE_Q_TAG; @@ -2316,15 +2283,11 @@ skipstate: break; } start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; out: - /* - * Block out any asynchronous callbacks - * while we touch the pending ccb list. - */ LIST_INSERT_HEAD(&softc->pending_ccbs, &start_ccb->ccb_h, periph_links.le); - softc->outstanding_cmds++; /* We expect a unit attention from this device */ if ((softc->flags & DA_FLAG_RETRY_UA) != 0) { @@ -2333,7 +2296,11 @@ out: } start_ccb->ccb_h.ccb_bp = bp; + softc->refcount++; + cam_periph_unlock(periph); xpt_action(start_ccb); + cam_periph_lock(periph); + softc->refcount--; /* May have more work to do, so ensure we stay scheduled */ daschedule(periph); @@ -2628,6 +2595,7 @@ da_delete_unmap(struct cam_periph *periph, union ccb *ccb, struct bio *bp) /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; + ccb->ccb_h.flags |= CAM_UNLOCKED; } static void @@ -2708,6 +2676,7 @@ da_delete_trim(struct cam_periph *periph, union ccb *ccb, struct bio *bp) /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; + ccb->ccb_h.flags |= CAM_UNLOCKED; } /* @@ -2764,6 +2733,7 @@ da_delete_ws(struct cam_periph *periph, union ccb *ccb, struct bio *bp) /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; + ccb->ccb_h.flags |= CAM_UNLOCKED; } static int @@ -2898,6 +2868,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb) { struct bio *bp, *bp1; + cam_periph_lock(periph); bp = (struct bio *)done_ccb->ccb_h.ccb_bp; if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { int error; @@ -2914,6 +2885,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb) * A retry was scheduled, so * just return. */ + cam_periph_unlock(periph); return; } bp = (struct bio *)done_ccb->ccb_h.ccb_bp; @@ -2981,18 +2953,22 @@ dadone(struct cam_periph *periph, union ccb *done_ccb) } } - /* - * Block out any asynchronous callbacks - * while we touch the pending ccb list. - */ LIST_REMOVE(&done_ccb->ccb_h, periph_links.le); - softc->outstanding_cmds--; - if (softc->outstanding_cmds == 0) - softc->flags |= DA_FLAG_WENT_IDLE; + if (LIST_EMPTY(&softc->pending_ccbs)) + softc->flags |= DA_FLAG_WAS_OTAG; + xpt_release_ccb(done_ccb); if (state == DA_CCB_DELETE) { - while ((bp1 = bioq_takefirst(&softc->delete_run_queue)) - != NULL) { + TAILQ_HEAD(, bio) queue; + + TAILQ_INIT(&queue); + TAILQ_CONCAT(&queue, &softc->delete_run_queue.queue, bio_queue); + softc->delete_run_queue.insert_point = NULL; + softc->delete_running = 0; + daschedule(periph); + cam_periph_unlock(periph); + while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { + TAILQ_REMOVE(&queue, bp1, bio_queue); bp1->bio_error = bp->bio_error; if (bp->bio_flags & BIO_ERROR) { bp1->bio_flags |= BIO_ERROR; @@ -3001,13 +2977,11 @@ dadone(struct cam_periph *periph, union ccb *done_ccb) bp1->bio_resid = 0; biodone(bp1); } - softc->delete_running = 0; - if (bp != NULL) - biodone(bp); - daschedule(periph); - } else if (bp != NULL) + } else + cam_periph_unlock(periph); + if (bp != NULL) biodone(bp); - break; + return; } case DA_CCB_PROBE_RC: case DA_CCB_PROBE_RC16: @@ -3457,12 +3431,6 @@ dadone(struct cam_periph *periph, union ccb *done_ccb) daprobedone(periph, done_ccb); return; } - case DA_CCB_WAITING: - { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } case DA_CCB_DUMP: /* No-op. We're polling */ return; @@ -3573,7 +3541,7 @@ damediapoll(void *arg) struct cam_periph *periph = arg; struct da_softc *softc = periph->softc; - if (!softc->tur && softc->outstanding_cmds == 0) { + if (!softc->tur && LIST_EMPTY(&softc->pending_ccbs)) { if (cam_periph_acquire(periph) == CAM_REQ_CMP) { softc->tur = 1; daschedule(periph); @@ -3745,14 +3713,11 @@ dasendorderedtag(void *arg) struct da_softc *softc = arg; if (da_send_ordered) { - if ((softc->ordered_tag_count == 0) - && ((softc->flags & DA_FLAG_WENT_IDLE) == 0)) { - softc->flags |= DA_FLAG_NEED_OTAG; + if (!LIST_EMPTY(&softc->pending_ccbs)) { + if ((softc->flags & DA_FLAG_WAS_OTAG) == 0) + softc->flags |= DA_FLAG_NEED_OTAG; + softc->flags &= ~DA_FLAG_WAS_OTAG; } - if (softc->outstanding_cmds > 0) - softc->flags &= ~DA_FLAG_WENT_IDLE; - - softc->ordered_tag_count = 0; } /* Queue us up again */ callout_reset(&softc->sendordered_c, diff --git a/sys/cam/scsi/scsi_enc.c b/sys/cam/scsi/scsi_enc.c index c39061f..2bf6b50 100644 --- a/sys/cam/scsi/scsi_enc.c +++ b/sys/cam/scsi/scsi_enc.c @@ -69,7 +69,6 @@ static periph_init_t enc_init; static periph_ctor_t enc_ctor; static periph_oninv_t enc_oninvalidate; static periph_dtor_t enc_dtor; -static periph_start_t enc_start; static void enc_async(void *, uint32_t, struct cam_path *, void *); static enctyp enc_type(struct ccb_getdev *); @@ -113,17 +112,16 @@ enc_init(void) static void enc_devgonecb(void *arg) { - struct cam_sim *sim; struct cam_periph *periph; struct enc_softc *enc; + struct mtx *mtx; int i; periph = (struct cam_periph *)arg; - sim = periph->sim; + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); enc = (struct enc_softc *)periph->softc; - mtx_lock(sim->mtx); - /* * When we get this callback, we will get no more close calls from * devfs. So if we have any dangling opens, we need to release the @@ -140,13 +138,13 @@ enc_devgonecb(void *arg) cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the final call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph * with a cam_periph_unlock() call would cause a page fault. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } static void @@ -243,8 +241,8 @@ enc_async(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) } status = cam_periph_alloc(enc_ctor, enc_oninvalidate, - enc_dtor, enc_start, "ses", CAM_PERIPH_BIO, - cgd->ccb_h.path, enc_async, AC_FOUND_DEVICE, cgd); + enc_dtor, NULL, "ses", CAM_PERIPH_BIO, + path, enc_async, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP && status != CAM_REQ_INPROG) { printf("enc_async: Unable to probe new device due to " @@ -299,25 +297,23 @@ out: static int enc_close(struct cdev *dev, int flag, int fmt, struct thread *td) { - struct cam_sim *sim; struct cam_periph *periph; struct enc_softc *enc; + struct mtx *mtx; periph = (struct cam_periph *)dev->si_drv1; if (periph == NULL) return (ENXIO); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); - sim = periph->sim; enc = periph->softc; - - mtx_lock(sim->mtx); - enc->open_count--; cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph @@ -328,34 +324,11 @@ enc_close(struct cdev *dev, int flag, int fmt, struct thread *td) * protect the open count and avoid another lock acquisition and * release. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); return (0); } -static void -enc_start(struct cam_periph *p, union ccb *sccb) -{ - struct enc_softc *enc; - - enc = p->softc; - ENC_DLOG(enc, "%s enter imm=%d prio=%d\n", - __func__, p->immediate_priority, p->pinfo.priority); - if (p->immediate_priority <= p->pinfo.priority) { - SLIST_INSERT_HEAD(&p->ccb_list, &sccb->ccb_h, periph_links.sle); - p->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&p->ccb_list); - } else - xpt_release_ccb(sccb); - ENC_DLOG(enc, "%s exit\n", __func__); -} - -void -enc_done(struct cam_periph *periph, union ccb *dccb) -{ - wakeup(&dccb->ccb_h.cbfcnp); -} - int enc_error(union ccb *ccb, uint32_t cflags, uint32_t sflags) { @@ -614,7 +587,7 @@ enc_runcmd(struct enc_softc *enc, char *cdb, int cdbl, char *dptr, int *dlenp) if (enc->enc_type == ENC_SEMB_SES || enc->enc_type == ENC_SEMB_SAFT) { tdlen = min(dlen, 1020); tdlen = (tdlen + 3) & ~3; - cam_fill_ataio(&ccb->ataio, 0, enc_done, ddf, 0, dptr, tdlen, + cam_fill_ataio(&ccb->ataio, 0, NULL, ddf, 0, dptr, tdlen, 30 * 1000); if (cdb[0] == RECEIVE_DIAGNOSTIC) ata_28bit_cmd(&ccb->ataio, @@ -632,7 +605,7 @@ enc_runcmd(struct enc_softc *enc, char *cdb, int cdbl, char *dptr, int *dlenp) 0x80, tdlen / 4); } else { tdlen = dlen; - cam_fill_csio(&ccb->csio, 0, enc_done, ddf, MSG_SIMPLE_Q_TAG, + cam_fill_csio(&ccb->csio, 0, NULL, ddf, MSG_SIMPLE_Q_TAG, dptr, dlen, sizeof (struct scsi_sense_data), cdbl, 60 * 1000); bcopy(cdb, ccb->csio.cdb_io.cdb_bytes, cdbl); @@ -886,7 +859,7 @@ enc_kproc_init(enc_softc_t *enc) { int result; - callout_init_mtx(&enc->status_updater, enc->periph->sim->mtx, 0); + callout_init_mtx(&enc->status_updater, cam_periph_mtx(enc->periph), 0); if (cam_periph_acquire(enc->periph) != CAM_REQ_CMP) return (ENXIO); diff --git a/sys/cam/scsi/scsi_enc_internal.h b/sys/cam/scsi/scsi_enc_internal.h index f1b8582..3bf8cfb 100644 --- a/sys/cam/scsi/scsi_enc_internal.h +++ b/sys/cam/scsi/scsi_enc_internal.h @@ -192,7 +192,6 @@ struct ses_mgmt_mode_page { /* Enclosure core interface for sub-drivers */ int enc_runcmd(struct enc_softc *, char *, int, char *, int *); void enc_log(struct enc_softc *, const char *, ...); -void enc_done(struct cam_periph *, union ccb *); int enc_error(union ccb *, uint32_t, uint32_t); void enc_update_request(enc_softc_t *, uint32_t); diff --git a/sys/cam/scsi/scsi_enc_safte.c b/sys/cam/scsi/scsi_enc_safte.c index 3c98ab8..8282d01 100644 --- a/sys/cam/scsi/scsi_enc_safte.c +++ b/sys/cam/scsi/scsi_enc_safte.c @@ -243,12 +243,12 @@ safte_fill_read_buf_io(enc_softc_t *enc, struct enc_fsm_state *state, if (enc->enc_type == ENC_SEMB_SAFT) { semb_read_buffer(&ccb->ataio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, + NULL, MSG_SIMPLE_Q_TAG, state->page_code, buf, state->buf_size, state->timeout); } else { scsi_read_buffer(&ccb->csio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, 1, + NULL, MSG_SIMPLE_Q_TAG, 1, state->page_code, 0, buf, state->buf_size, SSD_FULL_SIZE, state->timeout); } @@ -942,11 +942,11 @@ safte_fill_control_request(enc_softc_t *enc, struct enc_fsm_state *state, if (enc->enc_type == ENC_SEMB_SAFT) { semb_write_buffer(&ccb->ataio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, + NULL, MSG_SIMPLE_Q_TAG, buf, xfer_len, state->timeout); } else { scsi_write_buffer(&ccb->csio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, 1, + NULL, MSG_SIMPLE_Q_TAG, 1, 0, 0, buf, xfer_len, SSD_FULL_SIZE, state->timeout); } diff --git a/sys/cam/scsi/scsi_enc_ses.c b/sys/cam/scsi/scsi_enc_ses.c index 6917fff..5393244 100644 --- a/sys/cam/scsi/scsi_enc_ses.c +++ b/sys/cam/scsi/scsi_enc_ses.c @@ -888,7 +888,6 @@ ses_path_iter_devid_callback(enc_softc_t *enc, enc_element_t *elem, struct device_match_result *device_match; struct device_match_pattern *device_pattern; ses_path_iter_args_t *args; - struct cam_sim *sim; args = (ses_path_iter_args_t *)arg; match_pattern.type = DEV_MATCH_DEVICE; @@ -901,10 +900,10 @@ ses_path_iter_devid_callback(enc_softc_t *enc, enc_element_t *elem, device_pattern->data.devid_pat.id_len); memset(&cdm, 0, sizeof(cdm)); - if (xpt_create_path_unlocked(&cdm.ccb_h.path, /*periph*/NULL, - CAM_XPT_PATH_ID, - CAM_TARGET_WILDCARD, - CAM_LUN_WILDCARD) != CAM_REQ_CMP) + if (xpt_create_path(&cdm.ccb_h.path, /*periph*/NULL, + CAM_XPT_PATH_ID, + CAM_TARGET_WILDCARD, + CAM_LUN_WILDCARD) != CAM_REQ_CMP) return; cdm.ccb_h.func_code = XPT_DEV_MATCH; @@ -914,11 +913,8 @@ ses_path_iter_devid_callback(enc_softc_t *enc, enc_element_t *elem, cdm.match_buf_len = sizeof(match_result); cdm.matches = &match_result; - sim = xpt_path_sim(cdm.ccb_h.path); - CAM_SIM_LOCK(sim); xpt_action((union ccb *)&cdm); xpt_free_path(cdm.ccb_h.path); - CAM_SIM_UNLOCK(sim); if ((cdm.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP || (cdm.status != CAM_DEV_MATCH_LAST @@ -927,18 +923,15 @@ ses_path_iter_devid_callback(enc_softc_t *enc, enc_element_t *elem, return; device_match = &match_result.result.device_result; - if (xpt_create_path_unlocked(&cdm.ccb_h.path, /*periph*/NULL, - device_match->path_id, - device_match->target_id, - device_match->target_lun) != CAM_REQ_CMP) + if (xpt_create_path(&cdm.ccb_h.path, /*periph*/NULL, + device_match->path_id, + device_match->target_id, + device_match->target_lun) != CAM_REQ_CMP) return; args->callback(enc, elem, cdm.ccb_h.path, args->callback_arg); - sim = xpt_path_sim(cdm.ccb_h.path); - CAM_SIM_LOCK(sim); xpt_free_path(cdm.ccb_h.path); - CAM_SIM_UNLOCK(sim); } /** @@ -1186,7 +1179,7 @@ ses_set_timed_completion(enc_softc_t *enc, uint8_t tc_en) if (mode_buf == NULL) goto out; - scsi_mode_sense(&ccb->csio, /*retries*/4, enc_done, MSG_SIMPLE_Q_TAG, + scsi_mode_sense(&ccb->csio, /*retries*/4, NULL, MSG_SIMPLE_Q_TAG, /*dbd*/FALSE, SMS_PAGE_CTRL_CURRENT, SES_MGMT_MODE_PAGE_CODE, mode_buf, mode_buf_len, SSD_FULL_SIZE, /*timeout*/60 * 1000); @@ -1214,7 +1207,7 @@ ses_set_timed_completion(enc_softc_t *enc, uint8_t tc_en) /* SES2r20: a completion time of zero means as long as possible */ bzero(&mgmt->max_comp_time, sizeof(mgmt->max_comp_time)); - scsi_mode_select(&ccb->csio, 5, enc_done, MSG_SIMPLE_Q_TAG, + scsi_mode_select(&ccb->csio, 5, NULL, MSG_SIMPLE_Q_TAG, /*page_fmt*/FALSE, /*save_pages*/TRUE, mode_buf, mode_buf_len, SSD_FULL_SIZE, /*timeout*/60 * 1000); @@ -2030,12 +2023,12 @@ ses_fill_rcv_diag_io(enc_softc_t *enc, struct enc_fsm_state *state, if (enc->enc_type == ENC_SEMB_SES) { semb_receive_diagnostic_results(&ccb->ataio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, /*pcv*/1, + NULL, MSG_SIMPLE_Q_TAG, /*pcv*/1, state->page_code, buf, state->buf_size, state->timeout); } else { scsi_receive_diagnostic_results(&ccb->csio, /*retries*/5, - enc_done, MSG_SIMPLE_Q_TAG, /*pcv*/1, + NULL, MSG_SIMPLE_Q_TAG, /*pcv*/1, state->page_code, buf, state->buf_size, SSD_FULL_SIZE, state->timeout); } @@ -2153,12 +2146,12 @@ ses_fill_control_request(enc_softc_t *enc, struct enc_fsm_state *state, /* Fill out the ccb */ if (enc->enc_type == ENC_SEMB_SES) { - semb_send_diagnostic(&ccb->ataio, /*retries*/5, enc_done, + semb_send_diagnostic(&ccb->ataio, /*retries*/5, NULL, MSG_SIMPLE_Q_TAG, buf, ses_page_length(&ses_cache->status_page->hdr), state->timeout); } else { - scsi_send_diagnostic(&ccb->csio, /*retries*/5, enc_done, + scsi_send_diagnostic(&ccb->csio, /*retries*/5, NULL, MSG_SIMPLE_Q_TAG, /*unit_offline*/0, /*device_offline*/0, /*self_test*/0, /*page_format*/1, /*self_test_code*/0, diff --git a/sys/cam/scsi/scsi_pass.c b/sys/cam/scsi/scsi_pass.c index c135e68..158e90f 100644 --- a/sys/cam/scsi/scsi_pass.c +++ b/sys/cam/scsi/scsi_pass.c @@ -65,8 +65,7 @@ typedef enum { } pass_state; typedef enum { - PASS_CCB_BUFFER_IO, - PASS_CCB_WAITING + PASS_CCB_BUFFER_IO } pass_ccb_types; #define ccb_type ppriv_field0 @@ -94,12 +93,9 @@ static periph_init_t passinit; static periph_ctor_t passregister; static periph_oninv_t passoninvalidate; static periph_dtor_t passcleanup; -static periph_start_t passstart; static void pass_add_physpath(void *context, int pending); static void passasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg); -static void passdone(struct cam_periph *periph, - union ccb *done_ccb); static int passerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags); static int passsendccb(struct cam_periph *periph, union ccb *ccb, @@ -143,20 +139,19 @@ passinit(void) static void passdevgonecb(void *arg) { - struct cam_sim *sim; struct cam_periph *periph; + struct mtx *mtx; struct pass_softc *softc; int i; periph = (struct cam_periph *)arg; - sim = periph->sim; - softc = (struct pass_softc *)periph->softc; + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); + softc = (struct pass_softc *)periph->softc; KASSERT(softc->open_count >= 0, ("Negative open count %d", softc->open_count)); - mtx_lock(sim->mtx); - /* * When we get this callback, we will get no more close calls from * devfs. So if we have any dangling opens, we need to release the @@ -173,13 +168,13 @@ passdevgonecb(void *arg) cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the final call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph * with a cam_periph_unlock() call would cause a page fault. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } static void @@ -295,8 +290,8 @@ passasync(void *callback_arg, u_int32_t code, * process. */ status = cam_periph_alloc(passregister, passoninvalidate, - passcleanup, passstart, "pass", - CAM_PERIPH_BIO, cgd->ccb_h.path, + passcleanup, NULL, "pass", + CAM_PERIPH_BIO, path, passasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -498,25 +493,23 @@ passopen(struct cdev *dev, int flags, int fmt, struct thread *td) static int passclose(struct cdev *dev, int flag, int fmt, struct thread *td) { - struct cam_sim *sim; struct cam_periph *periph; struct pass_softc *softc; + struct mtx *mtx; periph = (struct cam_periph *)dev->si_drv1; if (periph == NULL) return (ENXIO); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); - sim = periph->sim; softc = periph->softc; - - mtx_lock(sim->mtx); - softc->open_count--; cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph @@ -527,46 +520,11 @@ passclose(struct cdev *dev, int flag, int fmt, struct thread *td) * protect the open count and avoid another lock acquisition and * release. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); return (0); } -static void -passstart(struct cam_periph *periph, union ccb *start_ccb) -{ - struct pass_softc *softc; - - softc = (struct pass_softc *)periph->softc; - - switch (softc->state) { - case PASS_STATE_NORMAL: - start_ccb->ccb_h.ccb_type = PASS_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - break; - } -} - -static void -passdone(struct cam_periph *periph, union ccb *done_ccb) -{ - struct pass_softc *softc; - struct ccb_scsiio *csio; - - softc = (struct pass_softc *)periph->softc; - csio = &done_ccb->csio; - switch (csio->ccb_h.ccb_type) { - case PASS_CCB_WAITING: - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } - xpt_release_ccb(done_ccb); -} - static int passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td) { @@ -686,12 +644,6 @@ passsendccb(struct cam_periph *periph, union ccb *ccb, union ccb *inccb) xpt_merge_ccb(ccb, inccb); /* - * There's no way for the user to have a completion - * function, so we put our own completion function in here. - */ - ccb->ccb_h.cbfcnp = passdone; - - /* * Let cam_periph_mapmem do a sanity check on the data pointer format. * Even if no data transfer is needed, it's a cheap check and it * simplifies the code. diff --git a/sys/cam/scsi/scsi_pt.c b/sys/cam/scsi/scsi_pt.c index 8d8feb2..f34748c 100644 --- a/sys/cam/scsi/scsi_pt.c +++ b/sys/cam/scsi/scsi_pt.c @@ -66,7 +66,6 @@ typedef enum { typedef enum { PT_CCB_BUFFER_IO = 0x01, - PT_CCB_WAITING = 0x02, PT_CCB_RETRY_UA = 0x04, PT_CCB_BUFFER_IO_UA = PT_CCB_BUFFER_IO|PT_CCB_RETRY_UA } pt_ccb_state; @@ -378,7 +377,7 @@ ptasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg) */ status = cam_periph_alloc(ptctor, ptoninvalidate, ptdtor, ptstart, "pt", CAM_PERIPH_BIO, - cgd->ccb_h.path, ptasync, + path, ptasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -423,15 +422,7 @@ ptstart(struct cam_periph *periph, union ccb *start_ccb) * See if there is a buf with work for us to do.. */ bp = bioq_first(&softc->bio_queue); - if (periph->immediate_priority <= periph->pinfo.priority) { - CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, - ("queuing for immediate ccb\n")); - start_ccb->ccb_h.ccb_state = PT_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } else if (bp == NULL) { + if (bp == NULL) { xpt_release_ccb(start_ccb); } else { bioq_remove(&softc->bio_queue, bp); @@ -554,10 +545,6 @@ ptdone(struct cam_periph *periph, union ccb *done_ccb) biofinish(bp, softc->device_stats, 0); break; } - case PT_CCB_WAITING: - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; } xpt_release_ccb(done_ccb); } diff --git a/sys/cam/scsi/scsi_sa.c b/sys/cam/scsi/scsi_sa.c index d91246b..c1cd0f0 100644 --- a/sys/cam/scsi/scsi_sa.c +++ b/sys/cam/scsi/scsi_sa.c @@ -114,7 +114,6 @@ typedef enum { #define ccb_bp ppriv_ptr1 #define SA_CCB_BUFFER_IO 0x0 -#define SA_CCB_WAITING 0x1 #define SA_CCB_TYPEMASK 0x1 #define SA_POSITION_UPDATED 0x2 @@ -1453,7 +1452,7 @@ saasync(void *callback_arg, u_int32_t code, */ status = cam_periph_alloc(saregister, saoninvalidate, sacleanup, sastart, - "sa", CAM_PERIPH_BIO, cgd->ccb_h.path, + "sa", CAM_PERIPH_BIO, path, saasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -1722,15 +1721,7 @@ sastart(struct cam_periph *periph, union ccb *start_ccb) * See if there is a buf with work for us to do.. */ bp = bioq_first(&softc->bio_queue); - if (periph->immediate_priority <= periph->pinfo.priority) { - CAM_DEBUG_PRINT(CAM_DEBUG_SUBTRACE, - ("queuing for immediate ccb\n")); - Set_CCB_Type(start_ccb, SA_CCB_WAITING); - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } else if (bp == NULL) { + if (bp == NULL) { xpt_release_ccb(start_ccb); } else if ((softc->flags & SA_FLAG_ERR_PENDING) != 0) { struct bio *done_bp; @@ -1953,12 +1944,6 @@ sadone(struct cam_periph *periph, union ccb *done_ccb) biofinish(bp, softc->device_stats, 0); break; } - case SA_CCB_WAITING: - { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } } xpt_release_ccb(done_ccb); } @@ -2545,7 +2530,8 @@ saerror(union ccb *ccb, u_int32_t cflgs, u_int32_t sflgs) /* * If a read/write command, we handle it here. */ - if (CCB_Type(csio) != SA_CCB_WAITING) { + if (csio->cdb_io.cdb_bytes[0] == SA_READ || + csio->cdb_io.cdb_bytes[0] == SA_WRITE) { break; } /* diff --git a/sys/cam/scsi/scsi_sg.c b/sys/cam/scsi/scsi_sg.c index b597b64..f1cb75b 100644 --- a/sys/cam/scsi/scsi_sg.c +++ b/sys/cam/scsi/scsi_sg.c @@ -76,8 +76,7 @@ typedef enum { } sg_rdwr_state; typedef enum { - SG_CCB_RDWR_IO, - SG_CCB_WAITING + SG_CCB_RDWR_IO } sg_ccb_types; #define ccb_type ppriv_field0 @@ -119,7 +118,6 @@ static periph_init_t sginit; static periph_ctor_t sgregister; static periph_oninv_t sgoninvalidate; static periph_dtor_t sgcleanup; -static periph_start_t sgstart; static void sgasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg); static void sgdone(struct cam_periph *periph, union ccb *done_ccb); @@ -172,20 +170,19 @@ sginit(void) static void sgdevgonecb(void *arg) { - struct cam_sim *sim; struct cam_periph *periph; struct sg_softc *softc; + struct mtx *mtx; int i; periph = (struct cam_periph *)arg; - sim = periph->sim; - softc = (struct sg_softc *)periph->softc; + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); + softc = (struct sg_softc *)periph->softc; KASSERT(softc->open_count >= 0, ("Negative open count %d", softc->open_count)); - mtx_lock(sim->mtx); - /* * When we get this callback, we will get no more close calls from * devfs. So if we have any dangling opens, we need to release the @@ -202,13 +199,13 @@ sgdevgonecb(void *arg) cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the final call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph * with a cam_periph_unlock() call would cause a page fault. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } @@ -277,8 +274,8 @@ sgasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg) * start the probe process. */ status = cam_periph_alloc(sgregister, sgoninvalidate, - sgcleanup, sgstart, "sg", - CAM_PERIPH_BIO, cgd->ccb_h.path, + sgcleanup, NULL, "sg", + CAM_PERIPH_BIO, path, sgasync, AC_FOUND_DEVICE, cgd); if ((status != CAM_REQ_CMP) && (status != CAM_REQ_INPROG)) { const struct cam_status_entry *entry; @@ -383,24 +380,6 @@ sgregister(struct cam_periph *periph, void *arg) } static void -sgstart(struct cam_periph *periph, union ccb *start_ccb) -{ - struct sg_softc *softc; - - softc = (struct sg_softc *)periph->softc; - - switch (softc->state) { - case SG_STATE_NORMAL: - start_ccb->ccb_h.ccb_type = SG_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - break; - } -} - -static void sgdone(struct cam_periph *periph, union ccb *done_ccb) { struct sg_softc *softc; @@ -409,10 +388,6 @@ sgdone(struct cam_periph *periph, union ccb *done_ccb) softc = (struct sg_softc *)periph->softc; csio = &done_ccb->csio; switch (csio->ccb_h.ccb_type) { - case SG_CCB_WAITING: - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; case SG_CCB_RDWR_IO: { struct sg_rdwr *rdwr; @@ -480,25 +455,23 @@ sgopen(struct cdev *dev, int flags, int fmt, struct thread *td) static int sgclose(struct cdev *dev, int flag, int fmt, struct thread *td) { - struct cam_sim *sim; struct cam_periph *periph; struct sg_softc *softc; + struct mtx *mtx; periph = (struct cam_periph *)dev->si_drv1; if (periph == NULL) return (ENXIO); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); - sim = periph->sim; softc = periph->softc; - - mtx_lock(sim->mtx); - softc->open_count--; cam_periph_release_locked(periph); /* - * We reference the SIM lock directly here, instead of using + * We reference the lock directly here, instead of using * cam_periph_unlock(). The reason is that the call to * cam_periph_release_locked() above could result in the periph * getting freed. If that is the case, dereferencing the periph @@ -509,7 +482,7 @@ sgclose(struct cdev *dev, int flag, int fmt, struct thread *td) * protect the open count and avoid another lock acquisition and * release. */ - mtx_unlock(sim->mtx); + mtx_unlock(mtx); return (0); } @@ -879,7 +852,7 @@ search: break; } if ((rdwr == NULL) || (rdwr->state != SG_RDWR_DONE)) { - if (msleep(rdwr, periph->sim->mtx, PCATCH, "sgread", 0) == ERESTART) + if (cam_periph_sleep(periph, rdwr, PCATCH, "sgread", 0) == ERESTART) return (EAGAIN); goto search; } diff --git a/sys/cam/scsi/scsi_targ_bh.c b/sys/cam/scsi/scsi_targ_bh.c index bcf4eea..5a4a39b 100644 --- a/sys/cam/scsi/scsi_targ_bh.c +++ b/sys/cam/scsi/scsi_targ_bh.c @@ -65,8 +65,7 @@ typedef enum { } targbh_flags; typedef enum { - TARGBH_CCB_WORKQ, - TARGBH_CCB_WAITING + TARGBH_CCB_WORKQ } targbh_ccb_types; #define MAX_ACCEPT 8 @@ -431,7 +430,7 @@ targbhdtor(struct cam_periph *periph) /* FALLTHROUGH */ default: /* XXX Wait for callback of targbhdislun() */ - msleep(softc, periph->sim->mtx, PRIBIO, "targbh", hz/2); + cam_periph_sleep(periph, softc, PRIBIO, "targbh", hz/2); free(softc, M_SCSIBH); break; } @@ -450,13 +449,7 @@ targbhstart(struct cam_periph *periph, union ccb *start_ccb) softc = (struct targbh_softc *)periph->softc; ccbh = TAILQ_FIRST(&softc->work_queue); - if (periph->immediate_priority <= periph->pinfo.priority) { - start_ccb->ccb_h.ccb_type = TARGBH_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - } else if (ccbh == NULL) { + if (ccbh == NULL) { xpt_release_ccb(start_ccb); } else { TAILQ_REMOVE(&softc->work_queue, ccbh, periph_links.tqe); @@ -535,12 +528,6 @@ targbhdone(struct cam_periph *periph, union ccb *done_ccb) softc = (struct targbh_softc *)periph->softc; - if (done_ccb->ccb_h.ccb_type == TARGBH_CCB_WAITING) { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } - switch (done_ccb->ccb_h.func_code) { case XPT_ACCEPT_TARGET_IO: { diff --git a/sys/cam/scsi/scsi_target.c b/sys/cam/scsi/scsi_target.c index 4b4ad78..42dc152 100644 --- a/sys/cam/scsi/scsi_target.c +++ b/sys/cam/scsi/scsi_target.c @@ -236,23 +236,21 @@ targioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t { struct ioc_enable_lun *new_lun; struct cam_path *path; - struct cam_sim *sim; new_lun = (struct ioc_enable_lun *)addr; - status = xpt_create_path_unlocked(&path, /*periph*/NULL, - new_lun->path_id, - new_lun->target_id, - new_lun->lun_id); + status = xpt_create_path(&path, /*periph*/NULL, + new_lun->path_id, + new_lun->target_id, + new_lun->lun_id); if (status != CAM_REQ_CMP) { printf("Couldn't create path, status %#x\n", status); break; } - sim = xpt_path_sim(path); - mtx_lock(sim->mtx); + xpt_path_lock(path); status = targenable(softc, path, new_lun->grp6_len, new_lun->grp7_len); + xpt_path_unlock(path); xpt_free_path(path); - mtx_unlock(sim->mtx); break; } case TARGIOCDISABLE: @@ -278,13 +276,10 @@ targioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t cdbg.flags = CAM_DEBUG_PERIPH; else cdbg.flags = CAM_DEBUG_NONE; - cam_periph_lock(softc->periph); xpt_setup_ccb(&cdbg.ccb_h, softc->path, CAM_PRIORITY_NORMAL); cdbg.ccb_h.func_code = XPT_DEBUG; cdbg.ccb_h.cbfcnp = targdone; - xpt_action((union ccb *)&cdbg); - cam_periph_unlock(softc->periph); status = cdbg.ccb_h.status & CAM_STATUS_MASK; break; } @@ -823,7 +818,7 @@ targread(struct cdev *dev, struct uio *uio, int ioflag) user_descr = TAILQ_FIRST(abort_queue); while (ccb_h == NULL && user_descr == NULL) { if ((ioflag & IO_NDELAY) == 0) { - error = msleep(user_queue, softc->periph->sim->mtx, + error = cam_periph_sleep(softc->periph, user_queue, PRIBIO | PCATCH, "targrd", 0); ccb_h = TAILQ_FIRST(user_queue); user_descr = TAILQ_FIRST(abort_queue); @@ -1019,7 +1014,6 @@ abort_all_pending(struct targ_softc *softc) struct targ_cmd_descr *descr; struct ccb_abort cab; struct ccb_hdr *ccb_h; - struct cam_sim *sim; CAM_DEBUG(softc->path, CAM_DEBUG_PERIPH, ("abort_all_pending\n")); @@ -1052,8 +1046,7 @@ abort_all_pending(struct targ_softc *softc) /* If we aborted at least one pending CCB ok, wait for it. */ if (cab.ccb_h.status == CAM_REQ_CMP) { - sim = xpt_path_sim(softc->path); - msleep(&softc->pending_ccb_queue, sim->mtx, + cam_periph_sleep(softc->periph, &softc->pending_ccb_queue, PRIBIO | PCATCH, "tgabrt", 0); } diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c index af0032e..69752e1 100644 --- a/sys/cam/scsi/scsi_xpt.c +++ b/sys/cam/scsi/scsi_xpt.c @@ -583,7 +583,7 @@ static struct cam_ed * lun_id_t lun_id); static void scsi_devise_transport(struct cam_path *path); static void scsi_set_transfer_settings(struct ccb_trans_settings *cts, - struct cam_ed *device, + struct cam_path *path, int async_update); static void scsi_toggle_tags(struct cam_path *path); static void scsi_dev_async(u_int32_t async_code, @@ -645,6 +645,7 @@ proberegister(struct cam_periph *periph, void *arg) return (status); } CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n")); + scsi_devise_transport(periph->path); /* * Ensure we've waited at least a bus settle @@ -1719,11 +1720,12 @@ probe_purge_old(struct cam_path *path, struct scsi_report_luns_data *new, if (path->target == NULL) { return; } - if (path->target->luns == NULL) { - path->target->luns = new; - return; - } + mtx_lock(&path->target->luns_mtx); old = path->target->luns; + path->target->luns = new; + mtx_unlock(&path->target->luns_mtx); + if (old == NULL) + return; nlun_old = scsi_4btoul(old->length) / 8; nlun_new = scsi_4btoul(new->length) / 8; @@ -1774,7 +1776,6 @@ probe_purge_old(struct cam_path *path, struct scsi_report_luns_data *new, } } free(old, M_CAMXPT); - path->target->luns = new; } static void @@ -1836,6 +1837,8 @@ typedef struct { static void scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) { + struct mtx *mtx; + CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("scsi_scan_bus\n")); switch (request_ccb->ccb_h.func_code) { @@ -1903,6 +1906,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) (work_ccb->cpi.max_target * sizeof (u_int)), M_CAMXPT, M_ZERO|M_NOWAIT); if (scan_info == NULL) { request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; + xpt_free_ccb(work_ccb); xpt_done(request_ccb); return; } @@ -1933,6 +1937,8 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) scan_info->counter--; } } + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); + mtx_unlock(mtx); for (i = low_target; i <= max_target; i++) { cam_status status; @@ -1965,10 +1971,13 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) request_ccb->ccb_h.pinfo.priority); work_ccb->ccb_h.func_code = XPT_SCAN_LUN; work_ccb->ccb_h.cbfcnp = scsi_scan_bus; + work_ccb->ccb_h.flags |= CAM_UNLOCKED; work_ccb->ccb_h.ppriv_ptr0 = scan_info; work_ccb->crcn.flags = request_ccb->crcn.flags; xpt_action(work_ccb); } + + mtx_lock(mtx); break; } case XPT_SCAN_LUN: @@ -2001,6 +2010,9 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) target = request_ccb->ccb_h.path->target; next_target = 1; + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); + mtx_lock(mtx); + mtx_lock(&target->luns_mtx); if (target->luns) { lun_id_t first; u_int nluns = scsi_4btoul(target->luns->length) / 8; @@ -2042,6 +2054,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) } if (scan_info->lunindex[target_id] < nluns) { + mtx_unlock(&target->luns_mtx); next_target = 0; CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_PROBE, @@ -2050,6 +2063,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) (uintmax_t)lun_id)); scan_info->lunindex[target_id]++; } else { + mtx_unlock(&target->luns_mtx); /* * We're done with scanning all luns. * @@ -2068,7 +2082,9 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) } } } - } else if (request_ccb->ccb_h.status != CAM_REQ_CMP) { + } else { + mtx_unlock(&target->luns_mtx); + if (request_ccb->ccb_h.status != CAM_REQ_CMP) { int phl; /* @@ -2100,7 +2116,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) if (lun_id == request_ccb->ccb_h.target_lun || lun_id > scan_info->cpi->max_lun) next_target = 1; - } else { + } else { device = request_ccb->ccb_h.path->device; @@ -2116,6 +2132,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) if (lun_id == request_ccb->ccb_h.target_lun || lun_id > scan_info->cpi->max_lun) next_target = 1; + } } /* @@ -2149,6 +2166,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) } } if (done) { + mtx_unlock(mtx); xpt_free_ccb(request_ccb); xpt_free_ccb((union ccb *)scan_info->cpi); request_ccb = scan_info->request_ccb; @@ -2162,6 +2180,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) } if ((scan_info->cpi->hba_misc & PIM_SEQSCAN) == 0) { + mtx_unlock(mtx); xpt_free_ccb(request_ccb); break; } @@ -2169,6 +2188,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) scan_info->request_ccb->ccb_h.path_id, scan_info->counter, 0); if (status != CAM_REQ_CMP) { + mtx_unlock(mtx); printf("scsi_scan_bus: xpt_create_path failed" " with status %#x, bus scan halted\n", status); @@ -2184,6 +2204,7 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) request_ccb->ccb_h.pinfo.priority); request_ccb->ccb_h.func_code = XPT_SCAN_LUN; request_ccb->ccb_h.cbfcnp = scsi_scan_bus; + request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->ccb_h.ppriv_ptr0 = scan_info; request_ccb->crcn.flags = scan_info->request_ccb->crcn.flags; @@ -2207,10 +2228,12 @@ scsi_scan_bus(struct cam_periph *periph, union ccb *request_ccb) request_ccb->ccb_h.pinfo.priority); request_ccb->ccb_h.func_code = XPT_SCAN_LUN; request_ccb->ccb_h.cbfcnp = scsi_scan_bus; + request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->ccb_h.ppriv_ptr0 = scan_info; request_ccb->crcn.flags = scan_info->request_ccb->crcn.flags; } + mtx_unlock(mtx); xpt_action(request_ccb); break; } @@ -2227,6 +2250,7 @@ scsi_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_status status; struct cam_path *new_path; struct cam_periph *old_periph; + int lock; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("scsi_scan_lun\n")); @@ -2274,9 +2298,13 @@ scsi_scan_lun(struct cam_periph *periph, struct cam_path *path, xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT); request_ccb->ccb_h.cbfcnp = xptscandone; request_ccb->ccb_h.func_code = XPT_SCAN_LUN; + request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->crcn.flags = flags; } + lock = (xpt_path_owned(path) == 0); + if (lock) + xpt_path_lock(path); if ((old_periph = cam_periph_find(path, "probe")) != NULL) { if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) { probe_softc *softc; @@ -2302,6 +2330,8 @@ scsi_scan_lun(struct cam_periph *periph, struct cam_path *path, xpt_done(request_ccb); } } + if (lock) + xpt_path_unlock(path); } static void @@ -2315,7 +2345,6 @@ xptscandone(struct cam_periph *periph, union ccb *done_ccb) static struct cam_ed * scsi_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { - struct cam_path path; struct scsi_quirk_entry *quirk; struct cam_ed *device; @@ -2340,22 +2369,6 @@ scsi_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) device->device_id_len = 0; device->supported_vpds = NULL; device->supported_vpds_len = 0; - - /* - * XXX should be limited by number of CCBs this bus can - * do. - */ - bus->sim->max_ccbs += device->ccbq.devq_openings; - if (lun_id != CAM_LUN_WILDCARD) { - xpt_compile_path(&path, - NULL, - bus->path_id, - target->target_id, - lun_id); - scsi_devise_transport(&path); - xpt_release_path(&path); - } - return (device); } @@ -2534,15 +2547,8 @@ scsi_dev_advinfo(union ccb *start_ccb) start_ccb->ccb_h.status = CAM_REQ_CMP; if (cdai->flags & CDAI_FLAG_STORE) { - int owned; - - owned = mtx_owned(start_ccb->ccb_h.path->bus->sim->mtx); - if (owned == 0) - mtx_lock(start_ccb->ccb_h.path->bus->sim->mtx); xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path, (void *)(uintptr_t)cdai->buftype); - if (owned == 0) - mtx_unlock(start_ccb->ccb_h.path->bus->sim->mtx); } } @@ -2554,7 +2560,7 @@ scsi_action(union ccb *start_ccb) case XPT_SET_TRAN_SETTINGS: { scsi_set_transfer_settings(&start_ccb->cts, - start_ccb->ccb_h.path->device, + start_ccb->ccb_h.path, /*async_update*/FALSE); break; } @@ -2567,14 +2573,6 @@ scsi_action(union ccb *start_ccb) start_ccb->ccb_h.path, start_ccb->crcn.flags, start_ccb); break; - case XPT_GET_TRAN_SETTINGS: - { - struct cam_sim *sim; - - sim = start_ccb->ccb_h.path->bus->sim; - (*(sim->sim_action))(sim, start_ccb); - break; - } case XPT_DEV_ADVINFO: { scsi_dev_advinfo(start_ccb); @@ -2587,17 +2585,17 @@ scsi_action(union ccb *start_ccb) } static void -scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, +scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path, int async_update) { struct ccb_pathinq cpi; struct ccb_trans_settings cur_cts; struct ccb_trans_settings_scsi *scsi; struct ccb_trans_settings_scsi *cur_scsi; - struct cam_sim *sim; struct scsi_inquiry_data *inq_data; + struct cam_ed *device; - if (device == NULL) { + if (path == NULL || (device = path->device) == NULL) { cts->ccb_h.status = CAM_PATH_INVALID; xpt_done((union ccb *)cts); return; @@ -2614,14 +2612,14 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device cts->protocol_version = device->protocol_version; if (cts->protocol != device->protocol) { - xpt_print(cts->ccb_h.path, "Uninitialized Protocol %x:%x?\n", + xpt_print(path, "Uninitialized Protocol %x:%x?\n", cts->protocol, device->protocol); cts->protocol = device->protocol; } if (cts->protocol_version > device->protocol_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Protocol " + xpt_print(path, "Down reving Protocol " "Version from %d to %d?\n", cts->protocol_version, device->protocol_version); } @@ -2639,22 +2637,20 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device cts->transport_version = device->transport_version; if (cts->transport != device->transport) { - xpt_print(cts->ccb_h.path, "Uninitialized Transport %x:%x?\n", + xpt_print(path, "Uninitialized Transport %x:%x?\n", cts->transport, device->transport); cts->transport = device->transport; } if (cts->transport_version > device->transport_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Transport " + xpt_print(path, "Down reving Transport " "Version from %d to %d?\n", cts->transport_version, device->transport_version); } cts->transport_version = device->transport_version; } - sim = cts->ccb_h.path->bus->sim; - /* * Nothing more of interest to do unless * this is a device connected via the @@ -2662,13 +2658,13 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device */ if (cts->protocol != PROTO_SCSI) { if (async_update == FALSE) - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); return; } inq_data = &device->inq_data; scsi = &cts->proto_specific.scsi; - xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); @@ -2689,7 +2685,7 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device * Perform sanity checking against what the * controller and device can do. */ - xpt_setup_ccb(&cur_cts.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&cur_cts.ccb_h, path, CAM_PRIORITY_NONE); cur_cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cur_cts.type = cts->type; xpt_action((union ccb *)&cur_cts); @@ -2810,7 +2806,7 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device && (spi->flags & (CTS_SPI_VALID_SYNC_RATE| CTS_SPI_VALID_SYNC_OFFSET| CTS_SPI_VALID_BUS_WIDTH)) != 0) - scsi_toggle_tags(cts->ccb_h.path); + scsi_toggle_tags(path); } if (cts->type == CTS_TYPE_CURRENT_SETTINGS @@ -2847,12 +2843,12 @@ scsi_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device device->tag_delay_count = CAM_TAG_DELAY_COUNT; device->flags |= CAM_DEV_TAG_AFTER_COUNT; } else { - xpt_stop_tags(cts->ccb_h.path); + xpt_stop_tags(path); } } } if (async_update == FALSE) - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); } static void @@ -2880,10 +2876,10 @@ scsi_toggle_tags(struct cam_path *path) cts.transport_version = XPORT_VERSION_UNSPECIFIED; cts.proto_specific.scsi.flags = 0; cts.proto_specific.scsi.valid = CTS_SCSI_VALID_TQ; - scsi_set_transfer_settings(&cts, path->device, + scsi_set_transfer_settings(&cts, path, /*async_update*/TRUE); cts.proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB; - scsi_set_transfer_settings(&cts, path->device, + scsi_set_transfer_settings(&cts, path, /*async_update*/TRUE); } } @@ -2954,10 +2950,14 @@ scsi_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, xpt_release_device(device); } else if (async_code == AC_TRANSFER_NEG) { struct ccb_trans_settings *settings; + struct cam_path path; settings = (struct ccb_trans_settings *)async_arg; - scsi_set_transfer_settings(settings, device, + xpt_compile_path(&path, NULL, bus->path_id, target->target_id, + device->lun_id); + scsi_set_transfer_settings(settings, &path, /*async_update*/TRUE); + xpt_release_path(&path); } } @@ -2971,7 +2971,7 @@ scsi_announce_periph(struct cam_periph *periph) u_int freq; u_int mb; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; |
