diff options
Diffstat (limited to 'sys/compat')
| -rw-r--r-- | sys/compat/ndis/kern_ndis.c | 142 | ||||
| -rw-r--r-- | sys/compat/ndis/kern_windrv.c | 56 | ||||
| -rw-r--r-- | sys/compat/ndis/ndis_var.h | 119 | ||||
| -rw-r--r-- | sys/compat/ndis/ntoskrnl_var.h | 142 | ||||
| -rw-r--r-- | sys/compat/ndis/subr_hal.c | 108 | ||||
| -rw-r--r-- | sys/compat/ndis/subr_ndis.c | 555 | ||||
| -rw-r--r-- | sys/compat/ndis/subr_ntoskrnl.c | 1286 | ||||
| -rw-r--r-- | sys/compat/ndis/winx32_wrap.S | 24 |
8 files changed, 1688 insertions, 744 deletions
diff --git a/sys/compat/ndis/kern_ndis.c b/sys/compat/ndis/kern_ndis.c index 573d78f..97f63f7 100644 --- a/sys/compat/ndis/kern_ndis.c +++ b/sys/compat/ndis/kern_ndis.c @@ -82,7 +82,7 @@ static void ndis_setdone_func(ndis_handle, ndis_status); static void ndis_getdone_func(ndis_handle, ndis_status); static void ndis_resetdone_func(ndis_handle, ndis_status, uint8_t); static void ndis_sendrsrcavail_func(ndis_handle); -static void ndis_intrhand(kdpc *, device_object *, +static void ndis_intrsetup(kdpc *, device_object *, irp *, struct ndis_softc *); static void ndis_return(kdpc *, void *, void *, void *); @@ -93,7 +93,7 @@ static image_patch_table kernndis_functbl[] = { IMPORT_SFUNC(ndis_getdone_func, 2), IMPORT_SFUNC(ndis_resetdone_func, 3), IMPORT_SFUNC(ndis_sendrsrcavail_func, 1), - IMPORT_SFUNC(ndis_intrhand, 4), + IMPORT_SFUNC(ndis_intrsetup, 4), IMPORT_SFUNC(ndis_return, 1), { NULL, NULL, NULL } @@ -183,34 +183,6 @@ ndis_modevent(module_t mod, int cmd, void *arg) DEV_MODULE(ndisapi, ndis_modevent, NULL); MODULE_VERSION(ndisapi, 1); -int -ndis_thsuspend(p, m, timo) - struct proc *p; - struct mtx *m; - int timo; -{ - int error; - - if (m != NULL) { - error = msleep(&p->p_siglist, m, - curthread->td_priority, "ndissp", timo); - } else { - PROC_LOCK(p); - error = msleep(&p->p_siglist, &p->p_mtx, - curthread->td_priority|PDROP, "ndissp", timo); - } - - return(error); -} - -void -ndis_thresume(p) - struct proc *p; -{ - wakeup(&p->p_siglist); - return; -} - static void ndis_sendrsrcavail_func(adapter) ndis_handle adapter; @@ -328,7 +300,6 @@ ndis_create_sysctls(arg) #endif /* Add the driver-specific registry keys. */ - vals = sc->ndis_regvals; while(1) { if (vals->nc_cfgkey == NULL) break; @@ -358,17 +329,8 @@ ndis_create_sysctls(arg) continue; } -#if __FreeBSD_version < 502113 - SYSCTL_ADD_STRING(&sc->ndis_ctx, - SYSCTL_CHILDREN(sc->ndis_tree), -#else - SYSCTL_ADD_STRING(device_get_sysctl_ctx(sc->ndis_dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(sc->ndis_dev)), -#endif - OID_AUTO, vals->nc_cfgkey, - CTLFLAG_RW, vals->nc_val, - sizeof(vals->nc_val), - vals->nc_cfgdesc); + ndis_add_sysctl(sc, vals->nc_cfgkey, vals->nc_cfgdesc, + vals->nc_val, CTLFLAG_RW); vals++; } @@ -420,8 +382,10 @@ ndis_add_sysctl(arg, key, desc, val, flag) cfg = malloc(sizeof(struct ndis_cfglist), M_DEVBUF, M_NOWAIT|M_ZERO); - if (cfg == NULL) + if (cfg == NULL) { + printf("failed for %s\n", key); return(ENOMEM); + } cfg->ndis_cfg.nc_cfgkey = strdup(key, M_DEVBUF); if (desc == NULL) { @@ -433,6 +397,7 @@ ndis_add_sysctl(arg, key, desc, val, flag) TAILQ_INSERT_TAIL(&sc->ndis_cfglist_head, cfg, link); + cfg->ndis_oid = #if __FreeBSD_version < 502113 SYSCTL_ADD_STRING(&sc->ndis_ctx, SYSCTL_CHILDREN(sc->ndis_tree), #else @@ -446,18 +411,34 @@ ndis_add_sysctl(arg, key, desc, val, flag) return(0); } +/* + * Somewhere, somebody decided "hey, let's automatically create + * a sysctl tree for each device instance as it's created -- it'll + * make life so much easier!" Lies. Why must they turn the kernel + * into a house of lies? + */ + int ndis_flush_sysctls(arg) void *arg; { struct ndis_softc *sc; struct ndis_cfglist *cfg; + struct sysctl_ctx_list *clist; sc = arg; +#if __FreeBSD_version < 502113 + clist = &sc->ndis_ctx; +#else + clist = device_get_sysctl_ctx(sc->ndis_dev); +#endif + while (!TAILQ_EMPTY(&sc->ndis_cfglist_head)) { cfg = TAILQ_FIRST(&sc->ndis_cfglist_head); TAILQ_REMOVE(&sc->ndis_cfglist_head, cfg, link); + sysctl_ctx_entry_del(clist, cfg->ndis_oid); + sysctl_remove_oid(cfg->ndis_oid, 1, 0); free(cfg->ndis_cfg.nc_cfgkey, M_DEVBUF); free(cfg->ndis_cfg.nc_cfgdesc, M_DEVBUF); free(cfg, M_DEVBUF); @@ -488,9 +469,11 @@ ndis_return(dpc, arg, sysarg1, sysarg2) returnfunc = sc->ndis_chars->nmc_return_packet_func; - KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); + if (NDIS_SERIALIZED(sc->ndis_block)) + KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); MSCALL2(returnfunc, adapter, p); - KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); + if (NDIS_SERIALIZED(sc->ndis_block)) + KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); return; } @@ -691,10 +674,13 @@ ndis_ptom(m0, p) struct mbuf **m0; ndis_packet *p; { - struct mbuf *m, *prev = NULL; + struct mbuf *m = NULL, *prev = NULL; ndis_buffer *buf; ndis_packet_private *priv; uint32_t totlen = 0; + struct ifnet *ifp; + struct ether_header *eh; + int diff; if (p == NULL || m0 == NULL) return(EINVAL); @@ -718,6 +704,7 @@ ndis_ptom(m0, p) MEXTADD(m, m->m_data, m->m_len, ndis_return_packet, p, 0, EXT_NDIS); p->np_refcnt++; + totlen += m->m_len; if (m->m_flags & M_PKTHDR) *m0 = m; @@ -726,6 +713,24 @@ ndis_ptom(m0, p) prev = m; } + /* + * This is a hack to deal with the Marvell 8335 driver + * which, when associated with an AP in WPA-PSK mode, + * seems to overpad its frames by 8 bytes. I don't know + * that the extra 8 bytes are for, and they're not there + * in open mode, so for now clamp the frame size at 1514 + * until I can figure out how to deal with this properly, + * otherwise if_ethersubr() will spank us by discarding + * the 'oversize' frames. + */ + + eh = mtod((*m0), struct ether_header *); + ifp = ((struct ndis_softc *)p->np_softc)->ifp; + if (totlen > ETHER_MAX_FRAME(ifp, eh->ether_type, FALSE)) { + diff = totlen - ETHER_MAX_FRAME(ifp, eh->ether_type, FALSE); + totlen -= diff; + m->m_len -= diff; + } (*m0)->m_pkthdr.len = totlen; return(0); @@ -770,6 +775,7 @@ ndis_mtop(m0, p) *p = NULL; return(ENOMEM); } + MmBuildMdlForNonPagedPool(buf); if (priv->npp_head == NULL) priv->npp_head = buf; @@ -875,7 +881,6 @@ ndis_set_info(arg, oid, buf, buflen) rval = sc->ndis_block->nmb_setstat; } - if (byteswritten) *buflen = byteswritten; if (bytesneeded) @@ -1080,7 +1085,7 @@ ndis_reset_nic(arg) return(0); } -#define NDIS_REAP_TIMERS +#undef NDIS_REAP_TIMERS int ndis_halt_nic(arg) @@ -1112,9 +1117,9 @@ ndis_halt_nic(arg) n->nmt_nexttimer = NULL; } sc->ndis_block->nmb_timerlist = NULL; +#endif if (!cold) KeFlushQueuedDpcs(); -#endif NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; @@ -1123,7 +1128,6 @@ ndis_halt_nic(arg) return(EIO); } - sc->ndis_block->nmb_miniportadapterctx = NULL; sc->ndis_block->nmb_devicectx = NULL; /* @@ -1137,6 +1141,10 @@ ndis_halt_nic(arg) MSCALL1(haltfunc, adapter); + NDIS_LOCK(sc); + sc->ndis_block->nmb_miniportadapterctx = NULL; + NDIS_UNLOCK(sc); + return(0); } @@ -1216,7 +1224,7 @@ ndis_init_nic(arg) * expects them to fire before the halt is called. */ - ndis_thsuspend(curthread->td_proc, NULL, hz); + tsleep(curthread->td_proc, PWAIT, "ndwait", hz); NDIS_LOCK(sc); sc->ndis_block->nmb_devicectx = sc; @@ -1292,33 +1300,26 @@ ndis_isr(arg, ourintr, callhandler) } static void -ndis_intrhand(dpc, dobj, ip, sc) +ndis_intrsetup(dpc, dobj, ip, sc) kdpc *dpc; device_object *dobj; irp *ip; struct ndis_softc *sc; { - ndis_handle adapter; - ndis_interrupt_handler intrfunc; - uint8_t irql; - - adapter = sc->ndis_block->nmb_miniportadapterctx; - intrfunc = sc->ndis_chars->nmc_interrupt_func; - - if (adapter == NULL || intrfunc == NULL) - return; - - if (NDIS_SERIALIZED(sc->ndis_block)) - KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); + ndis_miniport_interrupt *intr; - MSCALL1(intrfunc, adapter); + intr = sc->ndis_block->nmb_interrupt; - /* If there's a MiniportEnableInterrupt() routine, call it. */ + /* Sanity check. */ - ndis_enable_intr(sc); + if (intr == NULL) + return; - if (NDIS_SERIALIZED(sc->ndis_block)) - KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); + KeAcquireSpinLockAtDpcLevel(&intr->ni_dpccountlock); + KeResetEvent(&intr->ni_dpcevt); + if (KeInsertQueueDpc(&intr->ni_dpc, NULL, NULL) == TRUE) + intr->ni_dpccnt++; + KeReleaseSpinLockFromDpcLevel(&intr->ni_dpccountlock); return; } @@ -1424,6 +1425,7 @@ NdisAddDevice(drv, pdo) block->nmb_physdeviceobj = pdo; block->nmb_nextdeviceobj = IoAttachDeviceToDeviceStack(fdo, pdo); KeInitializeSpinLock(&block->nmb_lock); + InitializeListHead(&block->nmb_parmlist); /* * Stash pointers to the miniport block and miniport @@ -1447,7 +1449,7 @@ NdisAddDevice(drv, pdo) IoDeleteDevice(fdo); return(status); } - INIT_LIST_HEAD((&block->nmb_packetlist)); + InitializeListHead((&block->nmb_packetlist)); } /* Give interrupt handling priority over timers. */ diff --git a/sys/compat/ndis/kern_windrv.c b/sys/compat/ndis/kern_windrv.c index 90c977b..9202911 100644 --- a/sys/compat/ndis/kern_windrv.c +++ b/sys/compat/ndis/kern_windrv.c @@ -137,8 +137,8 @@ windrv_libfini(void) } mtx_unlock(&drvdb_mtx); - free(fake_pci_driver.dro_drivername.us_buf, M_DEVBUF); - free(fake_pccard_driver.dro_drivername.us_buf, M_DEVBUF); + RtlFreeUnicodeString(&fake_pci_driver.dro_drivername); + RtlFreeUnicodeString(&fake_pccard_driver.dro_drivername); mtx_destroy(&drvdb_mtx); @@ -161,16 +161,16 @@ windrv_lookup(img, name) { struct drvdb_ent *d; unicode_string us; + ansi_string as; bzero((char *)&us, sizeof(us)); /* Damn unicode. */ if (name != NULL) { - us.us_len = strlen(name) * 2; - us.us_maxlen = strlen(name) * 2; - us.us_buf = NULL; - ndis_ascii_to_unicode(name, &us.us_buf); + RtlInitAnsiString(&as, name); + if (RtlAnsiStringToUnicodeString(&us, &as, TRUE)) + return(NULL); } mtx_lock(&drvdb_mtx); @@ -187,7 +187,7 @@ windrv_lookup(img, name) mtx_unlock(&drvdb_mtx); if (name != NULL) - ExFreePool(us.us_buf); + RtlFreeUnicodeString(&us); return(NULL); } @@ -230,7 +230,7 @@ windrv_unload(mod, img, len) driver_object *drv; device_object *d, *pdo; device_t dev; - list_entry *e, *c; + list_entry *e; drv = windrv_lookup(img, NULL); @@ -267,7 +267,7 @@ windrv_unload(mod, img, len) mtx_lock(&drvdb_mtx); } } - + STAILQ_FOREACH(db, &drvdb_head, link) { if (db->windrv_object->dro_driverstart == (void *)img) { r = db; @@ -280,23 +280,23 @@ windrv_unload(mod, img, len) if (r == NULL) return (ENOENT); + if (drv == NULL) + return(ENOENT); + /* * Destroy any custom extensions that may have been added. */ drv = r->windrv_object; - e = drv->dro_driverext->dre_usrext.nle_flink; - while (e != &drv->dro_driverext->dre_usrext) { - c = e->nle_flink; - REMOVE_LIST_ENTRY(e); + while (!IsListEmpty(&drv->dro_driverext->dre_usrext)) { + e = RemoveHeadList(&drv->dro_driverext->dre_usrext); ExFreePool(e); - e = c; } /* Free the driver extension */ free(drv->dro_driverext, M_DEVBUF); /* Free the driver name */ - free(drv->dro_drivername.us_buf, M_DEVBUF); + RtlFreeUnicodeString(&drv->dro_drivername); /* Free driver object */ free(drv, M_DEVBUF); @@ -330,6 +330,7 @@ windrv_load(mod, img, len, bustype, devlist, regvals) struct driver_object *drv; int status; uint32_t *ptr; + ansi_string as; /* * First step: try to relocate and dynalink the executable @@ -396,16 +397,17 @@ skipreloc: return(ENOMEM); } - INIT_LIST_HEAD((&drv->dro_driverext->dre_usrext)); + InitializeListHead((&drv->dro_driverext->dre_usrext)); drv->dro_driverstart = (void *)img; drv->dro_driversize = len; - drv->dro_drivername.us_len = strlen(DUMMY_REGISTRY_PATH) * 2; - drv->dro_drivername.us_maxlen = strlen(DUMMY_REGISTRY_PATH) * 2; - drv->dro_drivername.us_buf = NULL; - ndis_ascii_to_unicode(DUMMY_REGISTRY_PATH, - &drv->dro_drivername.us_buf); + RtlInitAnsiString(&as, DUMMY_REGISTRY_PATH); + if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE)) { + free(new, M_DEVBUF); + free(drv, M_DEVBUF); + return(ENOMEM); + } new->windrv_object = drv; new->windrv_regvals = regvals; @@ -417,7 +419,7 @@ skipreloc: status = MSCALL2(entry, drv, &drv->dro_drivername); if (status != STATUS_SUCCESS) { - free(drv->dro_drivername.us_buf, M_DEVBUF); + RtlFreeUnicodeString(&drv->dro_drivername); free(drv, M_DEVBUF); free(new, M_DEVBUF); return(ENODEV); @@ -517,15 +519,15 @@ windrv_bus_attach(drv, name) char *name; { struct drvdb_ent *new; + ansi_string as; new = malloc(sizeof(struct drvdb_ent), M_DEVBUF, M_NOWAIT|M_ZERO); if (new == NULL) return (ENOMEM); - drv->dro_drivername.us_len = strlen(name) * 2; - drv->dro_drivername.us_maxlen = strlen(name) * 2; - drv->dro_drivername.us_buf = NULL; - ndis_ascii_to_unicode(name, &drv->dro_drivername.us_buf); + RtlInitAnsiString(&as, name); + if (RtlAnsiStringToUnicodeString(&drv->dro_drivername, &as, TRUE)) + return(ENOMEM); /* * Set up a fake image pointer to avoid false matches @@ -674,7 +676,7 @@ ctxsw_utow(void) * hasn't, we need to do it right now or else things will * explode. */ - + if (t->tid_self != t) x86_newldt(NULL); diff --git a/sys/compat/ndis/ndis_var.h b/sys/compat/ndis/ndis_var.h index 742fae2..4fd7106 100644 --- a/sys/compat/ndis/ndis_var.h +++ b/sys/compat/ndis/ndis_var.h @@ -268,6 +268,31 @@ typedef uint8_t ndis_kirql; #define OID_PNP_WAKE_UP_PATTERN_LIST 0xFD010105 #define OID_PNP_ENABLE_WAKE_UP 0xFD010106 +/* + * These are the possible power states for + * OID_PNP_SET_POWER and OID_PNP_QUERY_POWER. + */ +#define NDIS_POWERSTATE_UNSPEC 0 +#define NDIS_POWERSTATE_D0 1 +#define NDIS_POWERSTATE_D1 2 +#define NDIS_POWERSTATE_D2 3 +#define NDIS_POWERSTATE_D3 4 + +/* + * These are used with the MiniportPnpEventNotify() method. + */ + +#define NDIS_POWERPROFILE_BATTERY 0 +#define NDIS_POWERPROFILE_ACONLINE 1 + +#define NDIS_PNP_EVENT_QUERY_REMOVED 0 +#define NDIS_PNP_EVENT_REMOVED 1 +#define NDIS_PNP_EVENT_SURPRISE_REMOVED 2 +#define NDIS_PNP_EVENT_QUERY_STOPPED 3 +#define NDIS_PNP_EVENT_STOPPED 4 +#define NDIS_PNP_EVENT_PROFILECHANGED 5 + + /* PnP/PM Statistics (Optional). */ #define OID_PNP_WAKE_UP_OK 0xFD020200 #define OID_PNP_WAKE_UP_ERROR 0xFD020201 @@ -310,6 +335,8 @@ typedef uint8_t ndis_kirql; #define OID_802_11_REMOVE_KEY 0x0D01011E #define OID_802_11_ASSOCIATION_INFORMATION 0x0D01011F #define OID_802_11_TEST 0x0D010120 +#define OID_802_11_CAPABILITY 0x0D010122 +#define OID_802_11_PMKID 0x0D010123 /* structures/definitions for 802.11 */ #define NDIS_80211_NETTYPE_11FH 0x00000000 @@ -390,6 +417,8 @@ typedef struct ndis_80211_wep ndis_80211_wep; #define NDIS_80211_AUTHMODE_WPA 0x00000003 #define NDIS_80211_AUTHMODE_WPAPSK 0x00000004 #define NDIS_80211_AUTHMODE_WPANONE 0x00000005 +#define NDIS_80211_AUTHMODE_WPA2 0x00000006 +#define NDIS_80211_AUTHMODE_WPA2PSK 0x00000007 typedef uint8_t ndis_80211_rates[8]; typedef uint8_t ndis_80211_rates_ex[16]; @@ -483,6 +512,7 @@ typedef uint32_t ndis_80211_antenna; #define NDIS_80211_RELOADDEFAULT_WEP 0x00000000 #define NDIS_80211_STATUSTYPE_AUTH 0x00000000 +#define NDIS_80211_STATUSTYPE_PMKIDLIST 0x00000001 struct ndis_80211_status_indication { uint32_t nsi_type; @@ -490,6 +520,11 @@ struct ndis_80211_status_indication { typedef struct ndis_80211_status_indication ndis_80211_status_indication; +#define NDIS_802_11_AUTH_REQUEST_REAUTH 0x01 +#define NDIS_802_11_AUTH_REQUEST_KEYUPDATE 0x02 +#define NDIS_802_11_AUTH_REQUEST_PAIRWISE_ERROR 0x06 +#define NDIS_802_11_AUTH_REQUEST_GROUP_ERROR 0x0E + struct ndis_80211_auth_request { uint32_t nar_len; ndis_80211_macaddr nar_bssid; @@ -503,8 +538,9 @@ struct ndis_80211_key { uint32_t nk_keyidx; uint32_t nk_keylen; ndis_80211_macaddr nk_bssid; + uint8_t nk_pad[6]; uint64_t nk_keyrsc; - uint8_t nk_keydata[256]; + uint8_t nk_keydata[32]; }; typedef struct ndis_80211_key ndis_80211_key; @@ -573,6 +609,61 @@ struct ndis_80211_test { typedef struct ndis_80211_test ndis_80211_test; +struct ndis_80211_auth_encrypt { + uint32_t ne_authmode; + uint32_t ne_cryptstat; +}; + +typedef struct ndis_80211_auth_encrypt ndis_80211_auth_encrypt; + +struct ndis_80211_caps { + uint32_t nc_len; + uint32_t nc_ver; + uint32_t nc_numpmkids; + ndis_80211_auth_encrypt nc_authencs[1]; +}; + +typedef struct ndis_80211_caps ndis_80211_caps; + +struct ndis_80211_bssidinfo { + ndis_80211_macaddr nb_bssid; + uint8_t nb_pmkid[16]; +}; + +typedef struct ndis_80211_bssidinfo ndis_80211_bssidinfo; + +struct ndis_80211_pmkid { + uint32_t np_len; + uint32_t np_bssidcnt; + ndis_80211_bssidinfo np_bssidinfo[1]; +}; + +typedef struct ndis_80211_pmkid ndis_80211_pmkid; + +struct ndis_80211_pmkid_cand { + ndis_80211_macaddr npc_bssid; + uint32_t npc_flags; +}; + +typedef struct ndis_80211_pmkid_cand ndis_80211_pmkid_cand; + +#define NDIS_802_11_PMKID_CANDIDATE_PREAUTH_ENABLED (0x01) + +struct ndis_80211_pmkid_candidate_list { + uint32_t npcl_version; + uint32_t npcl_numcandidates; + ndis_80211_pmkid_cand npcl_candidatelist[1]; +}; + +typedef struct ndis_80211_pmkid_candidate_list ndis_80211_pmkid_candidate_list; + +struct ndis_80211_enc_indication { + uint32_t nei_statustype; + ndis_80211_pmkid_candidate_list nei_pmkidlist; +}; + +typedef struct ndis_80211_enc_indication ndis_80211_enc_indication; + /* TCP OIDs. */ #define OID_TCP_TASK_OFFLOAD 0xFC010201 @@ -682,6 +773,7 @@ typedef struct ndis_task_ipsec ndis_task_ipsec; * all attributes. */ +#define NDIS_ATTRIBUTE_IGNORE_PACKET_TIMEOUT 0x00000001 #define NDIS_ATTRIBUTE_IGNORE_REQUEST_TIMEOUT 0x00000002 #define NDIS_ATTRIBUTE_IGNORE_TOKEN_RING_ERRORS 0x00000004 #define NDIS_ATTRIBUTE_BUS_MASTER 0x00000008 @@ -805,8 +897,20 @@ struct ndis_config_parm { } ncp_parmdata; }; +/* + * Not part of Windows NDIS spec; we uses this to keep a + * list of ndis_config_parm structures that we've allocated. + */ + typedef struct ndis_config_parm ndis_config_parm; +struct ndis_parmlist_entry { + list_entry np_list; + ndis_config_parm np_parm; +}; + +typedef struct ndis_parmlist_entry ndis_parmlist_entry; + #ifdef notdef struct ndis_list_entry { struct ndis_list_entry *nle_flink; @@ -952,16 +1056,16 @@ typedef struct ndis_request ndis_request; * Filler, not used. */ struct ndis_miniport_interrupt { - void *ni_introbj; + kinterrupt *ni_introbj; ndis_kspin_lock ni_dpccountlock; void *ni_rsvd; void *ni_isrfunc; void *ni_dpcfunc; - struct ndis_kdpc ni_dpc; + kdpc ni_dpc; ndis_miniport_block *ni_block; uint8_t ni_dpccnt; uint8_t ni_filler1; - struct ndis_kevent ni_dpcsdoneevent; + struct nt_kevent ni_dpcevt; uint8_t ni_shared; uint8_t ni_isrreq; }; @@ -1506,8 +1610,7 @@ struct ndis_miniport_block { * End of windows-specific portion of miniport block. Everything * below is BSD-specific. */ - uint8_t nmb_dummybuf[128]; - ndis_config_parm nmb_replyparm; + list_entry nmb_parmlist; ndis_resource_list *nmb_rlist; ndis_status nmb_getstat; ndis_status nmb_setstat; @@ -1600,8 +1703,6 @@ extern image_patch_table ndis_functbl[]; __BEGIN_DECLS extern int ndis_libinit(void); extern int ndis_libfini(void); -extern int ndis_ascii_to_unicode(char *, uint16_t **); -extern int ndis_unicode_to_ascii(uint16_t *, int, char **); extern int ndis_load_driver(vm_offset_t, void *); extern int ndis_unload_driver(void *); extern int ndis_mtop(struct mbuf *, ndis_packet **); @@ -1629,8 +1730,6 @@ extern int ndis_destroy_dma(void *); extern int ndis_create_sysctls(void *); extern int ndis_add_sysctl(void *, char *, char *, char *, int); extern int ndis_flush_sysctls(void *); -extern int ndis_thsuspend(struct proc *, struct mtx *, int); -extern void ndis_thresume(struct proc *); extern int ndis_strcasecmp(const char *, const char *); extern int ndis_strncasecmp(const char *, const char *, size_t); diff --git a/sys/compat/ndis/ntoskrnl_var.h b/sys/compat/ndis/ntoskrnl_var.h index 622a387..7a1f3af 100644 --- a/sys/compat/ndis/ntoskrnl_var.h +++ b/sys/compat/ndis/ntoskrnl_var.h @@ -48,6 +48,14 @@ struct unicode_string { typedef struct unicode_string unicode_string; +struct ansi_string { + uint16_t as_len; + uint16_t as_maxlen; + char *as_buf; +}; + +typedef struct ansi_string ansi_string; + /* * Windows memory descriptor list. In Windows, it's possible for * buffers to be passed between user and kernel contexts without @@ -197,43 +205,54 @@ struct list_entry { typedef struct list_entry list_entry; -#define INIT_LIST_HEAD(l) \ +#define InitializeListHead(l) \ (l)->nle_flink = (l)->nle_blink = (l) -#define REMOVE_LIST_ENTRY(e) \ +#define IsListEmpty(h) \ + ((h)->nle_flink == (h)) + +#define RemoveEntryList(e) \ do { \ list_entry *b; \ list_entry *f; \ \ - f = e->nle_flink; \ - b = e->nle_blink; \ + f = (e)->nle_flink; \ + b = (e)->nle_blink; \ b->nle_flink = f; \ f->nle_blink = b; \ } while (0) -#define REMOVE_LIST_HEAD(l) \ - do { \ - list_entry *f; \ - list_entry *e; \ - \ - e = l->nle_flink; \ - f = e->nle_flink; \ - l->nle_flink = f; \ - f->nle_blink = l; \ - } while (0) +/* These two have to be inlined since they return things. */ -#define REMOVE_LIST_TAIL(l) \ - do { \ - list_entry *b; \ - list_entry *e; \ - \ - e = l->nle_blink; \ - b = e->nle_blink; \ - l->nle_blink = b; \ - b->nle_flink = l; \ - } while (0) +static __inline__ list_entry * +RemoveHeadList(list_entry *l) +{ + list_entry *f; + list_entry *e; + + e = l->nle_flink; + f = e->nle_flink; + l->nle_flink = f; + f->nle_blink = l; + + return (e); +} + +static __inline__ list_entry * +RemoveTailList(list_entry *l) +{ + list_entry *b; + list_entry *e; + + e = l->nle_blink; + b = e->nle_blink; + l->nle_blink = b; + b->nle_flink = l; -#define INSERT_LIST_TAIL(l, e) \ + return (e); +} + +#define InsertTailList(l, e) \ do { \ list_entry *b; \ \ @@ -244,7 +263,7 @@ typedef struct list_entry list_entry; l->nle_blink = (e); \ } while (0) -#define INSERT_LIST_HEAD(l, e) \ +#define InsertHeadList(l, e) \ do { \ list_entry *f; \ \ @@ -263,12 +282,24 @@ struct nt_dispatch_header { uint8_t dh_abs; uint8_t dh_size; uint8_t dh_inserted; - uint32_t dh_sigstate; + int32_t dh_sigstate; list_entry dh_waitlisthead; }; typedef struct nt_dispatch_header nt_dispatch_header; +/* Dispatcher object types */ + +#define DISP_TYPE_NOTIFICATION_EVENT 0 /* KEVENT */ +#define DISP_TYPE_SYNCHRONIZATION_EVENT 1 /* KEVENT */ +#define DISP_TYPE_MUTANT 2 /* KMUTANT/KMUTEX */ +#define DISP_TYPE_PROCESS 3 /* KPROCESS */ +#define DISP_TYPE_QUEUE 4 /* KQUEUE */ +#define DISP_TYPE_SEMAPHORE 5 /* KSEMAPHORE */ +#define DISP_TYPE_THREAD 6 /* KTHREAD */ +#define DISP_TYPE_NOTIFICATION_TIMER 8 /* KTIMER */ +#define DISP_TYPE_SYNCHRONIZATION_TIMER 9 /* KTIMER */ + #define OTYPE_EVENT 0 #define OTYPE_MUTEX 1 #define OTYPE_THREAD 2 @@ -334,14 +365,14 @@ struct ktimer { uint64_t k_duetime; union { list_entry k_timerlistentry; - struct callout_handle k_handle; + struct callout *k_callout; } u; void *k_dpc; uint32_t k_period; }; #define k_timerlistentry u.k_timerlistentry -#define k_handle u.k_handle +#define k_callout u.k_callout typedef struct ktimer ktimer; @@ -389,18 +420,12 @@ typedef struct kdpc kdpc; */ struct kmutant { nt_dispatch_header km_header; - union { - list_entry km_listentry; - uint32_t km_acquirecnt; - } u; + list_entry km_listentry; void *km_ownerthread; uint8_t km_abandoned; uint8_t km_apcdisable; }; -#define km_listentry u.km_listentry -#define km_acquirecnt u.km_acquirecnt - typedef struct kmutant kmutant; #define LOOKASIDE_DEPTH 256 @@ -485,18 +510,28 @@ struct wait_block { void *wb_kthread; nt_dispatch_header *wb_object; struct wait_block *wb_next; +#ifdef notdef uint16_t wb_waitkey; uint16_t wb_waittype; +#endif + uint8_t wb_waitkey; + uint8_t wb_waittype; + uint8_t wb_awakened; + uint8_t wb_oldpri; }; typedef struct wait_block wait_block; +#define wb_ext wb_kthread + #define THREAD_WAIT_OBJECTS 3 #define MAX_WAIT_OBJECTS 64 #define WAITTYPE_ALL 0 #define WAITTYPE_ANY 1 +#define WAITKEY_VALID 0x8000 + struct thread_context { void *tc_thrctx; void *tc_thrfunc; @@ -533,6 +568,23 @@ struct custom_extension { typedef struct custom_extension custom_extension; /* + * The KINTERRUPT structure in Windows is opaque to drivers. + * We define our own custom version with things we need. + */ + +struct kinterrupt { + device_t ki_dev; + void *ki_cookie; + struct resource *ki_irq; + kspin_lock ki_lock_priv; + kspin_lock *ki_lock; + void *ki_svcfunc; + void *ki_svcctx; +}; + +typedef struct kinterrupt kinterrupt; + +/* * In Windows, there are Physical Device Objects (PDOs) and * Functional Device Objects (FDOs). Physical Device Objects are * created and maintained by bus drivers. For example, the PCI @@ -1199,7 +1251,7 @@ typedef struct work_queue_item work_queue_item; do { \ (w)->wqi_func = (func); \ (w)->wqi_ctx = (ctx); \ - INIT_LIST_HEAD(&((w)->wqi_entry)); \ + InitializeListHead(&((w)->wqi_entry)); \ } while (0); \ @@ -1251,6 +1303,16 @@ extern void ctxsw_wtou(void); extern int ntoskrnl_libinit(void); extern int ntoskrnl_libfini(void); + +extern uint32_t RtlUnicodeStringToAnsiString(ansi_string *, + unicode_string *, uint8_t); +extern uint32_t RtlAnsiStringToUnicodeString(unicode_string *, + ansi_string *, uint8_t); +extern void RtlInitAnsiString(ansi_string *, char *); +extern void RtlInitUnicodeString(unicode_string *, + uint16_t *); +extern void RtlFreeUnicodeString(unicode_string *); +extern void RtlFreeAnsiString(ansi_string *); extern void KeInitializeDpc(kdpc *, void *, void *); extern uint8_t KeInsertQueueDpc(kdpc *, void *, void *); extern uint8_t KeRemoveQueueDpc(kdpc *); @@ -1280,10 +1342,16 @@ extern void KeAcquireSpinLockAtDpcLevel(kspin_lock *); extern void KeReleaseSpinLockFromDpcLevel(kspin_lock *); #endif extern void KeInitializeSpinLock(kspin_lock *); +extern uint8_t KeSynchronizeExecution(kinterrupt *, void *, void *); extern uintptr_t InterlockedExchange(volatile uint32_t *, uintptr_t); extern void *ExAllocatePoolWithTag(uint32_t, size_t, uint32_t); extern void ExFreePool(void *); +extern uint32_t IoConnectInterrupt(kinterrupt **, void *, void *, + kspin_lock *, uint32_t, uint8_t, uint8_t, uint8_t, uint8_t, + uint32_t, uint8_t); +extern void MmBuildMdlForNonPagedPool(mdl *); +extern void IoDisconnectInterrupt(kinterrupt *); extern uint32_t IoAllocateDriverObjectExtension(driver_object *, void *, uint32_t, void **); extern void *IoGetDriverObjectExtension(driver_object *, void *); diff --git a/sys/compat/ndis/subr_hal.c b/sys/compat/ndis/subr_hal.c index ca8f009..fa376bf 100644 --- a/sys/compat/ndis/subr_hal.c +++ b/sys/compat/ndis/subr_hal.c @@ -80,12 +80,18 @@ static void READ_PORT_BUFFER_UCHAR(uint8_t *, static uint64_t KeQueryPerformanceCounter(uint64_t *); static void dummy (void); -extern struct mtx_pool *ndis_mtxpool; +#define NDIS_MAXCPUS 64 +static struct mtx disp_lock[NDIS_MAXCPUS]; int hal_libinit() { image_patch_table *patch; + int i; + + for (i = 0; i < NDIS_MAXCPUS; i++) + mtx_init(&disp_lock[i], "HAL preemption lock", + "HAL lock", MTX_RECURSE|MTX_DEF); patch = hal_functbl; while (patch->ipt_func != NULL) { @@ -95,6 +101,7 @@ hal_libinit() patch++; } + return(0); } @@ -102,6 +109,10 @@ int hal_libfini() { image_patch_table *patch; + int i; + + for (i = 0; i < NDIS_MAXCPUS; i++) + mtx_destroy(&disp_lock[i]); patch = hal_functbl; while (patch->ipt_func != NULL) { @@ -285,6 +296,72 @@ READ_PORT_BUFFER_UCHAR(port, val, cnt) * KeAcquireSpinLock() must be running at IRQL <= DISPATCH_LEVEL. If * we detect someone trying to acquire a spinlock from DEVICE_LEVEL * or HIGH_LEVEL, we panic. + * + * Alternate sleep-lock-based spinlock implementation + * -------------------------------------------------- + * + * The earlier spinlock implementation was arguably a bit of a hack + * and presented several problems. It was basically designed to provide + * the functionality of spinlocks without incurring the wrath of + * WITNESS. We could get away with using both our spinlock implementation + * and FreeBSD sleep locks at the same time, but if WITNESS knew what + * we were really up to, it would have spanked us rather severely. + * + * There's another method we can use based entirely on sleep locks. + * First, it's important to realize that everything we're locking + * resides inside Project Evil itself: any critical data being locked + * by drivers belongs to the drivers, and should not be referenced + * by any other OS code outside of the NDISulator. The priority-based + * locking scheme has system-wide effects, just like real spinlocks + * (blocking preemption affects the whole CPU), but since we keep all + * our critical data private, we can use a simpler mechanism that + * affects only code/threads directly related to Project Evil. + * + * The idea is to create a sleep lock mutex for each CPU in the system. + * When a CPU running in the NDISulator wants to acquire a spinlock, it + * does the following: + * - Pin ourselves to the current CPU + * - Acquire the mutex for the current CPU + * - Spin on the spinlock variable using atomic test and set, just like + * a real spinlock. + * - Once we have the lock, we execute our critical code + * + * To give up the lock, we do: + * - Clear the spinlock variable with an atomic op + * - Release the per-CPU mutex + * - Unpin ourselves from the current CPU. + * + * On a uniprocessor system, this means all threads that access protected + * data are serialized through the per-CPU mutex. After one thread + * acquires the 'spinlock,' any other thread that uses a spinlock on the + * current CPU will block on the per-CPU mutex, which has the same general + * effect of blocking pre-emption, but _only_ for those threads that are + * running NDISulator code. + * + * On a multiprocessor system, threads on different CPUs all block on + * their respective per-CPU mutex, and the atomic test/set operation + * on the spinlock variable provides inter-CPU synchronization, though + * only for threads running NDISulator code. + * + * This method solves an important problem. In Windows, you're allowed + * to do an ExAllocatePoolWithTag() with a spinlock held, provided you + * allocate from NonPagedPool. This implies an atomic heap allocation + * that will not cause the current thread to sleep. (You can't sleep + * while holding real spinlock: clowns will eat you.) But in FreeBSD, + * malloc(9) _always_ triggers the acquisition of a sleep lock, even + * when you use M_NOWAIT. This is not a problem for FreeBSD native + * code: you're allowed to sleep in things like interrupt threads. But + * it is a problem with the old priority-based spinlock implementation: + * even though we get away with it most of the time, we really can't + * do a malloc(9) after doing a KeAcquireSpinLock() or KeRaiseIrql(). + * With the new implementation, it's not a problem: you're allowed to + * acquire more than one sleep lock (as long as you avoid lock order + * reversals). + * + * The one drawback to this approach is that now we have a lot of + * contention on one per-CPU mutex within the NDISulator code. Whether + * or not this is preferable to the expected Windows spinlock behavior + * of blocking pre-emption is debatable. */ uint8_t @@ -314,10 +391,10 @@ KfReleaseSpinLock(lock, newirql) return; } - uint8_t +uint8_t KeGetCurrentIrql() { - if (AT_DISPATCH_LEVEL(curthread)) + if (mtx_owned(&disp_lock[curthread->td_oncpu])) return(DISPATCH_LEVEL); return(PASSIVE_LEVEL); } @@ -338,19 +415,14 @@ KfRaiseIrql(irql) { uint8_t oldirql; - if (irql < KeGetCurrentIrql()) + oldirql = KeGetCurrentIrql(); + if (irql < oldirql) panic("IRQL_NOT_LESS_THAN"); - if (KeGetCurrentIrql() == DISPATCH_LEVEL) - return(DISPATCH_LEVEL); - - mtx_lock_spin(&sched_lock); - oldirql = curthread->td_base_pri; - sched_prio(curthread, PI_REALTIME); -#if __FreeBSD_version < 600000 - curthread->td_base_pri = PI_REALTIME; -#endif - mtx_unlock_spin(&sched_lock); + if (oldirql != DISPATCH_LEVEL) { + sched_pin(); + mtx_lock(&disp_lock[curthread->td_oncpu]); + } return(oldirql); } @@ -365,12 +437,8 @@ KfLowerIrql(oldirql) if (KeGetCurrentIrql() != DISPATCH_LEVEL) panic("IRQL_NOT_GREATER_THAN"); - mtx_lock_spin(&sched_lock); -#if __FreeBSD_version < 600000 - curthread->td_base_pri = oldirql; -#endif - sched_prio(curthread, oldirql); - mtx_unlock_spin(&sched_lock); + mtx_unlock(&disp_lock[curthread->td_oncpu]); + sched_unpin(); return; } diff --git a/sys/compat/ndis/subr_ndis.c b/sys/compat/ndis/subr_ndis.c index 5dfe8e5..7813515 100644 --- a/sys/compat/ndis/subr_ndis.c +++ b/sys/compat/ndis/subr_ndis.c @@ -104,6 +104,13 @@ __FBSDID("$FreeBSD$"); #include <compat/ndis/ndis_var.h> #include <dev/if_ndis/if_ndisvar.h> +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> +#include <vm/uma.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> + static char ndis_filepath[MAXPATHLEN]; extern struct nd_head ndis_devhead; @@ -124,17 +131,17 @@ static ndis_status NdisMSetAttributesEx(ndis_handle, ndis_handle, static void NdisOpenConfiguration(ndis_status *, ndis_handle *, ndis_handle); static void NdisOpenConfigurationKeyByIndex(ndis_status *, - ndis_handle, uint32_t, ndis_unicode_string *, ndis_handle *); + ndis_handle, uint32_t, unicode_string *, ndis_handle *); static void NdisOpenConfigurationKeyByName(ndis_status *, - ndis_handle, ndis_unicode_string *, ndis_handle *); + ndis_handle, unicode_string *, ndis_handle *); static ndis_status ndis_encode_parm(ndis_miniport_block *, struct sysctl_oid *, ndis_parm_type, ndis_config_parm **); static ndis_status ndis_decode_parm(ndis_miniport_block *, ndis_config_parm *, char *); static void NdisReadConfiguration(ndis_status *, ndis_config_parm **, - ndis_handle, ndis_unicode_string *, ndis_parm_type); + ndis_handle, unicode_string *, ndis_parm_type); static void NdisWriteConfiguration(ndis_status *, ndis_handle, - ndis_unicode_string *, ndis_config_parm *); + unicode_string *, ndis_config_parm *); static void NdisCloseConfiguration(ndis_handle); static void NdisAllocateSpinLock(ndis_spin_lock *); static void NdisFreeSpinLock(ndis_spin_lock *); @@ -213,11 +220,10 @@ static void NdisInitializeEvent(ndis_event *); static void NdisSetEvent(ndis_event *); static void NdisResetEvent(ndis_event *); static uint8_t NdisWaitEvent(ndis_event *, uint32_t); -static ndis_status NdisUnicodeStringToAnsiString(ndis_ansi_string *, - ndis_unicode_string *); +static ndis_status NdisUnicodeStringToAnsiString(ansi_string *, + unicode_string *); static ndis_status - NdisAnsiStringToUnicodeString(ndis_unicode_string *, - ndis_ansi_string *); + NdisAnsiStringToUnicodeString(unicode_string *, ansi_string *); static ndis_status NdisMPciAssignResources(ndis_handle, uint32_t, ndis_resource_list **); static ndis_status NdisMRegisterInterrupt(ndis_miniport_interrupt *, @@ -248,11 +254,10 @@ static uint8_t void *, void *); static void NdisGetCurrentSystemTime(uint64_t *); static void NdisGetSystemUpTime(uint32_t *); -static void NdisInitializeString(ndis_unicode_string *, char *); -static void NdisInitAnsiString(ndis_ansi_string *, char *); -static void NdisInitUnicodeString(ndis_unicode_string *, - uint16_t *); -static void NdisFreeString(ndis_unicode_string *); +static void NdisInitializeString(unicode_string *, char *); +static void NdisInitAnsiString(ansi_string *, char *); +static void NdisInitUnicodeString(unicode_string *, uint16_t *); +static void NdisFreeString(unicode_string *); static ndis_status NdisMRemoveMiniport(ndis_handle *); static void NdisTerminateWrapper(ndis_handle, void *); static void NdisMGetDeviceProperty(ndis_handle, device_object **, @@ -264,7 +269,7 @@ static void NdisGetFirstBufferFromPacketSafe(ndis_packet *, ndis_buffer **, void **, uint32_t *, uint32_t *, uint32_t); static int ndis_find_sym(linker_file_t, char *, char *, caddr_t *); static void NdisOpenFile(ndis_status *, ndis_handle *, uint32_t *, - ndis_unicode_string *, ndis_physaddr); + unicode_string *, ndis_physaddr); static void NdisMapFile(ndis_status *, void **, ndis_handle); static void NdisUnmapFile(ndis_handle); static void NdisCloseFile(ndis_handle); @@ -272,18 +277,19 @@ static uint8_t NdisSystemProcessorCount(void); static void NdisMIndicateStatusComplete(ndis_handle); static void NdisMIndicateStatus(ndis_handle, ndis_status, void *, uint32_t); +static void ndis_intr(void *); +static void ndis_intrhand(kdpc *, ndis_miniport_interrupt *, void *, void *); static funcptr ndis_findwrap(funcptr); static void NdisCopyFromPacketToPacket(ndis_packet *, uint32_t, uint32_t, ndis_packet *, uint32_t, uint32_t *); static void NdisCopyFromPacketToPacketSafe(ndis_packet *, uint32_t, uint32_t, ndis_packet *, uint32_t, uint32_t *, uint32_t); static ndis_status NdisMRegisterDevice(ndis_handle, - ndis_unicode_string *, ndis_unicode_string *, driver_dispatch **, + unicode_string *, unicode_string *, driver_dispatch **, void **, ndis_handle *); static ndis_status NdisMDeregisterDevice(ndis_handle); static ndis_status - NdisMQueryAdapterInstanceName(ndis_unicode_string *, - ndis_handle); + NdisMQueryAdapterInstanceName(unicode_string *, ndis_handle); static void NdisMRegisterUnloadHandler(ndis_handle, void *); static void dummy(void); @@ -346,57 +352,6 @@ ndis_findwrap(func) } /* - * NDIS deals with strings in unicode format, so we have - * do deal with them that way too. For now, we only handle - * conversion between unicode and ASCII since that's all - * that device drivers care about. - */ - -int -ndis_ascii_to_unicode(ascii, unicode) - char *ascii; - uint16_t **unicode; -{ - uint16_t *ustr; - int i; - - if (*unicode == NULL) - *unicode = malloc(strlen(ascii) * 2, M_DEVBUF, M_NOWAIT); - - if (*unicode == NULL) - return(ENOMEM); - ustr = *unicode; - for (i = 0; i < strlen(ascii); i++) { - *ustr = (uint16_t)ascii[i]; - ustr++; - } - - return(0); -} - -int -ndis_unicode_to_ascii(unicode, ulen, ascii) - uint16_t *unicode; - int ulen; - char **ascii; -{ - uint8_t *astr; - int i; - - if (*ascii == NULL) - *ascii = malloc((ulen / 2) + 1, M_DEVBUF, M_NOWAIT|M_ZERO); - if (*ascii == NULL) - return(ENOMEM); - astr = *ascii; - for (i = 0; i < ulen / 2; i++) { - *astr = (uint8_t)unicode[i]; - astr++; - } - - return(0); -} - -/* * This routine does the messy Windows Driver Model device attachment * stuff on behalf of NDIS drivers. We register our own AddDevice * routine here @@ -465,8 +420,10 @@ NdisMRegisterMiniport(handle, characteristics, len) if (IoAllocateDriverObjectExtension(drv, (void *)1, sizeof(ndis_miniport_characteristics), (void **)&ch) != - STATUS_SUCCESS) + STATUS_SUCCESS) { + printf("register error\n"); return(NDIS_STATUS_RESOURCES); + } bzero((char *)ch, sizeof(ndis_miniport_characteristics)); @@ -489,10 +446,10 @@ NdisAllocateMemoryWithTag(vaddr, len, tag) { void *mem; - mem = ExAllocatePoolWithTag(NonPagedPool, len, tag); - if (mem == NULL) + if (mem == NULL) { return(NDIS_STATUS_RESOURCES); + } *vaddr = mem; return(NDIS_STATUS_SUCCESS); @@ -568,7 +525,7 @@ static void NdisOpenConfigurationKeyByName(status, cfg, subkey, subhandle) ndis_status *status; ndis_handle cfg; - ndis_unicode_string *subkey; + unicode_string *subkey; ndis_handle *subhandle; { *subhandle = cfg; @@ -582,7 +539,7 @@ NdisOpenConfigurationKeyByIndex(status, cfg, idx, subkey, subhandle) ndis_status *status; ndis_handle cfg; uint32_t idx; - ndis_unicode_string *subkey; + unicode_string *subkey; ndis_handle *subhandle; { *status = NDIS_STATUS_FAILURE; @@ -597,36 +554,58 @@ ndis_encode_parm(block, oid, type, parm) ndis_parm_type type; ndis_config_parm **parm; { - uint16_t *unicode; - ndis_unicode_string *ustr; + ndis_config_parm *p; + ndis_parmlist_entry *np; + unicode_string *us; + ansi_string as; int base = 0; + uint32_t val; + char tmp[32]; - unicode = (uint16_t *)&block->nmb_dummybuf; + np = ExAllocatePoolWithTag(NonPagedPool, + sizeof(ndis_parmlist_entry), 0); + if (np == NULL) + return(NDIS_STATUS_RESOURCES); + InsertHeadList((&block->nmb_parmlist), (&np->np_list)); + *parm = p = &np->np_parm; switch(type) { case ndis_parm_string: - ndis_ascii_to_unicode((char *)oid->oid_arg1, &unicode); - (*parm)->ncp_type = ndis_parm_string; - ustr = &(*parm)->ncp_parmdata.ncp_stringdata; - ustr->us_len = strlen((char *)oid->oid_arg1) * 2; - ustr->us_buf = unicode; + /* See if this might be a number. */ + val = strtoul((char *)oid->oid_arg1, NULL, 10); + us = &p->ncp_parmdata.ncp_stringdata; + p->ncp_type = ndis_parm_string; + if (val) { + snprintf(tmp, 32, "%x", val); + RtlInitAnsiString(&as, tmp); + } else { + RtlInitAnsiString(&as, (char *)oid->oid_arg1); + } + + if (RtlAnsiStringToUnicodeString(us, &as, TRUE)) { + ExFreePool(np); + return(NDIS_STATUS_RESOURCES); + } break; case ndis_parm_int: if (strncmp((char *)oid->oid_arg1, "0x", 2) == 0) base = 16; else base = 10; - (*parm)->ncp_type = ndis_parm_int; - (*parm)->ncp_parmdata.ncp_intdata = + p->ncp_type = ndis_parm_int; + p->ncp_parmdata.ncp_intdata = strtol((char *)oid->oid_arg1, NULL, base); break; case ndis_parm_hexint: +#ifdef notdef if (strncmp((char *)oid->oid_arg1, "0x", 2) == 0) base = 16; else base = 10; - (*parm)->ncp_type = ndis_parm_hexint; - (*parm)->ncp_parmdata.ncp_intdata = +#endif + base = 16; + p->ncp_type = ndis_parm_hexint; + p->ncp_parmdata.ncp_intdata = strtoul((char *)oid->oid_arg1, NULL, base); break; default: @@ -689,15 +668,15 @@ NdisReadConfiguration(status, parm, cfg, key, type) ndis_status *status; ndis_config_parm **parm; ndis_handle cfg; - ndis_unicode_string *key; + unicode_string *key; ndis_parm_type type; { char *keystr = NULL; - uint16_t *unicode; ndis_miniport_block *block; struct ndis_softc *sc; struct sysctl_oid *oidp; struct sysctl_ctx_entry *e; + ansi_string as; block = (ndis_miniport_block *)cfg; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); @@ -707,10 +686,12 @@ NdisReadConfiguration(status, parm, cfg, key, type) return; } - ndis_unicode_to_ascii(key->us_buf, key->us_len, &keystr); - *parm = &block->nmb_replyparm; - bzero((char *)&block->nmb_replyparm, sizeof(ndis_config_parm)); - unicode = (uint16_t *)&block->nmb_dummybuf; + if (RtlUnicodeStringToAnsiString(&as, key, TRUE)) { + *status = NDIS_STATUS_RESOURCES; + return; + } + + keystr = as.as_buf; /* * See if registry key is already in a list of known keys @@ -724,12 +705,13 @@ NdisReadConfiguration(status, parm, cfg, key, type) oidp = e->entry; if (ndis_strcasecmp(oidp->oid_name, keystr) == 0) { if (strcmp((char *)oidp->oid_arg1, "UNSET") == 0) { - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); *status = NDIS_STATUS_FAILURE; return; } + *status = ndis_encode_parm(block, oidp, type, parm); - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); return; } } @@ -753,7 +735,7 @@ NdisReadConfiguration(status, parm, cfg, key, type) ndis_add_sysctl(sc, keystr, "(dynamic string key)", "UNSET", CTLFLAG_RW); - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); *status = NDIS_STATUS_FAILURE; return; @@ -765,15 +747,16 @@ ndis_decode_parm(block, parm, val) ndis_config_parm *parm; char *val; { - ndis_unicode_string *ustr; - char *astr = NULL; + unicode_string *ustr; + ansi_string as; switch(parm->ncp_type) { case ndis_parm_string: ustr = &parm->ncp_parmdata.ncp_stringdata; - ndis_unicode_to_ascii(ustr->us_buf, ustr->us_len, &astr); - bcopy(astr, val, 254); - free(astr, M_DEVBUF); + if (RtlUnicodeStringToAnsiString(&as, ustr, TRUE)) + return(NDIS_STATUS_RESOURCES); + bcopy(as.as_buf, val, as.as_len); + RtlFreeAnsiString(&as); break; case ndis_parm_int: sprintf(val, "%d", parm->ncp_parmdata.ncp_intdata); @@ -792,9 +775,10 @@ static void NdisWriteConfiguration(status, cfg, key, parm) ndis_status *status; ndis_handle cfg; - ndis_unicode_string *key; + unicode_string *key; ndis_config_parm *parm; { + ansi_string as; char *keystr = NULL; ndis_miniport_block *block; struct ndis_softc *sc; @@ -805,13 +789,18 @@ NdisWriteConfiguration(status, cfg, key, parm) block = (ndis_miniport_block *)cfg; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); - ndis_unicode_to_ascii(key->us_buf, key->us_len, &keystr); + if (RtlUnicodeStringToAnsiString(&as, key, TRUE)) { + *status = NDIS_STATUS_RESOURCES; + return; + } + + keystr = as.as_buf; /* Decode the parameter into a string. */ bzero(val, sizeof(val)); *status = ndis_decode_parm(block, parm, val); if (*status != NDIS_STATUS_SUCCESS) { - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); return; } @@ -826,7 +815,7 @@ NdisWriteConfiguration(status, cfg, key, parm) if (ndis_strcasecmp(oidp->oid_name, keystr) == 0) { /* Found it, set the value. */ strcpy((char *)oidp->oid_arg1, val); - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); return; } } @@ -835,7 +824,7 @@ NdisWriteConfiguration(status, cfg, key, parm) ndis_add_sysctl(sc, keystr, "(dynamically set key)", val, CTLFLAG_RW); - free(keystr, M_DEVBUF); + RtlFreeAnsiString(&as); *status = NDIS_STATUS_SUCCESS; return; } @@ -844,6 +833,22 @@ static void NdisCloseConfiguration(cfg) ndis_handle cfg; { + list_entry *e; + ndis_parmlist_entry *pe; + ndis_miniport_block *block; + ndis_config_parm *p; + + block = (ndis_miniport_block *)cfg; + + while (!IsListEmpty(&block->nmb_parmlist)) { + e = RemoveHeadList(&block->nmb_parmlist); + pe = CONTAINING_RECORD(e, ndis_parmlist_entry, np_list); + p = &pe->np_parm; + if (p->ncp_type == ndis_parm_string) + RtlFreeUnicodeString(&p->ncp_parmdata.ncp_stringdata); + ExFreePool(e); + } + return; } @@ -1038,6 +1043,7 @@ NdisWritePciSlotInformation(adapter, slot, offset, buf, len) * The errorlog routine uses a variable argument list, so we * have to declare it this way. */ + #define ERRMSGLEN 512 static void NdisWriteErrorLogEntry(ndis_handle adapter, ndis_error_code code, @@ -1046,13 +1052,15 @@ NdisWriteErrorLogEntry(ndis_handle adapter, ndis_error_code code, ndis_miniport_block *block; va_list ap; int i, error; - char *str = NULL, *ustr = NULL; + char *str = NULL; uint16_t flags; char msgbuf[ERRMSGLEN]; device_t dev; driver_object *drv; struct ndis_softc *sc; struct ifnet *ifp; + unicode_string us; + ansi_string as; block = (ndis_miniport_block *)adapter; dev = block->nmb_physdeviceobj->do_devext; @@ -1064,10 +1072,10 @@ NdisWriteErrorLogEntry(ndis_handle adapter, ndis_error_code code, code, &str, &i, &flags); if (error == 0 && flags & MESSAGE_RESOURCE_UNICODE && ifp->if_flags & IFF_DEBUG) { - ustr = msgbuf; - ndis_unicode_to_ascii((uint16_t *)str, - ((i / 2)) > (ERRMSGLEN - 1) ? ERRMSGLEN : i, &ustr); - str = ustr; + RtlInitUnicodeString(&us, (uint16_t *)msgbuf); + if (RtlUnicodeStringToAnsiString(&as, &us, TRUE)) + return; + str = as.as_buf; } device_printf (dev, "NDIS ERROR: %x (%s)\n", code, @@ -1082,6 +1090,9 @@ NdisWriteErrorLogEntry(ndis_handle adapter, ndis_error_code code, va_end(ap); } + if (str != NULL) + RtlFreeAnsiString(&as); + return; } @@ -1404,10 +1415,18 @@ NdisReadNetworkAddress(status, addr, addrlen, adapter) block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); +#ifdef IFP2ENADDR if (bcmp(IFP2ENADDR(sc->ifp), empty, ETHER_ADDR_LEN) == 0) +#else + if (bcmp(sc->arpcom.ac_enaddr, empty, ETHER_ADDR_LEN) == 0) +#endif *status = NDIS_STATUS_FAILURE; else { +#ifdef IFP2ENADDR *addr = IFP2ENADDR(sc->ifp); +#else + *addr = sc->arpcom.ac_enaddr; +#endif *addrlen = ETHER_ADDR_LEN; *status = NDIS_STATUS_SUCCESS; } @@ -1530,6 +1549,8 @@ NdisMAllocateSharedMemory(adapter, len, cached, vaddr, paddr) if (sh == NULL) return; + InitializeListHead(&sh->ndis_list); + /* * When performing shared memory allocations, create a tag * with a lowaddr limit that restricts physical memory mappings @@ -1582,10 +1603,11 @@ NdisMAllocateSharedMemory(adapter, len, cached, vaddr, paddr) * searching based on the virtual address fails. */ + NDIS_LOCK(sc); sh->ndis_paddr.np_quad = paddr->np_quad; sh->ndis_saddr = *vaddr; - sh->ndis_next = sc->ndis_shlist; - sc->ndis_shlist = sh; + InsertHeadList((&sc->ndis_shlist), (&sh->ndis_list)); + NDIS_UNLOCK(sc); return; } @@ -1674,21 +1696,24 @@ NdisMFreeSharedMemory(adapter, len, cached, vaddr, paddr) { ndis_miniport_block *block; struct ndis_softc *sc; - struct ndis_shmem *sh, *prev; + struct ndis_shmem *sh = NULL; + list_entry *l; if (vaddr == NULL || adapter == NULL) return; block = (ndis_miniport_block *)adapter; sc = device_get_softc(block->nmb_physdeviceobj->do_devext); - sh = prev = sc->ndis_shlist; /* Sanity check: is list empty? */ - if (sh == NULL) + if (IsListEmpty(&sc->ndis_shlist)) return; - while (sh) { + NDIS_LOCK(sc); + l = sc->ndis_shlist.nle_flink; + while (l != &sc->ndis_shlist) { + sh = CONTAINING_RECORD(l, struct ndis_shmem, ndis_list); if (sh->ndis_saddr == vaddr) break; /* @@ -1697,26 +1722,25 @@ NdisMFreeSharedMemory(adapter, len, cached, vaddr, paddr) */ if (sh->ndis_paddr.np_quad == paddr.np_quad) break; - prev = sh; - sh = sh->ndis_next; + l = l->nle_flink; } if (sh == NULL) { + NDIS_UNLOCK(sc); printf("NDIS: buggy driver tried to free " "invalid shared memory: vaddr: %p paddr: 0x%jx\n", vaddr, (uintmax_t)paddr.np_quad); return; } + RemoveEntryList(&sh->ndis_list); + + NDIS_UNLOCK(sc); + bus_dmamap_unload(sh->ndis_stag, sh->ndis_smap); bus_dmamem_free(sh->ndis_stag, sh->ndis_saddr, sh->ndis_smap); bus_dma_tag_destroy(sh->ndis_stag); - if (sh == sc->ndis_shlist) - sc->ndis_shlist = sh->ndis_next; - else - prev->ndis_next = sh->ndis_next; - free(sh, M_DEVBUF); return; @@ -2077,6 +2101,7 @@ NdisAllocateBufferPool(status, pool, descnum) ndis_handle *pool; uint32_t descnum; { + /* * The only thing we can really do here is verify that descnum * is a reasonable value, but I really don't know what to check @@ -2111,6 +2136,8 @@ NdisAllocateBuffer(status, buffer, pool, vaddr, len) return; } + MmBuildMdlForNonPagedPool(buf); + *buffer = buf; *status = NDIS_STATUS_SUCCESS; @@ -2260,37 +2287,35 @@ NdisWaitEvent(event, msecs) static ndis_status NdisUnicodeStringToAnsiString(dstr, sstr) - ndis_ansi_string *dstr; - ndis_unicode_string *sstr; + ansi_string *dstr; + unicode_string *sstr; { - if (dstr == NULL || sstr == NULL) - return(NDIS_STATUS_FAILURE); - if (ndis_unicode_to_ascii(sstr->us_buf, - sstr->us_len, &dstr->nas_buf)) + uint32_t rval; + + rval = RtlUnicodeStringToAnsiString(dstr, sstr, FALSE); + + if (rval == STATUS_INSUFFICIENT_RESOURCES) + return(NDIS_STATUS_RESOURCES); + if (rval) return(NDIS_STATUS_FAILURE); - dstr->nas_len = dstr->nas_maxlen = strlen(dstr->nas_buf); + return (NDIS_STATUS_SUCCESS); } static ndis_status NdisAnsiStringToUnicodeString(dstr, sstr) - ndis_unicode_string *dstr; - ndis_ansi_string *sstr; + unicode_string *dstr; + ansi_string *sstr; { - char *str; - if (dstr == NULL || sstr == NULL) - return(NDIS_STATUS_FAILURE); - str = malloc(sstr->nas_len + 1, M_DEVBUF, M_NOWAIT); - if (str == NULL) - return(NDIS_STATUS_FAILURE); - strncpy(str, sstr->nas_buf, sstr->nas_len); - *(str + sstr->nas_len) = '\0'; - if (ndis_ascii_to_unicode(str, &dstr->us_buf)) { - free(str, M_DEVBUF); + uint32_t rval; + + rval = RtlAnsiStringToUnicodeString(dstr, sstr, FALSE); + + if (rval == STATUS_INSUFFICIENT_RESOURCES) + return(NDIS_STATUS_RESOURCES); + if (rval) return(NDIS_STATUS_FAILURE); - } - dstr->us_len = dstr->us_maxlen = sstr->nas_len * 2; - free(str, M_DEVBUF); + return (NDIS_STATUS_SUCCESS); } @@ -2311,6 +2336,77 @@ NdisMPciAssignResources(adapter, slot, list) return (NDIS_STATUS_SUCCESS); } +static void +ndis_intr(arg) + void *arg; +{ + struct ndis_softc *sc; + struct ifnet *ifp; + int is_our_intr = 0; + int call_isr = 0; + ndis_miniport_interrupt *intr; + + sc = arg; + ifp = sc->ifp; + intr = sc->ndis_block->nmb_interrupt; + + if (intr == NULL || sc->ndis_block->nmb_miniportadapterctx == NULL) + return; + + if (sc->ndis_block->nmb_interrupt->ni_isrreq == TRUE) + ndis_isr(sc, &is_our_intr, &call_isr); + else { + ndis_disable_intr(sc); + call_isr = 1; + } + + if ((is_our_intr || call_isr)) + IoRequestDpc(sc->ndis_block->nmb_deviceobj, NULL, sc); + + return; +} + +static void +ndis_intrhand(dpc, intr, sysarg1, sysarg2) + kdpc *dpc; + ndis_miniport_interrupt *intr; + void *sysarg1; + void *sysarg2; +{ + struct ndis_softc *sc; + ndis_miniport_block *block; + ndis_handle adapter; + + block = intr->ni_block; + adapter = block->nmb_miniportadapterctx; + sc = device_get_softc(block->nmb_physdeviceobj->do_devext); + + if (NDIS_SERIALIZED(sc->ndis_block)) + KeAcquireSpinLockAtDpcLevel(&block->nmb_lock); + + MSCALL1(intr->ni_isrfunc, adapter); + + /* If there's a MiniportEnableInterrupt() routine, call it. */ + + ndis_enable_intr(sc); + + if (NDIS_SERIALIZED(sc->ndis_block)) + KeReleaseSpinLockFromDpcLevel(&block->nmb_lock); + + /* + * Set the completion event if we've drained all + * pending interrupts. + */ + + KeAcquireSpinLockAtDpcLevel(&intr->ni_dpccountlock); + intr->ni_dpccnt--; + if (intr->ni_dpccnt == 0) + KeSetEvent(&intr->ni_dpcevt, IO_NO_INCREMENT, FALSE); + KeReleaseSpinLockFromDpcLevel(&intr->ni_dpccountlock); + + return; +} + static ndis_status NdisMRegisterInterrupt(intr, adapter, ivec, ilevel, reqisr, shared, imode) ndis_miniport_interrupt *intr; @@ -2322,15 +2418,40 @@ NdisMRegisterInterrupt(intr, adapter, ivec, ilevel, reqisr, shared, imode) ndis_interrupt_mode imode; { ndis_miniport_block *block; + ndis_miniport_characteristics *ch; + struct ndis_softc *sc; + int error; block = adapter; + sc = device_get_softc(block->nmb_physdeviceobj->do_devext); + ch = IoGetDriverObjectExtension(block->nmb_deviceobj->do_drvobj, + (void *)1); + + intr->ni_rsvd = ExAllocatePoolWithTag(NonPagedPool, + sizeof(struct mtx), 0); + if (intr->ni_rsvd == NULL) + return(NDIS_STATUS_RESOURCES); intr->ni_block = adapter; intr->ni_isrreq = reqisr; intr->ni_shared = shared; - block->nmb_interrupt = intr; + intr->ni_dpccnt = 0; + intr->ni_isrfunc = ch->nmc_interrupt_func; + intr->ni_dpcfunc = ch->nmc_isr_func; + + KeInitializeEvent(&intr->ni_dpcevt, EVENT_TYPE_NOTIFY, TRUE); + KeInitializeDpc(&intr->ni_dpc, + ndis_findwrap((funcptr)ndis_intrhand), intr); + KeSetImportanceDpc(&intr->ni_dpc, KDPC_IMPORTANCE_LOW); - KeInitializeSpinLock(&intr->ni_dpccountlock); + error = IoConnectInterrupt(&intr->ni_introbj, + ndis_findwrap((funcptr)ndis_intr), sc, NULL, + ivec, ilevel, 0, imode, shared, 0, FALSE); + + if (error != STATUS_SUCCESS) + return(NDIS_STATUS_FAILURE); + + block->nmb_interrupt = intr; return(NDIS_STATUS_SUCCESS); } @@ -2339,6 +2460,29 @@ static void NdisMDeregisterInterrupt(intr) ndis_miniport_interrupt *intr; { + ndis_miniport_block *block; + struct ndis_softc *sc; + uint8_t irql; + + block = intr->ni_block; + sc = device_get_softc(block->nmb_physdeviceobj->do_devext); + + /* Should really be KeSynchronizeExecution() */ + + KeAcquireSpinLock(intr->ni_introbj->ki_lock, &irql); + block->nmb_interrupt = NULL; + KeReleaseSpinLock(intr->ni_introbj->ki_lock, irql); +/* + KeFlushQueuedDpcs(); +*/ + /* Disconnect our ISR */ + + IoDisconnectInterrupt(intr->ni_introbj); + + KeWaitForSingleObject((nt_dispatch_header *)&intr->ni_dpcevt, + 0, 0, FALSE, NULL); + KeResetEvent(&intr->ni_dpcevt); + return; } @@ -2429,23 +2573,22 @@ static void NdisMSleep(usecs) uint32_t usecs; { - struct timeval tv; + ktimer timer; /* * During system bootstrap, (i.e. cold == 1), we aren't - * allowed to msleep(), so calling ndis_thsuspend() here - * will return 0, and we won't actually have delayed. This - * is a problem because some drivers expect NdisMSleep() - * to always wait, and might fail if the expected delay - * period does not in fact elapse. As a workaround, if the - * attempt to sleep delay fails, we do a hard DELAY() instead. + * allowed to sleep, so we have to do a hard DELAY() + * instead. */ - tv.tv_sec = 0; - tv.tv_usec = usecs; - if (ndis_thsuspend(curthread->td_proc, NULL, tvtohz(&tv)) == 0) + if (cold) DELAY(usecs); - + else { + KeInitializeTimer(&timer); + KeSetTimer(&timer, ((int64_t)usecs * -10), NULL); + KeWaitForSingleObject((nt_dispatch_header *)&timer, + 0, 0, FALSE, NULL); + } return; } @@ -2571,19 +2714,7 @@ NdisMSynchronizeWithInterrupt(intr, syncfunc, syncctx) void *syncfunc; void *syncctx; { - uint8_t (*sync)(void *); - uint8_t rval; - uint8_t irql; - - if (syncfunc == NULL || syncctx == NULL) - return(0); - - sync = syncfunc; - KeAcquireSpinLock(&intr->ni_dpccountlock, &irql); - rval = MSCALL1(sync, syncctx); - KeReleaseSpinLock(&intr->ni_dpccountlock, irql); - - return(rval); + return(KeSynchronizeExecution(intr->ni_introbj, syncfunc, syncctx)); } /* @@ -2620,28 +2751,20 @@ NdisGetSystemUpTime(tval) static void NdisInitializeString(dst, src) - ndis_unicode_string *dst; + unicode_string *dst; char *src; { - ndis_unicode_string *u; - - u = dst; - u->us_buf = NULL; - if (ndis_ascii_to_unicode(src, &u->us_buf)) - return; - u->us_len = u->us_maxlen = strlen(src) * 2; + ansi_string as; + RtlInitAnsiString(&as, src); + RtlAnsiStringToUnicodeString(dst, &as, TRUE); return; } static void NdisFreeString(str) - ndis_unicode_string *str; + unicode_string *str; { - if (str == NULL) - return; - if (str->us_buf != NULL) - free(str->us_buf, M_DEVBUF); - free(str, M_DEVBUF); + RtlFreeUnicodeString(str); return; } @@ -2654,47 +2777,19 @@ NdisMRemoveMiniport(adapter) static void NdisInitAnsiString(dst, src) - ndis_ansi_string *dst; + ansi_string *dst; char *src; { - ndis_ansi_string *a; - - a = dst; - if (a == NULL) - return; - if (src == NULL) { - a->nas_len = a->nas_maxlen = 0; - a->nas_buf = NULL; - } else { - a->nas_buf = src; - a->nas_len = a->nas_maxlen = strlen(src); - } - + RtlInitAnsiString(dst, src); return; } static void NdisInitUnicodeString(dst, src) - ndis_unicode_string *dst; + unicode_string *dst; uint16_t *src; { - ndis_unicode_string *u; - int i; - - u = dst; - if (u == NULL) - return; - if (src == NULL) { - u->us_len = u->us_maxlen = 0; - u->us_buf = NULL; - } else { - i = 0; - while(src[i] != 0) - i++; - u->us_buf = src; - u->us_len = u->us_maxlen = i * 2; - } - + RtlInitUnicodeString(dst, src); return; } @@ -2806,9 +2901,10 @@ NdisOpenFile(status, filehandle, filelength, filename, highestaddr) ndis_status *status; ndis_handle *filehandle; uint32_t *filelength; - ndis_unicode_string *filename; + unicode_string *filename; ndis_physaddr highestaddr; { + ansi_string as; char *afilename = NULL; struct thread *td = curthread; struct nameidata nd; @@ -2820,8 +2916,13 @@ NdisOpenFile(status, filehandle, filelength, filename, highestaddr) linker_file_t head, lf; caddr_t kldstart, kldend; - ndis_unicode_to_ascii(filename->us_buf, - filename->us_len, &afilename); + if (RtlUnicodeStringToAnsiString(&as, filename, TRUE)) { + *status = NDIS_STATUS_RESOURCES; + return; + } + + afilename = strdup(as.as_buf, M_DEVBUF); + RtlFreeAnsiString(&as); fh = ExAllocatePoolWithTag(NonPagedPool, sizeof(ndis_fh), 0); if (fh == NULL) { @@ -3120,6 +3221,7 @@ NdisScheduleWorkItem(work) ExInitializeWorkItem(wqi, (work_item_func)work->nwi_func, work->nwi_ctx); ExQueueWorkItem(wqi, WORKQUEUE_DELAYED); + return(NDIS_STATUS_SUCCESS); } @@ -3230,8 +3332,8 @@ NdisCopyFromPacketToPacketSafe(dpkt, doff, reqlen, spkt, soff, cpylen, prio) static ndis_status NdisMRegisterDevice(handle, devname, symname, majorfuncs, devobj, devhandle) ndis_handle handle; - ndis_unicode_string *devname; - ndis_unicode_string *symname; + unicode_string *devname; + unicode_string *symname; driver_dispatch *majorfuncs[]; void **devobj; ndis_handle *devhandle; @@ -3260,18 +3362,19 @@ NdisMDeregisterDevice(handle) static ndis_status NdisMQueryAdapterInstanceName(name, handle) - ndis_unicode_string *name; + unicode_string *name; ndis_handle handle; { ndis_miniport_block *block; device_t dev; + ansi_string as; block = (ndis_miniport_block *)handle; dev = block->nmb_physdeviceobj->do_devext; - ndis_ascii_to_unicode(__DECONST(char *, - device_get_nameunit(dev)), &name->us_buf); - name->us_len = strlen(device_get_nameunit(dev)) * 2; + RtlInitAnsiString(&as, __DECONST(char *, device_get_nameunit(dev))); + if (RtlAnsiStringToUnicodeString(name, &as, TRUE)) + return(NDIS_STATUS_RESOURCES); return(NDIS_STATUS_SUCCESS); } @@ -3422,6 +3525,8 @@ image_patch_table ndis_functbl[] = { IMPORT_SFUNC(NdisMRegisterUnloadHandler, 2), IMPORT_SFUNC(ndis_timercall, 4), IMPORT_SFUNC(ndis_asyncmem_complete, 2), + IMPORT_SFUNC(ndis_intr, 1), + IMPORT_SFUNC(ndis_intrhand, 4), /* * This last entry is a catch-all for any function we haven't diff --git a/sys/compat/ndis/subr_ntoskrnl.c b/sys/compat/ndis/subr_ntoskrnl.c index 6ff0689..fd3955c 100644 --- a/sys/compat/ndis/subr_ntoskrnl.c +++ b/sys/compat/ndis/subr_ntoskrnl.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$"); #endif #include <sys/kernel.h> #include <sys/proc.h> +#include <sys/condvar.h> #include <sys/kthread.h> #include <sys/module.h> #include <sys/smp.h> @@ -58,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include <machine/clock.h> #include <machine/bus.h> #include <machine/stdarg.h> +#include <machine/resource.h> #include <sys/bus.h> #include <sys/rman.h> @@ -77,11 +79,8 @@ __FBSDID("$FreeBSD$"); #include <compat/ndis/ndis_var.h> struct kdpc_queue { - list_entry kq_high; - list_entry kq_low; - list_entry kq_med; + list_entry kq_disp; struct thread *kq_td; - int kq_state; int kq_cpu; int kq_exit; struct mtx kq_lock; @@ -92,14 +91,22 @@ struct kdpc_queue { typedef struct kdpc_queue kdpc_queue; -static uint8_t RtlEqualUnicodeString(ndis_unicode_string *, - ndis_unicode_string *, uint8_t); -static void RtlCopyUnicodeString(ndis_unicode_string *, - ndis_unicode_string *); -static ndis_status RtlUnicodeStringToAnsiString(ndis_ansi_string *, - ndis_unicode_string *, uint8_t); -static ndis_status RtlAnsiStringToUnicodeString(ndis_unicode_string *, - ndis_ansi_string *, uint8_t); +struct wb_ext { + struct cv we_cv; + struct thread *we_td; +}; + +typedef struct wb_ext wb_ext; + +#define NTOSKRNL_TIMEOUTS 256 +struct callout ntoskrnl_callout[NTOSKRNL_TIMEOUTS]; +int ntoskrnl_callidx; +#define CALLOUT_INC(i) (i) = ((i) + 1) % NTOSKRNL_TIMEOUTS + +static uint8_t RtlEqualUnicodeString(unicode_string *, + unicode_string *, uint8_t); +static void RtlCopyUnicodeString(unicode_string *, + unicode_string *); static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *, void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *); static irp *IoBuildAsynchronousFsdRequest(uint32_t, @@ -115,7 +122,10 @@ static irp *IoMakeAssociatedIrp(irp *, uint8_t); static uint32_t KeWaitForMultipleObjects(uint32_t, nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t, int64_t *, wait_block *); -static void ntoskrnl_wakeup(void *); +static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t); +static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *); +static void ntoskrnl_satisfy_multiple_waits(wait_block *); +static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *); static void ntoskrnl_timercall(void *); static void ntoskrnl_run_dpc(void *); static void ntoskrnl_dpc_thread(void *); @@ -123,7 +133,12 @@ static void ntoskrnl_destroy_dpc_threads(void); static void ntoskrnl_destroy_workitem_threads(void); static void ntoskrnl_workitem_thread(void *); static void ntoskrnl_workitem(device_object *, void *); +static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int); +static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int); static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *); +static device_t ntoskrnl_finddev(device_t, uint32_t, + uint8_t, uint8_t, struct resource **); +static void ntoskrnl_intr(void *); static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t); static uint16_t READ_REGISTER_USHORT(uint16_t *); static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t); @@ -163,19 +178,13 @@ static uint32_t InterlockedIncrement(volatile uint32_t *); static uint32_t InterlockedDecrement(volatile uint32_t *); static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t); static uint32_t MmSizeOfMdl(void *, size_t); -static void MmBuildMdlForNonPagedPool(mdl *); static void *MmMapLockedPages(mdl *, uint8_t); static void *MmMapLockedPagesSpecifyCache(mdl *, uint8_t, uint32_t, void *, uint32_t, uint32_t); static void MmUnmapLockedPages(void *, mdl *); static uint8_t MmIsAddressValid(void *); static size_t RtlCompareMemory(const void *, const void *, size_t); -static void RtlInitAnsiString(ndis_ansi_string *, char *); -static void RtlInitUnicodeString(ndis_unicode_string *, - uint16_t *); -static void RtlFreeUnicodeString(ndis_unicode_string *); -static void RtlFreeAnsiString(ndis_ansi_string *); -static ndis_status RtlUnicodeStringToInteger(ndis_unicode_string *, +static ndis_status RtlUnicodeStringToInteger(unicode_string *, uint32_t, uint32_t *); static int atoi (const char *); static long atol (const char *); @@ -198,6 +207,8 @@ static void ObfDereferenceObject(void *); static uint32_t ZwClose(ndis_handle); static void *ntoskrnl_memset(void *, int, size_t); static char *ntoskrnl_strstr(char *, char *); +static int ntoskrnl_toupper(int); +static int ntoskrnl_tolower(int); static funcptr ntoskrnl_findwrap(funcptr); static uint32_t DbgPrint(char *, ...); static void DbgBreakPoint(void); @@ -280,6 +291,9 @@ ntoskrnl_libinit() patch++; } + for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) + callout_init(&ntoskrnl_callout[i], CALLOUT_MPSAFE); + /* * MDLs are supposed to be variable size (they describe * buffers containing some number of pages, but we don't @@ -313,10 +327,10 @@ ntoskrnl_libfini() patch++; } - /* Stop the DPC queues. */ - ntoskrnl_destroy_dpc_threads(); /* Stop the workitem queues. */ ntoskrnl_destroy_workitem_threads(); + /* Stop the DPC queues. */ + ntoskrnl_destroy_dpc_threads(); ExFreePool(kq_queues); ExFreePool(wq_queues); @@ -362,11 +376,24 @@ ntoskrnl_strstr(s, find) return ((char *)s); } +static int +ntoskrnl_toupper(c) + int c; +{ + return(toupper(c)); +} + +static int +ntoskrnl_tolower(c) + int c; +{ + return(tolower(c)); +} static uint8_t RtlEqualUnicodeString(str1, str2, caseinsensitive) - ndis_unicode_string *str1; - ndis_unicode_string *str2; + unicode_string *str1; + unicode_string *str2; uint8_t caseinsensitive; { int i; @@ -390,8 +417,8 @@ RtlEqualUnicodeString(str1, str2, caseinsensitive) static void RtlCopyUnicodeString(dest, src) - ndis_unicode_string *dest; - ndis_unicode_string *src; + unicode_string *dest; + unicode_string *src; { if (dest->us_maxlen >= src->us_len) @@ -402,55 +429,99 @@ RtlCopyUnicodeString(dest, src) return; } -static ndis_status +static void +ntoskrnl_ascii_to_unicode(ascii, unicode, len) + char *ascii; + uint16_t *unicode; + int len; +{ + int i; + uint16_t *ustr; + + ustr = unicode; + for (i = 0; i < len; i++) { + *ustr = (uint16_t)ascii[i]; + ustr++; + } + + return; +} + +static void +ntoskrnl_unicode_to_ascii(unicode, ascii, len) + uint16_t *unicode; + char *ascii; + int len; +{ + int i; + uint8_t *astr; + + astr = ascii; + for (i = 0; i < len / 2; i++) { + *astr = (uint8_t)unicode[i]; + astr++; + } + + return; +} + +uint32_t RtlUnicodeStringToAnsiString(dest, src, allocate) - ndis_ansi_string *dest; - ndis_unicode_string *src; + ansi_string *dest; + unicode_string *src; uint8_t allocate; { - char *astr = NULL; - if (dest == NULL || src == NULL) return(NDIS_STATUS_FAILURE); + + dest->as_len = src->us_len / 2; + if (dest->as_maxlen < dest->as_len) + dest->as_len = dest->as_maxlen; + if (allocate == TRUE) { - if (ndis_unicode_to_ascii(src->us_buf, src->us_len, &astr)) - return(NDIS_STATUS_FAILURE); - dest->nas_buf = astr; - dest->nas_len = dest->nas_maxlen = strlen(astr); + dest->as_buf = ExAllocatePoolWithTag(NonPagedPool, + (src->us_len / 2) + 1, 0); + if (dest->as_buf == NULL) + return(STATUS_INSUFFICIENT_RESOURCES); + dest->as_len = dest->as_maxlen = src->us_len / 2; } else { - dest->nas_len = src->us_len / 2; /* XXX */ - if (dest->nas_maxlen < dest->nas_len) - dest->nas_len = dest->nas_maxlen; - ndis_unicode_to_ascii(src->us_buf, dest->nas_len * 2, - &dest->nas_buf); + dest->as_len = src->us_len / 2; /* XXX */ + if (dest->as_maxlen < dest->as_len) + dest->as_len = dest->as_maxlen; } - return (NDIS_STATUS_SUCCESS); + + ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf, + dest->as_len * 2); + + return (STATUS_SUCCESS); } -static ndis_status +uint32_t RtlAnsiStringToUnicodeString(dest, src, allocate) - ndis_unicode_string *dest; - ndis_ansi_string *src; + unicode_string *dest; + ansi_string *src; uint8_t allocate; { - uint16_t *ustr = NULL; - if (dest == NULL || src == NULL) return(NDIS_STATUS_FAILURE); if (allocate == TRUE) { - if (ndis_ascii_to_unicode(src->nas_buf, &ustr)) - return(NDIS_STATUS_FAILURE); - dest->us_buf = ustr; - dest->us_len = dest->us_maxlen = strlen(src->nas_buf) * 2; + dest->us_buf = ExAllocatePoolWithTag(NonPagedPool, + src->as_len * 2, 0); + if (dest->us_buf == NULL) + return(STATUS_INSUFFICIENT_RESOURCES); + dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2; } else { - dest->us_len = src->nas_len * 2; /* XXX */ + dest->us_len = src->as_len * 2; /* XXX */ if (dest->us_maxlen < dest->us_len) dest->us_len = dest->us_maxlen; - ndis_ascii_to_unicode(src->nas_buf, &dest->us_buf); } - return (NDIS_STATUS_SUCCESS); + + ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf, + dest->us_len / 2); + + return (STATUS_SUCCESS); } void * @@ -461,9 +532,10 @@ ExAllocatePoolWithTag(pooltype, len, tag) { void *buf; - buf = malloc(len, M_DEVBUF, M_NOWAIT); + buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO); if (buf == NULL) return(NULL); + return(buf); } @@ -491,7 +563,7 @@ IoAllocateDriverObjectExtension(drv, clid, extlen, ext) return(STATUS_INSUFFICIENT_RESOURCES); ce->ce_clid = clid; - INSERT_LIST_TAIL((&drv->dro_driverext->dre_usrext), (&ce->ce_list)); + InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list)); *ext = (void *)(ce + 1); @@ -902,7 +974,7 @@ IoInitializeIrp(io, psize, ssize) io->irp_size = psize; io->irp_stackcnt = ssize; io->irp_currentstackloc = ssize; - INIT_LIST_HEAD(&io->irp_thlist); + InitializeListHead(&io->irp_thlist); io->irp_tail.irp_overlay.irp_csl = (io_stack_location *)(io + 1) + ssize; @@ -1060,6 +1132,202 @@ IofCompleteRequest(ip, prioboost) return; } +static device_t +ntoskrnl_finddev(dev, vector, irql, shared, res) + device_t dev; + uint32_t vector; + uint8_t irql; + uint8_t shared; + struct resource **res; +{ + device_t *children; + device_t matching_dev; + int childcnt; + struct resource *r; + struct resource_list *rl; + struct resource_list_entry *rle; + uint32_t flags; + int i; + + /* We only want devices that have been successfully probed. */ + + if (device_is_alive(dev) == FALSE) + return(NULL); + + device_get_children(dev, &children, &childcnt); + + /* + * If this device has no children, it's a leaf: we can + * examine its resources. If the interrupt resource we + * want isn't here, we return NULL, otherwise we return + * the device to terminate the recursive search. + */ + + if (childcnt == 0) { + rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); + if (rl == NULL) + return(NULL); +#if __FreeBSD_version < 600022 + SLIST_FOREACH(rle, rl, link) { +#else + STAILQ_FOREACH(rle, rl, link) { +#endif + r = rle->res; + + if (r == NULL) + continue; + + flags = rman_get_flags(r); + + if (!(flags & RF_ACTIVE)) + continue; + + if (shared == TRUE && !(flags & RF_SHAREABLE)) + continue; + + if (rle->type == SYS_RES_IRQ && + rman_get_start(r) == irql) { + *res = r; + return(dev); + } + } + /* No match. */ + return (NULL); + } + + /* + * If this device has children, do another + * level of recursion to inspect them. + */ + + for (i = 0; i < childcnt; i++) { + matching_dev = ntoskrnl_finddev(children[i], + vector, irql, shared, res); + if (matching_dev != NULL) { + free(children, M_TEMP); + return(matching_dev); + } + } + + free(children, M_TEMP); + return(NULL); +} + +static void +ntoskrnl_intr(arg) + void *arg; +{ + kinterrupt *iobj; + uint8_t irql; + + iobj = arg; + + KeAcquireSpinLock(iobj->ki_lock, &irql); + MSCALL1(iobj->ki_svcfunc, iobj->ki_svcctx); + KeReleaseSpinLock(iobj->ki_lock, irql); + + return; +} + +uint8_t +KeSynchronizeExecution(iobj, syncfunc, syncctx) + kinterrupt *iobj; + void *syncfunc; + void *syncctx; +{ + uint8_t irql; + + KeAcquireSpinLock(iobj->ki_lock, &irql); + MSCALL1(syncfunc, syncctx); + KeReleaseSpinLock(iobj->ki_lock, irql); + + return(TRUE); +} + +/* + * This routine is a pain because the only thing we get passed + * here is the interrupt request level and vector, but bus_setup_intr() + * needs the device too. We can hack around this for now, but it's + * awkward. + */ + +uint32_t +IoConnectInterrupt(iobj, svcfunc, svcctx, lock, vector, irql, + syncirql, imode, shared, affinity, savefloat) + kinterrupt **iobj; + void *svcfunc; + void *svcctx; + uint32_t vector; + kspin_lock *lock; + uint8_t irql; + uint8_t syncirql; + uint8_t imode; + uint8_t shared; + uint32_t affinity; + uint8_t savefloat; +{ + devclass_t nexus_class; + device_t *nexus_devs, devp; + int nexus_count = 0; + device_t matching_dev = NULL; + struct resource *res; + int i, error; + + nexus_class = devclass_find("nexus"); + devclass_get_devices(nexus_class, &nexus_devs, &nexus_count); + + for (i = 0; i < nexus_count; i++) { + devp = nexus_devs[i]; + matching_dev = ntoskrnl_finddev(devp, vector, + irql, shared, &res); + if (matching_dev) + break; + } + + free(nexus_devs, M_TEMP); + + if (matching_dev == NULL) + return(STATUS_INVALID_PARAMETER); + + *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0); + if (*iobj == NULL) + return(STATUS_INSUFFICIENT_RESOURCES); + + (*iobj)->ki_dev = matching_dev; + (*iobj)->ki_irq = res; + (*iobj)->ki_svcfunc = svcfunc; + (*iobj)->ki_svcctx = svcctx; + + if (lock == NULL) { + KeInitializeSpinLock(&(*iobj)->ki_lock_priv); + (*iobj)->ki_lock = &(*iobj)->ki_lock_priv; + } else + (*iobj)->ki_lock = lock; + + error = bus_setup_intr(matching_dev, res, INTR_TYPE_NET | INTR_MPSAFE, + ntoskrnl_intr, *iobj, &(*iobj)->ki_cookie); + + if (error) { + ExFreePool(iobj); + return (STATUS_INVALID_PARAMETER); + } + + return(STATUS_SUCCESS); +} + +void +IoDisconnectInterrupt(iobj) + kinterrupt *iobj; +{ + if (iobj == NULL) + return; + + bus_teardown_intr(iobj->ki_dev, iobj->ki_irq, iobj->ki_cookie); + ExFreePool(iobj); + + return; +} + device_object * IoAttachDeviceToDeviceStack(src, dst) device_object *src; @@ -1094,7 +1362,7 @@ IoDetachDevice(topdev) topdev->do_attacheddev = tail->do_attacheddev; topdev->do_refcnt--; - /* Now reduce the stacksize count for the tail objects. */ + /* Now reduce the stacksize count for the takm_il objects. */ tail = topdev->do_attacheddev; while (tail != NULL) { @@ -1107,48 +1375,168 @@ IoDetachDevice(topdev) return; } -/* Always called with dispatcher lock held. */ -static void -ntoskrnl_wakeup(arg) - void *arg; +/* + * For the most part, an object is considered signalled if + * dh_sigstate == TRUE. The exception is for mutant objects + * (mutexes), where the logic works like this: + * + * - If the thread already owns the object and sigstate is + * less than or equal to 0, then the object is considered + * signalled (recursive acquisition). + * - If dh_sigstate == 1, the object is also considered + * signalled. + */ + +static int +ntoskrnl_is_signalled(obj, td) + nt_dispatch_header *obj; + struct thread *td; { + kmutant *km; + + if (obj->dh_type == DISP_TYPE_MUTANT) { + km = (kmutant *)obj; + if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) || + obj->dh_sigstate == 1) + return(TRUE); + return(FALSE); + } + + if (obj->dh_sigstate > 0) + return(TRUE); + return(FALSE); +} + +static void +ntoskrnl_satisfy_wait(obj, td) nt_dispatch_header *obj; - wait_block *w; - list_entry *e; struct thread *td; +{ + kmutant *km; - obj = arg; + switch (obj->dh_type) { + case DISP_TYPE_MUTANT: + km = (struct kmutant *)obj; + obj->dh_sigstate--; + /* + * If sigstate reaches 0, the mutex is now + * non-signalled (the new thread owns it). + */ + if (obj->dh_sigstate == 0) { + km->km_ownerthread = td; + if (km->km_abandoned == TRUE) + km->km_abandoned = FALSE; + } + break; + /* Synchronization objects get reset to unsignalled. */ + case DISP_TYPE_SYNCHRONIZATION_EVENT: + case DISP_TYPE_SYNCHRONIZATION_TIMER: + obj->dh_sigstate = 0; + break; + case DISP_TYPE_SEMAPHORE: + obj->dh_sigstate--; + break; + default: + break; + } - e = obj->dh_waitlisthead.nle_flink; + return; +} + +static void +ntoskrnl_satisfy_multiple_waits(wb) + wait_block *wb; +{ + wait_block *cur; + struct thread *td; + + cur = wb; + td = wb->wb_kthread; - obj->dh_sigstate = TRUE; + do { + ntoskrnl_satisfy_wait(wb->wb_object, td); + cur->wb_awakened = TRUE; + cur = cur->wb_next; + } while (cur != wb); + + return; +} + +/* Always called with dispatcher lock held. */ +static void +ntoskrnl_waittest(obj, increment) + nt_dispatch_header *obj; + uint32_t increment; +{ + wait_block *w, *next; + list_entry *e; + struct thread *td; + wb_ext *we; + int satisfied; /* - * What happens if someone tells us to wake up - * threads waiting on an object, but nobody's - * waiting on it at the moment? For sync events, - * the signal state is supposed to be automatically - * reset, but this only happens in the KeWaitXXX() - * functions. If nobody is waiting, the state never - * gets cleared. + * Once an object has been signalled, we walk its list of + * wait blocks. If a wait block can be awakened, then satisfy + * waits as necessary and wake the thread. + * + * The rules work like this: + * + * If a wait block is marked as WAITTYPE_ANY, then + * we can satisfy the wait conditions on the current + * object and wake the thread right away. Satisfying + * the wait also has the effect of breaking us out + * of the search loop. + * + * If the object is marked as WAITTYLE_ALL, then the + * wait block will be part of a circularly linked + * list of wait blocks belonging to a waiting thread + * that's sleeping in KeWaitForMultipleObjects(). In + * order to wake the thread, all the objects in the + * wait list must be in the signalled state. If they + * are, we then satisfy all of them and wake the + * thread. + * */ - if (e == &obj->dh_waitlisthead) { - if (obj->dh_type == EVENT_TYPE_SYNC) - obj->dh_sigstate = FALSE; - return; - } + e = obj->dh_waitlisthead.nle_flink; - while (e != &obj->dh_waitlisthead) { - w = (wait_block *)e; - td = w->wb_kthread; - ndis_thresume(td->td_proc); - /* - * For synchronization objects, only wake up - * the first waiter. - */ - if (obj->dh_type == EVENT_TYPE_SYNC) - break; + while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) { + w = CONTAINING_RECORD(e, wait_block, wb_waitlist); + we = w->wb_ext; + td = we->we_td; + satisfied = FALSE; + if (w->wb_waittype == WAITTYPE_ANY) { + /* + * Thread can be awakened if + * any wait is satisfied. + */ + ntoskrnl_satisfy_wait(obj, td); + satisfied = TRUE; + w->wb_awakened = TRUE; + } else { + /* + * Thread can only be woken up + * if all waits are satisfied. + * If the thread is waiting on multiple + * objects, they should all be linked + * through the wb_next pointers in the + * wait blocks. + */ + satisfied = TRUE; + next = w->wb_next; + while (next != w) { + if (ntoskrnl_is_signalled(obj, td) == FALSE) { + satisfied = FALSE; + break; + } + next = next->wb_next; + } + ntoskrnl_satisfy_multiple_waits(w); + } + + if (satisfied == TRUE) + cv_broadcastpri(&we->we_cv, w->wb_oldpri - + (increment * 4)); e = e->nle_flink; } @@ -1206,7 +1594,7 @@ ntoskrnl_time(tval) * * - For mutexes, we try to acquire the mutex and if we can't, we wait * on the mutex until it's available and then grab it. When a mutex is - * released, it enters the signaled state, which wakes up one of the + * released, it enters the signalled state, which wakes up one of the * threads waiting to acquire it. Mutexes are always synchronization * events. * @@ -1228,49 +1616,54 @@ KeWaitForSingleObject(obj, reason, mode, alertable, duetime) uint8_t alertable; int64_t *duetime; { - struct thread *td = curthread; - kmutant *km; wait_block w; + struct thread *td = curthread; struct timeval tv; int error = 0; uint64_t curtime; + wb_ext we; if (obj == NULL) return(STATUS_INVALID_PARAMETER); mtx_lock(&ntoskrnl_dispatchlock); + cv_init(&we.we_cv, "KeWFS"); + we.we_td = td; + /* - * See if the object is a mutex. If so, and we already own - * it, then just increment the acquisition count and return. - * - * For any other kind of object, see if it's already in the - * signalled state, and if it is, just return. If the object - * is marked as a synchronization event, reset the state to - * unsignalled. + * Check to see if this object is already signalled, + * and just return without waiting if it is. */ - - if (obj->dh_size == OTYPE_MUTEX) { - km = (kmutant *)obj; - if (km->km_ownerthread == NULL || - km->km_ownerthread == curthread->td_proc) { - obj->dh_sigstate = FALSE; - km->km_acquirecnt++; - km->km_ownerthread = curthread->td_proc; + if (ntoskrnl_is_signalled(obj, td) == TRUE) { + /* Sanity check the signal state value. */ + if (obj->dh_sigstate != INT32_MIN) { + ntoskrnl_satisfy_wait(obj, curthread); mtx_unlock(&ntoskrnl_dispatchlock); return (STATUS_SUCCESS); + } else { + /* + * There's a limit to how many times we can + * recursively acquire a mutant. If we hit + * the limit, something is very wrong. + */ + if (obj->dh_type == DISP_TYPE_MUTANT) { + mtx_unlock(&ntoskrnl_dispatchlock); + panic("mutant limit exceeded"); + } } - } else if (obj->dh_sigstate == TRUE) { - if (obj->dh_type == EVENT_TYPE_SYNC) - obj->dh_sigstate = FALSE; - mtx_unlock(&ntoskrnl_dispatchlock); - return (STATUS_SUCCESS); } + bzero((char *)&w, sizeof(wait_block)); w.wb_object = obj; - w.wb_kthread = td; + w.wb_ext = &we; + w.wb_waittype = WAITTYPE_ANY; + w.wb_next = &w; + w.wb_waitkey = 0; + w.wb_awakened = FALSE; + w.wb_oldpri = td->td_priority; - INSERT_LIST_TAIL((&obj->dh_waitlisthead), (&w.wb_waitlist)); + InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist)); /* * The timeout value is specified in 100 nanosecond units @@ -1298,41 +1691,30 @@ KeWaitForSingleObject(obj, reason, mode, alertable, duetime) } } - error = ndis_thsuspend(td->td_proc, &ntoskrnl_dispatchlock, - duetime == NULL ? 0 : tvtohz(&tv)); + if (duetime == NULL) + cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); + else + error = cv_timedwait(&we.we_cv, + &ntoskrnl_dispatchlock, tvtohz(&tv)); + + RemoveEntryList(&w.wb_waitlist); + + cv_destroy(&we.we_cv); /* We timed out. Leave the object alone and return status. */ if (error == EWOULDBLOCK) { - REMOVE_LIST_ENTRY((&w.wb_waitlist)); - INIT_LIST_HEAD((&w.wb_waitlist)); mtx_unlock(&ntoskrnl_dispatchlock); return(STATUS_TIMEOUT); } - /* - * Mutexes are always synchronization objects, which means - * if several threads are waiting to acquire it, only one will - * be woken up. If that one is us, and the mutex is up for grabs, - * grab it. - */ - - if (obj->dh_size == OTYPE_MUTEX) { - km = (kmutant *)obj; - if (km->km_ownerthread == NULL) { - km->km_ownerthread = curthread->td_proc; - km->km_acquirecnt++; - } - } - - if (obj->dh_type == EVENT_TYPE_SYNC) - obj->dh_sigstate = FALSE; - REMOVE_LIST_ENTRY((&w.wb_waitlist)); - INIT_LIST_HEAD((&w.wb_waitlist)); - mtx_unlock(&ntoskrnl_dispatchlock); return(STATUS_SUCCESS); +/* + return(KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason, + mode, alertable, duetime, &w)); +*/ } static uint32_t @@ -1348,13 +1730,15 @@ KeWaitForMultipleObjects(cnt, obj, wtype, reason, mode, wait_block *wb_array; { struct thread *td = curthread; - kmutant *km; - wait_block _wb_array[THREAD_WAIT_OBJECTS]; - wait_block *w; + wait_block *whead, *w; + wait_block _wb_array[MAX_WAIT_OBJECTS]; + nt_dispatch_header *cur; struct timeval tv; - int i, wcnt = 0, widx = 0, error = 0; + int i, wcnt = 0, error = 0; uint64_t curtime; struct timespec t1, t2; + uint32_t status = STATUS_SUCCESS; + wb_ext we; if (cnt > MAX_WAIT_OBJECTS) return(STATUS_INVALID_PARAMETER); @@ -1363,51 +1747,88 @@ KeWaitForMultipleObjects(cnt, obj, wtype, reason, mode, mtx_lock(&ntoskrnl_dispatchlock); + cv_init(&we.we_cv, "KeWFM"); + we.we_td = td; + if (wb_array == NULL) - w = &_wb_array[0]; + whead = _wb_array; else - w = wb_array; + whead = wb_array; + + bzero((char *)whead, sizeof(wait_block) * cnt); /* First pass: see if we can satisfy any waits immediately. */ + wcnt = 0; + w = whead; + for (i = 0; i < cnt; i++) { - if (obj[i]->dh_size == OTYPE_MUTEX) { - km = (kmutant *)obj[i]; - if (km->km_ownerthread == NULL || - km->km_ownerthread == curthread->td_proc) { - obj[i]->dh_sigstate = FALSE; - km->km_acquirecnt++; - km->km_ownerthread = curthread->td_proc; - if (wtype == WAITTYPE_ANY) { - mtx_unlock(&ntoskrnl_dispatchlock); - return (STATUS_WAIT_0 + i); - } + InsertTailList((&obj[i]->dh_waitlisthead), + (&w->wb_waitlist)); + w->wb_ext = &we; + w->wb_object = obj[i]; + w->wb_waittype = wtype; + w->wb_waitkey = i; + w->wb_awakened = FALSE; + w->wb_oldpri = td->td_priority; + w->wb_next = w + 1; + w++; + wcnt++; + if (ntoskrnl_is_signalled(obj[i], td)) { + /* + * There's a limit to how many times + * we can recursively acquire a mutant. + * If we hit the limit, something + * is very wrong. + */ + if (obj[i]->dh_sigstate == INT32_MIN && + obj[i]->dh_type == DISP_TYPE_MUTANT) { + mtx_unlock(&ntoskrnl_dispatchlock); + panic("mutant limit exceeded"); } - } else if (obj[i]->dh_sigstate == TRUE) { - if (obj[i]->dh_type == EVENT_TYPE_SYNC) - obj[i]->dh_sigstate = FALSE; + + /* + * If this is a WAITTYPE_ANY wait, then + * satisfy the waited object and exit + * right now. + */ + if (wtype == WAITTYPE_ANY) { - mtx_unlock(&ntoskrnl_dispatchlock); - return (STATUS_WAIT_0 + i); + ntoskrnl_satisfy_wait(obj[i], td); + status = STATUS_WAIT_0 + i; + goto wait_done; + } else { + w--; + wcnt--; + w->wb_object = NULL; + RemoveEntryList(&w->wb_waitlist); } } } /* - * Second pass: set up wait for anything we can't - * satisfy immediately. + * If this is a WAITTYPE_ALL wait and all objects are + * already signalled, satisfy the waits and exit now. */ - for (i = 0; i < cnt; i++) { - if (obj[i]->dh_sigstate == TRUE) - continue; - INSERT_LIST_TAIL((&obj[i]->dh_waitlisthead), - (&w[i].wb_waitlist)); - w[i].wb_kthread = td; - w[i].wb_object = obj[i]; - wcnt++; + if (wtype == WAITTYPE_ALL && wcnt == 0) { + for (i = 0; i < cnt; i++) + ntoskrnl_satisfy_wait(obj[i], td); + status = STATUS_SUCCESS; + goto wait_done; } + /* + * Create a circular waitblock list. The waitcount + * must always be non-zero when we get here. + */ + + (w - 1)->wb_next = whead; + + /* Wait on any objects that aren't yet signalled. */ + + /* Calculate timeout, if any. */ + if (duetime != NULL) { if (*duetime < 0) { tv.tv_sec = - (*duetime) / 10000000; @@ -1428,32 +1849,62 @@ KeWaitForMultipleObjects(cnt, obj, wtype, reason, mode, while (wcnt) { nanotime(&t1); - error = ndis_thsuspend(td->td_proc, &ntoskrnl_dispatchlock, - duetime == NULL ? 0 : tvtohz(&tv)); + if (duetime == NULL) + cv_wait(&we.we_cv, &ntoskrnl_dispatchlock); + else + error = cv_timedwait(&we.we_cv, + &ntoskrnl_dispatchlock, tvtohz(&tv)); + + /* Wait with timeout expired. */ + + if (error) { + status = STATUS_TIMEOUT; + goto wait_done; + } nanotime(&t2); - for (i = 0; i < cnt; i++) { - if (obj[i]->dh_size == OTYPE_MUTEX) { - km = (kmutant *)obj; - if (km->km_ownerthread == NULL) { - km->km_ownerthread = - curthread->td_proc; - km->km_acquirecnt++; + /* See what's been signalled. */ + + w = whead; + do { + cur = w->wb_object; + if (ntoskrnl_is_signalled(cur, td) == TRUE || + w->wb_awakened == TRUE) { + /* Sanity check the signal state value. */ + if (cur->dh_sigstate == INT32_MIN && + cur->dh_type == DISP_TYPE_MUTANT) { + mtx_unlock(&ntoskrnl_dispatchlock); + panic("mutant limit exceeded"); } - } - if (obj[i]->dh_sigstate == TRUE) { - widx = i; - if (obj[i]->dh_type == EVENT_TYPE_SYNC) - obj[i]->dh_sigstate = FALSE; - REMOVE_LIST_ENTRY((&w[i].wb_waitlist)); - INIT_LIST_HEAD((&w[i].wb_waitlist)); wcnt--; + if (wtype == WAITTYPE_ANY) { + status = w->wb_waitkey & + STATUS_WAIT_0; + goto wait_done; + } } + w = w->wb_next; + } while (w != whead); + + /* + * If all objects have been signalled, or if this + * is a WAITTYPE_ANY wait and we were woke up by + * someone, we can bail. + */ + + if (wcnt == 0) { + status = STATUS_SUCCESS; + goto wait_done; } - if (error || wtype == WAITTYPE_ANY) - break; + /* + * If this is WAITTYPE_ALL wait, and there's still + * objects that haven't been signalled, deduct the + * time that's elapsed so far from the timeout and + * wait again (or continue waiting indefinitely if + * there's no timeout). + */ if (duetime != NULL) { tv.tv_sec -= (t2.tv_sec - t1.tv_sec); @@ -1461,26 +1912,19 @@ KeWaitForMultipleObjects(cnt, obj, wtype, reason, mode, } } - if (wcnt) { - for (i = 0; i < cnt; i++) { - REMOVE_LIST_ENTRY((&w[i].wb_waitlist)); - INIT_LIST_HEAD((&w[i].wb_waitlist)); - } - } - if (error == EWOULDBLOCK) { - mtx_unlock(&ntoskrnl_dispatchlock); - return(STATUS_TIMEOUT); - } +wait_done: - if (wtype == WAITTYPE_ANY && wcnt) { - mtx_unlock(&ntoskrnl_dispatchlock); - return(STATUS_WAIT_0 + widx); - } + cv_destroy(&we.we_cv); + for (i = 0; i < cnt; i++) { + if (whead[i].wb_object != NULL) + RemoveEntryList(&whead[i].wb_waitlist); + + } mtx_unlock(&ntoskrnl_dispatchlock); - return(STATUS_SUCCESS); + return(status); } static void @@ -2047,7 +2491,7 @@ MmSizeOfMdl(vaddr, len) * Instead, we just fill in the page array with the kernel virtual * addresses of the buffers. */ -static void +void MmBuildMdlForNonPagedPool(m) mdl *m; { @@ -2135,16 +2579,14 @@ ntoskrnl_workitem_thread(arg) kq = arg; - INIT_LIST_HEAD(&kq->kq_med); + InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; - kq->kq_state = NDIS_PSTATE_SLEEPING; mtx_init(&kq->kq_lock, "NDIS thread lock", NULL, MTX_SPIN); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_dead, EVENT_TYPE_SYNC, FALSE); while (1) { - KeWaitForSingleObject((nt_dispatch_header *)&kq->kq_proc, 0, 0, TRUE, NULL); @@ -2152,34 +2594,27 @@ ntoskrnl_workitem_thread(arg) if (kq->kq_exit) { mtx_unlock_spin(&kq->kq_lock); - KeSetEvent(&kq->kq_dead, 0, FALSE); + KeSetEvent(&kq->kq_dead, IO_NO_INCREMENT, FALSE); break; } - kq->kq_state = NDIS_PSTATE_RUNNING; - - l = kq->kq_med.nle_flink; - while (l != & kq->kq_med) { + while (!IsListEmpty(&kq->kq_disp)) { + l = RemoveHeadList(&kq->kq_disp); iw = CONTAINING_RECORD(l, io_workitem, iw_listentry); - REMOVE_LIST_HEAD((&kq->kq_med)); - INIT_LIST_HEAD(l); - if (iw->iw_func == NULL) { - l = kq->kq_med.nle_flink; + InitializeListHead((&iw->iw_listentry)); + if (iw->iw_func == NULL) continue; - } mtx_unlock_spin(&kq->kq_lock); MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx); mtx_lock_spin(&kq->kq_lock); - l = kq->kq_med.nle_flink; } - kq->kq_state = NDIS_PSTATE_SLEEPING; - mtx_unlock_spin(&kq->kq_lock); } mtx_destroy(&kq->kq_lock); + #if __FreeBSD_version < 502113 mtx_lock(&Giant); #endif @@ -2195,9 +2630,8 @@ ntoskrnl_destroy_workitem_threads(void) for (i = 0; i < WORKITEM_THREADS; i++) { kq = wq_queues + i; - kq->kq_exit = 1; - KeSetEvent(&kq->kq_proc, 0, FALSE); + KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); KeWaitForSingleObject((nt_dispatch_header *)&kq->kq_dead, 0, 0, TRUE, NULL); } @@ -2215,7 +2649,7 @@ IoAllocateWorkItem(dobj) if (iw == NULL) return(NULL); - INIT_LIST_HEAD(&iw->iw_listentry); + InitializeListHead(&iw->iw_listentry); iw->iw_dobj = dobj; mtx_lock(&ntoskrnl_dispatchlock); @@ -2241,21 +2675,38 @@ IoQueueWorkItem(iw, iw_func, qtype, ctx) uint32_t qtype; void *ctx; { - int state; kdpc_queue *kq; + list_entry *l; + io_workitem *cur; iw->iw_func = iw_func; iw->iw_ctx = ctx; - kq = wq_queues + iw->iw_idx; mtx_lock_spin(&kq->kq_lock); - INSERT_LIST_TAIL((&kq->kq_med), (&iw->iw_listentry)); - state = kq->kq_state; + + /* + * Traverse the list and make sure this workitem hasn't + * already been inserted. Queuing the same workitem + * twice will hose the list but good. + */ + + l = kq->kq_disp.nle_flink; + while (l != &kq->kq_disp) { + cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); + if (cur == iw) { + /* Already queued -- do nothing. */ + mtx_unlock_spin(&kq->kq_lock); + return; + } + l = l->nle_flink; + } + + InsertTailList((&kq->kq_disp), (&iw->iw_listentry)); mtx_unlock_spin(&kq->kq_lock); - if (state == NDIS_PSTATE_SLEEPING) - KeSetEvent(&kq->kq_proc, 0, FALSE); + + KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); return; } @@ -2278,6 +2729,31 @@ ntoskrnl_workitem(dobj, arg) return; } +/* + * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft + * warns that it's unsafe and to use IoQueueWorkItem() instead. The + * problem with ExQueueWorkItem() is that it can't guard against + * the condition where a driver submits a job to the work queue and + * is then unloaded before the job is able to run. IoQueueWorkItem() + * acquires a reference to the device's device_object via the + * object manager and retains it until after the job has completed, + * which prevents the driver from being unloaded before the job + * runs. (We don't currently support this behavior, though hopefully + * that will change once the object manager API is fleshed out a bit.) + * + * Having said all that, the ExQueueWorkItem() API remains, because + * there are still other parts of Windows that use it, including + * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem(). + * We fake up the ExQueueWorkItem() API on top of our implementation + * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively + * for ExQueueWorkItem() jobs, and we pass a pointer to the work + * queue item (provided by the caller) in to IoAllocateWorkItem() + * instead of the device_object. We need to save this pointer so + * we can apply a sanity check: as with the DPC queue and other + * workitem queues, we can't allow the same work queue item to + * be queued twice. If it's already pending, we silently return + */ + void ExQueueWorkItem(w, qtype) work_queue_item *w; @@ -2285,6 +2761,33 @@ ExQueueWorkItem(w, qtype) { io_workitem *iw; io_workitem_func iwf; + kdpc_queue *kq; + list_entry *l; + io_workitem *cur; + + + /* + * We need to do a special sanity test to make sure + * the ExQueueWorkItem() API isn't used to queue + * the same workitem twice. Rather than checking the + * io_workitem pointer itself, we test the attached + * device object, which is really a pointer to the + * legacy work queue item structure. + */ + + kq = wq_queues + WORKITEM_LEGACY_THREAD; + mtx_lock_spin(&kq->kq_lock); + l = kq->kq_disp.nle_flink; + while (l != &kq->kq_disp) { + cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); + if (cur->iw_dobj == (device_object *)w) { + /* Already queued -- do nothing. */ + mtx_unlock_spin(&kq->kq_lock); + return; + } + l = l->nle_flink; + } + mtx_unlock_spin(&kq->kq_lock); iw = IoAllocateWorkItem((device_object *)w); if (iw == NULL) @@ -2316,33 +2819,33 @@ RtlCompareMemory(s1, s2, len) return(total); } -static void +void RtlInitAnsiString(dst, src) - ndis_ansi_string *dst; + ansi_string *dst; char *src; { - ndis_ansi_string *a; + ansi_string *a; a = dst; if (a == NULL) return; if (src == NULL) { - a->nas_len = a->nas_maxlen = 0; - a->nas_buf = NULL; + a->as_len = a->as_maxlen = 0; + a->as_buf = NULL; } else { - a->nas_buf = src; - a->nas_len = a->nas_maxlen = strlen(src); + a->as_buf = src; + a->as_len = a->as_maxlen = strlen(src); } return; } -static void +void RtlInitUnicodeString(dst, src) - ndis_unicode_string *dst; + unicode_string *dst; uint16_t *src; { - ndis_unicode_string *u; + unicode_string *u; int i; u = dst; @@ -2364,7 +2867,7 @@ RtlInitUnicodeString(dst, src) ndis_status RtlUnicodeStringToInteger(ustr, base, val) - ndis_unicode_string *ustr; + unicode_string *ustr; uint32_t base; uint32_t *val; { @@ -2410,31 +2913,31 @@ RtlUnicodeStringToInteger(ustr, base, val) astr++; } - ndis_unicode_to_ascii(uchr, len, &astr); + ntoskrnl_unicode_to_ascii(uchr, astr, len); *val = strtoul(abuf, NULL, base); return(NDIS_STATUS_SUCCESS); } -static void +void RtlFreeUnicodeString(ustr) - ndis_unicode_string *ustr; + unicode_string *ustr; { if (ustr->us_buf == NULL) return; - free(ustr->us_buf, M_DEVBUF); + ExFreePool(ustr->us_buf); ustr->us_buf = NULL; return; } -static void +void RtlFreeAnsiString(astr) - ndis_ansi_string *astr; + ansi_string *astr; { - if (astr->nas_buf == NULL) + if (astr->as_buf == NULL) return; - free(astr->nas_buf, M_DEVBUF); - astr->nas_buf = NULL; + ExFreePool(astr->as_buf); + astr->as_buf = NULL; return; } @@ -2512,13 +3015,12 @@ KeInitializeMutex(kmutex, level) kmutant *kmutex; uint32_t level; { - INIT_LIST_HEAD((&kmutex->km_header.dh_waitlisthead)); + InitializeListHead((&kmutex->km_header.dh_waitlisthead)); kmutex->km_abandoned = FALSE; kmutex->km_apcdisable = 1; - kmutex->km_header.dh_sigstate = TRUE; - kmutex->km_header.dh_type = EVENT_TYPE_SYNC; - kmutex->km_header.dh_size = OTYPE_MUTEX; - kmutex->km_acquirecnt = 0; + kmutex->km_header.dh_sigstate = 1; + kmutex->km_header.dh_type = DISP_TYPE_MUTANT; + kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t); kmutex->km_ownerthread = NULL; return; } @@ -2528,19 +3030,26 @@ KeReleaseMutex(kmutex, kwait) kmutant *kmutex; uint8_t kwait; { + uint32_t prevstate; + mtx_lock(&ntoskrnl_dispatchlock); - if (kmutex->km_ownerthread != curthread->td_proc) { + prevstate = kmutex->km_header.dh_sigstate; + if (kmutex->km_ownerthread != curthread) { mtx_unlock(&ntoskrnl_dispatchlock); return(STATUS_MUTANT_NOT_OWNED); } - kmutex->km_acquirecnt--; - if (kmutex->km_acquirecnt == 0) { + + kmutex->km_header.dh_sigstate++; + kmutex->km_abandoned = FALSE; + + if (kmutex->km_header.dh_sigstate == 1) { kmutex->km_ownerthread = NULL; - ntoskrnl_wakeup(&kmutex->km_header); + ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT); } + mtx_unlock(&ntoskrnl_dispatchlock); - return(kmutex->km_acquirecnt); + return(prevstate); } static uint32_t @@ -2556,10 +3065,13 @@ KeInitializeEvent(kevent, type, state) uint32_t type; uint8_t state; { - INIT_LIST_HEAD((&kevent->k_header.dh_waitlisthead)); + InitializeListHead((&kevent->k_header.dh_waitlisthead)); kevent->k_header.dh_sigstate = state; - kevent->k_header.dh_type = type; - kevent->k_header.dh_size = OTYPE_EVENT; + if (type == EVENT_TYPE_NOTIFY) + kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT; + else + kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT; + kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t); return; } @@ -2584,10 +3096,49 @@ KeSetEvent(kevent, increment, kwait) uint8_t kwait; { uint32_t prevstate; + wait_block *w; + nt_dispatch_header *dh; + struct thread *td; + wb_ext *we; mtx_lock(&ntoskrnl_dispatchlock); prevstate = kevent->k_header.dh_sigstate; - ntoskrnl_wakeup(&kevent->k_header); + dh = &kevent->k_header; + + if (IsListEmpty(&dh->dh_waitlisthead)) + /* + * If there's nobody in the waitlist, just set + * the state to signalled. + */ + dh->dh_sigstate = 1; + else { + /* + * Get the first waiter. If this is a synchronization + * event, just wake up that one thread (don't bother + * setting the state to signalled since we're supposed + * to automatically clear synchronization events anyway). + * + * If it's a notification event, or the the first + * waiter is doing a WAITTYPE_ALL wait, go through + * the full wait satisfaction process. + */ + w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink, + wait_block, wb_waitlist); + we = w->wb_ext; + td = we->we_td; + if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT || + w->wb_waittype == WAITTYPE_ALL) { + if (prevstate == 0) { + dh->dh_sigstate = 1; + ntoskrnl_waittest(dh, increment); + } + } else { + w->wb_awakened |= TRUE; + cv_broadcastpri(&we->we_cv, w->wb_oldpri - + (increment * 4)); + } + } + mtx_unlock(&ntoskrnl_dispatchlock); return(prevstate); @@ -2608,6 +3159,40 @@ KeReadStateEvent(kevent) return(kevent->k_header.dh_sigstate); } +/* + * The object manager in Windows is responsible for managing + * references and access to various types of objects, including + * device_objects, events, threads, timers and so on. However, + * there's a difference in the way objects are handled in user + * mode versus kernel mode. + * + * In user mode (i.e. Win32 applications), all objects are + * managed by the object manager. For example, when you create + * a timer or event object, you actually end up with an + * object_header (for the object manager's bookkeeping + * purposes) and an object body (which contains the actual object + * structure, e.g. ktimer, kevent, etc...). This allows Windows + * to manage resource quotas and to enforce access restrictions + * on basically every kind of system object handled by the kernel. + * + * However, in kernel mode, you only end up using the object + * manager some of the time. For example, in a driver, you create + * a timer object by simply allocating the memory for a ktimer + * structure and initializing it with KeInitializeTimer(). Hence, + * the timer has no object_header and no reference counting or + * security/resource checks are done on it. The assumption in + * this case is that if you're running in kernel mode, you know + * what you're doing, and you're already at an elevated privilege + * anyway. + * + * There are some exceptions to this. The two most important ones + * for our purposes are device_objects and threads. We need to use + * the object manager to do reference counting on device_objects, + * and for threads, you can only get a pointer to a thread's + * dispatch header by using ObReferenceObjectByHandle() on the + * handle returned by PsCreateSystemThread(). + */ + static ndis_status ObReferenceObjectByHandle(handle, reqaccess, otype, accessmode, object, handleinfo) @@ -2624,9 +3209,12 @@ ObReferenceObjectByHandle(handle, reqaccess, otype, if (nr == NULL) return(NDIS_STATUS_FAILURE); - INIT_LIST_HEAD((&nr->no_dh.dh_waitlisthead)); + InitializeListHead((&nr->no_dh.dh_waitlisthead)); nr->no_obj = handle; - nr->no_dh.dh_size = OTYPE_THREAD; + nr->no_dh.dh_type = DISP_TYPE_THREAD; + nr->no_dh.dh_sigstate = 0; + nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) / + sizeof(uint32_t)); TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link); *object = nr; @@ -2728,7 +3316,8 @@ PsTerminateSystemThread(status) TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) { if (nr->no_obj != curthread->td_proc) continue; - ntoskrnl_wakeup(&nr->no_dh); + nr->no_dh.dh_sigstate = 1; + ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT); break; } mtx_unlock(&ntoskrnl_dispatchlock); @@ -2773,14 +3362,21 @@ ntoskrnl_timercall(arg) ktimer *timer; struct timeval tv; - mtx_unlock(&Giant); - mtx_lock(&ntoskrnl_dispatchlock); timer = arg; + callout_init(timer->k_callout, CALLOUT_MPSAFE); + + /* Mark the timer as no longer being on the timer queue. */ + timer->k_header.dh_inserted = FALSE; + /* Now signal the object and satisfy any waits on it. */ + + timer->k_header.dh_sigstate = 1; + ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT); + /* * If this is a periodic timer, re-arm it * so it will fire again. We do this before @@ -2794,18 +3390,19 @@ ntoskrnl_timercall(arg) tv.tv_sec = 0; tv.tv_usec = timer->k_period * 1000; timer->k_header.dh_inserted = TRUE; - timer->k_handle = timeout(ntoskrnl_timercall, - timer, tvtohz(&tv)); + timer->k_callout = &ntoskrnl_callout[ntoskrnl_callidx]; + CALLOUT_INC(ntoskrnl_callidx); + callout_reset(timer->k_callout, tvtohz(&tv), + ntoskrnl_timercall, timer); } + /* If there's a DPC associated with the timer, queue it up. */ + if (timer->k_dpc != NULL) KeInsertQueueDpc(timer->k_dpc, NULL, NULL); - ntoskrnl_wakeup(&timer->k_header); mtx_unlock(&ntoskrnl_dispatchlock); - mtx_lock(&Giant); - return; } @@ -2830,12 +3427,14 @@ KeInitializeTimerEx(timer, type) return; bzero((char *)timer, sizeof(ktimer)); - INIT_LIST_HEAD((&timer->k_header.dh_waitlisthead)); + InitializeListHead((&timer->k_header.dh_waitlisthead)); timer->k_header.dh_sigstate = FALSE; timer->k_header.dh_inserted = FALSE; - timer->k_header.dh_type = type; - timer->k_header.dh_size = OTYPE_TIMER; - callout_handle_init(&timer->k_handle); + if (type == EVENT_TYPE_NOTIFY) + timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER; + else + timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER; + timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t); return; } @@ -2868,18 +3467,27 @@ ntoskrnl_dpc_thread(arg) kq = arg; - INIT_LIST_HEAD(&kq->kq_high); - INIT_LIST_HEAD(&kq->kq_low); - INIT_LIST_HEAD(&kq->kq_med); + InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; - kq->kq_state = NDIS_PSTATE_SLEEPING; mtx_init(&kq->kq_lock, "NDIS thread lock", NULL, MTX_SPIN); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_dead, EVENT_TYPE_SYNC, FALSE); + /* + * Elevate our priority. DPCs are used to run interrupt + * handlers, and they should trigger as soon as possible + * once scheduled by an ISR. + */ + + mtx_lock_spin(&sched_lock); sched_pin(); + sched_prio(curthread, PRI_MIN_KERN); +#if __FreeBSD_version < 600000 + curthread->td_base_pri = PRI_MIN_KERN; +#endif + mtx_unlock_spin(&sched_lock); while (1) { KeWaitForSingleObject((nt_dispatch_header *)&kq->kq_proc, @@ -2889,59 +3497,23 @@ ntoskrnl_dpc_thread(arg) if (kq->kq_exit) { mtx_unlock_spin(&kq->kq_lock); - KeSetEvent(&kq->kq_dead, 0, FALSE); + KeSetEvent(&kq->kq_dead, IO_NO_INCREMENT, FALSE); break; } - kq->kq_state = NDIS_PSTATE_RUNNING; - - /* Process high importance list first. */ - - l = kq->kq_high.nle_flink; - while (l != &kq->kq_high) { - d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); - REMOVE_LIST_ENTRY((&d->k_dpclistentry)); - INIT_LIST_HEAD((&d->k_dpclistentry)); - d->k_lock = NULL; - mtx_unlock_spin(&kq->kq_lock); - ntoskrnl_run_dpc(d); - mtx_lock_spin(&kq->kq_lock); - l = kq->kq_high.nle_flink; - } - - /* Now the medium importance list. */ - - l = kq->kq_med.nle_flink; - while (l != &kq->kq_med) { - d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); - REMOVE_LIST_ENTRY((&d->k_dpclistentry)); - INIT_LIST_HEAD((&d->k_dpclistentry)); - d->k_lock = NULL; - mtx_unlock_spin(&kq->kq_lock); - ntoskrnl_run_dpc(d); - mtx_lock_spin(&kq->kq_lock); - l = kq->kq_med.nle_flink; - } - - /* And finally the low importance list. */ - - l = kq->kq_low.nle_flink; - while (l != &kq->kq_low) { + while (!IsListEmpty(&kq->kq_disp)) { + l = RemoveHeadList((&kq->kq_disp)); d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); - REMOVE_LIST_ENTRY((&d->k_dpclistentry)); - INIT_LIST_HEAD((&d->k_dpclistentry)); + InitializeListHead((&d->k_dpclistentry)); d->k_lock = NULL; mtx_unlock_spin(&kq->kq_lock); ntoskrnl_run_dpc(d); mtx_lock_spin(&kq->kq_lock); - l = kq->kq_low.nle_flink; } - kq->kq_state = NDIS_PSTATE_SLEEPING; - mtx_unlock_spin(&kq->kq_lock); - KeSetEvent(&kq->kq_done, 0, FALSE); + KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE); } @@ -2995,7 +3567,6 @@ ntoskrnl_destroy_dpc_threads(void) KeInitializeDpc(&dpc, NULL, NULL); KeSetTargetProcessorDpc(&dpc, i); KeInsertQueueDpc(&dpc, NULL, NULL); - KeWaitForSingleObject((nt_dispatch_header *)&kq->kq_dead, 0, 0, TRUE, NULL); } @@ -3019,7 +3590,11 @@ ntoskrnl_insert_dpc(head, dpc) l = l->nle_flink; } - INSERT_LIST_TAIL((head), (&dpc->k_dpclistentry)); + if (dpc->k_importance == KDPC_IMPORTANCE_HIGH) + InsertHeadList((head), (&dpc->k_dpclistentry)); + else + InsertTailList((head), (&dpc->k_dpclistentry)); + return (TRUE); } @@ -3037,13 +3612,12 @@ KeInitializeDpc(dpc, dpcfunc, dpcctx) dpc->k_deferredctx = dpcctx; dpc->k_num = KDPC_CPU_DEFAULT; dpc->k_importance = KDPC_IMPORTANCE_MEDIUM; - dpc->k_num = KeGetCurrentProcessorNumber(); /* * In case someone tries to dequeue a DPC that * hasn't been queued yet. */ - dpc->k_lock = NULL /*&ntoskrnl_dispatchlock*/; - INIT_LIST_HEAD((&dpc->k_dpclistentry)); + dpc->k_lock = NULL; + InitializeListHead((&dpc->k_dpclistentry)); return; } @@ -3056,7 +3630,6 @@ KeInsertQueueDpc(dpc, sysarg1, sysarg2) { kdpc_queue *kq; uint8_t r; - int state; if (dpc == NULL) return(FALSE); @@ -3081,21 +3654,15 @@ KeInsertQueueDpc(dpc, sysarg1, sysarg2) */ mtx_lock_spin(&kq->kq_lock); - if (dpc->k_importance == KDPC_IMPORTANCE_HIGH) - r = ntoskrnl_insert_dpc(&kq->kq_high, dpc); - else if (dpc->k_importance == KDPC_IMPORTANCE_LOW) - r = ntoskrnl_insert_dpc(&kq->kq_low, dpc); - else - r = ntoskrnl_insert_dpc(&kq->kq_med, dpc); + r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc); if (r == TRUE) dpc->k_lock = &kq->kq_lock; - state = kq->kq_state; mtx_unlock_spin(&kq->kq_lock); + if (r == FALSE) return(r); - if (state == NDIS_PSTATE_SLEEPING) - KeSetEvent(&kq->kq_proc, 0, FALSE); + KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); return(r); } @@ -3104,21 +3671,26 @@ uint8_t KeRemoveQueueDpc(dpc) kdpc *dpc; { + struct mtx *lock; + if (dpc == NULL) return(FALSE); - if (dpc->k_lock == NULL) + lock = dpc->k_lock; + + if (lock == NULL) return(FALSE); - mtx_lock_spin((struct mtx*)(dpc->k_lock)); + mtx_lock_spin(lock); + dpc->k_lock = NULL; if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) { - mtx_unlock_spin((struct mtx*)(dpc->k_lock)); + mtx_unlock_spin(lock); return(FALSE); } - REMOVE_LIST_ENTRY((&dpc->k_dpclistentry)); - INIT_LIST_HEAD((&dpc->k_dpclistentry)); - mtx_unlock_spin((struct mtx*)(dpc->k_lock)); + RemoveEntryList((&dpc->k_dpclistentry)); + InitializeListHead((&dpc->k_dpclistentry)); + mtx_unlock_spin(lock); return(TRUE); } @@ -3162,7 +3734,7 @@ KeFlushQueuedDpcs(void) for (i = 0; i < mp_ncpus; i++) { kq = kq_queues + i; - KeSetEvent(&kq->kq_proc, 0, FALSE); + KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); KeWaitForSingleObject((nt_dispatch_header *)&kq->kq_done, 0, 0, TRUE, NULL); } @@ -3193,7 +3765,7 @@ KeSetTimerEx(timer, duetime, period, dpc) mtx_lock(&ntoskrnl_dispatchlock); if (timer->k_header.dh_inserted == TRUE) { - untimeout(ntoskrnl_timercall, timer, timer->k_handle); + callout_stop(timer->k_callout); timer->k_header.dh_inserted = FALSE; pending = TRUE; } else @@ -3220,7 +3792,10 @@ KeSetTimerEx(timer, duetime, period, dpc) } timer->k_header.dh_inserted = TRUE; - timer->k_handle = timeout(ntoskrnl_timercall, timer, tvtohz(&tv)); + timer->k_callout = &ntoskrnl_callout[ntoskrnl_callidx]; + CALLOUT_INC(ntoskrnl_callidx); + callout_reset(timer->k_callout, tvtohz(&tv), + ntoskrnl_timercall, timer); mtx_unlock(&ntoskrnl_dispatchlock); @@ -3236,6 +3811,12 @@ KeSetTimer(timer, duetime, dpc) return (KeSetTimerEx(timer, duetime, 0, dpc)); } +/* + * The Windows DDK documentation seems to say that cancelling + * a timer that has a DPC will result in the DPC also being + * cancelled, but this isn't really the case. + */ + uint8_t KeCancelTimer(timer) ktimer *timer; @@ -3247,14 +3828,12 @@ KeCancelTimer(timer) mtx_lock(&ntoskrnl_dispatchlock); + pending = timer->k_header.dh_inserted; + if (timer->k_header.dh_inserted == TRUE) { - untimeout(ntoskrnl_timercall, timer, timer->k_handle); - if (timer->k_dpc != NULL) - KeRemoveQueueDpc(timer->k_dpc); timer->k_header.dh_inserted = FALSE; - pending = TRUE; - } else - pending = KeRemoveQueueDpc(timer->k_dpc); + callout_stop(timer->k_callout); + } mtx_unlock(&ntoskrnl_dispatchlock); @@ -3299,6 +3878,8 @@ image_patch_table ntoskrnl_functbl[] = { IMPORT_CFUNC(strncpy, 0), IMPORT_CFUNC(strcpy, 0), IMPORT_CFUNC(strlen, 0), + IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0), + IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0), IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0), IMPORT_CFUNC_MAP(strchr, index, 0), IMPORT_CFUNC(memcpy, 0), @@ -3311,6 +3892,8 @@ image_patch_table ntoskrnl_functbl[] = { IMPORT_SFUNC(IoAcquireCancelSpinLock, 1), IMPORT_SFUNC(IoReleaseCancelSpinLock, 1), IMPORT_SFUNC(IoCancelIrp, 1), + IMPORT_SFUNC(IoConnectInterrupt, 11), + IMPORT_SFUNC(IoDisconnectInterrupt, 1), IMPORT_SFUNC(IoCreateDevice, 7), IMPORT_SFUNC(IoDeleteDevice, 1), IMPORT_SFUNC(IoGetAttachedDevice, 1), @@ -3324,6 +3907,7 @@ image_patch_table ntoskrnl_functbl[] = { IMPORT_SFUNC(IoMakeAssociatedIrp, 2), IMPORT_SFUNC(IoFreeIrp, 1), IMPORT_SFUNC(IoInitializeIrp, 3), + IMPORT_SFUNC(KeSynchronizeExecution, 3), IMPORT_SFUNC(KeWaitForSingleObject, 5), IMPORT_SFUNC(KeWaitForMultipleObjects, 8), IMPORT_SFUNC(_allmul, 4), diff --git a/sys/compat/ndis/winx32_wrap.S b/sys/compat/ndis/winx32_wrap.S index 4fadfb5..f5d24ba 100644 --- a/sys/compat/ndis/winx32_wrap.S +++ b/sys/compat/ndis/winx32_wrap.S @@ -82,6 +82,13 @@ * we only know the absolute addresses of the thunk and unthunk * routines. So we need to make sure the templates have enough * room in them for the full address. + * + * Also note that when we call the a thunk/unthunk routine after + * invoking a wrapped function, we have to make sure to preserve + * the value returned from that function. Most functions return + * a 32-bit value in %eax, however some routines return 64-bit + * values, which span both %eax and %edx. Consequently, we have + * to preserve both registers. */ /* @@ -114,13 +121,16 @@ ENTRY(x86_stdcall_wrap) x86_stdcall_wrap_call: movl $0,%eax call *%eax # jump to routine - mov %eax,%esi # preserve return val + push %eax # preserve return val + push %edx movl $ctxsw_utow, %eax call *%eax # thunk + pop %edx + pop %eax # restore return val + add $64,%esp # clean the stack - mov %esi,%eax # restore return val pop %edi pop %esi x86_stdcall_wrap_arg: @@ -151,12 +161,14 @@ ENTRY(x86_stdcall_call) call ctxsw_utow # thunk call *12(%edi) # branch to stdcall routine - mov %eax,%esi # preserve return val + push %eax # preserve return val + push %edx call ctxsw_wtou # unthunk + pop %edx + pop %eax # restore return val mov %edi,%esp # restore stack - mov %esi,%eax # restore return val pop %edi # restore %edi pop %esi # and %esi ret @@ -188,10 +200,12 @@ x86_fastcall_wrap_call: mov $0,%eax call *%eax # branch to fastcall routine push %eax # preserve return val + push %edx movl $ctxsw_utow, %eax call *%eax # thunk + pop %edx pop %eax # restore return val add $12,%esp # clean the stack x86_fastcall_wrap_arg: @@ -215,9 +229,11 @@ ENTRY(x86_fastcall_call) mov 16(%esp),%edx call *8(%esp) # branch to fastcall routine push %eax # preserve return val + push %edx call ctxsw_wtou # unthunk + pop %edx pop %eax # restore return val add $4,%esp # clean the stack ret |
