Implement IoCancelIrp(), IoAcquireCancelSpinLock(), IoReleaseCancelSpinLock()

and a machine-independent though inefficient InterlockedExchange().
In Windows, InterlockedExchange() appears to be implemented in header
files via inline assembly. I would prefer using an atomic.h macro for
this, but there doesn't seem to be one that just does a plain old
atomic exchange (as opposed to compare and exchange). Also implement
IoSetCancelRoutine(), which is just a macro that uses InterlockedExchange().

Fill in IoBuildSynchronousFsdRequest(), IoBuildAsynchronousFsdRequest()
and IoBuildDeviceIoControlRequest() so that they do something useful,
and add a bunch of #defines to ntoskrnl_var.h to help make these work.
These may require some tweaks later.

2 files changed, 260 insertions, 5 deletions

diff --git a/sys/compat/ndis/ntoskrnl_var.h b/sys/compat/ndis/ntoskrnl_var.h
index 54ae903..8e2771c 100644
--- a/sys/compat/ndis/ntoskrnl_var.h
+++ b/sys/compat/ndis/ntoskrnl_var.h
@@ -574,6 +574,26 @@ struct devobj_extension {
 
 typedef struct devobj_extension devobj_extension;
 
+/* Device object flags */
+
+#define DO_VERIFY_VOLUME		0x00000002      
+#define DO_BUFFERED_IO			0x00000004      
+#define DO_EXCLUSIVE			0x00000008      
+#define DO_DIRECT_IO			0x00000010      
+#define DO_MAP_IO_BUFFER		0x00000020      
+#define DO_DEVICE_HAS_NAME		0x00000040      
+#define DO_DEVICE_INITIALIZING		0x00000080      
+#define DO_SYSTEM_BOOT_PARTITION	0x00000100      
+#define DO_LONG_TERM_REQUESTS		0x00000200      
+#define DO_NEVER_LAST_DEVICE		0x00000400      
+#define DO_SHUTDOWN_REGISTERED		0x00000800      
+#define DO_BUS_ENUMERATED_DEVICE	0x00001000      
+#define DO_POWER_PAGABLE		0x00002000      
+#define DO_POWER_INRUSH			0x00004000      
+#define DO_LOW_PRIORITY_FILESYSTEM	0x00010000      
+
+/* Priority boosts */
+
 #define IO_NO_INCREMENT			0
 #define IO_CD_ROM_INCREMENT		1
 #define IO_DISK_INCREMENT		1
@@ -723,6 +743,15 @@ typedef struct devobj_extension devobj_extension;
 #define IRP_ALLOCATED_FIXED_SIZE        0x04
 #define IRP_LOOKASIDE_ALLOCATION        0x08
 
+/* I/O method types */
+
+#define METHOD_BUFFERED			0
+#define METHOD_IN_DIRECT		1
+#define METHOD_OUT_DIRECT		2
+#define METHOD_NEITHER			3
+
+#define IO_METHOD(x)			((x) & 0xFFFFFFFC)
+
 struct io_status_block {
 	union {
 		uint32_t		isb_status;
@@ -756,6 +785,8 @@ typedef struct kapc kapc;
 
 typedef __stdcall uint32_t (*completion_func)(device_object *,
 	struct irp *, void *);
+typedef __stdcall uint32_t (*cancel_func)(device_object *,
+	struct irp *);
 
 struct io_stack_location {
 	uint8_t			isl_major;
@@ -775,12 +806,28 @@ struct io_stack_location {
 
 	union {
 		struct {
+			uint32_t		isl_len;
+			uint32_t		*isl_key;
+			uint64_t		isl_byteoff;
+		} isl_read;
+		struct {
+			uint32_t		isl_len;
+			uint32_t		*isl_key;
+			uint64_t		isl_byteoff;
+		} isl_write;
+		struct {
+			uint32_t		isl_obuflen;
+			uint32_t		isl_ibuflen;
+			uint32_t		isl_iocode;
+			void			*isl_type3ibuf;
+		} isl_ioctl;
+		struct {
 			void			*isl_arg1;
 			void			*isl_arg2;
 			void			*isl_arg3;
 			void			*isl_arg4;
 		} isl_others;
-	} isl_parameters;
+	} isl_parameters __attribute__((packed));
 
 	void			*isl_devobj;
 	void			*isl_fileobj;
@@ -826,7 +873,7 @@ struct irp {
 		} irp_asyncparms;
 		uint64_t			irp_allocsz;
 	} irp_overlay;
-	void			*irp_cancelfunc;
+	cancel_func		irp_cancelfunc;
 	void			*irp_userbuf;
 
 	/* Windows kernel info */
@@ -860,9 +907,16 @@ struct irp {
 
 typedef struct irp irp;
 
+#define InterlockedExchangePointer(dst, val)				\
+	(void *)FASTCALL2(InterlockedExchange, (uint32_t *)(dst),	\
+	(uintptr_t)(val))
+
 #define IoSizeOfIrp(ssize)						\
 	((uint16_t) (sizeof(irp) + ((ssize) * (sizeof(io_stack_location)))))
 
+#define IoSetCancelRoutine(irp, func)					\
+	(cancel_func)InterlockedExchangePointer(			\
+	(void *)&(ip)->irp_cancelfunc, (void *)(func))
 
 #define IoGetCurrentIrpStackLocation(irp)				\
 	(irp)->irp_tail.irp_overlay.irp_csl
@@ -1104,6 +1158,8 @@ __stdcall extern uint32_t KeResetEvent(nt_kevent *);
 __fastcall extern void KefAcquireSpinLockAtDpcLevel(REGARGS1(kspin_lock *));
 __fastcall extern void KefReleaseSpinLockFromDpcLevel(REGARGS1(kspin_lock *));
 __stdcall extern void KeInitializeSpinLock(kspin_lock *);
+__fastcall extern uintptr_t InterlockedExchange(REGARGS2(volatile uint32_t *,
+	uintptr_t));
 __stdcall extern void *ExAllocatePoolWithTag(uint32_t, size_t, uint32_t);
 __stdcall extern void ExFreePool(void *);
 __stdcall extern uint32_t IoAllocateDriverObjectExtension(driver_object *,
@@ -1115,6 +1171,9 @@ __stdcall extern void IoDeleteDevice(device_object *);
 __stdcall extern device_object *IoGetAttachedDevice(device_object *);
 __fastcall extern uint32_t IofCallDriver(REGARGS2(device_object *, irp *));
 __fastcall extern void IofCompleteRequest(REGARGS2(irp *, uint8_t));
+__stdcall extern void IoAcquireCancelSpinLock(uint8_t *);
+__stdcall extern void IoReleaseCancelSpinLock(uint8_t);
+__stdcall extern uint8_t IoCancelIrp(irp *);
 __stdcall extern void IoDetachDevice(device_object *);
 __stdcall extern device_object *IoAttachDeviceToDeviceStack(device_object *,
 	device_object *);
diff --git a/sys/compat/ndis/subr_ntoskrnl.c b/sys/compat/ndis/subr_ntoskrnl.c
index eb7f5ba..4065147 100644
--- a/sys/compat/ndis/subr_ntoskrnl.c
+++ b/sys/compat/ndis/subr_ntoskrnl.c
@@ -183,6 +183,7 @@ __stdcall static void dummy(void);
 
 static struct mtx ntoskrnl_dispatchlock;
 static kspin_lock ntoskrnl_global;
+static kspin_lock ntoskrnl_cancellock;
 static int ntoskrnl_kth = 0;
 static struct nt_objref_head ntoskrnl_reflist;
 
@@ -531,7 +532,14 @@ IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
 	nt_kevent		*event;
 	io_status_block		*status;
 {
-	return(NULL);
+	irp			*ip;
+
+	ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
+	if (ip == NULL)
+		return(NULL);
+	ip->irp_usrevent = event;
+
+	return(ip);
 }
 
 __stdcall static irp *
@@ -543,7 +551,66 @@ IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
 	uint64_t		*off;
 	io_status_block		*status;
 {
-	return(NULL);
+	irp			*ip;
+	io_stack_location	*sl;
+
+	ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
+	if (ip == NULL)
+		return(NULL);
+
+	ip->irp_usriostat = status;
+	ip->irp_tail.irp_overlay.irp_thread = NULL;
+
+	sl = IoGetNextIrpStackLocation(ip);
+	sl->isl_major = func;
+	sl->isl_minor = 0;
+	sl->isl_flags = 0;
+	sl->isl_ctl = 0;
+	sl->isl_devobj = dobj;
+	sl->isl_fileobj = NULL;
+	sl->isl_completionfunc = NULL;
+
+	ip->irp_userbuf = buf;
+
+	if (dobj->do_flags & DO_BUFFERED_IO) {
+		ip->irp_assoc.irp_sysbuf =
+		    ExAllocatePoolWithTag(NonPagedPool, len, 0);
+		if (ip->irp_assoc.irp_sysbuf == NULL) {
+			IoFreeIrp(ip);
+			return(NULL);
+		}
+		bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
+	}
+
+	if (dobj->do_flags & DO_DIRECT_IO) {
+		ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
+		if (ip->irp_mdl == NULL) {
+			if (ip->irp_assoc.irp_sysbuf != NULL)
+				ExFreePool(ip->irp_assoc.irp_sysbuf);
+			IoFreeIrp(ip);
+			return(NULL);
+		}
+		ip->irp_userbuf = NULL;
+		ip->irp_assoc.irp_sysbuf = NULL;
+	}
+
+	if (func == IRP_MJ_READ) {
+		sl->isl_parameters.isl_read.isl_len = len;
+		if (off != NULL)
+			sl->isl_parameters.isl_read.isl_byteoff = *off;
+		else
+			sl->isl_parameters.isl_read.isl_byteoff = 0;
+	}
+
+	if (func == IRP_MJ_WRITE) {
+		sl->isl_parameters.isl_write.isl_len = len;
+		if (off != NULL)
+			sl->isl_parameters.isl_write.isl_byteoff = *off;
+		else
+			sl->isl_parameters.isl_write.isl_byteoff = 0;
+	}	
+
+	return(ip);
 }
 
 __stdcall static irp *
@@ -559,7 +626,87 @@ IoBuildDeviceIoControlRequest(iocode, dobj, ibuf, ilen, obuf, olen,
 	nt_kevent		*event;
 	io_status_block		*status;
 {
-	return (NULL);
+	irp			*ip;
+	io_stack_location	*sl;
+	uint32_t		buflen;
+
+	ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
+	if (ip == NULL)
+		return(NULL);
+	ip->irp_usrevent = event;
+	ip->irp_usriostat = status;
+	ip->irp_tail.irp_overlay.irp_thread = NULL;
+
+	sl = IoGetNextIrpStackLocation(ip);
+	sl->isl_major = isinternal == TRUE ?
+	    IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
+	sl->isl_minor = 0;
+	sl->isl_flags = 0;
+	sl->isl_ctl = 0;
+	sl->isl_devobj = dobj;
+	sl->isl_fileobj = NULL;
+	sl->isl_completionfunc = NULL;
+	sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
+	sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
+	sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
+
+	switch(IO_METHOD(iocode)) {
+	case METHOD_BUFFERED:
+		if (ilen > olen)
+			buflen = ilen;
+		else
+			buflen = olen;
+		if (buflen) {
+			ip->irp_assoc.irp_sysbuf =
+			    ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
+			if (ip->irp_assoc.irp_sysbuf == NULL) {
+				IoFreeIrp(ip);
+				return(NULL);
+			}
+		}
+		if (ilen && ibuf != NULL) {
+			bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
+			bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
+			    buflen - ilen);
+		} else
+			bzero(ip->irp_assoc.irp_sysbuf, ilen);
+		ip->irp_userbuf = obuf;
+		break;
+	case METHOD_IN_DIRECT:
+	case METHOD_OUT_DIRECT:
+		if (ilen && ibuf != NULL) {
+			ip->irp_assoc.irp_sysbuf =
+			    ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
+			if (ip->irp_assoc.irp_sysbuf == NULL) {
+				IoFreeIrp(ip);
+				return(NULL);
+			}
+			bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
+		}
+		if (olen && obuf != NULL) {
+			ip->irp_mdl = IoAllocateMdl(obuf, olen,
+			    FALSE, FALSE, ip);
+			/*
+			 * Normally we would MmProbeAndLockPages()
+			 * here, but we don't have to in our
+			 * imlementation.
+			 */
+		}
+		break;
+	case METHOD_NEITHER:
+		ip->irp_userbuf = obuf;
+		sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
+		break;
+	default:
+		break;
+	}
+
+	/*
+	 * Ideally, we should associate this IRP with the calling
+	 * thread here.
+	 */
+
+	return (ip);
 }
 
 __stdcall static irp *
@@ -639,6 +786,38 @@ IoReuseIrp(ip, status)
 	return;
 }
 
+__stdcall void
+IoAcquireCancelSpinLock(irql)
+	uint8_t			*irql;
+{
+	KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
+	return;
+}
+
+__stdcall void
+IoReleaseCancelSpinLock(irql)
+	uint8_t			irql;
+{
+	KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
+	return;
+}
+
+__stdcall uint8_t
+IoCancelIrp(irp *ip)
+{
+	cancel_func		cfunc;
+
+	IoAcquireCancelSpinLock(&ip->irp_cancelirql);
+	cfunc = IoSetCancelRoutine(ip, NULL);
+	ip->irp_cancel = TRUE;
+	if (ip->irp_cancelfunc == NULL) {
+		IoReleaseCancelSpinLock(ip->irp_cancelirql);
+		return(FALSE);
+	}
+	MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
+	return(TRUE);
+}
+
 __fastcall uint32_t
 IofCallDriver(REGARGS2(device_object *dobj, irp *ip))
 {
@@ -1498,6 +1677,20 @@ KefReleaseSpinLockFromDpcLevel(REGARGS1(kspin_lock *lock))
 	return;
 }
 
+__fastcall uintptr_t
+InterlockedExchange(REGARGS2(volatile uint32_t *dst, uintptr_t val))
+{
+	uint8_t			irql;
+	uintptr_t		r;
+
+	KeAcquireSpinLock(&ntoskrnl_global, &irql);
+	r = *dst;
+	*dst = val;
+	KeReleaseSpinLock(&ntoskrnl_global, irql);
+
+	return(r);
+}
+
 __fastcall static uint32_t
 InterlockedIncrement(REGARGS1(volatile uint32_t *addend))
 {
@@ -2376,6 +2569,9 @@ image_patch_table ntoskrnl_functbl[] = {
 	IMPORT_FUNC(IoGetDriverObjectExtension),
 	IMPORT_FUNC(IofCallDriver),
 	IMPORT_FUNC(IofCompleteRequest),
+	IMPORT_FUNC(IoAcquireCancelSpinLock),
+	IMPORT_FUNC(IoReleaseCancelSpinLock),
+	IMPORT_FUNC(IoCancelIrp),
 	IMPORT_FUNC(IoCreateDevice),
 	IMPORT_FUNC(IoDeleteDevice),
 	IMPORT_FUNC(IoGetAttachedDevice),