Update drm kernel drivers.

This is a sync to mesa/drm pre-gem, with a few fixes on top of that.
It also contains one local patch supplied by kib@ that I can't apply to
git.master shared code.

Approved by:	flz
Obtained from:	mesa/drm git.master
MFC after:	2 weeks

70 files changed, 13390 insertions, 6411 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 7da9135..e2b9924 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -572,6 +572,7 @@ dev/drm/i915_dma.c		optional i915drm
 dev/drm/i915_drv.c		optional i915drm
 dev/drm/i915_irq.c		optional i915drm
 dev/drm/i915_mem.c		optional i915drm
+dev/drm/i915_suspend.c		optional i915drm
 dev/drm/mach64_dma.c		optional mach64drm
 dev/drm/mach64_drv.c		optional mach64drm
 dev/drm/mach64_irq.c		optional mach64drm
diff --git a/sys/dev/drm/ati_pcigart.c b/sys/dev/drm/ati_pcigart.c
index 386702c..dac400d 100644
--- a/sys/dev/drm/ati_pcigart.c
+++ b/sys/dev/drm/ati_pcigart.c
@@ -1,6 +1,3 @@
-/* ati_pcigart.h -- ATI PCI GART support -*- linux-c -*-
- * Created: Wed Dec 13 21:52:19 2000 by gareth@valinux.com
- */
 /*-
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
@@ -32,72 +29,133 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file ati_pcigart.c
+ * Implementation of ATI's PCIGART, which provides an aperture in card virtual
+ * address space with addresses remapped to system memory.
+ */
+
 #include "dev/drm/drmP.h"
 
 #define ATI_PCIGART_PAGE_SIZE		4096	/* PCI GART page size */
-#define ATI_MAX_PCIGART_PAGES		8192	/* 32 MB aperture, 4K pages */
-#define ATI_PCIGART_TABLE_SIZE		32768
+#define ATI_PCIGART_PAGE_MASK		(~(ATI_PCIGART_PAGE_SIZE-1))
 
-int drm_ati_pcigart_init(drm_device_t *dev, drm_ati_pcigart_info *gart_info)
+#define ATI_PCIE_WRITE 0x4
+#define ATI_PCIE_READ 0x8
+
+static int drm_ati_alloc_pcigart_table(struct drm_device *dev,
+				       struct drm_ati_pcigart_info *gart_info)
 {
-	unsigned long pages;
-	u32 *pci_gart = NULL, page_base;
-	int i, j;
+	dev->sg->dmah = drm_pci_alloc(dev, gart_info->table_size,
+						PAGE_SIZE,
+						gart_info->table_mask);
+	if (dev->sg->dmah == NULL)
+		return ENOMEM;
+
+	return 0;
+}
+
+static void drm_ati_free_pcigart_table(struct drm_device *dev,
+				       struct drm_ati_pcigart_info *gart_info)
+{
+	drm_pci_free(dev, dev->sg->dmah);
+	dev->sg->dmah = NULL;
+}
 
+int drm_ati_pcigart_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info)
+{
+	/* we need to support large memory configurations */
 	if (dev->sg == NULL) {
-		DRM_ERROR( "no scatter/gather memory!\n" );
+		DRM_ERROR("no scatter/gather memory!\n");
 		return 0;
 	}
 
+	if (gart_info->bus_addr) {
+		if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) {
+			gart_info->bus_addr = 0;
+			if (dev->sg->dmah)
+				drm_ati_free_pcigart_table(dev, gart_info);
+		}
+	}
+
+	return 1;
+}
+
+int drm_ati_pcigart_init(struct drm_device *dev,
+			 struct drm_ati_pcigart_info *gart_info)
+{
+
+	void *address = NULL;
+	unsigned long pages;
+	u32 *pci_gart, page_base;
+	dma_addr_t bus_address = 0;
+	int i, j, ret = 0;
+	int max_pages;
+	dma_addr_t entry_addr;
+
+	/* we need to support large memory configurations */
+	if (dev->sg == NULL) {
+		DRM_ERROR("no scatter/gather memory!\n");
+		goto done;
+	}
+
 	if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) {
-		/* GART table in system memory */
-		dev->sg->dmah = drm_pci_alloc(dev, ATI_PCIGART_TABLE_SIZE, 0,
-		    0xfffffffful);
-		if (dev->sg->dmah == NULL) {
-			DRM_ERROR("cannot allocate PCI GART table!\n");
-			return 0;
+		DRM_DEBUG("PCI: no table in VRAM: using normal RAM\n");
+
+		ret = drm_ati_alloc_pcigart_table(dev, gart_info);
+		if (ret) {
+			DRM_ERROR("cannot allocate PCI GART page!\n");
+			goto done;
 		}
-	
-		gart_info->addr = (void *)dev->sg->dmah->vaddr;
-		gart_info->bus_addr = dev->sg->dmah->busaddr;
-		pci_gart = (u32 *)dev->sg->dmah->vaddr;
+
+		address = (void *)dev->sg->dmah->vaddr;
+		bus_address = dev->sg->dmah->busaddr;
 	} else {
-		/* GART table in framebuffer memory */
-		pci_gart = gart_info->addr;
+		address = gart_info->addr;
+		bus_address = gart_info->bus_addr;
+		DRM_DEBUG("PCI: Gart Table: VRAM %08X mapped at %08lX\n",
+			  (unsigned int)bus_address, (unsigned long)address);
 	}
-	
-	pages = DRM_MIN(dev->sg->pages, ATI_MAX_PCIGART_PAGES);
 
-	bzero(pci_gart, ATI_PCIGART_TABLE_SIZE);
+	pci_gart = (u32 *) address;
 
-	KASSERT(PAGE_SIZE >= ATI_PCIGART_PAGE_SIZE, ("page size too small"));
+	max_pages = (gart_info->table_size / sizeof(u32));
+	pages = (dev->sg->pages <= max_pages)
+	    ? dev->sg->pages : max_pages;
 
-	for ( i = 0 ; i < pages ; i++ ) {
-		page_base = (u32) dev->sg->busaddr[i];
+	memset(pci_gart, 0, max_pages * sizeof(u32));
+
+	KASSERT(PAGE_SIZE >= ATI_PCIGART_PAGE_SIZE, ("page size too small"));
 
+	for (i = 0; i < pages; i++) {
+		entry_addr = dev->sg->busaddr[i];
 		for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
-			if (gart_info->is_pcie)
-				*pci_gart = (cpu_to_le32(page_base) >> 8) | 0xc;
-			else
-				*pci_gart = cpu_to_le32(page_base);
+			page_base = (u32) entry_addr & ATI_PCIGART_PAGE_MASK;
+			switch(gart_info->gart_reg_if) {
+			case DRM_ATI_GART_IGP:
+				page_base |= (upper_32_bits(entry_addr) & 0xff) << 4;
+				page_base |= 0xc;
+				break;
+			case DRM_ATI_GART_PCIE:
+				page_base >>= 8;
+				page_base |= (upper_32_bits(entry_addr) & 0xff) << 24;
+				page_base |= ATI_PCIE_READ | ATI_PCIE_WRITE;
+				break;
+			default:
+			case DRM_ATI_GART_PCI:
+				break;
+			}
+			*pci_gart = cpu_to_le32(page_base);
 			pci_gart++;
-			page_base += ATI_PCIGART_PAGE_SIZE;
+			entry_addr += ATI_PCIGART_PAGE_SIZE;
 		}
 	}
 
 	DRM_MEMORYBARRIER();
 
-	return 1;
-}
-
-int drm_ati_pcigart_cleanup(drm_device_t *dev, drm_ati_pcigart_info *gart_info)
-{
-	if (dev->sg == NULL) {
-		DRM_ERROR( "no scatter/gather memory!\n" );
-		return 0;
-	}
+	ret = 1;
 
-	drm_pci_free(dev, dev->sg->dmah);
-
-	return 1;
+    done:
+	gart_info->addr = address;
+	gart_info->bus_addr = bus_address;
+	return ret;
 }
diff --git a/sys/dev/drm/drm-subprocess.pl b/sys/dev/drm/drm-subprocess.pl
index d053c59..16d27b5 100644
--- a/sys/dev/drm/drm-subprocess.pl
+++ b/sys/dev/drm/drm-subprocess.pl
@@ -37,6 +37,7 @@ while (<>) {
 		s/#include "(.*)_drm.h/#include "dev\/drm\/\1_drm.h/;
 		s/#include "mga_ucode.h/#include "dev\/drm\/mga_ucode.h/;
 		s/#include "r300_reg.h/#include "dev\/drm\/r300_reg.h/;
+		s/#include "radeon_microcode.h/#include "dev\/drm\/radeon_microcode.h/;
 		s/#include "sis_ds.h/#include "dev\/drm\/sis_ds.h/;
 		s/#include "drm/#include "dev\/drm\/drm/;
 		print;
@@ -47,4 +48,4 @@ while (<>) {
 # if we never found the copyright header, then we're still a line behind.
 if (!$foundopening) {
 	print $lastline;
-}
\ No newline at end of file
+}
diff --git a/sys/dev/drm/drm.h b/sys/dev/drm/drm.h
index d4dad8e..dffa18f 100644
--- a/sys/dev/drm/drm.h
+++ b/sys/dev/drm/drm.h
@@ -64,6 +64,9 @@ __FBSDID("$FreeBSD$");
 #ifndef __user
 #define __user
 #endif
+#ifndef __iomem
+#define __iomem
+#endif
 
 #ifdef __GNUC__
 # define DEPRECATED  __attribute__ ((deprecated))
@@ -72,9 +75,6 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #if defined(__linux__)
-#if defined(__KERNEL__)
-#include <linux/config.h>
-#endif
 #include <asm/ioctl.h>		/* For _IO* macros */
 #define DRM_IOCTL_NR(n)		_IOC_NR(n)
 #define DRM_IOC_VOID		_IOC_NONE
@@ -83,14 +83,7 @@ __FBSDID("$FreeBSD$");
 #define DRM_IOC_READWRITE	_IOC_READ|_IOC_WRITE
 #define DRM_IOC(dir, group, nr, size) _IOC(dir, group, nr, size)
 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
-#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) && defined(IN_MODULE)
-/* Prevent name collision when including sys/ioccom.h */
-#undef ioctl
 #include <sys/ioccom.h>
-#define ioctl(a,b,c)		xf86ioctl(a,b,c)
-#else
-#include <sys/ioccom.h>
-#endif				/* __FreeBSD__ && xf86ioctl */
 #define DRM_IOCTL_NR(n)		((n) & 0xff)
 #define DRM_IOC_VOID		IOC_VOID
 #define DRM_IOC_READ		IOC_OUT
@@ -99,24 +92,6 @@ __FBSDID("$FreeBSD$");
 #define DRM_IOC(dir, group, nr, size) _IOC(dir, group, nr, size)
 #endif
 
-#define XFREE86_VERSION(major,minor,patch,snap) \
-		((major << 16) | (minor << 8) | patch)
-
-#ifndef CONFIG_XFREE86_VERSION
-#define CONFIG_XFREE86_VERSION XFREE86_VERSION(4,1,0,0)
-#endif
-
-#if CONFIG_XFREE86_VERSION < XFREE86_VERSION(4,1,0,0)
-#define DRM_PROC_DEVICES "/proc/devices"
-#define DRM_PROC_MISC	 "/proc/misc"
-#define DRM_PROC_DRM	 "/proc/drm"
-#define DRM_DEV_DRM	 "/dev/drm"
-#define DRM_DEV_MODE	 (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP)
-#define DRM_DEV_UID	 0
-#define DRM_DEV_GID	 0
-#endif
-
-#if CONFIG_XFREE86_VERSION >= XFREE86_VERSION(4,1,0,0)
 #ifdef __OpenBSD__
 #define DRM_MAJOR       81
 #endif
@@ -124,7 +99,7 @@ __FBSDID("$FreeBSD$");
 #define DRM_MAJOR       226
 #endif
 #define DRM_MAX_MINOR   15
-#endif
+
 #define DRM_NAME	"drm"	  /**< Name in kernel, /dev, and /proc */
 #define DRM_MIN_ORDER	5	  /**< At least 2^5 bytes = 32 bytes */
 #define DRM_MAX_ORDER	22	  /**< Up to 2^22 bytes = 4MB */
@@ -139,6 +114,7 @@ __FBSDID("$FreeBSD$");
 #if defined(__linux__)
 typedef unsigned int drm_handle_t;
 #else
+#include <sys/types.h>
 typedef unsigned long drm_handle_t;	/**< To mapped regions */
 #endif
 typedef unsigned int drm_context_t;	/**< GLXContext handle */
@@ -154,23 +130,23 @@ typedef unsigned int drm_magic_t;	/**< Magic for authentication */
  * \note KW: Actually it's illegal to change either for
  * backwards-compatibility reasons.
  */
-typedef struct drm_clip_rect {
+struct drm_clip_rect {
 	unsigned short x1;
 	unsigned short y1;
 	unsigned short x2;
 	unsigned short y2;
-} drm_clip_rect_t;
+};
 
 /**
  * Texture region,
  */
-typedef struct drm_tex_region {
+struct drm_tex_region {
 	unsigned char next;
 	unsigned char prev;
 	unsigned char in_use;
 	unsigned char padding;
 	unsigned int age;
-} drm_tex_region_t;
+};
 
 /**
  * Hardware lock.
@@ -179,10 +155,10 @@ typedef struct drm_tex_region {
  * processor bus contention on a multiprocessor system, there should not be any
  * other data stored in the same cache line.
  */
-typedef struct drm_hw_lock {
+struct drm_hw_lock {
 	__volatile__ unsigned int lock;		/**< lock variable */
 	char padding[60];			/**< Pad to cache line */
-} drm_hw_lock_t;
+};
 
 /* This is beyond ugly, and only works on GCC.  However, it allows me to use
  * drm.h in places (i.e., in the X-server) where I can't use size_t.  The real
@@ -205,7 +181,7 @@ typedef struct drm_hw_lock {
  *
  * \sa drmGetVersion().
  */
-typedef struct drm_version {
+struct drm_version {
 	int version_major;	  /**< Major version */
 	int version_minor;	  /**< Minor version */
 	int version_patchlevel;	  /**< Patch level */
@@ -215,35 +191,35 @@ typedef struct drm_version {
 	char __user *date;		  /**< User-space buffer to hold date */
 	DRM_SIZE_T desc_len;	  /**< Length of desc buffer */
 	char __user *desc;		  /**< User-space buffer to hold desc */
-} drm_version_t;
+};
 
 /**
  * DRM_IOCTL_GET_UNIQUE ioctl argument type.
  *
  * \sa drmGetBusid() and drmSetBusId().
  */
-typedef struct drm_unique {
+struct drm_unique {
 	DRM_SIZE_T unique_len;	  /**< Length of unique */
 	char __user *unique;		  /**< Unique name for driver instantiation */
-} drm_unique_t;
+};
 
 #undef DRM_SIZE_T
 
-typedef struct drm_list {
+struct drm_list {
 	int count;		  /**< Length of user-space structures */
-	drm_version_t __user *version;
-} drm_list_t;
+	struct drm_version __user *version;
+};
 
-typedef struct drm_block {
+struct drm_block {
 	int unused;
-} drm_block_t;
+};
 
 /**
  * DRM_IOCTL_CONTROL ioctl argument type.
  *
  * \sa drmCtlInstHandler() and drmCtlUninstHandler().
  */
-typedef struct drm_control {
+struct drm_control {
 	enum {
 		DRM_ADD_COMMAND,
 		DRM_RM_COMMAND,
@@ -251,37 +227,39 @@ typedef struct drm_control {
 		DRM_UNINST_HANDLER
 	} func;
 	int irq;
-} drm_control_t;
+};
 
 /**
  * Type of memory to map.
  */
-typedef enum drm_map_type {
+enum drm_map_type {
 	_DRM_FRAME_BUFFER = 0,	  /**< WC (no caching), no core dump */
 	_DRM_REGISTERS = 1,	  /**< no caching, no core dump */
 	_DRM_SHM = 2,		  /**< shared, cached */
 	_DRM_AGP = 3,		  /**< AGP/GART */
 	_DRM_SCATTER_GATHER = 4,  /**< Scatter/gather memory for PCI DMA */
-	_DRM_CONSISTENT = 5	  /**< Consistent memory for PCI DMA */
-} drm_map_type_t;
+	_DRM_CONSISTENT = 5,	  /**< Consistent memory for PCI DMA */
+	_DRM_TTM = 6
+};
 
 /**
  * Memory mapping flags.
  */
-typedef enum drm_map_flags {
+enum drm_map_flags {
 	_DRM_RESTRICTED = 0x01,	     /**< Cannot be mapped to user-virtual */
 	_DRM_READ_ONLY = 0x02,
 	_DRM_LOCKED = 0x04,	     /**< shared, cached, locked */
 	_DRM_KERNEL = 0x08,	     /**< kernel requires access */
 	_DRM_WRITE_COMBINING = 0x10, /**< use write-combining if available */
 	_DRM_CONTAINS_LOCK = 0x20,   /**< SHM page that contains lock */
-	_DRM_REMOVABLE = 0x40	     /**< Removable mapping */
-} drm_map_flags_t;
+	_DRM_REMOVABLE = 0x40,	     /**< Removable mapping */
+	_DRM_DRIVER = 0x80	     /**< Managed by driver */
+};
 
-typedef struct drm_ctx_priv_map {
+struct drm_ctx_priv_map {
 	unsigned int ctx_id;	 /**< Context requesting private mapping */
 	void *handle;		 /**< Handle of map */
-} drm_ctx_priv_map_t;
+};
 
 /**
  * DRM_IOCTL_GET_MAP, DRM_IOCTL_ADD_MAP and DRM_IOCTL_RM_MAP ioctls
@@ -289,30 +267,30 @@ typedef struct drm_ctx_priv_map {
  *
  * \sa drmAddMap().
  */
-typedef struct drm_map {
+struct drm_map {
 	unsigned long offset;	 /**< Requested physical address (0 for SAREA)*/
 	unsigned long size;	 /**< Requested physical size (bytes) */
-	drm_map_type_t type;	 /**< Type of memory to map */
-	drm_map_flags_t flags;	 /**< Flags */
+	enum drm_map_type type;	 /**< Type of memory to map */
+	enum drm_map_flags flags;	 /**< Flags */
 	void *handle;		 /**< User-space: "Handle" to pass to mmap() */
 				 /**< Kernel-space: kernel-virtual address */
 	int mtrr;		 /**< MTRR slot used */
 	/*   Private data */
-} drm_map_t;
+};
 
 /**
  * DRM_IOCTL_GET_CLIENT ioctl argument type.
  */
-typedef struct drm_client {
+struct drm_client {
 	int idx;		/**< Which client desired? */
 	int auth;		/**< Is client authenticated? */
 	unsigned long pid;	/**< Process ID */
 	unsigned long uid;	/**< User ID */
 	unsigned long magic;	/**< Magic */
 	unsigned long iocs;	/**< Ioctl count */
-} drm_client_t;
+};
 
-typedef enum {
+enum drm_stat_type {
 	_DRM_STAT_LOCK,
 	_DRM_STAT_OPENS,
 	_DRM_STAT_CLOSES,
@@ -330,23 +308,23 @@ typedef enum {
 	_DRM_STAT_SPECIAL,	/**< Special DMA (e.g., priority or polled) */
 	_DRM_STAT_MISSED	/**< Missed DMA opportunity */
 	    /* Add to the *END* of the list */
-} drm_stat_type_t;
+};
 
 /**
  * DRM_IOCTL_GET_STATS ioctl argument type.
  */
-typedef struct drm_stats {
+struct drm_stats {
 	unsigned long count;
 	struct {
 		unsigned long value;
-		drm_stat_type_t type;
+		enum drm_stat_type type;
 	} data[15];
-} drm_stats_t;
+};
 
 /**
  * Hardware locking flags.
  */
-typedef enum drm_lock_flags {
+enum drm_lock_flags {
 	_DRM_LOCK_READY = 0x01,	     /**< Wait until hardware is ready for DMA */
 	_DRM_LOCK_QUIESCENT = 0x02,  /**< Wait until hardware quiescent */
 	_DRM_LOCK_FLUSH = 0x04,	     /**< Flush this context's DMA queue first */
@@ -356,17 +334,17 @@ typedef enum drm_lock_flags {
 	   full-screen DGA-like mode. */
 	_DRM_HALT_ALL_QUEUES = 0x10, /**< Halt all current and future queues */
 	_DRM_HALT_CUR_QUEUES = 0x20  /**< Halt all current queues */
-} drm_lock_flags_t;
+};
 
 /**
  * DRM_IOCTL_LOCK, DRM_IOCTL_UNLOCK and DRM_IOCTL_FINISH ioctl argument type.
  *
  * \sa drmGetLock() and drmUnlock().
  */
-typedef struct drm_lock {
+struct drm_lock {
 	int context;
-	drm_lock_flags_t flags;
-} drm_lock_t;
+	enum drm_lock_flags flags;
+};
 
 /**
  * DMA flags
@@ -376,7 +354,7 @@ typedef struct drm_lock {
  *
  * \sa drm_dma.
  */
-typedef enum drm_dma_flags {
+enum drm_dma_flags {
 	/* Flags for DMA buffer dispatch */
 	_DRM_DMA_BLOCK = 0x01,	      /**<
 				       * Block until buffer dispatched.
@@ -395,14 +373,14 @@ typedef enum drm_dma_flags {
 	_DRM_DMA_WAIT = 0x10,	      /**< Wait for free buffers */
 	_DRM_DMA_SMALLER_OK = 0x20,   /**< Smaller-than-requested buffers OK */
 	_DRM_DMA_LARGER_OK = 0x40     /**< Larger-than-requested buffers OK */
-} drm_dma_flags_t;
+};
 
 /**
  * DRM_IOCTL_ADD_BUFS and DRM_IOCTL_MARK_BUFS ioctl argument type.
  *
  * \sa drmAddBufs().
  */
-typedef struct drm_buf_desc {
+struct drm_buf_desc {
 	int count;		 /**< Number of buffers of this size */
 	int size;		 /**< Size in bytes */
 	int low_mark;		 /**< Low water mark */
@@ -411,54 +389,55 @@ typedef struct drm_buf_desc {
 		_DRM_PAGE_ALIGN = 0x01,	/**< Align on page boundaries for DMA */
 		_DRM_AGP_BUFFER = 0x02,	/**< Buffer is in AGP space */
 		_DRM_SG_BUFFER  = 0x04,	/**< Scatter/gather memory buffer */
-		_DRM_FB_BUFFER  = 0x08  /**< Buffer is in frame buffer */
+		_DRM_FB_BUFFER  = 0x08, /**< Buffer is in frame buffer */
+		_DRM_PCI_BUFFER_RO = 0x10 /**< Map PCI DMA buffer read-only */
 	} flags;
 	unsigned long agp_start; /**<
 				  * Start address of where the AGP buffers are
 				  * in the AGP aperture
 				  */
-} drm_buf_desc_t;
+};
 
 /**
  * DRM_IOCTL_INFO_BUFS ioctl argument type.
  */
-typedef struct drm_buf_info {
+struct drm_buf_info {
 	int count;		  /**< Number of buffers described in list */
-	drm_buf_desc_t __user *list;	  /**< List of buffer descriptions */
-} drm_buf_info_t;
+	struct drm_buf_desc __user *list; /**< List of buffer descriptions */
+};
 
 /**
  * DRM_IOCTL_FREE_BUFS ioctl argument type.
  */
-typedef struct drm_buf_free {
+struct drm_buf_free {
 	int count;
 	int __user *list;
-} drm_buf_free_t;
+};
 
 /**
  * Buffer information
  *
  * \sa drm_buf_map.
  */
-typedef struct drm_buf_pub {
+struct drm_buf_pub {
 	int idx;		       /**< Index into the master buffer list */
 	int total;		       /**< Buffer size */
 	int used;		       /**< Amount of buffer in use (for DMA) */
 	void __user *address;	       /**< Address of buffer */
-} drm_buf_pub_t;
+};
 
 /**
  * DRM_IOCTL_MAP_BUFS ioctl argument type.
  */
-typedef struct drm_buf_map {
+struct drm_buf_map {
 	int count;		/**< Length of the buffer list */
 #if defined(__cplusplus)
 	void __user *c_virtual;
 #else
 	void __user *virtual;		/**< Mmap'd area in user-virtual */
 #endif
-	drm_buf_pub_t __user *list;	/**< Buffer information */
-} drm_buf_map_t;
+	struct drm_buf_pub __user *list;	/**< Buffer information */
+};
 
 /**
  * DRM_IOCTL_DMA ioctl argument type.
@@ -467,84 +446,103 @@ typedef struct drm_buf_map {
  *
  * \sa drmDMA().
  */
-typedef struct drm_dma {
+struct drm_dma {
 	int context;			  /**< Context handle */
 	int send_count;			  /**< Number of buffers to send */
 	int __user *send_indices;	  /**< List of handles to buffers */
 	int __user *send_sizes;		  /**< Lengths of data to send */
-	drm_dma_flags_t flags;		  /**< Flags */
+	enum drm_dma_flags flags;	  /**< Flags */
 	int request_count;		  /**< Number of buffers requested */
 	int request_size;		  /**< Desired size for buffers */
 	int __user *request_indices;	 /**< Buffer information */
 	int __user *request_sizes;
 	int granted_count;		  /**< Number of buffers granted */
-} drm_dma_t;
+};
 
-typedef enum {
+enum drm_ctx_flags {
 	_DRM_CONTEXT_PRESERVED = 0x01,
 	_DRM_CONTEXT_2DONLY = 0x02
-} drm_ctx_flags_t;
+};
 
 /**
  * DRM_IOCTL_ADD_CTX ioctl argument type.
  *
  * \sa drmCreateContext() and drmDestroyContext().
  */
-typedef struct drm_ctx {
+struct drm_ctx {
 	drm_context_t handle;
-	drm_ctx_flags_t flags;
-} drm_ctx_t;
+	enum drm_ctx_flags flags;
+};
 
 /**
  * DRM_IOCTL_RES_CTX ioctl argument type.
  */
-typedef struct drm_ctx_res {
+struct drm_ctx_res {
 	int count;
-	drm_ctx_t __user *contexts;
-} drm_ctx_res_t;
+	struct drm_ctx __user *contexts;
+};
 
 /**
  * DRM_IOCTL_ADD_DRAW and DRM_IOCTL_RM_DRAW ioctl argument type.
  */
-typedef struct drm_draw {
+struct drm_draw {
 	drm_drawable_t handle;
-} drm_draw_t;
+};
+
+/**
+ * DRM_IOCTL_UPDATE_DRAW ioctl argument type.
+ */
+typedef enum {
+	DRM_DRAWABLE_CLIPRECTS,
+} drm_drawable_info_type_t;
+
+struct drm_update_draw {
+	drm_drawable_t handle;
+	unsigned int type;
+	unsigned int num;
+	unsigned long long data;
+};
 
 /**
  * DRM_IOCTL_GET_MAGIC and DRM_IOCTL_AUTH_MAGIC ioctl argument type.
  */
-typedef struct drm_auth {
+struct drm_auth {
 	drm_magic_t magic;
-} drm_auth_t;
+};
 
 /**
  * DRM_IOCTL_IRQ_BUSID ioctl argument type.
  *
  * \sa drmGetInterruptFromBusID().
  */
-typedef struct drm_irq_busid {
+struct drm_irq_busid {
 	int irq;	/**< IRQ number */
 	int busnum;	/**< bus number */
 	int devnum;	/**< device number */
 	int funcnum;	/**< function number */
-} drm_irq_busid_t;
+};
 
-typedef enum {
+enum drm_vblank_seq_type {
 	_DRM_VBLANK_ABSOLUTE = 0x0,	/**< Wait for specific vblank sequence number */
 	_DRM_VBLANK_RELATIVE = 0x1,	/**< Wait for given number of vblanks */
+	_DRM_VBLANK_FLIP = 0x8000000,	/**< Scheduled buffer swap should flip */
+	_DRM_VBLANK_NEXTONMISS = 0x10000000,	/**< If missed, wait for next vblank */
+	_DRM_VBLANK_SECONDARY = 0x20000000,	/**< Secondary display controller */
 	_DRM_VBLANK_SIGNAL = 0x40000000	/**< Send signal instead of blocking */
-} drm_vblank_seq_type_t;
+};
 
-#define _DRM_VBLANK_FLAGS_MASK _DRM_VBLANK_SIGNAL
+#define _DRM_VBLANK_TYPES_MASK (_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE)
+#define _DRM_VBLANK_FLAGS_MASK (_DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY | \
+				_DRM_VBLANK_NEXTONMISS)
 
 struct drm_wait_vblank_request {
-	drm_vblank_seq_type_t type;
+	enum drm_vblank_seq_type type;
 	unsigned int sequence;
 	unsigned long signal;
 };
 
 struct drm_wait_vblank_reply {
-	drm_vblank_seq_type_t type;
+	enum drm_vblank_seq_type type;
 	unsigned int sequence;
 	long tval_sec;
 	long tval_usec;
@@ -555,41 +553,55 @@ struct drm_wait_vblank_reply {
  *
  * \sa drmWaitVBlank().
  */
-typedef union drm_wait_vblank {
+union drm_wait_vblank {
 	struct drm_wait_vblank_request request;
 	struct drm_wait_vblank_reply reply;
-} drm_wait_vblank_t;
+};
+
+
+#define _DRM_PRE_MODESET 1
+#define _DRM_POST_MODESET 2
+
+/**
+ * DRM_IOCTL_MODESET_CTL ioctl argument type
+ *
+ * \sa drmModesetCtl().
+ */
+struct drm_modeset_ctl {
+	uint32_t crtc;
+	uint32_t cmd;
+};
 
 /**
  * DRM_IOCTL_AGP_ENABLE ioctl argument type.
  *
  * \sa drmAgpEnable().
  */
-typedef struct drm_agp_mode {
+struct drm_agp_mode {
 	unsigned long mode;	/**< AGP mode */
-} drm_agp_mode_t;
+};
 
 /**
  * DRM_IOCTL_AGP_ALLOC and DRM_IOCTL_AGP_FREE ioctls argument type.
  *
  * \sa drmAgpAlloc() and drmAgpFree().
  */
-typedef struct drm_agp_buffer {
+struct drm_agp_buffer {
 	unsigned long size;	/**< In bytes -- will round to page boundary */
 	unsigned long handle;	/**< Used for binding / unbinding */
 	unsigned long type;	/**< Type of memory to allocate */
 	unsigned long physical;	/**< Physical used by i810 */
-} drm_agp_buffer_t;
+};
 
 /**
  * DRM_IOCTL_AGP_BIND and DRM_IOCTL_AGP_UNBIND ioctls argument type.
  *
  * \sa drmAgpBind() and drmAgpUnbind().
  */
-typedef struct drm_agp_binding {
+struct drm_agp_binding {
 	unsigned long handle;	/**< From drm_agp_buffer */
 	unsigned long offset;	/**< In bytes -- will round to page boundary */
-} drm_agp_binding_t;
+};
 
 /**
  * DRM_IOCTL_AGP_INFO ioctl argument type.
@@ -598,7 +610,7 @@ typedef struct drm_agp_binding {
  * drmAgpBase(), drmAgpSize(), drmAgpMemoryUsed(), drmAgpMemoryAvail(),
  * drmAgpVendorId() and drmAgpDeviceId().
  */
-typedef struct drm_agp_info {
+struct drm_agp_info {
 	int agp_version_major;
 	int agp_version_minor;
 	unsigned long mode;
@@ -612,25 +624,343 @@ typedef struct drm_agp_info {
 	unsigned short id_vendor;
 	unsigned short id_device;
 	/*@} */
-} drm_agp_info_t;
+};
 
 /**
  * DRM_IOCTL_SG_ALLOC ioctl argument type.
  */
-typedef struct drm_scatter_gather {
+struct drm_scatter_gather {
 	unsigned long size;	/**< In bytes -- will round to page boundary */
 	unsigned long handle;	/**< Used for mapping / unmapping */
-} drm_scatter_gather_t;
+};
 
 /**
  * DRM_IOCTL_SET_VERSION ioctl argument type.
  */
-typedef struct drm_set_version {
+struct drm_set_version {
 	int drm_di_major;
 	int drm_di_minor;
 	int drm_dd_major;
 	int drm_dd_minor;
-} drm_set_version_t;
+};
+
+
+#define DRM_FENCE_FLAG_EMIT                0x00000001
+#define DRM_FENCE_FLAG_SHAREABLE           0x00000002
+/**
+ * On hardware with no interrupt events for operation completion,
+ * indicates that the kernel should sleep while waiting for any blocking
+ * operation to complete rather than spinning.
+ *
+ * Has no effect otherwise.
+ */
+#define DRM_FENCE_FLAG_WAIT_LAZY           0x00000004
+#define DRM_FENCE_FLAG_NO_USER             0x00000010
+
+/* Reserved for driver use */
+#define DRM_FENCE_MASK_DRIVER              0xFF000000
+
+#define DRM_FENCE_TYPE_EXE                 0x00000001
+
+struct drm_fence_arg {
+	unsigned int handle;
+	unsigned int fence_class;
+	unsigned int type;
+	unsigned int flags;
+	unsigned int signaled;
+	unsigned int error;
+	unsigned int sequence;
+	unsigned int pad64;
+	uint64_t expand_pad[2]; /*Future expansion */
+};
+
+/* Buffer permissions, referring to how the GPU uses the buffers.
+ * these translate to fence types used for the buffers.
+ * Typically a texture buffer is read, A destination buffer is write and
+ *  a command (batch-) buffer is exe. Can be or-ed together.
+ */
+
+#define DRM_BO_FLAG_READ        (1ULL << 0)
+#define DRM_BO_FLAG_WRITE       (1ULL << 1)
+#define DRM_BO_FLAG_EXE         (1ULL << 2)
+
+/*
+ * All of the bits related to access mode
+ */
+#define DRM_BO_MASK_ACCESS	(DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_EXE)
+/*
+ * Status flags. Can be read to determine the actual state of a buffer.
+ * Can also be set in the buffer mask before validation.
+ */
+
+/*
+ * Mask: Never evict this buffer. Not even with force. This type of buffer is only
+ * available to root and must be manually removed before buffer manager shutdown
+ * or lock.
+ * Flags: Acknowledge
+ */
+#define DRM_BO_FLAG_NO_EVICT    (1ULL << 4)
+
+/*
+ * Mask: Require that the buffer is placed in mappable memory when validated.
+ *       If not set the buffer may or may not be in mappable memory when validated.
+ * Flags: If set, the buffer is in mappable memory.
+ */
+#define DRM_BO_FLAG_MAPPABLE    (1ULL << 5)
+
+/* Mask: The buffer should be shareable with other processes.
+ * Flags: The buffer is shareable with other processes.
+ */
+#define DRM_BO_FLAG_SHAREABLE   (1ULL << 6)
+
+/* Mask: If set, place the buffer in cache-coherent memory if available.
+ *       If clear, never place the buffer in cache coherent memory if validated.
+ * Flags: The buffer is currently in cache-coherent memory.
+ */
+#define DRM_BO_FLAG_CACHED      (1ULL << 7)
+
+/* Mask: Make sure that every time this buffer is validated,
+ *       it ends up on the same location provided that the memory mask is the same.
+ *       The buffer will also not be evicted when claiming space for
+ *       other buffers. Basically a pinned buffer but it may be thrown out as
+ *       part of buffer manager shutdown or locking.
+ * Flags: Acknowledge.
+ */
+#define DRM_BO_FLAG_NO_MOVE     (1ULL << 8)
+
+/* Mask: Make sure the buffer is in cached memory when mapped.  In conjunction
+ * with DRM_BO_FLAG_CACHED it also allows the buffer to be bound into the GART
+ * with unsnooped PTEs instead of snooped, by using chipset-specific cache
+ * flushing at bind time.  A better name might be DRM_BO_FLAG_TT_UNSNOOPED,
+ * as the eviction to local memory (TTM unbind) on map is just a side effect
+ * to prevent aggressive cache prefetch from the GPU disturbing the cache
+ * management that the DRM is doing.
+ *
+ * Flags: Acknowledge.
+ * Buffers allocated with this flag should not be used for suballocators
+ * This type may have issues on CPUs with over-aggressive caching
+ * http://marc.info/?l=linux-kernel&m=102376926732464&w=2
+ */
+#define DRM_BO_FLAG_CACHED_MAPPED    (1ULL << 19)
+
+
+/* Mask: Force DRM_BO_FLAG_CACHED flag strictly also if it is set.
+ * Flags: Acknowledge.
+ */
+#define DRM_BO_FLAG_FORCE_CACHING  (1ULL << 13)
+
+/*
+ * Mask: Force DRM_BO_FLAG_MAPPABLE flag strictly also if it is clear.
+ * Flags: Acknowledge.
+ */
+#define DRM_BO_FLAG_FORCE_MAPPABLE (1ULL << 14)
+#define DRM_BO_FLAG_TILE           (1ULL << 15)
+
+/*
+ * Memory type flags that can be or'ed together in the mask, but only
+ * one appears in flags.
+ */
+
+/* System memory */
+#define DRM_BO_FLAG_MEM_LOCAL  (1ULL << 24)
+/* Translation table memory */
+#define DRM_BO_FLAG_MEM_TT     (1ULL << 25)
+/* Vram memory */
+#define DRM_BO_FLAG_MEM_VRAM   (1ULL << 26)
+/* Up to the driver to define. */
+#define DRM_BO_FLAG_MEM_PRIV0  (1ULL << 27)
+#define DRM_BO_FLAG_MEM_PRIV1  (1ULL << 28)
+#define DRM_BO_FLAG_MEM_PRIV2  (1ULL << 29)
+#define DRM_BO_FLAG_MEM_PRIV3  (1ULL << 30)
+#define DRM_BO_FLAG_MEM_PRIV4  (1ULL << 31)
+/* We can add more of these now with a 64-bit flag type */
+
+/*
+ * This is a mask covering all of the memory type flags; easier to just
+ * use a single constant than a bunch of | values. It covers
+ * DRM_BO_FLAG_MEM_LOCAL through DRM_BO_FLAG_MEM_PRIV4
+ */
+#define DRM_BO_MASK_MEM         0x00000000FF000000ULL
+/*
+ * This adds all of the CPU-mapping options in with the memory
+ * type to label all bits which change how the page gets mapped
+ */
+#define DRM_BO_MASK_MEMTYPE     (DRM_BO_MASK_MEM | \
+				 DRM_BO_FLAG_CACHED_MAPPED | \
+				 DRM_BO_FLAG_CACHED | \
+				 DRM_BO_FLAG_MAPPABLE)
+				 
+/* Driver-private flags */
+#define DRM_BO_MASK_DRIVER      0xFFFF000000000000ULL
+
+/*
+ * Don't block on validate and map. Instead, return EBUSY.
+ */
+#define DRM_BO_HINT_DONT_BLOCK  0x00000002
+/*
+ * Don't place this buffer on the unfenced list. This means
+ * that the buffer will not end up having a fence associated
+ * with it as a result of this operation
+ */
+#define DRM_BO_HINT_DONT_FENCE  0x00000004
+/**
+ * On hardware with no interrupt events for operation completion,
+ * indicates that the kernel should sleep while waiting for any blocking
+ * operation to complete rather than spinning.
+ *
+ * Has no effect otherwise.
+ */
+#define DRM_BO_HINT_WAIT_LAZY   0x00000008
+/*
+ * The client has compute relocations refering to this buffer using the
+ * offset in the presumed_offset field. If that offset ends up matching
+ * where this buffer lands, the kernel is free to skip executing those
+ * relocations
+ */
+#define DRM_BO_HINT_PRESUMED_OFFSET 0x00000010
+
+#define DRM_BO_INIT_MAGIC 0xfe769812
+#define DRM_BO_INIT_MAJOR 1
+#define DRM_BO_INIT_MINOR 0
+#define DRM_BO_INIT_PATCH 0
+
+
+struct drm_bo_info_req {
+	uint64_t mask;
+	uint64_t flags;
+	unsigned int handle;
+	unsigned int hint;
+	unsigned int fence_class;
+	unsigned int desired_tile_stride;
+	unsigned int tile_info;
+	unsigned int pad64;
+	uint64_t presumed_offset;
+};
+
+struct drm_bo_create_req {
+	uint64_t flags;
+	uint64_t size;
+	uint64_t buffer_start;
+	unsigned int hint;
+	unsigned int page_alignment;
+};
+
+
+/*
+ * Reply flags
+ */
+
+#define DRM_BO_REP_BUSY 0x00000001
+
+struct drm_bo_info_rep {
+	uint64_t flags;
+	uint64_t proposed_flags;
+	uint64_t size;
+	uint64_t offset;
+	uint64_t arg_handle;
+	uint64_t buffer_start;
+	unsigned int handle;
+	unsigned int fence_flags;
+	unsigned int rep_flags;
+	unsigned int page_alignment;
+	unsigned int desired_tile_stride;
+	unsigned int hw_tile_stride;
+	unsigned int tile_info;
+	unsigned int pad64;
+	uint64_t expand_pad[4]; /*Future expansion */
+};
+
+struct drm_bo_arg_rep {
+	struct drm_bo_info_rep bo_info;
+	int ret;
+	unsigned int pad64;
+};
+
+struct drm_bo_create_arg {
+	union {
+		struct drm_bo_create_req req;
+		struct drm_bo_info_rep rep;
+	} d;
+};
+
+struct drm_bo_handle_arg {
+	unsigned int handle;
+};
+
+struct drm_bo_reference_info_arg {
+	union {
+		struct drm_bo_handle_arg req;
+		struct drm_bo_info_rep rep;
+	} d;
+};
+
+struct drm_bo_map_wait_idle_arg {
+	union {
+		struct drm_bo_info_req req;
+		struct drm_bo_info_rep rep;
+	} d;
+};
+
+struct drm_bo_op_req {
+	enum {
+		drm_bo_validate,
+		drm_bo_fence,
+		drm_bo_ref_fence,
+	} op;
+	unsigned int arg_handle;
+	struct drm_bo_info_req bo_req;
+};
+
+
+struct drm_bo_op_arg {
+	uint64_t next;
+	union {
+		struct drm_bo_op_req req;
+		struct drm_bo_arg_rep rep;
+	} d;
+	int handled;
+	unsigned int pad64;
+};
+
+
+#define DRM_BO_MEM_LOCAL 0
+#define DRM_BO_MEM_TT 1
+#define DRM_BO_MEM_VRAM 2
+#define DRM_BO_MEM_PRIV0 3
+#define DRM_BO_MEM_PRIV1 4
+#define DRM_BO_MEM_PRIV2 5
+#define DRM_BO_MEM_PRIV3 6
+#define DRM_BO_MEM_PRIV4 7
+
+#define DRM_BO_MEM_TYPES 8 /* For now. */
+
+#define DRM_BO_LOCK_UNLOCK_BM       (1 << 0)
+#define DRM_BO_LOCK_IGNORE_NO_EVICT (1 << 1)
+
+struct drm_bo_version_arg {
+	uint32_t major;
+	uint32_t minor;
+	uint32_t patchlevel;
+};
+
+struct drm_mm_type_arg {
+	unsigned int mem_type;
+	unsigned int lock_flags;
+};
+
+struct drm_mm_init_arg {
+	unsigned int magic;
+	unsigned int major;
+	unsigned int minor;
+	unsigned int mem_type;
+	uint64_t p_offset;
+	uint64_t p_size;
+};
+
+struct drm_mm_info_arg {
+	unsigned int mem_type;
+	uint64_t p_size;
+};
 
 /**
  * \name Ioctls Definitions
@@ -643,69 +973,147 @@ typedef struct drm_set_version {
 #define DRM_IOW(nr,type)		_IOW(DRM_IOCTL_BASE,nr,type)
 #define DRM_IOWR(nr,type)		_IOWR(DRM_IOCTL_BASE,nr,type)
 
-#define DRM_IOCTL_VERSION		DRM_IOWR(0x00, drm_version_t)
-#define DRM_IOCTL_GET_UNIQUE		DRM_IOWR(0x01, drm_unique_t)
-#define DRM_IOCTL_GET_MAGIC		DRM_IOR( 0x02, drm_auth_t)
-#define DRM_IOCTL_IRQ_BUSID		DRM_IOWR(0x03, drm_irq_busid_t)
-#define DRM_IOCTL_GET_MAP               DRM_IOWR(0x04, drm_map_t)
-#define DRM_IOCTL_GET_CLIENT            DRM_IOWR(0x05, drm_client_t)
-#define DRM_IOCTL_GET_STATS             DRM_IOR( 0x06, drm_stats_t)
-#define DRM_IOCTL_SET_VERSION		DRM_IOWR(0x07, drm_set_version_t)
-
-#define DRM_IOCTL_SET_UNIQUE		DRM_IOW( 0x10, drm_unique_t)
-#define DRM_IOCTL_AUTH_MAGIC		DRM_IOW( 0x11, drm_auth_t)
-#define DRM_IOCTL_BLOCK			DRM_IOWR(0x12, drm_block_t)
-#define DRM_IOCTL_UNBLOCK		DRM_IOWR(0x13, drm_block_t)
-#define DRM_IOCTL_CONTROL		DRM_IOW( 0x14, drm_control_t)
-#define DRM_IOCTL_ADD_MAP		DRM_IOWR(0x15, drm_map_t)
-#define DRM_IOCTL_ADD_BUFS		DRM_IOWR(0x16, drm_buf_desc_t)
-#define DRM_IOCTL_MARK_BUFS		DRM_IOW( 0x17, drm_buf_desc_t)
-#define DRM_IOCTL_INFO_BUFS		DRM_IOWR(0x18, drm_buf_info_t)
-#define DRM_IOCTL_MAP_BUFS		DRM_IOWR(0x19, drm_buf_map_t)
-#define DRM_IOCTL_FREE_BUFS		DRM_IOW( 0x1a, drm_buf_free_t)
-
-#define DRM_IOCTL_RM_MAP		DRM_IOW( 0x1b, drm_map_t)
-
-#define DRM_IOCTL_SET_SAREA_CTX		DRM_IOW( 0x1c, drm_ctx_priv_map_t)
-#define DRM_IOCTL_GET_SAREA_CTX 	DRM_IOWR(0x1d, drm_ctx_priv_map_t)
-
-#define DRM_IOCTL_ADD_CTX		DRM_IOWR(0x20, drm_ctx_t)
-#define DRM_IOCTL_RM_CTX		DRM_IOWR(0x21, drm_ctx_t)
-#define DRM_IOCTL_MOD_CTX		DRM_IOW( 0x22, drm_ctx_t)
-#define DRM_IOCTL_GET_CTX		DRM_IOWR(0x23, drm_ctx_t)
-#define DRM_IOCTL_SWITCH_CTX		DRM_IOW( 0x24, drm_ctx_t)
-#define DRM_IOCTL_NEW_CTX		DRM_IOW( 0x25, drm_ctx_t)
-#define DRM_IOCTL_RES_CTX		DRM_IOWR(0x26, drm_ctx_res_t)
-#define DRM_IOCTL_ADD_DRAW		DRM_IOWR(0x27, drm_draw_t)
-#define DRM_IOCTL_RM_DRAW		DRM_IOWR(0x28, drm_draw_t)
-#define DRM_IOCTL_DMA			DRM_IOWR(0x29, drm_dma_t)
-#define DRM_IOCTL_LOCK			DRM_IOW( 0x2a, drm_lock_t)
-#define DRM_IOCTL_UNLOCK		DRM_IOW( 0x2b, drm_lock_t)
-#define DRM_IOCTL_FINISH		DRM_IOW( 0x2c, drm_lock_t)
+#define DRM_IOCTL_VERSION		DRM_IOWR(0x00, struct drm_version)
+#define DRM_IOCTL_GET_UNIQUE		DRM_IOWR(0x01, struct drm_unique)
+#define DRM_IOCTL_GET_MAGIC		DRM_IOR( 0x02, struct drm_auth)
+#define DRM_IOCTL_IRQ_BUSID		DRM_IOWR(0x03, struct drm_irq_busid)
+#define DRM_IOCTL_GET_MAP               DRM_IOWR(0x04, struct drm_map)
+#define DRM_IOCTL_GET_CLIENT            DRM_IOWR(0x05, struct drm_client)
+#define DRM_IOCTL_GET_STATS             DRM_IOR( 0x06, struct drm_stats)
+#define DRM_IOCTL_SET_VERSION		DRM_IOWR(0x07, struct drm_set_version)
+#define DRM_IOCTL_MODESET_CTL           DRM_IOW(0x08, struct drm_modeset_ctl)
+
+#define DRM_IOCTL_SET_UNIQUE		DRM_IOW( 0x10, struct drm_unique)
+#define DRM_IOCTL_AUTH_MAGIC		DRM_IOW( 0x11, struct drm_auth)
+#define DRM_IOCTL_BLOCK			DRM_IOWR(0x12, struct drm_block)
+#define DRM_IOCTL_UNBLOCK		DRM_IOWR(0x13, struct drm_block)
+#define DRM_IOCTL_CONTROL		DRM_IOW( 0x14, struct drm_control)
+#define DRM_IOCTL_ADD_MAP		DRM_IOWR(0x15, struct drm_map)
+#define DRM_IOCTL_ADD_BUFS		DRM_IOWR(0x16, struct drm_buf_desc)
+#define DRM_IOCTL_MARK_BUFS		DRM_IOW( 0x17, struct drm_buf_desc)
+#define DRM_IOCTL_INFO_BUFS		DRM_IOWR(0x18, struct drm_buf_info)
+#define DRM_IOCTL_MAP_BUFS		DRM_IOWR(0x19, struct drm_buf_map)
+#define DRM_IOCTL_FREE_BUFS		DRM_IOW( 0x1a, struct drm_buf_free)
+
+#define DRM_IOCTL_RM_MAP		DRM_IOW( 0x1b, struct drm_map)
+
+#define DRM_IOCTL_SET_SAREA_CTX		DRM_IOW( 0x1c, struct drm_ctx_priv_map)
+#define DRM_IOCTL_GET_SAREA_CTX		DRM_IOWR(0x1d, struct drm_ctx_priv_map)
+
+#define DRM_IOCTL_ADD_CTX		DRM_IOWR(0x20, struct drm_ctx)
+#define DRM_IOCTL_RM_CTX		DRM_IOWR(0x21, struct drm_ctx)
+#define DRM_IOCTL_MOD_CTX		DRM_IOW( 0x22, struct drm_ctx)
+#define DRM_IOCTL_GET_CTX		DRM_IOWR(0x23, struct drm_ctx)
+#define DRM_IOCTL_SWITCH_CTX		DRM_IOW( 0x24, struct drm_ctx)
+#define DRM_IOCTL_NEW_CTX		DRM_IOW( 0x25, struct drm_ctx)
+#define DRM_IOCTL_RES_CTX		DRM_IOWR(0x26, struct drm_ctx_res)
+#define DRM_IOCTL_ADD_DRAW		DRM_IOWR(0x27, struct drm_draw)
+#define DRM_IOCTL_RM_DRAW		DRM_IOWR(0x28, struct drm_draw)
+#define DRM_IOCTL_DMA			DRM_IOWR(0x29, struct drm_dma)
+#define DRM_IOCTL_LOCK			DRM_IOW( 0x2a, struct drm_lock)
+#define DRM_IOCTL_UNLOCK		DRM_IOW( 0x2b, struct drm_lock)
+#define DRM_IOCTL_FINISH		DRM_IOW( 0x2c, struct drm_lock)
 
 #define DRM_IOCTL_AGP_ACQUIRE		DRM_IO(  0x30)
 #define DRM_IOCTL_AGP_RELEASE		DRM_IO(  0x31)
-#define DRM_IOCTL_AGP_ENABLE		DRM_IOW( 0x32, drm_agp_mode_t)
-#define DRM_IOCTL_AGP_INFO		DRM_IOR( 0x33, drm_agp_info_t)
-#define DRM_IOCTL_AGP_ALLOC		DRM_IOWR(0x34, drm_agp_buffer_t)
-#define DRM_IOCTL_AGP_FREE		DRM_IOW( 0x35, drm_agp_buffer_t)
-#define DRM_IOCTL_AGP_BIND		DRM_IOW( 0x36, drm_agp_binding_t)
-#define DRM_IOCTL_AGP_UNBIND		DRM_IOW( 0x37, drm_agp_binding_t)
-
-#define DRM_IOCTL_SG_ALLOC		DRM_IOW( 0x38, drm_scatter_gather_t)
-#define DRM_IOCTL_SG_FREE		DRM_IOW( 0x39, drm_scatter_gather_t)
-
-#define DRM_IOCTL_WAIT_VBLANK		DRM_IOWR(0x3a, drm_wait_vblank_t)
+#define DRM_IOCTL_AGP_ENABLE		DRM_IOW( 0x32, struct drm_agp_mode)
+#define DRM_IOCTL_AGP_INFO		DRM_IOR( 0x33, struct drm_agp_info)
+#define DRM_IOCTL_AGP_ALLOC		DRM_IOWR(0x34, struct drm_agp_buffer)
+#define DRM_IOCTL_AGP_FREE		DRM_IOW( 0x35, struct drm_agp_buffer)
+#define DRM_IOCTL_AGP_BIND		DRM_IOW( 0x36, struct drm_agp_binding)
+#define DRM_IOCTL_AGP_UNBIND		DRM_IOW( 0x37, struct drm_agp_binding)
+
+#define DRM_IOCTL_SG_ALLOC		DRM_IOWR(0x38, struct drm_scatter_gather)
+#define DRM_IOCTL_SG_FREE		DRM_IOW( 0x39, struct drm_scatter_gather)
+
+#define DRM_IOCTL_WAIT_VBLANK		DRM_IOWR(0x3a, union drm_wait_vblank)
+
+#define DRM_IOCTL_UPDATE_DRAW           DRM_IOW(0x3f, struct drm_update_draw)
+
+#define DRM_IOCTL_MM_INIT               DRM_IOWR(0xc0, struct drm_mm_init_arg)
+#define DRM_IOCTL_MM_TAKEDOWN           DRM_IOWR(0xc1, struct drm_mm_type_arg)
+#define DRM_IOCTL_MM_LOCK               DRM_IOWR(0xc2, struct drm_mm_type_arg)
+#define DRM_IOCTL_MM_UNLOCK             DRM_IOWR(0xc3, struct drm_mm_type_arg)
+
+#define DRM_IOCTL_FENCE_CREATE          DRM_IOWR(0xc4, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_REFERENCE       DRM_IOWR(0xc6, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_UNREFERENCE     DRM_IOWR(0xc7, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_SIGNALED        DRM_IOWR(0xc8, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_FLUSH           DRM_IOWR(0xc9, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_WAIT            DRM_IOWR(0xca, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_EMIT            DRM_IOWR(0xcb, struct drm_fence_arg)
+#define DRM_IOCTL_FENCE_BUFFERS         DRM_IOWR(0xcc, struct drm_fence_arg)
+
+#define DRM_IOCTL_BO_CREATE             DRM_IOWR(0xcd, struct drm_bo_create_arg)
+#define DRM_IOCTL_BO_MAP                DRM_IOWR(0xcf, struct drm_bo_map_wait_idle_arg)
+#define DRM_IOCTL_BO_UNMAP              DRM_IOWR(0xd0, struct drm_bo_handle_arg)
+#define DRM_IOCTL_BO_REFERENCE          DRM_IOWR(0xd1, struct drm_bo_reference_info_arg)
+#define DRM_IOCTL_BO_UNREFERENCE        DRM_IOWR(0xd2, struct drm_bo_handle_arg)
+#define DRM_IOCTL_BO_SETSTATUS          DRM_IOWR(0xd3, struct drm_bo_map_wait_idle_arg)
+#define DRM_IOCTL_BO_INFO               DRM_IOWR(0xd4, struct drm_bo_reference_info_arg)
+#define DRM_IOCTL_BO_WAIT_IDLE          DRM_IOWR(0xd5, struct drm_bo_map_wait_idle_arg)
+#define DRM_IOCTL_BO_VERSION          DRM_IOR(0xd6, struct drm_bo_version_arg)
+#define DRM_IOCTL_MM_INFO               DRM_IOWR(0xd7, struct drm_mm_info_arg)
 
 /*@}*/
 
 /**
  * Device specific ioctls should only be in their respective headers
- * The device specific ioctl range is from 0x40 to 0x79.
+ * The device specific ioctl range is from 0x40 to 0x99.
+ * Generic IOCTLS restart at 0xA0.
  *
  * \sa drmCommandNone(), drmCommandRead(), drmCommandWrite(), and
  * drmCommandReadWrite().
  */
 #define DRM_COMMAND_BASE                0x40
+#define DRM_COMMAND_END                 0xA0
+
+/* typedef area */
+#if !defined(__KERNEL__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__)
+typedef struct drm_clip_rect drm_clip_rect_t;
+typedef struct drm_tex_region drm_tex_region_t;
+typedef struct drm_hw_lock drm_hw_lock_t;
+typedef struct drm_version drm_version_t;
+typedef struct drm_unique drm_unique_t;
+typedef struct drm_list drm_list_t;
+typedef struct drm_block drm_block_t;
+typedef struct drm_control drm_control_t;
+typedef enum drm_map_type drm_map_type_t;
+typedef enum drm_map_flags drm_map_flags_t;
+typedef struct drm_ctx_priv_map drm_ctx_priv_map_t;
+typedef struct drm_map drm_map_t;
+typedef struct drm_client drm_client_t;
+typedef enum drm_stat_type drm_stat_type_t;
+typedef struct drm_stats drm_stats_t;
+typedef enum drm_lock_flags drm_lock_flags_t;
+typedef struct drm_lock drm_lock_t;
+typedef enum drm_dma_flags drm_dma_flags_t;
+typedef struct drm_buf_desc drm_buf_desc_t;
+typedef struct drm_buf_info drm_buf_info_t;
+typedef struct drm_buf_free drm_buf_free_t;
+typedef struct drm_buf_pub drm_buf_pub_t;
+typedef struct drm_buf_map drm_buf_map_t;
+typedef struct drm_dma drm_dma_t;
+typedef union drm_wait_vblank drm_wait_vblank_t;
+typedef struct drm_agp_mode drm_agp_mode_t;
+typedef enum drm_ctx_flags drm_ctx_flags_t;
+typedef struct drm_ctx drm_ctx_t;
+typedef struct drm_ctx_res drm_ctx_res_t;
+typedef struct drm_draw drm_draw_t;
+typedef struct drm_update_draw drm_update_draw_t;
+typedef struct drm_auth drm_auth_t;
+typedef struct drm_irq_busid drm_irq_busid_t;
+typedef enum drm_vblank_seq_type drm_vblank_seq_type_t;
+typedef struct drm_agp_buffer drm_agp_buffer_t;
+typedef struct drm_agp_binding drm_agp_binding_t;
+typedef struct drm_agp_info drm_agp_info_t;
+typedef struct drm_scatter_gather drm_scatter_gather_t;
+typedef struct drm_set_version drm_set_version_t;
+
+typedef struct drm_fence_arg drm_fence_arg_t;
+typedef struct drm_mm_type_arg drm_mm_type_arg_t;
+typedef struct drm_mm_init_arg drm_mm_init_arg_t;
+typedef enum drm_bo_type drm_bo_type_t;
+#endif
 
 #endif
diff --git a/sys/dev/drm/drmP.h b/sys/dev/drm/drmP.h
index 44c0130..6132760 100644
--- a/sys/dev/drm/drmP.h
+++ b/sys/dev/drm/drmP.h
@@ -39,7 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #if defined(_KERNEL) || defined(__KERNEL__)
 
-typedef struct drm_device drm_device_t;
+struct drm_device;
 typedef struct drm_file drm_file_t;
 
 #include <sys/param.h>
@@ -62,6 +62,8 @@ typedef struct drm_file drm_file_t;
 #include <sys/bus.h>
 #include <sys/signalvar.h>
 #include <sys/poll.h>
+#include <sys/tree.h>
+#include <sys/taskqueue.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #include <vm/vm_extern.h>
@@ -79,9 +81,9 @@ typedef struct drm_file drm_file_t;
 #include <sys/memrange.h>
 #if __FreeBSD_version >= 800004
 #include <dev/agp/agpvar.h>
-#else
+#else /* __FreeBSD_version >= 800004 */
 #include <pci/agpvar.h>
-#endif
+#endif /* __FreeBSD_version >= 800004 */
 #include <sys/agpio.h>
 #if __FreeBSD_version >= 500000
 #include <sys/mutex.h>
@@ -113,6 +115,7 @@ typedef struct drm_file drm_file_t;
 #include "dev/drm/drm.h"
 #include "dev/drm/drm_linux_list.h"
 #include "dev/drm/drm_atomic.h"
+#include "dev/drm/drm_internal.h"
 
 #ifdef __FreeBSD__
 #include <opt_drm.h>
@@ -159,6 +162,7 @@ typedef struct drm_file drm_file_t;
 #define DRM_MEM_CTXBITMAP 17
 #define DRM_MEM_STUB	  18
 #define DRM_MEM_SGLISTS	  19
+#define DRM_MEM_DRAWABLE  20
 
 #define DRM_MAX_CTXBITMAP (PAGE_SIZE * 8)
 
@@ -191,10 +195,15 @@ MALLOC_DECLARE(M_DRM);
 #define DRM_CURPROC		curthread
 #define DRM_STRUCTPROC		struct thread
 #define DRM_SPINTYPE		struct mtx
-#define DRM_SPININIT(l,name)	mtx_init(&l, name, NULL, MTX_DEF)
-#define DRM_SPINUNINIT(l)	mtx_destroy(&l)
+#define DRM_SPININIT(l,name)	mtx_init(l, name, NULL, MTX_DEF)
+#define DRM_SPINUNINIT(l)	mtx_destroy(l)
 #define DRM_SPINLOCK(l)		mtx_lock(l)
-#define DRM_SPINUNLOCK(u)	mtx_unlock(u);
+#define DRM_SPINUNLOCK(u)	mtx_unlock(u)
+#define DRM_SPINLOCK_IRQSAVE(l, irqflags) do {		\
+	mtx_lock(l);					\
+	(void)irqflags;					\
+} while (0)
+#define DRM_SPINUNLOCK_IRQRESTORE(u, irqflags) mtx_unlock(u)
 #define DRM_SPINLOCK_ASSERT(l)	mtx_assert(l, MA_OWNED)
 #define DRM_CURRENTPID		curthread->td_proc->p_pid
 #define DRM_LOCK()		mtx_lock(&dev->dev_lock)
@@ -216,10 +225,6 @@ MALLOC_DECLARE(M_DRM);
 #define spldrm()		spltty()
 #endif /* __NetBSD__ || __OpenBSD__ */
 
-/* Currently our DRMFILE (filp) is a void * which is actually the pid
- * of the current process.  It should be a per-open unique pointer, but
- * code for that is not yet written */
-#define DRMFILE			void *
 #define DRM_IRQ_ARGS		void *arg
 typedef void			irqreturn_t;
 #define IRQ_HANDLED		/* nothing */
@@ -233,15 +238,19 @@ enum {
 #define DRM_AGP_MEM		struct agp_memory_info
 
 #if defined(__FreeBSD__)
-#define DRM_DEVICE							\
-	drm_device_t *dev = kdev->si_drv1
-#define DRM_IOCTL_ARGS		struct cdev *kdev, u_long cmd, caddr_t data, \
-				int flags, DRM_STRUCTPROC *p, DRMFILE filp
+#define drm_get_device_from_kdev(_kdev) (_kdev->si_drv1)
+#elif defined(__NetBSD__)
+#define drm_get_device_from_kdev(_kdev) device_lookup(&drm_cd, minor(_kdev))
+#elif defined(__OpenBSD__)
+#define drm_get_device_from_kdev(_kdev) device_lookup(&drm_cd, \
+    minor(_kdev)))->dv_cfdata->cf_driver->cd_devs[minor(_kdev)]
+#endif
 
+#if defined(__FreeBSD__)
 #define PAGE_ALIGN(addr) round_page(addr)
 /* DRM_SUSER returns true if the user is superuser */
 #if __FreeBSD_version >= 700000
-#define	DRM_SUSER(p)		(priv_check(p, PRIV_DRIVER) == 0)
+#define DRM_SUSER(p)		(priv_check(p, PRIV_DRIVER) == 0)
 #else
 #define DRM_SUSER(p)		(suser(p) == 0)
 #endif
@@ -251,17 +260,6 @@ enum {
 
 #else /* __FreeBSD__ */
 
-#if defined(__NetBSD__)
-#define DRM_DEVICE							\
-	drm_device_t *dev = device_lookup(&drm_cd, minor(kdev))
-#elif defined(__OpenBSD__)
-#define DRM_DEVICE							\
-	drm_device_t *dev = (device_lookup(&drm_cd,			\
-	    minor(kdev)))->dv_cfdata->cf_driver->cd_devs[minor(kdev)]
-#endif /* __OpenBSD__ */
-#define DRM_IOCTL_ARGS		dev_t kdev, u_long cmd, caddr_t data, \
-				int flags, DRM_STRUCTPROC *p, DRMFILE filp
-
 #define CDEV_MAJOR		34
 #define PAGE_ALIGN(addr)	(((addr) + PAGE_SIZE - 1) & PAGE_MASK)
 /* DRM_SUSER returns true if the user is superuser */
@@ -270,7 +268,7 @@ enum {
 #define DRM_MTRR_WC		MTRR_TYPE_WC
 #define jiffies			hardclock_ticks
 
-typedef drm_device_t *device_t;
+typedef struct drm_device *device_t;
 extern struct cfdriver drm_cd;
 #endif /* !__FreeBSD__ */
 
@@ -349,14 +347,6 @@ typedef vaddr_t vm_offset_t;
 	(!uvm_useracc((caddr_t)uaddr, size, VM_PROT_READ))
 #endif /* !__FreeBSD__ */
 
-#define DRM_COPY_TO_USER_IOCTL(user, kern, size)	\
-	if ( IOCPARM_LEN(cmd) != size)			\
-		return EINVAL;				\
-	*user = kern;
-#define DRM_COPY_FROM_USER_IOCTL(kern, user, size) \
-	if ( IOCPARM_LEN(cmd) != size)			\
-		return EINVAL;				\
-	kern = *user;
 #define DRM_COPY_TO_USER(user, kern, size) \
 	copyout(kern, user, size)
 #define DRM_COPY_FROM_USER(kern, user, size) \
@@ -376,7 +366,6 @@ typedef vaddr_t vm_offset_t;
 #define cpu_to_le32(x) htole32(x)
 #define le32_to_cpu(x) le32toh(x)
 
-#define DRM_ERR(v)		v
 #define DRM_HZ			hz
 #define DRM_UDELAY(udelay)	DELAY(udelay)
 #define DRM_TIME_SLICE		(hz/20)  /* Time slice for GLXContexts	  */
@@ -385,59 +374,35 @@ typedef vaddr_t vm_offset_t;
 	(_map) = (_dev)->context_sareas[_ctx];		\
 } while(0)
 
-#define DRM_GET_PRIV_WITH_RETURN(_priv, _filp)			\
-do {								\
-	if (_filp != (DRMFILE)(intptr_t)DRM_CURRENTPID) {	\
-		DRM_ERROR("filp doesn't match curproc\n");	\
-		return EINVAL;					\
-	}							\
-	_priv = drm_find_file_by_proc(dev, DRM_CURPROC);	\
-	if (_priv == NULL) {					\
-		DRM_ERROR("can't find authenticator\n");	\
-		return EINVAL;					\
-	}							\
-} while (0)
-
-#define LOCK_TEST_WITH_RETURN(dev, filp)				\
+#define LOCK_TEST_WITH_RETURN(dev, file_priv)				\
 do {									\
 	if (!_DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock) ||		\
-	     dev->lock.filp != filp) {					\
+	     dev->lock.file_priv != file_priv) {			\
 		DRM_ERROR("%s called without lock held\n",		\
 			   __FUNCTION__);				\
 		return EINVAL;						\
 	}								\
 } while (0)
 
-#define DRM_GETSAREA()					\
-do {								\
-	drm_local_map_t *map;					\
-	DRM_SPINLOCK_ASSERT(&dev->dev_lock);			\
-	TAILQ_FOREACH(map, &dev->maplist, link) {		\
-		if (map->type == _DRM_SHM &&			\
-			map->flags & _DRM_CONTAINS_LOCK) {	\
-			dev_priv->sarea = map;			\
-			break;					\
-		}						\
-	}							\
-} while (0)
-
 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
+/* Returns -errno to shared code */
 #define DRM_WAIT_ON( ret, queue, timeout, condition )		\
 for ( ret = 0 ; !ret && !(condition) ; ) {			\
 	DRM_UNLOCK();						\
 	mtx_lock(&dev->irq_lock);				\
 	if (!(condition))					\
-	   ret = msleep(&(queue), &dev->irq_lock, 		\
+	   ret = -mtx_sleep(&(queue), &dev->irq_lock, 		\
 			 PZERO | PCATCH, "drmwtq", (timeout));	\
 	mtx_unlock(&dev->irq_lock);				\
 	DRM_LOCK();						\
 }
 #else
+/* Returns -errno to shared code */
 #define DRM_WAIT_ON( ret, queue, timeout, condition )	\
 for ( ret = 0 ; !ret && !(condition) ; ) {		\
         int s = spldrm();				\
 	if (!(condition))				\
-	   ret = tsleep( &(queue), PZERO | PCATCH, 	\
+	   ret = -tsleep( &(queue), PZERO | PCATCH, 	\
 			 "drmwtq", (timeout) );		\
 	splx(s);					\
 }
@@ -467,9 +432,17 @@ typedef struct drm_pci_id_list
 #define DRM_MASTER	0x2
 #define DRM_ROOT_ONLY	0x4
 typedef struct drm_ioctl_desc {
-	int		     (*func)(DRM_IOCTL_ARGS);
+	unsigned long cmd;
+	int (*func)(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv);
 	int flags;
 } drm_ioctl_desc_t;
+/**
+ * Creates a driver or general drm_ioctl_desc array entry for the given
+ * ioctl, for use by drm_ioctl().
+ */
+#define DRM_IOCTL_DEF(ioctl, func, flags) \
+	[DRM_IOCTL_NR(ioctl)] = {ioctl, func, flags}
 
 typedef struct drm_magic_entry {
 	drm_magic_t	       magic;
@@ -492,7 +465,7 @@ typedef struct drm_buf {
 	unsigned long	  bus_address; /* Bus address of buffer		     */
 	struct drm_buf	  *next;       /* Kernel-only: used for free list    */
 	__volatile__ int  pending;     /* On hardware DMA queue		     */
-	DRMFILE		  filp;	       /* Unique identifier of holding process */
+	struct drm_file   *file_priv;  /* Unique identifier of holding process */
 	int		  context;     /* Kernel queue for this buffer	     */
 	enum {
 		DRM_LIST_NONE	 = 0,
@@ -554,12 +527,13 @@ struct drm_file {
 
 typedef struct drm_lock_data {
 	drm_hw_lock_t	  *hw_lock;	/* Hardware lock		   */
-	DRMFILE		  filp;	        /* Unique identifier of holding process (NULL is kernel)*/
+	struct drm_file   *file_priv;   /* Unique identifier of holding process (NULL is kernel)*/
 	int		  lock_queue;	/* Queue of blocked processes	   */
 	unsigned long	  lock_time;	/* Time of last lock in jiffies	   */
 } drm_lock_data_t;
 
-/* This structure, in the drm_device_t, is always initialized while the device
+/* This structure, in the struct drm_device, is always initialized while the
+ * device
  * is open.  dev->dma_lock protects the incrementing of dev->buf_use, which
  * when set marks that no further bufs may be allocated until device teardown
  * occurs (when the last open of the device has closed).  The high/low
@@ -637,28 +611,57 @@ typedef struct drm_vbl_sig {
 	int		pid;
 } drm_vbl_sig_t;
 
+struct drm_vblank_info {
+	wait_queue_head_t queue;	/* vblank wait queue */
+	atomic_t count;			/* number of VBLANK interrupts */
+					/* (driver must alloc the right number of counters) */
+	struct drm_vbl_sig_list sigs;	/* signal list to send on VBLANK */
+	atomic_t refcount;		/* number of users of vblank interrupts */
+	u32 last;			/* protected by dev->vbl_lock, used */
+					/* for wraparound handling */
+	int enabled;			/* so we don't call enable more than */
+					/* once per disable */
+	int inmodeset;			/* Display driver is setting mode */
+};
+
 /* location of GART table */
 #define DRM_ATI_GART_MAIN 1
 #define DRM_ATI_GART_FB   2
 
-typedef struct ati_pcigart_info {
+#define DRM_ATI_GART_PCI  1
+#define DRM_ATI_GART_PCIE 2
+#define DRM_ATI_GART_IGP  3
+
+struct drm_ati_pcigart_info {
 	int gart_table_location;
-	int is_pcie;
+	int gart_reg_if;
 	void *addr;
 	dma_addr_t bus_addr;
+	dma_addr_t table_mask;
+	dma_addr_t member_mask;
+	struct drm_dma_handle *table_handle;
 	drm_local_map_t mapping;
-} drm_ati_pcigart_info;
+	int table_size;
+};
+
+#ifndef DMA_BIT_MASK
+#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : (1ULL<<(n)) - 1)
+#endif
+
+#define upper_32_bits(n) ((u32)(((n) >> 16) >> 16))
 
 struct drm_driver_info {
 	int	(*load)(struct drm_device *, unsigned long flags);
 	int	(*firstopen)(struct drm_device *);
 	int	(*open)(struct drm_device *, drm_file_t *);
-	void	(*preclose)(struct drm_device *, void *filp);
+	void	(*preclose)(struct drm_device *, struct drm_file *file_priv);
 	void	(*postclose)(struct drm_device *, drm_file_t *);
 	void	(*lastclose)(struct drm_device *);
 	int	(*unload)(struct drm_device *);
-	void	(*reclaim_buffers_locked)(struct drm_device *, void *filp);
-	int	(*dma_ioctl)(DRM_IOCTL_ARGS);
+	void	(*reclaim_buffers_locked)(struct drm_device *,
+					  struct drm_file *file_priv);
+	int	(*dma_ioctl)(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
 	void	(*dma_ready)(struct drm_device *);
 	int	(*dma_quiescent)(struct drm_device *);
 	int	(*dma_flush_block_and_flush)(struct drm_device *, int context,
@@ -670,11 +673,13 @@ struct drm_driver_info {
 	int	(*kernel_context_switch)(struct drm_device *dev, int old,
 					 int new);
 	int	(*kernel_context_switch_unlock)(struct drm_device *dev);
-	void	(*irq_preinstall)(drm_device_t *dev);
-	void	(*irq_postinstall)(drm_device_t *dev);
-	void	(*irq_uninstall)(drm_device_t *dev);
+	void	(*irq_preinstall)(struct drm_device *dev);
+	int	(*irq_postinstall)(struct drm_device *dev);
+	void	(*irq_uninstall)(struct drm_device *dev);
 	void	(*irq_handler)(DRM_IRQ_ARGS);
-	int	(*vblank_wait)(drm_device_t *dev, unsigned int *sequence);
+	u32	(*get_vblank_counter)(struct drm_device *dev, int crtc);
+	int	(*enable_vblank)(struct drm_device *dev, int crtc);
+	void	(*disable_vblank)(struct drm_device *dev, int crtc);
 
 	drm_pci_id_list_t *id_entry;	/* PCI ID, name, and chipset private */
 
@@ -711,6 +716,7 @@ struct drm_driver_info {
 	unsigned use_dma_queue :1;
 	unsigned use_irq :1;
 	unsigned use_vbl_irq :1;
+	unsigned use_vbl_irq2 :1;
 	unsigned use_mtrr :1;
 };
 
@@ -743,10 +749,14 @@ struct drm_device {
 
 				/* Locks */
 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
+	struct mtx	  vbl_lock;	/* protects vblank operations */
 	struct mtx	  dma_lock;	/* protects dev->dma */
 	struct mtx	  irq_lock;	/* protects irq condition checks */
 	struct mtx	  dev_lock;	/* protects everything else */
 #endif
+	DRM_SPINTYPE	  drw_lock;
+	DRM_SPINTYPE	  tsk_lock;
+
 				/* Usage Counters */
 	int		  open_count;	/* Outstanding files open	   */
 	int		  buf_use;	/* Buffers in use -- cannot alloc  */
@@ -793,8 +803,13 @@ struct drm_device {
 
 	atomic_t	  context_flag;	/* Context swapping flag	   */
 	int		  last_context;	/* Last current context		   */
-   	int		  vbl_queue;	/* vbl wait channel */
-   	atomic_t          vbl_received;
+
+	int		  vblank_disable_allowed;
+	atomic_t 	  vbl_signal_pending;	/* number of signals pending on all crtcs */
+	struct callout	  vblank_disable_timer;
+	u32		  max_vblank_count;	/* size of vblank counter register */
+	struct drm_vblank_info *vblank;		/* per crtc vblank info */
+	int		  num_crtcs;
 
 #ifdef __FreeBSD__
 	struct sigio      *buf_sigio;	/* Processes waiting for SIGIO     */
@@ -811,6 +826,13 @@ struct drm_device {
 	void		  *dev_private;
 	unsigned int	  agp_buffer_token;
 	drm_local_map_t   *agp_buffer_map;
+
+	struct unrhdr	  *drw_unrhdr;
+	/* RB tree of drawable infos */
+	RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head;
+
+	struct task	  locked_task;
+	void		  (*locked_task_call)(struct drm_device *dev);
 };
 
 extern int	drm_debug_flag;
@@ -836,17 +858,20 @@ dev_type_read(drm_read);
 dev_type_poll(drm_poll);
 dev_type_mmap(drm_mmap);
 #endif
+extern drm_local_map_t	*drm_getsarea(struct drm_device *dev);
 
 /* File operations helpers (drm_fops.c) */
 #ifdef __FreeBSD__
-extern int		drm_open_helper(struct cdev *kdev, int flags, int fmt, 
-					 DRM_STRUCTPROC *p, drm_device_t *dev);
-extern drm_file_t	*drm_find_file_by_proc(drm_device_t *dev, 
-					 DRM_STRUCTPROC *p);
+extern int		drm_open_helper(struct cdev *kdev, int flags, int fmt,
+					 DRM_STRUCTPROC *p,
+					struct drm_device *dev);
+extern drm_file_t	*drm_find_file_by_proc(struct drm_device *dev,
+					       DRM_STRUCTPROC *p);
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
-extern int		drm_open_helper(dev_t kdev, int flags, int fmt, 
-					DRM_STRUCTPROC *p, drm_device_t *dev);
-extern drm_file_t	*drm_find_file_by_proc(drm_device_t *dev, 
+extern int		drm_open_helper(dev_t kdev, int flags, int fmt,
+					DRM_STRUCTPROC *p,
+					struct drm_device *dev);
+extern drm_file_t	*drm_find_file_by_proc(struct drm_device *dev,
 					       DRM_STRUCTPROC *p);
 #endif /* __NetBSD__ || __OpenBSD__ */
 
@@ -858,171 +883,238 @@ void	*drm_calloc(size_t nmemb, size_t size, int area);
 void	*drm_realloc(void *oldpt, size_t oldsize, size_t size,
 				   int area);
 void	drm_free(void *pt, size_t size, int area);
-void	*drm_ioremap(drm_device_t *dev, drm_local_map_t *map);
+void	*drm_ioremap(struct drm_device *dev, drm_local_map_t *map);
 void	drm_ioremapfree(drm_local_map_t *map);
 int	drm_mtrr_add(unsigned long offset, size_t size, int flags);
 int	drm_mtrr_del(int handle, unsigned long offset, size_t size, int flags);
 
-int	drm_context_switch(drm_device_t *dev, int old, int new);
-int	drm_context_switch_complete(drm_device_t *dev, int new);
+int	drm_context_switch(struct drm_device *dev, int old, int new);
+int	drm_context_switch_complete(struct drm_device *dev, int new);
 
-int	drm_ctxbitmap_init(drm_device_t *dev);
-void	drm_ctxbitmap_cleanup(drm_device_t *dev);
-void	drm_ctxbitmap_free(drm_device_t *dev, int ctx_handle);
-int	drm_ctxbitmap_next(drm_device_t *dev);
+int	drm_ctxbitmap_init(struct drm_device *dev);
+void	drm_ctxbitmap_cleanup(struct drm_device *dev);
+void	drm_ctxbitmap_free(struct drm_device *dev, int ctx_handle);
+int	drm_ctxbitmap_next(struct drm_device *dev);
 
 /* Locking IOCTL support (drm_lock.c) */
 int	drm_lock_take(__volatile__ unsigned int *lock,
 				    unsigned int context);
-int	drm_lock_transfer(drm_device_t *dev,
-					__volatile__ unsigned int *lock,
-					unsigned int context);
-int	drm_lock_free(drm_device_t *dev,
-				    __volatile__ unsigned int *lock,
-				    unsigned int context);
+int	drm_lock_transfer(struct drm_device *dev,
+			  __volatile__ unsigned int *lock,
+			  unsigned int context);
+int	drm_lock_free(struct drm_device *dev,
+		      __volatile__ unsigned int *lock,
+		      unsigned int context);
 
 /* Buffer management support (drm_bufs.c) */
-unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource);
-unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource);
-void	drm_rmmap(drm_device_t *dev, drm_local_map_t *map);
+unsigned long drm_get_resource_start(struct drm_device *dev,
+				     unsigned int resource);
+unsigned long drm_get_resource_len(struct drm_device *dev,
+				   unsigned int resource);
+void	drm_rmmap(struct drm_device *dev, drm_local_map_t *map);
 int	drm_order(unsigned long size);
-int	drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
+int	drm_addmap(struct drm_device *dev, unsigned long offset,
+		   unsigned long size,
 		   drm_map_type_t type, drm_map_flags_t flags,
 		   drm_local_map_t **map_ptr);
-int	drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request);
-int	drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request);
-int	drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request);
+int	drm_addbufs_pci(struct drm_device *dev, drm_buf_desc_t *request);
+int	drm_addbufs_sg(struct drm_device *dev, drm_buf_desc_t *request);
+int	drm_addbufs_agp(struct drm_device *dev, drm_buf_desc_t *request);
 
 /* DMA support (drm_dma.c) */
-int	drm_dma_setup(drm_device_t *dev);
-void	drm_dma_takedown(drm_device_t *dev);
-void	drm_free_buffer(drm_device_t *dev, drm_buf_t *buf);
-void	drm_reclaim_buffers(drm_device_t *dev, DRMFILE filp);
+int	drm_dma_setup(struct drm_device *dev);
+void	drm_dma_takedown(struct drm_device *dev);
+void	drm_free_buffer(struct drm_device *dev, drm_buf_t *buf);
+void	drm_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv);
 #define drm_core_reclaim_buffers drm_reclaim_buffers
 
 /* IRQ support (drm_irq.c) */
-int	drm_irq_install(drm_device_t *dev);
-int	drm_irq_uninstall(drm_device_t *dev);
+int	drm_irq_install(struct drm_device *dev);
+int	drm_irq_uninstall(struct drm_device *dev);
 irqreturn_t drm_irq_handler(DRM_IRQ_ARGS);
-void	drm_driver_irq_preinstall(drm_device_t *dev);
-void	drm_driver_irq_postinstall(drm_device_t *dev);
-void	drm_driver_irq_uninstall(drm_device_t *dev);
-int	drm_vblank_wait(drm_device_t *dev, unsigned int *vbl_seq);
-void	drm_vbl_send_signals(drm_device_t *dev);
+void	drm_driver_irq_preinstall(struct drm_device *dev);
+void	drm_driver_irq_postinstall(struct drm_device *dev);
+void	drm_driver_irq_uninstall(struct drm_device *dev);
+void	drm_handle_vblank(struct drm_device *dev, int crtc);
+u32	drm_vblank_count(struct drm_device *dev, int crtc);
+int	drm_vblank_get(struct drm_device *dev, int crtc);
+void	drm_vblank_put(struct drm_device *dev, int crtc);
+int	drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq);
+int	drm_vblank_init(struct drm_device *dev, int num_crtcs);
+void	drm_vbl_send_signals(struct drm_device *dev, int crtc);
+int 	drm_modeset_ctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
 
 /* AGP/PCI Express/GART support (drm_agpsupport.c) */
-int	drm_device_is_agp(drm_device_t *dev);
-int	drm_device_is_pcie(drm_device_t *dev);
+int	drm_device_is_agp(struct drm_device *dev);
+int	drm_device_is_pcie(struct drm_device *dev);
 drm_agp_head_t *drm_agp_init(void);
-int	drm_agp_acquire(drm_device_t *dev);
-int	drm_agp_release(drm_device_t *dev);
-int	drm_agp_info(drm_device_t * dev, drm_agp_info_t *info);
-int	drm_agp_enable(drm_device_t *dev, drm_agp_mode_t mode);
+int	drm_agp_acquire(struct drm_device *dev);
+int	drm_agp_release(struct drm_device *dev);
+int	drm_agp_info(struct drm_device * dev, drm_agp_info_t *info);
+int	drm_agp_enable(struct drm_device *dev, drm_agp_mode_t mode);
 void	*drm_agp_allocate_memory(size_t pages, u32 type);
 int	drm_agp_free_memory(void *handle);
 int	drm_agp_bind_memory(void *handle, off_t start);
 int	drm_agp_unbind_memory(void *handle);
-int	drm_agp_alloc(drm_device_t *dev, drm_agp_buffer_t *request);
-int	drm_agp_free(drm_device_t *dev, drm_agp_buffer_t *request);
-int	drm_agp_bind(drm_device_t *dev, drm_agp_binding_t *request);
-int	drm_agp_unbind(drm_device_t *dev, drm_agp_binding_t *request);
+int	drm_agp_alloc(struct drm_device *dev, drm_agp_buffer_t *request);
+int	drm_agp_free(struct drm_device *dev, drm_agp_buffer_t *request);
+int	drm_agp_bind(struct drm_device *dev, drm_agp_binding_t *request);
+int	drm_agp_unbind(struct drm_device *dev, drm_agp_binding_t *request);
 
 /* Scatter Gather Support (drm_scatter.c) */
 void	drm_sg_cleanup(drm_sg_mem_t *entry);
+int	drm_sg_alloc(struct drm_device *dev, drm_scatter_gather_t * request);
 
 #ifdef __FreeBSD__
 /* sysctl support (drm_sysctl.h) */
-extern int		drm_sysctl_init(drm_device_t *dev);
-extern int		drm_sysctl_cleanup(drm_device_t *dev);
+extern int		drm_sysctl_init(struct drm_device *dev);
+extern int		drm_sysctl_cleanup(struct drm_device *dev);
 #endif /* __FreeBSD__ */
 
 /* ATI PCIGART support (ati_pcigart.c) */
-int	drm_ati_pcigart_init(drm_device_t *dev,
-			     drm_ati_pcigart_info *gart_info);
-int	drm_ati_pcigart_cleanup(drm_device_t *dev,
-				drm_ati_pcigart_info *gart_info);
+int	drm_ati_pcigart_init(struct drm_device *dev,
+				struct drm_ati_pcigart_info *gart_info);
+int	drm_ati_pcigart_cleanup(struct drm_device *dev,
+				struct drm_ati_pcigart_info *gart_info);
 
 /* Locking IOCTL support (drm_drv.c) */
-int	drm_lock(DRM_IOCTL_ARGS);
-int	drm_unlock(DRM_IOCTL_ARGS);
-int	drm_version(DRM_IOCTL_ARGS);
-int	drm_setversion(DRM_IOCTL_ARGS);
+int	drm_lock(struct drm_device *dev, void *data,
+		 struct drm_file *file_priv);
+int	drm_unlock(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_version(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv);
+int	drm_setversion(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv);
 
 /* Misc. IOCTL support (drm_ioctl.c) */
-int	drm_irq_by_busid(DRM_IOCTL_ARGS);
-int	drm_getunique(DRM_IOCTL_ARGS);
-int	drm_setunique(DRM_IOCTL_ARGS);
-int	drm_getmap(DRM_IOCTL_ARGS);
-int	drm_getclient(DRM_IOCTL_ARGS);
-int	drm_getstats(DRM_IOCTL_ARGS);
-int	drm_noop(DRM_IOCTL_ARGS);
+int	drm_irq_by_busid(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+int	drm_getunique(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int	drm_setunique(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int	drm_getmap(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_getclient(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int	drm_getstats(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv);
+int	drm_noop(struct drm_device *dev, void *data,
+		 struct drm_file *file_priv);
 
 /* Context IOCTL support (drm_context.c) */
-int	drm_resctx(DRM_IOCTL_ARGS);
-int	drm_addctx(DRM_IOCTL_ARGS);
-int	drm_modctx(DRM_IOCTL_ARGS);
-int	drm_getctx(DRM_IOCTL_ARGS);
-int	drm_switchctx(DRM_IOCTL_ARGS);
-int	drm_newctx(DRM_IOCTL_ARGS);
-int	drm_rmctx(DRM_IOCTL_ARGS);
-int	drm_setsareactx(DRM_IOCTL_ARGS);
-int	drm_getsareactx(DRM_IOCTL_ARGS);
+int	drm_resctx(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_addctx(struct drm_device *dev, void *data,
+struct drm_file *file_priv);
+int	drm_modctx(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_getctx(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_switchctx(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
+int	drm_newctx(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_rmctx(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv);
+int	drm_setsareactx(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int	drm_getsareactx(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
 
 /* Drawable IOCTL support (drm_drawable.c) */
-int	drm_adddraw(DRM_IOCTL_ARGS);
-int	drm_rmdraw(DRM_IOCTL_ARGS);
+int	drm_adddraw(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv);
+int	drm_rmdraw(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv);
+int	drm_update_draw(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+struct drm_drawable_info *drm_get_drawable_info(struct drm_device *dev,
+						int handle);
+
+/* Drawable support (drm_drawable.c) */
+void drm_drawable_free_all(struct drm_device *dev);
 
 /* Authentication IOCTL support (drm_auth.c) */
-int	drm_getmagic(DRM_IOCTL_ARGS);
-int	drm_authmagic(DRM_IOCTL_ARGS);
+int	drm_getmagic(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv);
+int	drm_authmagic(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
 
 /* Buffer management support (drm_bufs.c) */
-int	drm_addmap_ioctl(DRM_IOCTL_ARGS);
-int	drm_rmmap_ioctl(DRM_IOCTL_ARGS);
-int	drm_addbufs_ioctl(DRM_IOCTL_ARGS);
-int	drm_infobufs(DRM_IOCTL_ARGS);
-int	drm_markbufs(DRM_IOCTL_ARGS);
-int	drm_freebufs(DRM_IOCTL_ARGS);
-int	drm_mapbufs(DRM_IOCTL_ARGS);
+int	drm_addmap_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+int	drm_rmmap_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+int	drm_addbufs_ioctl(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv);
+int	drm_infobufs(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv);
+int	drm_markbufs(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv);
+int	drm_freebufs(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv);
+int	drm_mapbufs(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv);
 
 /* DMA support (drm_dma.c) */
-int	drm_dma(DRM_IOCTL_ARGS);
+int	drm_dma(struct drm_device *dev, void *data, struct drm_file *file_priv);
 
 /* IRQ support (drm_irq.c) */
-int	drm_control(DRM_IOCTL_ARGS);
-int	drm_wait_vblank(DRM_IOCTL_ARGS);
+int	drm_control(struct drm_device *dev, void *data, struct drm_file *file_priv);
+int	drm_wait_vblank(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+void	drm_locked_tasklet(struct drm_device *dev,
+			   void (*tasklet)(struct drm_device *dev));
 
 /* AGP/GART support (drm_agpsupport.c) */
-int	drm_agp_acquire_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_release_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_enable_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_info_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_alloc_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_free_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_unbind_ioctl(DRM_IOCTL_ARGS);
-int	drm_agp_bind_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_acquire_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+int	drm_agp_release_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+int	drm_agp_enable_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
+int	drm_agp_info_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+int	drm_agp_alloc_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+int	drm_agp_free_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+int	drm_agp_unbind_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
+int	drm_agp_bind_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
 
 /* Scatter Gather Support (drm_scatter.c) */
-int	drm_sg_alloc(DRM_IOCTL_ARGS);
-int	drm_sg_free(DRM_IOCTL_ARGS);
+int	drm_sg_alloc_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+int	drm_sg_free(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv);
 
 /* consistent PCI memory functions (drm_pci.c) */
-drm_dma_handle_t *drm_pci_alloc(drm_device_t *dev, size_t size, size_t align,
-				dma_addr_t maxaddr);
-void	drm_pci_free(drm_device_t *dev, drm_dma_handle_t *dmah);
+drm_dma_handle_t *drm_pci_alloc(struct drm_device *dev, size_t size,
+				size_t align, dma_addr_t maxaddr);
+void	drm_pci_free(struct drm_device *dev, drm_dma_handle_t *dmah);
+
+#define drm_core_ioremap_wc drm_core_ioremap
 
 /* Inline replacements for DRM_IOREMAP macros */
-static __inline__ void drm_core_ioremap(struct drm_local_map *map, struct drm_device *dev)
+static __inline__ void
+drm_core_ioremap(struct drm_local_map *map, struct drm_device *dev)
 {
 	map->handle = drm_ioremap(dev, map);
 }
-static __inline__ void drm_core_ioremapfree(struct drm_local_map *map, struct drm_device *dev)
+static __inline__ void
+drm_core_ioremapfree(struct drm_local_map *map, struct drm_device *dev)
 {
 	if ( map->handle && map->size )
 		drm_ioremapfree(map);
 }
 
-static __inline__ struct drm_local_map *drm_core_findmap(struct drm_device *dev, unsigned long offset)
+static __inline__ struct drm_local_map *
+drm_core_findmap(struct drm_device *dev, unsigned long offset)
 {
 	drm_local_map_t *map;
 
diff --git a/sys/dev/drm/drm_agpsupport.c b/sys/dev/drm/drm_agpsupport.c
index 104f6b6..1614c21 100644
--- a/sys/dev/drm/drm_agpsupport.c
+++ b/sys/dev/drm/drm_agpsupport.c
@@ -1,6 +1,3 @@
-/* drm_agpsupport.h -- DRM support for AGP/GART backend -*- linux-c -*-
- * Created: Mon Dec 13 09:56:45 1999 by faith@precisioninsight.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,20 +31,25 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_agpsupport.c
+ * Support code for tying the kernel AGP support to DRM drivers and
+ * the DRM's AGP ioctls.
+ */
+
 #include "dev/drm/drmP.h"
 
 #ifdef __FreeBSD__
 #if __FreeBSD_version >= 800004
 #include <dev/agp/agpreg.h>
-#else
+#else /* __FreeBSD_version >= 800004 */
 #include <pci/agpreg.h>
-#endif
+#endif /* __FreeBSD_version >= 800004 */
 #include <dev/pci/pcireg.h>
 #endif
 
 /* Returns 1 if AGP or 0 if not. */
 static int
-drm_device_find_capability(drm_device_t *dev, int cap)
+drm_device_find_capability(struct drm_device *dev, int cap)
 {
 #ifdef __FreeBSD__
 #if __FreeBSD_version >= 602102
@@ -89,7 +91,7 @@ drm_device_find_capability(drm_device_t *dev, int cap)
 #endif
 }
 
-int drm_device_is_agp(drm_device_t *dev)
+int drm_device_is_agp(struct drm_device *dev)
 {
 	if (dev->driver.device_is_agp != NULL) {
 		int ret;
@@ -105,12 +107,12 @@ int drm_device_is_agp(drm_device_t *dev)
 	return (drm_device_find_capability(dev, PCIY_AGP));
 }
 
-int drm_device_is_pcie(drm_device_t *dev)
+int drm_device_is_pcie(struct drm_device *dev)
 {
 	return (drm_device_find_capability(dev, PCIY_EXPRESS));
 }
 
-int drm_agp_info(drm_device_t * dev, drm_agp_info_t *info)
+int drm_agp_info(struct drm_device * dev, drm_agp_info_t *info)
 {
 	struct agp_info *kern;
 
@@ -132,11 +134,11 @@ int drm_agp_info(drm_device_t * dev, drm_agp_info_t *info)
 	return 0;
 }
 
-int drm_agp_info_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_info_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
 	int err;
 	drm_agp_info_t info;
-	DRM_DEVICE;
 
 	err = drm_agp_info(dev, &info);
 	if (err != 0)
@@ -146,14 +148,14 @@ int drm_agp_info_ioctl(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-int drm_agp_acquire_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_acquire_ioctl(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 
 	return drm_agp_acquire(dev);
 }
 
-int drm_agp_acquire(drm_device_t *dev)
+int drm_agp_acquire(struct drm_device *dev)
 {
 	int retcode;
 
@@ -168,14 +170,14 @@ int drm_agp_acquire(drm_device_t *dev)
 	return 0;
 }
 
-int drm_agp_release_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_release_ioctl(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 
 	return drm_agp_release(dev);
 }
 
-int drm_agp_release(drm_device_t * dev)
+int drm_agp_release(struct drm_device * dev)
 {
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
@@ -184,7 +186,7 @@ int drm_agp_release(drm_device_t * dev)
 	return 0;
 }
 
-int drm_agp_enable(drm_device_t *dev, drm_agp_mode_t mode)
+int drm_agp_enable(struct drm_device *dev, drm_agp_mode_t mode)
 {
 
 	if (!dev->agp || !dev->agp->acquired)
@@ -192,22 +194,21 @@ int drm_agp_enable(drm_device_t *dev, drm_agp_mode_t mode)
 	
 	dev->agp->mode    = mode.mode;
 	agp_enable(dev->agp->agpdev, mode.mode);
-	dev->agp->base    = dev->agp->info.ai_aperture_base;
 	dev->agp->enabled = 1;
 	return 0;
 }
 
-int drm_agp_enable_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_enable_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
 	drm_agp_mode_t mode;
-	DRM_DEVICE;
 
 	mode = *(drm_agp_mode_t *) data;
 
 	return drm_agp_enable(dev, mode);
 }
 
-int drm_agp_alloc(drm_device_t *dev, drm_agp_buffer_t *request)
+int drm_agp_alloc(struct drm_device *dev, drm_agp_buffer_t *request)
 {
 	drm_agp_mem_t    *entry;
 	void	         *handle;
@@ -250,9 +251,9 @@ int drm_agp_alloc(drm_device_t *dev, drm_agp_buffer_t *request)
 	return 0;
 }
 
-int drm_agp_alloc_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_alloc_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_agp_buffer_t request;
 	int retcode;
 
@@ -267,7 +268,8 @@ int drm_agp_alloc_ioctl(DRM_IOCTL_ARGS)
 	return retcode;
 }
 
-static drm_agp_mem_t * drm_agp_lookup_entry(drm_device_t *dev, void *handle)
+static drm_agp_mem_t * drm_agp_lookup_entry(struct drm_device *dev,
+					    void *handle)
 {
 	drm_agp_mem_t *entry;
 
@@ -277,7 +279,7 @@ static drm_agp_mem_t * drm_agp_lookup_entry(drm_device_t *dev, void *handle)
 	return NULL;
 }
 
-int drm_agp_unbind(drm_device_t *dev, drm_agp_binding_t *request)
+int drm_agp_unbind(struct drm_device *dev, drm_agp_binding_t *request)
 {
 	drm_agp_mem_t     *entry;
 	int retcode;
@@ -299,9 +301,9 @@ int drm_agp_unbind(drm_device_t *dev, drm_agp_binding_t *request)
 	return retcode;
 }
 
-int drm_agp_unbind_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_unbind_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_agp_binding_t request;
 	int retcode;
 
@@ -314,7 +316,7 @@ int drm_agp_unbind_ioctl(DRM_IOCTL_ARGS)
 	return retcode;
 }
 
-int drm_agp_bind(drm_device_t *dev, drm_agp_binding_t *request)
+int drm_agp_bind(struct drm_device *dev, drm_agp_binding_t *request)
 {
 	drm_agp_mem_t     *entry;
 	int               retcode;
@@ -340,9 +342,9 @@ int drm_agp_bind(drm_device_t *dev, drm_agp_binding_t *request)
 	return retcode;
 }
 
-int drm_agp_bind_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_bind_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_agp_binding_t request;
 	int retcode;
 
@@ -355,7 +357,7 @@ int drm_agp_bind_ioctl(DRM_IOCTL_ARGS)
 	return retcode;
 }
 
-int drm_agp_free(drm_device_t *dev, drm_agp_buffer_t *request)
+int drm_agp_free(struct drm_device *dev, drm_agp_buffer_t *request)
 {
 	drm_agp_mem_t    *entry;
 	
@@ -385,9 +387,9 @@ int drm_agp_free(drm_device_t *dev, drm_agp_buffer_t *request)
 
 }
 
-int drm_agp_free_ioctl(DRM_IOCTL_ARGS)
+int drm_agp_free_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_agp_buffer_t request;
 	int retcode;
 
@@ -418,6 +420,7 @@ drm_agp_head_t *drm_agp_init(void)
 			return NULL;
 		head->agpdev = agpdev;
 		agp_get_info(agpdev, &head->info);
+		head->base = head->info.ai_aperture_base;
 		head->memory = NULL;
 		DRM_INFO("AGP at 0x%08lx %dMB\n",
 			 (long)head->info.ai_aperture_base,
diff --git a/sys/dev/drm/drm_auth.c b/sys/dev/drm/drm_auth.c
index 03b3a47..0fe93b8 100644
--- a/sys/dev/drm/drm_auth.c
+++ b/sys/dev/drm/drm_auth.c
@@ -1,6 +1,3 @@
-/* drm_auth.h -- IOCTLs for authentication -*- linux-c -*-
- * Created: Tue Feb  2 08:37:54 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,11 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_auth.c
+ * Implementation of the get/authmagic ioctls implementing the authentication
+ * scheme between the master and clients.
+ */
+
 #include "dev/drm/drmP.h"
 
 static int drm_hash_magic(drm_magic_t magic)
@@ -41,40 +43,46 @@ static int drm_hash_magic(drm_magic_t magic)
 	return magic & (DRM_HASH_SIZE-1);
 }
 
-static drm_file_t *drm_find_file(drm_device_t *dev, drm_magic_t magic)
+/**
+ * Returns the file private associated with the given magic number.
+ */
+static drm_file_t *drm_find_file(struct drm_device *dev, drm_magic_t magic)
 {
-	drm_file_t	  *retval = NULL;
 	drm_magic_entry_t *pt;
-	int		  hash;
+	int hash = drm_hash_magic(magic);
 
-	hash = drm_hash_magic(magic);
+	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
 
-	DRM_LOCK();
 	for (pt = dev->magiclist[hash].head; pt; pt = pt->next) {
 		if (pt->magic == magic) {
-			retval = pt->priv;
-			break;
+			return pt->priv;
 		}
 	}
-	DRM_UNLOCK();
-	return retval;
+
+	return NULL;
 }
 
-static int drm_add_magic(drm_device_t *dev, drm_file_t *priv, drm_magic_t magic)
+/**
+ * Inserts the given magic number into the hash table of used magic number
+ * lists.
+ */
+static int drm_add_magic(struct drm_device *dev, drm_file_t *priv,
+			 drm_magic_t magic)
 {
 	int		  hash;
 	drm_magic_entry_t *entry;
 
 	DRM_DEBUG("%d\n", magic);
 
+	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
+
 	hash = drm_hash_magic(magic);
 	entry = malloc(sizeof(*entry), M_DRM, M_ZERO | M_NOWAIT);
-	if (!entry) return DRM_ERR(ENOMEM);
+	if (!entry) return ENOMEM;
 	entry->magic = magic;
 	entry->priv  = priv;
 	entry->next  = NULL;
 
-	DRM_LOCK();
 	if (dev->magiclist[hash].tail) {
 		dev->magiclist[hash].tail->next = entry;
 		dev->magiclist[hash].tail	= entry;
@@ -82,21 +90,25 @@ static int drm_add_magic(drm_device_t *dev, drm_file_t *priv, drm_magic_t magic)
 		dev->magiclist[hash].head	= entry;
 		dev->magiclist[hash].tail	= entry;
 	}
-	DRM_UNLOCK();
 
 	return 0;
 }
 
-static int drm_remove_magic(drm_device_t *dev, drm_magic_t magic)
+/**
+ * Removes the given magic number from the hash table of used magic number
+ * lists.
+ */
+static int drm_remove_magic(struct drm_device *dev, drm_magic_t magic)
 {
 	drm_magic_entry_t *prev = NULL;
 	drm_magic_entry_t *pt;
 	int		  hash;
 
+	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
+
 	DRM_DEBUG("%d\n", magic);
 	hash = drm_hash_magic(magic);
 
-	DRM_LOCK();
 	for (pt = dev->magiclist[hash].head; pt; prev = pt, pt = pt->next) {
 		if (pt->magic == magic) {
 			if (dev->magiclist[hash].head == pt) {
@@ -108,68 +120,70 @@ static int drm_remove_magic(drm_device_t *dev, drm_magic_t magic)
 			if (prev) {
 				prev->next = pt->next;
 			}
-			DRM_UNLOCK();
+			free(pt, M_DRM);
 			return 0;
 		}
 	}
-	DRM_UNLOCK();
 
-	free(pt, M_DRM);
-	return DRM_ERR(EINVAL);
+	return EINVAL;
 }
 
-int drm_getmagic(DRM_IOCTL_ARGS)
+/**
+ * Called by the client, this returns a unique magic number to be authorized
+ * by the master.
+ *
+ * The master may use its own knowledge of the client (such as the X
+ * connection that the magic is passed over) to determine if the magic number
+ * should be authenticated.
+ */
+int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	static drm_magic_t sequence = 0;
-	drm_auth_t auth;
-	drm_file_t *priv;
-
-	DRM_LOCK();
-	priv = drm_find_file_by_proc(dev, p);
-	DRM_UNLOCK();
-	if (priv == NULL) {
-		DRM_ERROR("can't find authenticator\n");
-		return EINVAL;
-	}
+	drm_auth_t *auth = data;
 
 				/* Find unique magic */
-	if (priv->magic) {
-		auth.magic = priv->magic;
+	if (file_priv->magic) {
+		auth->magic = file_priv->magic;
 	} else {
+		DRM_LOCK();
 		do {
 			int old = sequence;
-			
-			auth.magic = old+1;
-			
-			if (!atomic_cmpset_int(&sequence, old, auth.magic))
+
+			auth->magic = old+1;
+
+			if (!atomic_cmpset_int(&sequence, old, auth->magic))
 				continue;
-		} while (drm_find_file(dev, auth.magic));
-		priv->magic = auth.magic;
-		drm_add_magic(dev, priv, auth.magic);
+		} while (drm_find_file(dev, auth->magic));
+		file_priv->magic = auth->magic;
+		drm_add_magic(dev, file_priv, auth->magic);
+		DRM_UNLOCK();
 	}
 
-	DRM_DEBUG("%u\n", auth.magic);
-
-	DRM_COPY_TO_USER_IOCTL((drm_auth_t *)data, auth, sizeof(auth));
+	DRM_DEBUG("%u\n", auth->magic);
 
 	return 0;
 }
 
-int drm_authmagic(DRM_IOCTL_ARGS)
+/**
+ * Marks the client associated with the given magic number as authenticated.
+ */
+int drm_authmagic(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	drm_auth_t	   auth;
-	drm_file_t	   *file;
-	DRM_DEVICE;
-
-	DRM_COPY_FROM_USER_IOCTL(auth, (drm_auth_t *)data, sizeof(auth));
+	drm_auth_t *auth = data;
+	drm_file_t *priv;
 
-	DRM_DEBUG("%u\n", auth.magic);
+	DRM_DEBUG("%u\n", auth->magic);
 
-	if ((file = drm_find_file(dev, auth.magic))) {
-		file->authenticated = 1;
-		drm_remove_magic(dev, auth.magic);
+	DRM_LOCK();
+	priv = drm_find_file(dev, auth->magic);
+	if (priv != NULL) {
+		priv->authenticated = 1;
+		drm_remove_magic(dev, auth->magic);
+		DRM_UNLOCK();
 		return 0;
+	} else {
+		DRM_UNLOCK();
+		return EINVAL;
 	}
-	return DRM_ERR(EINVAL);
 }
diff --git a/sys/dev/drm/drm_bufs.c b/sys/dev/drm/drm_bufs.c
index c8ef0e4..9ad13d7 100644
--- a/sys/dev/drm/drm_bufs.c
+++ b/sys/dev/drm/drm_bufs.c
@@ -1,6 +1,3 @@
-/* drm_bufs.h -- Generic buffer template -*- linux-c -*-
- * Created: Thu Nov 23 03:10:50 2000 by gareth@valinux.com
- */
 /*-
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,10 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_bufs.c
+ * Implementation of the ioctls for setup of DRM mappings and DMA buffers.
+ */
+
 #include "dev/pci/pcireg.h"
 
 #include "dev/drm/drmP.h"
@@ -58,7 +59,7 @@ int drm_order(unsigned long size)
  * drm_get_resource_*.  Note that they are not RF_ACTIVE, so there's no virtual
  * address for accessing them.  Cleaned up at unload.
  */
-static int drm_alloc_resource(drm_device_t *dev, int resource)
+static int drm_alloc_resource(struct drm_device *dev, int resource)
 {
 	if (resource >= DRM_MAX_PCI_RESOURCE) {
 		DRM_ERROR("Resource %d too large\n", resource);
@@ -84,7 +85,8 @@ static int drm_alloc_resource(drm_device_t *dev, int resource)
 	return 0;
 }
 
-unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource)
+unsigned long drm_get_resource_start(struct drm_device *dev,
+				     unsigned int resource)
 {
 	if (drm_alloc_resource(dev, resource) != 0)
 		return 0;
@@ -92,7 +94,8 @@ unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource)
 	return rman_get_start(dev->pcir[resource]);
 }
 
-unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource)
+unsigned long drm_get_resource_len(struct drm_device *dev,
+				   unsigned int resource)
 {
 	if (drm_alloc_resource(dev, resource) != 0)
 		return 0;
@@ -100,7 +103,8 @@ unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource)
 	return rman_get_size(dev->pcir[resource]);
 }
 
-int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
+int drm_addmap(struct drm_device * dev, unsigned long offset,
+	       unsigned long size,
     drm_map_type_t type, drm_map_flags_t flags, drm_local_map_t **map_ptr)
 {
 	drm_local_map_t *map;
@@ -151,8 +155,10 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 	 * initialization necessary.
 	 */
 	map = malloc(sizeof(*map), M_DRM, M_ZERO | M_NOWAIT);
-	if ( !map )
-		return DRM_ERR(ENOMEM);
+	if ( !map ) {
+		DRM_LOCK();
+		return ENOMEM;
+	}
 
 	map->offset = offset;
 	map->size = size;
@@ -175,7 +181,8 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 			   map->size, drm_order(map->size), map->handle );
 		if ( !map->handle ) {
 			free(map, M_DRM);
-			return DRM_ERR(ENOMEM);
+			DRM_LOCK();
+			return ENOMEM;
 		}
 		map->offset = (unsigned long)map->handle;
 		if ( map->flags & _DRM_CONTAINS_LOCK ) {
@@ -185,7 +192,7 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 				DRM_UNLOCK();
 				free(map->handle, M_DRM);
 				free(map, M_DRM);
-				return DRM_ERR(EBUSY);
+				return EBUSY;
 			}
 			dev->lock.hw_lock = map->handle; /* Pointer to lock */
 			DRM_UNLOCK();
@@ -193,7 +200,17 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 		break;
 	case _DRM_AGP:
 		/*valid = 0;*/
-		map->offset += dev->agp->base;
+		/* In some cases (i810 driver), user space may have already
+		 * added the AGP base itself, because dev->agp->base previously
+		 * only got set during AGP enable.  So, only add the base
+		 * address if the map's offset isn't already within the
+		 * aperture.
+		 */
+		if (map->offset < dev->agp->base ||
+		    map->offset > dev->agp->base +
+		    dev->agp->info.ai_aperture_size - 1) {
+			map->offset += dev->agp->base;
+		}
 		map->mtrr   = dev->agp->mtrr; /* for getmap */
 		/*for (entry = dev->agp->memory; entry; entry = entry->next) {
 			if ((map->offset >= entry->bound) &&
@@ -205,13 +222,15 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 		}
 		if (!valid) {
 			free(map, M_DRM);
-			return DRM_ERR(EACCES);
+			DRM_LOCK();
+			return EACCES;
 		}*/
 		break;
 	case _DRM_SCATTER_GATHER:
 		if (!dev->sg) {
 			free(map, M_DRM);
-			return DRM_ERR(EINVAL);
+			DRM_LOCK();
+			return EINVAL;
 		}
 		map->offset = map->offset + dev->sg->handle;
 		break;
@@ -228,7 +247,8 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 		map->dmah = drm_pci_alloc(dev, map->size, align, 0xfffffffful);
 		if (map->dmah == NULL) {
 			free(map, M_DRM);
-			return DRM_ERR(ENOMEM);
+			DRM_LOCK();
+			return ENOMEM;
 		}
 		map->handle = map->dmah->vaddr;
 		map->offset = map->dmah->busaddr;
@@ -236,7 +256,8 @@ int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
 	default:
 		DRM_ERROR("Bad map type %d\n", map->type);
 		free(map, M_DRM);
-		return DRM_ERR(EINVAL);
+		DRM_LOCK();
+		return EINVAL;
 	}
 
 	DRM_LOCK();
@@ -253,44 +274,41 @@ done:
 	return 0;
 }
 
-int drm_addmap_ioctl(DRM_IOCTL_ARGS)
+int drm_addmap_ioctl(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
 {
-	drm_map_t request;
+	drm_map_t *request = data;
 	drm_local_map_t *map;
 	int err;
-	DRM_DEVICE;
 
 	if (!(dev->flags & (FREAD|FWRITE)))
-		return DRM_ERR(EACCES); /* Require read/write */
-
-	DRM_COPY_FROM_USER_IOCTL(request, (drm_map_t *)data, sizeof(drm_map_t));
+		return EACCES; /* Require read/write */
 
-	if (!DRM_SUSER(p) && request.type != _DRM_AGP)
-		return DRM_ERR(EACCES);
+	if (!DRM_SUSER(DRM_CURPROC) && request->type != _DRM_AGP)
+		return EACCES;
 
 	DRM_LOCK();
-	err = drm_addmap(dev, request.offset, request.size, request.type,
-	    request.flags, &map);
+	err = drm_addmap(dev, request->offset, request->size, request->type,
+	    request->flags, &map);
 	DRM_UNLOCK();
 	if (err != 0)
 		return err;
 
-	request.offset = map->offset;
-	request.size = map->size;
-	request.type = map->type;
-	request.flags = map->flags;
-	request.mtrr   = map->mtrr;
-	request.handle = map->handle;
+	request->offset = map->offset;
+	request->size = map->size;
+	request->type = map->type;
+	request->flags = map->flags;
+	request->mtrr   = map->mtrr;
+	request->handle = map->handle;
 
-	if (request.type != _DRM_SHM) {
-		request.handle = (void *)request.offset;
+	if (request->type != _DRM_SHM) {
+		request->handle = (void *)request->offset;
 	}
-	DRM_COPY_TO_USER_IOCTL((drm_map_t *)data, request, sizeof(drm_map_t));
 
 	return 0;
 }
 
-void drm_rmmap(drm_device_t *dev, drm_local_map_t *map)
+void drm_rmmap(struct drm_device *dev, drm_local_map_t *map)
 {
 	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
 
@@ -319,6 +337,9 @@ void drm_rmmap(drm_device_t *dev, drm_local_map_t *map)
 	case _DRM_CONSISTENT:
 		drm_pci_free(dev, map->dmah);
 		break;
+	default:
+		DRM_ERROR("Bad map type %d\n", map->type);
+		break;
 	}
 
 	if (map->bsr != NULL) {
@@ -333,17 +354,15 @@ void drm_rmmap(drm_device_t *dev, drm_local_map_t *map)
  * isn't in use.
  */
 
-int drm_rmmap_ioctl(DRM_IOCTL_ARGS)
+int drm_rmmap_ioctl(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_local_map_t *map;
-	drm_map_t request;
-
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_map_t *)data, sizeof(request) );
+	drm_map_t *request = data;
 
 	DRM_LOCK();
 	TAILQ_FOREACH(map, &dev->maplist, link) {
-		if (map->handle == request.handle &&
+		if (map->handle == request->handle &&
 		    map->flags & _DRM_REMOVABLE)
 			break;
 	}
@@ -351,7 +370,7 @@ int drm_rmmap_ioctl(DRM_IOCTL_ARGS)
 	/* No match found. */
 	if (map == NULL) {
 		DRM_UNLOCK();
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}
 
 	drm_rmmap(dev, map);
@@ -362,7 +381,8 @@ int drm_rmmap_ioctl(DRM_IOCTL_ARGS)
 }
 
 
-static void drm_cleanup_buf_error(drm_device_t *dev, drm_buf_entry_t *entry)
+static void drm_cleanup_buf_error(struct drm_device *dev,
+				  drm_buf_entry_t *entry)
 {
 	int i;
 
@@ -385,7 +405,7 @@ static void drm_cleanup_buf_error(drm_device_t *dev, drm_buf_entry_t *entry)
 	}
 }
 
-static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_agp(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_entry_t *entry;
@@ -441,7 +461,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	}
 	if (!valid) {
 		DRM_DEBUG("zone invalid\n");
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}*/
 
 	entry = &dma->bufs[order];
@@ -449,7 +469,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	entry->buflist = malloc(count * sizeof(*entry->buflist), M_DRM,
 	    M_NOWAIT | M_ZERO);
 	if ( !entry->buflist ) {
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 
 	entry->buf_size = size;
@@ -469,7 +489,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 		buf->address = (void *)(agp_offset + offset);
 		buf->next    = NULL;
 		buf->pending = 0;
-		buf->filp    = NULL;
+		buf->file_priv = NULL;
 
 		buf->dev_priv_size = dev->driver.buf_priv_size;
 		buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
@@ -478,7 +498,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
 			drm_cleanup_buf_error(dev, entry);
-			return DRM_ERR(ENOMEM);
+			return ENOMEM;
 		}
 
 		offset += alignment;
@@ -494,7 +514,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	if (temp_buflist == NULL) {
 		/* Free the entry because it isn't valid */
 		drm_cleanup_buf_error(dev, entry);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 	dma->buflist = temp_buflist;
 
@@ -516,7 +536,7 @@ static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	return 0;
 }
 
-static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_pci(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	int count;
@@ -563,7 +583,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	    temp_pagelist == NULL) {
 		free(entry->buflist, M_DRM);
 		free(entry->seglist, M_DRM);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 	
 	memcpy(temp_pagelist, dma->pagelist, dma->page_count * 
@@ -586,7 +606,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 			entry->seg_count = count;
 			drm_cleanup_buf_error(dev, entry);
 			free(temp_pagelist, M_DRM);
-			return DRM_ERR(ENOMEM);
+			return ENOMEM;
 		}
 
 		entry->seglist[entry->seg_count++] = dmah;
@@ -610,7 +630,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 			buf->bus_address = dmah->busaddr + offset;
 			buf->next    = NULL;
 			buf->pending = 0;
-			buf->filp    = NULL;
+			buf->file_priv = NULL;
 
 			buf->dev_priv_size = dev->driver.buf_priv_size;
 			buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
@@ -621,7 +641,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 				entry->seg_count = count;
 				drm_cleanup_buf_error(dev, entry);
 				free(temp_pagelist, M_DRM);
-				return DRM_ERR(ENOMEM);
+				return ENOMEM;
 			}
 
 			DRM_DEBUG( "buffer %d @ %p\n",
@@ -637,7 +657,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 		/* Free the entry because it isn't valid */
 		drm_cleanup_buf_error(dev, entry);
 		free(temp_pagelist, M_DRM);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 	dma->buflist = temp_buflist;
 
@@ -663,7 +683,7 @@ static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 
 }
 
-static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_sg(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_entry_t *entry;
@@ -705,7 +725,7 @@ static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 	entry->buflist = malloc(count * sizeof(*entry->buflist), M_DRM,
 	    M_NOWAIT | M_ZERO);
 	if (entry->buflist == NULL)
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 
 	entry->buf_size = size;
 	entry->page_order = page_order;
@@ -724,7 +744,7 @@ static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 		buf->address = (void *)(agp_offset + offset + dev->sg->handle);
 		buf->next    = NULL;
 		buf->pending = 0;
-		buf->filp    = NULL;
+		buf->file_priv = NULL;
 
 		buf->dev_priv_size = dev->driver.buf_priv_size;
 		buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
@@ -733,7 +753,7 @@ static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
 			drm_cleanup_buf_error(dev, entry);
-			return DRM_ERR(ENOMEM);
+			return ENOMEM;
 		}
 
 		DRM_DEBUG( "buffer %d @ %p\n",
@@ -752,7 +772,7 @@ static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 	if (temp_buflist == NULL) {
 		/* Free the entry because it isn't valid */
 		drm_cleanup_buf_error(dev, entry);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 	dma->buflist = temp_buflist;
 
@@ -774,28 +794,28 @@ static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 	return 0;
 }
 
-int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
+int drm_addbufs_agp(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	int order, ret;
 
-	DRM_SPINLOCK(&dev->dma_lock);
-
 	if (request->count < 0 || request->count > 4096)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	
 	order = drm_order(request->size);
 	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
+
+	DRM_SPINLOCK(&dev->dma_lock);
 
 	/* No more allocations after first buffer-using ioctl. */
 	if (dev->buf_use != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(EBUSY);
+		return EBUSY;
 	}
 	/* No more than one allocation per order */
 	if (dev->dma->bufs[order].buf_count != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 
 	ret = drm_do_addbufs_agp(dev, request);
@@ -805,31 +825,31 @@ int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	return ret;
 }
 
-int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
+int drm_addbufs_sg(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	int order, ret;
 
-	DRM_SPINLOCK(&dev->dma_lock);
-
 	if (!DRM_SUSER(DRM_CURPROC))
-		return DRM_ERR(EACCES);
+		return EACCES;
 
 	if (request->count < 0 || request->count > 4096)
-		return DRM_ERR(EINVAL);
-	
+		return EINVAL;
+
 	order = drm_order(request->size);
 	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
+
+	DRM_SPINLOCK(&dev->dma_lock);
 
 	/* No more allocations after first buffer-using ioctl. */
 	if (dev->buf_use != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(EBUSY);
+		return EBUSY;
 	}
 	/* No more than one allocation per order */
 	if (dev->dma->bufs[order].buf_count != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 
 	ret = drm_do_addbufs_sg(dev, request);
@@ -839,31 +859,31 @@ int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 	return ret;
 }
 
-int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
+int drm_addbufs_pci(struct drm_device *dev, drm_buf_desc_t *request)
 {
 	int order, ret;
 
-	DRM_SPINLOCK(&dev->dma_lock);
-
 	if (!DRM_SUSER(DRM_CURPROC))
-		return DRM_ERR(EACCES);
+		return EACCES;
 
 	if (request->count < 0 || request->count > 4096)
-		return DRM_ERR(EINVAL);
-	
+		return EINVAL;
+
 	order = drm_order(request->size);
 	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
+
+	DRM_SPINLOCK(&dev->dma_lock);
 
 	/* No more allocations after first buffer-using ioctl. */
 	if (dev->buf_use != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(EBUSY);
+		return EBUSY;
 	}
 	/* No more than one allocation per order */
 	if (dev->dma->bufs[order].buf_count != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 
 	ret = drm_do_addbufs_pci(dev, request);
@@ -873,39 +893,30 @@ int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	return ret;
 }
 
-int drm_addbufs_ioctl(DRM_IOCTL_ARGS)
+int drm_addbufs_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_buf_desc_t request;
+	drm_buf_desc_t *request = data;
 	int err;
 
-	DRM_COPY_FROM_USER_IOCTL(request, (drm_buf_desc_t *)data,
-	    sizeof(request));
-
-	if (request.flags & _DRM_AGP_BUFFER)
-		err = drm_addbufs_agp(dev, &request);
-	else if (request.flags & _DRM_SG_BUFFER)
-		err = drm_addbufs_sg(dev, &request);
+	if (request->flags & _DRM_AGP_BUFFER)
+		err = drm_addbufs_agp(dev, request);
+	else if (request->flags & _DRM_SG_BUFFER)
+		err = drm_addbufs_sg(dev, request);
 	else
-		err = drm_addbufs_pci(dev, &request);
-
-	DRM_COPY_TO_USER_IOCTL((drm_buf_desc_t *)data, request,
-	    sizeof(request));
+		err = drm_addbufs_pci(dev, request);
 
 	return err;
 }
 
-int drm_infobufs(DRM_IOCTL_ARGS)
+int drm_infobufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_device_dma_t *dma = dev->dma;
-	drm_buf_info_t request;
+	drm_buf_info_t *request = data;
 	int i;
 	int count;
 	int retcode = 0;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_buf_info_t *)data, sizeof(request) );
-
 	DRM_SPINLOCK(&dev->dma_lock);
 	++dev->buf_use;		/* Can't allocate more after this call */
 	DRM_SPINUNLOCK(&dev->dma_lock);
@@ -916,7 +927,7 @@ int drm_infobufs(DRM_IOCTL_ARGS)
 
 	DRM_DEBUG( "count = %d\n", count );
 
-	if ( request.count >= count ) {
+	if ( request->count >= count ) {
 		for ( i = 0, count = 0 ; i < DRM_MAX_ORDER + 1 ; i++ ) {
 			if ( dma->bufs[i].buf_count ) {
 				drm_buf_desc_t from;
@@ -926,9 +937,9 @@ int drm_infobufs(DRM_IOCTL_ARGS)
 				from.low_mark = dma->bufs[i].freelist.low_mark;
 				from.high_mark = dma->bufs[i].freelist.high_mark;
 
-				if (DRM_COPY_TO_USER(&request.list[count], &from,
+				if (DRM_COPY_TO_USER(&request->list[count], &from,
 				    sizeof(drm_buf_desc_t)) != 0) {
-					retcode = DRM_ERR(EFAULT);
+					retcode = EFAULT;
 					break;
 				}
 
@@ -942,76 +953,69 @@ int drm_infobufs(DRM_IOCTL_ARGS)
 			}
 		}
 	}
-	request.count = count;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_buf_info_t *)data, request, sizeof(request) );
+	request->count = count;
 
 	return retcode;
 }
 
-int drm_markbufs(DRM_IOCTL_ARGS)
+int drm_markbufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_device_dma_t *dma = dev->dma;
-	drm_buf_desc_t request;
+	drm_buf_desc_t *request = data;
 	int order;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_buf_desc_t *)data, sizeof(request) );
-
 	DRM_DEBUG( "%d, %d, %d\n",
-		   request.size, request.low_mark, request.high_mark );
+		   request->size, request->low_mark, request->high_mark );
 	
 
-	order = drm_order(request.size);	
+	order = drm_order(request->size);	
 	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER ||
-	    request.low_mark < 0 || request.high_mark < 0) {
-		return DRM_ERR(EINVAL);
+	    request->low_mark < 0 || request->high_mark < 0) {
+		return EINVAL;
 	}
 
 	DRM_SPINLOCK(&dev->dma_lock);
-	if (request.low_mark > dma->bufs[order].buf_count ||
-	    request.high_mark > dma->bufs[order].buf_count) {
-		return DRM_ERR(EINVAL);
+	if (request->low_mark > dma->bufs[order].buf_count ||
+	    request->high_mark > dma->bufs[order].buf_count) {
+		DRM_SPINUNLOCK(&dev->dma_lock);
+		return EINVAL;
 	}
 
-	dma->bufs[order].freelist.low_mark  = request.low_mark;
-	dma->bufs[order].freelist.high_mark = request.high_mark;
+	dma->bufs[order].freelist.low_mark  = request->low_mark;
+	dma->bufs[order].freelist.high_mark = request->high_mark;
 	DRM_SPINUNLOCK(&dev->dma_lock);
 
 	return 0;
 }
 
-int drm_freebufs(DRM_IOCTL_ARGS)
+int drm_freebufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_device_dma_t *dma = dev->dma;
-	drm_buf_free_t request;
+	drm_buf_free_t *request = data;
 	int i;
 	int idx;
 	drm_buf_t *buf;
 	int retcode = 0;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_buf_free_t *)data, sizeof(request) );
-
-	DRM_DEBUG( "%d\n", request.count );
+	DRM_DEBUG( "%d\n", request->count );
 	
 	DRM_SPINLOCK(&dev->dma_lock);
-	for ( i = 0 ; i < request.count ; i++ ) {
-		if (DRM_COPY_FROM_USER(&idx, &request.list[i], sizeof(idx))) {
-			retcode = DRM_ERR(EFAULT);
+	for ( i = 0 ; i < request->count ; i++ ) {
+		if (DRM_COPY_FROM_USER(&idx, &request->list[i], sizeof(idx))) {
+			retcode = EFAULT;
 			break;
 		}
 		if ( idx < 0 || idx >= dma->buf_count ) {
 			DRM_ERROR( "Index %d (of %d max)\n",
 				   idx, dma->buf_count - 1 );
-			retcode = DRM_ERR(EINVAL);
+			retcode = EINVAL;
 			break;
 		}
 		buf = dma->buflist[idx];
-		if ( buf->filp != filp ) {
+		if ( buf->file_priv != file_priv ) {
 			DRM_ERROR("Process %d freeing buffer not owned\n",
 				   DRM_CURRENTPID);
-			retcode = DRM_ERR(EINVAL);
+			retcode = EINVAL;
 			break;
 		}
 		drm_free_buffer(dev, buf);
@@ -1021,9 +1025,8 @@ int drm_freebufs(DRM_IOCTL_ARGS)
 	return retcode;
 }
 
-int drm_mapbufs(DRM_IOCTL_ARGS)
+int drm_mapbufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_device_dma_t *dma = dev->dma;
 	int retcode = 0;
 	const int zero = 0;
@@ -1040,27 +1043,25 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	vaddr_t vaddr;
 #endif /* __NetBSD__ || __OpenBSD__ */
 
-	drm_buf_map_t request;
+	drm_buf_map_t *request = data;
 	int i;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_buf_map_t *)data, sizeof(request) );
-
 #if defined(__NetBSD__) || defined(__OpenBSD__)
 	if (!vfinddev(kdev, VCHR, &vn))
 		return 0;	/* FIXME: Shouldn't this be EINVAL or something? */
 #endif /* __NetBSD__ || __OpenBSD */
 
 #if defined(__FreeBSD__) && __FreeBSD_version >= 500000
-	vms = p->td_proc->p_vmspace;
+	vms = DRM_CURPROC->td_proc->p_vmspace;
 #else
-	vms = p->p_vmspace;
+	vms = DRM_CURPROC->p_vmspace;
 #endif
 
 	DRM_SPINLOCK(&dev->dma_lock);
 	dev->buf_use++;		/* Can't allocate more after this call */
 	DRM_SPINUNLOCK(&dev->dma_lock);
 
-	if (request.count < dma->buf_count)
+	if (request->count < dma->buf_count)
 		goto done;
 
 	if ((dev->driver.use_agp && (dma->flags & _DRM_DMA_USE_AGP)) ||
@@ -1082,10 +1083,11 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	vaddr = round_page((vm_offset_t)vms->vm_daddr + MAXDSIZ);
 #if __FreeBSD_version >= 600023
 	retcode = vm_mmap(&vms->vm_map, &vaddr, size, PROT_READ | PROT_WRITE,
-	    VM_PROT_ALL, MAP_SHARED, OBJT_DEVICE, kdev, foff );
+	    VM_PROT_ALL, MAP_SHARED, OBJT_DEVICE, dev->devnode, foff);
 #else
 	retcode = vm_mmap(&vms->vm_map, &vaddr, size, PROT_READ | PROT_WRITE,
-	    VM_PROT_ALL, MAP_SHARED, SLIST_FIRST(&kdev->si_hlist), foff );
+	    VM_PROT_ALL, MAP_SHARED, SLIST_FIRST(&dev->devnode->si_hlist),
+	    foff);
 #endif
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
 	vaddr = round_page((vaddr_t)vms->vm_daddr + MAXDSIZ);
@@ -1096,26 +1098,26 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	if (retcode)
 		goto done;
 
-	request.virtual = (void *)vaddr;
+	request->virtual = (void *)vaddr;
 
 	for ( i = 0 ; i < dma->buf_count ; i++ ) {
-		if (DRM_COPY_TO_USER(&request.list[i].idx,
-		    &dma->buflist[i]->idx, sizeof(request.list[0].idx))) {
+		if (DRM_COPY_TO_USER(&request->list[i].idx,
+		    &dma->buflist[i]->idx, sizeof(request->list[0].idx))) {
 			retcode = EFAULT;
 			goto done;
 		}
-		if (DRM_COPY_TO_USER(&request.list[i].total,
-		    &dma->buflist[i]->total, sizeof(request.list[0].total))) {
+		if (DRM_COPY_TO_USER(&request->list[i].total,
+		    &dma->buflist[i]->total, sizeof(request->list[0].total))) {
 			retcode = EFAULT;
 			goto done;
 		}
-		if (DRM_COPY_TO_USER(&request.list[i].used, &zero,
+		if (DRM_COPY_TO_USER(&request->list[i].used, &zero,
 		    sizeof(zero))) {
 			retcode = EFAULT;
 			goto done;
 		}
 		address = vaddr + dma->buflist[i]->offset; /* *** */
-		if (DRM_COPY_TO_USER(&request.list[i].address, &address,
+		if (DRM_COPY_TO_USER(&request->list[i].address, &address,
 		    sizeof(address))) {
 			retcode = EFAULT;
 			goto done;
@@ -1123,11 +1125,9 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	}
 
  done:
-	request.count = dma->buf_count;
-
-	DRM_DEBUG( "%d buffers, retcode = %d\n", request.count, retcode );
+	request->count = dma->buf_count;
 
-	DRM_COPY_TO_USER_IOCTL((drm_buf_map_t *)data, request, sizeof(request));
+	DRM_DEBUG( "%d buffers, retcode = %d\n", request->count, retcode );
 
-	return DRM_ERR(retcode);
+	return retcode;
 }
diff --git a/sys/dev/drm/drm_context.c b/sys/dev/drm/drm_context.c
index cf5bc15..7da4ae3 100644
--- a/sys/dev/drm/drm_context.c
+++ b/sys/dev/drm/drm_context.c
@@ -1,6 +1,3 @@
-/* drm_context.h -- IOCTLs for generic contexts -*- linux-c -*-
- * Created: Fri Nov 24 18:31:37 2000 by gareth@valinux.com
- */
 /*-
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,13 +31,17 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_context.c
+ * Implementation of the context management ioctls.
+ */
+
 #include "dev/drm/drmP.h"
 
 /* ================================================================
  * Context bitmap support
  */
 
-void drm_ctxbitmap_free(drm_device_t *dev, int ctx_handle)
+void drm_ctxbitmap_free(struct drm_device *dev, int ctx_handle)
 {
 	if (ctx_handle < 0 || ctx_handle >= DRM_MAX_CTXBITMAP || 
 	    dev->ctx_bitmap == NULL) {
@@ -56,7 +57,7 @@ void drm_ctxbitmap_free(drm_device_t *dev, int ctx_handle)
 	return;
 }
 
-int drm_ctxbitmap_next(drm_device_t *dev)
+int drm_ctxbitmap_next(struct drm_device *dev)
 {
 	int bit;
 
@@ -103,7 +104,7 @@ int drm_ctxbitmap_next(drm_device_t *dev)
 	return bit;
 }
 
-int drm_ctxbitmap_init(drm_device_t *dev)
+int drm_ctxbitmap_init(struct drm_device *dev)
 {
 	int i;
    	int temp;
@@ -112,7 +113,7 @@ int drm_ctxbitmap_init(drm_device_t *dev)
 	dev->ctx_bitmap = malloc(PAGE_SIZE, M_DRM, M_NOWAIT | M_ZERO);
 	if ( dev->ctx_bitmap == NULL ) {
 		DRM_UNLOCK();
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 	dev->context_sareas = NULL;
 	dev->max_context = -1;
@@ -126,7 +127,7 @@ int drm_ctxbitmap_init(drm_device_t *dev)
 	return 0;
 }
 
-void drm_ctxbitmap_cleanup(drm_device_t *dev)
+void drm_ctxbitmap_cleanup(struct drm_device *dev)
 {
 	DRM_LOCK();
 	if (dev->context_sareas != NULL)
@@ -139,48 +140,41 @@ void drm_ctxbitmap_cleanup(drm_device_t *dev)
  * Per Context SAREA Support
  */
 
-int drm_getsareactx( DRM_IOCTL_ARGS )
+int drm_getsareactx(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_priv_map_t request;
+	drm_ctx_priv_map_t *request = data;
 	drm_local_map_t *map;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_ctx_priv_map_t *)data, 
-			   sizeof(request) );
-
 	DRM_LOCK();
-	if (dev->max_context < 0 || request.ctx_id >= (unsigned) dev->max_context) {
+	if (dev->max_context < 0 ||
+	    request->ctx_id >= (unsigned) dev->max_context) {
 		DRM_UNLOCK();
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}
 
-	map = dev->context_sareas[request.ctx_id];
+	map = dev->context_sareas[request->ctx_id];
 	DRM_UNLOCK();
 
-	request.handle = map->handle;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_ctx_priv_map_t *)data, request, sizeof(request) );
+	request->handle = map->handle;
 
 	return 0;
 }
 
-int drm_setsareactx( DRM_IOCTL_ARGS )
+int drm_setsareactx(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_priv_map_t request;
+	drm_ctx_priv_map_t *request = data;
 	drm_local_map_t *map = NULL;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_ctx_priv_map_t *)data,
-			   sizeof(request) );
-
 	DRM_LOCK();
 	TAILQ_FOREACH(map, &dev->maplist, link) {
-		if (map->handle == request.handle) {
+		if (map->handle == request->handle) {
 			if (dev->max_context < 0)
 				goto bad;
-			if (request.ctx_id >= (unsigned) dev->max_context)
+			if (request->ctx_id >= (unsigned) dev->max_context)
 				goto bad;
-			dev->context_sareas[request.ctx_id] = map;
+			dev->context_sareas[request->ctx_id] = map;
 			DRM_UNLOCK();
 			return 0;
 		}
@@ -188,18 +182,18 @@ int drm_setsareactx( DRM_IOCTL_ARGS )
 
 bad:
 	DRM_UNLOCK();
-	return DRM_ERR(EINVAL);
+	return EINVAL;
 }
 
 /* ================================================================
  * The actual DRM context handling routines
  */
 
-int drm_context_switch(drm_device_t *dev, int old, int new)
+int drm_context_switch(struct drm_device *dev, int old, int new)
 {
         if ( test_and_set_bit( 0, &dev->context_flag ) ) {
                 DRM_ERROR( "Reentering -- FIXME\n" );
-                return DRM_ERR(EBUSY);
+                return EBUSY;
         }
 
         DRM_DEBUG( "Context switch from %d to %d\n", old, new );
@@ -212,7 +206,7 @@ int drm_context_switch(drm_device_t *dev, int old, int new)
         return 0;
 }
 
-int drm_context_switch_complete(drm_device_t *dev, int new)
+int drm_context_switch_complete(struct drm_device *dev, int new)
 {
         dev->last_context = new;  /* PRE/POST: This is the _only_ writer. */
 
@@ -228,120 +222,99 @@ int drm_context_switch_complete(drm_device_t *dev, int new)
         return 0;
 }
 
-int drm_resctx(DRM_IOCTL_ARGS)
+int drm_resctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	drm_ctx_res_t res;
+	drm_ctx_res_t *res = data;
 	drm_ctx_t ctx;
 	int i;
 
-	DRM_COPY_FROM_USER_IOCTL( res, (drm_ctx_res_t *)data, sizeof(res) );
-
-	if ( res.count >= DRM_RESERVED_CONTEXTS ) {
+	if ( res->count >= DRM_RESERVED_CONTEXTS ) {
 		bzero(&ctx, sizeof(ctx));
 		for ( i = 0 ; i < DRM_RESERVED_CONTEXTS ; i++ ) {
 			ctx.handle = i;
-			if ( DRM_COPY_TO_USER( &res.contexts[i],
+			if ( DRM_COPY_TO_USER( &res->contexts[i],
 					   &ctx, sizeof(ctx) ) )
-				return DRM_ERR(EFAULT);
+				return EFAULT;
 		}
 	}
-	res.count = DRM_RESERVED_CONTEXTS;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_ctx_res_t *)data, res, sizeof(res) );
+	res->count = DRM_RESERVED_CONTEXTS;
 
 	return 0;
 }
 
-int drm_addctx(DRM_IOCTL_ARGS)
+int drm_addctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_t ctx;
-
-	DRM_COPY_FROM_USER_IOCTL( ctx, (drm_ctx_t *)data, sizeof(ctx) );
+	drm_ctx_t *ctx = data;
 
-	ctx.handle = drm_ctxbitmap_next(dev);
-	if ( ctx.handle == DRM_KERNEL_CONTEXT ) {
+	ctx->handle = drm_ctxbitmap_next(dev);
+	if ( ctx->handle == DRM_KERNEL_CONTEXT ) {
 				/* Skip kernel's context and get a new one. */
-		ctx.handle = drm_ctxbitmap_next(dev);
+		ctx->handle = drm_ctxbitmap_next(dev);
 	}
-	DRM_DEBUG( "%d\n", ctx.handle );
-	if ( ctx.handle == -1 ) {
+	DRM_DEBUG( "%d\n", ctx->handle );
+	if ( ctx->handle == -1 ) {
 		DRM_DEBUG( "Not enough free contexts.\n" );
 				/* Should this return -EBUSY instead? */
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 	}
 
-	if (dev->driver.context_ctor && ctx.handle != DRM_KERNEL_CONTEXT) {
+	if (dev->driver.context_ctor && ctx->handle != DRM_KERNEL_CONTEXT) {
 		DRM_LOCK();
-		dev->driver.context_ctor(dev, ctx.handle);
+		dev->driver.context_ctor(dev, ctx->handle);
 		DRM_UNLOCK();
 	}
 
-	DRM_COPY_TO_USER_IOCTL( (drm_ctx_t *)data, ctx, sizeof(ctx) );
-
 	return 0;
 }
 
-int drm_modctx(DRM_IOCTL_ARGS)
+int drm_modctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	/* This does nothing */
 	return 0;
 }
 
-int drm_getctx(DRM_IOCTL_ARGS)
+int drm_getctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	drm_ctx_t ctx;
-
-	DRM_COPY_FROM_USER_IOCTL( ctx, (drm_ctx_t *)data, sizeof(ctx) );
+	drm_ctx_t *ctx = data;
 
 	/* This is 0, because we don't handle any context flags */
-	ctx.flags = 0;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_ctx_t *)data, ctx, sizeof(ctx) );
+	ctx->flags = 0;
 
 	return 0;
 }
 
-int drm_switchctx(DRM_IOCTL_ARGS)
+int drm_switchctx(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_t ctx;
+	drm_ctx_t *ctx = data;
 
-	DRM_COPY_FROM_USER_IOCTL( ctx, (drm_ctx_t *)data, sizeof(ctx) );
-
-	DRM_DEBUG( "%d\n", ctx.handle );
-	return drm_context_switch(dev, dev->last_context, ctx.handle);
+	DRM_DEBUG( "%d\n", ctx->handle );
+	return drm_context_switch(dev, dev->last_context, ctx->handle);
 }
 
-int drm_newctx(DRM_IOCTL_ARGS)
+int drm_newctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_t ctx;
-
-	DRM_COPY_FROM_USER_IOCTL( ctx, (drm_ctx_t *)data, sizeof(ctx) );
+	drm_ctx_t *ctx = data;
 
-	DRM_DEBUG( "%d\n", ctx.handle );
-	drm_context_switch_complete(dev, ctx.handle);
+	DRM_DEBUG( "%d\n", ctx->handle );
+	drm_context_switch_complete(dev, ctx->handle);
 
 	return 0;
 }
 
-int drm_rmctx(DRM_IOCTL_ARGS)
+int drm_rmctx(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_ctx_t ctx;
-
-	DRM_COPY_FROM_USER_IOCTL( ctx, (drm_ctx_t *)data, sizeof(ctx) );
+	drm_ctx_t *ctx = data;
 
-	DRM_DEBUG( "%d\n", ctx.handle );
-	if ( ctx.handle != DRM_KERNEL_CONTEXT ) {
+	DRM_DEBUG( "%d\n", ctx->handle );
+	if ( ctx->handle != DRM_KERNEL_CONTEXT ) {
 		if (dev->driver.context_dtor) {
 			DRM_LOCK();
-			dev->driver.context_dtor(dev, ctx.handle);
+			dev->driver.context_dtor(dev, ctx->handle);
 			DRM_UNLOCK();
 		}
 
-		drm_ctxbitmap_free(dev, ctx.handle);
+		drm_ctxbitmap_free(dev, ctx->handle);
 	}
 
 	return 0;
diff --git a/sys/dev/drm/drm_dma.c b/sys/dev/drm/drm_dma.c
index a8661ca..1ba3fdf 100644
--- a/sys/dev/drm/drm_dma.c
+++ b/sys/dev/drm/drm_dma.c
@@ -1,6 +1,3 @@
-/* drm_dma.c -- DMA IOCTL and function support -*- linux-c -*-
- * Created: Fri Mar 19 14:30:16 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,21 +31,29 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_dma.c
+ * Support code for DMA buffer management.
+ *
+ * The implementation used to be significantly more complicated, but the
+ * complexity has been moved into the drivers as different buffer management
+ * schemes evolved.
+ */
+
 #include "dev/drm/drmP.h"
 
-int drm_dma_setup(drm_device_t *dev)
+int drm_dma_setup(struct drm_device *dev)
 {
 
 	dev->dma = malloc(sizeof(*dev->dma), M_DRM, M_NOWAIT | M_ZERO);
 	if (dev->dma == NULL)
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 
-	DRM_SPININIT(dev->dma_lock, "drmdma");
+	DRM_SPININIT(&dev->dma_lock, "drmdma");
 
 	return 0;
 }
 
-void drm_dma_takedown(drm_device_t *dev)
+void drm_dma_takedown(struct drm_device *dev)
 {
 	drm_device_dma_t  *dma = dev->dma;
 	int		  i, j;
@@ -83,27 +88,27 @@ void drm_dma_takedown(drm_device_t *dev)
 	free(dma->pagelist, M_DRM);
 	free(dev->dma, M_DRM);
 	dev->dma = NULL;
-	DRM_SPINUNINIT(dev->dma_lock);
+	DRM_SPINUNINIT(&dev->dma_lock);
 }
 
 
-void drm_free_buffer(drm_device_t *dev, drm_buf_t *buf)
+void drm_free_buffer(struct drm_device *dev, drm_buf_t *buf)
 {
 	if (!buf) return;
 
 	buf->pending  = 0;
-	buf->filp     = NULL;
+	buf->file_priv= NULL;
 	buf->used     = 0;
 }
 
-void drm_reclaim_buffers(drm_device_t *dev, DRMFILE filp)
+void drm_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv)
 {
 	drm_device_dma_t *dma = dev->dma;
 	int		 i;
 
 	if (!dma) return;
 	for (i = 0; i < dma->buf_count; i++) {
-		if (dma->buflist[i]->filp == filp) {
+		if (dma->buflist[i]->file_priv == file_priv) {
 			switch (dma->buflist[i]->list) {
 			case DRM_LIST_NONE:
 				drm_free_buffer(dev, dma->buflist[i]);
@@ -120,12 +125,12 @@ void drm_reclaim_buffers(drm_device_t *dev, DRMFILE filp)
 }
 
 /* Call into the driver-specific DMA handler */
-int drm_dma(DRM_IOCTL_ARGS)
+int drm_dma(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 
 	if (dev->driver.dma_ioctl) {
-		return dev->driver.dma_ioctl(kdev, cmd, data, flags, p, filp);
+		/* shared code returns -errno */
+		return -dev->driver.dma_ioctl(dev, data, file_priv);
 	} else {
 		DRM_DEBUG("DMA ioctl on driver with no dma handler\n");
 		return EINVAL;
diff --git a/sys/dev/drm/drm_drawable.c b/sys/dev/drm/drm_drawable.c
index 8de8095..1223a41 100644
--- a/sys/dev/drm/drm_drawable.c
+++ b/sys/dev/drm/drm_drawable.c
@@ -1,6 +1,3 @@
-/* drm_drawable.h -- IOCTLs for drawables -*- linux-c -*-
- * Created: Tue Feb  2 08:37:54 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,21 +31,145 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_drawable.c
+ * This file implements ioctls to store information along with DRM drawables,
+ * such as the current set of cliprects for vblank-synced buffer swaps.
+ */
+
 #include "dev/drm/drmP.h"
 
-int drm_adddraw(DRM_IOCTL_ARGS)
+struct bsd_drm_drawable_info {
+	struct drm_drawable_info info;
+	int handle;
+	RB_ENTRY(bsd_drm_drawable_info) tree;
+};
+
+static int
+drm_drawable_compare(struct bsd_drm_drawable_info *a,
+    struct bsd_drm_drawable_info *b)
+{
+	if (a->handle > b->handle)
+		return 1;
+	if (a->handle < b->handle)
+		return -1;
+	return 0;
+}
+
+RB_GENERATE_STATIC(drawable_tree, bsd_drm_drawable_info, tree,
+    drm_drawable_compare);
+
+struct drm_drawable_info *
+drm_get_drawable_info(struct drm_device *dev, int handle)
+{
+	struct bsd_drm_drawable_info find, *result;
+
+	find.handle = handle;
+	result = RB_FIND(drawable_tree, &dev->drw_head, &find);
+
+	return &result->info;
+}
+
+int drm_adddraw(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	drm_draw_t draw;
+	drm_draw_t *draw = data;
+	struct bsd_drm_drawable_info *info;
+
+	info = drm_calloc(1, sizeof(struct bsd_drm_drawable_info),
+	    DRM_MEM_DRAWABLE);
+	if (info == NULL)
+		return ENOMEM;
 
-	draw.handle = 0;	/* NOOP */
-	DRM_DEBUG("%d\n", draw.handle);
-	
-	DRM_COPY_TO_USER_IOCTL( (drm_draw_t *)data, draw, sizeof(draw) );
+	info->handle = alloc_unr(dev->drw_unrhdr);
+	DRM_SPINLOCK(&dev->drw_lock);
+	RB_INSERT(drawable_tree, &dev->drw_head, info);
+	draw->handle = info->handle;
+	DRM_SPINUNLOCK(&dev->drw_lock);
+
+	DRM_DEBUG("%d\n", draw->handle);
 
 	return 0;
 }
 
-int drm_rmdraw(DRM_IOCTL_ARGS)
+int drm_rmdraw(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_draw_t *draw = (drm_draw_t *)data;
+	struct drm_drawable_info *info;
+
+	DRM_SPINLOCK(&dev->drw_lock);
+	info = drm_get_drawable_info(dev, draw->handle);
+	if (info != NULL) {
+		RB_REMOVE(drawable_tree, &dev->drw_head,
+		    (struct bsd_drm_drawable_info *)info);
+		DRM_SPINUNLOCK(&dev->drw_lock);
+		free_unr(dev->drw_unrhdr, draw->handle);
+		drm_free(info, sizeof(struct bsd_drm_drawable_info),
+		    DRM_MEM_DRAWABLE);
+		return 0;
+	} else {
+		DRM_SPINUNLOCK(&dev->drw_lock);
+		return EINVAL;
+	}
+}
+
+int drm_update_draw(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	return 0;		/* NOOP */
+	struct drm_drawable_info *info;
+	struct drm_update_draw *update = (struct drm_update_draw *)data;
+	int ret;
+
+	info = drm_get_drawable_info(dev, update->handle);
+	if (info == NULL)
+		return EINVAL;
+
+	switch (update->type) {
+	case DRM_DRAWABLE_CLIPRECTS:
+		DRM_SPINLOCK(&dev->drw_lock);
+		if (update->num != info->num_rects) {
+			drm_free(info->rects,
+			    sizeof(*info->rects) * info->num_rects,
+			    DRM_MEM_DRAWABLE);
+			info->rects = NULL;
+			info->num_rects = 0;
+		}
+		if (update->num == 0) {
+			DRM_SPINUNLOCK(&dev->drw_lock);
+			return 0;
+		}
+		if (info->rects == NULL) {
+			info->rects = drm_alloc(sizeof(*info->rects) *
+			    update->num, DRM_MEM_DRAWABLE);
+			if (info->rects == NULL) {
+				DRM_SPINUNLOCK(&dev->drw_lock);
+				return ENOMEM;
+			}
+			info->num_rects = update->num;
+		}
+		/* For some reason the pointer arg is unsigned long long. */
+		ret = copyin((void *)(intptr_t)update->data, info->rects,
+		    sizeof(*info->rects) * info->num_rects);
+		DRM_SPINUNLOCK(&dev->drw_lock);
+		return ret;
+	default:
+		return EINVAL;
+	}
+}
+
+void drm_drawable_free_all(struct drm_device *dev)
+{
+	struct bsd_drm_drawable_info *info, *next;
+
+	DRM_SPINLOCK(&dev->drw_lock);
+	for (info = RB_MIN(drawable_tree, &dev->drw_head);
+	    info != NULL ; info = next) {
+		next = RB_NEXT(drawable_tree, &dev->drw_head, info);
+		RB_REMOVE(drawable_tree, &dev->drw_head,
+		    (struct bsd_drm_drawable_info *)info);
+		DRM_SPINUNLOCK(&dev->drw_lock);
+		free_unr(dev->drw_unrhdr, info->handle);
+		drm_free(info, sizeof(struct bsd_drm_drawable_info),
+		    DRM_MEM_DRAWABLE);
+		DRM_SPINLOCK(&dev->drw_lock);
+	}
+	DRM_SPINUNLOCK(&dev->drw_lock);
 }
diff --git a/sys/dev/drm/drm_drv.c b/sys/dev/drm/drm_drv.c
index 71d9c75..8d5503d 100644
--- a/sys/dev/drm/drm_drv.c
+++ b/sys/dev/drm/drm_drv.c
@@ -1,6 +1,3 @@
-/* drm_drv.h -- Generic driver template -*- linux-c -*-
- * Created: Thu Nov 23 03:10:50 2000 by gareth@valinux.com
- */
 /*-
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,13 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_drv.c
+ * The catch-all file for DRM device support, including module setup/teardown,
+ * open/close, and ioctl dispatch.
+ */
+
+
+#include <sys/limits.h>
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/drm_sarea.h"
@@ -44,14 +48,14 @@ int drm_debug_flag = 1;
 int drm_debug_flag = 0;
 #endif
 
-static int drm_load(drm_device_t *dev);
-static void drm_unload(drm_device_t *dev);
+static int drm_load(struct drm_device *dev);
+static void drm_unload(struct drm_device *dev);
 static drm_pci_id_list_t *drm_find_description(int vendor, int device,
     drm_pci_id_list_t *idlist);
 
 #ifdef __FreeBSD__
 #define DRIVER_SOFTC(unit) \
-	((drm_device_t *)devclass_get_softc(drm_devclass, unit))
+	((struct drm_device *)devclass_get_softc(drm_devclass, unit))
 
 MODULE_VERSION(drm, 1);
 MODULE_DEPEND(drm, agp, 1, 1, 1);
@@ -63,67 +67,69 @@ MODULE_DEPEND(drm, mem, 1, 1, 1);
 
 #if defined(__NetBSD__) || defined(__OpenBSD__)
 #define DRIVER_SOFTC(unit) \
-	((drm_device_t *)device_lookup(&drm_cd, unit))
+	((struct drm_device *)device_lookup(&drm_cd, unit))
 #endif /* __NetBSD__ || __OpenBSD__ */
 
 static drm_ioctl_desc_t		  drm_ioctls[256] = {
-	[DRM_IOCTL_NR(DRM_IOCTL_VERSION)]       = { drm_version,     0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_UNIQUE)]    = { drm_getunique,   0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAGIC)]     = { drm_getmagic,    0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_IRQ_BUSID)]     = { drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAP)]       = { drm_getmap,      0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_CLIENT)]    = { drm_getclient,   0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_STATS)]     = { drm_getstats,    0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_VERSION)]   = { drm_setversion,  DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_UNIQUE)]    = { drm_setunique,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_BLOCK)]         = { drm_noop,        DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_UNBLOCK)]       = { drm_noop,        DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AUTH_MAGIC)]    = { drm_authmagic,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_MAP)]       = { drm_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_MAP)]        = { drm_rmmap_ioctl, DRM_AUTH },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_SAREA_CTX)] = { drm_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_SAREA_CTX)] = { drm_getsareactx, DRM_AUTH },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_CTX)]       = { drm_addctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_CTX)]        = { drm_rmctx,       DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_MOD_CTX)]       = { drm_modctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_CTX)]       = { drm_getctx,      DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_SWITCH_CTX)]    = { drm_switchctx,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_NEW_CTX)]       = { drm_newctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_RES_CTX)]       = { drm_resctx,      DRM_AUTH },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_DRAW)]      = { drm_adddraw,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_DRAW)]       = { drm_rmdraw,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_LOCK)]	        = { drm_lock,        DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_UNLOCK)]        = { drm_unlock,      DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_FINISH)]        = { drm_noop,        DRM_AUTH },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_BUFS)]      = { drm_addbufs_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_MARK_BUFS)]     = { drm_markbufs,    DRM_AUTH|DRM_MASTER },
-	[DRM_IOCTL_NR(DRM_IOCTL_INFO_BUFS)]     = { drm_infobufs,    DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_MAP_BUFS)]      = { drm_mapbufs,     DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_FREE_BUFS)]     = { drm_freebufs,    DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_DMA)]           = { drm_dma,         DRM_AUTH },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_CONTROL)]       = { drm_control,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ACQUIRE)]   = { drm_agp_acquire_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_RELEASE)]   = { drm_agp_release_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ENABLE)]    = { drm_agp_enable_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_INFO)]      = { drm_agp_info_ioctl, DRM_AUTH },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ALLOC)]     = { drm_agp_alloc_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_FREE)]      = { drm_agp_free_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_BIND)]      = { drm_agp_bind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_UNBIND)]    = { drm_agp_unbind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SG_ALLOC)]      = { drm_sg_alloc,    DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_IOCTL_SG_FREE)]       = { drm_sg_free,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_WAIT_VBLANK)]   = { drm_wait_vblank, 0 },
+	DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_getmap, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_CLIENT, drm_getclient, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_STATS, drm_getstats, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_SET_VERSION, drm_setversion, DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_SET_UNIQUE, drm_setunique, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_BLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_UNBLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_ADD_MAP, drm_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_RM_MAP, drm_rmmap_ioctl, DRM_AUTH),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_SET_SAREA_CTX, drm_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_SAREA_CTX, drm_getsareactx, DRM_AUTH),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_ADD_CTX, drm_addctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_RM_CTX, drm_rmctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_MOD_CTX, drm_modctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_GET_CTX, drm_getctx, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_SWITCH_CTX, drm_switchctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_NEW_CTX, drm_newctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_RES_CTX, drm_resctx, DRM_AUTH),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_ADD_DRAW, drm_adddraw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_RM_DRAW, drm_rmdraw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_LOCK, drm_lock, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_UNLOCK, drm_unlock, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_FINISH, drm_noop, DRM_AUTH),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_ADD_BUFS, drm_addbufs_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_MARK_BUFS, drm_markbufs, DRM_AUTH|DRM_MASTER),
+	DRM_IOCTL_DEF(DRM_IOCTL_INFO_BUFS, drm_infobufs, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_MAP_BUFS, drm_mapbufs, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_FREE_BUFS, drm_freebufs, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_DMA, drm_dma, DRM_AUTH),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_CONTROL, drm_control, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_ACQUIRE, drm_agp_acquire_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_RELEASE, drm_agp_release_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_ENABLE, drm_agp_enable_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_INFO, drm_agp_info_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_ALLOC, drm_agp_alloc_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_FREE, drm_agp_free_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_BIND, drm_agp_bind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_AGP_UNBIND, drm_agp_unbind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_SG_ALLOC, drm_sg_alloc_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_IOCTL_SG_FREE, drm_sg_free, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+
+	DRM_IOCTL_DEF(DRM_IOCTL_WAIT_VBLANK, drm_wait_vblank, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODESET_CTL, drm_modeset_ctl, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_UPDATE_DRAW, drm_update_draw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 };
 
 #ifdef __FreeBSD__
@@ -178,7 +184,7 @@ int drm_probe(device_t dev, drm_pci_id_list_t *idlist)
 
 int drm_attach(device_t nbdev, drm_pci_id_list_t *idlist)
 {
-	drm_device_t *dev;
+	struct drm_device *dev;
 	drm_pci_id_list_t *id_entry;
 	int unit;
 
@@ -200,7 +206,11 @@ int drm_attach(device_t nbdev, drm_pci_id_list_t *idlist)
 			DRM_DEV_MODE,
 			"dri/card%d", unit);
 #if __FreeBSD_version >= 500000
-	mtx_init(&dev->dev_lock, "drm device", NULL, MTX_DEF);
+	mtx_init(&dev->dev_lock, "drmdev", NULL, MTX_DEF);
+	mtx_init(&dev->irq_lock, "drmirq", NULL, MTX_DEF);
+	mtx_init(&dev->vbl_lock, "drmvbl", NULL, MTX_DEF);
+	mtx_init(&dev->drw_lock, "drmdrw", NULL, MTX_DEF);
+	mtx_init(&dev->tsk_lock, "drmtsk", NULL, MTX_DEF);
 #endif
 
 	id_entry = drm_find_description(pci_get_vendor(dev->device),
@@ -316,14 +326,15 @@ void drm_attach(struct pci_attach_args *pa, dev_t kdev,
     drm_pci_id_list_t *idlist)
 {
 	int i;
-	drm_device_t *dev;
+	struct drm_device *dev;
 	drm_pci_id_list_t *id_entry;
 
 	config_makeroom(kdev, &drm_cd);
-	drm_cd.cd_devs[(kdev)] = malloc(sizeof(drm_device_t), M_DRM, M_WAITOK);
+	drm_cd.cd_devs[(kdev)] = malloc(sizeof(struct drm_device),
+					M_DRM, M_WAITOK);
 	dev = DRIVER_SOFTC(kdev);
 
-	memset(dev, 0, sizeof(drm_device_t));
+	memset(dev, 0, sizeof(struct drm_device));
 	memcpy(&dev->pa, pa, sizeof(dev->pa));
 
 	dev->irq = pa->pa_intrline;
@@ -343,7 +354,7 @@ void drm_attach(struct pci_attach_args *pa, dev_t kdev,
 
 int drm_detach(struct device *self, int flags)
 {
-	drm_unload((drm_device_t *)self);
+	drm_unload((struct drm_device *)self);
 	return 0;
 }
 
@@ -376,7 +387,7 @@ drm_pci_id_list_t *drm_find_description(int vendor, int device,
 	return NULL;
 }
 
-static int drm_firstopen(drm_device_t *dev)
+static int drm_firstopen(struct drm_device *dev)
 {
 	drm_local_map_t *map;
 	int i;
@@ -400,17 +411,6 @@ static int drm_firstopen(drm_device_t *dev)
 			return i;
 	}
 
-	dev->counters  = 6;
-	dev->types[0]  = _DRM_STAT_LOCK;
-	dev->types[1]  = _DRM_STAT_OPENS;
-	dev->types[2]  = _DRM_STAT_CLOSES;
-	dev->types[3]  = _DRM_STAT_IOCTLS;
-	dev->types[4]  = _DRM_STAT_LOCKS;
-	dev->types[5]  = _DRM_STAT_UNLOCKS;
-
-	for ( i = 0 ; i < DRM_ARRAY_SIZE(dev->counts) ; i++ )
-		atomic_set( &dev->counts[i], 0 );
-
 	for ( i = 0 ; i < DRM_HASH_SIZE ; i++ ) {
 		dev->magiclist[i].head = NULL;
 		dev->magiclist[i].tail = NULL;
@@ -433,7 +433,7 @@ static int drm_firstopen(drm_device_t *dev)
 	return 0;
 }
 
-static int drm_lastclose(drm_device_t *dev)
+static int drm_lastclose(struct drm_device *dev)
 {
 	drm_magic_entry_t *pt, *next;
 	drm_local_map_t *map, *mapsave;
@@ -463,6 +463,8 @@ static int drm_lastclose(drm_device_t *dev)
 		dev->magiclist[i].head = dev->magiclist[i].tail = NULL;
 	}
 
+	drm_drawable_free_all(dev);
+
 				/* Clear AGP information */
 	if ( dev->agp ) {
 		drm_agp_mem_t *entry;
@@ -492,29 +494,32 @@ static int drm_lastclose(drm_device_t *dev)
 	}
 
 	TAILQ_FOREACH_SAFE(map, &dev->maplist, link, mapsave) {
-		drm_rmmap(dev, map);
+		if (!(map->flags & _DRM_DRIVER))
+			drm_rmmap(dev, map);
 	}
 
-
 	drm_dma_takedown(dev);
 	if ( dev->lock.hw_lock ) {
 		dev->lock.hw_lock = NULL; /* SHM removed */
-		dev->lock.filp = NULL;
+		dev->lock.file_priv = NULL;
 		DRM_WAKEUP_INT((void *)&dev->lock.lock_queue);
 	}
 
 	return 0;
 }
 
-static int drm_load(drm_device_t *dev)
+static int drm_load(struct drm_device *dev)
 {
-	int retcode;
+	int i, retcode;
 
 	DRM_DEBUG( "\n" );
 
 	dev->irq = pci_get_irq(dev->device);
-	/* XXX Fix domain number (alpha hoses) */
+#if defined(__FreeBSD__) && __FreeBSD_version >= 700053
+	dev->pci_domain = pci_get_domain(dev->device);
+#else
 	dev->pci_domain = 0;
+#endif
 	dev->pci_bus = pci_get_bus(dev->device);
 	dev->pci_slot = pci_get_slot(dev->device);
 	dev->pci_func = pci_get_function(dev->device);
@@ -530,9 +535,24 @@ static int drm_load(drm_device_t *dev)
 #endif
 	TAILQ_INIT(&dev->files);
 
+	dev->counters  = 6;
+	dev->types[0]  = _DRM_STAT_LOCK;
+	dev->types[1]  = _DRM_STAT_OPENS;
+	dev->types[2]  = _DRM_STAT_CLOSES;
+	dev->types[3]  = _DRM_STAT_IOCTLS;
+	dev->types[4]  = _DRM_STAT_LOCKS;
+	dev->types[5]  = _DRM_STAT_UNLOCKS;
+
+	for ( i = 0 ; i < DRM_ARRAY_SIZE(dev->counts) ; i++ )
+		atomic_set( &dev->counts[i], 0 );
+
 	if (dev->driver.load != NULL) {
 		DRM_LOCK();
-		retcode = dev->driver.load(dev, dev->id_entry->driver_private);
+		/* Shared code returns -errno. */
+		retcode = -dev->driver.load(dev,
+		    dev->id_entry->driver_private);
+		if (pci_enable_busmaster(dev->device))
+			DRM_ERROR("Request to enable bus-master failed.\n");
 		DRM_UNLOCK();
 		if (retcode != 0)
 			goto error;
@@ -544,7 +564,7 @@ static int drm_load(drm_device_t *dev)
 		if (dev->driver.require_agp && dev->agp == NULL) {
 			DRM_ERROR("Card isn't AGP, or couldn't initialize "
 			    "AGP.\n");
-			retcode = DRM_ERR(ENOMEM);
+			retcode = ENOMEM;
 			goto error;
 		}
 		if (dev->agp != NULL) {
@@ -559,7 +579,13 @@ static int drm_load(drm_device_t *dev)
 		DRM_ERROR("Cannot allocate memory for context bitmap.\n");
 		goto error;
 	}
-	
+
+	dev->drw_unrhdr = new_unrhdr(1, INT_MAX, NULL);
+	if (dev->drw_unrhdr == NULL) {
+		DRM_ERROR("Couldn't allocate drawable number allocator\n");
+		goto error;
+	}
+
 	DRM_INFO("Initialized %s %d.%d.%d %s\n",
 	  	dev->driver.name,
 	  	dev->driver.major,
@@ -579,13 +605,17 @@ error:
 #ifdef __FreeBSD__
 	destroy_dev(dev->devnode);
 #if __FreeBSD_version >= 500000
+	mtx_destroy(&dev->tsk_lock);
+	mtx_destroy(&dev->drw_lock);
+	mtx_destroy(&dev->vbl_lock);
+	mtx_destroy(&dev->irq_lock);
 	mtx_destroy(&dev->dev_lock);
 #endif
 #endif
 	return retcode;
 }
 
-static void drm_unload(drm_device_t *dev)
+static void drm_unload(struct drm_device *dev)
 {
 	int i;
 
@@ -631,46 +661,51 @@ static void drm_unload(drm_device_t *dev)
 	if (dev->driver.unload != NULL)
 		dev->driver.unload(dev);
 
+	delete_unrhdr(dev->drw_unrhdr);
+
 	drm_mem_uninit();
+
+	if (pci_disable_busmaster(dev->device))
+		DRM_ERROR("Request to disable bus-master failed.\n");
+
 #if defined(__FreeBSD__) &&  __FreeBSD_version >= 500000
+	mtx_destroy(&dev->tsk_lock);
+	mtx_destroy(&dev->drw_lock);
+	mtx_destroy(&dev->vbl_lock);
+	mtx_destroy(&dev->irq_lock);
 	mtx_destroy(&dev->dev_lock);
 #endif
 }
 
 
-int drm_version(DRM_IOCTL_ARGS)
+int drm_version(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_version_t version;
+	drm_version_t *version = data;
 	int len;
 
-	DRM_COPY_FROM_USER_IOCTL( version, (drm_version_t *)data, sizeof(version) );
-
 #define DRM_COPY( name, value )						\
 	len = strlen( value );						\
 	if ( len > name##_len ) len = name##_len;			\
 	name##_len = strlen( value );					\
 	if ( len && name ) {						\
 		if ( DRM_COPY_TO_USER( name, value, len ) )		\
-			return DRM_ERR(EFAULT);				\
+			return EFAULT;				\
 	}
 
-	version.version_major		= dev->driver.major;
-	version.version_minor		= dev->driver.minor;
-	version.version_patchlevel	= dev->driver.patchlevel;
+	version->version_major		= dev->driver.major;
+	version->version_minor		= dev->driver.minor;
+	version->version_patchlevel	= dev->driver.patchlevel;
 
-	DRM_COPY(version.name, dev->driver.name);
-	DRM_COPY(version.date, dev->driver.date);
-	DRM_COPY(version.desc, dev->driver.desc);
-
-	DRM_COPY_TO_USER_IOCTL( (drm_version_t *)data, version, sizeof(version) );
+	DRM_COPY(version->name, dev->driver.name);
+	DRM_COPY(version->date, dev->driver.date);
+	DRM_COPY(version->desc, dev->driver.desc);
 
 	return 0;
 }
 
 int drm_open(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 {
-	drm_device_t *dev = NULL;
+	struct drm_device *dev = NULL;
 	int retcode = 0;
 
 	dev = DRIVER_SOFTC(minor(kdev));
@@ -695,27 +730,26 @@ int drm_open(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 
 int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 {
-	drm_file_t *priv;
-	DRM_DEVICE;
+	struct drm_device *dev = drm_get_device_from_kdev(kdev);
+	drm_file_t *file_priv;
 	int retcode = 0;
-	DRMFILE filp = (void *)(uintptr_t)(DRM_CURRENTPID);
-	
+
 	DRM_DEBUG( "open_count = %d\n", dev->open_count );
 
 	DRM_LOCK();
 
-	priv = drm_find_file_by_proc(dev, p);
-	if (!priv) {
+	file_priv = drm_find_file_by_proc(dev, p);
+	if (!file_priv) {
 		DRM_UNLOCK();
 		DRM_ERROR("can't find authenticator\n");
 		return EINVAL;
 	}
 
-	if (--priv->refs != 0)
+	if (--file_priv->refs != 0)
 		goto done;
 
 	if (dev->driver.preclose != NULL)
-		dev->driver.preclose(dev, filp);
+		dev->driver.preclose(dev, file_priv);
 
 	/* ========================================================
 	 * Begin inline drm_release
@@ -730,12 +764,12 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 #endif
 
 	if (dev->lock.hw_lock && _DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock)
-	    && dev->lock.filp == filp) {
+	    && dev->lock.file_priv == file_priv) {
 		DRM_DEBUG("Process %d dead, freeing lock for context %d\n",
 			  DRM_CURRENTPID,
 			  _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock));
 		if (dev->driver.reclaim_buffers_locked != NULL)
-			dev->driver.reclaim_buffers_locked(dev, filp);
+			dev->driver.reclaim_buffers_locked(dev, file_priv);
 
 		drm_lock_free(dev, &dev->lock.hw_lock->lock,
 		    _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock));
@@ -750,19 +784,19 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 		for (;;) {
 			if ( !dev->lock.hw_lock ) {
 				/* Device has been unregistered */
-				retcode = DRM_ERR(EINTR);
+				retcode = EINTR;
 				break;
 			}
 			if (drm_lock_take(&dev->lock.hw_lock->lock,
 			    DRM_KERNEL_CONTEXT)) {
-				dev->lock.filp = filp;
+				dev->lock.file_priv = file_priv;
 				dev->lock.lock_time = jiffies;
                                 atomic_inc( &dev->counts[_DRM_STAT_LOCKS] );
 				break;	/* Got lock */
 			}
 				/* Contention */
 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
-			retcode = msleep((void *)&dev->lock.lock_queue,
+			retcode = mtx_sleep((void *)&dev->lock.lock_queue,
 			    &dev->dev_lock, PZERO | PCATCH, "drmlk2", 0);
 #else
 			retcode = tsleep((void *)&dev->lock.lock_queue,
@@ -772,14 +806,14 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 				break;
 		}
 		if (retcode == 0) {
-			dev->driver.reclaim_buffers_locked(dev, filp);
+			dev->driver.reclaim_buffers_locked(dev, file_priv);
 			drm_lock_free(dev, &dev->lock.hw_lock->lock,
 			    DRM_KERNEL_CONTEXT);
 		}
 	}
 
 	if (dev->driver.use_dma && !dev->driver.reclaim_buffers_locked)
-		drm_reclaim_buffers(dev, filp);
+		drm_reclaim_buffers(dev, file_priv);
 
 #if defined (__FreeBSD__) && (__FreeBSD_version >= 500000)
 	funsetown(&dev->buf_sigio);
@@ -790,16 +824,15 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 #endif /* __NetBSD__  || __OpenBSD__ */
 
 	if (dev->driver.postclose != NULL)
-		dev->driver.postclose(dev, priv);
-
-	TAILQ_REMOVE(&dev->files, priv, link);
-	free(priv, M_DRM);
+		dev->driver.postclose(dev, file_priv);
+	TAILQ_REMOVE(&dev->files, file_priv, link);
+	free(file_priv, M_DRM);
 
 	/* ========================================================
 	 * End inline drm_release
 	 */
 
-	done:
+done:
 	atomic_inc( &dev->counts[_DRM_STAT_CLOSES] );
 #ifdef __FreeBSD__
 	device_unbusy(dev->device);
@@ -818,32 +851,33 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 int drm_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int flags, 
     DRM_STRUCTPROC *p)
 {
-	DRM_DEVICE;
+	struct drm_device *dev = drm_get_device_from_kdev(kdev);
 	int retcode = 0;
 	drm_ioctl_desc_t *ioctl;
-	int (*func)(DRM_IOCTL_ARGS);
+	int (*func)(struct drm_device *dev, void *data, struct drm_file *file_priv);
 	int nr = DRM_IOCTL_NR(cmd);
 	int is_driver_ioctl = 0;
-	drm_file_t *priv;
-	DRMFILE filp = (DRMFILE)(uintptr_t)DRM_CURRENTPID;
+	drm_file_t *file_priv;
 
 	DRM_LOCK();
-	priv = drm_find_file_by_proc(dev, p);
+	file_priv = drm_find_file_by_proc(dev, p);
 	DRM_UNLOCK();
-	if (priv == NULL) {
+	if (file_priv == NULL) {
 		DRM_ERROR("can't find authenticator\n");
 		return EINVAL;
 	}
 
 	atomic_inc( &dev->counts[_DRM_STAT_IOCTLS] );
-	++priv->ioctl_count;
+	++file_priv->ioctl_count;
 
 #ifdef __FreeBSD__
 	DRM_DEBUG( "pid=%d, cmd=0x%02lx, nr=0x%02x, dev 0x%lx, auth=%d\n",
-		 DRM_CURRENTPID, cmd, nr, (long)dev->device, priv->authenticated );
+	    DRM_CURRENTPID, cmd, nr, (long)dev->device,
+	    file_priv->authenticated );
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
 	DRM_DEBUG( "pid=%d, cmd=0x%02lx, nr=0x%02x, dev 0x%lx, auth=%d\n",
-		 DRM_CURRENTPID, cmd, nr, (long)&dev->device, priv->authenticated );
+	    DRM_CURRENTPID, cmd, nr, (long)&dev->device,
+	    file_priv->authenticated );
 #endif
 
 	switch (cmd) {
@@ -898,26 +932,39 @@ int drm_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int flags,
 		DRM_DEBUG( "no function\n" );
 		return EINVAL;
 	}
-	/* ioctl->master check should be against something in the filp set up
-	 * for the first opener, but it doesn't matter yet.
-	 */
+
 	if (((ioctl->flags & DRM_ROOT_ONLY) && !DRM_SUSER(p)) ||
-	    ((ioctl->flags & DRM_AUTH) && !priv->authenticated) ||
-	    ((ioctl->flags & DRM_MASTER) && !priv->master))
+	    ((ioctl->flags & DRM_AUTH) && !file_priv->authenticated) ||
+	    ((ioctl->flags & DRM_MASTER) && !file_priv->master))
 		return EACCES;
 
-	if (is_driver_ioctl)
+	if (is_driver_ioctl) {
 		DRM_LOCK();
-	retcode = func(kdev, cmd, data, flags, p, filp);
-	if (is_driver_ioctl)
+		/* shared code returns -errno */
+		retcode = -func(dev, data, file_priv);
 		DRM_UNLOCK();
+	} else {
+		retcode = func(dev, data, file_priv);
+	}
 
 	if (retcode != 0)
 		DRM_DEBUG("    returning %d\n", retcode);
 
-	return DRM_ERR(retcode);
+	return retcode;
 }
 
+drm_local_map_t *drm_getsarea(struct drm_device *dev)
+{
+	drm_local_map_t *map;
+
+	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
+	TAILQ_FOREACH(map, &dev->maplist, link) {
+		if (map->type == _DRM_SHM && (map->flags & _DRM_CONTAINS_LOCK))
+			return map;
+	}
+
+	return NULL;
+}
 
 #if DRM_LINUX
 
diff --git a/sys/dev/drm/drm_fops.c b/sys/dev/drm/drm_fops.c
index 361f5a7..23cc1c0 100644
--- a/sys/dev/drm/drm_fops.c
+++ b/sys/dev/drm/drm_fops.c
@@ -1,6 +1,3 @@
-/* drm_fops.h -- File operations for DRM -*- linux-c -*-
- * Created: Mon Jan  4 08:58:31 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -35,9 +32,14 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_fops.c
+ * Support code for dealing with the file privates associated with each
+ * open of the DRM device.
+ */
+
 #include "dev/drm/drmP.h"
 
-drm_file_t *drm_find_file_by_proc(drm_device_t *dev, DRM_STRUCTPROC *p)
+drm_file_t *drm_find_file_by_proc(struct drm_device *dev, DRM_STRUCTPROC *p)
 {
 #if __FreeBSD_version >= 500021
 	uid_t uid = p->td_ucred->cr_svuid;
@@ -58,7 +60,7 @@ drm_file_t *drm_find_file_by_proc(drm_device_t *dev, DRM_STRUCTPROC *p)
 
 /* drm_open_helper is called whenever a process opens /dev/drm. */
 int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
-		    drm_device_t *dev)
+		    struct drm_device *dev)
 {
 	int	     m = minor(kdev);
 	drm_file_t   *priv;
@@ -78,7 +80,7 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		priv = malloc(sizeof(*priv), M_DRM, M_NOWAIT | M_ZERO);
 		if (priv == NULL) {
 			DRM_UNLOCK();
-			return DRM_ERR(ENOMEM);
+			return ENOMEM;
 		}
 #if __FreeBSD_version >= 500000
 		priv->uid		= p->td_ucred->cr_svuid;
@@ -96,7 +98,8 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		priv->authenticated	= DRM_SUSER(p);
 
 		if (dev->driver.open) {
-			retcode = dev->driver.open(dev, priv);
+			/* shared code returns -errno */
+			retcode = -dev->driver.open(dev, priv);
 			if (retcode != 0) {
 				free(priv, M_DRM);
 				DRM_UNLOCK();
diff --git a/sys/dev/drm/drm_internal.h b/sys/dev/drm/drm_internal.h
new file mode 100644
index 0000000..0ed1b6f
--- /dev/null
+++ b/sys/dev/drm/drm_internal.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright 2007 Red Hat, Inc
+ * All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/* This header file holds function prototypes and data types that are
+ * internal to the drm (not exported to user space) but shared across
+ * drivers and platforms */
+
+#ifndef __DRM_INTERNAL_H__
+#define __DRM_INTERNAL_H__
+
+/**
+ * Drawable information.
+ */
+struct drm_drawable_info {
+	unsigned int num_rects;
+	struct drm_clip_rect *rects;
+};
+
+#endif
diff --git a/sys/dev/drm/drm_ioctl.c b/sys/dev/drm/drm_ioctl.c
index 0ea2197..2d708d8 100644
--- a/sys/dev/drm/drm_ioctl.c
+++ b/sys/dev/drm/drm_ioctl.c
@@ -1,6 +1,3 @@
-/* drm_ioctl.h -- IOCTL processing for DRM -*- linux-c -*-
- * Created: Fri Jan  8 09:01:26 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,11 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_ioctl.c
+ * Varios minor DRM ioctls not applicable to other files, such as versioning
+ * information and reporting DRM information to userland.
+ */
+
 #include "dev/drm/drmP.h"
 
 /*
@@ -42,20 +44,16 @@ __FBSDID("$FreeBSD$");
  * before setunique has been called.  The format for the bus-specific part of
  * the unique is not defined for any other bus.
  */
-int drm_getunique(DRM_IOCTL_ARGS)
+int drm_getunique(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_unique_t	 u;
-
-	DRM_COPY_FROM_USER_IOCTL( u, (drm_unique_t *)data, sizeof(u) );
+	drm_unique_t	 *u = data;
 
-	if (u.unique_len >= dev->unique_len) {
-		if (DRM_COPY_TO_USER(u.unique, dev->unique, dev->unique_len))
-			return DRM_ERR(EFAULT);
+	if (u->unique_len >= dev->unique_len) {
+		if (DRM_COPY_TO_USER(u->unique, dev->unique, dev->unique_len))
+			return EFAULT;
 	}
-	u.unique_len = dev->unique_len;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_unique_t *)data, u, sizeof(u) );
+	u->unique_len = dev->unique_len;
 
 	return 0;
 }
@@ -63,28 +61,26 @@ int drm_getunique(DRM_IOCTL_ARGS)
 /* Deprecated in DRM version 1.1, and will return EBUSY when setversion has
  * requested version 1.1 or greater.
  */
-int drm_setunique(DRM_IOCTL_ARGS)
+int drm_setunique(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_unique_t u;
+	drm_unique_t *u = data;
 	int domain, bus, slot, func, ret;
 	char *busid;
 
-	DRM_COPY_FROM_USER_IOCTL( u, (drm_unique_t *)data, sizeof(u) );
-
 	/* Check and copy in the submitted Bus ID */
-	if (!u.unique_len || u.unique_len > 1024)
-		return DRM_ERR(EINVAL);
+	if (!u->unique_len || u->unique_len > 1024)
+		return EINVAL;
 
-	busid = malloc(u.unique_len + 1, M_DRM, M_WAITOK);
+	busid = malloc(u->unique_len + 1, M_DRM, M_WAITOK);
 	if (busid == NULL)
-		return DRM_ERR(ENOMEM);
+		return ENOMEM;
 
-	if (DRM_COPY_FROM_USER(busid, u.unique, u.unique_len)) {
+	if (DRM_COPY_FROM_USER(busid, u->unique, u->unique_len)) {
 		free(busid, M_DRM);
-		return DRM_ERR(EFAULT);
+		return EFAULT;
 	}
-	busid[u.unique_len] = '\0';
+	busid[u->unique_len] = '\0';
 
 	/* Return error if the busid submitted doesn't match the device's actual
 	 * busid.
@@ -92,7 +88,7 @@ int drm_setunique(DRM_IOCTL_ARGS)
 	ret = sscanf(busid, "PCI:%d:%d:%d", &bus, &slot, &func);
 	if (ret != 3) {
 		free(busid, M_DRM);
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}
 	domain = bus >> 8;
 	bus &= 0xff;
@@ -102,17 +98,17 @@ int drm_setunique(DRM_IOCTL_ARGS)
 	    (slot != dev->pci_slot) ||
 	    (func != dev->pci_func)) {
 		free(busid, M_DRM);
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}
 
 	/* Actually set the device's busid now. */
 	DRM_LOCK();
 	if (dev->unique_len || dev->unique) {
 		DRM_UNLOCK();
-		return DRM_ERR(EBUSY);
+		return EBUSY;
 	}
 
-	dev->unique_len = u.unique_len;
+	dev->unique_len = u->unique_len;
 	dev->unique = busid;
 	DRM_UNLOCK();
 
@@ -121,7 +117,7 @@ int drm_setunique(DRM_IOCTL_ARGS)
 
 
 static int
-drm_set_busid(drm_device_t *dev)
+drm_set_busid(struct drm_device *dev)
 {
 
 	DRM_LOCK();
@@ -146,32 +142,29 @@ drm_set_busid(drm_device_t *dev)
 	return 0;
 }
 
-int drm_getmap(DRM_IOCTL_ARGS)
+int drm_getmap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_map_t    map;
+	drm_map_t    *map = data;
 	drm_local_map_t    *mapinlist;
 	int          idx;
 	int	     i = 0;
 
-	DRM_COPY_FROM_USER_IOCTL( map, (drm_map_t *)data, sizeof(map) );
-
-	idx = map.offset;
+	idx = map->offset;
 
 	DRM_LOCK();
 	if (idx < 0) {
 		DRM_UNLOCK();
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 	}
 
 	TAILQ_FOREACH(mapinlist, &dev->maplist, link) {
 		if (i==idx) {
-			map.offset = mapinlist->offset;
-			map.size   = mapinlist->size;
-			map.type   = mapinlist->type;
-			map.flags  = mapinlist->flags;
-			map.handle = mapinlist->handle;
-			map.mtrr   = mapinlist->mtrr;
+			map->offset = mapinlist->offset;
+			map->size   = mapinlist->size;
+			map->type   = mapinlist->type;
+			map->flags  = mapinlist->flags;
+			map->handle = mapinlist->handle;
+			map->mtrr   = mapinlist->mtrr;
 			break;
 		}
 		i++;
@@ -182,100 +175,91 @@ int drm_getmap(DRM_IOCTL_ARGS)
  	if (mapinlist == NULL)
 		return EINVAL;
 
-	DRM_COPY_TO_USER_IOCTL( (drm_map_t *)data, map, sizeof(map) );
-
 	return 0;
 }
 
-int drm_getclient(DRM_IOCTL_ARGS)
+int drm_getclient(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_client_t client;
+	drm_client_t *client = data;
 	drm_file_t   *pt;
 	int          idx;
 	int          i = 0;
 
-	DRM_COPY_FROM_USER_IOCTL( client, (drm_client_t *)data, sizeof(client) );
-
-	idx = client.idx;
+	idx = client->idx;
 	DRM_LOCK();
 	TAILQ_FOREACH(pt, &dev->files, link) {
 		if (i==idx)
 		{
-			client.auth  = pt->authenticated;
-			client.pid   = pt->pid;
-			client.uid   = pt->uid;
-			client.magic = pt->magic;
-			client.iocs  = pt->ioctl_count;
+			client->auth  = pt->authenticated;
+			client->pid   = pt->pid;
+			client->uid   = pt->uid;
+			client->magic = pt->magic;
+			client->iocs  = pt->ioctl_count;
 			DRM_UNLOCK();
-
-			*(drm_client_t *)data = client;
 			return 0;
 		}
 		i++;
 	}
 	DRM_UNLOCK();
 
-	DRM_COPY_TO_USER_IOCTL( (drm_client_t *)data, client, sizeof(client) );
-
-	return 0;
+	return EINVAL;
 }
 
-int drm_getstats(DRM_IOCTL_ARGS)
+int drm_getstats(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_stats_t  stats;
+	drm_stats_t  *stats = data;
 	int          i;
 
-	memset(&stats, 0, sizeof(stats));
+	memset(stats, 0, sizeof(drm_stats_t));
 	
 	DRM_LOCK();
 
 	for (i = 0; i < dev->counters; i++) {
 		if (dev->types[i] == _DRM_STAT_LOCK)
-			stats.data[i].value
+			stats->data[i].value
 				= (dev->lock.hw_lock
 				   ? dev->lock.hw_lock->lock : 0);
 		else 
-			stats.data[i].value = atomic_read(&dev->counts[i]);
-		stats.data[i].type  = dev->types[i];
+			stats->data[i].value = atomic_read(&dev->counts[i]);
+		stats->data[i].type  = dev->types[i];
 	}
 	
-	stats.count = dev->counters;
+	stats->count = dev->counters;
 
 	DRM_UNLOCK();
 
-	DRM_COPY_TO_USER_IOCTL( (drm_stats_t *)data, stats, sizeof(stats) );
-
 	return 0;
 }
 
 #define DRM_IF_MAJOR	1
 #define DRM_IF_MINOR	2
 
-int drm_setversion(DRM_IOCTL_ARGS)
+int drm_setversion(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_set_version_t sv;
-	drm_set_version_t retv;
+	drm_set_version_t *sv = data;
+	drm_set_version_t ver;
 	int if_version;
 
-	DRM_COPY_FROM_USER_IOCTL(sv, (drm_set_version_t *)data, sizeof(sv));
-
-	retv.drm_di_major = DRM_IF_MAJOR;
-	retv.drm_di_minor = DRM_IF_MINOR;
-	retv.drm_dd_major = dev->driver.major;
-	retv.drm_dd_minor = dev->driver.minor;
-
-	DRM_COPY_TO_USER_IOCTL((drm_set_version_t *)data, retv, sizeof(sv));
-
-	if (sv.drm_di_major != -1) {
-		if (sv.drm_di_major != DRM_IF_MAJOR ||
-		    sv.drm_di_minor < 0 || sv.drm_di_minor > DRM_IF_MINOR)
+	/* Save the incoming data, and set the response before continuing
+	 * any further.
+	 */
+	ver = *sv;
+	sv->drm_di_major = DRM_IF_MAJOR;
+	sv->drm_di_minor = DRM_IF_MINOR;
+	sv->drm_dd_major = dev->driver.major;
+	sv->drm_dd_minor = dev->driver.minor;
+
+	if (ver.drm_di_major != -1) {
+		if (ver.drm_di_major != DRM_IF_MAJOR ||
+		    ver.drm_di_minor < 0 || ver.drm_di_minor > DRM_IF_MINOR) {
 			return EINVAL;
-		if_version = DRM_IF_VERSION(sv.drm_di_major, sv.drm_dd_minor);
+		}
+		if_version = DRM_IF_VERSION(ver.drm_di_major,
+		    ver.drm_dd_minor);
 		dev->if_version = DRM_MAX(if_version, dev->if_version);
-		if (sv.drm_di_minor >= 1) {
+		if (ver.drm_di_minor >= 1) {
 			/*
 			 * Version 1.1 includes tying of DRM to specific device
 			 */
@@ -283,16 +267,20 @@ int drm_setversion(DRM_IOCTL_ARGS)
 		}
 	}
 
-	if (sv.drm_dd_major != -1) {
-		if (sv.drm_dd_major != dev->driver.major ||
-		    sv.drm_dd_minor < 0 || sv.drm_dd_minor > dev->driver.minor)
+	if (ver.drm_dd_major != -1) {
+		if (ver.drm_dd_major != dev->driver.major ||
+		    ver.drm_dd_minor < 0 ||
+		    ver.drm_dd_minor > dev->driver.minor)
+		{
 			return EINVAL;
+		}
 	}
+
 	return 0;
 }
 
 
-int drm_noop(DRM_IOCTL_ARGS)
+int drm_noop(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	DRM_DEBUG("\n");
 	return 0;
diff --git a/sys/dev/drm/drm_irq.c b/sys/dev/drm/drm_irq.c
index 580980e..0cb8c70 100644
--- a/sys/dev/drm/drm_irq.c
+++ b/sys/dev/drm/drm_irq.c
@@ -1,6 +1,3 @@
-/* drm_irq.c -- IRQ IOCTL and function support
- * Created: Fri Oct 18 2003 by anholt@FreeBSD.org
- */
 /*-
  * Copyright 2003 Eric Anholt
  * All Rights Reserved.
@@ -31,28 +28,31 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_irq.c
+ * Support code for handling setup/teardown of interrupt handlers and
+ * handing interrupt handlers off to the drivers.
+ */
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
-int drm_irq_by_busid(DRM_IOCTL_ARGS)
-{
-	DRM_DEVICE;
-	drm_irq_busid_t irq;
+static void drm_locked_task(void *context, int pending __unused);
 
-	DRM_COPY_FROM_USER_IOCTL(irq, (drm_irq_busid_t *)data, sizeof(irq));
+int drm_irq_by_busid(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
+{
+	drm_irq_busid_t *irq = data;
 
-	if ((irq.busnum >> 8) != dev->pci_domain ||
-	    (irq.busnum & 0xff) != dev->pci_bus ||
-	    irq.devnum != dev->pci_slot ||
-	    irq.funcnum != dev->pci_func)
+	if ((irq->busnum >> 8) != dev->pci_domain ||
+	    (irq->busnum & 0xff) != dev->pci_bus ||
+	    irq->devnum != dev->pci_slot ||
+	    irq->funcnum != dev->pci_func)
 		return EINVAL;
 
-	irq.irq = dev->irq;
+	irq->irq = dev->irq;
 
 	DRM_DEBUG("%d:%d:%d => IRQ %d\n",
-		  irq.busnum, irq.devnum, irq.funcnum, irq.irq);
-
-	DRM_COPY_TO_USER_IOCTL( (drm_irq_busid_t *)data, irq, sizeof(irq) );
+		  irq->busnum, irq->devnum, irq->funcnum, irq->irq);
 
 	return 0;
 }
@@ -61,7 +61,7 @@ int drm_irq_by_busid(DRM_IOCTL_ARGS)
 static irqreturn_t
 drm_irq_handler_wrap(DRM_IRQ_ARGS)
 {
-	drm_device_t *dev = (drm_device_t *)arg;
+	struct drm_device *dev = arg;
 
 	DRM_SPINLOCK(&dev->irq_lock);
 	dev->driver.irq_handler(arg);
@@ -69,7 +69,89 @@ drm_irq_handler_wrap(DRM_IRQ_ARGS)
 }
 #endif
 
-int drm_irq_install(drm_device_t *dev)
+static void vblank_disable_fn(void *arg)
+{
+	struct drm_device *dev = (struct drm_device *)arg;
+	int i;
+
+	if (callout_pending(&dev->vblank_disable_timer)) {
+		/* callout was reset */
+		return;
+	}
+	if (!callout_active(&dev->vblank_disable_timer)) {
+		/* callout was stopped */
+		return;
+	}
+	callout_deactivate(&dev->vblank_disable_timer);
+
+	if (!dev->vblank_disable_allowed)
+		return;
+
+	for (i = 0; i < dev->num_crtcs; i++) {
+		if (atomic_read(&dev->vblank[i].refcount) == 0 &&
+		    dev->vblank[i].enabled) {
+			DRM_DEBUG("disabling vblank on crtc %d\n", i);
+			dev->vblank[i].last =
+			    dev->driver.get_vblank_counter(dev, i);
+			dev->driver.disable_vblank(dev, i);
+			dev->vblank[i].enabled = 0;
+		}
+	}
+}
+
+static void drm_vblank_cleanup(struct drm_device *dev)
+{
+	unsigned long irqflags;
+
+	/* Bail if the driver didn't call drm_vblank_init() */
+	if (dev->num_crtcs == 0)
+	    return;
+
+	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+	callout_stop(&dev->vblank_disable_timer);
+	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+
+	callout_drain(&dev->vblank_disable_timer);
+
+	vblank_disable_fn((void *)dev);
+
+	drm_free(dev->vblank, sizeof(struct drm_vblank_info) * dev->num_crtcs,
+	    DRM_MEM_DRIVER);
+
+	dev->num_crtcs = 0;
+}
+
+int drm_vblank_init(struct drm_device *dev, int num_crtcs)
+{
+	int i, ret = ENOMEM;
+
+	callout_init_mtx(&dev->vblank_disable_timer, &dev->vbl_lock, 0);
+	atomic_set(&dev->vbl_signal_pending, 0);
+	dev->num_crtcs = num_crtcs;
+
+	dev->vblank = drm_calloc(num_crtcs, sizeof(struct drm_vblank_info),
+	    DRM_MEM_DRIVER);
+	if (!dev->vblank)
+	    goto err;
+
+	/* Zero per-crtc vblank stuff */
+	for (i = 0; i < num_crtcs; i++) {
+		DRM_INIT_WAITQUEUE(&dev->vblank[i].queue);
+		TAILQ_INIT(&dev->vblank[i].sigs);
+		atomic_set(&dev->vblank[i].count, 0);
+		atomic_set(&dev->vblank[i].refcount, 0);
+	}
+
+	dev->vblank_disable_allowed = 0;
+
+	return 0;
+
+err:
+	drm_vblank_cleanup(dev);
+	return ret;
+}
+
+int drm_irq_install(struct drm_device *dev)
 {
 	int retcode;
 #ifdef __NetBSD__
@@ -77,21 +159,19 @@ int drm_irq_install(drm_device_t *dev)
 #endif
 
 	if (dev->irq == 0 || dev->dev_private == NULL)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 
 	DRM_DEBUG( "%s: irq=%d\n", __FUNCTION__, dev->irq );
 
 	DRM_LOCK();
 	if (dev->irq_enabled) {
 		DRM_UNLOCK();
-		return DRM_ERR(EBUSY);
+		return EBUSY;
 	}
 	dev->irq_enabled = 1;
 
 	dev->context_flag = 0;
 
-	DRM_SPININIT(dev->irq_lock, "DRM IRQ lock");
-
 				/* Before installing handler */
 	dev->driver.irq_preinstall(dev);
 	DRM_UNLOCK();
@@ -105,12 +185,14 @@ int drm_irq_install(drm_device_t *dev)
 		retcode = ENOENT;
 		goto err;
 	}
-#if __FreeBSD_version < 500000
-	retcode = bus_setup_intr(dev->device, dev->irqr, INTR_TYPE_TTY,
-				 dev->irq_handler, dev, &dev->irqh);
-#else
-	retcode = bus_setup_intr(dev->device, dev->irqr, INTR_TYPE_TTY | INTR_MPSAFE,
+#if __FreeBSD_version >= 700031
+	retcode = bus_setup_intr(dev->device, dev->irqr,
+				 INTR_TYPE_TTY | INTR_MPSAFE,
 				 NULL, drm_irq_handler_wrap, dev, &dev->irqh);
+#else
+	retcode = bus_setup_intr(dev->device, dev->irqr,
+				 INTR_TYPE_TTY | INTR_MPSAFE,
+				 drm_irq_handler_wrap, dev, &dev->irqh);
 #endif
 	if (retcode != 0)
 		goto err;
@@ -132,6 +214,7 @@ int drm_irq_install(drm_device_t *dev)
 	dev->driver.irq_postinstall(dev);
 	DRM_UNLOCK();
 
+	TASK_INIT(&dev->locked_task, 0, drm_locked_task, dev);
 	return 0;
 err:
 	DRM_LOCK();
@@ -143,19 +226,18 @@ err:
 		dev->irqrid = 0;
 	}
 #endif
-	DRM_SPINUNINIT(dev->irq_lock);
 	DRM_UNLOCK();
 	return retcode;
 }
 
-int drm_irq_uninstall(drm_device_t *dev)
+int drm_irq_uninstall(struct drm_device *dev)
 {
 #ifdef __FreeBSD__
 	int irqrid;
 #endif
 
 	if (!dev->irq_enabled)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
 
 	dev->irq_enabled = 0;
 #ifdef __FreeBSD__
@@ -175,20 +257,17 @@ int drm_irq_uninstall(drm_device_t *dev)
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
 	pci_intr_disestablish(&dev->pa.pa_pc, dev->irqh);
 #endif
-	DRM_SPINUNINIT(dev->irq_lock);
+	drm_vblank_cleanup(dev);
 
 	return 0;
 }
 
-int drm_control(DRM_IOCTL_ARGS)
+int drm_control(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_control_t ctl;
+	drm_control_t *ctl = data;
 	int err;
 
-	DRM_COPY_FROM_USER_IOCTL( ctl, (drm_control_t *) data, sizeof(ctl) );
-
-	switch ( ctl.func ) {
+	switch ( ctl->func ) {
 	case DRM_INST_HANDLER:
 		/* Handle drivers whose DRM used to require IRQ setup but the
 		 * no longer does.
@@ -196,8 +275,8 @@ int drm_control(DRM_IOCTL_ARGS)
 		if (!dev->driver.use_irq)
 			return 0;
 		if (dev->if_version < DRM_IF_VERSION(1, 2) &&
-		    ctl.irq != dev->irq)
-			return DRM_ERR(EINVAL);
+		    ctl->irq != dev->irq)
+			return EINVAL;
 		return drm_irq_install(dev);
 	case DRM_UNINST_HANDLER:
 		if (!dev->driver.use_irq)
@@ -207,29 +286,170 @@ int drm_control(DRM_IOCTL_ARGS)
 		DRM_UNLOCK();
 		return err;
 	default:
-		return DRM_ERR(EINVAL);
+		return EINVAL;
+	}
+}
+
+u32 drm_vblank_count(struct drm_device *dev, int crtc)
+{
+	return atomic_read(&dev->vblank[crtc].count);
+}
+
+static void drm_update_vblank_count(struct drm_device *dev, int crtc)
+{
+	u32 cur_vblank, diff;
+
+	/*
+	 * Interrupts were disabled prior to this call, so deal with counter
+	 * wrap if needed.
+	 * NOTE!  It's possible we lost a full dev->max_vblank_count events
+	 * here if the register is small or we had vblank interrupts off for
+	 * a long time.
+	 */
+	cur_vblank = dev->driver.get_vblank_counter(dev, crtc);
+	diff = cur_vblank - dev->vblank[crtc].last;
+	if (cur_vblank < dev->vblank[crtc].last) {
+		diff += dev->max_vblank_count;
+
+		DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
+		    crtc, dev->vblank[crtc].last, cur_vblank, diff);
+	}
+
+	DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
+	    crtc, diff);
+
+	atomic_add(diff, &dev->vblank[crtc].count);
+}
+
+int drm_vblank_get(struct drm_device *dev, int crtc)
+{
+	unsigned long irqflags;
+	int ret = 0;
+
+	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+	/* Going from 0->1 means we have to enable interrupts again */
+	atomic_add_acq_int(&dev->vblank[crtc].refcount, 1);
+	if (dev->vblank[crtc].refcount == 1 &&
+	    !dev->vblank[crtc].enabled) {
+		ret = dev->driver.enable_vblank(dev, crtc);
+		if (ret)
+			atomic_dec(&dev->vblank[crtc].refcount);
+		else {
+			dev->vblank[crtc].enabled = 1;
+			drm_update_vblank_count(dev, crtc);
+		}
+	}
+	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+
+	return ret;
+}
+
+void drm_vblank_put(struct drm_device *dev, int crtc)
+{
+	unsigned long irqflags;
+
+	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+	/* Last user schedules interrupt disable */
+	atomic_subtract_acq_int(&dev->vblank[crtc].refcount, 1);
+	if (dev->vblank[crtc].refcount == 0)
+	    callout_reset(&dev->vblank_disable_timer, 5 * DRM_HZ,
+		(timeout_t *)vblank_disable_fn, (void *)dev);
+	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+}
+
+int drm_modeset_ctl(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
+{
+	struct drm_modeset_ctl *modeset = data;
+	unsigned long irqflags;
+	int crtc, ret = 0;
+
+	/* If drm_vblank_init() hasn't been called yet, just no-op */
+	if (!dev->num_crtcs)
+	    goto out;
+
+	crtc = modeset->crtc;
+	if (crtc >= dev->num_crtcs) {
+		ret = EINVAL;
+		goto out;
+	}
+
+	/*
+	 * To avoid all the problems that might happen if interrupts
+	 * were enabled/disabled around or between these calls, we just
+	 * have the kernel take a reference on the CRTC (just once though
+	 * to avoid corrupting the count if multiple, mismatch calls occur),
+	 * so that interrupts remain enabled in the interim.
+	 */
+	switch (modeset->cmd) {
+	case _DRM_PRE_MODESET:
+		if (!dev->vblank[crtc].inmodeset) {
+			dev->vblank[crtc].inmodeset = 1;
+			drm_vblank_get(dev, crtc);
+		}
+		break;
+	case _DRM_POST_MODESET:
+		if (dev->vblank[crtc].inmodeset) {
+			DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+			dev->vblank_disable_allowed = 1;
+			dev->vblank[crtc].inmodeset = 0;
+			DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+			drm_vblank_put(dev, crtc);
+		}
+		break;
+	default:
+		ret = EINVAL;
+		break;
 	}
+
+out:
+	return ret;
 }
 
-int drm_wait_vblank(DRM_IOCTL_ARGS)
+int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_wait_vblank_t vblwait;
-	struct timeval now;
-	int ret;
+	drm_wait_vblank_t *vblwait = data;
+	int ret = 0;
+	int flags, seq, crtc;
 
 	if (!dev->irq_enabled)
-		return DRM_ERR(EINVAL);
+		return EINVAL;
+
+	if (vblwait->request.type &
+	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK)) {
+		DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
+		    vblwait->request.type,
+		    (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK));
+		return EINVAL;
+	}
 
-	DRM_COPY_FROM_USER_IOCTL( vblwait, (drm_wait_vblank_t *)data,
-				  sizeof(vblwait) );
+	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
+	crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
+
+	if (crtc >= dev->num_crtcs)
+		return EINVAL;
 
-	if (vblwait.request.type & _DRM_VBLANK_RELATIVE) {
-		vblwait.request.sequence += atomic_read(&dev->vbl_received);
-		vblwait.request.type &= ~_DRM_VBLANK_RELATIVE;
+	ret = drm_vblank_get(dev, crtc);
+	if (ret)
+	    return ret;
+	seq = drm_vblank_count(dev, crtc);
+
+	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
+	case _DRM_VBLANK_RELATIVE:
+		vblwait->request.sequence += seq;
+		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
+	case _DRM_VBLANK_ABSOLUTE:
+		break;
+	default:
+		ret = EINVAL;
+		goto done;
+	}
+
+	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
+	    (seq - vblwait->request.sequence) <= (1<<23)) {
+		vblwait->request.sequence = seq + 1;
 	}
 
-	flags = vblwait.request.type & _DRM_VBLANK_FLAGS_MASK;
 	if (flags & _DRM_VBLANK_SIGNAL) {
 #if 0 /* disabled */
 		drm_vbl_sig_t *vbl_sig = malloc(sizeof(drm_vbl_sig_t), M_DRM,
@@ -237,40 +457,48 @@ int drm_wait_vblank(DRM_IOCTL_ARGS)
 		if (vbl_sig == NULL)
 			return ENOMEM;
 
-		vbl_sig->sequence = vblwait.request.sequence;
-		vbl_sig->signo = vblwait.request.signal;
+		vbl_sig->sequence = vblwait->request.sequence;
+		vbl_sig->signo = vblwait->request.signal;
 		vbl_sig->pid = DRM_CURRENTPID;
 
-		vblwait.reply.sequence = atomic_read(&dev->vbl_received);
+		vblwait->reply.sequence = atomic_read(&dev->vbl_received);
 		
-		DRM_SPINLOCK(&dev->irq_lock);
+		DRM_SPINLOCK(&dev->vbl_lock);
 		TAILQ_INSERT_HEAD(&dev->vbl_sig_list, vbl_sig, link);
-		DRM_SPINUNLOCK(&dev->irq_lock);
+		DRM_SPINUNLOCK(&dev->vbl_lock);
 		ret = 0;
 #endif
 		ret = EINVAL;
 	} else {
 		DRM_LOCK();
-		ret = dev->driver.vblank_wait(dev, &vblwait.request.sequence);
+		/* shared code returns -errno */
+
+		DRM_WAIT_ON(ret, dev->vblank[crtc].queue, 3 * DRM_HZ,
+		    ((drm_vblank_count(dev, crtc)
+		      - vblwait->request.sequence) <= (1 << 23)));
 		DRM_UNLOCK();
 
-		microtime(&now);
-		vblwait.reply.tval_sec = now.tv_sec;
-		vblwait.reply.tval_usec = now.tv_usec;
-	}
+		if (ret != EINTR) {
+			struct timeval now;
 
-	DRM_COPY_TO_USER_IOCTL( (drm_wait_vblank_t *)data, vblwait,
-				sizeof(vblwait) );
+			microtime(&now);
+			vblwait->reply.tval_sec = now.tv_sec;
+			vblwait->reply.tval_usec = now.tv_usec;
+			vblwait->reply.sequence = drm_vblank_count(dev, crtc);
+		}
+	}
 
+done:
+	drm_vblank_put(dev, crtc);
 	return ret;
 }
 
-void drm_vbl_send_signals(drm_device_t *dev)
+void drm_vbl_send_signals(struct drm_device *dev, int crtc)
 {
 }
 
 #if 0 /* disabled */
-void drm_vbl_send_signals( drm_device_t *dev )
+void drm_vbl_send_signals(struct drm_device *dev, int crtc )
 {
 	drm_vbl_sig_t *vbl_sig;
 	unsigned int vbl_seq = atomic_read( &dev->vbl_received );
@@ -292,3 +520,54 @@ void drm_vbl_send_signals( drm_device_t *dev )
 	}
 }
 #endif
+
+void drm_handle_vblank(struct drm_device *dev, int crtc)
+{
+	atomic_inc(&dev->vblank[crtc].count);
+	DRM_WAKEUP(&dev->vblank[crtc].queue);
+	drm_vbl_send_signals(dev, crtc);
+}
+
+static void drm_locked_task(void *context, int pending __unused)
+{
+	struct drm_device *dev = context;
+
+	DRM_SPINLOCK(&dev->tsk_lock);
+
+	DRM_LOCK(); /* XXX drm_lock_take() should do it's own locking */
+	if (dev->locked_task_call == NULL ||
+	    drm_lock_take(&dev->lock.hw_lock->lock, DRM_KERNEL_CONTEXT) == 0) {
+		DRM_UNLOCK();
+		DRM_SPINUNLOCK(&dev->tsk_lock);
+		return;
+	}
+
+	dev->lock.file_priv = NULL; /* kernel owned */
+	dev->lock.lock_time = jiffies;
+	atomic_inc(&dev->counts[_DRM_STAT_LOCKS]);
+
+	DRM_UNLOCK();
+
+	dev->locked_task_call(dev);
+
+	drm_lock_free(dev, &dev->lock.hw_lock->lock, DRM_KERNEL_CONTEXT);
+
+	dev->locked_task_call = NULL;
+
+	DRM_SPINUNLOCK(&dev->tsk_lock);
+}
+
+void
+drm_locked_tasklet(struct drm_device *dev,
+		   void (*tasklet)(struct drm_device *dev))
+{
+	DRM_SPINLOCK(&dev->tsk_lock);
+	if (dev->locked_task_call != NULL) {
+		DRM_SPINUNLOCK(&dev->tsk_lock);
+		return;
+	}
+
+	dev->locked_task_call = tasklet;
+	DRM_SPINUNLOCK(&dev->tsk_lock);
+	taskqueue_enqueue(taskqueue_swi, &dev->locked_task);
+}
diff --git a/sys/dev/drm/drm_lock.c b/sys/dev/drm/drm_lock.c
index b075fb4..c82ab9b 100644
--- a/sys/dev/drm/drm_lock.c
+++ b/sys/dev/drm/drm_lock.c
@@ -1,6 +1,3 @@
-/* lock.c -- IOCTLs for locking -*- linux-c -*-
- * Created: Tue Feb  2 08:37:54 1999 by faith@valinux.com
- */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,25 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_lock.c
+ * Implementation of the ioctls and other support code for dealing with the
+ * hardware lock.
+ *
+ * The DRM hardware lock is a shared structure between the kernel and userland.
+ *
+ * On uncontended access where the new context was the last context, the
+ * client may take the lock without dropping down into the kernel, using atomic
+ * compare-and-set.
+ *
+ * If the client finds during compare-and-set that it was not the last owner
+ * of the lock, it calls the DRM lock ioctl, which may sleep waiting for the
+ * lock, and may have side-effects of kernel-managed context switching.
+ *
+ * When the client releases the lock, if the lock is marked as being contended
+ * by another client, then the DRM unlock ioctl is called so that the
+ * contending client may be woken up.
+ */
+
 #include "dev/drm/drmP.h"
 
 int drm_lock_take(__volatile__ unsigned int *lock, unsigned int context)
@@ -64,12 +80,12 @@ int drm_lock_take(__volatile__ unsigned int *lock, unsigned int context)
 
 /* This takes a lock forcibly and hands it to context.	Should ONLY be used
    inside *_unlock to give lock to kernel before calling *_dma_schedule. */
-int drm_lock_transfer(drm_device_t *dev,
+int drm_lock_transfer(struct drm_device *dev,
 		       __volatile__ unsigned int *lock, unsigned int context)
 {
 	unsigned int old, new;
 
-	dev->lock.filp = NULL;
+	dev->lock.file_priv = NULL;
 	do {
 		old  = *lock;
 		new  = context | _DRM_LOCK_HELD;
@@ -78,12 +94,12 @@ int drm_lock_transfer(drm_device_t *dev,
 	return 1;
 }
 
-int drm_lock_free(drm_device_t *dev,
+int drm_lock_free(struct drm_device *dev,
 		   __volatile__ unsigned int *lock, unsigned int context)
 {
 	unsigned int old, new;
 
-	dev->lock.filp = NULL;
+	dev->lock.file_priv = NULL;
 	do {
 		old  = *lock;
 		new  = 0;
@@ -98,30 +114,28 @@ int drm_lock_free(drm_device_t *dev,
 	return 0;
 }
 
-int drm_lock(DRM_IOCTL_ARGS)
+int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-        drm_lock_t lock;
+        drm_lock_t *lock = data;
         int ret = 0;
 
-	DRM_COPY_FROM_USER_IOCTL(lock, (drm_lock_t *)data, sizeof(lock));
-
-        if (lock.context == DRM_KERNEL_CONTEXT) {
+        if (lock->context == DRM_KERNEL_CONTEXT) {
                 DRM_ERROR("Process %d using kernel context %d\n",
-		    DRM_CURRENTPID, lock.context);
+		    DRM_CURRENTPID, lock->context);
                 return EINVAL;
         }
 
         DRM_DEBUG("%d (pid %d) requests lock (0x%08x), flags = 0x%08x\n",
-	    lock.context, DRM_CURRENTPID, dev->lock.hw_lock->lock, lock.flags);
+	    lock->context, DRM_CURRENTPID, dev->lock.hw_lock->lock,
+	    lock->flags);
 
-        if (dev->driver.use_dma_queue && lock.context < 0)
+        if (dev->driver.use_dma_queue && lock->context < 0)
                 return EINVAL;
 
 	DRM_LOCK();
 	for (;;) {
-		if (drm_lock_take(&dev->lock.hw_lock->lock, lock.context)) {
-			dev->lock.filp = (void *)(uintptr_t)DRM_CURRENTPID;
+		if (drm_lock_take(&dev->lock.hw_lock->lock, lock->context)) {
+			dev->lock.file_priv = file_priv;
 			dev->lock.lock_time = jiffies;
 			atomic_inc(&dev->counts[_DRM_STAT_LOCKS]);
 			break;  /* Got lock */
@@ -129,7 +143,7 @@ int drm_lock(DRM_IOCTL_ARGS)
 
 		/* Contention */
 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
-		ret = msleep((void *)&dev->lock.lock_queue, &dev->dev_lock,
+		ret = mtx_sleep((void *)&dev->lock.lock_queue, &dev->dev_lock,
 		    PZERO | PCATCH, "drmlk2", 0);
 #else
 		ret = tsleep((void *)&dev->lock.lock_queue, PZERO | PCATCH,
@@ -139,7 +153,7 @@ int drm_lock(DRM_IOCTL_ARGS)
 			break;
 	}
 	DRM_UNLOCK();
-	DRM_DEBUG("%d %s\n", lock.context, ret ? "interrupted" : "has lock");
+	DRM_DEBUG("%d %s\n", lock->context, ret ? "interrupted" : "has lock");
 
 	if (ret != 0)
 		return ret;
@@ -147,24 +161,34 @@ int drm_lock(DRM_IOCTL_ARGS)
 	/* XXX: Add signal blocking here */
 
 	if (dev->driver.dma_quiescent != NULL &&
-	    (lock.flags & _DRM_LOCK_QUIESCENT))
+	    (lock->flags & _DRM_LOCK_QUIESCENT))
 		dev->driver.dma_quiescent(dev);
 
 	return 0;
 }
 
-int drm_unlock(DRM_IOCTL_ARGS)
+int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_lock_t lock;
-
-	DRM_COPY_FROM_USER_IOCTL(lock, (drm_lock_t *)data, sizeof(lock));
+	drm_lock_t *lock = data;
 
-	if (lock.context == DRM_KERNEL_CONTEXT) {
+	if (lock->context == DRM_KERNEL_CONTEXT) {
 		DRM_ERROR("Process %d using kernel context %d\n",
-		    DRM_CURRENTPID, lock.context);
+		    DRM_CURRENTPID, lock->context);
 		return EINVAL;
 	}
+	/* Check that the context unlock being requested actually matches
+	 * who currently holds the lock.
+	 */
+	if (!_DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock) ||
+	    _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock) != lock->context)
+		return EINVAL;
+
+	DRM_SPINLOCK(&dev->tsk_lock);
+	if (dev->locked_task_call != NULL) {
+		dev->locked_task_call(dev);
+		dev->locked_task_call = NULL;
+	}
+	DRM_SPINUNLOCK(&dev->tsk_lock);
 
 	atomic_inc(&dev->counts[_DRM_STAT_UNLOCKS]);
 
diff --git a/sys/dev/drm/drm_memory.c b/sys/dev/drm/drm_memory.c
index 6f03538..461d9fd 100644
--- a/sys/dev/drm/drm_memory.c
+++ b/sys/dev/drm/drm_memory.c
@@ -1,6 +1,3 @@
-/* drm_memory.h -- Memory management wrappers for DRM -*- linux-c -*-
- * Created: Thu Feb  4 14:00:34 1999 by faith@valinux.com
- */
 /*-
  *Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -34,6 +31,14 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_memory.c
+ * Wrappers for kernel memory allocation routines, and MTRR management support.
+ *
+ * This file previously implemented a memory consumption tracking system using
+ * the "area" argument for various different types of allocations, but that
+ * has been stripped out for now.
+ */
+
 #include "dev/drm/drmP.h"
 
 MALLOC_DEFINE(M_DRM, "drm", "DRM Data Structures");
@@ -78,7 +83,7 @@ void drm_free(void *pt, size_t size, int area)
 	free(pt, M_DRM);
 }
 
-void *drm_ioremap(drm_device_t *dev, drm_local_map_t *map)
+void *drm_ioremap(struct drm_device *dev, drm_local_map_t *map)
 {
 #ifdef __FreeBSD__
 	return pmap_mapdev(map->offset, map->size);
diff --git a/sys/dev/drm/drm_pci.c b/sys/dev/drm/drm_pci.c
index 0dca37b..a6ddfdf 100644
--- a/sys/dev/drm/drm_pci.c
+++ b/sys/dev/drm/drm_pci.c
@@ -1,10 +1,3 @@
-/**
- * \file drm_pci.h
- * \brief PCI consistent, DMA-accessible memory functions.
- *
- * \author Eric Anholt <anholt@FreeBSD.org>
- */
-
 /*-
  * Copyright 2003 Eric Anholt.
  * All Rights Reserved.
@@ -31,6 +24,13 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/**
+ * \file drm_pci.h
+ * \brief PCI consistent, DMA-accessible memory allocation.
+ *
+ * \author Eric Anholt <anholt@FreeBSD.org>
+ */
+
 #include "dev/drm/drmP.h"
 
 /**********************************************************************/
@@ -56,7 +56,8 @@ drm_pci_busdma_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error
  * memory block.
  */
 drm_dma_handle_t *
-drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
+drm_pci_alloc(struct drm_device *dev, size_t size,
+	      size_t align, dma_addr_t maxaddr)
 {
 	drm_dma_handle_t *dmah;
 	int ret;
@@ -73,6 +74,7 @@ drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
 		return NULL;
 
 #ifdef __FreeBSD__
+	DRM_UNLOCK();
 	ret = bus_dma_tag_create(NULL, align, 0, /* tag, align, boundary */
 	    maxaddr, BUS_SPACE_MAXADDR, /* lowaddr, highaddr */
 	    NULL, NULL, /* filtfunc, filtfuncargs */
@@ -81,6 +83,7 @@ drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
 	    &dmah->tag);
 	if (ret != 0) {
 		free(dmah, M_DRM);
+		DRM_LOCK();
 		return NULL;
 	}
 
@@ -89,9 +92,10 @@ drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
 	if (ret != 0) {
 		bus_dma_tag_destroy(dmah->tag);
 		free(dmah, M_DRM);
+		DRM_LOCK();
 		return NULL;
 	}
-
+	DRM_LOCK();
 	ret = bus_dmamap_load(dmah->tag, dmah->map, dmah->vaddr, size,
 	    drm_pci_busdma_callback, dmah, 0);
 	if (ret != 0) {
@@ -126,7 +130,7 @@ drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
  * \brief Free a DMA-accessible consistent memory block.
  */
 void
-drm_pci_free(drm_device_t *dev, drm_dma_handle_t *dmah)
+drm_pci_free(struct drm_device *dev, drm_dma_handle_t *dmah)
 {
 	if (dmah == NULL)
 		return;
diff --git a/sys/dev/drm/drm_pciids.h b/sys/dev/drm/drm_pciids.h
index 75f0e2d..a27785f 100644
--- a/sys/dev/drm/drm_pciids.h
+++ b/sys/dev/drm/drm_pciids.h
@@ -6,13 +6,13 @@
    Please contact dri-devel@lists.sf.net to add new cards to this list
 */
 #define radeon_PCI_IDS \
-	{0x1002, 0x3150, CHIP_RV380|CHIP_IS_MOBILITY, "ATI Radeon Mobility X600 M24"}, \
-	{0x1002, 0x3152, CHIP_RV380|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon Mobility X300 M24"}, \
-	{0x1002, 0x3154, CHIP_RV380|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI FireGL M24 GL"}, \
-	{0x1002, 0x3E50, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI Radeon RV380 X600"}, \
-	{0x1002, 0x3E54, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI FireGL V3200 RV380"}, \
-	{0x1002, 0x4136, CHIP_RS100|CHIP_IS_IGP, "ATI Radeon RS100 IGP 320"}, \
-	{0x1002, 0x4137, CHIP_RS200|CHIP_IS_IGP, "ATI Radeon RS200 IGP 340"}, \
+	{0x1002, 0x3150, CHIP_RV380|RADEON_IS_MOBILITY, "ATI Radeon Mobility X600 M24"}, \
+	{0x1002, 0x3152, CHIP_RV380|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon Mobility X300 M24"}, \
+	{0x1002, 0x3154, CHIP_RV380|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI FireGL M24 GL"}, \
+	{0x1002, 0x3E50, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI Radeon RV380 X600"}, \
+	{0x1002, 0x3E54, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI FireGL V3200 RV380"}, \
+	{0x1002, 0x4136, CHIP_RS100|RADEON_IS_IGP, "ATI Radeon RS100 IGP 320"}, \
+	{0x1002, 0x4137, CHIP_RS200|RADEON_IS_IGP, "ATI Radeon RS200 IGP 340"}, \
 	{0x1002, 0x4144, CHIP_R300, "ATI Radeon AD 9500"}, \
 	{0x1002, 0x4145, CHIP_R300, "ATI Radeon AE 9700 Pro"}, \
 	{0x1002, 0x4146, CHIP_R300, "ATI Radeon AF R300 9600TX"}, \
@@ -28,35 +28,35 @@
 	{0x1002, 0x4154, CHIP_RV350, "ATI FireGL AT T2"}, \
 	{0x1002, 0x4155, CHIP_RV350, "ATI Radeon 9650"}, \
 	{0x1002, 0x4156, CHIP_RV350, "ATI FireGL AV RV360 T2"}, \
-	{0x1002, 0x4237, CHIP_RS200|CHIP_IS_IGP, "ATI Radeon RS250 IGP"}, \
+	{0x1002, 0x4237, CHIP_RS200|RADEON_IS_IGP, "ATI Radeon RS250 IGP"}, \
 	{0x1002, 0x4242, CHIP_R200, "ATI Radeon BB R200 AIW 8500DV"}, \
 	{0x1002, 0x4243, CHIP_R200, "ATI Radeon BC R200"}, \
-	{0x1002, 0x4336, CHIP_RS100|CHIP_IS_IGP|CHIP_IS_MOBILITY, "ATI Radeon RS100 Mobility U1"}, \
-	{0x1002, 0x4337, CHIP_RS200|CHIP_IS_IGP|CHIP_IS_MOBILITY, "ATI Radeon RS200 Mobility IGP 340M"}, \
-	{0x1002, 0x4437, CHIP_RS200|CHIP_IS_IGP|CHIP_IS_MOBILITY, "ATI Radeon RS250 Mobility IGP"}, \
+	{0x1002, 0x4336, CHIP_RS100|RADEON_IS_IGP|RADEON_IS_MOBILITY, "ATI Radeon RS100 Mobility U1"}, \
+	{0x1002, 0x4337, CHIP_RS200|RADEON_IS_IGP|RADEON_IS_MOBILITY, "ATI Radeon RS200 Mobility IGP 340M"}, \
+	{0x1002, 0x4437, CHIP_RS200|RADEON_IS_IGP|RADEON_IS_MOBILITY, "ATI Radeon RS250 Mobility IGP"}, \
 	{0x1002, 0x4966, CHIP_RV250, "ATI Radeon If RV250 9000"}, \
 	{0x1002, 0x4967, CHIP_RV250, "ATI Radeon Ig RV250 9000"}, \
-	{0x1002, 0x4A48, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JH R420 X800"}, \
-	{0x1002, 0x4A49, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JI R420 X800 Pro"}, \
-	{0x1002, 0x4A4A, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JJ R420 X800 SE"}, \
-	{0x1002, 0x4A4B, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JK R420 X800 XT"}, \
-	{0x1002, 0x4A4C, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JL R420 X800"}, \
-	{0x1002, 0x4A4D, CHIP_R420|CHIP_NEW_MEMMAP, "ATI FireGL JM X3-256"}, \
-	{0x1002, 0x4A4E, CHIP_R420|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon JN R420 Mobility M18"}, \
-	{0x1002, 0x4A4F, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JO R420 X800 SE"}, \
-	{0x1002, 0x4A50, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JP R420 X800 XT PE"}, \
-	{0x1002, 0x4A54, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon JT R420 AIW X800 VE"}, \
-	{0x1002, 0x4B49, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R481 X850 XT"}, \
-	{0x1002, 0x4B4A, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R481 X850 SE"}, \
-	{0x1002, 0x4B4B, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R481 X850 Pro"}, \
-	{0x1002, 0x4B4C, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R481 X850 XT PE"}, \
-	{0x1002, 0x4C57, CHIP_RV200|CHIP_IS_MOBILITY, "ATI Radeon LW RV200 Mobility 7500 M7"}, \
-	{0x1002, 0x4C58, CHIP_RV200|CHIP_IS_MOBILITY, "ATI Radeon LX RV200 Mobility FireGL 7800 M7"}, \
-	{0x1002, 0x4C59, CHIP_RV100|CHIP_IS_MOBILITY, "ATI Radeon LY RV100 Mobility M6"}, \
-	{0x1002, 0x4C5A, CHIP_RV100|CHIP_IS_MOBILITY, "ATI Radeon LZ RV100 Mobility M6"}, \
-	{0x1002, 0x4C64, CHIP_RV250|CHIP_IS_MOBILITY, "ATI Radeon Ld RV250 Mobility 9000 M9"}, \
+	{0x1002, 0x4A48, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JH R420 X800"}, \
+	{0x1002, 0x4A49, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JI R420 X800 Pro"}, \
+	{0x1002, 0x4A4A, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JJ R420 X800 SE"}, \
+	{0x1002, 0x4A4B, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JK R420 X800 XT"}, \
+	{0x1002, 0x4A4C, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JL R420 X800"}, \
+	{0x1002, 0x4A4D, CHIP_R420|RADEON_NEW_MEMMAP, "ATI FireGL JM X3-256"}, \
+	{0x1002, 0x4A4E, CHIP_R420|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon JN R420 Mobility M18"}, \
+	{0x1002, 0x4A4F, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JO R420 X800 SE"}, \
+	{0x1002, 0x4A50, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JP R420 X800 XT PE"}, \
+	{0x1002, 0x4A54, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon JT R420 AIW X800 VE"}, \
+	{0x1002, 0x4B49, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R481 X850 XT"}, \
+	{0x1002, 0x4B4A, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R481 X850 SE"}, \
+	{0x1002, 0x4B4B, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R481 X850 Pro"}, \
+	{0x1002, 0x4B4C, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R481 X850 XT PE"}, \
+	{0x1002, 0x4C57, CHIP_RV200|RADEON_IS_MOBILITY, "ATI Radeon LW RV200 Mobility 7500 M7"}, \
+	{0x1002, 0x4C58, CHIP_RV200|RADEON_IS_MOBILITY, "ATI Radeon LX RV200 Mobility FireGL 7800 M7"}, \
+	{0x1002, 0x4C59, CHIP_RV100|RADEON_IS_MOBILITY, "ATI Radeon LY RV100 Mobility M6"}, \
+	{0x1002, 0x4C5A, CHIP_RV100|RADEON_IS_MOBILITY, "ATI Radeon LZ RV100 Mobility M6"}, \
+	{0x1002, 0x4C64, CHIP_RV250|RADEON_IS_MOBILITY, "ATI Radeon Ld RV250 Mobility 9000 M9"}, \
 	{0x1002, 0x4C66, CHIP_RV250, "ATI Radeon Lf RV250 Mobility 9000 M9 / FireMV 2400 PCI"}, \
-	{0x1002, 0x4C67, CHIP_RV250|CHIP_IS_MOBILITY, "ATI Radeon Lg RV250 Mobility 9000 M9"}, \
+	{0x1002, 0x4C67, CHIP_RV250|RADEON_IS_MOBILITY, "ATI Radeon Lg RV250 Mobility 9000 M9"}, \
 	{0x1002, 0x4E44, CHIP_R300, "ATI Radeon ND R300 9700 Pro"}, \
 	{0x1002, 0x4E45, CHIP_R300, "ATI Radeon NE R300 9500 Pro / 9700"}, \
 	{0x1002, 0x4E46, CHIP_R300, "ATI Radeon NF R300 9600TX"}, \
@@ -65,16 +65,16 @@
 	{0x1002, 0x4E49, CHIP_R350, "ATI Radeon NI R350 9800"}, \
 	{0x1002, 0x4E4A, CHIP_R350, "ATI Radeon NJ R360 9800 XT"}, \
 	{0x1002, 0x4E4B, CHIP_R350, "ATI FireGL NK X2"}, \
-	{0x1002, 0x4E50, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NP"}, \
-	{0x1002, 0x4E51, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NQ"}, \
-	{0x1002, 0x4E52, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M11 NR"}, \
-	{0x1002, 0x4E53, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NS"}, \
-	{0x1002, 0x4E54, CHIP_RV350|CHIP_IS_MOBILITY, "ATI FireGL T2/T2e"}, \
-	{0x1002, 0x4E56, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon Mobility 9550"}, \
-	{0x1002, 0x5144, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QD R100"}, \
-	{0x1002, 0x5145, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QE R100"}, \
-	{0x1002, 0x5146, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QF R100"}, \
-	{0x1002, 0x5147, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QG R100"}, \
+	{0x1002, 0x4E50, CHIP_RV350|RADEON_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NP"}, \
+	{0x1002, 0x4E51, CHIP_RV350|RADEON_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NQ"}, \
+	{0x1002, 0x4E52, CHIP_RV350|RADEON_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M11 NR"}, \
+	{0x1002, 0x4E53, CHIP_RV350|RADEON_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NS"}, \
+	{0x1002, 0x4E54, CHIP_RV350|RADEON_IS_MOBILITY, "ATI FireGL T2/T2e"}, \
+	{0x1002, 0x4E56, CHIP_RV350|RADEON_IS_MOBILITY, "ATI Radeon Mobility 9550"}, \
+	{0x1002, 0x5144, CHIP_R100|RADEON_SINGLE_CRTC, "ATI Radeon QD R100"}, \
+	{0x1002, 0x5145, CHIP_R100|RADEON_SINGLE_CRTC, "ATI Radeon QE R100"}, \
+	{0x1002, 0x5146, CHIP_R100|RADEON_SINGLE_CRTC, "ATI Radeon QF R100"}, \
+	{0x1002, 0x5147, CHIP_R100|RADEON_SINGLE_CRTC, "ATI Radeon QG R100"}, \
 	{0x1002, 0x5148, CHIP_R200, "ATI Radeon QH R200 8500"}, \
 	{0x1002, 0x514C, CHIP_R200, "ATI Radeon QL R200 8500 LE"}, \
 	{0x1002, 0x514D, CHIP_R200, "ATI Radeon QM R200 9100"}, \
@@ -83,59 +83,165 @@
 	{0x1002, 0x5159, CHIP_RV100, "ATI Radeon QY RV100 7000/VE"}, \
 	{0x1002, 0x515A, CHIP_RV100, "ATI Radeon QZ RV100 7000/VE"}, \
 	{0x1002, 0x515E, CHIP_RV100, "ATI ES1000 RN50"}, \
-	{0x1002, 0x5460, CHIP_RV380|CHIP_IS_MOBILITY, "ATI Radeon Mobility X300 M22"}, \
-	{0x1002, 0x5462, CHIP_RV380|CHIP_IS_MOBILITY, "ATI Radeon Mobility X600 SE M24C"}, \
-	{0x1002, 0x5464, CHIP_RV380|CHIP_IS_MOBILITY, "ATI FireGL M22 GL 5464"}, \
-	{0x1002, 0x5548, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R423 X800"}, \
-	{0x1002, 0x5549, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R423 X800 Pro"}, \
-	{0x1002, 0x554A, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R423 X800 XT PE"}, \
-	{0x1002, 0x554B, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R423 X800 SE"}, \
-	{0x1002, 0x554C, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R430 X800 XTP"}, \
-	{0x1002, 0x554D, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R430 X800 XL"}, \
-	{0x1002, 0x554E, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R430 X800 SE"}, \
-	{0x1002, 0x554F, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R430 X800"}, \
-	{0x1002, 0x5550, CHIP_R420|CHIP_NEW_MEMMAP, "ATI FireGL V7100 R423"}, \
-	{0x1002, 0x5551, CHIP_R420|CHIP_NEW_MEMMAP, "ATI FireGL V5100 R423 UQ"}, \
-	{0x1002, 0x5552, CHIP_R420|CHIP_NEW_MEMMAP, "ATI FireGL unknown R423 UR"}, \
-	{0x1002, 0x5554, CHIP_R420|CHIP_NEW_MEMMAP, "ATI FireGL unknown R423 UT"}, \
-	{0x1002, 0x564A, CHIP_RV410|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Mobility FireGL V5000 M26"}, \
-	{0x1002, 0x564B, CHIP_RV410|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Mobility FireGL V5000 M26"}, \
-	{0x1002, 0x564F, CHIP_RV410|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon Mobility X700 XL M26"}, \
-	{0x1002, 0x5652, CHIP_RV410|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon Mobility X700 M26"}, \
-	{0x1002, 0x5653, CHIP_RV410|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon Mobility X700 M26"}, \
-	{0x1002, 0x5834, CHIP_RS300|CHIP_IS_IGP, "ATI Radeon RS300 9100 IGP"}, \
-	{0x1002, 0x5835, CHIP_RS300|CHIP_IS_IGP|CHIP_IS_MOBILITY, "ATI Radeon RS300 Mobility IGP"}, \
+	{0x1002, 0x5460, CHIP_RV380|RADEON_IS_MOBILITY, "ATI Radeon Mobility X300 M22"}, \
+	{0x1002, 0x5462, CHIP_RV380|RADEON_IS_MOBILITY, "ATI Radeon Mobility X600 SE M24C"}, \
+	{0x1002, 0x5464, CHIP_RV380|RADEON_IS_MOBILITY, "ATI FireGL M22 GL 5464"}, \
+	{0x1002, 0x5548, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R423 X800"}, \
+	{0x1002, 0x5549, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R423 X800 Pro"}, \
+	{0x1002, 0x554A, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R423 X800 XT PE"}, \
+	{0x1002, 0x554B, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R423 X800 SE"}, \
+	{0x1002, 0x554C, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R430 X800 XTP"}, \
+	{0x1002, 0x554D, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R430 X800 XL"}, \
+	{0x1002, 0x554E, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R430 X800 SE"}, \
+	{0x1002, 0x554F, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R430 X800"}, \
+	{0x1002, 0x5550, CHIP_R420|RADEON_NEW_MEMMAP, "ATI FireGL V7100 R423"}, \
+	{0x1002, 0x5551, CHIP_R420|RADEON_NEW_MEMMAP, "ATI FireGL V5100 R423 UQ"}, \
+	{0x1002, 0x5552, CHIP_R420|RADEON_NEW_MEMMAP, "ATI FireGL unknown R423 UR"}, \
+	{0x1002, 0x5554, CHIP_R420|RADEON_NEW_MEMMAP, "ATI FireGL unknown R423 UT"}, \
+	{0x1002, 0x564A, CHIP_RV410|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V5000 M26"}, \
+	{0x1002, 0x564B, CHIP_RV410|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V5000 M26"}, \
+	{0x1002, 0x564F, CHIP_RV410|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon Mobility X700 XL M26"}, \
+	{0x1002, 0x5652, CHIP_RV410|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon Mobility X700 M26"}, \
+	{0x1002, 0x5653, CHIP_RV410|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon Mobility X700 M26"}, \
+	{0x1002, 0x5657, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon X550XTX"}, \
+	{0x1002, 0x5834, CHIP_RS300|RADEON_IS_IGP, "ATI Radeon RS300 9100 IGP"}, \
+	{0x1002, 0x5835, CHIP_RS300|RADEON_IS_IGP|RADEON_IS_MOBILITY, "ATI Radeon RS300 Mobility IGP"}, \
+	{0x1002, 0x5954, CHIP_RS480|RADEON_IS_IGP|RADEON_IS_IGPGART, "ATI RS480 XPRESS 200G"}, \
+	{0x1002, 0x5955, CHIP_RS480|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_IS_IGPGART, "ATI Radeon XPRESS 200M 5955"}, \
+	{0x1002, 0x5974, CHIP_RS480|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_IS_IGPGART, "ATI Radeon RS482 XPRESS 200"}, \
+	{0x1002, 0x5975, CHIP_RS480|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_IS_IGPGART, "ATI Radeon RS485 XPRESS 1100 IGP"}, \
 	{0x1002, 0x5960, CHIP_RV280, "ATI Radeon RV280 9250"}, \
 	{0x1002, 0x5961, CHIP_RV280, "ATI Radeon RV280 9200"}, \
 	{0x1002, 0x5962, CHIP_RV280, "ATI Radeon RV280 9200"}, \
 	{0x1002, 0x5964, CHIP_RV280, "ATI Radeon RV280 9200 SE"}, \
 	{0x1002, 0x5965, CHIP_RV280, "ATI FireMV 2200 PCI"}, \
 	{0x1002, 0x5969, CHIP_RV100, "ATI ES1000 RN50"}, \
-	{0x1002, 0x5b60, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI Radeon RV370 X300 SE"}, \
-	{0x1002, 0x5b62, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI Radeon RV370 X600 Pro"}, \
-	{0x1002, 0x5b63, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI Radeon RV370 X550"}, \
-	{0x1002, 0x5b64, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI FireGL V3100 (RV370) 5B64"}, \
-	{0x1002, 0x5b65, CHIP_RV380|CHIP_NEW_MEMMAP, "ATI FireMV 2200 PCIE (RV370) 5B65"}, \
-	{0x1002, 0x5c61, CHIP_RV280|CHIP_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
-	{0x1002, 0x5c63, CHIP_RV280|CHIP_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
-	{0x1002, 0x5d48, CHIP_R420|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Mobility Radeon X800 XT M28"}, \
-	{0x1002, 0x5d49, CHIP_R420|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Mobility FireGL V5100 M28"}, \
-	{0x1002, 0x5d4a, CHIP_R420|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Mobility Radeon X800 M28"}, \
-	{0x1002, 0x5d4c, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R480 X850"}, \
-	{0x1002, 0x5d4d, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R480 X850 XT PE"}, \
-	{0x1002, 0x5d4e, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R480 X850 SE"}, \
-	{0x1002, 0x5d4f, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R480 X850 Pro"}, \
-	{0x1002, 0x5d50, CHIP_R420|CHIP_NEW_MEMMAP, "ATI unknown Radeon / FireGL R480"}, \
-	{0x1002, 0x5d52, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R480 X850 XT"}, \
-	{0x1002, 0x5d57, CHIP_R420|CHIP_NEW_MEMMAP, "ATI Radeon R423 X800 XT"}, \
-	{0x1002, 0x5e48, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI FireGL V5000 RV410"}, \
-	{0x1002, 0x5e4a, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI Radeon RV410 X700 XT"}, \
-	{0x1002, 0x5e4b, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI Radeon RV410 X700 Pro"}, \
-	{0x1002, 0x5e4c, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI Radeon RV410 X700 SE"}, \
-	{0x1002, 0x5e4d, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI Radeon RV410 X700"}, \
-	{0x1002, 0x5e4f, CHIP_RV410|CHIP_NEW_MEMMAP, "ATI Radeon RV410 X700 SE"}, \
-	{0x1002, 0x7834, CHIP_RS300|CHIP_IS_IGP|CHIP_NEW_MEMMAP, "ATI Radeon RS350 9000/9100 IGP"}, \
-	{0x1002, 0x7835, CHIP_RS300|CHIP_IS_IGP|CHIP_IS_MOBILITY|CHIP_NEW_MEMMAP, "ATI Radeon RS350 Mobility IGP"}, \
+	{0x1002, 0x5a41, CHIP_RS400|RADEON_IS_IGP|RADEON_IS_IGPGART, "ATI Radeon XPRESS 200 5A41 (PCIE)"}, \
+	{0x1002, 0x5a42, CHIP_RS400|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_IS_IGPGART, "ATI Radeon XPRESS 200M 5A42 (PCIE)"}, \
+	{0x1002, 0x5a61, CHIP_RS400|RADEON_IS_IGP|RADEON_IS_IGPGART, "ATI Radeon RC410 XPRESS 200"}, \
+	{0x1002, 0x5a62, CHIP_RS400|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_IS_IGPGART, "ATI Radeon RC410 XPRESS 200M"}, \
+	{0x1002, 0x5b60, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI Radeon RV370 X300 SE"}, \
+	{0x1002, 0x5b62, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI Radeon RV370 X600 Pro"}, \
+	{0x1002, 0x5b63, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI Radeon RV370 X550"}, \
+	{0x1002, 0x5b64, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI FireGL V3100 (RV370) 5B64"}, \
+	{0x1002, 0x5b65, CHIP_RV380|RADEON_NEW_MEMMAP, "ATI FireMV 2200 PCIE (RV370) 5B65"}, \
+	{0x1002, 0x5c61, CHIP_RV280|RADEON_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
+	{0x1002, 0x5c63, CHIP_RV280|RADEON_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
+	{0x1002, 0x5d48, CHIP_R420|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X800 XT M28"}, \
+	{0x1002, 0x5d49, CHIP_R420|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V5100 M28"}, \
+	{0x1002, 0x5d4a, CHIP_R420|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X800 M28"}, \
+	{0x1002, 0x5d4c, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R480 X850"}, \
+	{0x1002, 0x5d4d, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R480 X850 XT PE"}, \
+	{0x1002, 0x5d4e, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R480 X850 SE"}, \
+	{0x1002, 0x5d4f, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R480 X850 Pro"}, \
+	{0x1002, 0x5d50, CHIP_R420|RADEON_NEW_MEMMAP, "ATI unknown Radeon / FireGL R480"}, \
+	{0x1002, 0x5d52, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R480 X850 XT"}, \
+	{0x1002, 0x5d57, CHIP_R420|RADEON_NEW_MEMMAP, "ATI Radeon R423 X800 XT"}, \
+	{0x1002, 0x5e48, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI FireGL V5000 RV410"}, \
+	{0x1002, 0x5e4a, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon RV410 X700 XT"}, \
+	{0x1002, 0x5e4b, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon RV410 X700 Pro"}, \
+	{0x1002, 0x5e4c, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon RV410 X700 SE"}, \
+	{0x1002, 0x5e4d, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon RV410 X700"}, \
+	{0x1002, 0x5e4f, CHIP_RV410|RADEON_NEW_MEMMAP, "ATI Radeon RV410 X700 SE"}, \
+	{0x1002, 0x7100, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x7101, CHIP_R520|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1800 XT"}, \
+	{0x1002, 0x7102, CHIP_R520|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1800"}, \
+	{0x1002, 0x7103, CHIP_R520|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V7200"}, \
+	{0x1002, 0x7104, CHIP_R520|RADEON_NEW_MEMMAP, "ATI FireGL V7200"}, \
+	{0x1002, 0x7105, CHIP_R520|RADEON_NEW_MEMMAP, "ATI FireGL V5300"}, \
+	{0x1002, 0x7106, CHIP_R520|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V7100"}, \
+	{0x1002, 0x7108, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x7109, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x710A, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x710B, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x710C, CHIP_R520|RADEON_NEW_MEMMAP, "ATI Radeon X1800"}, \
+	{0x1002, 0x710E, CHIP_R520|RADEON_NEW_MEMMAP, "ATI FireGL V7300"}, \
+	{0x1002, 0x710F, CHIP_R520|RADEON_NEW_MEMMAP, "ATI FireGL V7350"}, \
+	{0x1002, 0x7140, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x7141, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI RV505"}, \
+	{0x1002, 0x7142, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300/X1550"}, \
+	{0x1002, 0x7143, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1550"}, \
+	{0x1002, 0x7144, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI M54-GL"}, \
+	{0x1002, 0x7145, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1400"}, \
+	{0x1002, 0x7146, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300/X1550"}, \
+	{0x1002, 0x7147, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1550 64-bit"}, \
+	{0x1002, 0x7149, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1300"}, \
+	{0x1002, 0x714A, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1300"}, \
+	{0x1002, 0x714B, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1300"}, \
+	{0x1002, 0x714C, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1300"}, \
+	{0x1002, 0x714D, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300"}, \
+	{0x1002, 0x714E, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300"}, \
+	{0x1002, 0x714F, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI RV505"}, \
+	{0x1002, 0x7151, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI RV505"}, \
+	{0x1002, 0x7152, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI FireGL V3300"}, \
+	{0x1002, 0x7153, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI FireGL V3350"}, \
+	{0x1002, 0x715E, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300"}, \
+	{0x1002, 0x715F, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1550 64-bit"}, \
+	{0x1002, 0x7180, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300/X1550"}, \
+	{0x1002, 0x7181, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x7183, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300/X1550"}, \
+	{0x1002, 0x7186, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1450"}, \
+	{0x1002, 0x7187, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300/X1550"}, \
+	{0x1002, 0x7188, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X2300"}, \
+	{0x1002, 0x718A, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X2300"}, \
+	{0x1002, 0x718B, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1350"}, \
+	{0x1002, 0x718C, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1350"}, \
+	{0x1002, 0x718D, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1450"}, \
+	{0x1002, 0x718F, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1300"}, \
+	{0x1002, 0x7193, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1550"}, \
+	{0x1002, 0x7196, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1350"}, \
+	{0x1002, 0x719B, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI FireMV 2250"}, \
+	{0x1002, 0x719F, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI Radeon X1550 64-bit"}, \
+	{0x1002, 0x71C0, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x71C1, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1650"}, \
+	{0x1002, 0x71C2, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x71C3, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x71C4, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V5200"}, \
+	{0x1002, 0x71C5, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1600"}, \
+	{0x1002, 0x71C6, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1650"}, \
+	{0x1002, 0x71C7, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1650"}, \
+	{0x1002, 0x71CD, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1600"}, \
+	{0x1002, 0x71CE, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI Radeon X1300 XT/X1600 Pro"}, \
+	{0x1002, 0x71D2, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI FireGL V3400"}, \
+	{0x1002, 0x71D4, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility FireGL V5250"}, \
+	{0x1002, 0x71D5, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1700"}, \
+	{0x1002, 0x71D6, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1700 XT"}, \
+	{0x1002, 0x71DA, CHIP_RV530|RADEON_NEW_MEMMAP, "ATI FireGL V5200"}, \
+	{0x1002, 0x71DE, CHIP_RV530|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1700"}, \
+	{0x1002, 0x7200, CHIP_RV515|RADEON_NEW_MEMMAP, "ATI  Radeon X2300HD"}, \
+	{0x1002, 0x7210, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon HD 2300"}, \
+	{0x1002, 0x7211, CHIP_RV515|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon HD 2300"}, \
+	{0x1002, 0x7240, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1950"}, \
+	{0x1002, 0x7243, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7244, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1950"}, \
+	{0x1002, 0x7245, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7246, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7247, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7248, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7249, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x724A, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x724B, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x724C, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x724D, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x724E, CHIP_R580|RADEON_NEW_MEMMAP, "ATI AMD Stream Processor"}, \
+	{0x1002, 0x724F, CHIP_R580|RADEON_NEW_MEMMAP, "ATI Radeon X1900"}, \
+	{0x1002, 0x7280, CHIP_RV570|RADEON_NEW_MEMMAP, "ATI Radeon X1950"}, \
+	{0x1002, 0x7281, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI RV560"}, \
+	{0x1002, 0x7283, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI RV560"}, \
+	{0x1002, 0x7284, CHIP_R580|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon X1900"}, \
+	{0x1002, 0x7287, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI RV560"}, \
+	{0x1002, 0x7288, CHIP_RV570|RADEON_NEW_MEMMAP, "ATI Radeon X1950 GT"}, \
+	{0x1002, 0x7289, CHIP_RV570|RADEON_NEW_MEMMAP, "ATI RV570"}, \
+	{0x1002, 0x728B, CHIP_RV570|RADEON_NEW_MEMMAP, "ATI RV570"}, \
+	{0x1002, 0x728C, CHIP_RV570|RADEON_NEW_MEMMAP, "ATI ATI FireGL V7400"}, \
+	{0x1002, 0x7290, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI RV560"}, \
+	{0x1002, 0x7291, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI Radeon X1650"}, \
+	{0x1002, 0x7293, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI Radeon X1650"}, \
+	{0x1002, 0x7297, CHIP_RV560|RADEON_NEW_MEMMAP, "ATI RV560"}, \
+	{0x1002, 0x7834, CHIP_RS300|RADEON_IS_IGP|RADEON_NEW_MEMMAP, "ATI Radeon RS350 9000/9100 IGP"}, \
+	{0x1002, 0x7835, CHIP_RS300|RADEON_IS_IGP|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Radeon RS350 Mobility IGP"}, \
+	{0x1002, 0x791e, CHIP_RS690|RADEON_IS_IGP|RADEON_NEW_MEMMAP|RADEON_IS_IGPGART, "ATI Radeon RS690 X1250 IGP"}, \
+	{0x1002, 0x791f, CHIP_RS690|RADEON_IS_IGP|RADEON_NEW_MEMMAP|RADEON_IS_IGPGART, "ATI Radeon RS690 X1270 IGP"}, \
 	{0, 0, 0, NULL}
 
 #define r128_PCI_IDS \
@@ -212,7 +318,10 @@
 	{0x1039, 0x0300, 0, "SiS 300/305"}, \
 	{0x1039, 0x5300, 0, "SiS 540"}, \
 	{0x1039, 0x6300, 0, "SiS 630"}, \
+	{0x1039, 0x6330, SIS_CHIP_315, "SiS 661"}, \
 	{0x1039, 0x7300, 0, "SiS 730"}, \
+	{0x18CA, 0x0040, SIS_CHIP_315, "Volari V3XT/V5/V8"}, \
+	{0x18CA, 0x0042, SIS_CHIP_315, "Volari Unknown"}, \
 	{0, 0, 0, NULL}
 
 #define tdfx_PCI_IDS \
@@ -230,7 +339,11 @@
 	{0x1106, 0x3122, 0, "VIA CLE266"}, \
 	{0x1106, 0x7205, 0, "VIA KM400"}, \
 	{0x1106, 0x3108, 0, "VIA K8M800"}, \
-	{0x1106, 0x3344, 0, "VIA P4VM800PRO"}, \
+	{0x1106, 0x3344, 0, "VIA CN700 / VM800 / P4M800Pro"}, \
+	{0x1106, 0x3343, 0, "VIA P4M890"}, \
+	{0x1106, 0x3230, VIA_DX9_0, "VIA K8M890"}, \
+	{0x1106, 0x3157, VIA_PRO_GROUP_A, "VIA CX700"}, \
+	{0x1106, 0x3371, VIA_DX9_0, "VIA P4M900 / VN896"}, \
 	{0, 0, 0, NULL}
 
 #define i810_PCI_IDS \
@@ -281,20 +394,29 @@
 	{0, 0, 0, NULL}
 
 #define i915_PCI_IDS \
-	{0x8086, 0x3577, 0, "Intel i830M GMCH"}, \
-	{0x8086, 0x2562, 0, "Intel i845G GMCH"}, \
-	{0x8086, 0x3582, 0, "Intel i852GM/i855GM GMCH"}, \
-	{0x8086, 0x2572, 0, "Intel i865G GMCH"}, \
-	{0x8086, 0x2582, 0, "Intel i915G"}, \
-	{0x8086, 0x2592, 0, "Intel i915GM"}, \
-	{0x8086, 0x2772, 0, "Intel i945G"}, \
-	{0x8086, 0x27A2, 0, "Intel i945GM"}, \
-	{0x8086, 0x2972, 0, "Intel i946GZ"}, \
-	{0x8086, 0x2982, 0, "Intel i965G"}, \
-	{0x8086, 0x2992, 0, "Intel i965Q"}, \
-	{0x8086, 0x29A2, 0, "Intel i965G"}, \
-	{0x8086, 0x2A02, 0, "Intel i965GM"}, \
-	{0x8086, 0x2A12, 0, "Intel i965GME"}, \
+	{0x8086, 0x3577, CHIP_I8XX, "Intel i830M GMCH"}, \
+	{0x8086, 0x2562, CHIP_I8XX, "Intel i845G GMCH"}, \
+	{0x8086, 0x3582, CHIP_I8XX, "Intel i852GM/i855GM GMCH"}, \
+	{0x8086, 0x2572, CHIP_I8XX, "Intel i865G GMCH"}, \
+	{0x8086, 0x2582, CHIP_I9XX|CHIP_I915, "Intel i915G"}, \
+	{0x8086, 0x258a, CHIP_I9XX|CHIP_I915, "Intel E7221 (i915)"}, \
+	{0x8086, 0x2592, CHIP_I9XX|CHIP_I915, "Intel i915GM"}, \
+	{0x8086, 0x2772, CHIP_I9XX|CHIP_I915, "Intel i945G"}, \
+	{0x8086, 0x27A2, CHIP_I9XX|CHIP_I915, "Intel i945GM"}, \
+	{0x8086, 0x27AE, CHIP_I9XX|CHIP_I915, "Intel i945GME"}, \
+	{0x8086, 0x2972, CHIP_I9XX|CHIP_I965, "Intel i946GZ"}, \
+	{0x8086, 0x2982, CHIP_I9XX|CHIP_I965, "Intel i965G"}, \
+	{0x8086, 0x2992, CHIP_I9XX|CHIP_I965, "Intel i965Q"}, \
+	{0x8086, 0x29A2, CHIP_I9XX|CHIP_I965, "Intel i965G"}, \
+	{0x8086, 0x2A02, CHIP_I9XX|CHIP_I965, "Intel i965GM"}, \
+	{0x8086, 0x2A12, CHIP_I9XX|CHIP_I965, "Intel i965GME/GLE"}, \
+	{0x8086, 0x29C2, CHIP_I9XX|CHIP_I915, "Intel G33"}, \
+	{0x8086, 0x29B2, CHIP_I9XX|CHIP_I915, "Intel Q35"}, \
+	{0x8086, 0x29D2, CHIP_I9XX|CHIP_I915, "Intel Q33"}, \
+	{0x8086, 0x2A42, CHIP_I9XX|CHIP_I965, "Mobile Intel® GM45 Express Chipset"}, \
+	{0x8086, 0x2E02, CHIP_I9XX|CHIP_I965, "Intel Integrated Graphics Device"}, \
+	{0x8086, 0x2E12, CHIP_I9XX|CHIP_I965, "Intel Q45/Q43"}, \
+	{0x8086, 0x2E22, CHIP_I9XX|CHIP_I965, "Intel G45/G43"}, \
 	{0, 0, 0, NULL}
 
 #define imagine_PCI_IDS \
@@ -374,7 +496,7 @@
 	{0x10DE, 0x0309, NV30, "NVidia Quadro FX 1000"}, \
 	{0x10DE, 0x0311, NV30, "NVidia GeForce FX 5600 Ultra"}, \
 	{0x10DE, 0x0312, NV30, "NVidia GeForce FX 5600"}, \
-	{0x10DE, 0x0313, NV30, "NVidia 0x0313"},}, \
+	{0x10DE, 0x0313, NV30, "NVidia 0x0313"}, \
 	{0x10DE, 0x0314, NV30, "NVidia GeForce FX 5600SE"}, \
 	{0x10DE, 0x0316, NV30, "NVidia 0x0316"}, \
 	{0x10DE, 0x0317, NV30, "NVidia 0x0317"}, \
@@ -484,3 +606,8 @@
 	{0x10DE, 0x009D, NV40, "NVidia Quadro FX 4500"}, \
 	{0x10DE, 0x009E, NV40, "NVidia 0x009E"}, \
 	{0, 0, 0, NULL}
+
+#define xgi_PCI_IDS \
+	{0x18ca, 0x2200, 0, "XP5"}, \
+	{0x18ca, 0x0047, 0, "XP10 / XG47"}, \
+	{0, 0, 0, NULL}
diff --git a/sys/dev/drm/drm_sarea.h b/sys/dev/drm/drm_sarea.h
index 842d3ac..bf5b464 100644
--- a/sys/dev/drm/drm_sarea.h
+++ b/sys/dev/drm/drm_sarea.h
@@ -44,38 +44,44 @@ __FBSDID("$FreeBSD$");
 #define SAREA_MAX                       0x10000	/* 64kB */
 #else
 /* Intel 830M driver needs at least 8k SAREA */
-#define SAREA_MAX                       0x2000
+#define SAREA_MAX                       0x2000UL
 #endif
 
 /** Maximum number of drawables in the SAREA */
-#define SAREA_MAX_DRAWABLES 		256
+#define SAREA_MAX_DRAWABLES		256
 
 #define SAREA_DRAWABLE_CLAIMED_ENTRY    0x80000000
 
 /** SAREA drawable */
-typedef struct drm_sarea_drawable {
+struct drm_sarea_drawable {
 	unsigned int stamp;
 	unsigned int flags;
-} drm_sarea_drawable_t;
+};
 
 /** SAREA frame */
-typedef struct drm_sarea_frame {
+struct drm_sarea_frame {
 	unsigned int x;
 	unsigned int y;
 	unsigned int width;
 	unsigned int height;
 	unsigned int fullscreen;
-} drm_sarea_frame_t;
+};
 
 /** SAREA */
-typedef struct drm_sarea {
+struct drm_sarea {
     /** first thing is always the DRM locking structure */
-	drm_hw_lock_t lock;
+	struct drm_hw_lock lock;
     /** \todo Use readers/writer lock for drm_sarea::drawable_lock */
-	drm_hw_lock_t drawable_lock;
-	drm_sarea_drawable_t drawableTable[SAREA_MAX_DRAWABLES];	/**< drawables */
-	drm_sarea_frame_t frame;	/**< frame */
+	struct drm_hw_lock drawable_lock;
+	struct drm_sarea_drawable drawableTable[SAREA_MAX_DRAWABLES];	/**< drawables */
+	struct drm_sarea_frame frame;	/**< frame */
 	drm_context_t dummy_context;
-} drm_sarea_t;
+};
+
+#ifndef __KERNEL__
+typedef struct drm_sarea_drawable drm_sarea_drawable_t;
+typedef struct drm_sarea_frame drm_sarea_frame_t;
+typedef struct drm_sarea drm_sarea_t;
+#endif
 
 #endif				/* _DRM_SAREA_H_ */
diff --git a/sys/dev/drm/drm_scatter.c b/sys/dev/drm/drm_scatter.c
index 85b8936..deef1c8 100644
--- a/sys/dev/drm/drm_scatter.c
+++ b/sys/dev/drm/drm_scatter.c
@@ -1,5 +1,3 @@
-/* drm_scatter.h -- IOCTLs to manage scatter/gather memory -*- linux-c -*-
- * Created: Mon Dec 18 23:20:54 2000 by gareth@valinux.com */
 /*-
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
@@ -32,6 +30,13 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_scatter.c
+ * Allocation of memory for scatter-gather mappings by the graphics chip.
+ *
+ * The memory allocated here is then made into an aperture in the card
+ * by drm_ati_pcigart_init().
+ */
+
 #include "dev/drm/drmP.h"
 
 #define DEBUG_SCATTER 0
@@ -43,28 +48,21 @@ void drm_sg_cleanup(drm_sg_mem_t *entry)
 	free(entry, M_DRM);
 }
 
-int drm_sg_alloc(DRM_IOCTL_ARGS)
+int drm_sg_alloc(struct drm_device * dev, drm_scatter_gather_t * request)
 {
-	DRM_DEVICE;
-	drm_scatter_gather_t request;
 	drm_sg_mem_t *entry;
 	unsigned long pages;
 	int i;
 
-	DRM_DEBUG( "%s\n", __FUNCTION__ );
-
 	if ( dev->sg )
 		return EINVAL;
 
-	DRM_COPY_FROM_USER_IOCTL(request, (drm_scatter_gather_t *)data,
-			     sizeof(request) );
-
 	entry = malloc(sizeof(*entry), M_DRM, M_WAITOK | M_ZERO);
 	if ( !entry )
 		return ENOMEM;
 
-	pages = round_page(request.size) / PAGE_SIZE;
-	DRM_DEBUG( "sg size=%ld pages=%ld\n", request.size, pages );
+	pages = round_page(request->size) / PAGE_SIZE;
+	DRM_DEBUG( "sg size=%ld pages=%ld\n", request->size, pages );
 
 	entry->pages = pages;
 
@@ -89,11 +87,7 @@ int drm_sg_alloc(DRM_IOCTL_ARGS)
 	DRM_DEBUG( "sg alloc handle  = %08lx\n", entry->handle );
 
 	entry->virtual = (void *)entry->handle;
-	request.handle = entry->handle;
-
-	DRM_COPY_TO_USER_IOCTL( (drm_scatter_gather_t *)data,
-			   request,
-			   sizeof(request) );
+	request->handle = entry->handle;
 
 	DRM_LOCK();
 	if (dev->sg) {
@@ -107,21 +101,29 @@ int drm_sg_alloc(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-int drm_sg_free(DRM_IOCTL_ARGS)
+int drm_sg_alloc_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_scatter_gather_t request;
-	drm_sg_mem_t *entry;
+	drm_scatter_gather_t *request = data;
+	int ret;
+
+	DRM_DEBUG( "%s\n", __FUNCTION__ );
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_scatter_gather_t *)data,
-			     sizeof(request) );
+	ret = drm_sg_alloc(dev, request);
+	return ret;
+}
+
+int drm_sg_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_scatter_gather_t *request = data;
+	drm_sg_mem_t *entry;
 
 	DRM_LOCK();
 	entry = dev->sg;
 	dev->sg = NULL;
 	DRM_UNLOCK();
 
-	if ( !entry || entry->handle != request.handle )
+	if ( !entry || entry->handle != request->handle )
 		return EINVAL;
 
 	DRM_DEBUG( "sg free virtual  = 0x%lx\n", entry->handle );
diff --git a/sys/dev/drm/drm_sysctl.c b/sys/dev/drm/drm_sysctl.c
index eb5bbe2..5c4eacb 100644
--- a/sys/dev/drm/drm_sysctl.c
+++ b/sys/dev/drm/drm_sysctl.c
@@ -24,6 +24,11 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_sysctl.c
+ * Implementation of various sysctls for controlling DRM behavior and reporting
+ * debug information.
+ */
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
@@ -50,7 +55,7 @@ struct drm_sysctl_info {
 	char		       name[2];
 };
 
-int drm_sysctl_init(drm_device_t *dev)
+int drm_sysctl_init(struct drm_device *dev)
 {
 	struct drm_sysctl_info *info;
 	struct sysctl_oid *oid;
@@ -104,7 +109,7 @@ int drm_sysctl_init(drm_device_t *dev)
 	return 0;
 }
 
-int drm_sysctl_cleanup(drm_device_t *dev)
+int drm_sysctl_cleanup(struct drm_device *dev)
 {
 	int error;
 	error = sysctl_ctx_free( &dev->sysctl->ctx );
@@ -125,7 +130,7 @@ do {								\
 
 static int drm_name_info DRM_SYSCTL_HANDLER_ARGS
 {
-	drm_device_t *dev = arg1;
+	struct drm_device *dev = arg1;
 	char buf[128];
 	int retcode;
 	int hasunique = 0;
@@ -150,7 +155,7 @@ done:
 
 static int drm_vm_info DRM_SYSCTL_HANDLER_ARGS
 {
-	drm_device_t *dev = arg1;
+	struct drm_device *dev = arg1;
 	drm_local_map_t *map, *tempmaps;
 	const char   *types[] = { "FB", "REG", "SHM", "AGP", "SG" };
 	const char *type, *yesno;
@@ -209,7 +214,7 @@ done:
 
 static int drm_bufs_info DRM_SYSCTL_HANDLER_ARGS
 {
-	drm_device_t	 *dev = arg1;
+	struct drm_device	 *dev = arg1;
 	drm_device_dma_t *dma = dev->dma;
 	drm_device_dma_t tempdma;
 	int *templists;
@@ -265,7 +270,7 @@ done:
 
 static int drm_clients_info DRM_SYSCTL_HANDLER_ARGS
 {
-	drm_device_t *dev = arg1;
+	struct drm_device *dev = arg1;
 	drm_file_t *priv, *tempprivs;
 	char buf[128];
 	int retcode;
diff --git a/sys/dev/drm/drm_vm.c b/sys/dev/drm/drm_vm.c
index e01257a3..c602eec 100644
--- a/sys/dev/drm/drm_vm.c
+++ b/sys/dev/drm/drm_vm.c
@@ -24,6 +24,10 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/** @file drm_vm.c
+ * Support code for mmaping of DRM maps.
+ */
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
@@ -36,7 +40,7 @@ int drm_mmap(dev_t kdev, vm_offset_t offset, int prot)
 paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 #endif
 {
-	DRM_DEVICE;
+	struct drm_device *dev = drm_get_device_from_kdev(kdev);
 	drm_local_map_t *map;
 	drm_file_t *priv;
 	drm_map_type_t type;
@@ -55,7 +59,7 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 	}
 
 	if (!priv->authenticated)
-		return DRM_ERR(EACCES);
+		return EACCES;
 
 	if (dev->dma && offset >= 0 && offset < ptoa(dev->dma->page_count)) {
 		drm_device_dma_t *dma = dev->dma;
@@ -66,9 +70,9 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 			unsigned long page = offset >> PAGE_SHIFT;
 			unsigned long phys = dma->pagelist[page];
 
+			DRM_SPINUNLOCK(&dev->dma_lock);
 #if defined(__FreeBSD__) && __FreeBSD_version >= 500102
 			*paddr = phys;
-			DRM_SPINUNLOCK(&dev->dma_lock);
 			return 0;
 #else
 			return atop(phys);
@@ -77,7 +81,6 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 			DRM_SPINUNLOCK(&dev->dma_lock);
 			return -1;
 		}
-		DRM_SPINUNLOCK(&dev->dma_lock);
 	}
 
 				/* A sequential search of a linked list is
diff --git a/sys/dev/drm/i915_dma.c b/sys/dev/drm/i915_dma.c
index a5e7da9..8cfdfb0 100644
--- a/sys/dev/drm/i915_dma.c
+++ b/sys/dev/drm/i915_dma.c
@@ -3,7 +3,7 @@
 /*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
@@ -11,11 +11,11 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
@@ -23,7 +23,7 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- * 
+ *
  */
 
 #include <sys/cdefs.h>
@@ -34,61 +34,51 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/i915_drm.h"
 #include "dev/drm/i915_drv.h"
 
-#define IS_I965G(dev)  (dev->pci_device == 0x2972 || \
-			dev->pci_device == 0x2982 || \
-			dev->pci_device == 0x2992 || \
-			dev->pci_device == 0x29A2 || \
-			dev->pci_device == 0x2A12 )
-
-
 /* Really want an OS-independent resettable timer.  Would like to have
  * this loop run for (eg) 3 sec, but have the timer reset every time
  * the head pointer changes, so that EBUSY only happens if the ring
  * actually stalls for (eg) 3 seconds.
  */
-int i915_wait_ring(drm_device_t * dev, int n, const char *caller)
+int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
-	u32 last_head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
+	u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 	int i;
 
 	for (i = 0; i < 10000; i++) {
-		ring->head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
+		ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
 		ring->space = ring->head - (ring->tail + 8);
 		if (ring->space < 0)
 			ring->space += ring->Size;
 		if (ring->space >= n)
 			return 0;
 
-		dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
-
 		if (ring->head != last_head)
 			i = 0;
 
 		last_head = ring->head;
+		DRM_UDELAY(1);
 	}
 
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
-void i915_kernel_lost_context(drm_device_t * dev)
+void i915_kernel_lost_context(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
 
-	ring->head = I915_READ(LP_RING + RING_HEAD) & HEAD_ADDR;
-	ring->tail = I915_READ(LP_RING + RING_TAIL) & TAIL_ADDR;
+	ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+	ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
 	ring->space = ring->head - (ring->tail + 8);
 	if (ring->space < 0)
 		ring->space += ring->Size;
-
-	if (ring->head == ring->tail)
-		dev_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY;
 }
 
-static int i915_dma_cleanup(drm_device_t * dev)
+static int i915_dma_cleanup(struct drm_device * dev)
 {
+	drm_i915_private_t *dev_priv = dev->dev_private;
 	/* Make sure interrupts are disabled here because the uninstall ioctl
 	 * may not have been called from userspace and after dev_private
 	 * is freed, it's too late.
@@ -96,67 +86,125 @@ static int i915_dma_cleanup(drm_device_t * dev)
 	if (dev->irq)
 		drm_irq_uninstall(dev);
 
-	if (dev->dev_private) {
-		drm_i915_private_t *dev_priv =
-		    (drm_i915_private_t *) dev->dev_private;
-
-		if (dev_priv->ring.virtual_start) {
-			drm_core_ioremapfree(&dev_priv->ring.map, dev);
-		}
-
-		if (dev_priv->status_page_dmah) {
-			drm_pci_free(dev, dev_priv->status_page_dmah);
-			/* Need to rewrite hardware status page */
-			I915_WRITE(0x02080, 0x1ffff000);
-		}
+	if (dev_priv->ring.virtual_start) {
+		drm_core_ioremapfree(&dev_priv->ring.map, dev);
+		dev_priv->ring.virtual_start = 0;
+		dev_priv->ring.map.handle = 0;
+		dev_priv->ring.map.size = 0;
+	}
 
-		drm_free(dev->dev_private, sizeof(drm_i915_private_t),
-			 DRM_MEM_DRIVER);
+	if (dev_priv->status_page_dmah) {
+		drm_pci_free(dev, dev_priv->status_page_dmah);
+		dev_priv->status_page_dmah = NULL;
+		/* Need to rewrite hardware status page */
+		I915_WRITE(0x02080, 0x1ffff000);
+	}
 
-		dev->dev_private = NULL;
+	if (dev_priv->status_gfx_addr) {
+		dev_priv->status_gfx_addr = 0;
+		drm_core_ioremapfree(&dev_priv->hws_map, dev);
+		I915_WRITE(0x02080, 0x1ffff000);
 	}
 
 	return 0;
 }
 
-static int i915_initialize(drm_device_t * dev,
-			   drm_i915_private_t * dev_priv,
-			   drm_i915_init_t * init)
+#if defined(I915_HAVE_BUFFER)
+#define DRI2_SAREA_BLOCK_TYPE(b) ((b) >> 16)
+#define DRI2_SAREA_BLOCK_SIZE(b) ((b) & 0xffff)
+#define DRI2_SAREA_BLOCK_NEXT(p)				\
+	((void *) ((unsigned char *) (p) +			\
+		   DRI2_SAREA_BLOCK_SIZE(*(unsigned int *) p)))
+
+#define DRI2_SAREA_BLOCK_END		0x0000
+#define DRI2_SAREA_BLOCK_LOCK		0x0001
+#define DRI2_SAREA_BLOCK_EVENT_BUFFER	0x0002
+
+static int
+setup_dri2_sarea(struct drm_device * dev,
+		 struct drm_file *file_priv,
+		 drm_i915_init_t * init)
 {
-	drm_dma_handle_t *dmah;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
+	unsigned int *p, *end, *next;
 
-	DRM_UNLOCK();
-	memset(dev_priv, 0, sizeof(drm_i915_private_t));
-	dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 
-	    0xffffffff);
-	DRM_LOCK();
-	if (!dmah) {
-		dev->dev_private = (void *)dev_priv;
-		i915_dma_cleanup(dev);
-		DRM_ERROR("Can not allocate hardware status page\n");
-		return DRM_ERR(ENOMEM);
+	mutex_lock(&dev->struct_mutex);
+	dev_priv->sarea_bo =
+		drm_lookup_buffer_object(file_priv,
+					 init->sarea_handle, 1);
+	mutex_unlock(&dev->struct_mutex);
+
+	if (!dev_priv->sarea_bo) {
+		DRM_ERROR("did not find sarea bo\n");
+		return -EINVAL;
 	}
 
-	dev_priv->status_page_dmah = dmah;
+	ret = drm_bo_kmap(dev_priv->sarea_bo, 0,
+			  dev_priv->sarea_bo->num_pages,
+			  &dev_priv->sarea_kmap);
+	if (ret) {
+		DRM_ERROR("could not map sarea bo\n");
+		return ret;
+	}
 
-	DRM_GETSAREA();
+	p = dev_priv->sarea_kmap.virtual;
+	end = (void *) p + (dev_priv->sarea_bo->num_pages << PAGE_SHIFT);
+	while (p < end && DRI2_SAREA_BLOCK_TYPE(*p) != DRI2_SAREA_BLOCK_END) {
+		switch (DRI2_SAREA_BLOCK_TYPE(*p)) {
+		case DRI2_SAREA_BLOCK_LOCK:
+			dev->lock.hw_lock = (void *) (p + 1);
+			dev->sigdata.lock = dev->lock.hw_lock;
+			break;
+		}
+		next = DRI2_SAREA_BLOCK_NEXT(p);
+		if (next <= p || end < next) {
+			DRM_ERROR("malformed dri2 sarea: next is %p should be within %p-%p\n",
+				  next, p, end);
+			return -EINVAL;
+		}
+		p = next;
+	}
+
+	return 0;
+}
+#endif
+
+static int i915_initialize(struct drm_device * dev,
+			   struct drm_file *file_priv,
+			   drm_i915_init_t * init)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+#if defined(I915_HAVE_BUFFER)
+	int ret;
+#endif
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("can not find sarea!\n");
-		dev->dev_private = (void *)dev_priv;
 		i915_dma_cleanup(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
+	if (init->mmio_offset != 0)
+		dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
 	if (!dev_priv->mmio_map) {
-		dev->dev_private = (void *)dev_priv;
 		i915_dma_cleanup(dev);
 		DRM_ERROR("can not find mmio map!\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	dev_priv->sarea_priv = (drm_i915_sarea_t *)
-	    ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset);
+#ifdef I915_HAVE_BUFFER
+	dev_priv->max_validate_buffers = I915_MAX_VALIDATE_BUFFERS;
+#endif
+
+	if (init->sarea_priv_offset)
+		dev_priv->sarea_priv = (drm_i915_sarea_t *)
+			((u8 *) dev_priv->sarea->handle +
+			 init->sarea_priv_offset);
+	else {
+		/* No sarea_priv for you! */
+		dev_priv->sarea_priv = NULL;
+	}
 
 	dev_priv->ring.Start = init->ring_start;
 	dev_priv->ring.End = init->ring_end;
@@ -172,96 +220,117 @@ static int i915_initialize(drm_device_t * dev,
 	drm_core_ioremap(&dev_priv->ring.map, dev);
 
 	if (dev_priv->ring.map.handle == NULL) {
-		dev->dev_private = (void *)dev_priv;
 		i915_dma_cleanup(dev);
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
 
-	dev_priv->back_offset = init->back_offset;
-	dev_priv->front_offset = init->front_offset;
-	dev_priv->current_page = 0;
-	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
+	dev_priv->cpp = init->cpp;
+
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->pf_current_page = 0;
 
 	/* We are using separate values as placeholders for mechanisms for
 	 * private backbuffer/depthbuffer usage.
 	 */
 	dev_priv->use_mi_batchbuffer_start = 0;
+	if (IS_I965G(dev)) /* 965 doesn't support older method */
+		dev_priv->use_mi_batchbuffer_start = 1;
 
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
 	dev_priv->allow_batchbuffer = 1;
 
+	/* Enable vblank on pipe A for older X servers
+	 */
+	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A;
+
 	/* Program Hardware Status Page */
-	dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr;
-	dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
-	
-	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
-	DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
+	if (!I915_NEED_GFX_HWS(dev)) {
+		dev_priv->status_page_dmah =
+			drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff);
+
+		if (!dev_priv->status_page_dmah) {
+			i915_dma_cleanup(dev);
+			DRM_ERROR("Can not allocate hardware status page\n");
+			return -ENOMEM;
+		}
+		dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr;
+		dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
 
-	I915_WRITE(0x02080, dev_priv->dma_status_page);
-	DRM_DEBUG("Enabled hardware status page\n");
+		memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 
-	dev->dev_private = (void *)dev_priv;
+		I915_WRITE(0x02080, dev_priv->dma_status_page);
+	}
+	DRM_DEBUG("Enabled hardware status page\n");
+#ifdef I915_HAVE_BUFFER
+	mutex_init(&dev_priv->cmdbuf_mutex);
+#endif
+#if defined(I915_HAVE_BUFFER)
+	if (init->func == I915_INIT_DMA2) {
+		ret = setup_dri2_sarea(dev, file_priv, init);
+		if (ret) {
+			i915_dma_cleanup(dev);
+			DRM_ERROR("could not set up dri2 sarea\n");
+			return ret;
+		}
+	}
+#endif
 
 	return 0;
 }
 
-static int i915_dma_resume(drm_device_t * dev)
+static int i915_dma_resume(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	if (!dev_priv->sarea) {
 		DRM_ERROR("can not find sarea!\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (!dev_priv->mmio_map) {
 		DRM_ERROR("can not find mmio map!\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (dev_priv->ring.map.handle == NULL) {
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	/* Program Hardware Status Page */
 	if (!dev_priv->hw_status_page) {
 		DRM_ERROR("Can not find hardware status page\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
 
-	I915_WRITE(0x02080, dev_priv->dma_status_page);
+	if (dev_priv->status_gfx_addr != 0)
+		I915_WRITE(0x02080, dev_priv->status_gfx_addr);
+	else
+		I915_WRITE(0x02080, dev_priv->dma_status_page);
 	DRM_DEBUG("Enabled hardware status page\n");
 
 	return 0;
 }
 
-static int i915_dma_init(DRM_IOCTL_ARGS)
+static int i915_dma_init(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_i915_private_t *dev_priv;
-	drm_i915_init_t init;
+	drm_i915_init_t *init = data;
 	int retcode = 0;
 
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_i915_init_t __user *) data,
-				 sizeof(init));
-
-	switch (init.func) {
+	switch (init->func) {
 	case I915_INIT_DMA:
-		dev_priv = drm_alloc(sizeof(drm_i915_private_t),
-				     DRM_MEM_DRIVER);
-		if (dev_priv == NULL)
-			return DRM_ERR(ENOMEM);
-		retcode = i915_initialize(dev, dev_priv, &init);
+	case I915_INIT_DMA2:
+		retcode = i915_initialize(dev, file_priv, init);
 		break;
 	case I915_CLEANUP_DMA:
 		retcode = i915_dma_cleanup(dev);
@@ -348,67 +417,59 @@ static int validate_cmd(int cmd)
 {
 	int ret = do_validate_cmd(cmd);
 
-/* 	printk("validate_cmd( %x ): %d\n", cmd, ret); */
+/*	printk("validate_cmd( %x ): %d\n", cmd, ret); */
 
 	return ret;
 }
 
-static int i915_emit_cmds(drm_device_t * dev, int __user * buffer, int dwords)
+static int i915_emit_cmds(struct drm_device *dev, int __user *buffer,
+			  int dwords)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	int i;
 	RING_LOCALS;
 
 	if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	BEGIN_LP_RING((dwords+1)&~1);
 
 	for (i = 0; i < dwords;) {
 		int cmd, sz;
 
-	     if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) {
-
-			return DRM_ERR(EINVAL);
-	      }
+		cmd = buffer[i];
 		if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 
 		OUT_RING(cmd);
 
 		while (++i, --sz) {
-			if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i],
-							 sizeof(cmd))) {
-				return DRM_ERR(EINVAL);
-			}
 			OUT_RING(cmd);
+			cmd = buffer[i];
 		}
 	}
-		
+
 	if (dwords & 1)
 		OUT_RING(0);
 
 	ADVANCE_LP_RING();
-		
+
 	return 0;
 }
 
-static int i915_emit_box(drm_device_t * dev,
-			 drm_clip_rect_t __user * boxes,
+static int i915_emit_box(struct drm_device * dev,
+			 struct drm_clip_rect __user * boxes,
 			 int i, int DR1, int DR4)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_clip_rect_t box;
+	struct drm_clip_rect box;
 	RING_LOCALS;
 
-	if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) {
-		return EFAULT;
-	}
-
+	box = boxes[i];
 	if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
 		DRM_ERROR("Bad box %d,%d..%d,%d\n",
 			  box.x1, box.y1, box.x2, box.y2);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (IS_I965G(dev)) {
@@ -433,36 +494,64 @@ static int i915_emit_box(drm_device_t * dev,
 }
 
 /* XXX: Emitting the counter should really be moved to part of the IRQ
- * emit.  For now, do it in both places:
+ * emit. For now, do it in both places:
  */
 
-static void i915_emit_breadcrumb(drm_device_t *dev)
+void i915_emit_breadcrumb(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
-	dev_priv->sarea_priv->last_enqueue = ++dev_priv->counter;
+	if (++dev_priv->counter > BREADCRUMB_MASK) {
+		 dev_priv->counter = 1;
+		 DRM_DEBUG("Breadcrumb counter wrapped around\n");
+	}
 
-	if (dev_priv->counter > 0x7FFFFFFFUL)
-		dev_priv->sarea_priv->last_enqueue = dev_priv->counter = 1;
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
 	BEGIN_LP_RING(4);
-	OUT_RING(CMD_STORE_DWORD_IDX);
+	OUT_RING(MI_STORE_DWORD_INDEX);
 	OUT_RING(20);
 	OUT_RING(dev_priv->counter);
 	OUT_RING(0);
 	ADVANCE_LP_RING();
 }
 
-static int i915_dispatch_cmdbuffer(drm_device_t * dev,
+
+int i915_emit_mi_flush(struct drm_device *dev, uint32_t flush)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t flush_cmd = MI_FLUSH;
+	RING_LOCALS;
+
+	flush_cmd |= flush;
+
+	i915_kernel_lost_context(dev);
+
+	BEGIN_LP_RING(4);
+	OUT_RING(flush_cmd);
+	OUT_RING(0);
+	OUT_RING(0);
+	OUT_RING(0);
+	ADVANCE_LP_RING();
+
+	return 0;
+}
+
+
+static int i915_dispatch_cmdbuffer(struct drm_device * dev,
 				   drm_i915_cmdbuffer_t * cmd)
 {
+#ifdef I915_HAVE_FENCE
+	drm_i915_private_t *dev_priv = dev->dev_private;
+#endif
 	int nbox = cmd->num_cliprects;
 	int i = 0, count, ret;
 
 	if (cmd->sz & 0x3) {
-		DRM_ERROR("alignment");
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("alignment\n");
+		return -EINVAL;
 	}
 
 	i915_kernel_lost_context(dev);
@@ -482,22 +571,26 @@ static int i915_dispatch_cmdbuffer(drm_device_t * dev,
 			return ret;
 	}
 
-	i915_emit_breadcrumb( dev );
+	i915_emit_breadcrumb(dev);
+#ifdef I915_HAVE_FENCE
+	if (unlikely((dev_priv->counter & 0xFF) == 0))
+		drm_fence_flush_old(dev, 0, dev_priv->counter);
+#endif
 	return 0;
 }
 
-static int i915_dispatch_batchbuffer(drm_device_t * dev,
-				     drm_i915_batchbuffer_t * batch)
+int i915_dispatch_batchbuffer(struct drm_device * dev,
+			      drm_i915_batchbuffer_t * batch)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_clip_rect_t __user *boxes = batch->cliprects;
+	struct drm_clip_rect __user *boxes = batch->cliprects;
 	int nbox = batch->num_cliprects;
 	int i = 0, count;
 	RING_LOCALS;
 
 	if ((batch->start | batch->used) & 0x7) {
-		DRM_ERROR("alignment");
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("alignment\n");
+		return -EINVAL;
 	}
 
 	i915_kernel_lost_context(dev);
@@ -514,9 +607,15 @@ static int i915_dispatch_batchbuffer(drm_device_t * dev,
 
 		if (dev_priv->use_mi_batchbuffer_start) {
 			BEGIN_LP_RING(2);
-			OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
-			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
+			if (IS_I965G(dev)) {
+				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
+				OUT_RING(batch->start);
+			} else {
+				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
+				OUT_RING(batch->start | MI_BATCH_NON_SECURE);
+			}
 			ADVANCE_LP_RING();
+
 		} else {
 			BEGIN_LP_RING(4);
 			OUT_RING(MI_BATCH_BUFFER);
@@ -527,59 +626,88 @@ static int i915_dispatch_batchbuffer(drm_device_t * dev,
 		}
 	}
 
-	i915_emit_breadcrumb( dev );
+	i915_emit_breadcrumb(dev);
+#ifdef I915_HAVE_FENCE
+	if (unlikely((dev_priv->counter & 0xFF) == 0))
+		drm_fence_flush_old(dev, 0, dev_priv->counter);
+#endif
 	return 0;
 }
 
-static int i915_dispatch_flip(drm_device_t * dev)
+static void i915_do_dispatch_flip(struct drm_device * dev, int plane, int sync)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	u32 num_pages, current_page, next_page, dspbase;
+	int shift = 2 * plane, x, y;
 	RING_LOCALS;
 
-	DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-		  __FUNCTION__,
-		  dev_priv->current_page,
-		  dev_priv->sarea_priv->pf_current_page);
-
-	i915_kernel_lost_context(dev);
+	/* Calculate display base offset */
+	num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
+	current_page = (dev_priv->sarea_priv->pf_current_page >> shift) & 0x3;
+	next_page = (current_page + 1) % num_pages;
 
-	BEGIN_LP_RING(2);
-	OUT_RING(INST_PARSER_CLIENT | INST_OP_FLUSH | INST_FLUSH_MAP_CACHE);
-	OUT_RING(0);
-	ADVANCE_LP_RING();
+	switch (next_page) {
+	default:
+	case 0:
+		dspbase = dev_priv->sarea_priv->front_offset;
+		break;
+	case 1:
+		dspbase = dev_priv->sarea_priv->back_offset;
+		break;
+	case 2:
+		dspbase = dev_priv->sarea_priv->third_offset;
+		break;
+	}
 
-	BEGIN_LP_RING(6);
-	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
-	OUT_RING(0);
-	if (dev_priv->current_page == 0) {
-		OUT_RING(dev_priv->back_offset);
-		dev_priv->current_page = 1;
+	if (plane == 0) {
+		x = dev_priv->sarea_priv->planeA_x;
+		y = dev_priv->sarea_priv->planeA_y;
 	} else {
-		OUT_RING(dev_priv->front_offset);
-		dev_priv->current_page = 0;
+		x = dev_priv->sarea_priv->planeB_x;
+		y = dev_priv->sarea_priv->planeB_y;
 	}
-	OUT_RING(0);
-	ADVANCE_LP_RING();
 
-	BEGIN_LP_RING(2);
-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
-	OUT_RING(0);
-	ADVANCE_LP_RING();
+	dspbase += (y * dev_priv->sarea_priv->pitch + x) * dev_priv->cpp;
 
-	dev_priv->sarea_priv->last_enqueue = dev_priv->counter++;
+	DRM_DEBUG("plane=%d current_page=%d dspbase=0x%x\n", plane, current_page,
+		  dspbase);
 
 	BEGIN_LP_RING(4);
-	OUT_RING(CMD_STORE_DWORD_IDX);
-	OUT_RING(20);
-	OUT_RING(dev_priv->counter);
-	OUT_RING(0);
+	OUT_RING(sync ? 0 :
+		 (MI_WAIT_FOR_EVENT | (plane ? MI_WAIT_FOR_PLANE_B_FLIP :
+				       MI_WAIT_FOR_PLANE_A_FLIP)));
+	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | (sync ? 0 : ASYNC_FLIP) |
+		 (plane ? DISPLAY_PLANE_B : DISPLAY_PLANE_A));
+	OUT_RING(dev_priv->sarea_priv->pitch * dev_priv->cpp);
+	OUT_RING(dspbase);
 	ADVANCE_LP_RING();
 
-	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
-	return 0;
+	dev_priv->sarea_priv->pf_current_page &= ~(0x3 << shift);
+	dev_priv->sarea_priv->pf_current_page |= next_page << shift;
 }
 
-static int i915_quiescent(drm_device_t * dev)
+void i915_dispatch_flip(struct drm_device * dev, int planes, int sync)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int i;
+
+	DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
+		  planes, dev_priv->sarea_priv->pf_current_page);
+
+	i915_emit_mi_flush(dev, MI_READ_FLUSH | MI_EXE_FLUSH);
+
+	for (i = 0; i < 2; i++)
+		if (planes & (1 << i))
+			i915_do_dispatch_flip(dev, i, sync);
+
+	i915_emit_breadcrumb(dev);
+#ifdef I915_HAVE_FENCE
+	if (unlikely(!sync && ((dev_priv->counter & 0xFF) == 0)))
+		drm_fence_flush_old(dev, 0, dev_priv->counter);
+#endif
+}
+
+int i915_quiescent(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 
@@ -587,123 +715,176 @@ static int i915_quiescent(drm_device_t * dev)
 	return i915_wait_ring(dev, dev_priv->ring.Size - 8, __FUNCTION__);
 }
 
-static int i915_flush_ioctl(DRM_IOCTL_ARGS)
+static int i915_flush_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return i915_quiescent(dev);
 }
 
-static int i915_batchbuffer(DRM_IOCTL_ARGS)
+static int i915_batchbuffer(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 *hw_status = dev_priv->hw_status_page;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
-	drm_i915_batchbuffer_t batch;
+	drm_i915_batchbuffer_t *batch = data;
+	size_t cliplen;
 	int ret;
 
 	if (!dev_priv->allow_batchbuffer) {
 		DRM_ERROR("Batchbuffer ioctl disabled\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(batch, (drm_i915_batchbuffer_t __user *) data,
-				 sizeof(batch));
-
 	DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d\n",
-		  batch.start, batch.used, batch.num_cliprects);
+		  batch->start, batch->used, batch->num_cliprects);
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (batch.num_cliprects && DRM_VERIFYAREA_READ(batch.cliprects,
-						       batch.num_cliprects *
-						       sizeof(drm_clip_rect_t)))
-		return DRM_ERR(EFAULT);
-
-	ret = i915_dispatch_batchbuffer(dev, &batch);
+	DRM_UNLOCK();
+	cliplen = batch->num_cliprects * sizeof(drm_clip_rect_t);
+	if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects,
+	    cliplen)) {
+		DRM_LOCK();
+		return -EFAULT;
+	}
+	if (batch->num_cliprects) {
+		ret = vslock(batch->cliprects, cliplen);
+		if (ret) {
+			DRM_ERROR("Fault wiring cliprects\n");
+			DRM_LOCK();
+			return -EFAULT;
+		}
+	}
+	DRM_LOCK();
+	ret = i915_dispatch_batchbuffer(dev, batch);
 
-	sarea_priv->last_dispatch = (int)hw_status[5];
+	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+	DRM_UNLOCK();
+	if (batch->num_cliprects)
+	    vsunlock(batch->cliprects, cliplen);
+	DRM_LOCK();
 	return ret;
 }
 
-static int i915_cmdbuffer(DRM_IOCTL_ARGS)
+static int i915_cmdbuffer(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 *hw_status = dev_priv->hw_status_page;
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
-	drm_i915_cmdbuffer_t cmdbuf;
+	drm_i915_cmdbuffer_t *cmdbuf = data;
+	size_t cliplen;
 	int ret;
 
-	DRM_COPY_FROM_USER_IOCTL(cmdbuf, (drm_i915_cmdbuffer_t __user *) data,
-				 sizeof(cmdbuf));
-
 	DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
-		  cmdbuf.buf, cmdbuf.sz, cmdbuf.num_cliprects);
+		  cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (cmdbuf.num_cliprects &&
-	    DRM_VERIFYAREA_READ(cmdbuf.cliprects,
-				cmdbuf.num_cliprects *
-				sizeof(drm_clip_rect_t))) {
+	DRM_UNLOCK();
+	cliplen = cmdbuf->num_cliprects * sizeof(drm_clip_rect_t);
+	if (cmdbuf->num_cliprects &&
+	    DRM_VERIFYAREA_READ(cmdbuf->cliprects, cliplen)) {
 		DRM_ERROR("Fault accessing cliprects\n");
-		return DRM_ERR(EFAULT);
+		DRM_LOCK();
+		return -EFAULT;
 	}
-
-	ret = i915_dispatch_cmdbuffer(dev, &cmdbuf);
-	if (ret) {
+	if (cmdbuf->num_cliprects) {
+		ret = vslock(cmdbuf->cliprects, cliplen);
+		if (ret) {
+			DRM_ERROR("Fault wiring cliprects\n");
+			DRM_LOCK();
+			return -EFAULT;
+		}
+		ret = vslock(cmdbuf->buf, cmdbuf->sz);
+		if (ret) {
+			vsunlock(cmdbuf->cliprects, cliplen);
+			DRM_ERROR("Fault wiring cmds\n");
+			DRM_LOCK();
+			return -EFAULT;
+		}
+	}
+	DRM_LOCK();
+	ret = i915_dispatch_cmdbuffer(dev, cmdbuf);
+	if (ret == 0)
+		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+	else
 		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
-		return ret;
+	DRM_UNLOCK();
+	if (cmdbuf->num_cliprects) {
+		vsunlock(cmdbuf->buf, cmdbuf->sz);
+		vsunlock(cmdbuf->cliprects, cliplen);
 	}
-
-	sarea_priv->last_dispatch = (int)hw_status[5];
-	return 0;
+	DRM_LOCK();
+	return (ret);
 }
 
-static int i915_do_cleanup_pageflip(drm_device_t * dev)
+#if defined(DRM_DEBUG_CODE)
+#define DRM_DEBUG_RELOCATION	(drm_debug != 0)
+#else
+#define DRM_DEBUG_RELOCATION	0
+#endif
+
+static int i915_do_cleanup_pageflip(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	int i, planes, num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
+
+	DRM_DEBUG("\n");
+
+	for (i = 0, planes = 0; i < 2; i++)
+		if (dev_priv->sarea_priv->pf_current_page & (0x3 << (2 * i))) {
+			dev_priv->sarea_priv->pf_current_page =
+				(dev_priv->sarea_priv->pf_current_page &
+				 ~(0x3 << (2 * i))) | ((num_pages - 1) << (2 * i));
+
+			planes |= 1 << i;
+		}
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
-	if (dev_priv->current_page != 0)
-		i915_dispatch_flip(dev);
+	if (planes)
+		i915_dispatch_flip(dev, planes, 0);
 
 	return 0;
 }
 
-static int i915_flip_bufs(DRM_IOCTL_ARGS)
+static int i915_flip_bufs(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
+	drm_i915_flip_t *param = data;
+
+	DRM_DEBUG("\n");
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	/* This is really planes */
+	if (param->pipes & ~0x3) {
+		DRM_ERROR("Invalid planes 0x%x, only <= 0x3 is valid\n",
+			  param->pipes);
+		return -EINVAL;
+	}
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	i915_dispatch_flip(dev, param->pipes, 0);
 
-	return i915_dispatch_flip(dev);
+	return 0;
 }
 
-static int i915_getparam(DRM_IOCTL_ARGS)
+
+static int i915_getparam(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_getparam_t param;
+	drm_i915_getparam_t *param = data;
 	int value;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_i915_getparam_t __user *) data,
-				 sizeof(param));
-
-	switch (param.param) {
+	switch (param->param) {
 	case I915_PARAM_IRQ_ACTIVE:
 		value = dev->irq ? 1 : 0;
 		break;
@@ -713,53 +894,156 @@ static int i915_getparam(DRM_IOCTL_ARGS)
 	case I915_PARAM_LAST_DISPATCH:
 		value = READ_BREADCRUMB(dev_priv);
 		break;
+	case I915_PARAM_CHIPSET_ID:
+		value = dev->pci_device;
+		break;
 	default:
-		DRM_ERROR("Unknown parameter %d\n", param.param);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("Unknown parameter %d\n", param->param);
+		return -EINVAL;
 	}
 
-	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
+	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("DRM_COPY_TO_USER failed\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-static int i915_setparam(DRM_IOCTL_ARGS)
+static int i915_setparam(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_setparam_t param;
+	drm_i915_setparam_t *param = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_i915_setparam_t __user *) data,
-				 sizeof(param));
-
-	switch (param.param) {
+	switch (param->param) {
 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
-		dev_priv->use_mi_batchbuffer_start = param.value;
+		if (!IS_I965G(dev))
+			dev_priv->use_mi_batchbuffer_start = param->value;
 		break;
 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
-		dev_priv->tex_lru_log_granularity = param.value;
+		dev_priv->tex_lru_log_granularity = param->value;
 		break;
 	case I915_SETPARAM_ALLOW_BATCHBUFFER:
-		dev_priv->allow_batchbuffer = param.value;
+		dev_priv->allow_batchbuffer = param->value;
 		break;
 	default:
-		DRM_ERROR("unknown parameter %d\n", param.param);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("unknown parameter %d\n", param->param);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+drm_i915_mmio_entry_t mmio_table[] = {
+	[MMIO_REGS_PS_DEPTH_COUNT] = {
+		I915_MMIO_MAY_READ|I915_MMIO_MAY_WRITE,
+		0x2350,
+		8
+	}
+};
+
+static int mmio_table_size = sizeof(mmio_table)/sizeof(drm_i915_mmio_entry_t);
+
+static int i915_mmio(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
+{
+	uint32_t buf[8];
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_mmio_entry_t *e;
+	drm_i915_mmio_t *mmio = data;
+	void __iomem *base;
+	int i;
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	if (mmio->reg >= mmio_table_size)
+		return -EINVAL;
+
+	e = &mmio_table[mmio->reg];
+	base = (u8 *) dev_priv->mmio_map->handle + e->offset;
+
+	switch (mmio->read_write) {
+	case I915_MMIO_READ:
+		if (!(e->flag & I915_MMIO_MAY_READ))
+			return -EINVAL;
+		for (i = 0; i < e->size / 4; i++)
+			buf[i] = I915_READ(e->offset + i * 4);
+		if (DRM_COPY_TO_USER(mmio->data, buf, e->size)) {
+			DRM_ERROR("DRM_COPY_TO_USER failed\n");
+			return -EFAULT;
+		}
+		break;
+		
+	case I915_MMIO_WRITE:
+		if (!(e->flag & I915_MMIO_MAY_WRITE))
+			return -EINVAL;
+		if (DRM_COPY_FROM_USER(buf, mmio->data, e->size)) {
+			DRM_ERROR("DRM_COPY_TO_USER failed\n");
+			return -EFAULT;
+		}
+		for (i = 0; i < e->size / 4; i++)
+			I915_WRITE(e->offset + i * 4, buf[i]);
+		break;
+	}
+	return 0;
+}
+
+static int i915_set_status_page(struct drm_device *dev, void *data,
+				struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_hws_addr_t *hws = data;
+
+	if (!I915_NEED_GFX_HWS(dev))
+		return -EINVAL;
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
+	DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws->addr);
+
+	dev_priv->status_gfx_addr = hws->addr & (0x1ffff<<12);
+
+	dev_priv->hws_map.offset = dev->agp->base + hws->addr;
+	dev_priv->hws_map.size = 4*1024;
+	dev_priv->hws_map.type = 0;
+	dev_priv->hws_map.flags = 0;
+	dev_priv->hws_map.mtrr = 0;
 
+	drm_core_ioremap(&dev_priv->hws_map, dev);
+	if (dev_priv->hws_map.handle == NULL) {
+		i915_dma_cleanup(dev);
+		dev_priv->status_gfx_addr = 0;
+		DRM_ERROR("can not ioremap virtual address for"
+				" G33 hw status page\n");
+		return -ENOMEM;
+	}
+	dev_priv->hw_status_page = dev_priv->hws_map.handle;
+
+	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
+	I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
+	DRM_DEBUG("load hws 0x2080 with gfx mem 0x%x\n",
+			dev_priv->status_gfx_addr);
+	DRM_DEBUG("load hws at %p\n", dev_priv->hw_status_page);
 	return 0;
 }
 
-int i915_driver_load(drm_device_t *dev, unsigned long flags)
+int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
+	struct drm_i915_private *dev_priv;
+	unsigned long base, size;
+	int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
+
 	/* i915 has 4 more counters */
 	dev->counters += 4;
 	dev->types[6] = _DRM_STAT_IRQ;
@@ -767,45 +1051,132 @@ int i915_driver_load(drm_device_t *dev, unsigned long flags)
 	dev->types[8] = _DRM_STAT_SECONDARY;
 	dev->types[9] = _DRM_STAT_DMA;
 
+	dev_priv = drm_alloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER);
+	if (dev_priv == NULL)
+		return -ENOMEM;
+
+	memset(dev_priv, 0, sizeof(drm_i915_private_t));
+
+	dev->dev_private = (void *)dev_priv;
+
+	/* Add register map (needed for suspend/resume) */
+	base = drm_get_resource_start(dev, mmio_bar);
+	size = drm_get_resource_len(dev, mmio_bar);
+
+	ret = drm_addmap(dev, base, size, _DRM_REGISTERS,
+		_DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map);
+
+	DRM_SPININIT(&dev_priv->swaps_lock, "swap");
+	DRM_SPININIT(&dev_priv->user_irq_lock, "userirq");
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+	intel_init_chipset_flush_compat(dev);
+#endif
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
+	intel_opregion_init(dev);
+#endif
+#endif
+
+	return ret;
+}
+
+int i915_driver_unload(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->mmio_map)
+		drm_rmmap(dev, dev_priv->mmio_map);
+
+	DRM_SPINUNINIT(&dev_priv->swaps_lock);
+	DRM_SPINUNINIT(&dev_priv->user_irq_lock);
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
+	intel_opregion_free(dev);
+#endif
+#endif
+
+	drm_free(dev->dev_private, sizeof(drm_i915_private_t),
+		 DRM_MEM_DRIVER);
+	dev->dev_private = NULL;
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+	intel_fini_chipset_flush_compat(dev);
+#endif
+#endif
 	return 0;
 }
 
-void i915_driver_lastclose(drm_device_t * dev)
+void i915_driver_lastclose(struct drm_device * dev)
 {
-	if (dev->dev_private) {
-		drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	/* agp off can use this to get called before dev_priv */
+	if (!dev_priv)
+		return;
+
+#ifdef I915_HAVE_BUFFER
+	if (dev_priv->val_bufs) {
+		vfree(dev_priv->val_bufs);
+		dev_priv->val_bufs = NULL;
+	}
+#endif
+
+	if (drm_getsarea(dev) && dev_priv->sarea_priv) {
+		i915_do_cleanup_pageflip(dev);
+		dev_priv->sarea_priv = NULL;
+		dev_priv->sarea = NULL;
+	}
+	if (dev_priv->agp_heap)
 		i915_mem_takedown(&(dev_priv->agp_heap));
+#if defined(I915_HAVE_BUFFER)
+	if (dev_priv->sarea_kmap.virtual) {
+		drm_bo_kunmap(&dev_priv->sarea_kmap);
+		dev_priv->sarea_kmap.virtual = NULL;
+		dev->lock.hw_lock = NULL;
+		dev->sigdata.lock = NULL;
 	}
+
+	if (dev_priv->sarea_bo) {
+		mutex_lock(&dev->struct_mutex);
+		drm_bo_usage_deref_locked(&dev_priv->sarea_bo);
+		mutex_unlock(&dev->struct_mutex);
+		dev_priv->sarea_bo = NULL;
+	}
+#endif
 	i915_dma_cleanup(dev);
 }
 
-void i915_driver_preclose(drm_device_t * dev, DRMFILE filp)
+void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
 {
-	if (dev->dev_private) {
-		drm_i915_private_t *dev_priv = dev->dev_private;
-		if (dev_priv->page_flipping) {
-			i915_do_cleanup_pageflip(dev);
-		}
-		i915_mem_release(dev, filp, dev_priv->agp_heap);
-	}
-}
-
-drm_ioctl_desc_t i915_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_I915_INIT)] = {i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_I915_FLUSH)] = {i915_flush_ioctl, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_FLIP)] = {i915_flip_bufs, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_BATCHBUFFER)] = {i915_batchbuffer, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_IRQ_EMIT)] = {i915_irq_emit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_IRQ_WAIT)] = {i915_irq_wait, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_GETPARAM)] = {i915_getparam, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_SETPARAM)] = {i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_I915_ALLOC)] = {i915_mem_alloc, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_FREE)] = {i915_mem_free, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_INIT_HEAP)] = {i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_I915_CMDBUFFER)] = {i915_cmdbuffer, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_I915_DESTROY_HEAP)] = { i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_I915_SET_VBLANK_PIPE)] = { i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
-	[DRM_IOCTL_NR(DRM_I915_GET_VBLANK_PIPE)] = { i915_vblank_pipe_get, DRM_AUTH },
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	i915_mem_release(dev, file_priv, dev_priv->agp_heap);
+}
+
+struct drm_ioctl_desc i915_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_FLIP, i915_flip_bufs, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GETPARAM, i915_getparam, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_I915_ALLOC, i915_mem_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_FREE, i915_mem_free, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP,  i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
+	DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE,  i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
+	DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE,  i915_vblank_pipe_get, DRM_AUTH ),
+	DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_MMIO, i915_mmio, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH),
+#ifdef I915_HAVE_BUFFER
+	DRM_IOCTL_DEF(DRM_I915_EXECBUFFER, i915_execbuffer, DRM_AUTH),
+#endif
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
@@ -821,7 +1192,15 @@ int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
  * \returns
  * A value of 1 is always retured to indictate every i9x5 is AGP.
  */
-int i915_driver_device_is_agp(drm_device_t * dev)
+int i915_driver_device_is_agp(struct drm_device * dev)
 {
 	return 1;
 }
+
+int i915_driver_firstopen(struct drm_device *dev)
+{
+#ifdef I915_HAVE_BUFFER
+	drm_bo_driver_init(dev);
+#endif
+	return 0;
+}
diff --git a/sys/dev/drm/i915_drm.h b/sys/dev/drm/i915_drm.h
index 778088f..99092e4 100644
--- a/sys/dev/drm/i915_drm.h
+++ b/sys/dev/drm/i915_drm.h
@@ -1,7 +1,7 @@
 /*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
@@ -9,11 +9,11 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
@@ -21,7 +21,7 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- * 
+ *
  */
 
 #include <sys/cdefs.h>
@@ -46,7 +46,12 @@ typedef struct _drm_i915_init {
 	enum {
 		I915_INIT_DMA = 0x01,
 		I915_CLEANUP_DMA = 0x02,
-		I915_RESUME_DMA = 0x03
+		I915_RESUME_DMA = 0x03,
+
+		/* Since this struct isn't versioned, just used a new
+		 * 'func' code to indicate the presence of dri2 sarea
+		 * info. */
+		I915_INIT_DMA2 = 0x04
 	} func;
 	unsigned int mmio_offset;
 	int sarea_priv_offset;
@@ -64,10 +69,11 @@ typedef struct _drm_i915_init {
 	unsigned int depth_pitch;
 	unsigned int cpp;
 	unsigned int chipset;
+	unsigned int sarea_handle;
 } drm_i915_init_t;
 
-typedef struct _drm_i915_sarea {
-	drm_tex_region_t texList[I915_NR_TEX_REGIONS + 1];
+typedef struct drm_i915_sarea {
+	struct drm_tex_region texList[I915_NR_TEX_REGIONS + 1];
 	int last_upload;	/* last time texture was uploaded */
 	int last_enqueue;	/* last time a buffer was enqueued */
 	int last_dispatch;	/* age of the most recently dispatched buffer */
@@ -107,8 +113,42 @@ typedef struct _drm_i915_sarea {
 	unsigned int depth_tiled;
 	unsigned int rotated_tiled;
 	unsigned int rotated2_tiled;
+
+	int planeA_x;
+	int planeA_y;
+	int planeA_w;
+	int planeA_h;
+	int planeB_x;
+	int planeB_y;
+	int planeB_w;
+	int planeB_h;
+
+	/* Triple buffering */
+	drm_handle_t third_handle;
+	int third_offset;
+	int third_size;
+	unsigned int third_tiled;
+
+	/* buffer object handles for the static buffers.  May change
+	 * over the lifetime of the client, though it doesn't in our current
+	 * implementation.
+	 */
+	unsigned int front_bo_handle;
+	unsigned int back_bo_handle;
+	unsigned int third_bo_handle;
+	unsigned int depth_bo_handle;
 } drm_i915_sarea_t;
 
+/* Driver specific fence types and classes.
+ */
+
+/* The only fence class we support */
+#define DRM_I915_FENCE_CLASS_ACCEL 0
+/* Fence type that guarantees read-write flush */
+#define DRM_I915_FENCE_TYPE_RW 2
+/* MI_FLUSH programmed just before the fence */
+#define DRM_I915_FENCE_FLAG_FLUSHED 0x01000000
+
 /* Flags for perf_boxes
  */
 #define I915_BOX_RING_EMPTY    0x1
@@ -135,10 +175,14 @@ typedef struct _drm_i915_sarea {
 #define DRM_I915_DESTROY_HEAP	0x0c
 #define DRM_I915_SET_VBLANK_PIPE	0x0d
 #define DRM_I915_GET_VBLANK_PIPE	0x0e
+#define DRM_I915_VBLANK_SWAP	0x0f
+#define DRM_I915_MMIO		0x10
+#define DRM_I915_HWS_ADDR	0x11
+#define DRM_I915_EXECBUFFER	0x12
 
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
-#define DRM_IOCTL_I915_FLIP		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLIP)
+#define DRM_IOCTL_I915_FLIP		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_FLIP, drm_i915_flip_t)
 #define DRM_IOCTL_I915_BATCHBUFFER	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_BATCHBUFFER, drm_i915_batchbuffer_t)
 #define DRM_IOCTL_I915_IRQ_EMIT         DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_IRQ_EMIT, drm_i915_irq_emit_t)
 #define DRM_IOCTL_I915_IRQ_WAIT         DRM_IOW( DRM_COMMAND_BASE + DRM_I915_IRQ_WAIT, drm_i915_irq_wait_t)
@@ -151,18 +195,32 @@ typedef struct _drm_i915_sarea {
 #define DRM_IOCTL_I915_DESTROY_HEAP	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_DESTROY_HEAP, drm_i915_mem_destroy_heap_t)
 #define DRM_IOCTL_I915_SET_VBLANK_PIPE	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
 #define DRM_IOCTL_I915_GET_VBLANK_PIPE	DRM_IOR( DRM_COMMAND_BASE + DRM_I915_GET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
+#define DRM_IOCTL_I915_VBLANK_SWAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_VBLANK_SWAP, drm_i915_vblank_swap_t)
+#define DRM_IOCTL_I915_MMIO             DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_MMIO, drm_i915_mmio)
+#define DRM_IOCTL_I915_EXECBUFFER	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_EXECBUFFER, struct drm_i915_execbuffer)
 
+/* Asynchronous page flipping:
+ */
+typedef struct drm_i915_flip {
+	/*
+	 * This is really talking about planes, and we could rename it
+	 * except for the fact that some of the duplicated i915_drm.h files
+	 * out there check for HAVE_I915_FLIP and so might pick up this
+	 * version.
+	 */
+	int pipes;
+} drm_i915_flip_t;
 
 /* Allow drivers to submit batchbuffers directly to hardware, relying
  * on the security mechanisms provided by hardware.
  */
-typedef struct _drm_i915_batchbuffer {
+typedef struct drm_i915_batchbuffer {
 	int start;		/* agp offset */
 	int used;		/* nr bytes in use */
 	int DR1;		/* hw flags for GFX_OP_DRAWRECT_INFO */
 	int DR4;		/* window origin for GFX_OP_DRAWRECT_INFO */
 	int num_cliprects;	/* mulitpass with multiple cliprects? */
-	drm_clip_rect_t __user *cliprects;	/* pointer to userspace cliprects */
+	struct drm_clip_rect __user *cliprects;	/* pointer to userspace cliprects */
 } drm_i915_batchbuffer_t;
 
 /* As above, but pass a pointer to userspace buffer which can be
@@ -174,7 +232,7 @@ typedef struct _drm_i915_cmdbuffer {
 	int DR1;		/* hw flags for GFX_OP_DRAWRECT_INFO */
 	int DR4;		/* window origin for GFX_OP_DRAWRECT_INFO */
 	int num_cliprects;	/* mulitpass with multiple cliprects? */
-	drm_clip_rect_t __user *cliprects;	/* pointer to userspace cliprects */
+	struct drm_clip_rect __user *cliprects;	/* pointer to userspace cliprects */
 } drm_i915_cmdbuffer_t;
 
 /* Userspace can request & wait on irq's:
@@ -192,6 +250,7 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_IRQ_ACTIVE            1
 #define I915_PARAM_ALLOW_BATCHBUFFER     2
 #define I915_PARAM_LAST_DISPATCH         3
+#define I915_PARAM_CHIPSET_ID            4
 
 typedef struct drm_i915_getparam {
 	int param;
@@ -235,7 +294,7 @@ typedef struct drm_i915_mem_init_heap {
  * rotate):
  */
 typedef struct drm_i915_mem_destroy_heap {
-	        int region;
+	int region;
 } drm_i915_mem_destroy_heap_t;
 
 /* Allow X server to configure which pipes to monitor for vblank signals
@@ -247,4 +306,96 @@ typedef struct drm_i915_vblank_pipe {
 	int pipe;
 } drm_i915_vblank_pipe_t;
 
+/* Schedule buffer swap at given vertical blank:
+ */
+typedef struct drm_i915_vblank_swap {
+	drm_drawable_t drawable;
+	enum drm_vblank_seq_type seqtype;
+	unsigned int sequence;
+} drm_i915_vblank_swap_t;
+
+#define I915_MMIO_READ	0
+#define I915_MMIO_WRITE 1
+
+#define I915_MMIO_MAY_READ	0x1
+#define I915_MMIO_MAY_WRITE	0x2
+
+#define MMIO_REGS_IA_PRIMATIVES_COUNT		0
+#define MMIO_REGS_IA_VERTICES_COUNT		1
+#define MMIO_REGS_VS_INVOCATION_COUNT		2
+#define MMIO_REGS_GS_PRIMITIVES_COUNT		3
+#define MMIO_REGS_GS_INVOCATION_COUNT		4
+#define MMIO_REGS_CL_PRIMITIVES_COUNT		5
+#define MMIO_REGS_CL_INVOCATION_COUNT		6
+#define MMIO_REGS_PS_INVOCATION_COUNT		7
+#define MMIO_REGS_PS_DEPTH_COUNT		8
+
+typedef struct drm_i915_mmio_entry {
+	unsigned int flag;
+	unsigned int offset;
+	unsigned int size;
+} drm_i915_mmio_entry_t;
+
+typedef struct drm_i915_mmio {
+	unsigned int read_write:1;
+	unsigned int reg:31;
+	void __user *data;
+} drm_i915_mmio_t;
+
+typedef struct drm_i915_hws_addr {
+	uint64_t addr;
+} drm_i915_hws_addr_t;
+
+/*
+ * Relocation header is 4 uint32_ts
+ * 0 - 32 bit reloc count
+ * 1 - 32-bit relocation type
+ * 2-3 - 64-bit user buffer handle ptr for another list of relocs.
+ */
+#define I915_RELOC_HEADER 4
+
+/*
+ * type 0 relocation has 4-uint32_t stride
+ * 0 - offset into buffer
+ * 1 - delta to add in
+ * 2 - buffer handle
+ * 3 - reserved (for optimisations later).
+ */
+/*
+ * type 1 relocation has 4-uint32_t stride.
+ * Hangs off the first item in the op list.
+ * Performed after all valiations are done.
+ * Try to group relocs into the same relocatee together for
+ * performance reasons.
+ * 0 - offset into buffer
+ * 1 - delta to add in
+ * 2 - buffer index in op list.
+ * 3 - relocatee index in op list.
+ */
+#define I915_RELOC_TYPE_0 0
+#define I915_RELOC0_STRIDE 4
+#define I915_RELOC_TYPE_1 1
+#define I915_RELOC1_STRIDE 4
+
+
+struct drm_i915_op_arg {
+	uint64_t next;
+	uint64_t reloc_ptr;
+	int handled;
+	unsigned int pad64;
+	union {
+		struct drm_bo_op_req req;
+		struct drm_bo_arg_rep rep;
+	} d;
+
+};
+
+struct drm_i915_execbuffer {
+	uint64_t ops_list;
+	uint32_t num_buffers;
+	struct drm_i915_batchbuffer batch;
+	drm_context_t context; /* for lockless use in the future */
+	struct drm_fence_arg fence_arg;
+};
+
 #endif				/* _I915_DRM_H_ */
diff --git a/sys/dev/drm/i915_drv.c b/sys/dev/drm/i915_drv.c
index 8a20330..0556431 100644
--- a/sys/dev/drm/i915_drv.c
+++ b/sys/dev/drm/i915_drv.c
@@ -1,4 +1,4 @@
-/* i915_drv.c -- ATI Radeon driver -*- linux-c -*-
+/* i915_drv.c -- Intel i915 driver -*- linux-c -*-
  * Created: Wed Feb 14 17:10:04 2001 by gareth@valinux.com
  */
 /*-
@@ -43,14 +43,44 @@ static drm_pci_id_list_t i915_pciidlist[] = {
 	i915_PCI_IDS
 };
 
-static void i915_configure(drm_device_t *dev)
+static int i915_suspend(device_t nbdev)
 {
-	dev->driver.buf_priv_size	= 1;	/* No dev_priv */
+	struct drm_device *dev = device_get_softc(nbdev);
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (!dev || !dev_priv) {
+		DRM_ERROR("dev: 0x%lx, dev_priv: 0x%lx\n",
+			(unsigned long) dev, (unsigned long) dev_priv);
+		DRM_ERROR("DRM not initialized, aborting suspend.\n");
+		return -ENODEV;
+	}
+
+	i915_save_state(dev);
+
+	return (bus_generic_suspend(nbdev));
+}
+
+static int i915_resume(device_t nbdev)
+{
+	struct drm_device *dev = device_get_softc(nbdev);
+
+	i915_restore_state(dev);
+
+	return (bus_generic_resume(nbdev));
+}
+
+static void i915_configure(struct drm_device *dev)
+{
+	dev->driver.buf_priv_size	= sizeof(drm_i915_private_t);
 	dev->driver.load		= i915_driver_load;
+	dev->driver.unload		= i915_driver_unload;
+	dev->driver.firstopen		= i915_driver_firstopen;
 	dev->driver.preclose		= i915_driver_preclose;
 	dev->driver.lastclose		= i915_driver_lastclose;
-	dev->driver.device_is_agp	= i915_driver_device_is_agp,
-	dev->driver.vblank_wait		= i915_driver_vblank_wait;
+	dev->driver.device_is_agp	= i915_driver_device_is_agp;
+	dev->driver.get_vblank_counter	= i915_get_vblank_counter;
+	dev->driver.enable_vblank	= i915_enable_vblank;
+	dev->driver.disable_vblank	= i915_disable_vblank;
 	dev->driver.irq_preinstall	= i915_driver_irq_preinstall;
 	dev->driver.irq_postinstall	= i915_driver_irq_postinstall;
 	dev->driver.irq_uninstall	= i915_driver_irq_uninstall;
@@ -71,6 +101,7 @@ static void i915_configure(drm_device_t *dev)
 	dev->driver.use_mtrr		= 1;
 	dev->driver.use_irq		= 1;
 	dev->driver.use_vbl_irq		= 1;
+	dev->driver.use_vbl_irq2	= 1;
 }
 
 #ifdef __FreeBSD__
@@ -83,9 +114,9 @@ i915_probe(device_t dev)
 static int
 i915_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	i915_configure(dev);
 	return drm_attach(nbdev, i915_pciidlist);
 }
@@ -94,6 +125,8 @@ static device_method_t i915_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		i915_probe),
 	DEVMETHOD(device_attach,	i915_attach),
+	DEVMETHOD(device_suspend,	i915_suspend),
+	DEVMETHOD(device_resume,	i915_resume),
 	DEVMETHOD(device_detach,	drm_detach),
 
 	{ 0, 0 }
@@ -106,7 +139,7 @@ static driver_t i915_driver = {
 	"drmsub",
 #endif
 	i915_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
diff --git a/sys/dev/drm/i915_drv.h b/sys/dev/drm/i915_drv.h
index 481bd2b..42706cd 100644
--- a/sys/dev/drm/i915_drv.h
+++ b/sys/dev/drm/i915_drv.h
@@ -1,10 +1,10 @@
 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*-
  */
 /*
- * 
+ *
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
@@ -12,11 +12,11 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
@@ -24,7 +24,7 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- * 
+ *
  */
 
 #include <sys/cdefs.h>
@@ -40,7 +40,12 @@ __FBSDID("$FreeBSD$");
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20060119"
+#define DRIVER_DATE		"20080312"
+
+#if defined(__linux__)
+#define I915_HAVE_FENCE
+#define I915_HAVE_BUFFER
+#endif
 
 /* Interface history:
  *
@@ -49,11 +54,32 @@ __FBSDID("$FreeBSD$");
  * 1.3: Add vblank support
  * 1.4: Fix cmdbuffer path, add heap destroy
  * 1.5: Add vblank pipe configuration
+ * 1.6: - New ioctl for scheduling buffer swaps on vertical blank
+ *      - Support vertical blank on secondary display pipe
+ * 1.8: New ioctl for ARB_Occlusion_Query
+ * 1.9: Usable page flipping and triple buffering
+ * 1.10: Plane/pipe disentangling
+ * 1.11: TTM superioctl
+ * 1.12: TTM relocation optimization
  */
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		5
+#if defined(I915_HAVE_FENCE) && defined(I915_HAVE_BUFFER)
+#define DRIVER_MINOR		13
+#else
+#define DRIVER_MINOR		6
+#endif
 #define DRIVER_PATCHLEVEL	0
 
+enum pipe {
+    PIPE_A = 0,
+    PIPE_B,
+};
+
+#ifdef I915_HAVE_BUFFER
+#define I915_MAX_VALIDATE_BUFFERS 4096
+struct drm_i915_validate_buffer;
+#endif
+
 typedef struct _drm_i915_ring_buffer {
 	int tail_mask;
 	unsigned long Start;
@@ -71,9 +97,33 @@ struct mem_block {
 	struct mem_block *prev;
 	int start;
 	int size;
-	DRMFILE filp;		/* 0: free, -1: heap, other: real files */
+	struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */
 };
 
+typedef struct _drm_i915_vbl_swap {
+	struct list_head head;
+	drm_drawable_t drw_id;
+	unsigned int plane;
+	unsigned int sequence;
+	int flip;
+} drm_i915_vbl_swap_t;
+
+#ifdef __linux__
+struct opregion_header;
+struct opregion_acpi;
+struct opregion_swsci;
+struct opregion_asle;
+
+struct intel_opregion {
+	struct opregion_header *header;
+	struct opregion_acpi *acpi;
+	struct opregion_swsci *swsci;
+	struct opregion_asle *asle;
+
+	int enabled;
+};
+#endif
+
 typedef struct drm_i915_private {
 	drm_local_map_t *sarea;
 	drm_local_map_t *mmio_map;
@@ -84,12 +134,11 @@ typedef struct drm_i915_private {
 	drm_dma_handle_t *status_page_dmah;
 	void *hw_status_page;
 	dma_addr_t dma_status_page;
-	unsigned long counter;
+	uint32_t counter;
+	unsigned int status_gfx_addr;
+	drm_local_map_t hws_map;
 
-	int back_offset;
-	int front_offset;
-	int current_page;
-	int page_flipping;
+	unsigned int cpp;
 	int use_mi_batchbuffer_start;
 
 	wait_queue_head_t irq_queue;
@@ -101,55 +150,259 @@ typedef struct drm_i915_private {
 	struct mem_block *agp_heap;
 	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
 	int vblank_pipe;
+	DRM_SPINTYPE user_irq_lock;
+	int user_irq_refcount;
+	int fence_irq_on;
+	uint32_t irq_enable_reg;
+	int irq_enabled;
+
+#ifdef I915_HAVE_FENCE
+	uint32_t flush_sequence;
+	uint32_t flush_flags;
+	uint32_t flush_pending;
+	uint32_t saved_flush_status;
+#endif
+#ifdef I915_HAVE_BUFFER
+	void *agp_iomap;
+	unsigned int max_validate_buffers;
+	struct mutex cmdbuf_mutex;
+	struct drm_i915_validate_buffer *val_bufs;
+#endif
+
+	DRM_SPINTYPE swaps_lock;
+	drm_i915_vbl_swap_t vbl_swaps;
+	unsigned int swaps_pending;
+#if defined(I915_HAVE_BUFFER)
+	/* DRI2 sarea */
+	struct drm_buffer_object *sarea_bo;
+	struct drm_bo_kmap_obj sarea_kmap;
+#endif
+
+#ifdef __linux__
+	struct intel_opregion opregion;
+#endif
+
+	/* Register state */
+	u8 saveLBB;
+	u32 saveDSPACNTR;
+	u32 saveDSPBCNTR;
+	u32 saveDSPARB;
+	u32 savePIPEACONF;
+	u32 savePIPEBCONF;
+	u32 savePIPEASRC;
+	u32 savePIPEBSRC;
+	u32 saveFPA0;
+	u32 saveFPA1;
+	u32 saveDPLL_A;
+	u32 saveDPLL_A_MD;
+	u32 saveHTOTAL_A;
+	u32 saveHBLANK_A;
+	u32 saveHSYNC_A;
+	u32 saveVTOTAL_A;
+	u32 saveVBLANK_A;
+	u32 saveVSYNC_A;
+	u32 saveBCLRPAT_A;
+	u32 savePIPEASTAT;
+	u32 saveDSPASTRIDE;
+	u32 saveDSPASIZE;
+	u32 saveDSPAPOS;
+	u32 saveDSPAADDR;
+	u32 saveDSPASURF;
+	u32 saveDSPATILEOFF;
+	u32 savePFIT_PGM_RATIOS;
+	u32 saveBLC_PWM_CTL;
+	u32 saveBLC_PWM_CTL2;
+	u32 saveFPB0;
+	u32 saveFPB1;
+	u32 saveDPLL_B;
+	u32 saveDPLL_B_MD;
+	u32 saveHTOTAL_B;
+	u32 saveHBLANK_B;
+	u32 saveHSYNC_B;
+	u32 saveVTOTAL_B;
+	u32 saveVBLANK_B;
+	u32 saveVSYNC_B;
+	u32 saveBCLRPAT_B;
+	u32 savePIPEBSTAT;
+	u32 saveDSPBSTRIDE;
+	u32 saveDSPBSIZE;
+	u32 saveDSPBPOS;
+	u32 saveDSPBADDR;
+	u32 saveDSPBSURF;
+	u32 saveDSPBTILEOFF;
+	u32 saveVGA0;
+	u32 saveVGA1;
+	u32 saveVGA_PD;
+	u32 saveVGACNTRL;
+	u32 saveADPA;
+	u32 saveLVDS;
+	u32 savePP_ON_DELAYS;
+	u32 savePP_OFF_DELAYS;
+	u32 saveDVOA;
+	u32 saveDVOB;
+	u32 saveDVOC;
+	u32 savePP_ON;
+	u32 savePP_OFF;
+	u32 savePP_CONTROL;
+	u32 savePP_DIVISOR;
+	u32 savePFIT_CONTROL;
+	u32 save_palette_a[256];
+	u32 save_palette_b[256];
+	u32 saveFBC_CFB_BASE;
+	u32 saveFBC_LL_BASE;
+	u32 saveFBC_CONTROL;
+	u32 saveFBC_CONTROL2;
+	u32 saveIER;
+	u32 saveIIR;
+	u32 saveIMR;
+	u32 saveCACHE_MODE_0;
+	u32 saveD_STATE;
+	u32 saveCG_2D_DIS;
+	u32 saveMI_ARB_STATE;
+	u32 saveSWF0[16];
+	u32 saveSWF1[16];
+	u32 saveSWF2[3];
+	u8 saveMSR;
+	u8 saveSR[8];
+	u8 saveGR[25];
+	u8 saveAR_INDEX;
+	u8 saveAR[21];
+	u8 saveDACMASK;
+	u8 saveDACDATA[256*3]; /* 256 3-byte colors */
+	u8 saveCR[37];
 } drm_i915_private_t;
 
-extern drm_ioctl_desc_t i915_ioctls[];
+enum intel_chip_family {
+	CHIP_I8XX = 0x01,
+	CHIP_I9XX = 0x02,
+	CHIP_I915 = 0x04,
+	CHIP_I965 = 0x08,
+};
+
+extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
 
 				/* i915_dma.c */
-extern void i915_kernel_lost_context(drm_device_t * dev);
+extern void i915_kernel_lost_context(struct drm_device * dev);
 extern int i915_driver_load(struct drm_device *, unsigned long flags);
-extern void i915_driver_lastclose(drm_device_t * dev);
-extern void i915_driver_preclose(drm_device_t * dev, DRMFILE filp);
-extern int i915_driver_device_is_agp(drm_device_t * dev);
+extern int i915_driver_unload(struct drm_device *);
+extern void i915_driver_lastclose(struct drm_device * dev);
+extern void i915_driver_preclose(struct drm_device *dev,
+				 struct drm_file *file_priv);
+extern int i915_driver_device_is_agp(struct drm_device * dev);
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
+extern void i915_emit_breadcrumb(struct drm_device *dev);
+extern void i915_dispatch_flip(struct drm_device * dev, int pipes, int sync);
+extern int i915_emit_mi_flush(struct drm_device *dev, uint32_t flush);
+extern int i915_driver_firstopen(struct drm_device *dev);
+extern int i915_dispatch_batchbuffer(struct drm_device * dev,
+				     drm_i915_batchbuffer_t * batch);
+extern int i915_quiescent(struct drm_device *dev);
 
 /* i915_irq.c */
-extern int i915_irq_emit(DRM_IOCTL_ARGS);
-extern int i915_irq_wait(DRM_IOCTL_ARGS);
+extern int i915_irq_emit(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+extern int i915_irq_wait(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
 
-extern int i915_driver_vblank_wait(drm_device_t *dev, unsigned int *sequence);
 extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS);
-extern void i915_driver_irq_preinstall(drm_device_t * dev);
-extern void i915_driver_irq_postinstall(drm_device_t * dev);
-extern void i915_driver_irq_uninstall(drm_device_t * dev);
-extern int i915_vblank_pipe_set(DRM_IOCTL_ARGS);
-extern int i915_vblank_pipe_get(DRM_IOCTL_ARGS);
+extern void i915_driver_irq_preinstall(struct drm_device * dev);
+extern int i915_driver_irq_postinstall(struct drm_device * dev);
+extern void i915_driver_irq_uninstall(struct drm_device * dev);
+extern int i915_vblank_pipe_set(struct drm_device *dev, void *data,
+				struct drm_file *file_priv);
+extern int i915_vblank_pipe_get(struct drm_device *dev, void *data,
+				struct drm_file *file_priv);
+extern int i915_emit_irq(struct drm_device * dev);
+extern int i915_enable_vblank(struct drm_device *dev, int crtc);
+extern void i915_disable_vblank(struct drm_device *dev, int crtc);
+extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int i915_vblank_swap(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+extern void i915_user_irq_on(drm_i915_private_t *dev_priv);
+extern void i915_user_irq_off(drm_i915_private_t *dev_priv);
 
 /* i915_mem.c */
-extern int i915_mem_alloc(DRM_IOCTL_ARGS);
-extern int i915_mem_free(DRM_IOCTL_ARGS);
-extern int i915_mem_init_heap(DRM_IOCTL_ARGS);
-extern int i915_mem_destroy_heap(DRM_IOCTL_ARGS);
+extern int i915_mem_alloc(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv);
+extern int i915_mem_free(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+extern int i915_mem_init_heap(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+extern int i915_mem_destroy_heap(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv);
 extern void i915_mem_takedown(struct mem_block **heap);
-extern void i915_mem_release(drm_device_t * dev,
-			     DRMFILE filp, struct mem_block *heap);
+extern void i915_mem_release(struct drm_device * dev,
+			     struct drm_file *file_priv,
+			     struct mem_block *heap);
+
+/* i915_suspend.c */
+extern int i915_save_state(struct drm_device *dev);
+extern int i915_restore_state(struct drm_device *dev);
+
+#ifdef I915_HAVE_FENCE
+/* i915_fence.c */
+extern void i915_fence_handler(struct drm_device *dev);
+extern void i915_invalidate_reported_sequence(struct drm_device *dev);
+
+#endif
+
+#ifdef I915_HAVE_BUFFER
+/* i915_buffer.c */
+extern struct drm_ttm_backend *i915_create_ttm_backend_entry(struct drm_device *dev);
+extern int i915_fence_type(struct drm_buffer_object *bo, uint32_t *fclass,
+			   uint32_t *type);
+extern int i915_invalidate_caches(struct drm_device *dev, uint64_t buffer_flags);
+extern int i915_init_mem_type(struct drm_device *dev, uint32_t type,
+			       struct drm_mem_type_manager *man);
+extern uint64_t i915_evict_flags(struct drm_buffer_object *bo);
+extern int i915_move(struct drm_buffer_object *bo, int evict,
+		int no_wait, struct drm_bo_mem_reg *new_mem);
+void i915_flush_ttm(struct drm_ttm *ttm);
+/* i915_execbuf.c */
+int i915_execbuffer(struct drm_device *dev, void *data,
+				   struct drm_file *file_priv);
+
+#endif
+
+#ifdef __linux__
+/* i915_opregion.c */
+extern int intel_opregion_init(struct drm_device *dev);
+extern void intel_opregion_free(struct drm_device *dev);
+extern void opregion_asle_intr(struct drm_device *dev);
+extern void opregion_enable_asle(struct drm_device *dev);
+#endif
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+extern void intel_init_chipset_flush_compat(struct drm_device *dev);
+extern void intel_fini_chipset_flush_compat(struct drm_device *dev);
+#endif
+#endif
 
 #define I915_READ(reg)          DRM_READ32(dev_priv->mmio_map, (reg))
 #define I915_WRITE(reg,val)     DRM_WRITE32(dev_priv->mmio_map, (reg), (val))
-#define I915_READ16(reg) 	DRM_READ16(dev_priv->mmio_map, (reg))
+#define I915_READ16(reg)	DRM_READ16(dev_priv->mmio_map, (reg))
 #define I915_WRITE16(reg,val)	DRM_WRITE16(dev_priv->mmio_map, (reg), (val))
+#define I915_READ8(reg)		DRM_READ8(dev_priv->mmio_map, (reg))
+#define I915_WRITE8(reg,val)	DRM_WRITE8(dev_priv->mmio_map, (reg), (val))
+
+#if defined(__FreeBSD__)
+typedef boolean_t bool;
+#endif
 
 #define I915_VERBOSE 0
 
+#define PRIMARY_RINGBUFFER_SIZE         (128*1024)
+
 #define RING_LOCALS	unsigned int outring, ringmask, outcount; \
 			volatile char *virt;
 
 #define BEGIN_LP_RING(n) do {				\
 	if (I915_VERBOSE)				\
-		DRM_DEBUG("BEGIN_LP_RING(%d) in %s\n",	\
-	                         (n), __FUNCTION__);           \
+		DRM_DEBUG("BEGIN_LP_RING(%d)\n",	\
+	                         (n));		        \
 	if (dev_priv->ring.space < (n)*4)                      \
 		i915_wait_ring(dev, (n)*4, __FUNCTION__);      \
 	outcount = 0;					\
@@ -170,35 +423,367 @@ extern void i915_mem_release(drm_device_t * dev,
 	if (I915_VERBOSE) DRM_DEBUG("ADVANCE_LP_RING %x\n", outring);	\
 	dev_priv->ring.tail = outring;					\
 	dev_priv->ring.space -= outcount * 4;				\
-	I915_WRITE(LP_RING + RING_TAIL, outring);			\
+	I915_WRITE(PRB0_TAIL, outring);			\
 } while(0)
 
-extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
+extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
+
+/*
+ * The Bridge device's PCI config space has information about the
+ * fb aperture size and the amount of pre-reserved memory.
+ */
+#define INTEL_GMCH_CTRL		0x52
+#define INTEL_GMCH_ENABLED	0x4
+#define INTEL_GMCH_MEM_MASK	0x1
+#define INTEL_GMCH_MEM_64M	0x1
+#define INTEL_GMCH_MEM_128M	0
+
+#define INTEL_855_GMCH_GMS_MASK		(0x7 << 4)
+#define INTEL_855_GMCH_GMS_DISABLED	(0x0 << 4)
+#define INTEL_855_GMCH_GMS_STOLEN_1M	(0x1 << 4)
+#define INTEL_855_GMCH_GMS_STOLEN_4M	(0x2 << 4)
+#define INTEL_855_GMCH_GMS_STOLEN_8M	(0x3 << 4)
+#define INTEL_855_GMCH_GMS_STOLEN_16M	(0x4 << 4)
+#define INTEL_855_GMCH_GMS_STOLEN_32M	(0x5 << 4)
+
+#define INTEL_915G_GMCH_GMS_STOLEN_48M	(0x6 << 4)
+#define INTEL_915G_GMCH_GMS_STOLEN_64M	(0x7 << 4)
+
+/* PCI config space */
+
+#define HPLLCC	0xc0 /* 855 only */
+#define   GC_CLOCK_CONTROL_MASK		(3 << 0)
+#define   GC_CLOCK_133_200		(0 << 0)
+#define   GC_CLOCK_100_200		(1 << 0)
+#define   GC_CLOCK_100_133		(2 << 0)
+#define   GC_CLOCK_166_250		(3 << 0)
+#define GCFGC	0xf0 /* 915+ only */
+#define   GC_LOW_FREQUENCY_ENABLE	(1 << 7)
+#define   GC_DISPLAY_CLOCK_190_200_MHZ	(0 << 4)
+#define   GC_DISPLAY_CLOCK_333_MHZ	(4 << 4)
+#define   GC_DISPLAY_CLOCK_MASK		(7 << 4)
+#define LBB	0xf4
+
+/* VGA stuff */
+
+#define VGA_ST01_MDA 0x3ba
+#define VGA_ST01_CGA 0x3da
+
+#define VGA_MSR_WRITE 0x3c2
+#define VGA_MSR_READ 0x3cc
+#define   VGA_MSR_MEM_EN (1<<1)
+#define   VGA_MSR_CGA_MODE (1<<0)
+
+#define VGA_SR_INDEX 0x3c4
+#define VGA_SR_DATA 0x3c5
+
+#define VGA_AR_INDEX 0x3c0
+#define   VGA_AR_VID_EN (1<<5)
+#define VGA_AR_DATA_WRITE 0x3c0
+#define VGA_AR_DATA_READ 0x3c1
+
+#define VGA_GR_INDEX 0x3ce
+#define VGA_GR_DATA 0x3cf
+/* GR05 */
+#define   VGA_GR_MEM_READ_MODE_SHIFT 3
+#define     VGA_GR_MEM_READ_MODE_PLANE 1
+/* GR06 */
+#define   VGA_GR_MEM_MODE_MASK 0xc
+#define   VGA_GR_MEM_MODE_SHIFT 2
+#define   VGA_GR_MEM_A0000_AFFFF 0
+#define   VGA_GR_MEM_A0000_BFFFF 1
+#define   VGA_GR_MEM_B0000_B7FFF 2
+#define   VGA_GR_MEM_B0000_BFFFF 3
+
+#define VGA_DACMASK 0x3c6
+#define VGA_DACRX 0x3c7
+#define VGA_DACWX 0x3c8
+#define VGA_DACDATA 0x3c9
+
+#define VGA_CR_INDEX_MDA 0x3b4
+#define VGA_CR_DATA_MDA 0x3b5
+#define VGA_CR_INDEX_CGA 0x3d4
+#define VGA_CR_DATA_CGA 0x3d5
+
+/*
+ * Memory interface instructions used by the kernel
+ */
+#define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags))
+
+#define MI_NOOP			MI_INSTR(0, 0)
+#define MI_USER_INTERRUPT	MI_INSTR(0x02, 0)
+#define MI_WAIT_FOR_EVENT       MI_INSTR(0x03, 0)
+#define   MI_WAIT_FOR_PLANE_B_FLIP      (1<<6)
+#define   MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
+#define   MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
+#define MI_FLUSH		MI_INSTR(0x04, 0)
+#define   MI_READ_FLUSH		(1 << 0)
+#define   MI_EXE_FLUSH		(1 << 1)
+#define   MI_NO_WRITE_FLUSH	(1 << 2)
+#define   MI_SCENE_COUNT	(1 << 3) /* just increment scene count */
+#define   MI_END_SCENE		(1 << 4) /* flush binner and incr scene count */
+#define MI_BATCH_BUFFER_END	MI_INSTR(0x0a, 0)
+#define MI_REPORT_HEAD		MI_INSTR(0x07, 0)
+#define MI_LOAD_SCAN_LINES_INCL MI_INSTR(0x12, 0)
+#define MI_STORE_DWORD_IMM	MI_INSTR(0x20, 1)
+#define   MI_MEM_VIRTUAL	(1 << 22) /* 965+ only */
+#define MI_STORE_DWORD_INDEX	MI_INSTR(0x21, 1)
+#define MI_LOAD_REGISTER_IMM	MI_INSTR(0x22, 1)
+#define MI_BATCH_BUFFER		MI_INSTR(0x30, 1)
+#define   MI_BATCH_NON_SECURE	(1)
+#define   MI_BATCH_NON_SECURE_I965 (1<<8)
+#define MI_BATCH_BUFFER_START	MI_INSTR(0x31, 0)
+
+#define BREADCRUMB_BITS 31
+#define BREADCRUMB_MASK ((1U << BREADCRUMB_BITS) - 1)
+
+#define READ_BREADCRUMB(dev_priv)  (((volatile u32*)(dev_priv->hw_status_page))[5])
+
+/**
+ * Reads a dword out of the status page, which is written to from the command
+ * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
+ * MI_STORE_DATA_IMM.
+ *
+ * The following dwords have a reserved meaning:
+ * 0: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
+ * 4: ring 0 head pointer
+ * 5: ring 1 head pointer (915-class)
+ * 6: ring 2 head pointer (915-class)
+ *
+ * The area from dword 0x10 to 0x3ff is available for driver usage.
+ */
+#define READ_HWSP(dev_priv, reg)  (((volatile u32*)(dev_priv->hw_status_page))[reg])
+#define I915_GEM_HWS_INDEX		0x10
+
+/*
+ * 3D instructions used by the kernel
+ */
+#define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags))
 
-#define GFX_OP_USER_INTERRUPT 		((0<<29)|(2<<23))
-#define GFX_OP_BREAKPOINT_INTERRUPT	((0<<29)|(1<<23))
-#define CMD_REPORT_HEAD			(7<<23)
-#define CMD_STORE_DWORD_IDX		((0x21<<23) | 0x1)
-#define CMD_OP_BATCH_BUFFER  ((0x0<<29)|(0x30<<23)|0x1)
+#define GFX_OP_RASTER_RULES    ((0x3<<29)|(0x7<<24))
+#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
+#define   SC_UPDATE_SCISSOR       (0x1<<1)
+#define   SC_ENABLE_MASK          (0x1<<0)
+#define   SC_ENABLE               (0x1<<0)
+#define GFX_OP_LOAD_INDIRECT   ((0x3<<29)|(0x1d<<24)|(0x7<<16))
+#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
+#define   SCI_YMIN_MASK      (0xffff<<16)
+#define   SCI_XMIN_MASK      (0xffff<<0)
+#define   SCI_YMAX_MASK      (0xffff<<16)
+#define   SCI_XMAX_MASK      (0xffff<<0)
+#define GFX_OP_SCISSOR_ENABLE	 ((0x3<<29)|(0x1c<<24)|(0x10<<19))
+#define GFX_OP_SCISSOR_RECT	 ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
+#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
+#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
+#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
+#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
+#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
+#define GFX_OP_DRAWRECT_INFO_I965  ((0x7900<<16)|0x2)
+#define SRC_COPY_BLT_CMD                ((2<<29)|(0x43<<22)|4)
+#define XY_SRC_COPY_BLT_CMD		((2<<29)|(0x53<<22)|6)
+#define XY_MONO_SRC_COPY_IMM_BLT	((2<<29)|(0x71<<22)|5)
+#define XY_SRC_COPY_BLT_WRITE_ALPHA	(1<<21)
+#define XY_SRC_COPY_BLT_WRITE_RGB	(1<<20)
+#define   BLT_DEPTH_8			(0<<24)
+#define   BLT_DEPTH_16_565		(1<<24)
+#define   BLT_DEPTH_16_1555		(2<<24)
+#define   BLT_DEPTH_32			(3<<24)
+#define   BLT_ROP_GXCOPY		(0xcc<<16)
+#define XY_SRC_COPY_BLT_SRC_TILED	(1<<15) /* 965+ only */
+#define XY_SRC_COPY_BLT_DST_TILED	(1<<11) /* 965+ only */
+#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
+#define   ASYNC_FLIP                (1<<22)
+#define   DISPLAY_PLANE_A           (0<<20)
+#define   DISPLAY_PLANE_B           (1<<20)
 
-#define INST_PARSER_CLIENT   0x00000000
-#define INST_OP_FLUSH        0x02000000
-#define INST_FLUSH_MAP_CACHE 0x00000001
+/*
+ * Instruction and interrupt control regs
+ */
+
+#define PRB0_TAIL	0x02030
+#define PRB0_HEAD	0x02034
+#define PRB0_START	0x02038
+#define PRB0_CTL	0x0203c
+#define   TAIL_ADDR		0x001FFFF8
+#define   HEAD_WRAP_COUNT	0xFFE00000
+#define   HEAD_WRAP_ONE		0x00200000
+#define   HEAD_ADDR		0x001FFFFC
+#define   RING_NR_PAGES		0x001FF000
+#define   RING_REPORT_MASK	0x00000006
+#define   RING_REPORT_64K	0x00000002
+#define   RING_REPORT_128K	0x00000004
+#define   RING_NO_REPORT	0x00000000
+#define   RING_VALID_MASK	0x00000001
+#define   RING_VALID		0x00000001
+#define   RING_INVALID		0x00000000
+#define PRB1_TAIL	0x02040 /* 915+ only */
+#define PRB1_HEAD	0x02044 /* 915+ only */
+#define PRB1_START	0x02048 /* 915+ only */
+#define PRB1_CTL	0x0204c /* 915+ only */
+#define HWS_PGA		0x02080
+#define IPEIR		0x02088
+#define NOPID		0x02094
+#define HWSTAM		0x02098
+#define SCPD0		0x0209c /* 915+ only */
+#define IER		0x020a0
+#define IIR		0x020a4
+#define IMR		0x020a8
+#define ISR		0x020ac
+#define   I915_PIPE_CONTROL_NOTIFY_INTERRUPT		(1<<18)
+#define   I915_DISPLAY_PORT_INTERRUPT			(1<<17)
+#define   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT	(1<<15)
+#define   I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT	(1<<14)
+#define   I915_HWB_OOM_INTERRUPT			(1<<13)
+#define   I915_SYNC_STATUS_INTERRUPT			(1<<12)
+#define   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT	(1<<11)
+#define   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT	(1<<10)
+#define   I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT	(1<<9)
+#define   I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT	(1<<8)
+#define   I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT		(1<<7)
+#define   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT		(1<<6)
+#define   I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT		(1<<5)
+#define   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT		(1<<4)
+#define   I915_DEBUG_INTERRUPT				(1<<2)
+#define   I915_USER_INTERRUPT				(1<<1)
+#define   I915_ASLE_INTERRUPT				(1<<0)
+#define EIR		0x020b0
+#define EMR		0x020b4
+#define ESR		0x020b8
+#define INSTPM	        0x020c0
+#define FW_BLC		0x020d8
+#define FW_BLC_SELF	0x020e0 /* 915+ only */
+#define MI_ARB_STATE	0x020e4 /* 915+ only */
+#define CACHE_MODE_0	0x02120 /* 915+ only */
+#define   CM0_MASK_SHIFT          16
+#define   CM0_IZ_OPT_DISABLE      (1<<6)
+#define   CM0_ZR_OPT_DISABLE      (1<<5)
+#define   CM0_DEPTH_EVICT_DISABLE (1<<4)
+#define   CM0_COLOR_EVICT_DISABLE (1<<3)
+#define   CM0_DEPTH_WRITE_DISABLE (1<<1)
+#define   CM0_RC_OP_FLUSH_DISABLE (1<<0)
+#define GFX_FLSH_CNTL	0x02170 /* 915+ only */
+
+/*
+ * Framebuffer compression (915+ only)
+ */
+
+#define FBC_CFB_BASE		0x03200 /* 4k page aligned */
+#define FBC_LL_BASE		0x03204 /* 4k page aligned */
+#define FBC_CONTROL		0x03208
+#define   FBC_CTL_EN		(1<<31)
+#define   FBC_CTL_PERIODIC	(1<<30)
+#define   FBC_CTL_INTERVAL_SHIFT (16)
+#define   FBC_CTL_UNCOMPRESSIBLE (1<<14)
+#define   FBC_CTL_STRIDE_SHIFT	(5)
+#define   FBC_CTL_FENCENO	(1<<0)
+#define FBC_COMMAND		0x0320c
+#define   FBC_CMD_COMPRESS	(1<<0)
+#define FBC_STATUS		0x03210
+#define   FBC_STAT_COMPRESSING	(1<<31)
+#define   FBC_STAT_COMPRESSED	(1<<30)
+#define   FBC_STAT_MODIFIED	(1<<29)
+#define   FBC_STAT_CURRENT_LINE	(1<<0)
+#define FBC_CONTROL2		0x03214
+#define   FBC_CTL_FENCE_DBL	(0<<4)
+#define   FBC_CTL_IDLE_IMM	(0<<2)
+#define   FBC_CTL_IDLE_FULL	(1<<2)
+#define   FBC_CTL_IDLE_LINE	(2<<2)
+#define   FBC_CTL_IDLE_DEBUG	(3<<2)
+#define   FBC_CTL_CPU_FENCE	(1<<1)
+#define   FBC_CTL_PLANEA	(0<<0)
+#define   FBC_CTL_PLANEB	(1<<0)
+#define FBC_FENCE_OFF		0x0321b
+
+#define FBC_LL_SIZE		(1536)
+
+/*
+ * GPIO regs
+ */
+
+#define GPIOA			0x5010
+#define GPIOB			0x5014
+#define GPIOC			0x5018
+#define GPIOD			0x501c
+#define GPIOE			0x5020
+#define GPIOF			0x5024
+#define GPIOG			0x5028
+#define GPIOH			0x502c
+# define GPIO_CLOCK_DIR_MASK		(1 << 0)
+# define GPIO_CLOCK_DIR_IN		(0 << 1)
+# define GPIO_CLOCK_DIR_OUT		(1 << 1)
+# define GPIO_CLOCK_VAL_MASK		(1 << 2)
+# define GPIO_CLOCK_VAL_OUT		(1 << 3)
+# define GPIO_CLOCK_VAL_IN		(1 << 4)
+# define GPIO_CLOCK_PULLUP_DISABLE	(1 << 5)
+# define GPIO_DATA_DIR_MASK		(1 << 8)
+# define GPIO_DATA_DIR_IN		(0 << 9)
+# define GPIO_DATA_DIR_OUT		(1 << 9)
+# define GPIO_DATA_VAL_MASK		(1 << 10)
+# define GPIO_DATA_VAL_OUT		(1 << 11)
+# define GPIO_DATA_VAL_IN		(1 << 12)
+# define GPIO_DATA_PULLUP_DISABLE	(1 << 13)
+
+/*
+ * Clock control & power management
+ */
 
-#define BB1_START_ADDR_MASK   (~0x7)
-#define BB1_PROTECTED         (1<<0)
-#define BB1_UNPROTECTED       (0<<0)
-#define BB2_END_ADDR_MASK     (~0x7)
+#define VGA0	0x6000
+#define VGA1	0x6004
+#define VGA_PD	0x6010
+#define   VGA0_PD_P2_DIV_4	(1 << 7)
+#define   VGA0_PD_P1_DIV_2	(1 << 5)
+#define   VGA0_PD_P1_SHIFT	0
+#define   VGA0_PD_P1_MASK	(0x1f << 0)
+#define   VGA1_PD_P2_DIV_4	(1 << 15)
+#define   VGA1_PD_P1_DIV_2	(1 << 13)
+#define   VGA1_PD_P1_SHIFT	8
+#define   VGA1_PD_P1_MASK	(0x1f << 8)
+#define DPLL_A	0x06014
+#define DPLL_B	0x06018
+#define   DPLL_VCO_ENABLE		(1 << 31)
+#define   DPLL_DVO_HIGH_SPEED		(1 << 30)
+#define   DPLL_SYNCLOCK_ENABLE		(1 << 29)
+#define   DPLL_VGA_MODE_DIS		(1 << 28)
+#define   DPLLB_MODE_DAC_SERIAL		(1 << 26) /* i915 */
+#define   DPLLB_MODE_LVDS		(2 << 26) /* i915 */
+#define   DPLL_MODE_MASK		(3 << 26)
+#define   DPLL_DAC_SERIAL_P2_CLOCK_DIV_10 (0 << 24) /* i915 */
+#define   DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 (1 << 24) /* i915 */
+#define   DPLLB_LVDS_P2_CLOCK_DIV_14	(0 << 24) /* i915 */
+#define   DPLLB_LVDS_P2_CLOCK_DIV_7	(1 << 24) /* i915 */
+#define   DPLL_P2_CLOCK_DIV_MASK	0x03000000 /* i915 */
+#define   DPLL_FPA01_P1_POST_DIV_MASK	0x00ff0000 /* i915 */
 
-#define I915REG_HWSTAM		0x02098
-#define I915REG_INT_IDENTITY_R	0x020a4
-#define I915REG_INT_MASK_R 	0x020a8
-#define I915REG_INT_ENABLE_R	0x020a0
+#define I915_FIFO_UNDERRUN_STATUS		(1UL<<31)
+#define I915_CRC_ERROR_ENABLE			(1UL<<29)
+#define I915_CRC_DONE_ENABLE			(1UL<<28)
+#define I915_GMBUS_EVENT_ENABLE			(1UL<<27)
+#define I915_VSYNC_INTERRUPT_ENABLE		(1UL<<25)
+#define I915_DISPLAY_LINE_COMPARE_ENABLE	(1UL<<24)
+#define I915_DPST_EVENT_ENABLE			(1UL<<23)
+#define I915_LEGACY_BLC_EVENT_ENABLE		(1UL<<22)
+#define I915_ODD_FIELD_INTERRUPT_ENABLE		(1UL<<21)
+#define I915_EVEN_FIELD_INTERRUPT_ENABLE	(1UL<<20)
+#define I915_START_VBLANK_INTERRUPT_ENABLE	(1UL<<18)	/* 965 or later */
+#define I915_VBLANK_INTERRUPT_ENABLE		(1UL<<17)
+#define I915_OVERLAY_UPDATED_ENABLE		(1UL<<16)
+#define I915_CRC_ERROR_INTERRUPT_STATUS		(1UL<<13)
+#define I915_CRC_DONE_INTERRUPT_STATUS		(1UL<<12)
+#define I915_GMBUS_INTERRUPT_STATUS		(1UL<<11)
+#define I915_VSYNC_INTERRUPT_STATUS		(1UL<<9)
+#define I915_DISPLAY_LINE_COMPARE_STATUS	(1UL<<8)
+#define I915_DPST_EVENT_STATUS			(1UL<<7)
+#define I915_LEGACY_BLC_EVENT_STATUS		(1UL<<6)
+#define I915_ODD_FIELD_INTERRUPT_STATUS		(1UL<<5)
+#define I915_EVEN_FIELD_INTERRUPT_STATUS	(1UL<<4)
+#define I915_START_VBLANK_INTERRUPT_STATUS	(1UL<<2)	/* 965 or later */
+#define I915_VBLANK_INTERRUPT_STATUS		(1UL<<1)
+#define I915_OVERLAY_UPDATED_STATUS		(1UL<<0)
 
 #define SRX_INDEX		0x3c4
 #define SRX_DATA		0x3c5
 #define SR01			1
-#define SR01_SCREEN_OFF 	(1<<5)
+#define SR01_SCREEN_OFF		(1<<5)
 
 #define PPCR			0x61204
 #define PPCR_ON			(1<<0)
@@ -217,64 +802,1054 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define ADPA_DPMS_STANDBY	(2<<10)
 #define ADPA_DPMS_OFF		(3<<10)
 
-#define NOPID                   0x2094
-#define LP_RING     		0x2030
-#define HP_RING     		0x2040
-#define RING_TAIL      		0x00
+#define RING_TAIL		0x00
 #define TAIL_ADDR		0x001FFFF8
-#define RING_HEAD      		0x04
-#define HEAD_WRAP_COUNT     	0xFFE00000
-#define HEAD_WRAP_ONE       	0x00200000
-#define HEAD_ADDR           	0x001FFFFC
-#define RING_START     		0x08
-#define START_ADDR          	0x0xFFFFF000
-#define RING_LEN       		0x0C
-#define RING_NR_PAGES       	0x001FF000
-#define RING_REPORT_MASK    	0x00000006
-#define RING_REPORT_64K     	0x00000002
-#define RING_REPORT_128K    	0x00000004
-#define RING_NO_REPORT      	0x00000000
-#define RING_VALID_MASK     	0x00000001
-#define RING_VALID          	0x00000001
-#define RING_INVALID        	0x00000000
+#define RING_HEAD		0x04
+#define HEAD_WRAP_COUNT		0xFFE00000
+#define HEAD_WRAP_ONE		0x00200000
+#define HEAD_ADDR		0x001FFFFC
+#define RING_START		0x08
+#define START_ADDR		0xFFFFF000
+#define RING_LEN		0x0C
+#define RING_NR_PAGES		0x001FF000
+#define RING_REPORT_MASK	0x00000006
+#define RING_REPORT_64K		0x00000002
+#define RING_REPORT_128K	0x00000004
+#define RING_NO_REPORT		0x00000000
+#define RING_VALID_MASK		0x00000001
+#define RING_VALID		0x00000001
+#define RING_INVALID		0x00000000
 
-#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define SC_UPDATE_SCISSOR       (0x1<<1)
-#define SC_ENABLE_MASK          (0x1<<0)
-#define SC_ENABLE               (0x1<<0)
+/* Scratch pad debug 0 reg:
+ */
+#define   DPLL_FPA01_P1_POST_DIV_MASK_I830	0x001f0000
+/*
+ * The i830 generation, in LVDS mode, defines P1 as the bit number set within
+ * this field (only one bit may be set).
+ */
+#define   DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS	0x003f0000
+#define   DPLL_FPA01_P1_POST_DIV_SHIFT	16
+/* i830, required in DVO non-gang */
+#define   PLL_P2_DIVIDE_BY_4		(1 << 23)
+#define   PLL_P1_DIVIDE_BY_TWO		(1 << 21) /* i830 */
+#define   PLL_REF_INPUT_DREFCLK		(0 << 13)
+#define   PLL_REF_INPUT_TVCLKINA	(1 << 13) /* i830 */
+#define   PLL_REF_INPUT_TVCLKINBC	(2 << 13) /* SDVO TVCLKIN */
+#define   PLLB_REF_INPUT_SPREADSPECTRUMIN (3 << 13)
+#define   PLL_REF_INPUT_MASK		(3 << 13)
+#define   PLL_LOAD_PULSE_PHASE_SHIFT		9
+/*
+ * Parallel to Serial Load Pulse phase selection.
+ * Selects the phase for the 10X DPLL clock for the PCIe
+ * digital display port. The range is 4 to 13; 10 or more
+ * is just a flip delay. The default is 6
+ */
+#define   PLL_LOAD_PULSE_PHASE_MASK		(0xf << PLL_LOAD_PULSE_PHASE_SHIFT)
+#define   DISPLAY_RATE_SELECT_FPA1		(1 << 8)
+/*
+ * SDVO multiplier for 945G/GM. Not used on 965.
+ */
+#define   SDVO_MULTIPLIER_MASK			0x000000ff
+#define   SDVO_MULTIPLIER_SHIFT_HIRES		4
+#define   SDVO_MULTIPLIER_SHIFT_VGA		0
+#define DPLL_A_MD 0x0601c /* 965+ only */
+/*
+ * UDI pixel divider, controlling how many pixels are stuffed into a packet.
+ *
+ * Value is pixels minus 1.  Must be set to 1 pixel for SDVO.
+ */
+#define   DPLL_MD_UDI_DIVIDER_MASK		0x3f000000
+#define   DPLL_MD_UDI_DIVIDER_SHIFT		24
+/* UDI pixel divider for VGA, same as DPLL_MD_UDI_DIVIDER_MASK. */
+#define   DPLL_MD_VGA_UDI_DIVIDER_MASK		0x003f0000
+#define   DPLL_MD_VGA_UDI_DIVIDER_SHIFT		16
+/*
+ * SDVO/UDI pixel multiplier.
+ *
+ * SDVO requires that the bus clock rate be between 1 and 2 Ghz, and the bus
+ * clock rate is 10 times the DPLL clock.  At low resolution/refresh rate
+ * modes, the bus rate would be below the limits, so SDVO allows for stuffing
+ * dummy bytes in the datastream at an increased clock rate, with both sides of
+ * the link knowing how many bytes are fill.
+ *
+ * So, for a mode with a dotclock of 65Mhz, we would want to double the clock
+ * rate to 130Mhz to get a bus rate of 1.30Ghz.  The DPLL clock rate would be
+ * set to 130Mhz, and the SDVO multiplier set to 2x in this register and
+ * through an SDVO command.
+ *
+ * This register field has values of multiplication factor minus 1, with
+ * a maximum multiplier of 5 for SDVO.
+ */
+#define   DPLL_MD_UDI_MULTIPLIER_MASK		0x00003f00
+#define   DPLL_MD_UDI_MULTIPLIER_SHIFT		8
+/*
+ * SDVO/UDI pixel multiplier for VGA, same as DPLL_MD_UDI_MULTIPLIER_MASK.
+ * This best be set to the default value (3) or the CRT won't work. No,
+ * I don't entirely understand what this does...
+ */
+#define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK	0x0000003f
+#define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT	0
+#define DPLL_B_MD 0x06020 /* 965+ only */
+#define FPA0	0x06040
+#define FPA1	0x06044
+#define FPB0	0x06048
+#define FPB1	0x0604c
+#define   FP_N_DIV_MASK		0x003f0000
+#define   FP_N_DIV_SHIFT		16
+#define   FP_M1_DIV_MASK	0x00003f00
+#define   FP_M1_DIV_SHIFT		 8
+#define   FP_M2_DIV_MASK	0x0000003f
+#define   FP_M2_DIV_SHIFT		 0
+#define DPLL_TEST	0x606c
+#define   DPLLB_TEST_SDVO_DIV_1		(0 << 22)
+#define   DPLLB_TEST_SDVO_DIV_2		(1 << 22)
+#define   DPLLB_TEST_SDVO_DIV_4		(2 << 22)
+#define   DPLLB_TEST_SDVO_DIV_MASK	(3 << 22)
+#define   DPLLB_TEST_N_BYPASS		(1 << 19)
+#define   DPLLB_TEST_M_BYPASS		(1 << 18)
+#define   DPLLB_INPUT_BUFFER_ENABLE	(1 << 16)
+#define   DPLLA_TEST_N_BYPASS		(1 << 3)
+#define   DPLLA_TEST_M_BYPASS		(1 << 2)
+#define   DPLLA_INPUT_BUFFER_ENABLE	(1 << 0)
+#define D_STATE		0x6104
+#define CG_2D_DIS	0x6200
+#define CG_3D_DIS	0x6204
 
-#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
-#define SCI_YMIN_MASK      (0xffff<<16)
-#define SCI_XMIN_MASK      (0xffff<<0)
-#define SCI_YMAX_MASK      (0xffff<<16)
-#define SCI_XMAX_MASK      (0xffff<<0)
+/*
+ * Palette regs
+ */
 
-#define GFX_OP_SCISSOR_ENABLE	 ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define GFX_OP_SCISSOR_RECT	 ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
-#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
-#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
-#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
+#define PALETTE_A		0x0a000
+#define PALETTE_B		0x0a800
 
-#define GFX_OP_DRAWRECT_INFO_I965  ((0x7900<<16)|0x2)
+/*
+ * Overlay regs
+ */
 
-#define MI_BATCH_BUFFER 	((0x30<<23)|1)
-#define MI_BATCH_BUFFER_START 	(0x31<<23)
-#define MI_BATCH_BUFFER_END 	(0xA<<23)
-#define MI_BATCH_NON_SECURE	(1)
+#define OVADD			0x30000
+#define DOVSTA			0x30008
+#define OC_BUF			(0x3<<20)
+#define OGAMC5			0x30010
+#define OGAMC4			0x30014
+#define OGAMC3			0x30018
+#define OGAMC2			0x3001c
+#define OGAMC1			0x30020
+#define OGAMC0			0x30024
 
-#define MI_WAIT_FOR_EVENT       ((0x3<<23))
-#define MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
-#define MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
+/*
+ * Display engine regs
+ */
 
-#define MI_LOAD_SCAN_LINES_INCL  ((0x12<<23))
+/* Pipe A timing regs */
+#define HTOTAL_A	0x60000
+#define HBLANK_A	0x60004
+#define HSYNC_A		0x60008
+#define VTOTAL_A	0x6000c
+#define VBLANK_A	0x60010
+#define VSYNC_A		0x60014
+#define PIPEASRC	0x6001c
+#define BCLRPAT_A	0x60020
 
-#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
-#define ASYNC_FLIP                (1<<22)
+/* Pipe B timing regs */
+#define HTOTAL_B	0x61000
+#define HBLANK_B	0x61004
+#define HSYNC_B		0x61008
+#define VTOTAL_B	0x6100c
+#define VBLANK_B	0x61010
+#define VSYNC_B		0x61014
+#define PIPEBSRC	0x6101c
+#define BCLRPAT_B	0x61020
+
+/* VGA port control */
+#define ADPA			0x61100
+#define   ADPA_DAC_ENABLE	(1<<31)
+#define   ADPA_DAC_DISABLE	0
+#define   ADPA_PIPE_SELECT_MASK	(1<<30)
+#define   ADPA_PIPE_A_SELECT	0
+#define   ADPA_PIPE_B_SELECT	(1<<30)
+#define   ADPA_USE_VGA_HVPOLARITY (1<<15)
+#define   ADPA_SETS_HVPOLARITY	0
+#define   ADPA_VSYNC_CNTL_DISABLE (1<<11)
+#define   ADPA_VSYNC_CNTL_ENABLE 0
+#define   ADPA_HSYNC_CNTL_DISABLE (1<<10)
+#define   ADPA_HSYNC_CNTL_ENABLE 0
+#define   ADPA_VSYNC_ACTIVE_HIGH (1<<4)
+#define   ADPA_VSYNC_ACTIVE_LOW	0
+#define   ADPA_HSYNC_ACTIVE_HIGH (1<<3)
+#define   ADPA_HSYNC_ACTIVE_LOW	0
+#define   ADPA_DPMS_MASK	(~(3<<10))
+#define   ADPA_DPMS_ON		(0<<10)
+#define   ADPA_DPMS_SUSPEND	(1<<10)
+#define   ADPA_DPMS_STANDBY	(2<<10)
+#define   ADPA_DPMS_OFF		(3<<10)
+
+/* Hotplug control (945+ only) */
+#define PORT_HOTPLUG_EN		0x61110
+#define   SDVOB_HOTPLUG_INT_EN			(1 << 26)
+#define   SDVOC_HOTPLUG_INT_EN			(1 << 25)
+#define   TV_HOTPLUG_INT_EN			(1 << 18)
+#define   CRT_HOTPLUG_INT_EN			(1 << 9)
+#define   CRT_HOTPLUG_FORCE_DETECT		(1 << 3)
+
+#define PORT_HOTPLUG_STAT	0x61114
+#define   CRT_HOTPLUG_INT_STATUS		(1 << 11)
+#define   TV_HOTPLUG_INT_STATUS			(1 << 10)
+#define   CRT_HOTPLUG_MONITOR_MASK		(3 << 8)
+#define   CRT_HOTPLUG_MONITOR_COLOR		(3 << 8)
+#define   CRT_HOTPLUG_MONITOR_MONO		(2 << 8)
+#define   CRT_HOTPLUG_MONITOR_NONE		(0 << 8)
+#define   SDVOC_HOTPLUG_INT_STATUS		(1 << 7)
+#define   SDVOB_HOTPLUG_INT_STATUS		(1 << 6)
+
+/* SDVO port control */
+#define SDVOB			0x61140
+#define SDVOC			0x61160
+#define   SDVO_ENABLE		(1 << 31)
+#define   SDVO_PIPE_B_SELECT	(1 << 30)
+#define   SDVO_STALL_SELECT	(1 << 29)
+#define   SDVO_INTERRUPT_ENABLE	(1 << 26)
+/**
+ * 915G/GM SDVO pixel multiplier.
+ *
+ * Programmed value is multiplier - 1, up to 5x.
+ *
+ * \sa DPLL_MD_UDI_MULTIPLIER_MASK
+ */
+#define   SDVO_PORT_MULTIPLY_MASK	(7 << 23)
+#define   SDVO_PORT_MULTIPLY_SHIFT		23
+#define   SDVO_PHASE_SELECT_MASK	(15 << 19)
+#define   SDVO_PHASE_SELECT_DEFAULT	(6 << 19)
+#define   SDVO_CLOCK_OUTPUT_INVERT	(1 << 18)
+#define   SDVOC_GANG_MODE		(1 << 16)
+#define   SDVO_BORDER_ENABLE		(1 << 7)
+#define   SDVOB_PCIE_CONCURRENCY	(1 << 3)
+#define   SDVO_DETECTED			(1 << 2)
+/* Bits to be preserved when writing */
+#define   SDVOB_PRESERVE_MASK ((1 << 17) | (1 << 16) | (1 << 14) | (1 << 26))
+#define   SDVOC_PRESERVE_MASK ((1 << 17) | (1 << 26))
+
+/* DVO port control */
+#define DVOA			0x61120
+#define DVOB			0x61140
+#define DVOC			0x61160
+#define   DVO_ENABLE			(1 << 31)
+#define   DVO_PIPE_B_SELECT		(1 << 30)
+#define   DVO_PIPE_STALL_UNUSED		(0 << 28)
+#define   DVO_PIPE_STALL		(1 << 28)
+#define   DVO_PIPE_STALL_TV		(2 << 28)
+#define   DVO_PIPE_STALL_MASK		(3 << 28)
+#define   DVO_USE_VGA_SYNC		(1 << 15)
+#define   DVO_DATA_ORDER_I740		(0 << 14)
+#define   DVO_DATA_ORDER_FP		(1 << 14)
+#define   DVO_VSYNC_DISABLE		(1 << 11)
+#define   DVO_HSYNC_DISABLE		(1 << 10)
+#define   DVO_VSYNC_TRISTATE		(1 << 9)
+#define   DVO_HSYNC_TRISTATE		(1 << 8)
+#define   DVO_BORDER_ENABLE		(1 << 7)
+#define   DVO_DATA_ORDER_GBRG		(1 << 6)
+#define   DVO_DATA_ORDER_RGGB		(0 << 6)
+#define   DVO_DATA_ORDER_GBRG_ERRATA	(0 << 6)
+#define   DVO_DATA_ORDER_RGGB_ERRATA	(1 << 6)
+#define   DVO_VSYNC_ACTIVE_HIGH		(1 << 4)
+#define   DVO_HSYNC_ACTIVE_HIGH		(1 << 3)
+#define   DVO_BLANK_ACTIVE_HIGH		(1 << 2)
+#define   DVO_OUTPUT_CSTATE_PIXELS	(1 << 1)	/* SDG only */
+#define   DVO_OUTPUT_SOURCE_SIZE_PIXELS	(1 << 0)	/* SDG only */
+#define   DVO_PRESERVE_MASK		(0x7<<24)
+#define DVOA_SRCDIM		0x61124
+#define DVOB_SRCDIM		0x61144
+#define DVOC_SRCDIM		0x61164
+#define   DVO_SRCDIM_HORIZONTAL_SHIFT	12
+#define   DVO_SRCDIM_VERTICAL_SHIFT	0
+
+/* LVDS port control */
+#define LVDS			0x61180
+/*
+ * Enables the LVDS port.  This bit must be set before DPLLs are enabled, as
+ * the DPLL semantics change when the LVDS is assigned to that pipe.
+ */
+#define   LVDS_PORT_EN			(1 << 31)
+/* Selects pipe B for LVDS data.  Must be set on pre-965. */
+#define   LVDS_PIPEB_SELECT		(1 << 30)
+/*
+ * Enables the A0-A2 data pairs and CLKA, containing 18 bits of color data per
+ * pixel.
+ */
+#define   LVDS_A0A2_CLKA_POWER_MASK	(3 << 8)
+#define   LVDS_A0A2_CLKA_POWER_DOWN	(0 << 8)
+#define   LVDS_A0A2_CLKA_POWER_UP	(3 << 8)
+/*
+ * Controls the A3 data pair, which contains the additional LSBs for 24 bit
+ * mode.  Only enabled if LVDS_A0A2_CLKA_POWER_UP also indicates it should be
+ * on.
+ */
+#define   LVDS_A3_POWER_MASK		(3 << 6)
+#define   LVDS_A3_POWER_DOWN		(0 << 6)
+#define   LVDS_A3_POWER_UP		(3 << 6)
+/*
+ * Controls the CLKB pair.  This should only be set when LVDS_B0B3_POWER_UP
+ * is set.
+ */
+#define   LVDS_CLKB_POWER_MASK		(3 << 4)
+#define   LVDS_CLKB_POWER_DOWN		(0 << 4)
+#define   LVDS_CLKB_POWER_UP		(3 << 4)
+/*
+ * Controls the B0-B3 data pairs.  This must be set to match the DPLL p2
+ * setting for whether we are in dual-channel mode.  The B3 pair will
+ * additionally only be powered up when LVDS_A3_POWER_UP is set.
+ */
+#define   LVDS_B0B3_POWER_MASK		(3 << 2)
+#define   LVDS_B0B3_POWER_DOWN		(0 << 2)
+#define   LVDS_B0B3_POWER_UP		(3 << 2)
+
+/* Panel power sequencing */
+#define PP_STATUS	0x61200
+#define   PP_ON		(1 << 31)
+/*
+ * Indicates that all dependencies of the panel are on:
+ *
+ * - PLL enabled
+ * - pipe enabled
+ * - LVDS/DVOB/DVOC on
+ */
+#define   PP_READY		(1 << 30)
+#define   PP_SEQUENCE_NONE	(0 << 28)
+#define   PP_SEQUENCE_ON	(1 << 28)
+#define   PP_SEQUENCE_OFF	(2 << 28)
+#define   PP_SEQUENCE_MASK	0x30000000
+#define PP_CONTROL	0x61204
+#define   POWER_TARGET_ON	(1 << 0)
+#define PP_ON_DELAYS	0x61208
+#define PP_OFF_DELAYS	0x6120c
+#define PP_DIVISOR	0x61210
+
+/* Panel fitting */
+#define PFIT_CONTROL	0x61230
+#define   PFIT_ENABLE		(1 << 31)
+#define   PFIT_PIPE_MASK	(3 << 29)
+#define   PFIT_PIPE_SHIFT	29
+#define   VERT_INTERP_DISABLE	(0 << 10)
+#define   VERT_INTERP_BILINEAR	(1 << 10)
+#define   VERT_INTERP_MASK	(3 << 10)
+#define   VERT_AUTO_SCALE	(1 << 9)
+#define   HORIZ_INTERP_DISABLE	(0 << 6)
+#define   HORIZ_INTERP_BILINEAR	(1 << 6)
+#define   HORIZ_INTERP_MASK	(3 << 6)
+#define   HORIZ_AUTO_SCALE	(1 << 5)
+#define   PANEL_8TO6_DITHER_ENABLE (1 << 3)
+#define PFIT_PGM_RATIOS	0x61234
+#define   PFIT_VERT_SCALE_MASK			0xfff00000
+#define   PFIT_HORIZ_SCALE_MASK			0x0000fff0
+#define PFIT_AUTO_RATIOS 0x61238
+
+/* Backlight control */
+#define BLC_PWM_CTL		0x61254
+#define   BACKLIGHT_MODULATION_FREQ_SHIFT		(17)
+#define BLC_PWM_CTL2		0x61250 /* 965+ only */
+/*
+ * This is the most significant 15 bits of the number of backlight cycles in a
+ * complete cycle of the modulated backlight control.
+ *
+ * The actual value is this field multiplied by two.
+ */
+#define   BACKLIGHT_MODULATION_FREQ_MASK		(0x7fff << 17)
+#define   BLM_LEGACY_MODE				(1 << 16)
+/*
+ * This is the number of cycles out of the backlight modulation cycle for which
+ * the backlight is on.
+ *
+ * This field must be no greater than the number of cycles in the complete
+ * backlight modulation cycle.
+ */
+#define   BACKLIGHT_DUTY_CYCLE_SHIFT		(0)
+#define   BACKLIGHT_DUTY_CYCLE_MASK		(0xffff)
+
+/* TV port control */
+#define TV_CTL			0x68000
+/** Enables the TV encoder */
+# define TV_ENC_ENABLE			(1 << 31)
+/** Sources the TV encoder input from pipe B instead of A. */
+# define TV_ENC_PIPEB_SELECT		(1 << 30)
+/** Outputs composite video (DAC A only) */
+# define TV_ENC_OUTPUT_COMPOSITE	(0 << 28)
+/** Outputs SVideo video (DAC B/C) */
+# define TV_ENC_OUTPUT_SVIDEO		(1 << 28)
+/** Outputs Component video (DAC A/B/C) */
+# define TV_ENC_OUTPUT_COMPONENT	(2 << 28)
+/** Outputs Composite and SVideo (DAC A/B/C) */
+# define TV_ENC_OUTPUT_SVIDEO_COMPOSITE	(3 << 28)
+# define TV_TRILEVEL_SYNC		(1 << 21)
+/** Enables slow sync generation (945GM only) */
+# define TV_SLOW_SYNC			(1 << 20)
+/** Selects 4x oversampling for 480i and 576p */
+# define TV_OVERSAMPLE_4X		(0 << 18)
+/** Selects 2x oversampling for 720p and 1080i */
+# define TV_OVERSAMPLE_2X		(1 << 18)
+/** Selects no oversampling for 1080p */
+# define TV_OVERSAMPLE_NONE		(2 << 18)
+/** Selects 8x oversampling */
+# define TV_OVERSAMPLE_8X		(3 << 18)
+/** Selects progressive mode rather than interlaced */
+# define TV_PROGRESSIVE			(1 << 17)
+/** Sets the colorburst to PAL mode.  Required for non-M PAL modes. */
+# define TV_PAL_BURST			(1 << 16)
+/** Field for setting delay of Y compared to C */
+# define TV_YC_SKEW_MASK		(7 << 12)
+/** Enables a fix for 480p/576p standard definition modes on the 915GM only */
+# define TV_ENC_SDP_FIX			(1 << 11)
+/**
+ * Enables a fix for the 915GM only.
+ *
+ * Not sure what it does.
+ */
+# define TV_ENC_C0_FIX			(1 << 10)
+/** Bits that must be preserved by software */
+# define TV_CTL_SAVE			((3 << 8) | (3 << 6))
+# define TV_FUSE_STATE_MASK		(3 << 4)
+/** Read-only state that reports all features enabled */
+# define TV_FUSE_STATE_ENABLED		(0 << 4)
+/** Read-only state that reports that Macrovision is disabled in hardware*/
+# define TV_FUSE_STATE_NO_MACROVISION	(1 << 4)
+/** Read-only state that reports that TV-out is disabled in hardware. */
+# define TV_FUSE_STATE_DISABLED		(2 << 4)
+/** Normal operation */
+# define TV_TEST_MODE_NORMAL		(0 << 0)
+/** Encoder test pattern 1 - combo pattern */
+# define TV_TEST_MODE_PATTERN_1		(1 << 0)
+/** Encoder test pattern 2 - full screen vertical 75% color bars */
+# define TV_TEST_MODE_PATTERN_2		(2 << 0)
+/** Encoder test pattern 3 - full screen horizontal 75% color bars */
+# define TV_TEST_MODE_PATTERN_3		(3 << 0)
+/** Encoder test pattern 4 - random noise */
+# define TV_TEST_MODE_PATTERN_4		(4 << 0)
+/** Encoder test pattern 5 - linear color ramps */
+# define TV_TEST_MODE_PATTERN_5		(5 << 0)
+/**
+ * This test mode forces the DACs to 50% of full output.
+ *
+ * This is used for load detection in combination with TVDAC_SENSE_MASK
+ */
+# define TV_TEST_MODE_MONITOR_DETECT	(7 << 0)
+# define TV_TEST_MODE_MASK		(7 << 0)
+
+#define TV_DAC			0x68004
+/**
+ * Reports that DAC state change logic has reported change (RO).
+ *
+ * This gets cleared when TV_DAC_STATE_EN is cleared
+*/
+# define TVDAC_STATE_CHG		(1 << 31)
+# define TVDAC_SENSE_MASK		(7 << 28)
+/** Reports that DAC A voltage is above the detect threshold */
+# define TVDAC_A_SENSE			(1 << 30)
+/** Reports that DAC B voltage is above the detect threshold */
+# define TVDAC_B_SENSE			(1 << 29)
+/** Reports that DAC C voltage is above the detect threshold */
+# define TVDAC_C_SENSE			(1 << 28)
+/**
+ * Enables DAC state detection logic, for load-based TV detection.
+ *
+ * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set
+ * to off, for load detection to work.
+ */
+# define TVDAC_STATE_CHG_EN		(1 << 27)
+/** Sets the DAC A sense value to high */
+# define TVDAC_A_SENSE_CTL		(1 << 26)
+/** Sets the DAC B sense value to high */
+# define TVDAC_B_SENSE_CTL		(1 << 25)
+/** Sets the DAC C sense value to high */
+# define TVDAC_C_SENSE_CTL		(1 << 24)
+/** Overrides the ENC_ENABLE and DAC voltage levels */
+# define DAC_CTL_OVERRIDE		(1 << 7)
+/** Sets the slew rate.  Must be preserved in software */
+# define ENC_TVDAC_SLEW_FAST		(1 << 6)
+# define DAC_A_1_3_V			(0 << 4)
+# define DAC_A_1_1_V			(1 << 4)
+# define DAC_A_0_7_V			(2 << 4)
+# define DAC_A_OFF			(3 << 4)
+# define DAC_B_1_3_V			(0 << 2)
+# define DAC_B_1_1_V			(1 << 2)
+# define DAC_B_0_7_V			(2 << 2)
+# define DAC_B_OFF			(3 << 2)
+# define DAC_C_1_3_V			(0 << 0)
+# define DAC_C_1_1_V			(1 << 0)
+# define DAC_C_0_7_V			(2 << 0)
+# define DAC_C_OFF			(3 << 0)
+
+/**
+ * CSC coefficients are stored in a floating point format with 9 bits of
+ * mantissa and 2 or 3 bits of exponent.  The exponent is represented as 2**-n,
+ * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with
+ * -1 (0x3) being the only legal negative value.
+ */
+#define TV_CSC_Y		0x68010
+# define TV_RY_MASK			0x07ff0000
+# define TV_RY_SHIFT			16
+# define TV_GY_MASK			0x00000fff
+# define TV_GY_SHIFT			0
+
+#define TV_CSC_Y2		0x68014
+# define TV_BY_MASK			0x07ff0000
+# define TV_BY_SHIFT			16
+/**
+ * Y attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AY_MASK			0x000003ff
+# define TV_AY_SHIFT			0
+
+#define TV_CSC_U		0x68018
+# define TV_RU_MASK			0x07ff0000
+# define TV_RU_SHIFT			16
+# define TV_GU_MASK			0x000007ff
+# define TV_GU_SHIFT			0
+
+#define TV_CSC_U2		0x6801c
+# define TV_BU_MASK			0x07ff0000
+# define TV_BU_SHIFT			16
+/**
+ * U attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AU_MASK			0x000003ff
+# define TV_AU_SHIFT			0
+
+#define TV_CSC_V		0x68020
+# define TV_RV_MASK			0x0fff0000
+# define TV_RV_SHIFT			16
+# define TV_GV_MASK			0x000007ff
+# define TV_GV_SHIFT			0
+
+#define TV_CSC_V2		0x68024
+# define TV_BV_MASK			0x07ff0000
+# define TV_BV_SHIFT			16
+/**
+ * V attenuation for component video.
+ *
+ * Stored in 1.9 fixed point.
+ */
+# define TV_AV_MASK			0x000007ff
+# define TV_AV_SHIFT			0
+
+#define TV_CLR_KNOBS		0x68028
+/** 2s-complement brightness adjustment */
+# define TV_BRIGHTNESS_MASK		0xff000000
+# define TV_BRIGHTNESS_SHIFT		24
+/** Contrast adjustment, as a 2.6 unsigned floating point number */
+# define TV_CONTRAST_MASK		0x00ff0000
+# define TV_CONTRAST_SHIFT		16
+/** Saturation adjustment, as a 2.6 unsigned floating point number */
+# define TV_SATURATION_MASK		0x0000ff00
+# define TV_SATURATION_SHIFT		8
+/** Hue adjustment, as an integer phase angle in degrees */
+# define TV_HUE_MASK			0x000000ff
+# define TV_HUE_SHIFT			0
+
+#define TV_CLR_LEVEL		0x6802c
+/** Controls the DAC level for black */
+# define TV_BLACK_LEVEL_MASK		0x01ff0000
+# define TV_BLACK_LEVEL_SHIFT		16
+/** Controls the DAC level for blanking */
+# define TV_BLANK_LEVEL_MASK		0x000001ff
+# define TV_BLANK_LEVEL_SHIFT		0
+
+#define TV_H_CTL_1		0x68030
+/** Number of pixels in the hsync. */
+# define TV_HSYNC_END_MASK		0x1fff0000
+# define TV_HSYNC_END_SHIFT		16
+/** Total number of pixels minus one in the line (display and blanking). */
+# define TV_HTOTAL_MASK			0x00001fff
+# define TV_HTOTAL_SHIFT		0
+
+#define TV_H_CTL_2		0x68034
+/** Enables the colorburst (needed for non-component color) */
+# define TV_BURST_ENA			(1 << 31)
+/** Offset of the colorburst from the start of hsync, in pixels minus one. */
+# define TV_HBURST_START_SHIFT		16
+# define TV_HBURST_START_MASK		0x1fff0000
+/** Length of the colorburst */
+# define TV_HBURST_LEN_SHIFT		0
+# define TV_HBURST_LEN_MASK		0x0001fff
+
+#define TV_H_CTL_3		0x68038
+/** End of hblank, measured in pixels minus one from start of hsync */
+# define TV_HBLANK_END_SHIFT		16
+# define TV_HBLANK_END_MASK		0x1fff0000
+/** Start of hblank, measured in pixels minus one from start of hsync */
+# define TV_HBLANK_START_SHIFT		0
+# define TV_HBLANK_START_MASK		0x0001fff
+
+#define TV_V_CTL_1		0x6803c
+/** XXX */
+# define TV_NBR_END_SHIFT		16
+# define TV_NBR_END_MASK		0x07ff0000
+/** XXX */
+# define TV_VI_END_F1_SHIFT		8
+# define TV_VI_END_F1_MASK		0x00003f00
+/** XXX */
+# define TV_VI_END_F2_SHIFT		0
+# define TV_VI_END_F2_MASK		0x0000003f
+
+#define TV_V_CTL_2		0x68040
+/** Length of vsync, in half lines */
+# define TV_VSYNC_LEN_MASK		0x07ff0000
+# define TV_VSYNC_LEN_SHIFT		16
+/** Offset of the start of vsync in field 1, measured in one less than the
+ * number of half lines.
+ */
+# define TV_VSYNC_START_F1_MASK		0x00007f00
+# define TV_VSYNC_START_F1_SHIFT	8
+/**
+ * Offset of the start of vsync in field 2, measured in one less than the
+ * number of half lines.
+ */
+# define TV_VSYNC_START_F2_MASK		0x0000007f
+# define TV_VSYNC_START_F2_SHIFT	0
+
+#define TV_V_CTL_3		0x68044
+/** Enables generation of the equalization signal */
+# define TV_EQUAL_ENA			(1 << 31)
+/** Length of vsync, in half lines */
+# define TV_VEQ_LEN_MASK		0x007f0000
+# define TV_VEQ_LEN_SHIFT		16
+/** Offset of the start of equalization in field 1, measured in one less than
+ * the number of half lines.
+ */
+# define TV_VEQ_START_F1_MASK		0x0007f00
+# define TV_VEQ_START_F1_SHIFT		8
+/**
+ * Offset of the start of equalization in field 2, measured in one less than
+ * the number of half lines.
+ */
+# define TV_VEQ_START_F2_MASK		0x000007f
+# define TV_VEQ_START_F2_SHIFT		0
+
+#define TV_V_CTL_4		0x68048
+/**
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F1_MASK	0x003f0000
+# define TV_VBURST_START_F1_SHIFT	16
+/**
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F1_MASK		0x000000ff
+# define TV_VBURST_END_F1_SHIFT		0
+
+#define TV_V_CTL_5		0x6804c
+/**
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F2_MASK	0x003f0000
+# define TV_VBURST_START_F2_SHIFT	16
+/**
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F2_MASK		0x000000ff
+# define TV_VBURST_END_F2_SHIFT		0
+
+#define TV_V_CTL_6		0x68050
+/**
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F3_MASK	0x003f0000
+# define TV_VBURST_START_F3_SHIFT	16
+/**
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F3_MASK		0x000000ff
+# define TV_VBURST_END_F3_SHIFT		0
+
+#define TV_V_CTL_7		0x68054
+/**
+ * Offset to start of vertical colorburst, measured in one less than the
+ * number of lines from vertical start.
+ */
+# define TV_VBURST_START_F4_MASK	0x003f0000
+# define TV_VBURST_START_F4_SHIFT	16
+/**
+ * Offset to the end of vertical colorburst, measured in one less than the
+ * number of lines from the start of NBR.
+ */
+# define TV_VBURST_END_F4_MASK		0x000000ff
+# define TV_VBURST_END_F4_SHIFT		0
+
+#define TV_SC_CTL_1		0x68060
+/** Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA1_EN			(1 << 31)
+/** Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA2_EN			(1 << 30)
+/** Turns on the first subcarrier phase generation DDA */
+# define TV_SC_DDA3_EN			(1 << 29)
+/** Sets the subcarrier DDA to reset frequency every other field */
+# define TV_SC_RESET_EVERY_2		(0 << 24)
+/** Sets the subcarrier DDA to reset frequency every fourth field */
+# define TV_SC_RESET_EVERY_4		(1 << 24)
+/** Sets the subcarrier DDA to reset frequency every eighth field */
+# define TV_SC_RESET_EVERY_8		(2 << 24)
+/** Sets the subcarrier DDA to never reset the frequency */
+# define TV_SC_RESET_NEVER		(3 << 24)
+/** Sets the peak amplitude of the colorburst.*/
+# define TV_BURST_LEVEL_MASK		0x00ff0000
+# define TV_BURST_LEVEL_SHIFT		16
+/** Sets the increment of the first subcarrier phase generation DDA */
+# define TV_SCDDA1_INC_MASK		0x00000fff
+# define TV_SCDDA1_INC_SHIFT		0
+
+#define TV_SC_CTL_2		0x68064
+/** Sets the rollover for the second subcarrier phase generation DDA */
+# define TV_SCDDA2_SIZE_MASK		0x7fff0000
+# define TV_SCDDA2_SIZE_SHIFT		16
+/** Sets the increent of the second subcarrier phase generation DDA */
+# define TV_SCDDA2_INC_MASK		0x00007fff
+# define TV_SCDDA2_INC_SHIFT		0
+
+#define TV_SC_CTL_3		0x68068
+/** Sets the rollover for the third subcarrier phase generation DDA */
+# define TV_SCDDA3_SIZE_MASK		0x7fff0000
+# define TV_SCDDA3_SIZE_SHIFT		16
+/** Sets the increent of the third subcarrier phase generation DDA */
+# define TV_SCDDA3_INC_MASK		0x00007fff
+# define TV_SCDDA3_INC_SHIFT		0
+
+#define TV_WIN_POS		0x68070
+/** X coordinate of the display from the start of horizontal active */
+# define TV_XPOS_MASK			0x1fff0000
+# define TV_XPOS_SHIFT			16
+/** Y coordinate of the display from the start of vertical active (NBR) */
+# define TV_YPOS_MASK			0x00000fff
+# define TV_YPOS_SHIFT			0
+
+#define TV_WIN_SIZE		0x68074
+/** Horizontal size of the display window, measured in pixels*/
+# define TV_XSIZE_MASK			0x1fff0000
+# define TV_XSIZE_SHIFT			16
+/**
+ * Vertical size of the display window, measured in pixels.
+ *
+ * Must be even for interlaced modes.
+ */
+# define TV_YSIZE_MASK			0x00000fff
+# define TV_YSIZE_SHIFT			0
+
+#define TV_FILTER_CTL_1		0x68080
+/**
+ * Enables automatic scaling calculation.
+ *
+ * If set, the rest of the registers are ignored, and the calculated values can
+ * be read back from the register.
+ */
+# define TV_AUTO_SCALE			(1 << 31)
+/**
+ * Disables the vertical filter.
+ *
+ * This is required on modes more than 1024 pixels wide */
+# define TV_V_FILTER_BYPASS		(1 << 29)
+/** Enables adaptive vertical filtering */
+# define TV_VADAPT			(1 << 28)
+# define TV_VADAPT_MODE_MASK		(3 << 26)
+/** Selects the least adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_LEAST		(0 << 26)
+/** Selects the moderately adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_MODERATE	(1 << 26)
+/** Selects the most adaptive vertical filtering mode */
+# define TV_VADAPT_MODE_MOST		(3 << 26)
+/**
+ * Sets the horizontal scaling factor.
+ *
+ * This should be the fractional part of the horizontal scaling factor divided
+ * by the oversampling rate.  TV_HSCALE should be less than 1, and set to:
+ *
+ * (src width - 1) / ((oversample * dest width) - 1)
+ */
+# define TV_HSCALE_FRAC_MASK		0x00003fff
+# define TV_HSCALE_FRAC_SHIFT		0
+
+#define TV_FILTER_CTL_2		0x68084
+/**
+ * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1)
+ */
+# define TV_VSCALE_INT_MASK		0x00038000
+# define TV_VSCALE_INT_SHIFT		15
+/**
+ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * \sa TV_VSCALE_INT_MASK
+ */
+# define TV_VSCALE_FRAC_MASK		0x00007fff
+# define TV_VSCALE_FRAC_SHIFT		0
+
+#define TV_FILTER_CTL_3		0x68088
+/**
+ * Sets the integer part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1))
+ *
+ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
+ */
+# define TV_VSCALE_IP_INT_MASK		0x00038000
+# define TV_VSCALE_IP_INT_SHIFT		15
+/**
+ * Sets the fractional part of the 3.15 fixed-point vertical scaling factor.
+ *
+ * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes.
+ *
+ * \sa TV_VSCALE_IP_INT_MASK
+ */
+# define TV_VSCALE_IP_FRAC_MASK		0x00007fff
+# define TV_VSCALE_IP_FRAC_SHIFT		0
+
+#define TV_CC_CONTROL		0x68090
+# define TV_CC_ENABLE			(1 << 31)
+/**
+ * Specifies which field to send the CC data in.
+ *
+ * CC data is usually sent in field 0.
+ */
+# define TV_CC_FID_MASK			(1 << 27)
+# define TV_CC_FID_SHIFT		27
+/** Sets the horizontal position of the CC data.  Usually 135. */
+# define TV_CC_HOFF_MASK		0x03ff0000
+# define TV_CC_HOFF_SHIFT		16
+/** Sets the vertical position of the CC data.  Usually 21 */
+# define TV_CC_LINE_MASK		0x0000003f
+# define TV_CC_LINE_SHIFT		0
+
+#define TV_CC_DATA		0x68094
+# define TV_CC_RDY			(1 << 31)
+/** Second word of CC data to be transmitted. */
+# define TV_CC_DATA_2_MASK		0x007f0000
+# define TV_CC_DATA_2_SHIFT		16
+/** First word of CC data to be transmitted. */
+# define TV_CC_DATA_1_MASK		0x0000007f
+# define TV_CC_DATA_1_SHIFT		0
+
+#define TV_H_LUMA_0		0x68100
+#define TV_H_LUMA_59		0x681ec
+#define TV_H_CHROMA_0		0x68200
+#define TV_H_CHROMA_59		0x682ec
+#define TV_V_LUMA_0		0x68300
+#define TV_V_LUMA_42		0x683a8
+#define TV_V_CHROMA_0		0x68400
+#define TV_V_CHROMA_42		0x684a8
+
+/* Display & cursor control */
+
+/* Pipe A */
+#define PIPEADSL		0x70000
+#define PIPEACONF		 0x70008
+#define   PIPEACONF_ENABLE	(1<<31)
+#define   PIPEACONF_DISABLE	0
+#define   PIPEACONF_DOUBLE_WIDE	(1<<30)
+#define   I965_PIPECONF_ACTIVE	(1<<30)
+#define   PIPEACONF_SINGLE_WIDE	0
+#define   PIPEACONF_PIPE_UNLOCKED 0
+#define   PIPEACONF_PIPE_LOCKED	(1<<25)
+#define   PIPEACONF_PALETTE	0
+#define   PIPEACONF_GAMMA		(1<<24)
+#define   PIPECONF_FORCE_BORDER	(1<<25)
+#define   PIPECONF_PROGRESSIVE	(0 << 21)
+#define   PIPECONF_INTERLACE_W_FIELD_INDICATION	(6 << 21)
+#define   PIPECONF_INTERLACE_FIELD_0_ONLY		(7 << 21)
+#define PIPEASTAT		0x70024
+#define   PIPE_FIFO_UNDERRUN_STATUS		(1UL<<31)
+#define   PIPE_CRC_ERROR_ENABLE			(1UL<<29)
+#define   PIPE_CRC_DONE_ENABLE			(1UL<<28)
+#define   PIPE_GMBUS_EVENT_ENABLE		(1UL<<27)
+#define   PIPE_HOTPLUG_INTERRUPT_ENABLE		(1UL<<26)
+#define   PIPE_VSYNC_INTERRUPT_ENABLE		(1UL<<25)
+#define   PIPE_DISPLAY_LINE_COMPARE_ENABLE	(1UL<<24)
+#define   PIPE_DPST_EVENT_ENABLE		(1UL<<23)
+#define   PIPE_LEGACY_BLC_EVENT_ENABLE		(1UL<<22)
+#define   PIPE_ODD_FIELD_INTERRUPT_ENABLE	(1UL<<21)
+#define   PIPE_EVEN_FIELD_INTERRUPT_ENABLE	(1UL<<20)
+#define   PIPE_HOTPLUG_TV_INTERRUPT_ENABLE	(1UL<<18) /* pre-965 */
+#define   PIPE_START_VBLANK_INTERRUPT_ENABLE	(1UL<<18) /* 965 or later */
+#define   PIPE_VBLANK_INTERRUPT_ENABLE		(1UL<<17)
+#define   PIPE_OVERLAY_UPDATED_ENABLE		(1UL<<16)
+#define   PIPE_CRC_ERROR_INTERRUPT_STATUS	(1UL<<13)
+#define   PIPE_CRC_DONE_INTERRUPT_STATUS	(1UL<<12)
+#define   PIPE_GMBUS_INTERRUPT_STATUS		(1UL<<11)
+#define   PIPE_HOTPLUG_INTERRUPT_STATUS		(1UL<<10)
+#define   PIPE_VSYNC_INTERRUPT_STATUS		(1UL<<9)
+#define   PIPE_DISPLAY_LINE_COMPARE_STATUS	(1UL<<8)
+#define   PIPE_DPST_EVENT_STATUS		(1UL<<7)
+#define   PIPE_LEGACY_BLC_EVENT_STATUS		(1UL<<6)
+#define   PIPE_ODD_FIELD_INTERRUPT_STATUS	(1UL<<5)
+#define   PIPE_EVEN_FIELD_INTERRUPT_STATUS	(1UL<<4)
+#define   PIPE_HOTPLUG_TV_INTERRUPT_STATUS	(1UL<<2) /* pre-965 */
+#define   PIPE_START_VBLANK_INTERRUPT_STATUS	(1UL<<2) /* 965 or later */
+#define   PIPE_VBLANK_INTERRUPT_STATUS		(1UL<<1)
+#define   PIPE_OVERLAY_UPDATED_STATUS		(1UL<<0)
+
+#define DSPARB			0x70030
+#define   DSPARB_CSTART_MASK	(0x7f << 7)
+#define   DSPARB_CSTART_SHIFT	7
+#define   DSPARB_BSTART_MASK	(0x7f)		 
+#define   DSPARB_BSTART_SHIFT	0
+/*
+ * The two pipe frame counter registers are not synchronized, so
+ * reading a stable value is somewhat tricky. The following code 
+ * should work:
+ *
+ *  do {
+ *    high1 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >>
+ *             PIPE_FRAME_HIGH_SHIFT;
+ *    low1 =  ((INREG(PIPEAFRAMEPIXEL) & PIPE_FRAME_LOW_MASK) >>
+ *             PIPE_FRAME_LOW_SHIFT);
+ *    high2 = ((INREG(PIPEAFRAMEHIGH) & PIPE_FRAME_HIGH_MASK) >>
+ *             PIPE_FRAME_HIGH_SHIFT);
+ *  } while (high1 != high2);
+ *  frame = (high1 << 8) | low1;
+ */
+#define PIPEAFRAMEHIGH          0x70040
+#define   PIPE_FRAME_HIGH_MASK    0x0000ffff
+#define   PIPE_FRAME_HIGH_SHIFT   0
+#define PIPEAFRAMEPIXEL         0x70044
+#define   PIPE_FRAME_LOW_MASK     0xff000000
+#define   PIPE_FRAME_LOW_SHIFT    24
+#define   PIPE_PIXEL_MASK         0x00ffffff
+#define   PIPE_PIXEL_SHIFT        0
+
+/* Cursor A & B regs */
+#define CURACNTR		0x70080
+#define   CURSOR_MODE_DISABLE   0x00
+#define   CURSOR_MODE_64_32B_AX 0x07
+#define   CURSOR_MODE_64_ARGB_AX ((1 << 5) | CURSOR_MODE_64_32B_AX)
+#define   MCURSOR_GAMMA_ENABLE  (1 << 26)
+#define CURABASE		0x70084
+#define CURAPOS			0x70088
+#define   CURSOR_POS_MASK       0x007FF
+#define   CURSOR_POS_SIGN       0x8000
+#define   CURSOR_X_SHIFT        0
+#define   CURSOR_Y_SHIFT        16
+#define CURBCNTR		0x700c0
+#define CURBBASE		0x700c4
+#define CURBPOS			0x700c8
+
+/* Display A control */
+#define DSPACNTR                0x70180
+#define   DISPLAY_PLANE_ENABLE			(1<<31)
+#define   DISPLAY_PLANE_DISABLE			0
+#define   DISPPLANE_GAMMA_ENABLE		(1<<30)
+#define   DISPPLANE_GAMMA_DISABLE		0
+#define   DISPPLANE_PIXFORMAT_MASK		(0xf<<26)
+#define   DISPPLANE_8BPP			(0x2<<26)
+#define   DISPPLANE_15_16BPP			(0x4<<26)
+#define   DISPPLANE_16BPP			(0x5<<26)
+#define   DISPPLANE_32BPP_NO_ALPHA		(0x6<<26)
+#define   DISPPLANE_32BPP			(0x7<<26)
+#define   DISPPLANE_STEREO_ENABLE		(1<<25)
+#define   DISPPLANE_STEREO_DISABLE		0
+#define   DISPPLANE_SEL_PIPE_MASK		(1<<24)
+#define   DISPPLANE_SEL_PIPE_A			0
+#define   DISPPLANE_SEL_PIPE_B			(1<<24)
+#define   DISPPLANE_SRC_KEY_ENABLE		(1<<22)
+#define   DISPPLANE_SRC_KEY_DISABLE		0
+#define   DISPPLANE_LINE_DOUBLE			(1<<20)
+#define   DISPPLANE_NO_LINE_DOUBLE		0
+#define   DISPPLANE_STEREO_POLARITY_FIRST	0
+#define   DISPPLANE_STEREO_POLARITY_SECOND	(1<<18)
+#define DSPAADDR		0x70184
+#define DSPASTRIDE		0x70188
+#define DSPAPOS			0x7018C /* reserved */
+#define DSPASIZE		0x70190
+#define DSPASURF		0x7019C /* 965+ only */
+#define DSPATILEOFF		0x701A4 /* 965+ only */
+
+/* VBIOS flags */
+#define SWF00			0x71410
+#define SWF01			0x71414
+#define SWF02			0x71418
+#define SWF03			0x7141c
+#define SWF04			0x71420
+#define SWF05			0x71424
+#define SWF06			0x71428
+#define SWF10			0x70410
+#define SWF11			0x70414
+#define SWF14			0x71420
+#define SWF30			0x72414
+#define SWF31			0x72418
+#define SWF32			0x7241c
+
+/* Pipe B */
+#define PIPEBDSL		0x71000
+#define PIPEBCONF		0x71008
+#define PIPEBSTAT		0x71024
+#define PIPEBFRAMEHIGH		0x71040
+#define PIPEBFRAMEPIXEL		0x71044
+
+/* Display B control */
+#define DSPBCNTR		0x71180
+#define   DISPPLANE_ALPHA_TRANS_ENABLE		(1<<15)
+#define   DISPPLANE_ALPHA_TRANS_DISABLE		0
+#define   DISPPLANE_SPRITE_ABOVE_DISPLAY	0
+#define   DISPPLANE_SPRITE_ABOVE_OVERLAY	(1)
+#define DSPBADDR		0x71184
+#define DSPBSTRIDE		0x71188
+#define DSPBPOS			0x7118C
+#define DSPBSIZE		0x71190
+#define DSPBSURF		0x7119C
+#define DSPBTILEOFF		0x711A4
+
+/* VBIOS regs */
+#define VGACNTRL		0x71400
+# define VGA_DISP_DISABLE			(1 << 31)
+# define VGA_2X_MODE				(1 << 30)
+# define VGA_PIPE_B_SELECT			(1 << 29)
+
+/* Chipset type macros */
+
+#define IS_I830(dev) ((dev)->pci_device == 0x3577)
+#define IS_845G(dev) ((dev)->pci_device == 0x2562)
+#define IS_I85X(dev) ((dev)->pci_device == 0x3582)
+#define IS_I855(dev) ((dev)->pci_device == 0x3582)
+#define IS_I865G(dev) ((dev)->pci_device == 0x2572)
+
+#define IS_I915G(dev) ((dev)->pci_device == 0x2582 || (dev)->pci_device == 0x258a)
+#define IS_I915GM(dev) ((dev)->pci_device == 0x2592)
+#define IS_I945G(dev) ((dev)->pci_device == 0x2772)
+#define IS_I945GM(dev) ((dev)->pci_device == 0x27A2 ||\
+		        (dev)->pci_device == 0x27AE)
+#define IS_I965G(dev) ((dev)->pci_device == 0x2972 || \
+		       (dev)->pci_device == 0x2982 || \
+		       (dev)->pci_device == 0x2992 || \
+		       (dev)->pci_device == 0x29A2 || \
+		       (dev)->pci_device == 0x2A02 || \
+		       (dev)->pci_device == 0x2A12 || \
+		       (dev)->pci_device == 0x2A42 || \
+		       (dev)->pci_device == 0x2E02 || \
+		       (dev)->pci_device == 0x2E12 || \
+		       (dev)->pci_device == 0x2E22)
+
+#define IS_I965GM(dev) ((dev)->pci_device == 0x2A02)
+
+#define IS_GM45(dev) ((dev)->pci_device == 0x2A42)
+
+#define IS_G4X(dev) ((dev)->pci_device == 0x2E02 || \
+		     (dev)->pci_device == 0x2E12 || \
+		     (dev)->pci_device == 0x2E22)
+
+#define IS_G33(dev)    ((dev)->pci_device == 0x29C2 ||	\
+			(dev)->pci_device == 0x29B2 ||	\
+			(dev)->pci_device == 0x29D2)
+
+#define IS_I9XX(dev) (IS_I915G(dev) || IS_I915GM(dev) || IS_I945G(dev) || \
+		      IS_I945GM(dev) || IS_I965G(dev) || IS_G33(dev))
 
-#define CMD_OP_DESTBUFFER_INFO	 ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
+#define IS_MOBILE(dev) (IS_I830(dev) || IS_I85X(dev) || IS_I915GM(dev) || \
+			IS_I945GM(dev) || IS_I965GM(dev) || IS_GM45(dev))
 
-#define READ_BREADCRUMB(dev_priv)  (((u32*)(dev_priv->hw_status_page))[5])
+#define I915_NEED_GFX_HWS(dev) (IS_G33(dev) || IS_GM45(dev) || IS_G4X(dev))
 
 #endif
diff --git a/sys/dev/drm/i915_irq.c b/sys/dev/drm/i915_irq.c
index b40669c..99e22c4 100644
--- a/sys/dev/drm/i915_irq.c
+++ b/sys/dev/drm/i915_irq.c
@@ -3,7 +3,7 @@
 /*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
@@ -11,11 +11,11 @@
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
@@ -23,7 +23,7 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- * 
+ *
  */
 
 #include <sys/cdefs.h>
@@ -34,143 +34,543 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/i915_drm.h"
 #include "dev/drm/i915_drv.h"
 
-#define USER_INT_FLAG (1<<1)
-#define VSYNC_PIPEB_FLAG (1<<5)
-#define VSYNC_PIPEA_FLAG (1<<7)
-
 #define MAX_NOPID ((u32)~0)
 
-irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
+/**
+ * i915_get_pipe - return the the pipe associated with a given plane
+ * @dev: DRM device
+ * @plane: plane to look for
+ *
+ * The Intel Mesa & 2D drivers call the vblank routines with a plane number
+ * rather than a pipe number, since they may not always be equal.  This routine
+ * maps the given @plane back to a pipe number.
+ */
+static int
+i915_get_pipe(struct drm_device *dev, int plane)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 dspcntr;
+
+	dspcntr = plane ? I915_READ(DSPBCNTR) : I915_READ(DSPACNTR);
+
+	return dspcntr & DISPPLANE_SEL_PIPE_MASK ? 1 : 0;
+}
+
+/**
+ * i915_get_plane - return the the plane associated with a given pipe
+ * @dev: DRM device
+ * @pipe: pipe to look for
+ *
+ * The Intel Mesa & 2D drivers call the vblank routines with a plane number
+ * rather than a plane number, since they may not always be equal.  This routine
+ * maps the given @pipe back to a plane number.
+ */
+static int
+i915_get_plane(struct drm_device *dev, int pipe)
+{
+	if (i915_get_pipe(dev, 0) == pipe)
+		return 0;
+	return 1;
+}
+
+/**
+ * i915_pipe_enabled - check if a pipe is enabled
+ * @dev: DRM device
+ * @pipe: pipe to check
+ *
+ * Reading certain registers when the pipe is disabled can hang the chip.
+ * Use this routine to make sure the PLL is running and the pipe is active
+ * before reading such registers if unsure.
+ */
+static int
+i915_pipe_enabled(struct drm_device *dev, int pipe)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
+
+	if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
+		return 1;
+
+	return 0;
+}
+
+/**
+ * Emit a synchronous flip.
+ *
+ * This function must be called with the drawable spinlock held.
+ */
+static void
+i915_dispatch_vsync_flip(struct drm_device *dev, struct drm_drawable_info *drw,
+			 int plane)
 {
-	drm_device_t *dev = (drm_device_t *) arg;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u16 temp;
+	drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
+	u16 x1, y1, x2, y2;
+	int pf_planes = 1 << plane;
+
+	DRM_SPINLOCK_ASSERT(&dev->drw_lock);
+
+	/* If the window is visible on the other plane, we have to flip on that
+	 * plane as well.
+	 */
+	if (plane == 1) {
+		x1 = sarea_priv->planeA_x;
+		y1 = sarea_priv->planeA_y;
+		x2 = x1 + sarea_priv->planeA_w;
+		y2 = y1 + sarea_priv->planeA_h;
+	} else {
+		x1 = sarea_priv->planeB_x;
+		y1 = sarea_priv->planeB_y;
+		x2 = x1 + sarea_priv->planeB_w;
+		y2 = y1 + sarea_priv->planeB_h;
+	}
+
+	if (x2 > 0 && y2 > 0) {
+		int i, num_rects = drw->num_rects;
+		struct drm_clip_rect *rect = drw->rects;
+
+		for (i = 0; i < num_rects; i++)
+			if (!(rect[i].x1 >= x2 || rect[i].y1 >= y2 ||
+			      rect[i].x2 <= x1 || rect[i].y2 <= y1)) {
+				pf_planes = 0x3;
+
+				break;
+			}
+	}
+
+	i915_dispatch_flip(dev, pf_planes, 1);
+}
+
+/**
+ * Emit blits for scheduled buffer swaps.
+ *
+ * This function will be called with the HW lock held.
+ */
+static void i915_vblank_tasklet(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	struct list_head *list, *tmp, hits, *hit;
+	int nhits, nrects, slice[2], upper[2], lower[2], i, num_pages;
+	unsigned counter[2];
+	struct drm_drawable_info *drw;
+	drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
+	u32 cpp = dev_priv->cpp,  offsets[3];
+	u32 cmd = (cpp == 4) ? (XY_SRC_COPY_BLT_CMD |
+				XY_SRC_COPY_BLT_WRITE_ALPHA |
+				XY_SRC_COPY_BLT_WRITE_RGB)
+			     : XY_SRC_COPY_BLT_CMD;
+	u32 src_pitch = sarea_priv->pitch * cpp;
+	u32 dst_pitch = sarea_priv->pitch * cpp;
+	/* COPY rop (0xcc), map cpp to magic color depth constants */
+	u32 ropcpp = (0xcc << 16) | ((cpp - 1) << 24);
+	RING_LOCALS;
+	
+	if (IS_I965G(dev) && sarea_priv->front_tiled) {
+		cmd |= XY_SRC_COPY_BLT_DST_TILED;
+		dst_pitch >>= 2;
+	}
+	if (IS_I965G(dev) && sarea_priv->back_tiled) {
+		cmd |= XY_SRC_COPY_BLT_SRC_TILED;
+		src_pitch >>= 2;
+	}
+	
+	counter[0] = drm_vblank_count(dev, 0);
+	counter[1] = drm_vblank_count(dev, 1);
+
+	DRM_DEBUG("\n");
+
+	INIT_LIST_HEAD(&hits);
+
+	nhits = nrects = 0;
+
+	/* No irqsave/restore necessary.  This tasklet may be run in an
+	 * interrupt context or normal context, but we don't have to worry
+	 * about getting interrupted by something acquiring the lock, because
+	 * we are the interrupt context thing that acquires the lock.
+	 */
+	DRM_SPINLOCK(&dev_priv->swaps_lock);
+
+	/* Find buffer swaps scheduled for this vertical blank */
+	list_for_each_safe(list, tmp, &dev_priv->vbl_swaps.head) {
+		drm_i915_vbl_swap_t *vbl_swap =
+			list_entry(list, drm_i915_vbl_swap_t, head);
+		int pipe = i915_get_pipe(dev, vbl_swap->plane);
+
+		if ((counter[pipe] - vbl_swap->sequence) > (1<<23))
+			continue;
+
+		list_del(list);
+		dev_priv->swaps_pending--;
+		drm_vblank_put(dev, pipe);
+
+		DRM_SPINUNLOCK(&dev_priv->swaps_lock);
+		DRM_SPINLOCK(&dev->drw_lock);
 
-	temp = I915_READ16(I915REG_INT_IDENTITY_R);
+		drw = drm_get_drawable_info(dev, vbl_swap->drw_id);
 
-	temp &= (USER_INT_FLAG | VSYNC_PIPEA_FLAG | VSYNC_PIPEB_FLAG);
+		if (!drw) {
+			DRM_SPINUNLOCK(&dev->drw_lock);
+			drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
+			DRM_SPINLOCK(&dev_priv->swaps_lock);
+			continue;
+		}
 
-	DRM_DEBUG("%s flag=%08x\n", __FUNCTION__, temp);
+		list_for_each(hit, &hits) {
+			drm_i915_vbl_swap_t *swap_cmp =
+				list_entry(hit, drm_i915_vbl_swap_t, head);
+			struct drm_drawable_info *drw_cmp =
+				drm_get_drawable_info(dev, swap_cmp->drw_id);
 
-	if (temp == 0)
+			if (drw_cmp &&
+			    drw_cmp->rects[0].y1 > drw->rects[0].y1) {
+				list_add_tail(list, hit);
+				break;
+			}
+		}
+
+		DRM_SPINUNLOCK(&dev->drw_lock);
+
+		/* List of hits was empty, or we reached the end of it */
+		if (hit == &hits)
+			list_add_tail(list, hits.prev);
+
+		nhits++;
+
+		DRM_SPINLOCK(&dev_priv->swaps_lock);
+	}
+
+	DRM_SPINUNLOCK(&dev_priv->swaps_lock);
+
+	if (nhits == 0) {
+		return;
+	}
+
+	i915_kernel_lost_context(dev);
+
+	upper[0] = upper[1] = 0;
+	slice[0] = max(sarea_priv->planeA_h / nhits, 1);
+	slice[1] = max(sarea_priv->planeB_h / nhits, 1);
+	lower[0] = sarea_priv->planeA_y + slice[0];
+	lower[1] = sarea_priv->planeB_y + slice[0];
+
+	offsets[0] = sarea_priv->front_offset;
+	offsets[1] = sarea_priv->back_offset;
+	offsets[2] = sarea_priv->third_offset;
+	num_pages = sarea_priv->third_handle ? 3 : 2;
+
+	DRM_SPINLOCK(&dev->drw_lock);
+
+	/* Emit blits for buffer swaps, partitioning both outputs into as many
+	 * slices as there are buffer swaps scheduled in order to avoid tearing
+	 * (based on the assumption that a single buffer swap would always
+	 * complete before scanout starts).
+	 */
+	for (i = 0; i++ < nhits;
+	     upper[0] = lower[0], lower[0] += slice[0],
+	     upper[1] = lower[1], lower[1] += slice[1]) {
+		int init_drawrect = 1;
+
+		if (i == nhits)
+			lower[0] = lower[1] = sarea_priv->height;
+
+		list_for_each(hit, &hits) {
+			drm_i915_vbl_swap_t *swap_hit =
+				list_entry(hit, drm_i915_vbl_swap_t, head);
+			struct drm_clip_rect *rect;
+			int num_rects, plane, front, back;
+			unsigned short top, bottom;
+
+			drw = drm_get_drawable_info(dev, swap_hit->drw_id);
+
+			if (!drw)
+				continue;
+
+			plane = swap_hit->plane;
+
+			if (swap_hit->flip) {
+				i915_dispatch_vsync_flip(dev, drw, plane);
+				continue;
+			}
+
+			if (init_drawrect) {
+				int width  = sarea_priv->width;
+				int height = sarea_priv->height;
+				if (IS_I965G(dev)) {
+					BEGIN_LP_RING(4);
+
+					OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
+					OUT_RING(0);
+					OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
+					OUT_RING(0);
+					
+					ADVANCE_LP_RING();
+				} else {
+					BEGIN_LP_RING(6);
+	
+					OUT_RING(GFX_OP_DRAWRECT_INFO);
+					OUT_RING(0);
+					OUT_RING(0);
+					OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
+					OUT_RING(0);
+					OUT_RING(0);
+					
+					ADVANCE_LP_RING();
+				}
+
+				sarea_priv->ctxOwner = DRM_KERNEL_CONTEXT;
+
+				init_drawrect = 0;
+			}
+
+			rect = drw->rects;
+			top = upper[plane];
+			bottom = lower[plane];
+
+			front = (dev_priv->sarea_priv->pf_current_page >>
+				 (2 * plane)) & 0x3;
+			back = (front + 1) % num_pages;
+
+			for (num_rects = drw->num_rects; num_rects--; rect++) {
+				int y1 = max(rect->y1, top);
+				int y2 = min(rect->y2, bottom);
+
+				if (y1 >= y2)
+					continue;
+
+				BEGIN_LP_RING(8);
+
+				OUT_RING(cmd);
+				OUT_RING(ropcpp | dst_pitch);
+				OUT_RING((y1 << 16) | rect->x1);
+				OUT_RING((y2 << 16) | rect->x2);
+				OUT_RING(offsets[front]);
+				OUT_RING((y1 << 16) | rect->x1);
+				OUT_RING(src_pitch);
+				OUT_RING(offsets[back]);
+
+				ADVANCE_LP_RING();
+			}
+		}
+	}
+
+	DRM_SPINUNLOCK(&dev->drw_lock);
+
+	list_for_each_safe(hit, tmp, &hits) {
+		drm_i915_vbl_swap_t *swap_hit =
+			list_entry(hit, drm_i915_vbl_swap_t, head);
+
+		list_del(hit);
+
+		drm_free(swap_hit, sizeof(*swap_hit), DRM_MEM_DRIVER);
+	}
+}
+
+u32 i915_get_vblank_counter(struct drm_device *dev, int plane)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	unsigned long high_frame;
+	unsigned long low_frame;
+	u32 high1, high2, low, count;
+	int pipe;
+
+	pipe = i915_get_pipe(dev, plane);
+	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
+	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
+
+	if (!i915_pipe_enabled(dev, pipe)) {
+	    DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
+	    return 0;
+	}
+
+	/*
+	 * High & low register fields aren't synchronized, so make sure
+	 * we get a low value that's stable across two reads of the high
+	 * register.
+	 */
+	do {
+		high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
+			 PIPE_FRAME_HIGH_SHIFT);
+		low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
+			PIPE_FRAME_LOW_SHIFT);
+		high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
+			 PIPE_FRAME_HIGH_SHIFT);
+	} while (high1 != high2);
+
+	count = (high1 << 8) | low;
+
+	return count;
+}
+
+irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
+{
+	struct drm_device *dev = (struct drm_device *) arg;
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 iir;
+	u32 pipea_stats, pipeb_stats;
+	int vblank = 0;
+
+	iir = I915_READ(IIR);
+	if (iir == 0)
 		return IRQ_NONE;
 
-	I915_WRITE16(I915REG_INT_IDENTITY_R, temp);
+	/*
+	 * Clear the PIPE(A|B)STAT regs before the IIR otherwise
+	 * we may get extra interrupts.
+	 */
+	if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
+		pipea_stats = I915_READ(PIPEASTAT);
+		if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
+				   PIPE_VBLANK_INTERRUPT_STATUS))
+		{
+			vblank++;
+			drm_handle_vblank(dev, i915_get_plane(dev, 0));
+		}
+
+		I915_WRITE(PIPEASTAT, pipea_stats);
+	}
+	if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
+		pipeb_stats = I915_READ(PIPEBSTAT);
+		/* Ack the event */
+		I915_WRITE(PIPEBSTAT, pipeb_stats);
+
+		/* The vblank interrupt gets enabled even if we didn't ask for
+		   it, so make sure it's shut down again */
+		if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
+			pipeb_stats &= ~(I915_VBLANK_INTERRUPT_ENABLE);
+
+		if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
+				   PIPE_VBLANK_INTERRUPT_STATUS))
+		{
+			vblank++;
+			drm_handle_vblank(dev, i915_get_plane(dev, 1));
+		}
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
+		if (pipeb_stats & I915_LEGACY_BLC_EVENT_ENABLE)
+			opregion_asle_intr(dev);
+#endif
+#endif
+		I915_WRITE(PIPEBSTAT, pipeb_stats);
+	}
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
+	if (iir & I915_ASLE_INTERRUPT)
+		opregion_asle_intr(dev);
+#endif
+#endif
+
+	if (dev_priv->sarea_priv)
+	    dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 
-	dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+	I915_WRITE(IIR, iir);
+	(void) I915_READ(IIR);
 
-	if (temp & USER_INT_FLAG)
+	if (iir & I915_USER_INTERRUPT) {
 		DRM_WAKEUP(&dev_priv->irq_queue);
+#ifdef I915_HAVE_FENCE
+		i915_fence_handler(dev);
+#endif
+	}
 
-	if (temp & (VSYNC_PIPEA_FLAG | VSYNC_PIPEB_FLAG)) {
-		atomic_inc(&dev->vbl_received);
-		DRM_WAKEUP(&dev->vbl_queue);
-		drm_vbl_send_signals(dev);
+	if (vblank) {
+		if (dev_priv->swaps_pending > 0)
+			drm_locked_tasklet(dev, i915_vblank_tasklet);
 	}
 
 	return IRQ_HANDLED;
 }
 
-static int i915_emit_irq(drm_device_t * dev)
+int i915_emit_irq(struct drm_device *dev)
 {
-	
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
 	i915_kernel_lost_context(dev);
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
-
-	dev_priv->sarea_priv->last_enqueue = ++dev_priv->counter;
-
-	if (dev_priv->counter > 0x7FFFFFFFUL)
-		 dev_priv->sarea_priv->last_enqueue = dev_priv->counter = 1;
+	DRM_DEBUG("\n");
 
-	BEGIN_LP_RING(6);
-	OUT_RING(CMD_STORE_DWORD_IDX);
-	OUT_RING(20);
-	OUT_RING(dev_priv->counter);
+	i915_emit_breadcrumb(dev);
 
+	BEGIN_LP_RING(2);
 	OUT_RING(0);
-	OUT_RING(0);
-	OUT_RING(GFX_OP_USER_INTERRUPT);
+	OUT_RING(MI_USER_INTERRUPT);
 	ADVANCE_LP_RING();
 
 	return dev_priv->counter;
+}
 
+void i915_user_irq_on(drm_i915_private_t *dev_priv)
+{
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
+	if (dev_priv->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
+		dev_priv->irq_enable_reg |= I915_USER_INTERRUPT;
+		I915_WRITE(IER, dev_priv->irq_enable_reg);
+	}
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 
 }
 
-static int i915_wait_irq(drm_device_t * dev, int irq_nr)
+void i915_user_irq_off(drm_i915_private_t *dev_priv)
+{
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
+	if (dev_priv->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
+		//		dev_priv->irq_enable_reg &= ~I915_USER_INTERRUPT;
+		//		I915_WRITE(IER, dev_priv->irq_enable_reg);
+	}
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
+}
+
+
+static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	int ret = 0;
 
-	DRM_DEBUG("%s irq_nr=%d breadcrumb=%d\n", __FUNCTION__, irq_nr,
+	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
 		  READ_BREADCRUMB(dev_priv));
 
 	if (READ_BREADCRUMB(dev_priv) >= irq_nr)
 		return 0;
 
-	dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
-
+	i915_user_irq_on(dev_priv);
 	DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
 		    READ_BREADCRUMB(dev_priv) >= irq_nr);
+	i915_user_irq_off(dev_priv);
 
-	if (ret == DRM_ERR(EBUSY)) {
-		DRM_ERROR("%s: EBUSY -- rec: %d emitted: %d\n",
-			  __FUNCTION__,
+	if (ret == -EBUSY) {
+		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
 			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
 	}
 
-	dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
-	return ret;
-}
-
-int i915_driver_vblank_wait(drm_device_t *dev, unsigned int *sequence)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	unsigned int cur_vblank;
-	int ret = 0;
-
-	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
-	}
-
-	DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-		    (((cur_vblank = atomic_read(&dev->vbl_received))
-			- *sequence) <= (1<<23)));
-	
-	*sequence = cur_vblank;
-
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_dispatch =
+			READ_BREADCRUMB(dev_priv);
 	return ret;
 }
 
 /* Needs the lock as it touches the ring.
  */
-int i915_irq_emit(DRM_IOCTL_ARGS)
+int i915_irq_emit(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_irq_emit_t emit;
+	drm_i915_irq_emit_t *emit = data;
 	int result;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(emit, (drm_i915_irq_emit_t __user *) data,
-				 sizeof(emit));
-
 	result = i915_emit_irq(dev);
 
-	if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
+	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
@@ -178,115 +578,368 @@ int i915_irq_emit(DRM_IOCTL_ARGS)
 
 /* Doesn't need the hardware lock.
  */
-int i915_irq_wait(DRM_IOCTL_ARGS)
+int i915_irq_wait(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_irq_wait_t irqwait;
+	drm_i915_irq_wait_t *irqwait = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	return i915_wait_irq(dev, irqwait->irq_seq);
+}
+
+int i915_enable_vblank(struct drm_device *dev, int plane)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe = i915_get_pipe(dev, plane);
+	u32	pipestat_reg = 0;
+	u32	pipestat;
+
+	switch (pipe) {
+	case 0:
+		pipestat_reg = PIPEASTAT;
+		dev_priv->irq_enable_reg |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
+		break;
+	case 1:
+		pipestat_reg = PIPEBSTAT;
+		dev_priv->irq_enable_reg |= I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
+		break;
+	default:
+		DRM_ERROR("tried to enable vblank on non-existent pipe %d\n",
+			  pipe);
+		break;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(irqwait, (drm_i915_irq_wait_t __user *) data,
-				 sizeof(irqwait));
+	if (pipestat_reg)
+	{
+		pipestat = I915_READ (pipestat_reg);
+		/*
+		 * Older chips didn't have the start vblank interrupt,
+		 * but 
+		 */
+		if (IS_I965G (dev))
+			pipestat |= PIPE_START_VBLANK_INTERRUPT_ENABLE;
+		else
+			pipestat |= PIPE_VBLANK_INTERRUPT_ENABLE;
+		/*
+		 * Clear any pending status
+		 */
+		pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
+			     PIPE_VBLANK_INTERRUPT_STATUS);
+		I915_WRITE(pipestat_reg, pipestat);
+	}
+	I915_WRITE(IER, dev_priv->irq_enable_reg);
 
-	return i915_wait_irq(dev, irqwait.irq_seq);
+	return 0;
 }
 
-static int i915_enable_interrupt (drm_device_t *dev)
+void i915_disable_vblank(struct drm_device *dev, int plane)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe = i915_get_pipe(dev, plane);
+	u32	pipestat_reg = 0;
+	u32	pipestat;
+
+	switch (pipe) {
+	case 0:
+		pipestat_reg = PIPEASTAT;
+		dev_priv->irq_enable_reg &= ~I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
+		break;
+	case 1:
+		pipestat_reg = PIPEBSTAT;
+		dev_priv->irq_enable_reg &= ~I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
+		break;
+	default:
+		DRM_ERROR("tried to disable vblank on non-existent pipe %d\n",
+			  pipe);
+		break;
+	}
+
+	I915_WRITE(IER, dev_priv->irq_enable_reg);
+
+	if (pipestat_reg)
+	{
+		pipestat = I915_READ (pipestat_reg);
+		pipestat &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
+			      PIPE_VBLANK_INTERRUPT_ENABLE);
+		/*
+		 * Clear any pending status
+		 */
+		pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
+			     PIPE_VBLANK_INTERRUPT_STATUS);
+		I915_WRITE(pipestat_reg, pipestat);
+	}
+}
+
+static void i915_enable_interrupt (struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u16 flag;
 	
-	flag = 0;
-	if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A)
-		flag |= VSYNC_PIPEA_FLAG;
-	if (dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B)
-		flag |= VSYNC_PIPEB_FLAG;
-	if (dev_priv->vblank_pipe & ~(DRM_I915_VBLANK_PIPE_A|DRM_I915_VBLANK_PIPE_B)) {
-		DRM_ERROR("%s called with invalid pipe 0x%x\n", 
-			  __FUNCTION__, dev_priv->vblank_pipe);
-		return DRM_ERR(EINVAL);
-	}
-	I915_WRITE16(I915REG_INT_ENABLE_R, USER_INT_FLAG | flag);
-	return 0;
+	dev_priv->irq_enable_reg |= I915_USER_INTERRUPT;
+
+#ifdef __linux__
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
+	opregion_enable_asle(dev);
+#endif
+#endif
+
+	I915_WRITE(IER, dev_priv->irq_enable_reg);
+	dev_priv->irq_enabled = 1;
 }
 
 /* Set the vblank monitor pipe
  */
-int i915_vblank_pipe_set(DRM_IOCTL_ARGS)
+int i915_vblank_pipe_set(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_vblank_pipe_t pipe;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(pipe, (drm_i915_vblank_pipe_t __user *) data,
-				 sizeof(pipe));
+	return 0;
+}
 
-	dev_priv->vblank_pipe = pipe.pipe;
-	return i915_enable_interrupt (dev);
+int i915_vblank_pipe_get(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_vblank_pipe_t *pipe = data;
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
+
+	return 0;
 }
 
-int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
+/**
+ * Schedule buffer swap at given vertical blank.
+ */
+int i915_vblank_swap(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_vblank_pipe_t pipe;
-	u16 flag;
+	drm_i915_vblank_swap_t *swap = data;
+	drm_i915_vbl_swap_t *vbl_swap;
+	unsigned int pipe, seqtype, curseq, plane;
+	unsigned long irqflags;
+	struct list_head *list;
+	int ret;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
-	}
-
-	flag = I915_READ(I915REG_INT_ENABLE_R);
-	pipe.pipe = 0;
-	if (flag & VSYNC_PIPEA_FLAG)
-		pipe.pipe |= DRM_I915_VBLANK_PIPE_A;
-	if (flag & VSYNC_PIPEB_FLAG)
-		pipe.pipe |= DRM_I915_VBLANK_PIPE_B;
-	DRM_COPY_TO_USER_IOCTL((drm_i915_vblank_pipe_t __user *) data, pipe,
-				 sizeof(pipe));
+		DRM_ERROR("%s called with no initialization\n", __func__);
+		return -EINVAL;
+	}
+
+	if (!dev_priv->sarea_priv || dev_priv->sarea_priv->rotation) {
+		DRM_DEBUG("Rotation not supported\n");
+		return -EINVAL;
+	}
+
+	if (swap->seqtype & ~(_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE |
+			     _DRM_VBLANK_SECONDARY | _DRM_VBLANK_NEXTONMISS |
+			     _DRM_VBLANK_FLIP)) {
+		DRM_ERROR("Invalid sequence type 0x%x\n", swap->seqtype);
+		return -EINVAL;
+	}
+
+	plane = (swap->seqtype & _DRM_VBLANK_SECONDARY) ? 1 : 0;
+	pipe = i915_get_pipe(dev, plane);
+
+	seqtype = swap->seqtype & (_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE);
+
+	if (!(dev_priv->vblank_pipe & (1 << pipe))) {
+		DRM_ERROR("Invalid pipe %d\n", pipe);
+		return -EINVAL;
+	}
+
+	DRM_SPINLOCK_IRQSAVE(&dev->drw_lock, irqflags);
+
+	/* It makes no sense to schedule a swap for a drawable that doesn't have
+	 * valid information at this point. E.g. this could mean that the X
+	 * server is too old to push drawable information to the DRM, in which
+	 * case all such swaps would become ineffective.
+	 */
+	if (!drm_get_drawable_info(dev, swap->drawable)) {
+		DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
+		DRM_DEBUG("Invalid drawable ID %d\n", swap->drawable);
+		return -EINVAL;
+	}
+
+	DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
+
+	/*
+	 * We take the ref here and put it when the swap actually completes
+	 * in the tasklet.
+	 */
+	ret = drm_vblank_get(dev, pipe);
+	if (ret)
+		return ret;
+	curseq = drm_vblank_count(dev, pipe);
+
+	if (seqtype == _DRM_VBLANK_RELATIVE)
+		swap->sequence += curseq;
+
+	if ((curseq - swap->sequence) <= (1<<23)) {
+		if (swap->seqtype & _DRM_VBLANK_NEXTONMISS) {
+			swap->sequence = curseq + 1;
+		} else {
+			DRM_DEBUG("Missed target sequence\n");
+			drm_vblank_put(dev, pipe);
+			return -EINVAL;
+		}
+	}
+
+	if (swap->seqtype & _DRM_VBLANK_FLIP) {
+		swap->sequence--;
+
+		if ((curseq - swap->sequence) <= (1<<23)) {
+			struct drm_drawable_info *drw;
+
+			LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+			DRM_SPINLOCK_IRQSAVE(&dev->drw_lock, irqflags);
+
+			drw = drm_get_drawable_info(dev, swap->drawable);
+
+			if (!drw) {
+				DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock,
+				    irqflags);
+				DRM_DEBUG("Invalid drawable ID %d\n",
+					  swap->drawable);
+				drm_vblank_put(dev, pipe);
+				return -EINVAL;
+			}
+
+			i915_dispatch_vsync_flip(dev, drw, plane);
+
+			DRM_SPINUNLOCK_IRQRESTORE(&dev->drw_lock, irqflags);
+
+			drm_vblank_put(dev, pipe);
+			return 0;
+		}
+	}
+
+	DRM_SPINLOCK_IRQSAVE(&dev_priv->swaps_lock, irqflags);
+
+	list_for_each(list, &dev_priv->vbl_swaps.head) {
+		vbl_swap = list_entry(list, drm_i915_vbl_swap_t, head);
+
+		if (vbl_swap->drw_id == swap->drawable &&
+		    vbl_swap->plane == plane &&
+		    vbl_swap->sequence == swap->sequence) {
+			vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
+			DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
+			DRM_DEBUG("Already scheduled\n");
+			return 0;
+		}
+	}
+
+	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
+
+	if (dev_priv->swaps_pending >= 100) {
+		DRM_DEBUG("Too many swaps queued\n");
+		drm_vblank_put(dev, pipe);
+		return -EBUSY;
+	}
+
+	vbl_swap = drm_calloc(1, sizeof(*vbl_swap), DRM_MEM_DRIVER);
+
+	if (!vbl_swap) {
+		DRM_ERROR("Failed to allocate memory to queue swap\n");
+		drm_vblank_put(dev, pipe);
+		return -ENOMEM;
+	}
+
+	DRM_DEBUG("\n");
+
+	vbl_swap->drw_id = swap->drawable;
+	vbl_swap->plane = plane;
+	vbl_swap->sequence = swap->sequence;
+	vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
+
+	if (vbl_swap->flip)
+		swap->sequence++;
+
+	DRM_SPINLOCK_IRQSAVE(&dev_priv->swaps_lock, irqflags);
+
+	list_add_tail(&vbl_swap->head, &dev_priv->vbl_swaps.head);
+	dev_priv->swaps_pending++;
+
+	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->swaps_lock, irqflags);
+
 	return 0;
 }
 
 /* drm_dma.h hooks
 */
-void i915_driver_irq_preinstall(drm_device_t * dev)
+void i915_driver_irq_preinstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
-	I915_WRITE16(I915REG_HWSTAM, 0xfffe);
-	I915_WRITE16(I915REG_INT_MASK_R, 0x0);
-	I915_WRITE16(I915REG_INT_ENABLE_R, 0x0);
+	I915_WRITE16(HWSTAM, 0xeffe);
+	I915_WRITE16(IMR, 0x0);
+	I915_WRITE16(IER, 0x0);
 }
 
-void i915_driver_irq_postinstall(drm_device_t * dev)
+int i915_driver_irq_postinstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int ret, num_pipes = 2;
+
+	INIT_LIST_HEAD(&dev_priv->vbl_swaps.head);
+	dev_priv->swaps_pending = 0;
+
+	dev_priv->user_irq_refcount = 0;
+	dev_priv->irq_enable_reg = 0;
+
+	ret = drm_vblank_init(dev, num_pipes);
+	if (ret)
+		return ret;
+
+	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
+	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
 
 	i915_enable_interrupt(dev);
 	DRM_INIT_WAITQUEUE(&dev_priv->irq_queue);
+
+	/*
+	 * Initialize the hardware status page IRQ location.
+	 */
+
+	I915_WRITE(INSTPM, (1 << 5) | (1 << 21));
+	return 0;
 }
 
-void i915_driver_irq_uninstall(drm_device_t * dev)
+void i915_driver_irq_uninstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u16 temp;
+	u32 temp;
+
 	if (!dev_priv)
 		return;
 
-	I915_WRITE16(I915REG_HWSTAM, 0xffff);
-	I915_WRITE16(I915REG_INT_MASK_R, 0xffff);
-	I915_WRITE16(I915REG_INT_ENABLE_R, 0x0);
+	dev_priv->vblank_pipe = 0;
+
+	dev_priv->irq_enabled = 0;
+	I915_WRITE(HWSTAM, 0xffffffff);
+	I915_WRITE(IMR, 0xffffffff);
+	I915_WRITE(IER, 0x0);
 
-	temp = I915_READ16(I915REG_INT_IDENTITY_R);
-	I915_WRITE16(I915REG_INT_IDENTITY_R, temp);
+	temp = I915_READ(PIPEASTAT);
+	I915_WRITE(PIPEASTAT, temp);
+	temp = I915_READ(PIPEBSTAT);
+	I915_WRITE(PIPEBSTAT, temp);
+	temp = I915_READ(IIR);
+	I915_WRITE(IIR, temp);
 }
diff --git a/sys/dev/drm/i915_mem.c b/sys/dev/drm/i915_mem.c
index fd234c2..42dd5a8 100644
--- a/sys/dev/drm/i915_mem.c
+++ b/sys/dev/drm/i915_mem.c
@@ -46,11 +46,11 @@ __FBSDID("$FreeBSD$");
  * block to allocate, and the ring is drained prior to allocations --
  * in other words allocation is expensive.
  */
-static void mark_block(drm_device_t * dev, struct mem_block *p, int in_use)
+static void mark_block(struct drm_device * dev, struct mem_block *p, int in_use)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_tex_region_t *list;
+	struct drm_tex_region *list;
 	unsigned shift, nr;
 	unsigned start;
 	unsigned end;
@@ -92,7 +92,7 @@ static void mark_block(drm_device_t * dev, struct mem_block *p, int in_use)
  */
 
 static struct mem_block *split_block(struct mem_block *p, int start, int size,
-				     DRMFILE filp)
+				     struct drm_file *file_priv)
 {
 	/* Maybe cut off the start of an existing block */
 	if (start > p->start) {
@@ -102,7 +102,7 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 			goto out;
 		newblock->start = start;
 		newblock->size = p->size - (start - p->start);
-		newblock->filp = NULL;
+		newblock->file_priv = NULL;
 		newblock->next = p->next;
 		newblock->prev = p;
 		p->next->prev = newblock;
@@ -119,7 +119,7 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 			goto out;
 		newblock->start = start + size;
 		newblock->size = p->size - size;
-		newblock->filp = NULL;
+		newblock->file_priv = NULL;
 		newblock->next = p->next;
 		newblock->prev = p;
 		p->next->prev = newblock;
@@ -129,20 +129,20 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 
       out:
 	/* Our block is in the middle */
-	p->filp = filp;
+	p->file_priv = file_priv;
 	return p;
 }
 
 static struct mem_block *alloc_block(struct mem_block *heap, int size,
-				     int align2, DRMFILE filp)
+				     int align2, struct drm_file *file_priv)
 {
 	struct mem_block *p;
 	int mask = (1 << align2) - 1;
 
 	for (p = heap->next; p != heap; p = p->next) {
 		int start = (p->start + mask) & ~mask;
-		if (p->filp == NULL && start + size <= p->start + p->size)
-			return split_block(p, start, size, filp);
+		if (p->file_priv == NULL && start + size <= p->start + p->size)
+			return split_block(p, start, size, file_priv);
 	}
 
 	return NULL;
@@ -161,12 +161,12 @@ static struct mem_block *find_block(struct mem_block *heap, int start)
 
 static void free_block(struct mem_block *p)
 {
-	p->filp = NULL;
+	p->file_priv = NULL;
 
-	/* Assumes a single contiguous range.  Needs a special filp in
+	/* Assumes a single contiguous range.  Needs a special file_priv in
 	 * 'heap' to stop it being subsumed.
 	 */
-	if (p->next->filp == NULL) {
+	if (p->next->file_priv == NULL) {
 		struct mem_block *q = p->next;
 		p->size += q->size;
 		p->next = q->next;
@@ -174,7 +174,7 @@ static void free_block(struct mem_block *p)
 		drm_free(q, sizeof(*q), DRM_MEM_BUFLISTS);
 	}
 
-	if (p->prev->filp == NULL) {
+	if (p->prev->file_priv == NULL) {
 		struct mem_block *q = p->prev;
 		q->size += p->size;
 		q->next = p->next;
@@ -200,18 +200,19 @@ static int init_heap(struct mem_block **heap, int start, int size)
 
 	blocks->start = start;
 	blocks->size = size;
-	blocks->filp = NULL;
+	blocks->file_priv = NULL;
 	blocks->next = blocks->prev = *heap;
 
 	memset(*heap, 0, sizeof(**heap));
-	(*heap)->filp = (DRMFILE) - 1;
+	(*heap)->file_priv = (struct drm_file *) - 1;
 	(*heap)->next = (*heap)->prev = blocks;
 	return 0;
 }
 
 /* Free all blocks associated with the releasing file.
  */
-void i915_mem_release(drm_device_t * dev, DRMFILE filp, struct mem_block *heap)
+void i915_mem_release(struct drm_device * dev, struct drm_file *file_priv,
+		      struct mem_block *heap)
 {
 	struct mem_block *p;
 
@@ -219,17 +220,17 @@ void i915_mem_release(drm_device_t * dev, DRMFILE filp, struct mem_block *heap)
 		return;
 
 	for (p = heap->next; p != heap; p = p->next) {
-		if (p->filp == filp) {
-			p->filp = NULL;
+		if (p->file_priv == file_priv) {
+			p->file_priv = NULL;
 			mark_block(dev, p, 0);
 		}
 	}
 
-	/* Assumes a single contiguous range.  Needs a special filp in
+	/* Assumes a single contiguous range.  Needs a special file_priv in
 	 * 'heap' to stop it being subsumed.
 	 */
 	for (p = heap->next; p != heap; p = p->next) {
-		while (p->filp == NULL && p->next->filp == NULL) {
+		while (p->file_priv == NULL && p->next->file_priv == NULL) {
 			struct mem_block *q = p->next;
 			p->size += q->size;
 			p->next = q->next;
@@ -270,129 +271,117 @@ static struct mem_block **get_heap(drm_i915_private_t * dev_priv, int region)
 
 /* IOCTL HANDLERS */
 
-int i915_mem_alloc(DRM_IOCTL_ARGS)
+int i915_mem_alloc(struct drm_device *dev, void *data,
+		   struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_mem_alloc_t alloc;
+	drm_i915_mem_alloc_t *alloc = data;
 	struct mem_block *block, **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(alloc, (drm_i915_mem_alloc_t __user *) data,
-				 sizeof(alloc));
-
-	heap = get_heap(dev_priv, alloc.region);
+	heap = get_heap(dev_priv, alloc->region);
 	if (!heap || !*heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
 	/* Make things easier on ourselves: all allocations at least
 	 * 4k aligned.
 	 */
-	if (alloc.alignment < 12)
-		alloc.alignment = 12;
+	if (alloc->alignment < 12)
+		alloc->alignment = 12;
 
-	block = alloc_block(*heap, alloc.size, alloc.alignment, filp);
+	block = alloc_block(*heap, alloc->size, alloc->alignment, file_priv);
 
 	if (!block)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	mark_block(dev, block, 1);
 
-	if (DRM_COPY_TO_USER(alloc.region_offset, &block->start, sizeof(int))) {
+	if (DRM_COPY_TO_USER(alloc->region_offset, &block->start,
+			     sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-int i915_mem_free(DRM_IOCTL_ARGS)
+int i915_mem_free(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_mem_free_t memfree;
+	drm_i915_mem_free_t *memfree = data;
 	struct mem_block *block, **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(memfree, (drm_i915_mem_free_t __user *) data,
-				 sizeof(memfree));
-
-	heap = get_heap(dev_priv, memfree.region);
+	heap = get_heap(dev_priv, memfree->region);
 	if (!heap || !*heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
-	block = find_block(*heap, memfree.region_offset);
+	block = find_block(*heap, memfree->region_offset);
 	if (!block)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
-	if (block->filp != filp)
-		return DRM_ERR(EPERM);
+	if (block->file_priv != file_priv)
+		return -EPERM;
 
 	mark_block(dev, block, 0);
 	free_block(block);
 	return 0;
 }
 
-int i915_mem_init_heap(DRM_IOCTL_ARGS)
+int i915_mem_init_heap(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_mem_init_heap_t initheap;
+	drm_i915_mem_init_heap_t *initheap = data;
 	struct mem_block **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(initheap,
-				 (drm_i915_mem_init_heap_t __user *) data,
-				 sizeof(initheap));
-
-	heap = get_heap(dev_priv, initheap.region);
+	heap = get_heap(dev_priv, initheap->region);
 	if (!heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
 	if (*heap) {
 		DRM_ERROR("heap already initialized?");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
-	return init_heap(heap, initheap.start, initheap.size);
+	return init_heap(heap, initheap->start, initheap->size);
 }
 
-int i915_mem_destroy_heap( DRM_IOCTL_ARGS )
+int i915_mem_destroy_heap( struct drm_device *dev, void *data,
+			   struct drm_file *file_priv )
 {
-	DRM_DEVICE;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_mem_destroy_heap_t destroyheap;
+	drm_i915_mem_destroy_heap_t *destroyheap = data;
 	struct mem_block **heap;
 
 	if ( !dev_priv ) {
-		DRM_ERROR( "%s called with no initialization\n", __FUNCTION__ );
-		return DRM_ERR(EINVAL);
+		DRM_ERROR( "called with no initialization\n" );
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL( destroyheap, (drm_i915_mem_destroy_heap_t *)data,
-				  sizeof(destroyheap) );
-
-	heap = get_heap( dev_priv, destroyheap.region );
+	heap = get_heap( dev_priv, destroyheap->region );
 	if (!heap) {
 		DRM_ERROR("get_heap failed");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
-	
+
 	if (!*heap) {
 		DRM_ERROR("heap not initialized?");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	i915_mem_takedown( heap );
diff --git a/sys/dev/drm/i915_suspend.c b/sys/dev/drm/i915_suspend.c
new file mode 100644
index 0000000..5e518ee
--- /dev/null
+++ b/sys/dev/drm/i915_suspend.c
@@ -0,0 +1,521 @@
+/*-
+ *
+ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/i915_drm.h"
+#include "dev/drm/i915_drv.h"
+
+static bool i915_pipe_enabled(struct drm_device *dev, enum pipe pipe)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (pipe == PIPE_A)
+		return (I915_READ(DPLL_A) & DPLL_VCO_ENABLE);
+	else
+		return (I915_READ(DPLL_B) & DPLL_VCO_ENABLE);
+}
+
+static void i915_save_palette(struct drm_device *dev, enum pipe pipe)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
+	u32 *array;
+	int i;
+
+	if (!i915_pipe_enabled(dev, pipe))
+		return;
+
+	if (pipe == PIPE_A)
+		array = dev_priv->save_palette_a;
+	else
+		array = dev_priv->save_palette_b;
+
+	for(i = 0; i < 256; i++)
+		array[i] = I915_READ(reg + (i << 2));
+}
+
+static void i915_restore_palette(struct drm_device *dev, enum pipe pipe)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
+	u32 *array;
+	int i;
+
+	if (!i915_pipe_enabled(dev, pipe))
+		return;
+
+	if (pipe == PIPE_A)
+		array = dev_priv->save_palette_a;
+	else
+		array = dev_priv->save_palette_b;
+
+	for(i = 0; i < 256; i++)
+		I915_WRITE(reg + (i << 2), array[i]);
+}
+
+static u8 i915_read_indexed(struct drm_device *dev, u16 index_port, u16 data_port, u8 reg)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE8(index_port, reg);
+	return I915_READ8(data_port);
+}
+
+static u8 i915_read_ar(struct drm_device *dev, u16 st01, u8 reg, u16 palette_enable)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_READ8(st01);
+	I915_WRITE8(VGA_AR_INDEX, palette_enable | reg);
+	return I915_READ8(VGA_AR_DATA_READ);
+}
+
+static void i915_write_ar(struct drm_device *dev, u16 st01, u8 reg, u8 val, u16 palette_enable)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_READ8(st01);
+	I915_WRITE8(VGA_AR_INDEX, palette_enable | reg);
+	I915_WRITE8(VGA_AR_DATA_WRITE, val);
+}
+
+static void i915_write_indexed(struct drm_device *dev, u16 index_port, u16 data_port, u8 reg, u8 val)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE8(index_port, reg);
+	I915_WRITE8(data_port, val);
+}
+
+static void i915_save_vga(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int i;
+	u16 cr_index, cr_data, st01;
+
+	/* VGA color palette registers */
+	dev_priv->saveDACMASK = I915_READ8(VGA_DACMASK);
+	/* DACCRX automatically increments during read */
+	I915_WRITE8(VGA_DACRX, 0);
+	/* Read 3 bytes of color data from each index */
+	for (i = 0; i < 256 * 3; i++)
+		dev_priv->saveDACDATA[i] = I915_READ8(VGA_DACDATA);
+
+	/* MSR bits */
+	dev_priv->saveMSR = I915_READ8(VGA_MSR_READ);
+	if (dev_priv->saveMSR & VGA_MSR_CGA_MODE) {
+		cr_index = VGA_CR_INDEX_CGA;
+		cr_data = VGA_CR_DATA_CGA;
+		st01 = VGA_ST01_CGA;
+	} else {
+		cr_index = VGA_CR_INDEX_MDA;
+		cr_data = VGA_CR_DATA_MDA;
+		st01 = VGA_ST01_MDA;
+	}
+
+	/* CRT controller regs */
+	i915_write_indexed(dev, cr_index, cr_data, 0x11,
+			   i915_read_indexed(dev, cr_index, cr_data, 0x11) &
+			   (~0x80));
+	for (i = 0; i <= 0x24; i++)
+		dev_priv->saveCR[i] =
+			i915_read_indexed(dev, cr_index, cr_data, i);
+	/* Make sure we don't turn off CR group 0 writes */
+	dev_priv->saveCR[0x11] &= ~0x80;
+
+	/* Attribute controller registers */
+	I915_READ8(st01);
+	dev_priv->saveAR_INDEX = I915_READ8(VGA_AR_INDEX);
+	for (i = 0; i <= 0x14; i++)
+		dev_priv->saveAR[i] = i915_read_ar(dev, st01, i, 0);
+	I915_READ8(st01);
+	I915_WRITE8(VGA_AR_INDEX, dev_priv->saveAR_INDEX);
+	I915_READ8(st01);
+
+	/* Graphics controller registers */
+	for (i = 0; i < 9; i++)
+		dev_priv->saveGR[i] =
+			i915_read_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, i);
+
+	dev_priv->saveGR[0x10] =
+		i915_read_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x10);
+	dev_priv->saveGR[0x11] =
+		i915_read_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x11);
+	dev_priv->saveGR[0x18] =
+		i915_read_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x18);
+
+	/* Sequencer registers */
+	for (i = 0; i < 8; i++)
+		dev_priv->saveSR[i] =
+			i915_read_indexed(dev, VGA_SR_INDEX, VGA_SR_DATA, i);
+}
+
+static void i915_restore_vga(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int i;
+	u16 cr_index, cr_data, st01;
+
+	/* MSR bits */
+	I915_WRITE8(VGA_MSR_WRITE, dev_priv->saveMSR);
+	if (dev_priv->saveMSR & VGA_MSR_CGA_MODE) {
+		cr_index = VGA_CR_INDEX_CGA;
+		cr_data = VGA_CR_DATA_CGA;
+		st01 = VGA_ST01_CGA;
+	} else {
+		cr_index = VGA_CR_INDEX_MDA;
+		cr_data = VGA_CR_DATA_MDA;
+		st01 = VGA_ST01_MDA;
+	}
+
+	/* Sequencer registers, don't write SR07 */
+	for (i = 0; i < 7; i++)
+		i915_write_indexed(dev, VGA_SR_INDEX, VGA_SR_DATA, i,
+				   dev_priv->saveSR[i]);
+
+	/* CRT controller regs */
+	/* Enable CR group 0 writes */
+	i915_write_indexed(dev, cr_index, cr_data, 0x11, dev_priv->saveCR[0x11]);
+	for (i = 0; i <= 0x24; i++)
+		i915_write_indexed(dev, cr_index, cr_data, i, dev_priv->saveCR[i]);
+
+	/* Graphics controller regs */
+	for (i = 0; i < 9; i++)
+		i915_write_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, i,
+				   dev_priv->saveGR[i]);
+
+	i915_write_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x10,
+			   dev_priv->saveGR[0x10]);
+	i915_write_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x11,
+			   dev_priv->saveGR[0x11]);
+	i915_write_indexed(dev, VGA_GR_INDEX, VGA_GR_DATA, 0x18,
+			   dev_priv->saveGR[0x18]);
+
+	/* Attribute controller registers */
+	I915_READ8(st01); /* switch back to index mode */
+	for (i = 0; i <= 0x14; i++)
+		i915_write_ar(dev, st01, i, dev_priv->saveAR[i], 0);
+	I915_READ8(st01); /* switch back to index mode */
+	I915_WRITE8(VGA_AR_INDEX, dev_priv->saveAR_INDEX | 0x20);
+	I915_READ8(st01);
+
+	/* VGA color palette registers */
+	I915_WRITE8(VGA_DACMASK, dev_priv->saveDACMASK);
+	/* DACCRX automatically increments during read */
+	I915_WRITE8(VGA_DACWX, 0);
+	/* Read 3 bytes of color data from each index */
+	for (i = 0; i < 256 * 3; i++)
+		I915_WRITE8(VGA_DACDATA, dev_priv->saveDACDATA[i]);
+
+}
+
+int i915_save_state(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int i;
+
+#if defined(__FreeBSD__)
+	dev_priv->saveLBB = (u8) pci_read_config(dev->device, LBB, 1);
+#else
+	pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
+#endif
+
+	/* Display arbitration control */
+	dev_priv->saveDSPARB = I915_READ(DSPARB);
+
+	/* Pipe & plane A info */
+	dev_priv->savePIPEACONF = I915_READ(PIPEACONF);
+	dev_priv->savePIPEASRC = I915_READ(PIPEASRC);
+	dev_priv->saveFPA0 = I915_READ(FPA0);
+	dev_priv->saveFPA1 = I915_READ(FPA1);
+	dev_priv->saveDPLL_A = I915_READ(DPLL_A);
+	if (IS_I965G(dev))
+		dev_priv->saveDPLL_A_MD = I915_READ(DPLL_A_MD);
+	dev_priv->saveHTOTAL_A = I915_READ(HTOTAL_A);
+	dev_priv->saveHBLANK_A = I915_READ(HBLANK_A);
+	dev_priv->saveHSYNC_A = I915_READ(HSYNC_A);
+	dev_priv->saveVTOTAL_A = I915_READ(VTOTAL_A);
+	dev_priv->saveVBLANK_A = I915_READ(VBLANK_A);
+	dev_priv->saveVSYNC_A = I915_READ(VSYNC_A);
+	dev_priv->saveBCLRPAT_A = I915_READ(BCLRPAT_A);
+
+	dev_priv->saveDSPACNTR = I915_READ(DSPACNTR);
+	dev_priv->saveDSPASTRIDE = I915_READ(DSPASTRIDE);
+	dev_priv->saveDSPASIZE = I915_READ(DSPASIZE);
+	dev_priv->saveDSPAPOS = I915_READ(DSPAPOS);
+	dev_priv->saveDSPAADDR = I915_READ(DSPAADDR);
+	if (IS_I965G(dev)) {
+		dev_priv->saveDSPASURF = I915_READ(DSPASURF);
+		dev_priv->saveDSPATILEOFF = I915_READ(DSPATILEOFF);
+	}
+	i915_save_palette(dev, PIPE_A);
+	dev_priv->savePIPEASTAT = I915_READ(PIPEASTAT);
+
+	/* Pipe & plane B info */
+	dev_priv->savePIPEBCONF = I915_READ(PIPEBCONF);
+	dev_priv->savePIPEBSRC = I915_READ(PIPEBSRC);
+	dev_priv->saveFPB0 = I915_READ(FPB0);
+	dev_priv->saveFPB1 = I915_READ(FPB1);
+	dev_priv->saveDPLL_B = I915_READ(DPLL_B);
+	if (IS_I965G(dev))
+		dev_priv->saveDPLL_B_MD = I915_READ(DPLL_B_MD);
+	dev_priv->saveHTOTAL_B = I915_READ(HTOTAL_B);
+	dev_priv->saveHBLANK_B = I915_READ(HBLANK_B);
+	dev_priv->saveHSYNC_B = I915_READ(HSYNC_B);
+	dev_priv->saveVTOTAL_B = I915_READ(VTOTAL_B);
+	dev_priv->saveVBLANK_B = I915_READ(VBLANK_B);
+	dev_priv->saveVSYNC_B = I915_READ(VSYNC_B);
+	dev_priv->saveBCLRPAT_A = I915_READ(BCLRPAT_A);
+
+	dev_priv->saveDSPBCNTR = I915_READ(DSPBCNTR);
+	dev_priv->saveDSPBSTRIDE = I915_READ(DSPBSTRIDE);
+	dev_priv->saveDSPBSIZE = I915_READ(DSPBSIZE);
+	dev_priv->saveDSPBPOS = I915_READ(DSPBPOS);
+	dev_priv->saveDSPBADDR = I915_READ(DSPBADDR);
+	if (IS_I965GM(dev) || IS_GM45(dev)) {
+		dev_priv->saveDSPBSURF = I915_READ(DSPBSURF);
+		dev_priv->saveDSPBTILEOFF = I915_READ(DSPBTILEOFF);
+	}
+	i915_save_palette(dev, PIPE_B);
+	dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
+
+	/* CRT state */
+	dev_priv->saveADPA = I915_READ(ADPA);
+
+	/* LVDS state */
+	dev_priv->savePP_CONTROL = I915_READ(PP_CONTROL);
+	dev_priv->savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
+	dev_priv->saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
+	if (IS_I965G(dev))
+		dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
+	if (IS_MOBILE(dev) && !IS_I830(dev))
+		dev_priv->saveLVDS = I915_READ(LVDS);
+	if (!IS_I830(dev) && !IS_845G(dev))
+		dev_priv->savePFIT_CONTROL = I915_READ(PFIT_CONTROL);
+	dev_priv->savePP_ON_DELAYS = I915_READ(PP_ON_DELAYS);
+	dev_priv->savePP_OFF_DELAYS = I915_READ(PP_OFF_DELAYS);
+	dev_priv->savePP_DIVISOR = I915_READ(PP_DIVISOR);
+
+	/* FIXME: save TV & SDVO state */
+
+	/* FBC state */
+	dev_priv->saveFBC_CFB_BASE = I915_READ(FBC_CFB_BASE);
+	dev_priv->saveFBC_LL_BASE = I915_READ(FBC_LL_BASE);
+	dev_priv->saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2);
+	dev_priv->saveFBC_CONTROL = I915_READ(FBC_CONTROL);
+
+	/* Interrupt state */
+	dev_priv->saveIIR = I915_READ(IIR);
+	dev_priv->saveIER = I915_READ(IER);
+	dev_priv->saveIMR = I915_READ(IMR);
+
+	/* VGA state */
+	dev_priv->saveVGA0 = I915_READ(VGA0);
+	dev_priv->saveVGA1 = I915_READ(VGA1);
+	dev_priv->saveVGA_PD = I915_READ(VGA_PD);
+	dev_priv->saveVGACNTRL = I915_READ(VGACNTRL);
+
+	/* Clock gating state */
+	dev_priv->saveD_STATE = I915_READ(D_STATE);
+	dev_priv->saveCG_2D_DIS = I915_READ(CG_2D_DIS);
+
+	/* Cache mode state */
+	dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
+
+	/* Memory Arbitration state */
+	dev_priv->saveMI_ARB_STATE = I915_READ(MI_ARB_STATE);
+
+	/* Scratch space */
+	for (i = 0; i < 16; i++) {
+		dev_priv->saveSWF0[i] = I915_READ(SWF00 + (i << 2));
+		dev_priv->saveSWF1[i] = I915_READ(SWF10 + (i << 2));
+	}
+	for (i = 0; i < 3; i++)
+		dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
+
+	i915_save_vga(dev);
+
+	return 0;
+}
+
+int i915_restore_state(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int i;
+
+#if defined(__FreeBSD__)
+	pci_write_config(dev->device, LBB, dev_priv->saveLBB, 1);
+#else
+	pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
+#endif
+
+	I915_WRITE(DSPARB, dev_priv->saveDSPARB);
+
+	/* Pipe & plane A info */
+	/* Prime the clock */
+	if (dev_priv->saveDPLL_A & DPLL_VCO_ENABLE) {
+		I915_WRITE(DPLL_A, dev_priv->saveDPLL_A &
+			   ~DPLL_VCO_ENABLE);
+		DRM_UDELAY(150);
+	}
+	I915_WRITE(FPA0, dev_priv->saveFPA0);
+	I915_WRITE(FPA1, dev_priv->saveFPA1);
+	/* Actually enable it */
+	I915_WRITE(DPLL_A, dev_priv->saveDPLL_A);
+	DRM_UDELAY(150);
+	if (IS_I965G(dev))
+		I915_WRITE(DPLL_A_MD, dev_priv->saveDPLL_A_MD);
+	DRM_UDELAY(150);
+
+	/* Restore mode */
+	I915_WRITE(HTOTAL_A, dev_priv->saveHTOTAL_A);
+	I915_WRITE(HBLANK_A, dev_priv->saveHBLANK_A);
+	I915_WRITE(HSYNC_A, dev_priv->saveHSYNC_A);
+	I915_WRITE(VTOTAL_A, dev_priv->saveVTOTAL_A);
+	I915_WRITE(VBLANK_A, dev_priv->saveVBLANK_A);
+	I915_WRITE(VSYNC_A, dev_priv->saveVSYNC_A);
+	I915_WRITE(BCLRPAT_A, dev_priv->saveBCLRPAT_A);
+
+	/* Restore plane info */
+	I915_WRITE(DSPASIZE, dev_priv->saveDSPASIZE);
+	I915_WRITE(DSPAPOS, dev_priv->saveDSPAPOS);
+	I915_WRITE(PIPEASRC, dev_priv->savePIPEASRC);
+	I915_WRITE(DSPAADDR, dev_priv->saveDSPAADDR);
+	I915_WRITE(DSPASTRIDE, dev_priv->saveDSPASTRIDE);
+	if (IS_I965G(dev)) {
+		I915_WRITE(DSPASURF, dev_priv->saveDSPASURF);
+		I915_WRITE(DSPATILEOFF, dev_priv->saveDSPATILEOFF);
+	}
+
+	I915_WRITE(PIPEACONF, dev_priv->savePIPEACONF);
+
+	i915_restore_palette(dev, PIPE_A);
+	/* Enable the plane */
+	I915_WRITE(DSPACNTR, dev_priv->saveDSPACNTR);
+	I915_WRITE(DSPAADDR, I915_READ(DSPAADDR));
+
+	/* Pipe & plane B info */
+	if (dev_priv->saveDPLL_B & DPLL_VCO_ENABLE) {
+		I915_WRITE(DPLL_B, dev_priv->saveDPLL_B &
+			   ~DPLL_VCO_ENABLE);
+		DRM_UDELAY(150);
+	}
+	I915_WRITE(FPB0, dev_priv->saveFPB0);
+	I915_WRITE(FPB1, dev_priv->saveFPB1);
+	/* Actually enable it */
+	I915_WRITE(DPLL_B, dev_priv->saveDPLL_B);
+	DRM_UDELAY(150);
+	if (IS_I965G(dev))
+		I915_WRITE(DPLL_B_MD, dev_priv->saveDPLL_B_MD);
+	DRM_UDELAY(150);
+
+	/* Restore mode */
+	I915_WRITE(HTOTAL_B, dev_priv->saveHTOTAL_B);
+	I915_WRITE(HBLANK_B, dev_priv->saveHBLANK_B);
+	I915_WRITE(HSYNC_B, dev_priv->saveHSYNC_B);
+	I915_WRITE(VTOTAL_B, dev_priv->saveVTOTAL_B);
+	I915_WRITE(VBLANK_B, dev_priv->saveVBLANK_B);
+	I915_WRITE(VSYNC_B, dev_priv->saveVSYNC_B);
+	I915_WRITE(BCLRPAT_B, dev_priv->saveBCLRPAT_B);
+
+	/* Restore plane info */
+	I915_WRITE(DSPBSIZE, dev_priv->saveDSPBSIZE);
+	I915_WRITE(DSPBPOS, dev_priv->saveDSPBPOS);
+	I915_WRITE(PIPEBSRC, dev_priv->savePIPEBSRC);
+	I915_WRITE(DSPBADDR, dev_priv->saveDSPBADDR);
+	I915_WRITE(DSPBSTRIDE, dev_priv->saveDSPBSTRIDE);
+	if (IS_I965G(dev)) {
+		I915_WRITE(DSPBSURF, dev_priv->saveDSPBSURF);
+		I915_WRITE(DSPBTILEOFF, dev_priv->saveDSPBTILEOFF);
+	}
+
+	I915_WRITE(PIPEBCONF, dev_priv->savePIPEBCONF);
+
+	i915_restore_palette(dev, PIPE_B);
+	/* Enable the plane */
+	I915_WRITE(DSPBCNTR, dev_priv->saveDSPBCNTR);
+	I915_WRITE(DSPBADDR, I915_READ(DSPBADDR));
+
+	/* CRT state */
+	I915_WRITE(ADPA, dev_priv->saveADPA);
+
+	/* LVDS state */
+	if (IS_I965G(dev))
+		I915_WRITE(BLC_PWM_CTL2, dev_priv->saveBLC_PWM_CTL2);
+	if (IS_MOBILE(dev) && !IS_I830(dev))
+		I915_WRITE(LVDS, dev_priv->saveLVDS);
+	if (!IS_I830(dev) && !IS_845G(dev))
+		I915_WRITE(PFIT_CONTROL, dev_priv->savePFIT_CONTROL);
+
+	I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS);
+	I915_WRITE(BLC_PWM_CTL, dev_priv->saveBLC_PWM_CTL);
+	I915_WRITE(PP_ON_DELAYS, dev_priv->savePP_ON_DELAYS);
+	I915_WRITE(PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS);
+	I915_WRITE(PP_DIVISOR, dev_priv->savePP_DIVISOR);
+	I915_WRITE(PP_CONTROL, dev_priv->savePP_CONTROL);
+
+	/* FIXME: restore TV & SDVO state */
+
+	/* FBC info */
+	I915_WRITE(FBC_CFB_BASE, dev_priv->saveFBC_CFB_BASE);
+	I915_WRITE(FBC_LL_BASE, dev_priv->saveFBC_LL_BASE);
+	I915_WRITE(FBC_CONTROL2, dev_priv->saveFBC_CONTROL2);
+	I915_WRITE(FBC_CONTROL, dev_priv->saveFBC_CONTROL);
+
+	/* VGA state */
+	I915_WRITE(VGACNTRL, dev_priv->saveVGACNTRL);
+	I915_WRITE(VGA0, dev_priv->saveVGA0);
+	I915_WRITE(VGA1, dev_priv->saveVGA1);
+	I915_WRITE(VGA_PD, dev_priv->saveVGA_PD);
+	DRM_UDELAY(150);
+
+	/* Clock gating state */
+	I915_WRITE (D_STATE, dev_priv->saveD_STATE);
+	I915_WRITE (CG_2D_DIS, dev_priv->saveCG_2D_DIS);
+
+	/* Cache mode state */
+	I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);
+
+	/* Memory arbitration state */
+	I915_WRITE (MI_ARB_STATE, dev_priv->saveMI_ARB_STATE | 0xffff0000);
+
+	for (i = 0; i < 16; i++) {
+		I915_WRITE(SWF00 + (i << 2), dev_priv->saveSWF0[i]);
+		I915_WRITE(SWF10 + (i << 2), dev_priv->saveSWF1[i+7]);
+	}
+	for (i = 0; i < 3; i++)
+		I915_WRITE(SWF30 + (i << 2), dev_priv->saveSWF2[i]);
+
+	i915_restore_vga(dev);
+
+	return 0;
+}
+
diff --git a/sys/dev/drm/mach64_dma.c b/sys/dev/drm/mach64_dma.c
index 77966a1..1cb7779 100644
--- a/sys/dev/drm/mach64_dma.c
+++ b/sys/dev/drm/mach64_dma.c
@@ -6,7 +6,7 @@
  * \author Gareth Hughes <gareth@valinux.com>
  * \author Frank C. Earl <fearl@airmail.net>
  * \author Leif Delgass <ldelgass@retinalburn.net>
- * \author Jose Fonseca <j_r_fonseca@yahoo.co.uk>
+ * \author José Fonseca <j_r_fonseca@yahoo.co.uk>
  */
 
 /*-
@@ -56,11 +56,11 @@ __FBSDID("$FreeBSD$");
  *
  * \param dev_priv pointer to device private data structure.
  * \param entries number of free entries in the FIFO to wait for.
- * 
+ *
  * \returns zero on success, or -EBUSY if the timeout (specificed by
  * drm_mach64_private::usec_timeout) occurs.
  */
-int mach64_do_wait_for_fifo(drm_mach64_private_t * dev_priv, int entries)
+int mach64_do_wait_for_fifo(drm_mach64_private_t *dev_priv, int entries)
 {
 	int slots = 0, i;
 
@@ -71,15 +71,14 @@ int mach64_do_wait_for_fifo(drm_mach64_private_t * dev_priv, int entries)
 		DRM_UDELAY(1);
 	}
 
-	DRM_INFO("%s failed! slots=%d entries=%d\n", __FUNCTION__, slots,
-		 entries);
-	return DRM_ERR(EBUSY);
+	DRM_INFO("failed! slots=%d entries=%d\n", slots, entries);
+	return -EBUSY;
 }
 
 /**
  * Wait for the draw engine to be idle.
  */
-int mach64_do_wait_for_idle(drm_mach64_private_t * dev_priv)
+int mach64_do_wait_for_idle(drm_mach64_private_t *dev_priv)
 {
 	int i, ret;
 
@@ -88,16 +87,14 @@ int mach64_do_wait_for_idle(drm_mach64_private_t * dev_priv)
 		return ret;
 
 	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		if (!(MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE)) {
+		if (!(MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE))
 			return 0;
-		}
 		DRM_UDELAY(1);
 	}
 
-	DRM_INFO("%s failed! GUI_STAT=0x%08x\n", __FUNCTION__,
-		 MACH64_READ(MACH64_GUI_STAT));
+	DRM_INFO("failed! GUI_STAT=0x%08x\n", MACH64_READ(MACH64_GUI_STAT));
 	mach64_dump_ring_info(dev_priv);
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /**
@@ -110,16 +107,16 @@ int mach64_do_wait_for_idle(drm_mach64_private_t * dev_priv)
  *
  * This function should be called before writing new entries to the ring
  * buffer.
- * 
+ *
  * \param dev_priv pointer to device private data structure.
  * \param n number of free entries in the ring buffer to wait for.
- * 
+ *
  * \returns zero on success, or -EBUSY if the timeout (specificed by
  * drm_mach64_private_t::usec_timeout) occurs.
  *
  * \sa mach64_dump_ring_info()
  */
-int mach64_wait_ring(drm_mach64_private_t * dev_priv, int n)
+int mach64_wait_ring(drm_mach64_private_t *dev_priv, int n)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 	int i;
@@ -127,9 +124,8 @@ int mach64_wait_ring(drm_mach64_private_t * dev_priv, int n)
 	for (i = 0; i < dev_priv->usec_timeout; i++) {
 		mach64_update_ring_snapshot(dev_priv);
 		if (ring->space >= n) {
-			if (i > 0) {
-				DRM_DEBUG("%s: %d usecs\n", __FUNCTION__, i);
-			}
+			if (i > 0)
+				DRM_DEBUG("%d usecs\n", i);
 			return 0;
 		}
 		DRM_UDELAY(1);
@@ -138,15 +134,15 @@ int mach64_wait_ring(drm_mach64_private_t * dev_priv, int n)
 	/* FIXME: This is being ignored... */
 	DRM_ERROR("failed!\n");
 	mach64_dump_ring_info(dev_priv);
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /**
- * Wait until all DMA requests have been processed... 
+ * Wait until all DMA requests have been processed...
  *
  * \sa mach64_wait_ring()
  */
-static int mach64_ring_idle(drm_mach64_private_t * dev_priv)
+static int mach64_ring_idle(drm_mach64_private_t *dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 	u32 head;
@@ -158,9 +154,8 @@ static int mach64_ring_idle(drm_mach64_private_t * dev_priv)
 		mach64_update_ring_snapshot(dev_priv);
 		if (ring->head == ring->tail &&
 		    !(MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE)) {
-			if (i > 0) {
-				DRM_DEBUG("%s: %d usecs\n", __FUNCTION__, i);
-			}
+			if (i > 0)
+				DRM_DEBUG("%d usecs\n", i);
 			return 0;
 		}
 		if (ring->head == head) {
@@ -172,10 +167,9 @@ static int mach64_ring_idle(drm_mach64_private_t * dev_priv)
 		DRM_UDELAY(1);
 	}
 
-	DRM_INFO("%s failed! GUI_STAT=0x%08x\n", __FUNCTION__,
-		 MACH64_READ(MACH64_GUI_STAT));
+	DRM_INFO("failed! GUI_STAT=0x%08x\n", MACH64_READ(MACH64_GUI_STAT));
 	mach64_dump_ring_info(dev_priv);
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /**
@@ -183,7 +177,7 @@ static int mach64_ring_idle(drm_mach64_private_t * dev_priv)
  *
  * \sa mach64_do_engine_reset()
  */
-static void mach64_ring_reset(drm_mach64_private_t * dev_priv)
+static void mach64_ring_reset(drm_mach64_private_t *dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 
@@ -201,7 +195,7 @@ static void mach64_ring_reset(drm_mach64_private_t * dev_priv)
 /**
  * Ensure the all the queued commands will be processed.
  */
-int mach64_do_dma_flush(drm_mach64_private_t * dev_priv)
+int mach64_do_dma_flush(drm_mach64_private_t *dev_priv)
 {
 	/* FIXME: It's not necessary to wait for idle when flushing
 	 * we just need to ensure the ring will be completely processed
@@ -213,14 +207,14 @@ int mach64_do_dma_flush(drm_mach64_private_t * dev_priv)
 /**
  * Stop all DMA activity.
  */
-int mach64_do_dma_idle(drm_mach64_private_t * dev_priv)
+int mach64_do_dma_idle(drm_mach64_private_t *dev_priv)
 {
 	int ret;
 
 	/* wait for completion */
 	if ((ret = mach64_ring_idle(dev_priv)) < 0) {
-		DRM_ERROR("%s failed BM_GUI_TABLE=0x%08x tail: %u\n",
-			  __FUNCTION__, MACH64_READ(MACH64_BM_GUI_TABLE),
+		DRM_ERROR("failed BM_GUI_TABLE=0x%08x tail: %u\n",
+			  MACH64_READ(MACH64_BM_GUI_TABLE),
 			  dev_priv->ring.tail);
 		return ret;
 	}
@@ -235,11 +229,11 @@ int mach64_do_dma_idle(drm_mach64_private_t * dev_priv)
 /**
  * Reset the engine.  This will stop the DMA if it is running.
  */
-int mach64_do_engine_reset(drm_mach64_private_t * dev_priv)
+int mach64_do_engine_reset(drm_mach64_private_t *dev_priv)
 {
 	u32 tmp;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	/* Kill off any outstanding DMA transfers.
 	 */
@@ -279,7 +273,7 @@ int mach64_do_engine_reset(drm_mach64_private_t * dev_priv)
 /**
  * Dump engine registers values.
  */
-void mach64_dump_engine_info(drm_mach64_private_t * dev_priv)
+void mach64_dump_engine_info(drm_mach64_private_t *dev_priv)
 {
 	DRM_INFO("\n");
 	if (!dev_priv->is_pci) {
@@ -420,8 +414,8 @@ void mach64_dump_engine_info(drm_mach64_private_t * dev_priv)
  * Used by mach64_dump_ring_info() to dump the contents of the current buffer
  * pointed by the ring head.
  */
-static void mach64_dump_buf_info(drm_mach64_private_t * dev_priv,
-				 drm_buf_t * buf)
+static void mach64_dump_buf_info(drm_mach64_private_t *dev_priv,
+				 struct drm_buf *buf)
 {
 	u32 addr = GETBUFADDR(buf);
 	u32 used = buf->used >> 2;
@@ -480,7 +474,7 @@ static void mach64_dump_buf_info(drm_mach64_private_t * dev_priv,
  * Dump the ring state and contents, including the contents of the buffer being
  * processed by the graphics engine.
  */
-void mach64_dump_ring_info(drm_mach64_private_t * dev_priv)
+void mach64_dump_ring_info(drm_mach64_private_t *dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 	int i, skipped;
@@ -525,13 +519,12 @@ void mach64_dump_ring_info(drm_mach64_private_t * dev_priv)
 		list_for_each(ptr, &dev_priv->pending) {
 			drm_mach64_freelist_t *entry =
 			    list_entry(ptr, drm_mach64_freelist_t, list);
-			drm_buf_t *buf = entry->buf;
+			struct drm_buf *buf = entry->buf;
 
 			u32 buf_addr = GETBUFADDR(buf);
 
-			if (buf_addr <= addr && addr < buf_addr + buf->used) {
+			if (buf_addr <= addr && addr < buf_addr + buf->used)
 				mach64_dump_buf_info(dev_priv, buf);
-			}
 		}
 	}
 
@@ -562,6 +555,259 @@ void mach64_dump_ring_info(drm_mach64_private_t * dev_priv)
 
 
 /*******************************************************************/
+/** \name DMA descriptor ring macros */
+/*@{*/
+
+/**
+ * Add the end mark to the ring's new tail position.
+ *
+ * The bus master engine will keep processing the DMA buffers listed in the ring
+ * until it finds this mark, making it stop.
+ *
+ * \sa mach64_clear_dma_eol
+ */ 
+static __inline__ void mach64_set_dma_eol(volatile u32 *addr)
+{
+#if defined(__i386__)
+	int nr = 31;
+
+	/* Taken from include/asm-i386/bitops.h linux header */
+	__asm__ __volatile__("lock;" "btsl %1,%0":"=m"(*addr)
+			     :"Ir"(nr));
+#elif defined(__powerpc__)
+	u32 old;
+	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
+
+	/* Taken from the include/asm-ppc/bitops.h linux header */
+	__asm__ __volatile__("\n\
+1:	lwarx	%0,0,%3 \n\
+	or	%0,%0,%2 \n\
+	stwcx.	%0,0,%3 \n\
+	bne-	1b":"=&r"(old), "=m"(*addr)
+			     :"r"(mask), "r"(addr), "m"(*addr)
+			     :"cc");
+#elif defined(__alpha__)
+	u32 temp;
+	u32 mask = MACH64_DMA_EOL;
+
+	/* Taken from the include/asm-alpha/bitops.h linux header */
+	__asm__ __volatile__("1:	ldl_l %0,%3\n"
+			     "	bis %0,%2,%0\n"
+			     "	stl_c %0,%1\n"
+			     "	beq %0,2f\n"
+			     ".subsection 2\n"
+			     "2:	br 1b\n"
+			     ".previous":"=&r"(temp), "=m"(*addr)
+			     :"Ir"(mask), "m"(*addr));
+#else
+	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
+
+	*addr |= mask;
+#endif
+}
+
+/**
+ * Remove the end mark from the ring's old tail position.
+ *
+ * It should be called after calling mach64_set_dma_eol to mark the ring's new
+ * tail position.
+ *
+ * We update the end marks while the bus master engine is in operation. Since
+ * the bus master engine may potentially be reading from the same position
+ * that we write, we must change atomically to avoid having intermediary bad
+ * data.
+ */
+static __inline__ void mach64_clear_dma_eol(volatile u32 *addr)
+{
+#if defined(__i386__)
+	int nr = 31;
+
+	/* Taken from include/asm-i386/bitops.h linux header */
+	__asm__ __volatile__("lock;" "btrl %1,%0":"=m"(*addr)
+			     :"Ir"(nr));
+#elif defined(__powerpc__)
+	u32 old;
+	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
+
+	/* Taken from the include/asm-ppc/bitops.h linux header */
+	__asm__ __volatile__("\n\
+1:	lwarx	%0,0,%3 \n\
+	andc	%0,%0,%2 \n\
+	stwcx.	%0,0,%3 \n\
+	bne-	1b":"=&r"(old), "=m"(*addr)
+			     :"r"(mask), "r"(addr), "m"(*addr)
+			     :"cc");
+#elif defined(__alpha__)
+	u32 temp;
+	u32 mask = ~MACH64_DMA_EOL;
+
+	/* Taken from the include/asm-alpha/bitops.h linux header */
+	__asm__ __volatile__("1:	ldl_l %0,%3\n"
+			     "	and %0,%2,%0\n"
+			     "	stl_c %0,%1\n"
+			     "	beq %0,2f\n"
+			     ".subsection 2\n"
+			     "2:	br 1b\n"
+			     ".previous":"=&r"(temp), "=m"(*addr)
+			     :"Ir"(mask), "m"(*addr));
+#else
+	u32 mask = cpu_to_le32(~MACH64_DMA_EOL);
+
+	*addr &= mask;
+#endif
+}
+
+#define RING_LOCALS							\
+	int _ring_tail, _ring_write; unsigned int _ring_mask; volatile u32 *_ring
+
+#define RING_WRITE_OFS  _ring_write
+
+#define BEGIN_RING(n)							\
+	do {								\
+		if (MACH64_VERBOSE) {					\
+			DRM_INFO( "BEGIN_RING( %d ) \n",		\
+				  (n) );				\
+		}							\
+		if (dev_priv->ring.space <= (n) * sizeof(u32)) {	\
+			int ret;					\
+			if ((ret = mach64_wait_ring( dev_priv, (n) * sizeof(u32))) < 0 ) { \
+				DRM_ERROR( "wait_ring failed, resetting engine\n"); \
+				mach64_dump_engine_info( dev_priv );	\
+				mach64_do_engine_reset( dev_priv );	\
+				return ret;				\
+			}						\
+		}							\
+		dev_priv->ring.space -= (n) * sizeof(u32);		\
+		_ring = (u32 *) dev_priv->ring.start;			\
+		_ring_tail = _ring_write = dev_priv->ring.tail;		\
+		_ring_mask = dev_priv->ring.tail_mask;			\
+	} while (0)
+
+#define OUT_RING( x )						\
+do {								\
+	if (MACH64_VERBOSE) {					\
+		DRM_INFO( "   OUT_RING( 0x%08x ) at 0x%x\n",	\
+			   (unsigned int)(x), _ring_write );	\
+	}							\
+	_ring[_ring_write++] = cpu_to_le32( x );		\
+	_ring_write &= _ring_mask;				\
+} while (0)
+
+#define ADVANCE_RING()							\
+do {									\
+	if (MACH64_VERBOSE) {						\
+		DRM_INFO( "ADVANCE_RING() wr=0x%06x tail=0x%06x\n",	\
+			  _ring_write, _ring_tail );			\
+	}								\
+	DRM_MEMORYBARRIER();						\
+	mach64_clear_dma_eol( &_ring[(_ring_tail - 2) & _ring_mask] );	\
+	DRM_MEMORYBARRIER();						\
+	dev_priv->ring.tail = _ring_write;				\
+	mach64_ring_tick( dev_priv, &(dev_priv)->ring );		\
+} while (0)
+
+/**
+ * Queue a DMA buffer of registers writes into the ring buffer.
+ */ 
+int mach64_add_buf_to_ring(drm_mach64_private_t *dev_priv,
+                           drm_mach64_freelist_t *entry)
+{
+	int bytes, pages, remainder;
+	u32 address, page;
+	int i;
+	struct drm_buf *buf = entry->buf;
+	RING_LOCALS;
+
+	bytes = buf->used;
+	address = GETBUFADDR( buf );
+	pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;
+
+	BEGIN_RING( pages * 4 );
+
+	for ( i = 0 ; i < pages-1 ; i++ ) {
+		page = address + i * MACH64_DMA_CHUNKSIZE;
+		OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
+		OUT_RING( page );
+		OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );
+		OUT_RING( 0 );
+	}
+
+	/* generate the final descriptor for any remaining commands in this buffer */
+	page = address + i * MACH64_DMA_CHUNKSIZE;
+	remainder = bytes - i * MACH64_DMA_CHUNKSIZE;
+
+	/* Save dword offset of last descriptor for this buffer.
+	 * This is needed to check for completion of the buffer in freelist_get
+	 */
+	entry->ring_ofs = RING_WRITE_OFS;
+
+	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
+	OUT_RING( page );
+	OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );
+	OUT_RING( 0 );
+
+	ADVANCE_RING();
+	
+	return 0;
+}
+
+/**
+ * Queue DMA buffer controlling host data tranfers (e.g., blit).
+ * 
+ * Almost identical to mach64_add_buf_to_ring.
+ */
+int mach64_add_hostdata_buf_to_ring(drm_mach64_private_t *dev_priv,
+                                    drm_mach64_freelist_t *entry)
+{
+	int bytes, pages, remainder;
+	u32 address, page;
+	int i;
+	struct drm_buf *buf = entry->buf;
+	RING_LOCALS;
+	
+	bytes = buf->used - MACH64_HOSTDATA_BLIT_OFFSET;
+	pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;
+	address = GETBUFADDR( buf );
+	
+	BEGIN_RING( 4 + pages * 4 );
+	
+	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
+	OUT_RING( address );
+	OUT_RING( MACH64_HOSTDATA_BLIT_OFFSET | MACH64_DMA_HOLD_OFFSET );
+	OUT_RING( 0 );
+	address += MACH64_HOSTDATA_BLIT_OFFSET;
+	
+	for ( i = 0 ; i < pages-1 ; i++ ) {
+		page = address + i * MACH64_DMA_CHUNKSIZE;
+		OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );
+		OUT_RING( page );
+		OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );
+		OUT_RING( 0 );
+	}
+	
+	/* generate the final descriptor for any remaining commands in this buffer */
+	page = address + i * MACH64_DMA_CHUNKSIZE;
+	remainder = bytes - i * MACH64_DMA_CHUNKSIZE;
+	
+	/* Save dword offset of last descriptor for this buffer.
+	 * This is needed to check for completion of the buffer in freelist_get
+	 */
+	entry->ring_ofs = RING_WRITE_OFS;
+	
+	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );
+	OUT_RING( page );
+	OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );
+	OUT_RING( 0 );
+	
+	ADVANCE_RING();
+	
+	return 0;
+}
+
+/*@}*/
+
+
+/*******************************************************************/
 /** \name DMA test and initialization */
 /*@{*/
 
@@ -575,7 +821,7 @@ void mach64_dump_ring_info(drm_mach64_private_t * dev_priv)
  * DMA operation. It is left here since it so tricky to get DMA operating
  * properly in some architectures and hardware.
  */
-static int mach64_bm_dma_test(drm_device_t * dev)
+static int mach64_bm_dma_test(struct drm_device * dev)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_dma_handle_t *cpu_addr_dmah;
@@ -585,7 +831,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 	u32 src_cntl, pat_reg0, pat_reg1;
 	int i, count, failed;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	table = (u32 *) dev_priv->ring.start;
 
@@ -595,7 +841,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 	    drm_pci_alloc(dev, 0x1000, 0x1000, 0xfffffffful);
 	if (!cpu_addr_dmah) {
 		DRM_INFO("data-memory allocation failed!\n");
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	} else {
 		data = (u32 *) cpu_addr_dmah->vaddr;
 		data_addr = (u32) cpu_addr_dmah->busaddr;
@@ -627,7 +873,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 			mach64_do_engine_reset(dev_priv);
 			DRM_INFO("freeing data buffer memory.\n");
 			drm_pci_free(dev, cpu_addr_dmah);
-			return DRM_ERR(EIO);
+			return -EIO;
 		}
 	}
 
@@ -755,17 +1001,17 @@ static int mach64_bm_dma_test(drm_device_t * dev)
  * Called during the DMA initialization ioctl to initialize all the necessary
  * software and hardware state for DMA operation.
  */
-static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
+static int mach64_do_dma_init(struct drm_device * dev, drm_mach64_init_t * init)
 {
 	drm_mach64_private_t *dev_priv;
 	u32 tmp;
 	int i, ret;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	dev_priv = drm_alloc(sizeof(drm_mach64_private_t), DRM_MEM_DRIVER);
 	if (dev_priv == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv, 0, sizeof(drm_mach64_private_t));
 
@@ -795,55 +1041,56 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 	INIT_LIST_HEAD(&dev_priv->placeholders);
 	INIT_LIST_HEAD(&dev_priv->pending);
 
-	DRM_GETSAREA();
-
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("can not find sarea!\n");
 		dev->dev_private = (void *)dev_priv;
 		mach64_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev_priv->fb = drm_core_findmap(dev, init->fb_offset);
 	if (!dev_priv->fb) {
 		DRM_ERROR("can not find frame buffer map!\n");
 		dev->dev_private = (void *)dev_priv;
 		mach64_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
 	if (!dev_priv->mmio) {
 		DRM_ERROR("can not find mmio map!\n");
 		dev->dev_private = (void *)dev_priv;
 		mach64_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
+	}
+
+	dev_priv->ring_map = drm_core_findmap(dev, init->ring_offset);
+	if (!dev_priv->ring_map) {
+		DRM_ERROR("can not find ring map!\n");
+		dev->dev_private = (void *)dev_priv;
+		mach64_do_cleanup_dma(dev);
+		return -EINVAL;
 	}
 
 	dev_priv->sarea_priv = (drm_mach64_sarea_t *)
 	    ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset);
 
 	if (!dev_priv->is_pci) {
-		dev_priv->ring_map = drm_core_findmap(dev, init->ring_offset);
-		if (!dev_priv->ring_map) {
-			DRM_ERROR("can not find ring map!\n");
-			dev->dev_private = (void *)dev_priv;
-			mach64_do_cleanup_dma(dev);
-			return DRM_ERR(EINVAL);
-		}
 		drm_core_ioremap(dev_priv->ring_map, dev);
 		if (!dev_priv->ring_map->handle) {
 			DRM_ERROR("can not ioremap virtual address for"
 				  " descriptor ring\n");
 			dev->dev_private = (void *)dev_priv;
 			mach64_do_cleanup_dma(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
+		dev->agp_buffer_token = init->buffers_offset;
 		dev->agp_buffer_map =
 		    drm_core_findmap(dev, init->buffers_offset);
 		if (!dev->agp_buffer_map) {
 			DRM_ERROR("can not find dma buffer map!\n");
 			dev->dev_private = (void *)dev_priv;
 			mach64_do_cleanup_dma(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		/* there might be a nicer way to do this -
 		   dev isn't passed all the way though the mach64 - DA */
@@ -855,7 +1102,7 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 				  " dma buffer\n");
 			dev->dev_private = (void *)dev_priv;
 			mach64_do_cleanup_dma(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 		dev_priv->agp_textures =
 		    drm_core_findmap(dev, init->agp_textures_offset);
@@ -863,7 +1110,7 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 			DRM_ERROR("can not find agp texture region!\n");
 			dev->dev_private = (void *)dev_priv;
 			mach64_do_cleanup_dma(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -893,27 +1140,9 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 		}
 	}
 
-	/* allocate descriptor memory from pci pool */
-	DRM_DEBUG("Allocating dma descriptor ring\n");
 	dev_priv->ring.size = 0x4000;	/* 16KB */
-
-	if (dev_priv->is_pci) {
-		dev_priv->ring.dmah = drm_pci_alloc(dev, dev_priv->ring.size,
-						     dev_priv->ring.size,
-						     0xfffffffful);
-
-		if (!dev_priv->ring.dmah) {
-			DRM_ERROR("Allocating dma descriptor ring failed\n");
-			return DRM_ERR(ENOMEM);
-		} else {
-			dev_priv->ring.start = dev_priv->ring.dmah->vaddr;
-			dev_priv->ring.start_addr =
-			    (u32) dev_priv->ring.dmah->busaddr;
-		}
-	} else {
-		dev_priv->ring.start = dev_priv->ring_map->handle;
-		dev_priv->ring.start_addr = (u32) dev_priv->ring_map->offset;
-	}
+	dev_priv->ring.start = dev_priv->ring_map->handle;
+	dev_priv->ring.start_addr = (u32) dev_priv->ring_map->offset;
 
 	memset(dev_priv->ring.start, 0, dev_priv->ring.size);
 	DRM_INFO("descriptor ring: cpu addr %p, bus addr: 0x%08x\n",
@@ -988,13 +1217,13 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 /** MMIO Pseudo-DMA (intended primarily for debugging, not performance)
  */
 
-int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t * dev_priv)
+int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t *dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 	volatile u32 *ring_read;
 	struct list_head *ptr;
 	drm_mach64_freelist_t *entry;
-	drm_buf_t *buf = NULL;
+	struct drm_buf *buf = NULL;
 	u32 *buf_ptr;
 	u32 used, reg, target;
 	int fifo, count, found, ret, no_idle_wait;
@@ -1003,9 +1232,7 @@ int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t * dev_priv)
 	target = MACH64_BM_ADDR;
 
 	if ((ret = mach64_do_wait_for_idle(dev_priv)) < 0) {
-		DRM_INFO
-		    ("%s: idle failed before pseudo-dma dispatch, resetting engine\n",
-		     __FUNCTION__);
+		DRM_INFO("idle failed before pseudo-dma dispatch, resetting engine\n");
 		mach64_dump_engine_info(dev_priv);
 		mach64_do_engine_reset(dev_priv);
 		return ret;
@@ -1055,7 +1282,7 @@ int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t * dev_priv)
 			     head, ring->tail, buf_addr, (eol ? "eol" : ""));
 			mach64_dump_ring_info(dev_priv);
 			mach64_do_engine_reset(dev_priv);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		/* Hand feed the buffer to the card via MMIO, waiting for the fifo
@@ -1126,7 +1353,7 @@ int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t * dev_priv)
 	MACH64_WRITE(MACH64_BM_GUI_TABLE_CMD,
 		     ring->head_addr | MACH64_CIRCULAR_BUF_SIZE_16KB);
 
-	DRM_DEBUG("%s completed\n", __FUNCTION__);
+	DRM_DEBUG("completed\n");
 	return 0;
 }
 
@@ -1137,9 +1364,9 @@ int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t * dev_priv)
 /** \name DMA cleanup */
 /*@{*/
 
-int mach64_do_cleanup_dma(drm_device_t * dev)
+int mach64_do_cleanup_dma(struct drm_device * dev)
 {
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	/* Make sure interrupts are disabled here because the uninstall ioctl
 	 * may not have been called from userspace and after dev_private
@@ -1151,18 +1378,14 @@ int mach64_do_cleanup_dma(drm_device_t * dev)
 	if (dev->dev_private) {
 		drm_mach64_private_t *dev_priv = dev->dev_private;
 
-		if (dev_priv->is_pci) {
-			if (dev_priv->ring.dmah) {
-				drm_pci_free(dev, dev_priv->ring.dmah);
-			}
-		} else {
+		if (!dev_priv->is_pci) {
 			if (dev_priv->ring_map)
 				drm_core_ioremapfree(dev_priv->ring_map, dev);
-		}
 
-		if (dev->agp_buffer_map) {
-			drm_core_ioremapfree(dev->agp_buffer_map, dev);
-			dev->agp_buffer_map = NULL;
+			if (dev->agp_buffer_map) {
+				drm_core_ioremapfree(dev->agp_buffer_map, dev);
+				dev->agp_buffer_map = NULL;
+			}
 		}
 
 		mach64_destroy_freelist(dev);
@@ -1182,60 +1405,57 @@ int mach64_do_cleanup_dma(drm_device_t * dev)
 /** \name IOCTL handlers */
 /*@{*/
 
-int mach64_dma_init(DRM_IOCTL_ARGS)
+int mach64_dma_init(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_mach64_init_t init;
-
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	drm_mach64_init_t *init = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	DRM_DEBUG("\n");
 
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_mach64_init_t *) data,
-				 sizeof(init));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	switch (init.func) {
+	switch (init->func) {
 	case DRM_MACH64_INIT_DMA:
-		return mach64_do_dma_init(dev, &init);
+		return mach64_do_dma_init(dev, init);
 	case DRM_MACH64_CLEANUP_DMA:
 		return mach64_do_cleanup_dma(dev);
 	}
 
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
-int mach64_dma_idle(DRM_IOCTL_ARGS)
+int mach64_dma_idle(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return mach64_do_dma_idle(dev_priv);
 }
 
-int mach64_dma_flush(DRM_IOCTL_ARGS)
+int mach64_dma_flush(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return mach64_do_dma_flush(dev_priv);
 }
 
-int mach64_engine_reset(DRM_IOCTL_ARGS)
+int mach64_engine_reset(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return mach64_do_engine_reset(dev_priv);
 }
@@ -1247,23 +1467,22 @@ int mach64_engine_reset(DRM_IOCTL_ARGS)
 /** \name Freelist management */
 /*@{*/
 
-int mach64_init_freelist(drm_device_t * dev)
+int mach64_init_freelist(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_freelist_t *entry;
 	struct list_head *ptr;
 	int i;
 
-	DRM_DEBUG("%s: adding %d buffers to freelist\n", __FUNCTION__,
-		  dma->buf_count);
+	DRM_DEBUG("adding %d buffers to freelist\n", dma->buf_count);
 
 	for (i = 0; i < dma->buf_count; i++) {
 		if ((entry =
 		     (drm_mach64_freelist_t *)
 		     drm_alloc(sizeof(drm_mach64_freelist_t),
 			       DRM_MEM_BUFLISTS)) == NULL)
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		memset(entry, 0, sizeof(drm_mach64_freelist_t));
 		entry->buf = dma->buflist[i];
 		ptr = &entry->list;
@@ -1273,14 +1492,14 @@ int mach64_init_freelist(drm_device_t * dev)
 	return 0;
 }
 
-void mach64_destroy_freelist(drm_device_t * dev)
+void mach64_destroy_freelist(struct drm_device * dev)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_freelist_t *entry;
 	struct list_head *ptr;
 	struct list_head *tmp;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	list_for_each_safe(ptr, tmp, &dev_priv->pending) {
 		list_del(ptr);
@@ -1303,7 +1522,7 @@ void mach64_destroy_freelist(drm_device_t * dev)
 /* IMPORTANT: This function should only be called when the engine is idle or locked up,
  * as it assumes all buffers in the pending list have been completed by the hardware.
  */
-int mach64_do_release_used_buffers(drm_mach64_private_t * dev_priv)
+int mach64_do_release_used_buffers(drm_mach64_private_t *dev_priv)
 {
 	struct list_head *ptr;
 	struct list_head *tmp;
@@ -1325,23 +1544,92 @@ int mach64_do_release_used_buffers(drm_mach64_private_t * dev_priv)
 		}
 	}
 
-	DRM_DEBUG("%s: released %d buffers from pending list\n", __FUNCTION__,
-		  i);
+	DRM_DEBUG("released %d buffers from pending list\n", i);
 
 	return 0;
 }
 
-drm_buf_t *mach64_freelist_get(drm_mach64_private_t * dev_priv)
+static int mach64_do_reclaim_completed(drm_mach64_private_t *dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
-	drm_mach64_freelist_t *entry;
 	struct list_head *ptr;
 	struct list_head *tmp;
+	drm_mach64_freelist_t *entry;
+	u32 head, tail, ofs;
+
+	mach64_ring_tick(dev_priv, ring);
+	head = ring->head;
+	tail = ring->tail;
+
+	if (head == tail) {
+#if MACH64_EXTRA_CHECKING
+		if (MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE) {
+			DRM_ERROR("Empty ring with non-idle engine!\n");
+			mach64_dump_ring_info(dev_priv);
+			return -1;
+		}
+#endif
+		/* last pass is complete, so release everything */
+		mach64_do_release_used_buffers(dev_priv);
+		DRM_DEBUG("idle engine, freed all buffers.\n");
+		if (list_empty(&dev_priv->free_list)) {
+			DRM_ERROR("Freelist empty with idle engine\n");
+			return -1;
+		}
+		return 0;
+	}
+	/* Look for a completed buffer and bail out of the loop
+	 * as soon as we find one -- don't waste time trying
+	 * to free extra bufs here, leave that to do_release_used_buffers
+	 */
+	list_for_each_safe(ptr, tmp, &dev_priv->pending) {
+		entry = list_entry(ptr, drm_mach64_freelist_t, list);
+		ofs = entry->ring_ofs;
+		if (entry->discard &&
+		    ((head < tail && (ofs < head || ofs >= tail)) ||
+		     (head > tail && (ofs < head && ofs >= tail)))) {
+#if MACH64_EXTRA_CHECKING
+			int i;
+
+			for (i = head; i != tail; i = (i + 4) & ring->tail_mask)
+			{
+				u32 o1 = le32_to_cpu(((u32 *) ring->
+						 start)[i + 1]);
+				u32 o2 = GETBUFADDR(entry->buf);
+
+				if (o1 == o2) {
+					DRM_ERROR
+					    ("Attempting to free used buffer: "
+					     "i=%d  buf=0x%08x\n",
+					     i, o1);
+					mach64_dump_ring_info(dev_priv);
+					return -1;
+				}
+			}
+#endif
+			/* found a processed buffer */
+			entry->buf->pending = 0;
+			list_del(ptr);
+			list_add_tail(ptr, &dev_priv->free_list);
+			DRM_DEBUG
+			    ("freed processed buffer (head=%d tail=%d "
+			     "buf ring ofs=%d).\n",
+			     head, tail, ofs);
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
+struct drm_buf *mach64_freelist_get(drm_mach64_private_t *dev_priv)
+{
+	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
+	drm_mach64_freelist_t *entry;
+	struct list_head *ptr;
 	int t;
 
 	if (list_empty(&dev_priv->free_list)) {
-		u32 head, tail, ofs;
-
 		if (list_empty(&dev_priv->pending)) {
 			DRM_ERROR
 			    ("Couldn't get buffer - pending and free lists empty\n");
@@ -1353,81 +1641,15 @@ drm_buf_t *mach64_freelist_get(drm_mach64_private_t * dev_priv)
 			return NULL;
 		}
 
-		tail = ring->tail;
 		for (t = 0; t < dev_priv->usec_timeout; t++) {
-			mach64_ring_tick(dev_priv, ring);
-			head = ring->head;
+			int ret;
 
-			if (head == tail) {
-#if MACH64_EXTRA_CHECKING
-				if (MACH64_READ(MACH64_GUI_STAT) &
-				    MACH64_GUI_ACTIVE) {
-					DRM_ERROR
-					    ("Empty ring with non-idle engine!\n");
-					mach64_dump_ring_info(dev_priv);
-					return NULL;
-				}
-#endif
-				/* last pass is complete, so release everything */
-				mach64_do_release_used_buffers(dev_priv);
-				DRM_DEBUG
-				    ("%s: idle engine, freed all buffers.\n",
-				     __FUNCTION__);
-				if (list_empty(&dev_priv->free_list)) {
-					DRM_ERROR
-					    ("Freelist empty with idle engine\n");
-					return NULL;
-				}
+			ret = mach64_do_reclaim_completed(dev_priv);
+			if (ret == 0)
 				goto _freelist_entry_found;
-			}
-			/* Look for a completed buffer and bail out of the loop
-			 * as soon as we find one -- don't waste time trying
-			 * to free extra bufs here, leave that to do_release_used_buffers
-			 */
-			list_for_each_safe(ptr, tmp, &dev_priv->pending) {
-				entry =
-				    list_entry(ptr, drm_mach64_freelist_t,
-					       list);
-				ofs = entry->ring_ofs;
-				if (entry->discard &&
-				    ((head < tail
-				      && (ofs < head || ofs >= tail))
-				     || (head > tail
-					 && (ofs < head && ofs >= tail)))) {
-#if MACH64_EXTRA_CHECKING
-					int i;
-
-					for (i = head; i != tail;
-					     i = (i + 4) & ring->tail_mask) {
-						u32 o1 =
-						    le32_to_cpu(((u32 *) ring->
-								 start)[i + 1]);
-						u32 o2 = GETBUFADDR(entry->buf);
-
-						if (o1 == o2) {
-							DRM_ERROR
-							    ("Attempting to free used buffer: "
-							     "i=%d  buf=0x%08x\n",
-							     i, o1);
-							mach64_dump_ring_info
-							    (dev_priv);
-							return NULL;
-						}
-					}
-#endif
-					/* found a processed buffer */
-					entry->buf->pending = 0;
-					list_del(ptr);
-					entry->buf->used = 0;
-					list_add_tail(ptr,
-						      &dev_priv->placeholders);
-					DRM_DEBUG
-					    ("%s: freed processed buffer (head=%d tail=%d "
-					     "buf ring ofs=%d).\n",
-					     __FUNCTION__, head, tail, ofs);
-					return entry->buf;
-				}
-			}
+			if (ret < 0)
+				return NULL;
+
 			DRM_UDELAY(1);
 		}
 		mach64_dump_ring_info(dev_priv);
@@ -1446,6 +1668,32 @@ drm_buf_t *mach64_freelist_get(drm_mach64_private_t * dev_priv)
 	return entry->buf;
 }
 
+int mach64_freelist_put(drm_mach64_private_t *dev_priv, struct drm_buf *copy_buf)
+{
+	struct list_head *ptr;
+	drm_mach64_freelist_t *entry;
+
+#if MACH64_EXTRA_CHECKING
+	list_for_each(ptr, &dev_priv->pending) {
+		entry = list_entry(ptr, drm_mach64_freelist_t, list);
+		if (copy_buf == entry->buf) {
+			DRM_ERROR("Trying to release a pending buf\n");
+			return -EFAULT;
+		}
+	}
+#endif
+	ptr = dev_priv->placeholders.next;
+	entry = list_entry(ptr, drm_mach64_freelist_t, list);
+	copy_buf->pending = 0;
+	copy_buf->used = 0;
+	entry->buf = copy_buf;
+	entry->discard = 1;
+	list_del(ptr);
+	list_add_tail(ptr, &dev_priv->free_list);
+
+	return 0;
+}
+
 /*@}*/
 
 
@@ -1453,76 +1701,73 @@ drm_buf_t *mach64_freelist_get(drm_mach64_private_t * dev_priv)
 /** \name DMA buffer request and submission IOCTL handler */
 /*@{*/
 
-static int mach64_dma_get_buffers(DRMFILE filp, drm_device_t * dev,
-				  drm_dma_t * d)
+static int mach64_dma_get_buffers(struct drm_device *dev,
+				  struct drm_file *file_priv,
+				  struct drm_dma * d)
 {
 	int i;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 
 	for (i = d->granted_count; i < d->request_count; i++) {
 		buf = mach64_freelist_get(dev_priv);
 #if MACH64_EXTRA_CHECKING
 		if (!buf)
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 #else
 		if (!buf)
-			return DRM_ERR(EAGAIN);
+			return -EAGAIN;
 #endif
 
-		buf->filp = filp;
+		buf->file_priv = file_priv;
 
 		if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
 				     sizeof(buf->idx)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
 				     sizeof(buf->total)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 
 		d->granted_count++;
 	}
 	return 0;
 }
 
-int mach64_dma_buffers(DRM_IOCTL_ARGS)
+int mach64_dma_buffers(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
-	drm_dma_t d;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_dma *d = data;
 	int ret = 0;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(d, (drm_dma_t *) data, sizeof(d));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Please don't send us buffers.
 	 */
-	if (d.send_count != 0) {
+	if (d->send_count != 0) {
 		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d.send_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->send_count);
+		return -EINVAL;
 	}
 
 	/* We'll send you buffers.
 	 */
-	if (d.request_count < 0 || d.request_count > dma->buf_count) {
+	if (d->request_count < 0 || d->request_count > dma->buf_count) {
 		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d.request_count, dma->buf_count);
-		ret = DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->request_count, dma->buf_count);
+		ret = -EINVAL;
 	}
 
-	d.granted_count = 0;
+	d->granted_count = 0;
 
-	if (d.request_count) {
-		ret = mach64_dma_get_buffers(filp, dev, &d);
+	if (d->request_count) {
+		ret = mach64_dma_get_buffers(dev, file_priv, d);
 	}
 
-	DRM_COPY_TO_USER_IOCTL((drm_dma_t *) data, d, sizeof(d));
-
 	return ret;
 }
 
-void mach64_driver_lastclose(drm_device_t * dev)
+void mach64_driver_lastclose(struct drm_device * dev)
 {
 	mach64_do_cleanup_dma(dev);
 }
diff --git a/sys/dev/drm/mach64_drm.h b/sys/dev/drm/mach64_drm.h
index 042c91e..44ba505 100644
--- a/sys/dev/drm/mach64_drm.h
+++ b/sys/dev/drm/mach64_drm.h
@@ -133,7 +133,7 @@ typedef struct drm_mach64_sarea {
 
 	/* The current cliprects, or a subset thereof.
 	 */
-	drm_clip_rect_t boxes[MACH64_NR_SAREA_CLIPRECTS];
+	struct drm_clip_rect boxes[MACH64_NR_SAREA_CLIPRECTS];
 	unsigned int nbox;
 
 	/* Counters for client-side throttling of rendering clients.
@@ -142,7 +142,7 @@ typedef struct drm_mach64_sarea {
 
 	/* Texture memory LRU.
 	 */
-	drm_tex_region_t tex_list[MACH64_NR_TEX_HEAPS][MACH64_NR_TEX_REGIONS +
+	struct drm_tex_region tex_list[MACH64_NR_TEX_HEAPS][MACH64_NR_TEX_REGIONS +
 						       1];
 	unsigned int tex_age[MACH64_NR_TEX_HEAPS];
 	int ctx_owner;
@@ -240,7 +240,7 @@ typedef struct drm_mach64_vertex {
 } drm_mach64_vertex_t;
 
 typedef struct drm_mach64_blit {
-	int idx;
+	void *buf;
 	int pitch;
 	int offset;
 	int format;
diff --git a/sys/dev/drm/mach64_drv.c b/sys/dev/drm/mach64_drv.c
index 8ba4a21..11d2437 100644
--- a/sys/dev/drm/mach64_drv.c
+++ b/sys/dev/drm/mach64_drv.c
@@ -47,11 +47,13 @@ static drm_pci_id_list_t mach64_pciidlist[] = {
 	mach64_PCI_IDS
 };
 
-static void mach64_configure(drm_device_t *dev)
+static void mach64_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= 1; /* No dev_priv */
 	dev->driver.lastclose		= mach64_driver_lastclose;
-	dev->driver.vblank_wait		= mach64_driver_vblank_wait;
+	dev->driver.get_vblank_counter	= mach64_get_vblank_counter;
+	dev->driver.enable_vblank	= mach64_enable_vblank;
+	dev->driver.disable_vblank	= mach64_disable_vblank;
 	dev->driver.irq_preinstall	= mach64_driver_irq_preinstall;
 	dev->driver.irq_postinstall	= mach64_driver_irq_postinstall;
 	dev->driver.irq_uninstall	= mach64_driver_irq_uninstall;
@@ -86,9 +88,9 @@ mach64_probe(device_t dev)
 static int
 mach64_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	mach64_configure(dev);
 	return drm_attach(nbdev, mach64_pciidlist);
 }
@@ -105,7 +107,7 @@ static device_method_t mach64_methods[] = {
 static driver_t mach64_driver = {
 	"drm",
 	mach64_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
diff --git a/sys/dev/drm/mach64_drv.h b/sys/dev/drm/mach64_drv.h
index 24e524e..9c85454 100644
--- a/sys/dev/drm/mach64_drv.h
+++ b/sys/dev/drm/mach64_drv.h
@@ -29,7 +29,7 @@
  *    Gareth Hughes <gareth@valinux.com>
  *    Frank C. Earl <fearl@airmail.net>
  *    Leif Delgass <ldelgass@retinalburn.net>
- *    Jos�Fonseca <j_r_fonseca@yahoo.co.uk>
+ *    José Fonseca <j_r_fonseca@yahoo.co.uk>
  */
 
 #include <sys/cdefs.h>
@@ -45,9 +45,9 @@ __FBSDID("$FreeBSD$");
 
 #define DRIVER_NAME		"mach64"
 #define DRIVER_DESC		"DRM module for the ATI Rage Pro"
-#define DRIVER_DATE		"20020904"
+#define DRIVER_DATE		"20060718"
 
-#define DRIVER_MAJOR		1
+#define DRIVER_MAJOR		2
 #define DRIVER_MINOR		0
 #define DRIVER_PATCHLEVEL	0
 
@@ -58,13 +58,12 @@ __FBSDID("$FreeBSD$");
 
 typedef struct drm_mach64_freelist {
 	struct list_head list;	/* List pointers for free_list, placeholders, or pending list */
-	drm_buf_t *buf;		/* Pointer to the buffer */
+	struct drm_buf *buf;		/* Pointer to the buffer */
 	int discard;		/* This flag is set when we're done (re)using a buffer */
 	u32 ring_ofs;		/* dword offset in ring of last descriptor for this buffer */
 } drm_mach64_freelist_t;
 
 typedef struct drm_mach64_descriptor_ring {
-	drm_dma_handle_t *dmah;	/* Handle to pci dma memory */
 	void *start;		/* write pointer (cpu address) to start of descriptor ring */
 	u32 start_addr;		/* bus address of beginning of descriptor ring */
 	int size;		/* size of ring in bytes */
@@ -100,6 +99,8 @@ typedef struct drm_mach64_private {
 	unsigned int depth_bpp;
 	unsigned int depth_offset, depth_pitch;
 
+	atomic_t vbl_received;          /**< Number of vblanks received. */
+
 	u32 front_offset_pitch;
 	u32 back_offset_pitch;
 	u32 depth_offset_pitch;
@@ -112,20 +113,27 @@ typedef struct drm_mach64_private {
 	drm_local_map_t *agp_textures;
 } drm_mach64_private_t;
 
-extern drm_ioctl_desc_t mach64_ioctls[];
+extern struct drm_ioctl_desc mach64_ioctls[];
 extern int mach64_max_ioctl;
 
 				/* mach64_dma.c */
-extern int mach64_dma_init(DRM_IOCTL_ARGS);
-extern int mach64_dma_idle(DRM_IOCTL_ARGS);
-extern int mach64_dma_flush(DRM_IOCTL_ARGS);
-extern int mach64_engine_reset(DRM_IOCTL_ARGS);
-extern int mach64_dma_buffers(DRM_IOCTL_ARGS);
-extern void mach64_driver_lastclose(drm_device_t * dev);
-
-extern int mach64_init_freelist(drm_device_t * dev);
-extern void mach64_destroy_freelist(drm_device_t * dev);
-extern drm_buf_t *mach64_freelist_get(drm_mach64_private_t * dev_priv);
+extern int mach64_dma_init(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+extern int mach64_dma_idle(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+extern int mach64_dma_flush(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+extern int mach64_engine_reset(struct drm_device *dev, void *data,
+			       struct drm_file *file_priv);
+extern int mach64_dma_buffers(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+extern void mach64_driver_lastclose(struct drm_device * dev);
+
+extern int mach64_init_freelist(struct drm_device * dev);
+extern void mach64_destroy_freelist(struct drm_device * dev);
+extern struct drm_buf *mach64_freelist_get(drm_mach64_private_t * dev_priv);
+extern int mach64_freelist_put(drm_mach64_private_t * dev_priv,
+			       struct drm_buf * copy_buf);
 
 extern int mach64_do_wait_for_fifo(drm_mach64_private_t * dev_priv,
 				   int entries);
@@ -137,23 +145,34 @@ extern void mach64_dump_engine_info(drm_mach64_private_t * dev_priv);
 extern void mach64_dump_ring_info(drm_mach64_private_t * dev_priv);
 extern int mach64_do_engine_reset(drm_mach64_private_t * dev_priv);
 
+extern int mach64_add_buf_to_ring(drm_mach64_private_t *dev_priv,
+                                  drm_mach64_freelist_t *_entry);
+extern int mach64_add_hostdata_buf_to_ring(drm_mach64_private_t *dev_priv,
+                                           drm_mach64_freelist_t *_entry);
+
 extern int mach64_do_dma_idle(drm_mach64_private_t * dev_priv);
 extern int mach64_do_dma_flush(drm_mach64_private_t * dev_priv);
-extern int mach64_do_cleanup_dma(drm_device_t * dev);
+extern int mach64_do_cleanup_dma(struct drm_device * dev);
 
 				/* mach64_state.c */
-extern int mach64_dma_clear(DRM_IOCTL_ARGS);
-extern int mach64_dma_swap(DRM_IOCTL_ARGS);
-extern int mach64_dma_vertex(DRM_IOCTL_ARGS);
-extern int mach64_dma_blit(DRM_IOCTL_ARGS);
-extern int mach64_get_param(DRM_IOCTL_ARGS);
-extern int mach64_driver_vblank_wait(drm_device_t * dev,
-				     unsigned int *sequence);
-
+extern int mach64_dma_clear(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+extern int mach64_dma_swap(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+extern int mach64_dma_vertex(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
+extern int mach64_dma_blit(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+extern int mach64_get_param(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+
+extern u32 mach64_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int mach64_enable_vblank(struct drm_device *dev, int crtc);
+extern void mach64_disable_vblank(struct drm_device *dev, int crtc);
 extern irqreturn_t mach64_driver_irq_handler(DRM_IRQ_ARGS);
-extern void mach64_driver_irq_preinstall(drm_device_t * dev);
-extern void mach64_driver_irq_postinstall(drm_device_t * dev);
-extern void mach64_driver_irq_uninstall(drm_device_t * dev);
+extern void mach64_driver_irq_preinstall(struct drm_device *dev);
+extern int mach64_driver_irq_postinstall(struct drm_device *dev);
+extern void mach64_driver_irq_uninstall(struct drm_device *dev);
 
 /* ================================================================
  * Registers
@@ -163,14 +182,14 @@ extern void mach64_driver_irq_uninstall(drm_device_t * dev);
 #define MACH64_AGP_CNTL				0x014c
 #define MACH64_ALPHA_TST_CNTL			0x0550
 
-#define MACH64_DSP_CONFIG 			0x0420
-#define MACH64_DSP_ON_OFF 			0x0424
-#define MACH64_EXT_MEM_CNTL 			0x04ac
-#define MACH64_GEN_TEST_CNTL 			0x04d0
-#define MACH64_HW_DEBUG 			0x047c
-#define MACH64_MEM_ADDR_CONFIG 			0x0434
-#define MACH64_MEM_BUF_CNTL 			0x042c
-#define MACH64_MEM_CNTL 			0x04b0
+#define MACH64_DSP_CONFIG			0x0420
+#define MACH64_DSP_ON_OFF			0x0424
+#define MACH64_EXT_MEM_CNTL			0x04ac
+#define MACH64_GEN_TEST_CNTL			0x04d0
+#define MACH64_HW_DEBUG				0x047c
+#define MACH64_MEM_ADDR_CONFIG			0x0434
+#define MACH64_MEM_BUF_CNTL			0x042c
+#define MACH64_MEM_CNTL				0x04b0
 
 #define MACH64_BM_ADDR				0x0648
 #define MACH64_BM_COMMAND			0x0188
@@ -197,16 +216,16 @@ extern void mach64_driver_irq_uninstall(drm_device_t * dev);
 #define MACH64_CLR_CMP_CLR			0x0700
 #define MACH64_CLR_CMP_CNTL			0x0708
 #define MACH64_CLR_CMP_MASK			0x0704
-#define MACH64_CONFIG_CHIP_ID 			0x04e0
-#define MACH64_CONFIG_CNTL 			0x04dc
-#define MACH64_CONFIG_STAT0 			0x04e4
-#define MACH64_CONFIG_STAT1 			0x0494
-#define MACH64_CONFIG_STAT2 			0x0498
+#define MACH64_CONFIG_CHIP_ID			0x04e0
+#define MACH64_CONFIG_CNTL			0x04dc
+#define MACH64_CONFIG_STAT0			0x04e4
+#define MACH64_CONFIG_STAT1			0x0494
+#define MACH64_CONFIG_STAT2			0x0498
 #define MACH64_CONTEXT_LOAD_CNTL		0x072c
 #define MACH64_CONTEXT_MASK			0x0720
 #define MACH64_COMPOSITE_SHADOW_ID		0x0798
-#define MACH64_CRC_SIG 				0x04e8
-#define MACH64_CUSTOM_MACRO_CNTL 		0x04d4
+#define MACH64_CRC_SIG				0x04e8
+#define MACH64_CUSTOM_MACRO_CNTL		0x04d4
 
 #define MACH64_DP_BKGD_CLR			0x06c0
 #define MACH64_DP_FOG_CLR			0x06c4
@@ -350,7 +369,7 @@ extern void mach64_driver_irq_uninstall(drm_device_t * dev);
 #define MACH64_TEX_0_OFF			0x05c0
 #define MACH64_TEX_CNTL				0x0774
 #define MACH64_TEX_SIZE_PITCH			0x0770
-#define MACH64_TIMER_CONFIG 			0x0428
+#define MACH64_TIMER_CONFIG			0x0428
 
 #define MACH64_VERTEX_1_ARGB			0x0254
 #define MACH64_VERTEX_1_S			0x0240
@@ -513,95 +532,17 @@ extern void mach64_driver_irq_uninstall(drm_device_t * dev);
 #define MACH64_APERTURE_OFFSET	        0x7ff800	/* frame-buffer offset for gui-masters */
 
 /* ================================================================
- * Misc helper macros
+ * Ring operations
+ *
+ * Since the Mach64 bus master engine requires polling, these functions end
+ * up being called frequently, hence being inline.
  */
 
-static __inline__ void mach64_set_dma_eol(volatile u32 * addr)
-{
-#if defined(__i386__)
-	int nr = 31;
-
-	/* Taken from include/asm-i386/bitops.h linux header */
-	__asm__ __volatile__("lock;" "btsl %1,%0":"=m"(*addr)
-			     :"Ir"(nr));
-#elif defined(__powerpc__)
-	u32 old;
-	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
-
-	/* Taken from the include/asm-ppc/bitops.h linux header */
-	__asm__ __volatile__("\n\
-1:	lwarx	%0,0,%3 \n\
-	or	%0,%0,%2 \n\
-	stwcx.	%0,0,%3 \n\
-	bne-	1b":"=&r"(old), "=m"(*addr)
-			     :"r"(mask), "r"(addr), "m"(*addr)
-			     :"cc");
-#elif defined(__alpha__)
-	u32 temp;
-	u32 mask = MACH64_DMA_EOL;
-
-	/* Taken from the include/asm-alpha/bitops.h linux header */
-	__asm__ __volatile__("1:	ldl_l %0,%3\n"
-			     "	bis %0,%2,%0\n"
-			     "	stl_c %0,%1\n"
-			     "	beq %0,2f\n"
-			     ".subsection 2\n"
-			     "2:	br 1b\n"
-			     ".previous":"=&r"(temp), "=m"(*addr)
-			     :"Ir"(mask), "m"(*addr));
-#else
-	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
-
-	*addr |= mask;
-#endif
-}
-
-static __inline__ void mach64_clear_dma_eol(volatile u32 * addr)
-{
-#if defined(__i386__)
-	int nr = 31;
-
-	/* Taken from include/asm-i386/bitops.h linux header */
-	__asm__ __volatile__("lock;" "btrl %1,%0":"=m"(*addr)
-			     :"Ir"(nr));
-#elif defined(__powerpc__)
-	u32 old;
-	u32 mask = cpu_to_le32(MACH64_DMA_EOL);
-
-	/* Taken from the include/asm-ppc/bitops.h linux header */
-	__asm__ __volatile__("\n\
-1:	lwarx	%0,0,%3 \n\
-	andc	%0,%0,%2 \n\
-	stwcx.	%0,0,%3 \n\
-	bne-	1b":"=&r"(old), "=m"(*addr)
-			     :"r"(mask), "r"(addr), "m"(*addr)
-			     :"cc");
-#elif defined(__alpha__)
-	u32 temp;
-	u32 mask = ~MACH64_DMA_EOL;
-
-	/* Taken from the include/asm-alpha/bitops.h linux header */
-	__asm__ __volatile__("1:	ldl_l %0,%3\n"
-			     "	and %0,%2,%0\n"
-			     "	stl_c %0,%1\n"
-			     "	beq %0,2f\n"
-			     ".subsection 2\n"
-			     "2:	br 1b\n"
-			     ".previous":"=&r"(temp), "=m"(*addr)
-			     :"Ir"(mask), "m"(*addr));
-#else
-	u32 mask = cpu_to_le32(~MACH64_DMA_EOL);
-
-	*addr &= mask;
-#endif
-}
-
 static __inline__ void mach64_ring_start(drm_mach64_private_t * dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 
-	DRM_DEBUG("%s: head_addr: 0x%08x head: %d tail: %d space: %d\n",
-		  __FUNCTION__,
+	DRM_DEBUG("head_addr: 0x%08x head: %d tail: %d space: %d\n",
 		  ring->head_addr, ring->head, ring->tail, ring->space);
 
 	if (mach64_do_wait_for_idle(dev_priv) < 0) {
@@ -627,8 +568,7 @@ static __inline__ void mach64_ring_start(drm_mach64_private_t * dev_priv)
 static __inline__ void mach64_ring_resume(drm_mach64_private_t * dev_priv,
 					  drm_mach64_descriptor_ring_t * ring)
 {
-	DRM_DEBUG("%s: head_addr: 0x%08x head: %d tail: %d space: %d\n",
-		  __FUNCTION__,
+	DRM_DEBUG("head_addr: 0x%08x head: %d tail: %d space: %d\n",
 		  ring->head_addr, ring->head, ring->tail, ring->space);
 
 	/* reset descriptor table ring head */
@@ -647,8 +587,7 @@ static __inline__ void mach64_ring_resume(drm_mach64_private_t * dev_priv,
 		MACH64_WRITE(MACH64_DST_HEIGHT_WIDTH, 0);
 		if (dev_priv->driver_mode == MACH64_MODE_DMA_SYNC) {
 			if ((mach64_do_wait_for_idle(dev_priv)) < 0) {
-				DRM_ERROR("%s: idle failed, resetting engine\n",
-					  __FUNCTION__);
+				DRM_ERROR("idle failed, resetting engine\n");
 				mach64_dump_engine_info(dev_priv);
 				mach64_do_engine_reset(dev_priv);
 				return;
@@ -658,11 +597,22 @@ static __inline__ void mach64_ring_resume(drm_mach64_private_t * dev_priv,
 	}
 }
 
+/**
+ * Poll the ring head and make sure the bus master is alive.
+ * 
+ * Mach64's bus master engine will stop if there are no more entries to process.
+ * This function polls the engine for the last processed entry and calls 
+ * mach64_ring_resume if there is an unprocessed entry.
+ * 
+ * Note also that, since we update the ring tail while the bus master engine is 
+ * in operation, it is possible that the last tail update was too late to be 
+ * processed, and the bus master engine stops at the previous tail position. 
+ * Therefore it is important to call this function frequently. 
+ */
 static __inline__ void mach64_ring_tick(drm_mach64_private_t * dev_priv,
 					drm_mach64_descriptor_ring_t * ring)
 {
-	DRM_DEBUG("%s: head_addr: 0x%08x head: %d tail: %d space: %d\n",
-		  __FUNCTION__,
+	DRM_DEBUG("head_addr: 0x%08x head: %d tail: %d space: %d\n",
 		  ring->head_addr, ring->head, ring->tail, ring->space);
 
 	if (!dev_priv->ring_running) {
@@ -709,8 +659,7 @@ static __inline__ void mach64_ring_tick(drm_mach64_private_t * dev_priv,
 
 static __inline__ void mach64_ring_stop(drm_mach64_private_t * dev_priv)
 {
-	DRM_DEBUG("%s: head_addr: 0x%08x head: %d tail: %d space: %d\n",
-		  __FUNCTION__,
+	DRM_DEBUG("head_addr: 0x%08x head: %d tail: %d space: %d\n",
 		  dev_priv->ring.head_addr, dev_priv->ring.head,
 		  dev_priv->ring.tail, dev_priv->ring.space);
 
@@ -731,7 +680,7 @@ mach64_update_ring_snapshot(drm_mach64_private_t * dev_priv)
 {
 	drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	mach64_ring_tick(dev_priv, ring);
 
@@ -742,70 +691,22 @@ mach64_update_ring_snapshot(drm_mach64_private_t * dev_priv)
 }
 
 /* ================================================================
- * DMA descriptor ring macros
- */
-
-#define RING_LOCALS									\
-	int _ring_tail, _ring_write; unsigned int _ring_mask; volatile u32 *_ring
-
-#define RING_WRITE_OFS  _ring_write
-
-#define BEGIN_RING( n ) 								\
-do {											\
-	if ( MACH64_VERBOSE ) {								\
-		DRM_INFO( "BEGIN_RING( %d ) in %s\n",					\
-			   (n), __FUNCTION__ );						\
-	}										\
-	if ( dev_priv->ring.space <= (n) * sizeof(u32) ) {				\
-		int ret;								\
-		if ((ret=mach64_wait_ring( dev_priv, (n) * sizeof(u32))) < 0 ) {	\
-			DRM_ERROR( "wait_ring failed, resetting engine\n");		\
-			mach64_dump_engine_info( dev_priv );				\
-			mach64_do_engine_reset( dev_priv );				\
-			return ret;							\
-		}									\
-	}										\
-	dev_priv->ring.space -= (n) * sizeof(u32);					\
-	_ring = (u32 *) dev_priv->ring.start;						\
-	_ring_tail = _ring_write = dev_priv->ring.tail;					\
-	_ring_mask = dev_priv->ring.tail_mask;						\
-} while (0)
-
-#define OUT_RING( x )						\
-do {								\
-	if ( MACH64_VERBOSE ) {					\
-		DRM_INFO( "   OUT_RING( 0x%08x ) at 0x%x\n",	\
-			   (unsigned int)(x), _ring_write );	\
-	}							\
-	_ring[_ring_write++] = cpu_to_le32( x );		\
-	_ring_write &= _ring_mask;				\
-} while (0)
-
-#define ADVANCE_RING() 							\
-do {									\
-	if ( MACH64_VERBOSE ) {						\
-		DRM_INFO( "ADVANCE_RING() wr=0x%06x tail=0x%06x\n",	\
-			  _ring_write, _ring_tail );			\
-	}								\
-	DRM_MEMORYBARRIER();						\
-	mach64_clear_dma_eol( &_ring[(_ring_tail - 2) & _ring_mask] );	\
-	DRM_MEMORYBARRIER();						\
-	dev_priv->ring.tail = _ring_write;				\
-	mach64_ring_tick( dev_priv, &(dev_priv)->ring );		\
-} while (0)
-
-/* ================================================================
  * DMA macros
+ * 
+ * Mach64's ring buffer doesn't take register writes directly. These 
+ * have to be written indirectly in DMA buffers. These macros simplify 
+ * the task of setting up a buffer, writing commands to it, and 
+ * queuing the buffer in the ring. 
  */
 
 #define DMALOCALS				\
 	drm_mach64_freelist_t *_entry = NULL;	\
-	drm_buf_t *_buf = NULL; 		\
+	struct drm_buf *_buf = NULL;		\
 	u32 *_buf_wptr; int _outcount
 
 #define GETBUFPTR( __buf )						\
-((dev_priv->is_pci) ? 							\
-	((u32 *)(__buf)->address) : 					\
+((dev_priv->is_pci) ?							\
+	((u32 *)(__buf)->address) :					\
 	((u32 *)((char *)dev_priv->dev_buffers->handle + (__buf)->offset)))
 
 #define GETBUFADDR( __buf ) ((u32)(__buf)->bus_address)
@@ -815,20 +716,20 @@ do {									\
 static __inline__ int mach64_find_pending_buf_entry(drm_mach64_private_t *
 						    dev_priv,
 						    drm_mach64_freelist_t **
-						    entry, drm_buf_t * buf)
+						    entry, struct drm_buf * buf)
 {
 	struct list_head *ptr;
 #if MACH64_EXTRA_CHECKING
 	if (list_empty(&dev_priv->pending)) {
-		DRM_ERROR("Empty pending list in %s\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("Empty pending list in \n");
+		return -EINVAL;
 	}
 #endif
 	ptr = dev_priv->pending.prev;
 	*entry = list_entry(ptr, drm_mach64_freelist_t, list);
 	while ((*entry)->buf != buf) {
 		if (ptr == &dev_priv->pending) {
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		}
 		ptr = ptr->prev;
 		*entry = list_entry(ptr, drm_mach64_freelist_t, list);
@@ -836,7 +737,7 @@ static __inline__ int mach64_find_pending_buf_entry(drm_mach64_private_t *
 	return 0;
 }
 
-#define DMASETPTR( _p ) 			\
+#define DMASETPTR( _p )				\
 do {						\
 	_buf = (_p);				\
 	_outcount = 0;				\
@@ -844,24 +745,21 @@ do {						\
 } while(0)
 
 /* FIXME: use a private set of smaller buffers for state emits, clears, and swaps? */
-#define DMAGETPTR( filp, dev_priv, n )					\
+#define DMAGETPTR( file_priv, dev_priv, n )				\
 do {									\
 	if ( MACH64_VERBOSE ) {						\
-		DRM_INFO( "DMAGETPTR( %d ) in %s\n",			\
-			  n, __FUNCTION__ );				\
+		DRM_INFO( "DMAGETPTR( %d )\n", (n) );			\
 	}								\
 	_buf = mach64_freelist_get( dev_priv );				\
 	if (_buf == NULL) {						\
-		DRM_ERROR("%s: couldn't get buffer in DMAGETPTR\n",	\
-			   __FUNCTION__ );				\
-		return DRM_ERR(EAGAIN);					\
+		DRM_ERROR("couldn't get buffer in DMAGETPTR\n");	\
+		return -EAGAIN;					\
 	}								\
 	if (_buf->pending) {						\
-	        DRM_ERROR("%s: pending buf in DMAGETPTR\n",		\
-			   __FUNCTION__ );				\
-		return DRM_ERR(EFAULT);					\
+	        DRM_ERROR("pending buf in DMAGETPTR\n");		\
+		return -EFAULT;					\
 	}								\
-	_buf->filp = filp;						\
+	_buf->file_priv = file_priv;					\
 	_outcount = 0;							\
 									\
         _buf_wptr = GETBUFPTR( _buf );					\
@@ -878,173 +776,87 @@ do {								\
 	_buf->used += 8;					\
 } while (0)
 
-#define DMAADVANCE( dev_priv, _discard )						     \
-do {											     \
-	struct list_head *ptr;								     \
-	RING_LOCALS;									     \
-											     \
-	if ( MACH64_VERBOSE ) {								     \
-		DRM_INFO( "DMAADVANCE() in %s\n", __FUNCTION__ );			     \
-	}										     \
-											     \
-	if (_buf->used <= 0) {								     \
-		DRM_ERROR( "DMAADVANCE() in %s: sending empty buf %d\n",		     \
-				   __FUNCTION__, _buf->idx );				     \
-		return DRM_ERR(EFAULT);							     \
-	}										     \
-	if (_buf->pending) {								     \
-                /* This is a resued buffer, so we need to find it in the pending list */     \
-		int ret;								     \
-		if ( (ret=mach64_find_pending_buf_entry(dev_priv, &_entry, _buf)) ) {	     \
-			DRM_ERROR( "DMAADVANCE() in %s: couldn't find pending buf %d\n",     \
-				   __FUNCTION__, _buf->idx );				     \
-			return ret;							     \
-		}									     \
-		if (_entry->discard) {							     \
-			DRM_ERROR( "DMAADVANCE() in %s: sending discarded pending buf %d\n", \
-				   __FUNCTION__, _buf->idx );				     \
-			return DRM_ERR(EFAULT);						     \
-		}									     \
-     	} else {									     \
-		if (list_empty(&dev_priv->placeholders)) {				     \
-			DRM_ERROR( "DMAADVANCE() in %s: empty placeholder list\n",	     \
-			   	__FUNCTION__ );						     \
-			return DRM_ERR(EFAULT);						     \
-		}									     \
-		ptr = dev_priv->placeholders.next;					     \
-		list_del(ptr);								     \
-		_entry = list_entry(ptr, drm_mach64_freelist_t, list);			     \
-		_buf->pending = 1;							     \
-		_entry->buf = _buf;							     \
-		list_add_tail(ptr, &dev_priv->pending);					     \
-	}										     \
-	_entry->discard = (_discard);							     \
-	ADD_BUF_TO_RING( dev_priv );							     \
-} while (0)
-
-#define DMADISCARDBUF()									\
-do {											\
-	if (_entry == NULL) {								\
-		int ret;								\
-		if ( (ret=mach64_find_pending_buf_entry(dev_priv, &_entry, _buf)) ) {	\
-			DRM_ERROR( "%s: couldn't find pending buf %d\n",		\
-				   __FUNCTION__, _buf->idx );				\
-			return ret;							\
-		}									\
-	}										\
-	_entry->discard = 1;								\
-} while(0)
-
-#define ADD_BUF_TO_RING( dev_priv )							\
-do {											\
-	int bytes, pages, remainder;							\
-	u32 address, page;								\
-	int i;										\
-											\
-	bytes = _buf->used;								\
-	address = GETBUFADDR( _buf );							\
-											\
-	pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;		\
-											\
-	BEGIN_RING( pages * 4 );							\
-											\
-	for ( i = 0 ; i < pages-1 ; i++ ) {						\
-		page = address + i * MACH64_DMA_CHUNKSIZE;				\
-		OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );			\
-		OUT_RING( page );							\
-		OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );		\
-		OUT_RING( 0 );								\
-	}										\
-											\
-	/* generate the final descriptor for any remaining commands in this buffer */	\
-	page = address + i * MACH64_DMA_CHUNKSIZE;					\
-	remainder = bytes - i * MACH64_DMA_CHUNKSIZE;					\
-											\
-	/* Save dword offset of last descriptor for this buffer.			\
-	 * This is needed to check for completion of the buffer in freelist_get		\
-	 */										\
-	_entry->ring_ofs = RING_WRITE_OFS;						\
-											\
-	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );				\
-	OUT_RING( page );								\
-	OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );		\
-	OUT_RING( 0 );									\
-											\
-	ADVANCE_RING();									\
-} while(0)
-
-#define DMAADVANCEHOSTDATA( dev_priv )							\
-do {											\
-	struct list_head *ptr;								\
-	RING_LOCALS;									\
-											\
-	if ( MACH64_VERBOSE ) {								\
-		DRM_INFO( "DMAADVANCEHOSTDATA() in %s\n", __FUNCTION__ );		\
-	}										\
-											\
-	if (_buf->used <= 0) {								\
-		DRM_ERROR( "DMAADVANCEHOSTDATA() in %s: sending empty buf %d\n",	\
-				   __FUNCTION__, _buf->idx );				\
-		return DRM_ERR(EFAULT);							\
-	}										\
-	if (list_empty(&dev_priv->placeholders)) {					\
-		DRM_ERROR( "%s: empty placeholder list in DMAADVANCEHOSTDATA()\n",	\
-			   __FUNCTION__ );						\
-		return DRM_ERR(EFAULT);							\
-	}										\
-											\
-        ptr = dev_priv->placeholders.next;						\
-	list_del(ptr);									\
-	_entry = list_entry(ptr, drm_mach64_freelist_t, list);				\
-	_entry->buf = _buf;								\
-	_entry->buf->pending = 1;							\
-	list_add_tail(ptr, &dev_priv->pending);						\
-	_entry->discard = 1;								\
-	ADD_HOSTDATA_BUF_TO_RING( dev_priv );						\
-} while (0)
-
-#define ADD_HOSTDATA_BUF_TO_RING( dev_priv )						 \
-do {											 \
-	int bytes, pages, remainder;							 \
-	u32 address, page;								 \
-	int i;										 \
-											 \
-	bytes = _buf->used - MACH64_HOSTDATA_BLIT_OFFSET;				 \
-	pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;		 \
-	address = GETBUFADDR( _buf );							 \
-											 \
-	BEGIN_RING( 4 + pages * 4 );							 \
-											 \
-	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );				 \
-	OUT_RING( address );								 \
-	OUT_RING( MACH64_HOSTDATA_BLIT_OFFSET | MACH64_DMA_HOLD_OFFSET );		 \
-	OUT_RING( 0 );									 \
-											 \
-	address += MACH64_HOSTDATA_BLIT_OFFSET;						 \
-											 \
-	for ( i = 0 ; i < pages-1 ; i++ ) {						 \
-		page = address + i * MACH64_DMA_CHUNKSIZE;				 \
-		OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );		 \
-		OUT_RING( page );							 \
-		OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );		 \
-		OUT_RING( 0 );								 \
-	}										 \
-											 \
-	/* generate the final descriptor for any remaining commands in this buffer */	 \
-	page = address + i * MACH64_DMA_CHUNKSIZE;					 \
-	remainder = bytes - i * MACH64_DMA_CHUNKSIZE;					 \
-											 \
-	/* Save dword offset of last descriptor for this buffer.			 \
-	 * This is needed to check for completion of the buffer in freelist_get		 \
-	 */										 \
-	_entry->ring_ofs = RING_WRITE_OFS;						 \
-											 \
-	OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );			 \
-	OUT_RING( page );								 \
-	OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );		 \
-	OUT_RING( 0 );									 \
-											 \
-	ADVANCE_RING();									 \
-} while(0)
+#define DMAADVANCE( dev_priv, _discard )				\
+	do {								\
+		struct list_head *ptr;					\
+		int ret;						\
+									\
+		if ( MACH64_VERBOSE ) {					\
+			DRM_INFO( "DMAADVANCE() in \n" );		\
+		}							\
+									\
+		if (_buf->used <= 0) {					\
+			DRM_ERROR( "DMAADVANCE(): sending empty buf %d\n", \
+				   _buf->idx );				\
+			return -EFAULT;					\
+		}							\
+		if (_buf->pending) {					\
+			/* This is a resued buffer, so we need to find it in the pending list */ \
+			if ((ret = mach64_find_pending_buf_entry(dev_priv, &_entry, _buf))) { \
+				DRM_ERROR( "DMAADVANCE(): couldn't find pending buf %d\n", _buf->idx );	\
+				return ret;				\
+			}						\
+			if (_entry->discard) {				\
+				DRM_ERROR( "DMAADVANCE(): sending discarded pending buf %d\n", _buf->idx ); \
+				return -EFAULT;				\
+			}						\
+		} else {						\
+			if (list_empty(&dev_priv->placeholders)) {	\
+				DRM_ERROR( "DMAADVANCE(): empty placeholder list\n"); \
+				return -EFAULT;				\
+			}						\
+			ptr = dev_priv->placeholders.next;		\
+			list_del(ptr);					\
+			_entry = list_entry(ptr, drm_mach64_freelist_t, list); \
+			_buf->pending = 1;				\
+			_entry->buf = _buf;				\
+			list_add_tail(ptr, &dev_priv->pending);		\
+		}							\
+		_entry->discard = (_discard);				\
+		if ((ret = mach64_add_buf_to_ring( dev_priv, _entry ))) \
+			return ret;					\
+	} while (0)
+
+#define DMADISCARDBUF()							\
+	do {								\
+		if (_entry == NULL) {					\
+			int ret;					\
+			if ((ret = mach64_find_pending_buf_entry(dev_priv, &_entry, _buf))) { \
+				DRM_ERROR( "couldn't find pending buf %d\n", \
+					   _buf->idx );			\
+				return ret;				\
+			}						\
+		}							\
+		_entry->discard = 1;					\
+	} while(0)
+
+#define DMAADVANCEHOSTDATA( dev_priv )					\
+	do {								\
+		struct list_head *ptr;					\
+		int ret;						\
+									\
+		if ( MACH64_VERBOSE ) {					\
+			DRM_INFO( "DMAADVANCEHOSTDATA() in \n" );	\
+		}							\
+									\
+		if (_buf->used <= 0) {					\
+			DRM_ERROR( "DMAADVANCEHOSTDATA(): sending empty buf %d\n", _buf->idx );	\
+			return -EFAULT;					\
+		}							\
+		if (list_empty(&dev_priv->placeholders)) {		\
+			DRM_ERROR( "empty placeholder list in DMAADVANCEHOSTDATA()\n" ); \
+			return -EFAULT;					\
+		}							\
+									\
+		ptr = dev_priv->placeholders.next;			\
+		list_del(ptr);						\
+		_entry = list_entry(ptr, drm_mach64_freelist_t, list);	\
+		_entry->buf = _buf;					\
+		_entry->buf->pending = 1;				\
+		list_add_tail(ptr, &dev_priv->pending);			\
+		_entry->discard = 1;					\
+		if ((ret = mach64_add_hostdata_buf_to_ring( dev_priv, _entry ))) \
+			return ret;					\
+	} while (0)
 
 #endif				/* __MACH64_DRV_H__ */
diff --git a/sys/dev/drm/mach64_irq.c b/sys/dev/drm/mach64_irq.c
index c6563ce..d433724 100644
--- a/sys/dev/drm/mach64_irq.c
+++ b/sys/dev/drm/mach64_irq.c
@@ -45,9 +45,8 @@ __FBSDID("$FreeBSD$");
 
 irqreturn_t mach64_driver_irq_handler(DRM_IRQ_ARGS)
 {
-	drm_device_t *dev = (drm_device_t *) arg;
-	drm_mach64_private_t *dev_priv =
-	    (drm_mach64_private_t *) dev->dev_private;
+	struct drm_device *dev = arg;
+	drm_mach64_private_t *dev_priv = dev->dev_private;
 	int status;
 
 	status = MACH64_READ(MACH64_CRTC_INT_CNTL);
@@ -65,74 +64,98 @@ irqreturn_t mach64_driver_irq_handler(DRM_IRQ_ARGS)
 			     (status & ~MACH64_CRTC_INT_ACKS)
 			     | MACH64_CRTC_VBLANK_INT);
 
-		atomic_inc(&dev->vbl_received);
-		DRM_WAKEUP(&dev->vbl_queue);
-		drm_vbl_send_signals(dev);
+		atomic_inc(&dev_priv->vbl_received);
+		drm_handle_vblank(dev, 0);
 		return IRQ_HANDLED;
 	}
 	return IRQ_NONE;
 }
 
-int mach64_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
+u32 mach64_get_vblank_counter(struct drm_device * dev, int crtc)
 {
-	unsigned int cur_vblank;
-	int ret = 0;
+	const drm_mach64_private_t *const dev_priv = dev->dev_private;
 
-	/* Assume that the user has missed the current sequence number
-	 * by about a day rather than she wants to wait for years
-	 * using vertical blanks...
-	 */
-	DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-		    (((cur_vblank = atomic_read(&dev->vbl_received))
-		      - *sequence) <= (1 << 23)));
+	if (crtc != 0)
+		return 0;
+
+	return atomic_read(&dev_priv->vbl_received);
+}
 
-	*sequence = cur_vblank;
+int mach64_enable_vblank(struct drm_device * dev, int crtc)
+{
+	drm_mach64_private_t *dev_priv = dev->dev_private;
+	u32 status = MACH64_READ(MACH64_CRTC_INT_CNTL);
 
-	return ret;
+	if (crtc != 0) {
+		DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+			  crtc);
+		return -EINVAL;
+	}
+
+	DRM_DEBUG("before enable vblank CRTC_INT_CTNL: 0x%08x\n", status);
+
+	/* Turn on VBLANK interrupt */
+	MACH64_WRITE(MACH64_CRTC_INT_CNTL, MACH64_READ(MACH64_CRTC_INT_CNTL)
+		     | MACH64_CRTC_VBLANK_INT_EN);
+
+	return 0;
 }
 
-/* drm_dma.h hooks
-*/
-void mach64_driver_irq_preinstall(drm_device_t * dev)
+void mach64_disable_vblank(struct drm_device * dev, int crtc)
 {
-	drm_mach64_private_t *dev_priv =
-	    (drm_mach64_private_t *) dev->dev_private;
+	if (crtc != 0) {
+		DRM_ERROR("tried to disable vblank on non-existent crtc %d\n",
+			  crtc);
+		return;
+	}
+
+	/*
+	 * FIXME: implement proper interrupt disable by using the vblank
+	 * counter register (if available).
+	 */
+}
 
+static void mach64_disable_vblank_local(struct drm_device * dev, int crtc)
+{
+	drm_mach64_private_t *dev_priv = dev->dev_private;
 	u32 status = MACH64_READ(MACH64_CRTC_INT_CNTL);
 
-	DRM_DEBUG("before install CRTC_INT_CTNL: 0x%08x\n", status);
+	if (crtc != 0) {
+		DRM_ERROR("tried to disable vblank on non-existent crtc %d\n",
+			  crtc);
+		return;
+	}
+
+	DRM_DEBUG("before disable vblank CRTC_INT_CTNL: 0x%08x\n", status);
 
 	/* Disable and clear VBLANK interrupt */
 	MACH64_WRITE(MACH64_CRTC_INT_CNTL, (status & ~MACH64_CRTC_VBLANK_INT_EN)
 		     | MACH64_CRTC_VBLANK_INT);
 }
 
-void mach64_driver_irq_postinstall(drm_device_t * dev)
+void mach64_driver_irq_preinstall(struct drm_device * dev)
 {
-	drm_mach64_private_t *dev_priv =
-	    (drm_mach64_private_t *) dev->dev_private;
+	drm_mach64_private_t *dev_priv = dev->dev_private;
 
-	/* Turn on VBLANK interrupt */
-	MACH64_WRITE(MACH64_CRTC_INT_CNTL, MACH64_READ(MACH64_CRTC_INT_CNTL)
-		     | MACH64_CRTC_VBLANK_INT_EN);
+	u32 status = MACH64_READ(MACH64_CRTC_INT_CNTL);
 
-	DRM_DEBUG("after install CRTC_INT_CTNL: 0x%08x\n",
-		  MACH64_READ(MACH64_CRTC_INT_CNTL));
+	DRM_DEBUG("before install CRTC_INT_CTNL: 0x%08x\n", status);
+
+	mach64_disable_vblank_local(dev, 0);
+}
 
+int mach64_driver_irq_postinstall(struct drm_device * dev)
+{
+	return drm_vblank_init(dev, 1);
 }
 
-void mach64_driver_irq_uninstall(drm_device_t * dev)
+void mach64_driver_irq_uninstall(struct drm_device * dev)
 {
-	drm_mach64_private_t *dev_priv =
-	    (drm_mach64_private_t *) dev->dev_private;
+	drm_mach64_private_t *dev_priv = dev->dev_private;
 	if (!dev_priv)
 		return;
 
-	/* Disable and clear VBLANK interrupt */
-	MACH64_WRITE(MACH64_CRTC_INT_CNTL,
-		     (MACH64_READ(MACH64_CRTC_INT_CNTL) &
-		      ~MACH64_CRTC_VBLANK_INT_EN)
-		     | MACH64_CRTC_VBLANK_INT);
+	mach64_disable_vblank_local(dev, 0);
 
 	DRM_DEBUG("after uninstall CRTC_INT_CTNL: 0x%08x\n",
 		  MACH64_READ(MACH64_CRTC_INT_CNTL));
diff --git a/sys/dev/drm/mach64_state.c b/sys/dev/drm/mach64_state.c
index 9d5ab63..d59d55e 100644
--- a/sys/dev/drm/mach64_state.c
+++ b/sys/dev/drm/mach64_state.c
@@ -27,7 +27,7 @@
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
  *    Leif Delgass <ldelgass@retinalburn.net>
- *    Jos�Fonseca <j_r_fonseca@yahoo.co.uk>
+ *    José Fonseca <j_r_fonseca@yahoo.co.uk>
  */
 
 #include <sys/cdefs.h>
@@ -43,16 +43,16 @@ __FBSDID("$FreeBSD$");
  * 1.0 - Initial mach64 DRM
  *
  */
-drm_ioctl_desc_t mach64_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_MACH64_INIT)] = {mach64_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_MACH64_CLEAR)] = {mach64_dma_clear, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_SWAP)] = {mach64_dma_swap, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_IDLE)] = {mach64_dma_idle, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_RESET)] = {mach64_engine_reset, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_VERTEX)] = {mach64_dma_vertex, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_BLIT)] = {mach64_dma_blit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_FLUSH)] = {mach64_dma_flush, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MACH64_GETPARAM)] = {mach64_get_param, DRM_AUTH},
+struct drm_ioctl_desc mach64_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_MACH64_INIT, mach64_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_MACH64_CLEAR, mach64_dma_clear, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_SWAP, mach64_dma_swap, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_IDLE, mach64_dma_idle, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_RESET, mach64_engine_reset, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_VERTEX, mach64_dma_vertex, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_BLIT, mach64_dma_blit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_FLUSH, mach64_dma_flush, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MACH64_GETPARAM, mach64_get_param, DRM_AUTH),
 };
 
 int mach64_max_ioctl = DRM_ARRAY_SIZE(mach64_ioctls);
@@ -88,16 +88,17 @@ static void mach64_print_dirty(const char *msg, unsigned int flags)
 /* This function returns 0 on success, 1 for no intersection, and
  * negative for an error
  */
-static int mach64_emit_cliprect(DRMFILE filp, drm_mach64_private_t * dev_priv,
-				drm_clip_rect_t * box)
+static int mach64_emit_cliprect(struct drm_file *file_priv,
+				drm_mach64_private_t * dev_priv,
+				struct drm_clip_rect * box)
 {
 	u32 sc_left_right, sc_top_bottom;
-	drm_clip_rect_t scissor;
+	struct drm_clip_rect scissor;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mach64_context_regs_t *regs = &sarea_priv->context_state;
 	DMALOCALS;
 
-	DRM_DEBUG("%s: box=%p\n", __FUNCTION__, box);
+	DRM_DEBUG("box=%p\n", box);
 
 	/* Get GL scissor */
 	/* FIXME: store scissor in SAREA as a cliprect instead of in
@@ -123,7 +124,7 @@ static int mach64_emit_cliprect(DRMFILE filp, drm_mach64_private_t * dev_priv,
 	if (scissor.y1 >= scissor.y2)
 		return 1;
 
-	DMAGETPTR(filp, dev_priv, 2);	/* returns on failure to get buffer */
+	DMAGETPTR(file_priv, dev_priv, 2);	/* returns on failure to get buffer */
 
 	sc_left_right = ((scissor.x1 << 0) | (scissor.x2 << 16));
 	sc_top_bottom = ((scissor.y1 << 0) | (scissor.y2 << 16));
@@ -136,7 +137,7 @@ static int mach64_emit_cliprect(DRMFILE filp, drm_mach64_private_t * dev_priv,
 	return 0;
 }
 
-static __inline__ int mach64_emit_state(DRMFILE filp,
+static __inline__ int mach64_emit_state(struct drm_file *file_priv,
 					drm_mach64_private_t * dev_priv)
 {
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
@@ -148,10 +149,10 @@ static __inline__ int mach64_emit_state(DRMFILE filp,
 	if (MACH64_VERBOSE) {
 		mach64_print_dirty(__FUNCTION__, dirty);
 	} else {
-		DRM_DEBUG("%s: dirty=0x%08x\n", __FUNCTION__, dirty);
+		DRM_DEBUG("dirty=0x%08x\n", dirty);
 	}
 
-	DMAGETPTR(filp, dev_priv, 17);	/* returns on failure to get buffer */
+	DMAGETPTR(file_priv, dev_priv, 17);	/* returns on failure to get buffer */
 
 	if (dirty & MACH64_UPLOAD_MISC) {
 		DMAOUTREG(MACH64_DP_MIX, regs->dp_mix);
@@ -215,7 +216,8 @@ static __inline__ int mach64_emit_state(DRMFILE filp,
  * DMA command dispatch functions
  */
 
-static int mach64_dma_dispatch_clear(DRMFILE filp, drm_device_t * dev,
+static int mach64_dma_dispatch_clear(struct drm_device * dev,
+				     struct drm_file *file_priv,
 				     unsigned int flags,
 				     int cx, int cy, int cw, int ch,
 				     unsigned int clear_color,
@@ -225,12 +227,12 @@ static int mach64_dma_dispatch_clear(DRMFILE filp, drm_device_t * dev,
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mach64_context_regs_t *ctx = &sarea_priv->context_state;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	u32 fb_bpp, depth_bpp;
 	int i;
 	DMALOCALS;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	switch (dev_priv->fb_bpp) {
 	case 16:
@@ -240,7 +242,7 @@ static int mach64_dma_dispatch_clear(DRMFILE filp, drm_device_t * dev,
 		fb_bpp = MACH64_DATATYPE_ARGB8888;
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	switch (dev_priv->depth_bpp) {
 	case 16:
@@ -251,13 +253,13 @@ static int mach64_dma_dispatch_clear(DRMFILE filp, drm_device_t * dev,
 		depth_bpp = MACH64_DATATYPE_ARGB8888;
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (!nbox)
 		return 0;
 
-	DMAGETPTR(filp, dev_priv, nbox * 31);	/* returns on failure to get buffer */
+	DMAGETPTR(file_priv, dev_priv, nbox * 31);	/* returns on failure to get buffer */
 
 	for (i = 0; i < nbox; i++) {
 		int x = pbox[i].x1;
@@ -358,17 +360,18 @@ static int mach64_dma_dispatch_clear(DRMFILE filp, drm_device_t * dev,
 	return 0;
 }
 
-static int mach64_dma_dispatch_swap(DRMFILE filp, drm_device_t * dev)
+static int mach64_dma_dispatch_swap(struct drm_device * dev,
+				    struct drm_file *file_priv)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	u32 fb_bpp;
 	int i;
 	DMALOCALS;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	switch (dev_priv->fb_bpp) {
 	case 16:
@@ -383,7 +386,7 @@ static int mach64_dma_dispatch_swap(DRMFILE filp, drm_device_t * dev)
 	if (!nbox)
 		return 0;
 
-	DMAGETPTR(filp, dev_priv, 13 + nbox * 4);	/* returns on failure to get buffer */
+	DMAGETPTR(file_priv, dev_priv, 13 + nbox * 4);	/* returns on failure to get buffer */
 
 	DMAOUTREG(MACH64_Z_CNTL, 0);
 	DMAOUTREG(MACH64_SCALE_3D_CNTL, 0);
@@ -445,7 +448,7 @@ static int mach64_do_get_frames_queued(drm_mach64_private_t * dev_priv)
 	int i, start;
 	u32 head, tail, ofs;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	if (sarea_priv->frames_queued == 0)
 		return 0;
@@ -483,20 +486,20 @@ static int mach64_do_get_frames_queued(drm_mach64_private_t * dev_priv)
 /* Copy and verify a client submited buffer.
  * FIXME: Make an assembly optimized version
  */
-static __inline__ int copy_and_verify_from_user(u32 *to,
-						const u32 __user *ufrom,
-						unsigned long bytes)
+static __inline__ int copy_from_user_vertex(u32 *to,
+					    const u32 __user *ufrom,
+					    unsigned long bytes)
 {
 	unsigned long n = bytes;	/* dwords remaining in buffer */
 	u32 *from, *orig_from;
 
 	from = drm_alloc(bytes, DRM_MEM_DRIVER);
 	if (from == NULL)
-		return ENOMEM;
+		return -ENOMEM;
 
 	if (DRM_COPY_FROM_USER(from, ufrom, bytes)) {
 		drm_free(from, bytes, DRM_MEM_DRIVER);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 	orig_from = from; /* we'll be modifying the "from" ptr, so save it */
 
@@ -525,17 +528,16 @@ static __inline__ int copy_and_verify_from_user(u32 *to,
 				from += count;
 				to += count;
 			} else {
-				DRM_ERROR("%s: Got bad command: 0x%04x\n",
-					  __FUNCTION__, reg);
+				DRM_ERROR("Got bad command: 0x%04x\n", reg);
 				drm_free(orig_from, bytes, DRM_MEM_DRIVER);
-				return DRM_ERR(EACCES);
+				return -EACCES;
 			}
 		} else {
 			DRM_ERROR
-			    ("%s: Got bad command count(=%u) dwords remaining=%lu\n",
-			     __FUNCTION__, count, n);
+			    ("Got bad command count(=%u) dwords remaining=%lu\n",
+			     count, n);
 			drm_free(orig_from, bytes, DRM_MEM_DRIVER);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -543,119 +545,120 @@ static __inline__ int copy_and_verify_from_user(u32 *to,
 	if (n == 0)
 		return 0;
 	else {
-		DRM_ERROR("%s: Bad buf->used(=%lu)\n", __FUNCTION__, bytes);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("Bad buf->used(=%lu)\n", bytes);
+		return -EINVAL;
 	}
 }
 
-static int mach64_dma_dispatch_vertex(DRMFILE filp, drm_device_t * dev,
-				      int prim, void *buf, unsigned long used,
-				      int discard)
+static int mach64_dma_dispatch_vertex(struct drm_device * dev,
+				      struct drm_file *file_priv,
+				      drm_mach64_vertex_t * vertex)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_buf_t *copy_buf;
+	struct drm_buf *copy_buf;
+	void *buf = vertex->buf;
+	unsigned long used = vertex->used;
+	int ret = 0;
+	int i = 0;
 	int done = 0;
 	int verify_ret = 0;
 	DMALOCALS;
 
-	DRM_DEBUG("%s: buf=%p used=%lu nbox=%d\n",
-		  __FUNCTION__, buf, used, sarea_priv->nbox);
+	DRM_DEBUG("buf=%p used=%lu nbox=%d\n",
+		  buf, used, sarea_priv->nbox);
 
-	if (used) {
-		int ret = 0;
-		int i = 0;
+	if (!used)
+		goto _vertex_done;
 
-		copy_buf = mach64_freelist_get(dev_priv);
-		if (copy_buf == NULL) {
-			DRM_ERROR("%s: couldn't get buffer in DMAGETPTR\n",
-				  __FUNCTION__);
-			return DRM_ERR(EAGAIN);
-		}
+	copy_buf = mach64_freelist_get(dev_priv);
+	if (copy_buf == NULL) {
+		DRM_ERROR("couldn't get buffer\n");
+		return -EAGAIN;
+	}
 
-		if ((verify_ret =
-		     copy_and_verify_from_user(GETBUFPTR(copy_buf), buf,
-					       used)) == 0) {
+	/* Mach64's vertex data is actually register writes. To avoid security
+	 * compromises these register writes have to be verified and copied from
+	 * user space into a private DMA buffer.
+	 */
+	verify_ret = copy_from_user_vertex(GETBUFPTR(copy_buf), buf, used);
 
-			copy_buf->used = used;
+	if (verify_ret != 0) {
+		mach64_freelist_put(dev_priv, copy_buf);
+		goto _vertex_done;
+	}
 
-			DMASETPTR(copy_buf);
+	copy_buf->used = used;
 
-			if (sarea_priv->dirty & ~MACH64_UPLOAD_CLIPRECTS) {
-				ret = mach64_emit_state(filp, dev_priv);
-				if (ret < 0)
-					return ret;
-			}
+	DMASETPTR(copy_buf);
 
-			do {
-				/* Emit the next cliprect */
-				if (i < sarea_priv->nbox) {
-					ret =
-					    mach64_emit_cliprect(filp, dev_priv,
-								 &sarea_priv->
-								 boxes[i]);
-					if (ret < 0) {
-						/* failed to get buffer */
-						return ret;
-					} else if (ret != 0) {
-						/* null intersection with scissor */
-						continue;
-					}
-				}
-				if ((i >= sarea_priv->nbox - 1))
-					done = 1;
-
-				/* Add the buffer to the DMA queue */
-				DMAADVANCE(dev_priv, done);
-
-			} while (++i < sarea_priv->nbox);
+	if (sarea_priv->dirty & ~MACH64_UPLOAD_CLIPRECTS) {
+		ret = mach64_emit_state(file_priv, dev_priv);
+		if (ret < 0)
+			return ret;
+	}
+
+	do {
+		/* Emit the next cliprect */
+		if (i < sarea_priv->nbox) {
+			ret = mach64_emit_cliprect(file_priv, dev_priv,
+						   &sarea_priv->boxes[i]);
+			if (ret < 0) {
+				/* failed to get buffer */
+				return ret;
+			} else if (ret != 0) {
+				/* null intersection with scissor */
+				continue;
+			}
 		}
+		if ((i >= sarea_priv->nbox - 1))
+			done = 1;
+
+		/* Add the buffer to the DMA queue */
+		DMAADVANCE(dev_priv, done);
+
+	} while (++i < sarea_priv->nbox);
 
-		if (copy_buf->pending && !done) {
+	if (!done) {
+		if (copy_buf->pending) {
 			DMADISCARDBUF();
-		} else if (!done) {
-			/* This buffer wasn't used (no cliprects or verify failed), so place it back
-			 * on the free list
+		} else {
+			/* This buffer wasn't used (no cliprects), so place it
+			 * back on the free list
 			 */
-			struct list_head *ptr;
-			drm_mach64_freelist_t *entry;
-#if MACH64_EXTRA_CHECKING
-			list_for_each(ptr, &dev_priv->pending) {
-				entry =
-				    list_entry(ptr, drm_mach64_freelist_t,
-					       list);
-				if (copy_buf == entry->buf) {
-					DRM_ERROR
-					    ("%s: Trying to release a pending buf\n",
-					     __FUNCTION__);
-					return DRM_ERR(EFAULT);
-				}
-			}
-#endif
-			ptr = dev_priv->placeholders.next;
-			entry = list_entry(ptr, drm_mach64_freelist_t, list);
-			copy_buf->pending = 0;
-			copy_buf->used = 0;
-			entry->buf = copy_buf;
-			entry->discard = 1;
-			list_del(ptr);
-			list_add_tail(ptr, &dev_priv->free_list);
+			mach64_freelist_put(dev_priv, copy_buf);
 		}
 	}
 
+_vertex_done:
 	sarea_priv->dirty &= ~MACH64_UPLOAD_CLIPRECTS;
 	sarea_priv->nbox = 0;
 
 	return verify_ret;
 }
 
-static int mach64_dma_dispatch_blit(DRMFILE filp, drm_device_t * dev,
+static __inline__ int copy_from_user_blit(u32 *to,
+					  const u32 __user *ufrom,
+					  unsigned long bytes)
+{
+	to = (u32 *)((char *)to + MACH64_HOSTDATA_BLIT_OFFSET);
+
+	if (DRM_COPY_FROM_USER(to, ufrom, bytes)) {
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int mach64_dma_dispatch_blit(struct drm_device * dev,
+				    struct drm_file *file_priv,
 				    drm_mach64_blit_t * blit)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
 	int dword_shift, dwords;
-	drm_buf_t *buf;
+	unsigned long used;
+	struct drm_buf *copy_buf;
+	int verify_ret = 0;
 	DMALOCALS;
 
 	/* The compiler won't optimize away a division by a variable,
@@ -679,22 +682,7 @@ static int mach64_dma_dispatch_blit(DRMFILE filp, drm_device_t * dev,
 		break;
 	default:
 		DRM_ERROR("invalid blit format %d\n", blit->format);
-		return DRM_ERR(EINVAL);
-	}
-
-	/* Dispatch the blit buffer.
-	 */
-	buf = dma->buflist[blit->idx];
-
-	if (buf->filp != filp) {
-		DRM_ERROR("process %d (filp %p) using buffer with filp %p\n",
-			  DRM_CURRENTPID, filp, buf->filp);
-		return DRM_ERR(EINVAL);
-	}
-
-	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", blit->idx);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	/* Set buf->used to the bytes of blit data based on the blit dimensions
@@ -703,13 +691,38 @@ static int mach64_dma_dispatch_blit(DRMFILE filp, drm_device_t * dev,
 	 * used for setup as well as the blit data.
 	 */
 	dwords = (blit->width * blit->height) >> dword_shift;
-	buf->used = dwords << 2;
-	if (buf->used <= 0 ||
-	    buf->used > MACH64_BUFFER_SIZE - MACH64_HOSTDATA_BLIT_OFFSET) {
-		DRM_ERROR("Invalid blit size: %d bytes\n", buf->used);
-		return DRM_ERR(EINVAL);
+	used = dwords << 2;
+	if (used <= 0 ||
+	    used > MACH64_BUFFER_SIZE - MACH64_HOSTDATA_BLIT_OFFSET) {
+		DRM_ERROR("Invalid blit size: %lu bytes\n", used);
+		return -EINVAL;
+	}
+
+	copy_buf = mach64_freelist_get(dev_priv);
+	if (copy_buf == NULL) {
+		DRM_ERROR("couldn't get buffer\n");
+		return -EAGAIN;
+	}
+
+	/* Copy the blit data from userspace.
+	 * 
+	 * XXX: This is overkill. The most efficient solution would be having 
+	 * two sets of buffers (one set private for vertex data, the other set 
+	 * client-writable for blits). However that would bring more complexity 
+	 * and would break backward compatability. The solution currently 
+	 * implemented is keeping all buffers private, allowing to secure the
+	 * driver, without increasing complexity at the expense of some speed 
+	 * transfering data.
+	 */
+	verify_ret = copy_from_user_blit(GETBUFPTR(copy_buf), blit->buf, used);
+
+	if (verify_ret != 0) {
+		mach64_freelist_put(dev_priv, copy_buf);
+		goto _blit_done;
 	}
 
+	copy_buf->used = used;
+
 	/* FIXME: Use a last buffer flag and reduce the state emitted for subsequent,
 	 * continuation buffers?
 	 */
@@ -718,7 +731,7 @@ static int mach64_dma_dispatch_blit(DRMFILE filp, drm_device_t * dev,
 	 * a register command every 16 dwords.  State setup is added at the start of the
 	 * buffer -- the client leaves space for this based on MACH64_HOSTDATA_BLIT_OFFSET
 	 */
-	DMASETPTR(buf);
+	DMASETPTR(copy_buf);
 
 	DMAOUTREG(MACH64_Z_CNTL, 0);
 	DMAOUTREG(MACH64_SCALE_3D_CNTL, 0);
@@ -748,39 +761,38 @@ static int mach64_dma_dispatch_blit(DRMFILE filp, drm_device_t * dev,
 	DMAOUTREG(MACH64_DST_X_Y, (blit->y << 16) | blit->x);
 	DMAOUTREG(MACH64_DST_WIDTH_HEIGHT, (blit->height << 16) | blit->width);
 
-	DRM_DEBUG("%s: %d bytes\n", __FUNCTION__, buf->used);
+	DRM_DEBUG("%lu bytes\n", used);
 
 	/* Add the buffer to the queue */
 	DMAADVANCEHOSTDATA(dev_priv);
 
-	return 0;
+_blit_done:
+	return verify_ret;
 }
 
 /* ================================================================
  * IOCTL functions
  */
 
-int mach64_dma_clear(DRM_IOCTL_ARGS)
+int mach64_dma_clear(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_mach64_clear_t clear;
+	drm_mach64_clear_t *clear = data;
 	int ret;
 
-	DRM_DEBUG("%s: pid=%d\n", __FUNCTION__, DRM_CURRENTPID);
-
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	DRM_COPY_FROM_USER_IOCTL(clear, (drm_mach64_clear_t *) data,
-				 sizeof(clear));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (sarea_priv->nbox > MACH64_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MACH64_NR_SAREA_CLIPRECTS;
 
-	ret = mach64_dma_dispatch_clear(filp, dev, clear.flags,
-					clear.x, clear.y, clear.w, clear.h,
-					clear.clear_color, clear.clear_depth);
+	ret = mach64_dma_dispatch_clear(dev, file_priv, clear->flags,
+					clear->x, clear->y, clear->w, clear->h,
+					clear->clear_color,
+					clear->clear_depth);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -788,21 +800,21 @@ int mach64_dma_clear(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-int mach64_dma_swap(DRM_IOCTL_ARGS)
+int mach64_dma_swap(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	int ret;
 
-	DRM_DEBUG("%s: pid=%d\n", __FUNCTION__, DRM_CURRENTPID);
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (sarea_priv->nbox > MACH64_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MACH64_NR_SAREA_CLIPRECTS;
 
-	ret = mach64_dma_dispatch_swap(filp, dev);
+	ret = mach64_dma_dispatch_swap(dev, file_priv);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -810,69 +822,52 @@ int mach64_dma_swap(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-int mach64_dma_vertex(DRM_IOCTL_ARGS)
+int mach64_dma_vertex(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_mach64_vertex_t vertex;
+	drm_mach64_vertex_t *vertex = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(vertex, (drm_mach64_vertex_t *) data,
-				 sizeof(vertex));
+	DRM_DEBUG("pid=%d buf=%p used=%lu discard=%d\n",
+		  DRM_CURRENTPID,
+		  vertex->buf, vertex->used, vertex->discard);
 
-	DRM_DEBUG("%s: pid=%d buf=%p used=%lu discard=%d\n",
-		  __FUNCTION__, DRM_CURRENTPID,
-		  vertex.buf, vertex.used, vertex.discard);
-
-	if (vertex.prim < 0 || vertex.prim > MACH64_PRIM_POLYGON) {
-		DRM_ERROR("buffer prim %d\n", vertex.prim);
-		return DRM_ERR(EINVAL);
+	if (vertex->prim < 0 || vertex->prim > MACH64_PRIM_POLYGON) {
+		DRM_ERROR("buffer prim %d\n", vertex->prim);
+		return -EINVAL;
 	}
 
-	if (vertex.used > MACH64_BUFFER_SIZE || (vertex.used & 3) != 0) {
+	if (vertex->used > MACH64_BUFFER_SIZE || (vertex->used & 3) != 0) {
 		DRM_ERROR("Invalid vertex buffer size: %lu bytes\n",
-			  vertex.used);
-		return DRM_ERR(EINVAL);
+			  vertex->used);
+		return -EINVAL;
 	}
 
 	if (sarea_priv->nbox > MACH64_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MACH64_NR_SAREA_CLIPRECTS;
 
-	return mach64_dma_dispatch_vertex(filp, dev, vertex.prim, vertex.buf,
-					  vertex.used, vertex.discard);
+	return mach64_dma_dispatch_vertex(dev, file_priv, vertex);
 }
 
-int mach64_dma_blit(DRM_IOCTL_ARGS)
+int mach64_dma_blit(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
 	drm_mach64_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_mach64_blit_t blit;
+	drm_mach64_blit_t *blit = data;
 	int ret;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(blit, (drm_mach64_blit_t *) data,
-				 sizeof(blit));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_DEBUG("%s: pid=%d index=%d\n",
-		  __FUNCTION__, DRM_CURRENTPID, blit.idx);
-
-	if (blit.idx < 0 || blit.idx >= dma->buf_count) {
-		DRM_ERROR("buffer index %d (of %d max)\n",
-			  blit.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
-	}
-
-	ret = mach64_dma_dispatch_blit(filp, dev, &blit);
+	ret = mach64_dma_dispatch_blit(dev, file_priv, blit);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -882,39 +877,36 @@ int mach64_dma_blit(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-int mach64_get_param(DRM_IOCTL_ARGS)
+int mach64_get_param(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mach64_private_t *dev_priv = dev->dev_private;
-	drm_mach64_getparam_t param;
+	drm_mach64_getparam_t *param = data;
 	int value;
 
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_mach64_getparam_t *) data,
-				 sizeof(param));
-
-	switch (param.param) {
+	switch (param->param) {
 	case MACH64_PARAM_FRAMES_QUEUED:
 		/* Needs lock since it calls mach64_ring_tick() */
-		LOCK_TEST_WITH_RETURN(dev, filp);
+		LOCK_TEST_WITH_RETURN(dev, file_priv);
 		value = mach64_do_get_frames_queued(dev_priv);
 		break;
 	case MACH64_PARAM_IRQ_NR:
 		value = dev->irq;
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
+	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
diff --git a/sys/dev/drm/mga_dma.c b/sys/dev/drm/mga_dma.c
index 093dd25..14c2d73 100644
--- a/sys/dev/drm/mga_dma.c
+++ b/sys/dev/drm/mga_dma.c
@@ -31,7 +31,7 @@ __FBSDID("$FreeBSD$");
 /**
  * \file mga_dma.c
  * DMA support for MGA G200 / G400.
- * 
+ *
  * \author Rickard E. (Rik) Faith <faith@valinux.com>
  * \author Jeff Hartmann <jhartmann@valinux.com>
  * \author Keith Whitwell <keith@tungstengraphics.com>
@@ -49,7 +49,7 @@ __FBSDID("$FreeBSD$");
 
 #define MINIMAL_CLEANUP    0
 #define FULL_CLEANUP       1
-static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup);
+static int mga_do_cleanup_dma(struct drm_device *dev, int full_cleanup);
 
 /* ================================================================
  * Engine control
@@ -74,7 +74,7 @@ int mga_do_wait_for_idle(drm_mga_private_t * dev_priv)
 	DRM_ERROR("failed!\n");
 	DRM_INFO("   status=0x%08x\n", status);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int mga_do_dma_reset(drm_mga_private_t * dev_priv)
@@ -227,7 +227,7 @@ void mga_do_dma_wrap_end(drm_mga_private_t * dev_priv)
 #define MGA_BUFFER_FREE		0
 
 #if MGA_FREELIST_DEBUG
-static void mga_freelist_print(drm_device_t * dev)
+static void mga_freelist_print(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_freelist_t *entry;
@@ -248,10 +248,10 @@ static void mga_freelist_print(drm_device_t * dev)
 }
 #endif
 
-static int mga_freelist_init(drm_device_t * dev, drm_mga_private_t * dev_priv)
+static int mga_freelist_init(struct drm_device * dev, drm_mga_private_t * dev_priv)
 {
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_mga_buf_priv_t *buf_priv;
 	drm_mga_freelist_t *entry;
 	int i;
@@ -259,7 +259,7 @@ static int mga_freelist_init(drm_device_t * dev, drm_mga_private_t * dev_priv)
 
 	dev_priv->head = drm_alloc(sizeof(drm_mga_freelist_t), DRM_MEM_DRIVER);
 	if (dev_priv->head == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv->head, 0, sizeof(drm_mga_freelist_t));
 	SET_AGE(&dev_priv->head->age, MGA_BUFFER_USED, 0);
@@ -270,7 +270,7 @@ static int mga_freelist_init(drm_device_t * dev, drm_mga_private_t * dev_priv)
 
 		entry = drm_alloc(sizeof(drm_mga_freelist_t), DRM_MEM_DRIVER);
 		if (entry == NULL)
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 
 		memset(entry, 0, sizeof(drm_mga_freelist_t));
 
@@ -294,7 +294,7 @@ static int mga_freelist_init(drm_device_t * dev, drm_mga_private_t * dev_priv)
 	return 0;
 }
 
-static void mga_freelist_cleanup(drm_device_t * dev)
+static void mga_freelist_cleanup(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_freelist_t *entry;
@@ -314,10 +314,10 @@ static void mga_freelist_cleanup(drm_device_t * dev)
 #if 0
 /* FIXME: Still needed?
  */
-static void mga_freelist_reset(drm_device_t * dev)
+static void mga_freelist_reset(struct drm_device * dev)
 {
 	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	drm_mga_buf_priv_t *buf_priv;
 	int i;
 
@@ -329,7 +329,7 @@ static void mga_freelist_reset(drm_device_t * dev)
 }
 #endif
 
-static drm_buf_t *mga_freelist_get(drm_device_t * dev)
+static struct drm_buf *mga_freelist_get(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_freelist_t *next;
@@ -362,7 +362,7 @@ static drm_buf_t *mga_freelist_get(drm_device_t * dev)
 	return NULL;
 }
 
-int mga_freelist_put(drm_device_t * dev, drm_buf_t * buf)
+int mga_freelist_put(struct drm_device * dev, struct drm_buf * buf)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_buf_priv_t *buf_priv = buf->dev_private;
@@ -396,13 +396,13 @@ int mga_freelist_put(drm_device_t * dev, drm_buf_t * buf)
  * DMA initialization, cleanup
  */
 
-int mga_driver_load(drm_device_t *dev, unsigned long flags)
+int mga_driver_load(struct drm_device *dev, unsigned long flags)
 {
-	drm_mga_private_t * dev_priv;
+	drm_mga_private_t *dev_priv;
 
 	dev_priv = drm_alloc(sizeof(drm_mga_private_t), DRM_MEM_DRIVER);
 	if (!dev_priv)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	dev->dev_private = (void *)dev_priv;
 	memset(dev_priv, 0, sizeof(drm_mga_private_t));
@@ -423,7 +423,7 @@ int mga_driver_load(drm_device_t *dev, unsigned long flags)
 
 /**
  * Bootstrap the driver for AGP DMA.
- * 
+ *
  * \todo
  * Investigate whether there is any benifit to storing the WARP microcode in
  * AGP memory.  If not, the microcode may as well always be put in PCI
@@ -436,21 +436,22 @@ int mga_driver_load(drm_device_t *dev, unsigned long flags)
  *
  * \sa mga_do_dma_bootstrap, mga_do_pci_dma_bootstrap
  */
-static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
+static int mga_do_agp_dma_bootstrap(struct drm_device *dev,
 				    drm_mga_dma_bootstrap_t * dma_bs)
 {
-	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	drm_mga_private_t *const dev_priv =
+		(drm_mga_private_t *)dev->dev_private;
 	unsigned int warp_size = mga_warp_microcode_size(dev_priv);
 	int err;
-	unsigned  offset;
+	unsigned offset;
 	const unsigned secondary_size = dma_bs->secondary_bin_count
 		* dma_bs->secondary_bin_size;
 	const unsigned agp_size = (dma_bs->agp_size << 20);
-	drm_buf_desc_t req;
-	drm_agp_mode_t mode;
-	drm_agp_info_t info;
-	drm_agp_buffer_t agp_req;
-	drm_agp_binding_t bind_req;
+	struct drm_buf_desc req;
+	struct drm_agp_mode mode;
+	struct drm_agp_info info;
+	struct drm_agp_buffer agp_req;
+	struct drm_agp_binding bind_req;
 
 	/* Acquire AGP. */
 	err = drm_agp_acquire(dev);
@@ -484,11 +485,10 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 		}
 	}
 
-
 	/* Allocate and bind AGP memory. */
 	agp_req.size = agp_size;
 	agp_req.type = 0;
-	err = drm_agp_alloc( dev, & agp_req );
+	err = drm_agp_alloc(dev, &agp_req);
 	if (err) {
 		dev_priv->agp_size = 0;
 		DRM_ERROR("Unable to allocate %uMB AGP memory\n",
@@ -514,36 +514,36 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 		warp_size = PAGE_SIZE;
 
 	offset = 0;
-	err = drm_addmap( dev, offset, warp_size,
-			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->warp );
+	err = drm_addmap(dev, offset, warp_size,
+			 _DRM_AGP, _DRM_READ_ONLY, &dev_priv->warp);
 	if (err) {
 		DRM_ERROR("Unable to map WARP microcode: %d\n", err);
 		return err;
 	}
 
 	offset += warp_size;
-	err = drm_addmap( dev, offset, dma_bs->primary_size,
-			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->primary );
+	err = drm_addmap(dev, offset, dma_bs->primary_size,
+			 _DRM_AGP, _DRM_READ_ONLY, & dev_priv->primary);
 	if (err) {
 		DRM_ERROR("Unable to map primary DMA region: %d\n", err);
 		return err;
 	}
 
 	offset += dma_bs->primary_size;
-	err = drm_addmap( dev, offset, secondary_size,
-			  _DRM_AGP, 0, & dev->agp_buffer_map );
+	err = drm_addmap(dev, offset, secondary_size,
+			 _DRM_AGP, 0, & dev->agp_buffer_map);
 	if (err) {
 		DRM_ERROR("Unable to map secondary DMA region: %d\n", err);
 		return err;
 	}
 
-	(void) memset( &req, 0, sizeof(req) );
+	(void)memset( &req, 0, sizeof(req) );
 	req.count = dma_bs->secondary_bin_count;
 	req.size = dma_bs->secondary_bin_size;
 	req.flags = _DRM_AGP_BUFFER;
 	req.agp_start = offset;
 
-	err = drm_addbufs_agp( dev, & req );
+	err = drm_addbufs_agp(dev, &req);
 	if (err) {
 		DRM_ERROR("Unable to add secondary DMA buffers: %d\n", err);
 		return err;
@@ -551,10 +551,10 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 
 #ifdef __linux__
 	{
-		drm_map_list_t *_entry;
+		struct drm_map_list *_entry;
 		unsigned long agp_token = 0;
 
-		list_for_each_entry(_entry, &dev->maplist->head, head) {
+		list_for_each_entry(_entry, &dev->maplist, head) {
 			if (_entry->map == dev->agp_buffer_map)
 				agp_token = _entry->user_token;
 		}
@@ -566,8 +566,8 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 #endif
 
 	offset += secondary_size;
-	err = drm_addmap( dev, offset, agp_size - offset,
-			  _DRM_AGP, 0, & dev_priv->agp_textures );
+	err = drm_addmap(dev, offset, agp_size - offset,
+			 _DRM_AGP, 0, & dev_priv->agp_textures);
 	if (err) {
 		DRM_ERROR("Unable to map AGP texture region: %d\n", err);
 		return err;
@@ -582,7 +582,7 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 		DRM_ERROR("failed to ioremap agp regions! (%p, %p, %p)\n",
 			  dev_priv->warp->handle, dev_priv->primary->handle,
 			  dev->agp_buffer_map->handle);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	dev_priv->dma_access = MGA_PAGPXFER;
@@ -594,7 +594,7 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
 
 /**
  * Bootstrap the driver for PCI DMA.
- * 
+ *
  * \todo
  * The algorithm for decreasing the size of the primary DMA buffer could be
  * better.  The size should be rounded up to the nearest page size, then
@@ -603,23 +603,24 @@ static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
  * \todo
  * Determine whether the maximum address passed to drm_pci_alloc is correct.
  * The same goes for drm_addbufs_pci.
- * 
+ *
  * \sa mga_do_dma_bootstrap, mga_do_agp_dma_bootstrap
  */
-static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
+static int mga_do_pci_dma_bootstrap(struct drm_device * dev,
 				    drm_mga_dma_bootstrap_t * dma_bs)
 {
-	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	drm_mga_private_t *const dev_priv =
+		(drm_mga_private_t *) dev->dev_private;
 	unsigned int warp_size = mga_warp_microcode_size(dev_priv);
 	unsigned int primary_size;
 	unsigned int bin_count;
 	int err;
-	drm_buf_desc_t req;
+	struct drm_buf_desc req;
+
 
-	
 	if (dev->dma == NULL) {
 		DRM_ERROR("dev->dma is NULL\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	/* Make drm_addbufs happy by not trying to create a mapping for less
@@ -642,9 +643,8 @@ static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
 	 * alignment of the primary or secondary DMA buffers.
 	 */
 
-	for ( primary_size = dma_bs->primary_size
-	      ; primary_size != 0
-	      ; primary_size >>= 1 ) {
+	for (primary_size = dma_bs->primary_size; primary_size != 0;
+	     primary_size >>= 1 ) {
 		/* The proper alignment for this mapping is 0x04 */
 		err = drm_addmap(dev, 0, primary_size, _DRM_CONSISTENT,
 				 _DRM_READ_ONLY, &dev_priv->primary);
@@ -654,29 +654,28 @@ static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
 
 	if (err != 0) {
 		DRM_ERROR("Unable to allocate primary DMA region: %d\n", err);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	if (dev_priv->primary->size != dma_bs->primary_size) {
 		DRM_INFO("Primary DMA buffer size reduced from %u to %u.\n",
-			 dma_bs->primary_size, 
-			 (unsigned) dev_priv->primary->size);
+			 dma_bs->primary_size,
+			 (unsigned)dev_priv->primary->size);
 		dma_bs->primary_size = dev_priv->primary->size;
 	}
 
-	for ( bin_count = dma_bs->secondary_bin_count
-	      ; bin_count > 0 
-	      ; bin_count-- ) {
-		(void) memset( &req, 0, sizeof(req) );
+	for (bin_count = dma_bs->secondary_bin_count; bin_count > 0;
+	     bin_count-- ) {
+		(void)memset(&req, 0, sizeof(req));
 		req.count = bin_count;
 		req.size = dma_bs->secondary_bin_size;
 
-		err = drm_addbufs_pci( dev, & req );
+		err = drm_addbufs_pci(dev, &req);
 		if (!err) {
 			break;
 		}
 	}
-	
+
 	if (bin_count == 0) {
 		DRM_ERROR("Unable to add secondary DMA buffers: %d\n", err);
 		return err;
@@ -699,12 +698,12 @@ static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
 }
 
 
-static int mga_do_dma_bootstrap(drm_device_t * dev,
-				drm_mga_dma_bootstrap_t * dma_bs)
+static int mga_do_dma_bootstrap(struct drm_device *dev,
+				drm_mga_dma_bootstrap_t *dma_bs)
 {
 	const int is_agp = (dma_bs->agp_mode != 0) && drm_device_is_agp(dev);
 	int err;
-	drm_mga_private_t * const dev_priv =
+	drm_mga_private_t *const dev_priv =
 		(drm_mga_private_t *) dev->dev_private;
 
 
@@ -713,17 +712,17 @@ static int mga_do_dma_bootstrap(drm_device_t * dev,
 	/* The first steps are the same for both PCI and AGP based DMA.  Map
 	 * the cards MMIO registers and map a status page.
 	 */
-	err = drm_addmap( dev, dev_priv->mmio_base, dev_priv->mmio_size,
-			  _DRM_REGISTERS, _DRM_READ_ONLY, & dev_priv->mmio );
+	err = drm_addmap(dev, dev_priv->mmio_base, dev_priv->mmio_size,
+			 _DRM_REGISTERS, _DRM_READ_ONLY, & dev_priv->mmio);
 	if (err) {
 		DRM_ERROR("Unable to map MMIO region: %d\n", err);
 		return err;
 	}
 
 
-	err = drm_addmap( dev, 0, SAREA_MAX, _DRM_SHM,
-			  _DRM_READ_ONLY | _DRM_LOCKED | _DRM_KERNEL,
-			  & dev_priv->status );
+	err = drm_addmap(dev, 0, SAREA_MAX, _DRM_SHM,
+			 _DRM_READ_ONLY | _DRM_LOCKED | _DRM_KERNEL,
+			 & dev_priv->status);
 	if (err) {
 		DRM_ERROR("Unable to map status region: %d\n", err);
 		return err;
@@ -739,7 +738,7 @@ static int mga_do_dma_bootstrap(drm_device_t * dev,
 	if (is_agp) {
 		err = mga_do_agp_dma_bootstrap(dev, dma_bs);
 	}
-	
+
 	/* If we attempted to initialize the card for AGP DMA but failed,
 	 * clean-up any mess that may have been created.
 	 */
@@ -765,44 +764,37 @@ static int mga_do_dma_bootstrap(drm_device_t * dev,
 	return err;
 }
 
-int mga_dma_bootstrap(DRM_IOCTL_ARGS)
+int mga_dma_bootstrap(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_mga_dma_bootstrap_t bootstrap;
+	drm_mga_dma_bootstrap_t *bootstrap = data;
 	int err;
 	static const int modes[] = { 0, 1, 2, 2, 4, 4, 4, 4 };
-	const drm_mga_private_t * const dev_priv = 
+	const drm_mga_private_t *const dev_priv =
 		(drm_mga_private_t *) dev->dev_private;
 
 
-	DRM_COPY_FROM_USER_IOCTL(bootstrap,
-				 (drm_mga_dma_bootstrap_t __user *) data,
-				 sizeof(bootstrap));
-
-	err = mga_do_dma_bootstrap(dev, & bootstrap);
+	err = mga_do_dma_bootstrap(dev, bootstrap);
 	if (err) {
 		mga_do_cleanup_dma(dev, FULL_CLEANUP);
 		return err;
 	}
 
 	if (dev_priv->agp_textures != NULL) {
-		bootstrap.texture_handle = dev_priv->agp_textures->offset;
-		bootstrap.texture_size = dev_priv->agp_textures->size;
+		bootstrap->texture_handle = dev_priv->agp_textures->offset;
+		bootstrap->texture_size = dev_priv->agp_textures->size;
 	} else {
-		bootstrap.texture_handle = 0;
-		bootstrap.texture_size = 0;
+		bootstrap->texture_handle = 0;
+		bootstrap->texture_size = 0;
 	}
 
-	bootstrap.agp_mode = modes[bootstrap.agp_mode & 0x07];
-
-	DRM_COPY_TO_USER_IOCTL((drm_mga_dma_bootstrap_t __user *)data,
-			       bootstrap, sizeof(bootstrap));
+	bootstrap->agp_mode = modes[bootstrap->agp_mode & 0x07];
 
 	return 0;
 }
 
 
-static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
+static int mga_do_init_dma(struct drm_device * dev, drm_mga_init_t * init)
 {
 	drm_mga_private_t *dev_priv;
 	int ret;
@@ -833,14 +825,13 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 	dev_priv->texture_offset = init->texture_offset[0];
 	dev_priv->texture_size = init->texture_size[0];
 
-	DRM_GETSAREA();
-
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("failed to find sarea!\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (! dev_priv->used_new_dma_init) {
+	if (!dev_priv->used_new_dma_init) {
 
 		dev_priv->dma_access = MGA_PAGPXFER;
 		dev_priv->wagp_enable = MGA_WAGP_ENABLE;
@@ -848,28 +839,29 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 		dev_priv->status = drm_core_findmap(dev, init->status_offset);
 		if (!dev_priv->status) {
 			DRM_ERROR("failed to find status page!\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
 		if (!dev_priv->mmio) {
 			DRM_ERROR("failed to find mmio region!\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		dev_priv->warp = drm_core_findmap(dev, init->warp_offset);
 		if (!dev_priv->warp) {
 			DRM_ERROR("failed to find warp microcode region!\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		dev_priv->primary = drm_core_findmap(dev, init->primary_offset);
 		if (!dev_priv->primary) {
 			DRM_ERROR("failed to find primary dma region!\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		dev->agp_buffer_token = init->buffers_offset;
-		dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
+		dev->agp_buffer_map =
+			drm_core_findmap(dev, init->buffers_offset);
 		if (!dev->agp_buffer_map) {
 			DRM_ERROR("failed to find dma buffer region!\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		drm_core_ioremap(dev_priv->warp, dev);
@@ -887,7 +879,7 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 	     ((dev->agp_buffer_map == NULL) ||
 	      (dev->agp_buffer_map->handle == NULL)))) {
 		DRM_ERROR("failed to ioremap agp regions!\n");
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	ret = mga_warp_install_microcode(dev_priv);
@@ -909,10 +901,6 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 	/* Init the primary DMA registers.
 	 */
 	MGA_WRITE(MGA_PRIMADDRESS, dev_priv->primary->offset | MGA_DMA_GENERAL);
-#if 0
-	MGA_WRITE(MGA_PRIMPTR, virt_to_bus((void *)dev_priv->prim.status) | MGA_PRIMPTREN0 |	/* Soft trap, SECEND, SETUPEND */
-		  MGA_PRIMPTREN1);	/* DWGSYNC */
-#endif
 
 	dev_priv->prim.start = (u8 *) dev_priv->primary->handle;
 	dev_priv->prim.end = ((u8 *) dev_priv->primary->handle
@@ -937,13 +925,13 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 
 	if (mga_freelist_init(dev, dev_priv) < 0) {
 		DRM_ERROR("could not initialize freelist\n");
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	return 0;
 }
 
-static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup)
+static int mga_do_cleanup_dma(struct drm_device *dev, int full_cleanup)
 {
 	int err = 0;
 	DRM_DEBUG("\n");
@@ -962,7 +950,7 @@ static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup)
 		    && (dev_priv->warp->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->warp, dev);
 
-		if ((dev_priv->primary != NULL) 
+		if ((dev_priv->primary != NULL)
 		    && (dev_priv->primary->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->primary, dev);
 
@@ -971,8 +959,8 @@ static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup)
 
 		if (dev_priv->used_new_dma_init) {
 			if (dev_priv->agp_handle != 0) {
-				drm_agp_binding_t unbind_req;
-				drm_agp_buffer_t free_req;
+				struct drm_agp_binding unbind_req;
+				struct drm_agp_buffer free_req;
 
 				unbind_req.handle = dev_priv->agp_handle;
 				drm_agp_unbind(dev, &unbind_req);
@@ -1004,72 +992,67 @@ static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup)
 
 		memset(&dev_priv->prim, 0, sizeof(dev_priv->prim));
 		dev_priv->warp_pipe = 0;
-		memset(dev_priv->warp_pipe_phys, 0, sizeof(dev_priv->warp_pipe_phys));
+		memset(dev_priv->warp_pipe_phys, 0,
+		       sizeof(dev_priv->warp_pipe_phys));
 
 		if (dev_priv->head != NULL) {
 			mga_freelist_cleanup(dev);
 		}
 	}
 
-	return 0;
+	return err;
 }
 
-int mga_dma_init(DRM_IOCTL_ARGS)
+int mga_dma_init(struct drm_device *dev, void *data,
+		 struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_mga_init_t init;
+	drm_mga_init_t *init = data;
 	int err;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_mga_init_t __user *) data,
-				 sizeof(init));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	switch (init.func) {
+	switch (init->func) {
 	case MGA_INIT_DMA:
-		err = mga_do_init_dma(dev, &init);
+		err = mga_do_init_dma(dev, init);
 		if (err) {
-			(void) mga_do_cleanup_dma(dev, FULL_CLEANUP);
+			(void)mga_do_cleanup_dma(dev, FULL_CLEANUP);
 		}
 		return err;
 	case MGA_CLEANUP_DMA:
 		return mga_do_cleanup_dma(dev, FULL_CLEANUP);
 	}
 
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
 /* ================================================================
  * Primary DMA stream management
  */
 
-int mga_dma_flush(DRM_IOCTL_ARGS)
+int mga_dma_flush(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
-	drm_lock_t lock;
+	struct drm_lock *lock = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(lock, (drm_lock_t __user *) data,
-				 sizeof(lock));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	DRM_DEBUG("%s%s%s\n",
-		  (lock.flags & _DRM_LOCK_FLUSH) ? "flush, " : "",
-		  (lock.flags & _DRM_LOCK_FLUSH_ALL) ? "flush all, " : "",
-		  (lock.flags & _DRM_LOCK_QUIESCENT) ? "idle, " : "");
+		  (lock->flags & _DRM_LOCK_FLUSH) ? "flush, " : "",
+		  (lock->flags & _DRM_LOCK_FLUSH_ALL) ? "flush all, " : "",
+		  (lock->flags & _DRM_LOCK_QUIESCENT) ? "idle, " : "");
 
 	WRAP_WAIT_WITH_RETURN(dev_priv);
 
-	if (lock.flags & (_DRM_LOCK_FLUSH | _DRM_LOCK_FLUSH_ALL)) {
+	if (lock->flags & (_DRM_LOCK_FLUSH | _DRM_LOCK_FLUSH_ALL)) {
 		mga_do_dma_flush(dev_priv);
 	}
 
-	if (lock.flags & _DRM_LOCK_QUIESCENT) {
+	if (lock->flags & _DRM_LOCK_QUIESCENT) {
 #if MGA_DMA_DEBUG
 		int ret = mga_do_wait_for_idle(dev_priv);
 		if (ret < 0)
-			DRM_INFO("%s: -EBUSY\n", __FUNCTION__);
+			DRM_INFO("-EBUSY\n");
 		return ret;
 #else
 		return mga_do_wait_for_idle(dev_priv);
@@ -1079,12 +1062,12 @@ int mga_dma_flush(DRM_IOCTL_ARGS)
 	}
 }
 
-int mga_dma_reset(DRM_IOCTL_ARGS)
+int mga_dma_reset(struct drm_device *dev, void *data,
+		  struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return mga_do_dma_reset(dev_priv);
 }
@@ -1093,76 +1076,72 @@ int mga_dma_reset(DRM_IOCTL_ARGS)
  * DMA buffer management
  */
 
-static int mga_dma_get_buffers(DRMFILE filp, drm_device_t * dev, drm_dma_t * d)
+static int mga_dma_get_buffers(struct drm_device * dev,
+			       struct drm_file *file_priv, struct drm_dma * d)
 {
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	int i;
 
 	for (i = d->granted_count; i < d->request_count; i++) {
 		buf = mga_freelist_get(dev);
 		if (!buf)
-			return DRM_ERR(EAGAIN);
+			return -EAGAIN;
 
-		buf->filp = filp;
+		buf->file_priv = file_priv;
 
 		if (DRM_COPY_TO_USER(&d->request_indices[i],
 				     &buf->idx, sizeof(buf->idx)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		if (DRM_COPY_TO_USER(&d->request_sizes[i],
 				     &buf->total, sizeof(buf->total)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 
 		d->granted_count++;
 	}
 	return 0;
 }
 
-int mga_dma_buffers(DRM_IOCTL_ARGS)
+int mga_dma_buffers(struct drm_device *dev, void *data,
+		    struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
-	drm_dma_t __user *argp = (void __user *)data;
-	drm_dma_t d;
+	struct drm_dma *d = data;
 	int ret = 0;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(d, argp, sizeof(d));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Please don't send us buffers.
 	 */
-	if (d.send_count != 0) {
+	if (d->send_count != 0) {
 		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d.send_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->send_count);
+		return -EINVAL;
 	}
 
 	/* We'll send you buffers.
 	 */
-	if (d.request_count < 0 || d.request_count > dma->buf_count) {
+	if (d->request_count < 0 || d->request_count > dma->buf_count) {
 		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d.request_count, dma->buf_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->request_count, dma->buf_count);
+		return -EINVAL;
 	}
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
 
-	d.granted_count = 0;
+	d->granted_count = 0;
 
-	if (d.request_count) {
-		ret = mga_dma_get_buffers(filp, dev, &d);
+	if (d->request_count) {
+		ret = mga_dma_get_buffers(dev, file_priv, d);
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, d, sizeof(d));
-
 	return ret;
 }
 
 /**
  * Called just before the module is unloaded.
  */
-int mga_driver_unload(drm_device_t * dev)
+int mga_driver_unload(struct drm_device * dev)
 {
 	drm_free(dev->dev_private, sizeof(drm_mga_private_t), DRM_MEM_DRIVER);
 	dev->dev_private = NULL;
@@ -1173,12 +1152,12 @@ int mga_driver_unload(drm_device_t * dev)
 /**
  * Called when the last opener of the device is closed.
  */
-void mga_driver_lastclose(drm_device_t * dev)
+void mga_driver_lastclose(struct drm_device * dev)
 {
 	mga_do_cleanup_dma(dev, FULL_CLEANUP);
 }
 
-int mga_driver_dma_quiescent(drm_device_t * dev)
+int mga_driver_dma_quiescent(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	return mga_do_wait_for_idle(dev_priv);
diff --git a/sys/dev/drm/mga_drm.h b/sys/dev/drm/mga_drm.h
index 6c466e0..16e8990 100644
--- a/sys/dev/drm/mga_drm.h
+++ b/sys/dev/drm/mga_drm.h
@@ -184,7 +184,7 @@ typedef struct _drm_mga_sarea {
 
 	/* The current cliprects, or a subset thereof.
 	 */
-	drm_clip_rect_t boxes[MGA_NR_SAREA_CLIPRECTS];
+	struct drm_clip_rect boxes[MGA_NR_SAREA_CLIPRECTS];
 	unsigned int nbox;
 
 	/* Information about the most recently used 3d drawable.  The
@@ -205,7 +205,7 @@ typedef struct _drm_mga_sarea {
 	unsigned int exported_nback;
 	int exported_back_x, exported_front_x, exported_w;
 	int exported_back_y, exported_front_y, exported_h;
-	drm_clip_rect_t exported_boxes[MGA_NR_SAREA_CLIPRECTS];
+	struct drm_clip_rect exported_boxes[MGA_NR_SAREA_CLIPRECTS];
 
 	/* Counters for aging textures and for client-side throttling.
 	 */
@@ -219,7 +219,7 @@ typedef struct _drm_mga_sarea {
 
 	/* LRU lists for texture memory in agp space and on the card.
 	 */
-	drm_tex_region_t texList[MGA_NR_TEX_HEAPS][MGA_NR_TEX_REGIONS + 1];
+	struct drm_tex_region texList[MGA_NR_TEX_HEAPS][MGA_NR_TEX_REGIONS + 1];
 	unsigned int texAge[MGA_NR_TEX_HEAPS];
 
 	/* Mechanism to validate card state.
@@ -305,10 +305,10 @@ typedef struct drm_mga_init {
 typedef struct drm_mga_dma_bootstrap {
 	/**
 	 * \name AGP texture region
-	 * 
+	 *
 	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, these fields will
 	 * be filled in with the actual AGP texture settings.
-	 * 
+	 *
 	 * \warning
 	 * If these fields are non-zero, but dma_mga_dma_bootstrap::agp_mode
 	 * is zero, it means that PCI memory (most likely through the use of
@@ -322,7 +322,7 @@ typedef struct drm_mga_dma_bootstrap {
 
 	/**
 	 * Requested size of the primary DMA region.
-	 * 
+	 *
 	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
 	 * filled in with the actual AGP mode.  If AGP was not available
 	 */
@@ -331,18 +331,18 @@ typedef struct drm_mga_dma_bootstrap {
 
 	/**
 	 * Requested number of secondary DMA buffers.
-	 * 
+	 *
 	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
 	 * filled in with the actual number of secondary DMA buffers
 	 * allocated.  Particularly when PCI DMA is used, this may be
 	 * (subtantially) less than the number requested.
 	 */
 	uint32_t secondary_bin_count;
-	
-	
+
+
 	/**
 	 * Requested size of each secondary DMA buffer.
-	 * 
+	 *
 	 * While the kernel \b is free to reduce
 	 * dma_mga_dma_bootstrap::secondary_bin_count, it is \b not allowed
 	 * to reduce dma_mga_dma_bootstrap::secondary_bin_size.
@@ -355,7 +355,7 @@ typedef struct drm_mga_dma_bootstrap {
 	 * \c AGPSTAT2_2X, and \c AGPSTAT2_4X are supported.  If this value is
 	 * zero, it means that PCI DMA should be used, even if AGP is
 	 * possible.
-	 * 
+	 *
 	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
 	 * filled in with the actual AGP mode.  If AGP was not available
 	 * (i.e., PCI DMA was used), this value will be zero.
diff --git a/sys/dev/drm/mga_drv.c b/sys/dev/drm/mga_drv.c
index 8bde4fd..e5de7d7 100644
--- a/sys/dev/drm/mga_drv.c
+++ b/sys/dev/drm/mga_drv.c
@@ -62,7 +62,7 @@ static drm_pci_id_list_t mga_pciidlist[] = {
  * This function needs to be filled in!  The implementation in
  * linux-core/mga_drv.c shows what needs to be done.
  */
-static int mga_driver_device_is_agp(drm_device_t * dev)
+static int mga_driver_device_is_agp(struct drm_device * dev)
 {
 	device_t bus;
 
@@ -87,13 +87,15 @@ static int mga_driver_device_is_agp(drm_device_t * dev)
 		return DRM_MIGHT_BE_AGP;
 }
 
-static void mga_configure(drm_device_t *dev)
+static void mga_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= sizeof(drm_mga_buf_priv_t);
 	dev->driver.load		= mga_driver_load;
 	dev->driver.unload		= mga_driver_unload;
 	dev->driver.lastclose		= mga_driver_lastclose;
-	dev->driver.vblank_wait		= mga_driver_vblank_wait;
+	dev->driver.get_vblank_counter	= mga_get_vblank_counter;
+	dev->driver.enable_vblank	= mga_enable_vblank;
+	dev->driver.disable_vblank	= mga_disable_vblank;
 	dev->driver.irq_preinstall	= mga_driver_irq_preinstall;
 	dev->driver.irq_postinstall	= mga_driver_irq_postinstall;
 	dev->driver.irq_uninstall	= mga_driver_irq_uninstall;
@@ -132,9 +134,9 @@ mga_probe(device_t dev)
 static int
 mga_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	mga_configure(dev);
 	return drm_attach(nbdev, mga_pciidlist);
 }
@@ -151,7 +153,7 @@ static device_method_t mga_methods[] = {
 static driver_t mga_driver = {
 	"drm",
 	mga_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
@@ -166,7 +168,7 @@ MODULE_DEPEND(mga, drm, 1, 1, 1);
 #ifdef _LKM
 CFDRIVER_DECL(mga, DV_TTY, NULL);
 #else
-CFATTACH_DECL(mga, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+CFATTACH_DECL(mga, sizeof(struct drm_device), drm_probe, drm_attach, drm_detach,
     drm_activate);
 #endif
 #endif
diff --git a/sys/dev/drm/mga_drv.h b/sys/dev/drm/mga_drv.h
index 95726e2..61b6e82 100644
--- a/sys/dev/drm/mga_drv.h
+++ b/sys/dev/drm/mga_drv.h
@@ -68,7 +68,7 @@ typedef struct drm_mga_freelist {
 	struct drm_mga_freelist *next;
 	struct drm_mga_freelist *prev;
 	drm_mga_age_t age;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 } drm_mga_freelist_t;
 
 typedef struct {
@@ -112,17 +112,18 @@ typedef struct drm_mga_private {
 
 	/**
 	 * \name MMIO region parameters.
-	 * 
+	 *
 	 * \sa drm_mga_private_t::mmio
 	 */
 	/*@{*/
-	u32 mmio_base;             /**< Bus address of base of MMIO. */
-	u32 mmio_size;             /**< Size of the MMIO region. */
+	u32 mmio_base;			/**< Bus address of base of MMIO. */
+	u32 mmio_size;			/**< Size of the MMIO region. */
 	/*@}*/
 
 	u32 clear_cmd;
 	u32 maccess;
 
+	atomic_t vbl_received;		/**< Number of vblanks received. */
 	wait_queue_head_t fence_queue;
 	atomic_t last_fence_retired;
 	u32 next_fence_to_post;
@@ -146,24 +147,29 @@ typedef struct drm_mga_private {
 	drm_local_map_t *warp;
 	drm_local_map_t *primary;
 	drm_local_map_t *agp_textures;
-	
+
 	unsigned long agp_handle;
 	unsigned int agp_size;
 } drm_mga_private_t;
 
-extern drm_ioctl_desc_t mga_ioctls[];
+extern struct drm_ioctl_desc mga_ioctls[];
 extern int mga_max_ioctl;
 
 				/* mga_dma.c */
-extern int mga_dma_bootstrap(DRM_IOCTL_ARGS);
-extern int mga_dma_init(DRM_IOCTL_ARGS);
-extern int mga_dma_flush(DRM_IOCTL_ARGS);
-extern int mga_dma_reset(DRM_IOCTL_ARGS);
-extern int mga_dma_buffers(DRM_IOCTL_ARGS);
-extern int mga_driver_load(drm_device_t *dev, unsigned long flags);
-extern int mga_driver_unload(drm_device_t * dev);
-extern void mga_driver_lastclose(drm_device_t * dev);
-extern int mga_driver_dma_quiescent(drm_device_t * dev);
+extern int mga_dma_bootstrap(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv);
+extern int mga_dma_init(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+extern int mga_dma_flush(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+extern int mga_dma_reset(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+extern int mga_dma_buffers(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+extern int mga_driver_load(struct drm_device *dev, unsigned long flags);
+extern int mga_driver_unload(struct drm_device * dev);
+extern void mga_driver_lastclose(struct drm_device * dev);
+extern int mga_driver_dma_quiescent(struct drm_device * dev);
 
 extern int mga_do_wait_for_idle(drm_mga_private_t * dev_priv);
 
@@ -171,7 +177,7 @@ extern void mga_do_dma_flush(drm_mga_private_t * dev_priv);
 extern void mga_do_dma_wrap_start(drm_mga_private_t * dev_priv);
 extern void mga_do_dma_wrap_end(drm_mga_private_t * dev_priv);
 
-extern int mga_freelist_put(drm_device_t * dev, drm_buf_t * buf);
+extern int mga_freelist_put(struct drm_device * dev, struct drm_buf * buf);
 
 				/* mga_warp.c */
 extern unsigned int mga_warp_microcode_size(const drm_mga_private_t * dev_priv);
@@ -179,12 +185,15 @@ extern int mga_warp_install_microcode(drm_mga_private_t * dev_priv);
 extern int mga_warp_init(drm_mga_private_t * dev_priv);
 
 				/* mga_irq.c */
-extern int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence);
-extern int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence);
+extern int mga_enable_vblank(struct drm_device *dev, int crtc);
+extern void mga_disable_vblank(struct drm_device *dev, int crtc);
+extern u32 mga_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int mga_driver_fence_wait(struct drm_device * dev, unsigned int *sequence);
+extern int mga_driver_vblank_wait(struct drm_device * dev, unsigned int *sequence);
 extern irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS);
-extern void mga_driver_irq_preinstall(drm_device_t * dev);
-extern void mga_driver_irq_postinstall(drm_device_t * dev);
-extern void mga_driver_irq_uninstall(drm_device_t * dev);
+extern void mga_driver_irq_preinstall(struct drm_device * dev);
+extern int mga_driver_irq_postinstall(struct drm_device * dev);
+extern void mga_driver_irq_uninstall(struct drm_device * dev);
 extern long mga_compat_ioctl(struct file *filp, unsigned int cmd,
 			     unsigned long arg);
 
@@ -214,8 +223,8 @@ static inline u32 _MGA_READ(u32 * addr)
 #define MGA_WRITE( reg, val )	DRM_WRITE32(dev_priv->mmio, (reg), (val))
 #endif
 
-#define DWGREG0 	0x1c00
-#define DWGREG0_END 	0x1dff
+#define DWGREG0		0x1c00
+#define DWGREG0_END	0x1dff
 #define DWGREG1		0x2c00
 #define DWGREG1_END	0x2dff
 
@@ -247,8 +256,8 @@ do {									\
 		} else if ( dev_priv->prim.space <			\
 			    dev_priv->prim.high_mark ) {		\
 			if ( MGA_DMA_DEBUG )				\
-				DRM_INFO( "%s: wrap...\n", __FUNCTION__ );	\
-			return DRM_ERR(EBUSY);			\
+				DRM_INFO( "wrap...\n");		\
+			return -EBUSY;			\
 		}							\
 	}								\
 } while (0)
@@ -258,8 +267,8 @@ do {									\
 	if ( test_bit( 0, &dev_priv->prim.wrapped ) ) {			\
 		if ( mga_do_wait_for_idle( dev_priv ) < 0 ) {		\
 			if ( MGA_DMA_DEBUG )				\
-				DRM_INFO( "%s: wrap...\n", __FUNCTION__ );	\
-			return DRM_ERR(EBUSY);			\
+				DRM_INFO( "wrap...\n");		\
+			return -EBUSY;			\
 		}							\
 		mga_do_dma_wrap_end( dev_priv );			\
 	}								\
@@ -278,8 +287,7 @@ do {									\
 #define BEGIN_DMA( n )							\
 do {									\
 	if ( MGA_VERBOSE ) {						\
-		DRM_INFO( "BEGIN_DMA( %d ) in %s\n",			\
-			  (n), __FUNCTION__ );				\
+		DRM_INFO( "BEGIN_DMA( %d )\n", (n) );		\
 		DRM_INFO( "   space=0x%x req=0x%Zx\n",			\
 			  dev_priv->prim.space, (n) * DMA_BLOCK_SIZE );	\
 	}								\
@@ -290,7 +298,7 @@ do {									\
 #define BEGIN_DMA_WRAP()						\
 do {									\
 	if ( MGA_VERBOSE ) {						\
-		DRM_INFO( "BEGIN_DMA() in %s\n", __FUNCTION__ );		\
+		DRM_INFO( "BEGIN_DMA()\n" );				\
 		DRM_INFO( "   space=0x%x\n", dev_priv->prim.space );	\
 	}								\
 	prim = dev_priv->prim.start;					\
@@ -309,7 +317,7 @@ do {									\
 #define FLUSH_DMA()							\
 do {									\
 	if ( 0 ) {							\
-		DRM_INFO( "%s:\n", __FUNCTION__ );				\
+		DRM_INFO( "\n" );					\
 		DRM_INFO( "   tail=0x%06x head=0x%06lx\n",		\
 			  dev_priv->prim.tail,				\
 			  MGA_READ( MGA_PRIMADDRESS ) -			\
@@ -392,22 +400,22 @@ do {									\
 #define MGA_VINTCLR			(1 << 4)
 #define MGA_VINTEN			(1 << 5)
 
-#define MGA_ALPHACTRL 			0x2c7c
-#define MGA_AR0 			0x1c60
-#define MGA_AR1 			0x1c64
-#define MGA_AR2 			0x1c68
-#define MGA_AR3 			0x1c6c
-#define MGA_AR4 			0x1c70
-#define MGA_AR5 			0x1c74
-#define MGA_AR6 			0x1c78
+#define MGA_ALPHACTRL			0x2c7c
+#define MGA_AR0				0x1c60
+#define MGA_AR1				0x1c64
+#define MGA_AR2				0x1c68
+#define MGA_AR3				0x1c6c
+#define MGA_AR4				0x1c70
+#define MGA_AR5				0x1c74
+#define MGA_AR6				0x1c78
 
 #define MGA_CXBNDRY			0x1c80
-#define MGA_CXLEFT 			0x1ca0
+#define MGA_CXLEFT			0x1ca0
 #define MGA_CXRIGHT			0x1ca4
 
-#define MGA_DMAPAD 			0x1c54
-#define MGA_DSTORG 			0x2cb8
-#define MGA_DWGCTL 			0x1c00
+#define MGA_DMAPAD			0x1c54
+#define MGA_DSTORG			0x2cb8
+#define MGA_DWGCTL			0x1c00
 #	define MGA_OPCOD_MASK			(15 << 0)
 #	define MGA_OPCOD_TRAP			(4 << 0)
 #	define MGA_OPCOD_TEXTURE_TRAP		(6 << 0)
@@ -453,27 +461,27 @@ do {									\
 #	define MGA_CLIPDIS			(1 << 31)
 #define MGA_DWGSYNC			0x2c4c
 
-#define MGA_FCOL 			0x1c24
-#define MGA_FIFOSTATUS 			0x1e10
-#define MGA_FOGCOL 			0x1cf4
+#define MGA_FCOL			0x1c24
+#define MGA_FIFOSTATUS			0x1e10
+#define MGA_FOGCOL			0x1cf4
 #define MGA_FXBNDRY			0x1c84
-#define MGA_FXLEFT 			0x1ca8
+#define MGA_FXLEFT			0x1ca8
 #define MGA_FXRIGHT			0x1cac
 
-#define MGA_ICLEAR 			0x1e18
+#define MGA_ICLEAR			0x1e18
 #	define MGA_SOFTRAPICLR			(1 << 0)
 #	define MGA_VLINEICLR			(1 << 5)
-#define MGA_IEN 			0x1e1c
+#define MGA_IEN				0x1e1c
 #	define MGA_SOFTRAPIEN			(1 << 0)
 #	define MGA_VLINEIEN			(1 << 5)
 
-#define MGA_LEN 			0x1c5c
+#define MGA_LEN				0x1c5c
 
 #define MGA_MACCESS			0x1c04
 
-#define MGA_PITCH 			0x1c8c
-#define MGA_PLNWT 			0x1c1c
-#define MGA_PRIMADDRESS 		0x1e58
+#define MGA_PITCH			0x1c8c
+#define MGA_PLNWT			0x1c1c
+#define MGA_PRIMADDRESS			0x1e58
 #	define MGA_DMA_GENERAL			(0 << 0)
 #	define MGA_DMA_BLIT			(1 << 0)
 #	define MGA_DMA_VECTOR			(2 << 0)
@@ -485,43 +493,43 @@ do {									\
 #	define MGA_PRIMPTREN0			(1 << 0)
 #	define MGA_PRIMPTREN1			(1 << 1)
 
-#define MGA_RST 			0x1e40
+#define MGA_RST				0x1e40
 #	define MGA_SOFTRESET			(1 << 0)
 #	define MGA_SOFTEXTRST			(1 << 1)
 
-#define MGA_SECADDRESS 			0x2c40
-#define MGA_SECEND 			0x2c44
-#define MGA_SETUPADDRESS 		0x2cd0
-#define MGA_SETUPEND 			0x2cd4
+#define MGA_SECADDRESS			0x2c40
+#define MGA_SECEND			0x2c44
+#define MGA_SETUPADDRESS		0x2cd0
+#define MGA_SETUPEND			0x2cd4
 #define MGA_SGN				0x1c58
 #define MGA_SOFTRAP			0x2c48
-#define MGA_SRCORG 			0x2cb4
+#define MGA_SRCORG			0x2cb4
 #	define MGA_SRMMAP_MASK			(1 << 0)
 #	define MGA_SRCMAP_FB			(0 << 0)
 #	define MGA_SRCMAP_SYSMEM		(1 << 0)
 #	define MGA_SRCACC_MASK			(1 << 1)
 #	define MGA_SRCACC_PCI			(0 << 1)
 #	define MGA_SRCACC_AGP			(1 << 1)
-#define MGA_STATUS 			0x1e14
+#define MGA_STATUS			0x1e14
 #	define MGA_SOFTRAPEN			(1 << 0)
 #	define MGA_VSYNCPEN			(1 << 4)
 #	define MGA_VLINEPEN			(1 << 5)
 #	define MGA_DWGENGSTS			(1 << 16)
 #	define MGA_ENDPRDMASTS			(1 << 17)
 #define MGA_STENCIL			0x2cc8
-#define MGA_STENCILCTL 			0x2ccc
+#define MGA_STENCILCTL			0x2ccc
 
-#define MGA_TDUALSTAGE0 		0x2cf8
-#define MGA_TDUALSTAGE1 		0x2cfc
-#define MGA_TEXBORDERCOL 		0x2c5c
-#define MGA_TEXCTL 			0x2c30
+#define MGA_TDUALSTAGE0			0x2cf8
+#define MGA_TDUALSTAGE1			0x2cfc
+#define MGA_TEXBORDERCOL		0x2c5c
+#define MGA_TEXCTL			0x2c30
 #define MGA_TEXCTL2			0x2c3c
 #	define MGA_DUALTEX			(1 << 7)
 #	define MGA_G400_TC2_MAGIC		(1 << 15)
 #	define MGA_MAP1_ENABLE			(1 << 31)
-#define MGA_TEXFILTER 			0x2c58
-#define MGA_TEXHEIGHT 			0x2c2c
-#define MGA_TEXORG 			0x2c24
+#define MGA_TEXFILTER			0x2c58
+#define MGA_TEXHEIGHT			0x2c2c
+#define MGA_TEXORG			0x2c24
 #	define MGA_TEXORGMAP_MASK		(1 << 0)
 #	define MGA_TEXORGMAP_FB			(0 << 0)
 #	define MGA_TEXORGMAP_SYSMEM		(1 << 0)
@@ -532,45 +540,45 @@ do {									\
 #define MGA_TEXORG2			0x2ca8
 #define MGA_TEXORG3			0x2cac
 #define MGA_TEXORG4			0x2cb0
-#define MGA_TEXTRANS 			0x2c34
-#define MGA_TEXTRANSHIGH 		0x2c38
-#define MGA_TEXWIDTH 			0x2c28
-
-#define MGA_WACCEPTSEQ 			0x1dd4
-#define MGA_WCODEADDR 			0x1e6c
-#define MGA_WFLAG 			0x1dc4
-#define MGA_WFLAG1 			0x1de0
+#define MGA_TEXTRANS			0x2c34
+#define MGA_TEXTRANSHIGH		0x2c38
+#define MGA_TEXWIDTH			0x2c28
+
+#define MGA_WACCEPTSEQ			0x1dd4
+#define MGA_WCODEADDR			0x1e6c
+#define MGA_WFLAG			0x1dc4
+#define MGA_WFLAG1			0x1de0
 #define MGA_WFLAGNB			0x1e64
-#define MGA_WFLAGNB1 			0x1e08
+#define MGA_WFLAGNB1			0x1e08
 #define MGA_WGETMSB			0x1dc8
-#define MGA_WIADDR 			0x1dc0
+#define MGA_WIADDR			0x1dc0
 #define MGA_WIADDR2			0x1dd8
 #	define MGA_WMODE_SUSPEND		(0 << 0)
 #	define MGA_WMODE_RESUME			(1 << 0)
 #	define MGA_WMODE_JUMP			(2 << 0)
 #	define MGA_WMODE_START			(3 << 0)
 #	define MGA_WAGP_ENABLE			(1 << 2)
-#define MGA_WMISC 			0x1e70
+#define MGA_WMISC			0x1e70
 #	define MGA_WUCODECACHE_ENABLE		(1 << 0)
 #	define MGA_WMASTER_ENABLE		(1 << 1)
 #	define MGA_WCACHEFLUSH_ENABLE		(1 << 3)
 #define MGA_WVRTXSZ			0x1dcc
 
-#define MGA_YBOT 			0x1c9c
-#define MGA_YDST 			0x1c90
+#define MGA_YBOT			0x1c9c
+#define MGA_YDST			0x1c90
 #define MGA_YDSTLEN			0x1c88
 #define MGA_YDSTORG			0x1c94
-#define MGA_YTOP 			0x1c98
+#define MGA_YTOP			0x1c98
 
-#define MGA_ZORG 			0x1c0c
+#define MGA_ZORG			0x1c0c
 
 /* This finishes the current batch of commands
  */
-#define MGA_EXEC 			0x0100
+#define MGA_EXEC			0x0100
 
 /* AGP PLL encoding (for G200 only).
  */
-#define MGA_AGP_PLL 			0x1e4c
+#define MGA_AGP_PLL			0x1e4c
 #	define MGA_AGP2XPLL_DISABLE		(0 << 0)
 #	define MGA_AGP2XPLL_ENABLE		(1 << 0)
 
diff --git a/sys/dev/drm/mga_irq.c b/sys/dev/drm/mga_irq.c
index f8a88ed..0d59993 100644
--- a/sys/dev/drm/mga_irq.c
+++ b/sys/dev/drm/mga_irq.c
@@ -39,9 +39,23 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/mga_drm.h"
 #include "dev/drm/mga_drv.h"
 
+u32 mga_get_vblank_counter(struct drm_device *dev, int crtc)
+{
+	const drm_mga_private_t *const dev_priv = 
+		(drm_mga_private_t *) dev->dev_private;
+
+	if (crtc != 0) {
+		return 0;
+	}
+
+
+	return atomic_read(&dev_priv->vbl_received);
+}
+
+
 irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 {
-	drm_device_t *dev = (drm_device_t *) arg;
+	struct drm_device *dev = (struct drm_device *) arg;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 	int status;
 	int handled = 0;
@@ -51,16 +65,15 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 	/* VBLANK interrupt */
 	if (status & MGA_VLINEPEN) {
 		MGA_WRITE(MGA_ICLEAR, MGA_VLINEICLR);
-		atomic_inc(&dev->vbl_received);
-		DRM_WAKEUP(&dev->vbl_queue);
-		drm_vbl_send_signals(dev);
+		atomic_inc(&dev_priv->vbl_received);
+		drm_handle_vblank(dev, 0);
 		handled = 1;
 	}
 
 	/* SOFTRAP interrupt */
 	if (status & MGA_SOFTRAPEN) {
 		const u32 prim_start = MGA_READ(MGA_PRIMADDRESS);
-		const u32 prim_end   = MGA_READ(MGA_PRIMEND);
+		const u32 prim_end = MGA_READ(MGA_PRIMEND);
 
 
 		MGA_WRITE(MGA_ICLEAR, MGA_SOFTRAPICLR);
@@ -68,7 +81,7 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 		/* In addition to clearing the interrupt-pending bit, we
 		 * have to write to MGA_PRIMEND to re-start the DMA operation.
 		 */
-		if ( (prim_start & ~0x03) != (prim_end & ~0x03) ) {
+		if ((prim_start & ~0x03) != (prim_end & ~0x03)) {
 			MGA_WRITE(MGA_PRIMEND, prim_end);
 		}
 
@@ -77,31 +90,42 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 		handled = 1;
 	}
 
-	if ( handled ) {
+	if (handled)
 		return IRQ_HANDLED;
-	}
 	return IRQ_NONE;
 }
 
-int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
+int mga_enable_vblank(struct drm_device *dev, int crtc)
 {
-	unsigned int cur_vblank;
-	int ret = 0;
+	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 
-	/* Assume that the user has missed the current sequence number
-	 * by about a day rather than she wants to wait for years
-	 * using vertical blanks...
-	 */
-	DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-		    (((cur_vblank = atomic_read(&dev->vbl_received))
-		      - *sequence) <= (1 << 23)));
+	if (crtc != 0) {
+		DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+			  crtc);
+		return 0;
+	}
 
-	*sequence = cur_vblank;
+	MGA_WRITE(MGA_IEN, MGA_VLINEIEN | MGA_SOFTRAPEN);
+	return 0;
+}
 
-	return ret;
+
+void mga_disable_vblank(struct drm_device *dev, int crtc)
+{
+	if (crtc != 0) {
+		DRM_ERROR("tried to disable vblank on non-existent crtc %d\n",
+			  crtc);
+	}
+
+	/* Do *NOT* disable the vertical refresh interrupt.  MGA doesn't have
+	 * a nice hardware counter that tracks the number of refreshes when
+	 * the interrupt is disabled, and the kernel doesn't know the refresh
+	 * rate to calculate an estimate.
+	 */
+	/* MGA_WRITE(MGA_IEN, MGA_VLINEIEN | MGA_SOFTRAPEN); */
 }
 
-int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence)
+int mga_driver_fence_wait(struct drm_device * dev, unsigned int *sequence)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 	unsigned int cur_fence;
@@ -120,7 +144,7 @@ int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence)
 	return ret;
 }
 
-void mga_driver_irq_preinstall(drm_device_t * dev)
+void mga_driver_irq_preinstall(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 
@@ -130,17 +154,25 @@ void mga_driver_irq_preinstall(drm_device_t * dev)
 	MGA_WRITE(MGA_ICLEAR, ~0);
 }
 
-void mga_driver_irq_postinstall(drm_device_t * dev)
+int mga_driver_irq_postinstall(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
+	int ret;
 
-	DRM_INIT_WAITQUEUE( &dev_priv->fence_queue );
+	ret = drm_vblank_init(dev, 1);
+	if (ret)
+		return ret;
 
-	/* Turn on vertical blank interrupt and soft trap interrupt. */
-	MGA_WRITE(MGA_IEN, MGA_VLINEIEN | MGA_SOFTRAPEN);
+	DRM_INIT_WAITQUEUE(&dev_priv->fence_queue);
+
+	/* Turn on soft trap interrupt.  Vertical blank interrupts are enabled
+	 * in mga_enable_vblank.
+	 */
+	MGA_WRITE(MGA_IEN, MGA_SOFTRAPEN);
+	return 0;
 }
 
-void mga_driver_irq_uninstall(drm_device_t * dev)
+void mga_driver_irq_uninstall(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 	if (!dev_priv)
@@ -148,6 +180,6 @@ void mga_driver_irq_uninstall(drm_device_t * dev)
 
 	/* Disable *all* interrupts */
 	MGA_WRITE(MGA_IEN, 0);
-	
+
 	dev->irq_enabled = 0;
 }
diff --git a/sys/dev/drm/mga_state.c b/sys/dev/drm/mga_state.c
index 951a4c8..4d90197 100644
--- a/sys/dev/drm/mga_state.c
+++ b/sys/dev/drm/mga_state.c
@@ -46,7 +46,7 @@ __FBSDID("$FreeBSD$");
  */
 
 static void mga_emit_clip_rect(drm_mga_private_t * dev_priv,
-			       drm_clip_rect_t * box)
+			       struct drm_clip_rect * box)
 {
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mga_context_regs_t *ctx = &sarea_priv->context_state;
@@ -65,8 +65,7 @@ static void mga_emit_clip_rect(drm_mga_private_t * dev_priv,
 	}
 	DMA_BLOCK(MGA_DMAPAD, 0x00000000,
 		  MGA_CXBNDRY, ((box->x2 - 1) << 16) | box->x1,
-		  MGA_YTOP, box->y1 * pitch,
-		  MGA_YBOT, (box->y2 - 1) * pitch);
+		  MGA_YTOP, box->y1 * pitch, MGA_YBOT, (box->y2 - 1) * pitch);
 
 	ADVANCE_DMA();
 }
@@ -81,18 +80,15 @@ static __inline__ void mga_g200_emit_context(drm_mga_private_t * dev_priv)
 
 	DMA_BLOCK(MGA_DSTORG, ctx->dstorg,
 		  MGA_MACCESS, ctx->maccess,
-		  MGA_PLNWT, ctx->plnwt,
-		  MGA_DWGCTL, ctx->dwgctl);
+		  MGA_PLNWT, ctx->plnwt, MGA_DWGCTL, ctx->dwgctl);
 
 	DMA_BLOCK(MGA_ALPHACTRL, ctx->alphactrl,
 		  MGA_FOGCOL, ctx->fogcolor,
-		  MGA_WFLAG, ctx->wflag,
-		  MGA_ZORG, dev_priv->depth_offset);
+		  MGA_WFLAG, ctx->wflag, MGA_ZORG, dev_priv->depth_offset);
 
 	DMA_BLOCK(MGA_FCOL, ctx->fcol,
 		  MGA_DMAPAD, 0x00000000,
-		  MGA_DMAPAD, 0x00000000,
-		  MGA_DMAPAD, 0x00000000);
+		  MGA_DMAPAD, 0x00000000, MGA_DMAPAD, 0x00000000);
 
 	ADVANCE_DMA();
 }
@@ -165,8 +161,8 @@ static __inline__ void mga_g400_emit_tex0(drm_mga_private_t * dev_priv)
 	drm_mga_texture_regs_t *tex = &sarea_priv->tex_state[0];
 	DMA_LOCALS;
 
-/*  	printk("mga_g400_emit_tex0 %x %x %x\n", tex->texorg, */
-/*  	       tex->texctl, tex->texctl2); */
+/*	printk("mga_g400_emit_tex0 %x %x %x\n", tex->texorg, */
+/*	       tex->texctl, tex->texctl2); */
 
 	BEGIN_DMA(6);
 
@@ -209,8 +205,8 @@ static __inline__ void mga_g400_emit_tex1(drm_mga_private_t * dev_priv)
 	drm_mga_texture_regs_t *tex = &sarea_priv->tex_state[1];
 	DMA_LOCALS;
 
-/*  	printk("mga_g400_emit_tex1 %x %x %x\n", tex->texorg,  */
-/*  	       tex->texctl, tex->texctl2); */
+/*	printk("mga_g400_emit_tex1 %x %x %x\n", tex->texorg,  */
+/*	       tex->texctl, tex->texctl2); */
 
 	BEGIN_DMA(5);
 
@@ -279,7 +275,7 @@ static __inline__ void mga_g400_emit_pipe(drm_mga_private_t * dev_priv)
 	unsigned int pipe = sarea_priv->warp_pipe;
 	DMA_LOCALS;
 
-/*  	printk("mga_g400_emit_pipe %x\n", pipe); */
+/*	printk("mga_g400_emit_pipe %x\n", pipe); */
 
 	BEGIN_DMA(10);
 
@@ -419,7 +415,7 @@ static int mga_verify_context(drm_mga_private_t * dev_priv)
 			  ctx->dstorg, dev_priv->front_offset,
 			  dev_priv->back_offset);
 		ctx->dstorg = 0;
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -438,7 +434,7 @@ static int mga_verify_tex(drm_mga_private_t * dev_priv, int unit)
 	if (org == (MGA_TEXORGMAP_SYSMEM | MGA_TEXORGACC_PCI)) {
 		DRM_ERROR("*** bad TEXORG: 0x%x, unit %d\n", tex->texorg, unit);
 		tex->texorg = 0;
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -480,13 +476,13 @@ static int mga_verify_iload(drm_mga_private_t * dev_priv,
 	    dstorg + length > (dev_priv->texture_offset +
 			       dev_priv->texture_size)) {
 		DRM_ERROR("*** bad iload DSTORG: 0x%x\n", dstorg);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (length & MGA_ILOAD_MASK) {
 		DRM_ERROR("*** bad iload length: 0x%x\n",
 			  length & MGA_ILOAD_MASK);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -498,7 +494,7 @@ static int mga_verify_blit(drm_mga_private_t * dev_priv,
 	if ((srcorg & 0x3) == (MGA_SRCACC_PCI | MGA_SRCMAP_SYSMEM) ||
 	    (dstorg & 0x3) == (MGA_SRCACC_PCI | MGA_SRCMAP_SYSMEM)) {
 		DRM_ERROR("*** bad blit: src=0x%x dst=0x%x\n", srcorg, dstorg);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	return 0;
 }
@@ -507,12 +503,12 @@ static int mga_verify_blit(drm_mga_private_t * dev_priv,
  *
  */
 
-static void mga_dma_dispatch_clear(drm_device_t * dev, drm_mga_clear_t * clear)
+static void mga_dma_dispatch_clear(struct drm_device * dev, drm_mga_clear_t * clear)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mga_context_regs_t *ctx = &sarea_priv->context_state;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	int nbox = sarea_priv->nbox;
 	int i;
 	DMA_LOCALS;
@@ -528,7 +524,7 @@ static void mga_dma_dispatch_clear(drm_device_t * dev, drm_mga_clear_t * clear)
 	ADVANCE_DMA();
 
 	for (i = 0; i < nbox; i++) {
-		drm_clip_rect_t *box = &pbox[i];
+		struct drm_clip_rect *box = &pbox[i];
 		u32 height = box->y2 - box->y1;
 
 		DRM_DEBUG("   from=%d,%d to=%d,%d\n",
@@ -597,12 +593,12 @@ static void mga_dma_dispatch_clear(drm_device_t * dev, drm_mga_clear_t * clear)
 	FLUSH_DMA();
 }
 
-static void mga_dma_dispatch_swap(drm_device_t * dev)
+static void mga_dma_dispatch_swap(struct drm_device * dev)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mga_context_regs_t *ctx = &sarea_priv->context_state;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	int nbox = sarea_priv->nbox;
 	int i;
 	DMA_LOCALS;
@@ -629,7 +625,7 @@ static void mga_dma_dispatch_swap(drm_device_t * dev)
 		  MGA_DWGCTL, MGA_DWGCTL_COPY);
 
 	for (i = 0; i < nbox; i++) {
-		drm_clip_rect_t *box = &pbox[i];
+		struct drm_clip_rect *box = &pbox[i];
 		u32 height = box->y2 - box->y1;
 		u32 start = box->y1 * dev_priv->front_pitch;
 
@@ -651,10 +647,10 @@ static void mga_dma_dispatch_swap(drm_device_t * dev)
 
 	FLUSH_DMA();
 
-	DRM_DEBUG("%s... done.\n", __FUNCTION__);
+	DRM_DEBUG("... done.\n");
 }
 
-static void mga_dma_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
+static void mga_dma_dispatch_vertex(struct drm_device * dev, struct drm_buf * buf)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_buf_priv_t *buf_priv = buf->dev_private;
@@ -663,7 +659,7 @@ static void mga_dma_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 	u32 length = (u32) buf->used;
 	int i = 0;
 	DMA_LOCALS;
-	DRM_DEBUG("vertex: buf=%d used=%d\n", buf->idx, buf->used);
+	DRM_DEBUG("buf=%d used=%d\n", buf->idx, buf->used);
 
 	if (buf->used) {
 		buf_priv->dispatched = 1;
@@ -701,7 +697,7 @@ static void mga_dma_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 	FLUSH_DMA();
 }
 
-static void mga_dma_dispatch_indices(drm_device_t * dev, drm_buf_t * buf,
+static void mga_dma_dispatch_indices(struct drm_device * dev, struct drm_buf * buf,
 				     unsigned int start, unsigned int end)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
@@ -710,7 +706,7 @@ static void mga_dma_dispatch_indices(drm_device_t * dev, drm_buf_t * buf,
 	u32 address = (u32) buf->bus_address;
 	int i = 0;
 	DMA_LOCALS;
-	DRM_DEBUG("indices: buf=%d start=%d end=%d\n", buf->idx, start, end);
+	DRM_DEBUG("buf=%d start=%d end=%d\n", buf->idx, start, end);
 
 	if (start != end) {
 		buf_priv->dispatched = 1;
@@ -750,7 +746,7 @@ static void mga_dma_dispatch_indices(drm_device_t * dev, drm_buf_t * buf,
 /* This copies a 64 byte aligned agp region to the frambuffer with a
  * standard blit, the ioctl needs to do checking.
  */
-static void mga_dma_dispatch_iload(drm_device_t * dev, drm_buf_t * buf,
+static void mga_dma_dispatch_iload(struct drm_device * dev, struct drm_buf * buf,
 				   unsigned int dstorg, unsigned int length)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
@@ -803,12 +799,12 @@ static void mga_dma_dispatch_iload(drm_device_t * dev, drm_buf_t * buf,
 	FLUSH_DMA();
 }
 
-static void mga_dma_dispatch_blit(drm_device_t * dev, drm_mga_blit_t * blit)
+static void mga_dma_dispatch_blit(struct drm_device * dev, drm_mga_blit_t * blit)
 {
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_mga_context_regs_t *ctx = &sarea_priv->context_state;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	int nbox = sarea_priv->nbox;
 	u32 scandir = 0, i;
 	DMA_LOCALS;
@@ -868,24 +864,20 @@ static void mga_dma_dispatch_blit(drm_device_t * dev, drm_mga_blit_t * blit)
  *
  */
 
-static int mga_dma_clear(DRM_IOCTL_ARGS)
+static int mga_dma_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_mga_clear_t clear;
-
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	drm_mga_clear_t *clear = data;
 
-	DRM_COPY_FROM_USER_IOCTL(clear, (drm_mga_clear_t __user *) data,
-				 sizeof(clear));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (sarea_priv->nbox > MGA_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MGA_NR_SAREA_CLIPRECTS;
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
 
-	mga_dma_dispatch_clear(dev, &clear);
+	mga_dma_dispatch_clear(dev, clear);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -894,13 +886,12 @@ static int mga_dma_clear(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int mga_dma_swap(DRM_IOCTL_ARGS)
+static int mga_dma_swap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (sarea_priv->nbox > MGA_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MGA_NR_SAREA_CLIPRECTS;
@@ -916,37 +907,32 @@ static int mga_dma_swap(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int mga_dma_vertex(DRM_IOCTL_ARGS)
+static int mga_dma_vertex(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_mga_buf_priv_t *buf_priv;
-	drm_mga_vertex_t vertex;
+	drm_mga_vertex_t *vertex = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_COPY_FROM_USER_IOCTL(vertex,
-				 (drm_mga_vertex_t __user *) data,
-				 sizeof(vertex));
-
-	if (vertex.idx < 0 || vertex.idx > dma->buf_count)
-		return DRM_ERR(EINVAL);
-	buf = dma->buflist[vertex.idx];
+	if (vertex->idx < 0 || vertex->idx > dma->buf_count)
+		return -EINVAL;
+	buf = dma->buflist[vertex->idx];
 	buf_priv = buf->dev_private;
 
-	buf->used = vertex.used;
-	buf_priv->discard = vertex.discard;
+	buf->used = vertex->used;
+	buf_priv->discard = vertex->discard;
 
 	if (!mga_verify_state(dev_priv)) {
-		if (vertex.discard) {
+		if (vertex->discard) {
 			if (buf_priv->dispatched == 1)
 				AGE_BUFFER(buf_priv);
 			buf_priv->dispatched = 0;
 			mga_freelist_put(dev, buf);
 		}
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
@@ -956,82 +942,73 @@ static int mga_dma_vertex(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int mga_dma_indices(DRM_IOCTL_ARGS)
+static int mga_dma_indices(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_mga_buf_priv_t *buf_priv;
-	drm_mga_indices_t indices;
-
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	drm_mga_indices_t *indices = data;
 
-	DRM_COPY_FROM_USER_IOCTL(indices,
-				 (drm_mga_indices_t __user *) data,
-				 sizeof(indices));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (indices.idx < 0 || indices.idx > dma->buf_count)
-		return DRM_ERR(EINVAL);
+	if (indices->idx < 0 || indices->idx > dma->buf_count)
+		return -EINVAL;
 
-	buf = dma->buflist[indices.idx];
+	buf = dma->buflist[indices->idx];
 	buf_priv = buf->dev_private;
 
-	buf_priv->discard = indices.discard;
+	buf_priv->discard = indices->discard;
 
 	if (!mga_verify_state(dev_priv)) {
-		if (indices.discard) {
+		if (indices->discard) {
 			if (buf_priv->dispatched == 1)
 				AGE_BUFFER(buf_priv);
 			buf_priv->dispatched = 0;
 			mga_freelist_put(dev, buf);
 		}
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
 
-	mga_dma_dispatch_indices(dev, buf, indices.start, indices.end);
+	mga_dma_dispatch_indices(dev, buf, indices->start, indices->end);
 
 	return 0;
 }
 
-static int mga_dma_iload(DRM_IOCTL_ARGS)
+static int mga_dma_iload(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	drm_mga_buf_priv_t *buf_priv;
-	drm_mga_iload_t iload;
+	drm_mga_iload_t *iload = data;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(iload, (drm_mga_iload_t __user *) data,
-				 sizeof(iload));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 #if 0
 	if (mga_do_wait_for_idle(dev_priv) < 0) {
 		if (MGA_DMA_DEBUG)
-			DRM_INFO("%s: -EBUSY\n", __FUNCTION__);
-		return DRM_ERR(EBUSY);
+			DRM_INFO("-EBUSY\n");
+		return -EBUSY;
 	}
 #endif
-	if (iload.idx < 0 || iload.idx > dma->buf_count)
-		return DRM_ERR(EINVAL);
+	if (iload->idx < 0 || iload->idx > dma->buf_count)
+		return -EINVAL;
 
-	buf = dma->buflist[iload.idx];
+	buf = dma->buflist[iload->idx];
 	buf_priv = buf->dev_private;
 
-	if (mga_verify_iload(dev_priv, iload.dstorg, iload.length)) {
+	if (mga_verify_iload(dev_priv, iload->dstorg, iload->length)) {
 		mga_freelist_put(dev, buf);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
 
-	mga_dma_dispatch_iload(dev, buf, iload.dstorg, iload.length);
+	mga_dma_dispatch_iload(dev, buf, iload->dstorg, iload->length);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -1040,28 +1017,24 @@ static int mga_dma_iload(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int mga_dma_blit(DRM_IOCTL_ARGS)
+static int mga_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_mga_blit_t blit;
+	drm_mga_blit_t *blit = data;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(blit, (drm_mga_blit_t __user *) data,
-				 sizeof(blit));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (sarea_priv->nbox > MGA_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = MGA_NR_SAREA_CLIPRECTS;
 
-	if (mga_verify_blit(dev_priv, blit.srcorg, blit.dstorg))
-		return DRM_ERR(EINVAL);
+	if (mga_verify_blit(dev_priv, blit->srcorg, blit->dstorg))
+		return -EINVAL;
 
 	WRAP_TEST_WITH_RETURN(dev_priv);
 
-	mga_dma_dispatch_blit(dev, &blit);
+	mga_dma_dispatch_blit(dev, blit);
 
 	/* Make sure we restore the 3D state next time.
 	 */
@@ -1070,24 +1043,20 @@ static int mga_dma_blit(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int mga_getparam(DRM_IOCTL_ARGS)
+static int mga_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	drm_mga_getparam_t param;
+	drm_mga_getparam_t *param = data;
 	int value;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_mga_getparam_t __user *) data,
-				 sizeof(param));
-
 	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	switch (param.param) {
+	switch (param->param) {
 	case MGA_PARAM_IRQ_NR:
 		value = dev->irq;
 		break;
@@ -1095,36 +1064,35 @@ static int mga_getparam(DRM_IOCTL_ARGS)
 		value = dev_priv->chipset;
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
+	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-static int mga_set_fence(DRM_IOCTL_ARGS)
+static int mga_set_fence(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	u32 temp;
+	u32 *fence = data;
 	DMA_LOCALS;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
 	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	/* I would normal do this assignment in the declaration of temp,
+	/* I would normal do this assignment in the declaration of fence,
 	 * but dev_priv may be NULL.
 	 */
 
-	temp = dev_priv->next_fence_to_post;
+	*fence = dev_priv->next_fence_to_post;
 	dev_priv->next_fence_to_post++;
 
 	BEGIN_DMA(1);
@@ -1134,47 +1102,40 @@ static int mga_set_fence(DRM_IOCTL_ARGS)
 		  MGA_SOFTRAP, 0x00000000);
 	ADVANCE_DMA();
 
-	DRM_COPY_TO_USER_IOCTL((u32 __user *)data, temp, sizeof(u32));
-
 	return 0;
 }
 
-static int mga_wait_fence(DRM_IOCTL_ARGS)
+static int mga_wait_fence(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_mga_private_t *dev_priv = dev->dev_private;
-	u32 fence;
+	u32 *fence = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(fence, (u32 __user *) data, sizeof(u32));
-
 	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	mga_driver_fence_wait(dev, & fence);
-
-	DRM_COPY_TO_USER_IOCTL((u32 __user *)data, fence, sizeof(u32));
+	mga_driver_fence_wait(dev, fence);
 
 	return 0;
 }
 
-drm_ioctl_desc_t mga_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_MGA_INIT)] = {mga_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_MGA_FLUSH)] = {mga_dma_flush, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_RESET)] = {mga_dma_reset, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_SWAP)] = {mga_dma_swap, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_CLEAR)] = {mga_dma_clear, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_VERTEX)] = {mga_dma_vertex, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_INDICES)] = {mga_dma_indices, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_ILOAD)] = {mga_dma_iload, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_BLIT)] = {mga_dma_blit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_GETPARAM)] = {mga_getparam, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_SET_FENCE)] = {mga_set_fence, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_WAIT_FENCE)] = {mga_wait_fence, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_MGA_DMA_BOOTSTRAP)] = {mga_dma_bootstrap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+struct drm_ioctl_desc mga_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_MGA_INIT, mga_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_MGA_FLUSH, mga_dma_flush, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_RESET, mga_dma_reset, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_SWAP, mga_dma_swap, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_CLEAR, mga_dma_clear, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_VERTEX, mga_dma_vertex, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_INDICES, mga_dma_indices, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_ILOAD, mga_dma_iload, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_BLIT, mga_dma_blit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_GETPARAM, mga_getparam, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_SET_FENCE, mga_set_fence, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_WAIT_FENCE, mga_wait_fence, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_MGA_DMA_BOOTSTRAP, mga_dma_bootstrap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
 };
 
diff --git a/sys/dev/drm/mga_warp.c b/sys/dev/drm/mga_warp.c
index 2339739..dd3e734 100644
--- a/sys/dev/drm/mga_warp.c
+++ b/sys/dev/drm/mga_warp.c
@@ -149,7 +149,7 @@ int mga_warp_install_microcode(drm_mga_private_t * dev_priv)
 	if (size > dev_priv->warp->size) {
 		DRM_ERROR("microcode too large! (%u > %lu)\n",
 			  size, dev_priv->warp->size);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	switch (dev_priv->chipset) {
@@ -159,7 +159,7 @@ int mga_warp_install_microcode(drm_mga_private_t * dev_priv)
 	case MGA_CARD_TYPE_G200:
 		return mga_warp_install_g200_microcode(dev_priv);
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 }
 
@@ -185,7 +185,7 @@ int mga_warp_init(drm_mga_private_t * dev_priv)
 		MGA_WRITE(MGA_WVRTXSZ, 7);
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	MGA_WRITE(MGA_WMISC, (MGA_WUCODECACHE_ENABLE |
@@ -194,7 +194,7 @@ int mga_warp_init(drm_mga_private_t * dev_priv)
 	if (wmisc != WMISC_EXPECTED) {
 		DRM_ERROR("WARP engine config failed! 0x%x != 0x%x\n",
 			  wmisc, WMISC_EXPECTED);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
diff --git a/sys/dev/drm/r128_cce.c b/sys/dev/drm/r128_cce.c
index 9ac647e..c799bd1 100644
--- a/sys/dev/drm/r128_cce.c
+++ b/sys/dev/drm/r128_cce.c
@@ -84,7 +84,7 @@ static u32 r128_cce_microcode[] = {
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 };
 
-static int R128_READ_PLL(drm_device_t * dev, int addr)
+static int R128_READ_PLL(struct drm_device * dev, int addr)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 
@@ -132,7 +132,7 @@ static int r128_do_pixcache_flush(drm_r128_private_t * dev_priv)
 #if R128_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int r128_do_wait_for_fifo(drm_r128_private_t * dev_priv, int entries)
@@ -149,7 +149,7 @@ static int r128_do_wait_for_fifo(drm_r128_private_t * dev_priv, int entries)
 #if R128_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int r128_do_wait_for_idle(drm_r128_private_t * dev_priv)
@@ -171,7 +171,7 @@ static int r128_do_wait_for_idle(drm_r128_private_t * dev_priv)
 #if R128_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /* ================================================================
@@ -230,7 +230,7 @@ int r128_do_cce_idle(drm_r128_private_t * dev_priv)
 	DRM_ERROR("failed!\n");
 	r128_status(dev_priv);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /* Start the Concurrent Command Engine.
@@ -274,7 +274,7 @@ static void r128_do_cce_stop(drm_r128_private_t * dev_priv)
 
 /* Reset the engine.  This will stop the CCE if it is running.
  */
-static int r128_do_engine_reset(drm_device_t * dev)
+static int r128_do_engine_reset(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	u32 clock_cntl_index, mclk_cntl, gen_reset_cntl;
@@ -311,7 +311,7 @@ static int r128_do_engine_reset(drm_device_t * dev)
 	return 0;
 }
 
-static void r128_cce_init_ring_buffer(drm_device_t * dev,
+static void r128_cce_init_ring_buffer(struct drm_device * dev,
 				      drm_r128_private_t * dev_priv)
 {
 	u32 ring_start;
@@ -327,7 +327,7 @@ static void r128_cce_init_ring_buffer(drm_device_t * dev,
 		ring_start = dev_priv->cce_ring->offset - dev->agp->base;
 	else
 #endif
-		ring_start = dev_priv->cce_ring->offset - 
+		ring_start = dev_priv->cce_ring->offset -
 				(unsigned long)dev->sg->virtual;
 
 	R128_WRITE(R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET);
@@ -350,7 +350,7 @@ static void r128_cce_init_ring_buffer(drm_device_t * dev,
 	R128_WRITE(R128_BUS_CNTL, tmp);
 }
 
-static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
+static int r128_do_init_cce(struct drm_device * dev, drm_r128_init_t * init)
 {
 	drm_r128_private_t *dev_priv;
 
@@ -358,7 +358,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 
 	dev_priv = drm_alloc(sizeof(drm_r128_private_t), DRM_MEM_DRIVER);
 	if (dev_priv == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv, 0, sizeof(drm_r128_private_t));
 
@@ -368,7 +368,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		DRM_ERROR("PCI GART memory not allocated!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->usec_timeout = init->usec_timeout;
@@ -377,7 +377,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		DRM_DEBUG("TIMEOUT problem!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->cce_mode = init->cce_mode;
@@ -397,7 +397,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		DRM_DEBUG("Bad cce_mode!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	switch (init->cce_mode) {
@@ -459,13 +459,12 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 	dev_priv->span_pitch_offset_c = (((dev_priv->depth_pitch / 8) << 21) |
 					 (dev_priv->span_offset >> 5));
 
-	DRM_GETSAREA();
-
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("could not find sarea!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
@@ -473,21 +472,21 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		DRM_ERROR("could not find mmio region!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev_priv->cce_ring = drm_core_findmap(dev, init->ring_offset);
 	if (!dev_priv->cce_ring) {
 		DRM_ERROR("could not find cce ring region!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
 	if (!dev_priv->ring_rptr) {
 		DRM_ERROR("could not find ring read pointer!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev->agp_buffer_token = init->buffers_offset;
 	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
@@ -495,7 +494,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		DRM_ERROR("could not find dma buffer region!\n");
 		dev->dev_private = (void *)dev_priv;
 		r128_do_cleanup_cce(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (!dev_priv->is_pci) {
@@ -505,7 +504,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 			DRM_ERROR("could not find agp texture region!\n");
 			dev->dev_private = (void *)dev_priv;
 			r128_do_cleanup_cce(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -524,7 +523,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 			DRM_ERROR("Could not ioremap agp regions!\n");
 			dev->dev_private = (void *)dev_priv;
 			r128_do_cleanup_cce(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 	} else
 #endif
@@ -562,15 +561,17 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 #if __OS_HAS_AGP
 	if (dev_priv->is_pci) {
 #endif
+		dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 		dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
+		dev_priv->gart_info.table_size = R128_PCIGART_TABLE_SIZE;
 		dev_priv->gart_info.addr = NULL;
 		dev_priv->gart_info.bus_addr = 0;
-		dev_priv->gart_info.is_pcie = 0;
+		dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 		if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 			DRM_ERROR("failed to init PCI GART!\n");
 			dev->dev_private = (void *)dev_priv;
 			r128_do_cleanup_cce(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 		R128_WRITE(R128_PCI_GART_PAGE, dev_priv->gart_info.bus_addr);
 #if __OS_HAS_AGP
@@ -587,7 +588,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 	return 0;
 }
 
-int r128_do_cleanup_cce(drm_device_t * dev)
+int r128_do_cleanup_cce(struct drm_device * dev)
 {
 
 	/* Make sure interrupts are disabled here because the uninstall ioctl
@@ -626,38 +627,33 @@ int r128_do_cleanup_cce(drm_device_t * dev)
 	return 0;
 }
 
-int r128_cce_init(DRM_IOCTL_ARGS)
+int r128_cce_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_r128_init_t init;
+	drm_r128_init_t *init = data;
 
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_r128_init_t __user *) data,
-				 sizeof(init));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	switch (init.func) {
+	switch (init->func) {
 	case R128_INIT_CCE:
-		return r128_do_init_cce(dev, &init);
+		return r128_do_init_cce(dev, init);
 	case R128_CLEANUP_CCE:
 		return r128_do_cleanup_cce(dev);
 	}
 
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
-int r128_cce_start(DRM_IOCTL_ARGS)
+int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4) {
-		DRM_DEBUG("%s while CCE running\n", __FUNCTION__);
+		DRM_DEBUG("while CCE running\n");
 		return 0;
 	}
 
@@ -669,30 +665,26 @@ int r128_cce_start(DRM_IOCTL_ARGS)
 /* Stop the CCE.  The engine must have been idled before calling this
  * routine.
  */
-int r128_cce_stop(DRM_IOCTL_ARGS)
+int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_r128_cce_stop_t stop;
+	drm_r128_cce_stop_t *stop = data;
 	int ret;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(stop, (drm_r128_cce_stop_t __user *) data,
-				 sizeof(stop));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Flush any pending CCE commands.  This ensures any outstanding
 	 * commands are exectuted by the engine before we turn it off.
 	 */
-	if (stop.flush) {
+	if (stop->flush) {
 		r128_do_cce_flush(dev_priv);
 	}
 
 	/* If we fail to make the engine go idle, we return an error
 	 * code so that the DRM ioctl wrapper can try again.
 	 */
-	if (stop.idle) {
+	if (stop->idle) {
 		ret = r128_do_cce_idle(dev_priv);
 		if (ret)
 			return ret;
@@ -712,17 +704,16 @@ int r128_cce_stop(DRM_IOCTL_ARGS)
 
 /* Just reset the CCE ring.  Called as part of an X Server engine reset.
  */
-int r128_cce_reset(DRM_IOCTL_ARGS)
+int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_DEBUG("%s called before init done\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_DEBUG("called before init done\n");
+		return -EINVAL;
 	}
 
 	r128_do_cce_reset(dev_priv);
@@ -733,13 +724,12 @@ int r128_cce_reset(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-int r128_cce_idle(DRM_IOCTL_ARGS)
+int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (dev_priv->cce_running) {
 		r128_do_cce_flush(dev_priv);
@@ -748,19 +738,18 @@ int r128_cce_idle(DRM_IOCTL_ARGS)
 	return r128_do_cce_idle(dev_priv);
 }
 
-int r128_engine_reset(DRM_IOCTL_ARGS)
+int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return r128_do_engine_reset(dev);
 }
 
-int r128_fullscreen(DRM_IOCTL_ARGS)
+int r128_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
 /* ================================================================
@@ -770,18 +759,18 @@ int r128_fullscreen(DRM_IOCTL_ARGS)
 #define R128_BUFFER_FREE	0
 
 #if 0
-static int r128_freelist_init(drm_device_t * dev)
+static int r128_freelist_init(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	drm_r128_buf_priv_t *buf_priv;
 	drm_r128_freelist_t *entry;
 	int i;
 
 	dev_priv->head = drm_alloc(sizeof(drm_r128_freelist_t), DRM_MEM_DRIVER);
 	if (dev_priv->head == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv->head, 0, sizeof(drm_r128_freelist_t));
 	dev_priv->head->age = R128_BUFFER_USED;
@@ -792,7 +781,7 @@ static int r128_freelist_init(drm_device_t * dev)
 
 		entry = drm_alloc(sizeof(drm_r128_freelist_t), DRM_MEM_DRIVER);
 		if (!entry)
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 
 		entry->age = R128_BUFFER_FREE;
 		entry->buf = buf;
@@ -816,12 +805,12 @@ static int r128_freelist_init(drm_device_t * dev)
 }
 #endif
 
-static drm_buf_t *r128_freelist_get(drm_device_t * dev)
+static struct drm_buf *r128_freelist_get(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_buf_priv_t *buf_priv;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	int i, t;
 
 	/* FIXME: Optimize -- use freelist code */
@@ -829,7 +818,7 @@ static drm_buf_t *r128_freelist_get(drm_device_t * dev)
 	for (i = 0; i < dma->buf_count; i++) {
 		buf = dma->buflist[i];
 		buf_priv = buf->dev_private;
-		if (buf->filp == 0)
+		if (buf->file_priv == 0)
 			return buf;
 	}
 
@@ -854,13 +843,13 @@ static drm_buf_t *r128_freelist_get(drm_device_t * dev)
 	return NULL;
 }
 
-void r128_freelist_reset(drm_device_t * dev)
+void r128_freelist_reset(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	int i;
 
 	for (i = 0; i < dma->buf_count; i++) {
-		drm_buf_t *buf = dma->buflist[i];
+		struct drm_buf *buf = dma->buflist[i];
 		drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 		buf_priv->age = 0;
 	}
@@ -884,68 +873,64 @@ int r128_wait_ring(drm_r128_private_t * dev_priv, int n)
 
 	/* FIXME: This is being ignored... */
 	DRM_ERROR("failed!\n");
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
-static int r128_cce_get_buffers(DRMFILE filp, drm_device_t * dev, drm_dma_t * d)
+static int r128_cce_get_buffers(struct drm_device * dev,
+				struct drm_file *file_priv,
+				struct drm_dma * d)
 {
 	int i;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 
 	for (i = d->granted_count; i < d->request_count; i++) {
 		buf = r128_freelist_get(dev);
 		if (!buf)
-			return DRM_ERR(EAGAIN);
+			return -EAGAIN;
 
-		buf->filp = filp;
+		buf->file_priv = file_priv;
 
 		if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
 				     sizeof(buf->idx)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
 				     sizeof(buf->total)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 
 		d->granted_count++;
 	}
 	return 0;
 }
 
-int r128_cce_buffers(DRM_IOCTL_ARGS)
+int r128_cce_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	int ret = 0;
-	drm_dma_t __user *argp = (void __user *)data;
-	drm_dma_t d;
+	struct drm_dma *d = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(d, argp, sizeof(d));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Please don't send us buffers.
 	 */
-	if (d.send_count != 0) {
+	if (d->send_count != 0) {
 		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d.send_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->send_count);
+		return -EINVAL;
 	}
 
 	/* We'll send you buffers.
 	 */
-	if (d.request_count < 0 || d.request_count > dma->buf_count) {
+	if (d->request_count < 0 || d->request_count > dma->buf_count) {
 		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d.request_count, dma->buf_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->request_count, dma->buf_count);
+		return -EINVAL;
 	}
 
-	d.granted_count = 0;
+	d->granted_count = 0;
 
-	if (d.request_count) {
-		ret = r128_cce_get_buffers(filp, dev, &d);
+	if (d->request_count) {
+		ret = r128_cce_get_buffers(dev, file_priv, d);
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, d, sizeof(d));
-
 	return ret;
 }
diff --git a/sys/dev/drm/r128_drm.h b/sys/dev/drm/r128_drm.h
index c9ad817..3afbefa 100644
--- a/sys/dev/drm/r128_drm.h
+++ b/sys/dev/drm/r128_drm.h
@@ -156,7 +156,7 @@ typedef struct drm_r128_sarea {
 
 	/* The current cliprects, or a subset thereof.
 	 */
-	drm_clip_rect_t boxes[R128_NR_SAREA_CLIPRECTS];
+	struct drm_clip_rect boxes[R128_NR_SAREA_CLIPRECTS];
 	unsigned int nbox;
 
 	/* Counters for client-side throttling of rendering clients.
@@ -164,7 +164,7 @@ typedef struct drm_r128_sarea {
 	unsigned int last_frame;
 	unsigned int last_dispatch;
 
-	drm_tex_region_t tex_list[R128_NR_TEX_HEAPS][R128_NR_TEX_REGIONS + 1];
+	struct drm_tex_region tex_list[R128_NR_TEX_HEAPS][R128_NR_TEX_REGIONS + 1];
 	unsigned int tex_age[R128_NR_TEX_HEAPS];
 	int ctx_owner;
 	int pfAllowPageFlip;	/* number of 3d windows (0,1,2 or more) */
@@ -225,11 +225,7 @@ typedef struct drm_r128_init {
 		R128_INIT_CCE = 0x01,
 		R128_CLEANUP_CCE = 0x02
 	} func;
-#if CONFIG_XFREE86_VERSION < XFREE86_VERSION(4,1,0,0)
-	int sarea_priv_offset;
-#else
 	unsigned long sarea_priv_offset;
-#endif
 	int is_pci;
 	int cce_mode;
 	int cce_secure;
@@ -243,21 +239,12 @@ typedef struct drm_r128_init {
 	unsigned int depth_offset, depth_pitch;
 	unsigned int span_offset;
 
-#if CONFIG_XFREE86_VERSION < XFREE86_VERSION(4,1,0,0)
-	unsigned int fb_offset;
-	unsigned int mmio_offset;
-	unsigned int ring_offset;
-	unsigned int ring_rptr_offset;
-	unsigned int buffers_offset;
-	unsigned int agp_textures_offset;
-#else
 	unsigned long fb_offset;
 	unsigned long mmio_offset;
 	unsigned long ring_offset;
 	unsigned long ring_rptr_offset;
 	unsigned long buffers_offset;
 	unsigned long agp_textures_offset;
-#endif
 } drm_r128_init_t;
 
 typedef struct drm_r128_cce_stop {
@@ -267,15 +254,10 @@ typedef struct drm_r128_cce_stop {
 
 typedef struct drm_r128_clear {
 	unsigned int flags;
-#if CONFIG_XFREE86_VERSION < XFREE86_VERSION(4,1,0,0)
-	int x, y, w, h;
-#endif
 	unsigned int clear_color;
 	unsigned int clear_depth;
-#if CONFIG_XFREE86_VERSION >= XFREE86_VERSION(4,1,0,0)
 	unsigned int color_mask;
 	unsigned int depth_mask;
-#endif
 } drm_r128_clear_t;
 
 typedef struct drm_r128_vertex {
diff --git a/sys/dev/drm/r128_drv.c b/sys/dev/drm/r128_drv.c
index 71b4259..a290d22 100644
--- a/sys/dev/drm/r128_drv.c
+++ b/sys/dev/drm/r128_drv.c
@@ -45,12 +45,14 @@ static drm_pci_id_list_t r128_pciidlist[] = {
 	r128_PCI_IDS
 };
 
-static void r128_configure(drm_device_t *dev)
+static void r128_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= sizeof(drm_r128_buf_priv_t);
 	dev->driver.preclose		= r128_driver_preclose;
 	dev->driver.lastclose		= r128_driver_lastclose;
-	dev->driver.vblank_wait		= r128_driver_vblank_wait;
+	dev->driver.get_vblank_counter	= r128_get_vblank_counter;
+	dev->driver.enable_vblank	= r128_enable_vblank;
+	dev->driver.disable_vblank	= r128_disable_vblank;
 	dev->driver.irq_preinstall	= r128_driver_irq_preinstall;
 	dev->driver.irq_postinstall	= r128_driver_irq_postinstall;
 	dev->driver.irq_uninstall	= r128_driver_irq_uninstall;
@@ -86,9 +88,9 @@ r128_probe(device_t dev)
 static int
 r128_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	r128_configure(dev);
 	return drm_attach(nbdev, r128_pciidlist);
 }
@@ -105,7 +107,7 @@ static device_method_t r128_methods[] = {
 static driver_t r128_driver = {
 	"drm",
 	r128_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
@@ -120,7 +122,7 @@ MODULE_DEPEND(r128, drm, 1, 1, 1);
 #ifdef _LKM
 CFDRIVER_DECL(r128, DV_TTY, NULL);
 #else
-CFATTACH_DECL(r128, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
-    drm_activate);
+CFATTACH_DECL(r128, sizeof(struct drm_device), drm_probe, drm_attach,
+    drm_detach, drm_activate);
 #endif
 #endif
diff --git a/sys/dev/drm/r128_drv.h b/sys/dev/drm/r128_drv.h
index ba738fa..0c223c6 100644
--- a/sys/dev/drm/r128_drv.h
+++ b/sys/dev/drm/r128_drv.h
@@ -60,7 +60,7 @@ __FBSDID("$FreeBSD$");
 
 typedef struct drm_r128_freelist {
 	unsigned int age;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	struct drm_r128_freelist *next;
 	struct drm_r128_freelist *prev;
 } drm_r128_freelist_t;
@@ -100,6 +100,8 @@ typedef struct drm_r128_private {
 	u32 crtc_offset;
 	u32 crtc_offset_cntl;
 
+	atomic_t vbl_received;
+
 	u32 color_fmt;
 	unsigned int front_offset;
 	unsigned int front_pitch;
@@ -121,7 +123,7 @@ typedef struct drm_r128_private {
 	drm_local_map_t *cce_ring;
 	drm_local_map_t *ring_rptr;
 	drm_local_map_t *agp_textures;
-	drm_ati_pcigart_info gart_info;
+	struct drm_ati_pcigart_info gart_info;
 } drm_r128_private_t;
 
 typedef struct drm_r128_buf_priv {
@@ -132,34 +134,36 @@ typedef struct drm_r128_buf_priv {
 	drm_r128_freelist_t *list_entry;
 } drm_r128_buf_priv_t;
 
-extern drm_ioctl_desc_t r128_ioctls[];
+extern struct drm_ioctl_desc r128_ioctls[];
 extern int r128_max_ioctl;
 
 				/* r128_cce.c */
-extern int r128_cce_init(DRM_IOCTL_ARGS);
-extern int r128_cce_start(DRM_IOCTL_ARGS);
-extern int r128_cce_stop(DRM_IOCTL_ARGS);
-extern int r128_cce_reset(DRM_IOCTL_ARGS);
-extern int r128_cce_idle(DRM_IOCTL_ARGS);
-extern int r128_engine_reset(DRM_IOCTL_ARGS);
-extern int r128_fullscreen(DRM_IOCTL_ARGS);
-extern int r128_cce_buffers(DRM_IOCTL_ARGS);
+extern int r128_cce_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int r128_cce_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv);
 
-extern void r128_freelist_reset(drm_device_t * dev);
+extern void r128_freelist_reset(struct drm_device * dev);
 
 extern int r128_wait_ring(drm_r128_private_t * dev_priv, int n);
 
 extern int r128_do_cce_idle(drm_r128_private_t * dev_priv);
-extern int r128_do_cleanup_cce(drm_device_t * dev);
-
-extern int r128_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence);
+extern int r128_do_cleanup_cce(struct drm_device * dev);
 
+extern int r128_enable_vblank(struct drm_device *dev, int crtc);
+extern void r128_disable_vblank(struct drm_device *dev, int crtc);
+extern u32 r128_get_vblank_counter(struct drm_device *dev, int crtc);
 extern irqreturn_t r128_driver_irq_handler(DRM_IRQ_ARGS);
-extern void r128_driver_irq_preinstall(drm_device_t * dev);
-extern void r128_driver_irq_postinstall(drm_device_t * dev);
-extern void r128_driver_irq_uninstall(drm_device_t * dev);
-extern void r128_driver_lastclose(drm_device_t * dev);
-extern void r128_driver_preclose(drm_device_t * dev, DRMFILE filp);
+extern void r128_driver_irq_preinstall(struct drm_device * dev);
+extern int r128_driver_irq_postinstall(struct drm_device * dev);
+extern void r128_driver_irq_uninstall(struct drm_device * dev);
+extern void r128_driver_lastclose(struct drm_device * dev);
+extern void r128_driver_preclose(struct drm_device * dev,
+				 struct drm_file *file_priv);
 
 extern long r128_compat_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
@@ -386,6 +390,8 @@ extern long r128_compat_ioctl(struct file *filp, unsigned int cmd,
 
 #define R128_PERFORMANCE_BOXES		0
 
+#define R128_PCIGART_TABLE_SIZE         32768
+
 #define R128_READ(reg)		DRM_READ32(  dev_priv->mmio, (reg) )
 #define R128_WRITE(reg,val)	DRM_WRITE32( dev_priv->mmio, (reg), (val) )
 #define R128_READ8(reg)		DRM_READ8(   dev_priv->mmio, (reg) )
@@ -429,7 +435,7 @@ do {									\
 			DRM_UDELAY(1);				\
 		}							\
 		DRM_ERROR( "ring space check failed!\n" );		\
-		return DRM_ERR(EBUSY);				\
+		return -EBUSY;				\
 	}								\
  __ring_space_done:							\
 	;								\
@@ -462,8 +468,7 @@ do {									\
 
 #define BEGIN_RING( n ) do {						\
 	if ( R128_VERBOSE ) {						\
-		DRM_INFO( "BEGIN_RING( %d ) in %s\n",			\
-			   (n), __FUNCTION__ );				\
+		DRM_INFO( "BEGIN_RING( %d )\n", (n));			\
 	}								\
 	if ( dev_priv->ring.space <= (n) * sizeof(u32) ) {		\
 		COMMIT_RING();						\
@@ -493,7 +498,7 @@ do {									\
 			write * sizeof(u32) );				\
 	}								\
 	if (((dev_priv->ring.tail + _nr) & tail_mask) != write) {	\
-		DRM_ERROR( 						\
+		DRM_ERROR(						\
 			"ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n",	\
 			((dev_priv->ring.tail + _nr) & tail_mask),	\
 			write, __LINE__);				\
diff --git a/sys/dev/drm/r128_irq.c b/sys/dev/drm/r128_irq.c
index c3b4129..ec5eea1 100644
--- a/sys/dev/drm/r128_irq.c
+++ b/sys/dev/drm/r128_irq.c
@@ -1,5 +1,4 @@
-/* r128_irq.c -- IRQ handling for radeon -*- linux-c -*-
- */
+/* r128_irq.c -- IRQ handling for radeon -*- linux-c -*- */
 /*-
  * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
  *
@@ -39,9 +38,19 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/r128_drm.h"
 #include "dev/drm/r128_drv.h"
 
+u32 r128_get_vblank_counter(struct drm_device *dev, int crtc)
+{
+	const drm_r128_private_t *dev_priv = dev->dev_private;
+
+	if (crtc != 0)
+		return 0;
+
+	return atomic_read(&dev_priv->vbl_received);
+}
+
 irqreturn_t r128_driver_irq_handler(DRM_IRQ_ARGS)
 {
-	drm_device_t *dev = (drm_device_t *) arg;
+	struct drm_device *dev = (struct drm_device *) arg;
 	drm_r128_private_t *dev_priv = (drm_r128_private_t *) dev->dev_private;
 	int status;
 
@@ -50,33 +59,41 @@ irqreturn_t r128_driver_irq_handler(DRM_IRQ_ARGS)
 	/* VBLANK interrupt */
 	if (status & R128_CRTC_VBLANK_INT) {
 		R128_WRITE(R128_GEN_INT_STATUS, R128_CRTC_VBLANK_INT_AK);
-		atomic_inc(&dev->vbl_received);
-		DRM_WAKEUP(&dev->vbl_queue);
-		drm_vbl_send_signals(dev);
+		atomic_inc(&dev_priv->vbl_received);
+		drm_handle_vblank(dev, 0);
 		return IRQ_HANDLED;
 	}
 	return IRQ_NONE;
 }
 
-int r128_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
+int r128_enable_vblank(struct drm_device *dev, int crtc)
 {
-	unsigned int cur_vblank;
-	int ret = 0;
+	drm_r128_private_t *dev_priv = dev->dev_private;
 
-	/* Assume that the user has missed the current sequence number
-	 * by about a day rather than she wants to wait for years
-	 * using vertical blanks...
-	 */
-	DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-		    (((cur_vblank = atomic_read(&dev->vbl_received))
-		      - *sequence) <= (1 << 23)));
+	if (crtc != 0) {
+		DRM_ERROR("%s:  bad crtc %d\n", __FUNCTION__, crtc);
+		return -EINVAL;
+	}
 
-	*sequence = cur_vblank;
+	R128_WRITE(R128_GEN_INT_CNTL, R128_CRTC_VBLANK_INT_EN);
+	return 0;
+}
+
+void r128_disable_vblank(struct drm_device *dev, int crtc)
+{
+	if (crtc != 0)
+		DRM_ERROR("%s:  bad crtc %d\n", __FUNCTION__, crtc);
 
-	return ret;
+	/*
+	 * FIXME: implement proper interrupt disable by using the vblank
+	 * counter register (if available)
+	 *
+	 * R128_WRITE(R128_GEN_INT_CNTL,
+	 *            R128_READ(R128_GEN_INT_CNTL) & ~R128_CRTC_VBLANK_INT_EN);
+	 */
 }
 
-void r128_driver_irq_preinstall(drm_device_t * dev)
+void r128_driver_irq_preinstall(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = (drm_r128_private_t *) dev->dev_private;
 
@@ -86,15 +103,12 @@ void r128_driver_irq_preinstall(drm_device_t * dev)
 	R128_WRITE(R128_GEN_INT_STATUS, R128_CRTC_VBLANK_INT_AK);
 }
 
-void r128_driver_irq_postinstall(drm_device_t * dev)
+int r128_driver_irq_postinstall(struct drm_device * dev)
 {
-	drm_r128_private_t *dev_priv = (drm_r128_private_t *) dev->dev_private;
-
-	/* Turn on VBL interrupt */
-	R128_WRITE(R128_GEN_INT_CNTL, R128_CRTC_VBLANK_INT_EN);
+	return drm_vblank_init(dev, 1);
 }
 
-void r128_driver_irq_uninstall(drm_device_t * dev)
+void r128_driver_irq_uninstall(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = (drm_r128_private_t *) dev->dev_private;
 	if (!dev_priv)
diff --git a/sys/dev/drm/r128_state.c b/sys/dev/drm/r128_state.c
index 62f17e7..6aef11c 100644
--- a/sys/dev/drm/r128_state.c
+++ b/sys/dev/drm/r128_state.c
@@ -41,11 +41,11 @@ __FBSDID("$FreeBSD$");
  */
 
 static void r128_emit_clip_rects(drm_r128_private_t * dev_priv,
-				 drm_clip_rect_t * boxes, int count)
+				 struct drm_clip_rect * boxes, int count)
 {
 	u32 aux_sc_cntl = 0x00000000;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING((count < 3 ? count : 3) * 5 + 2);
 
@@ -88,7 +88,7 @@ static __inline__ void r128_emit_core(drm_r128_private_t * dev_priv)
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(2);
 
@@ -103,7 +103,7 @@ static __inline__ void r128_emit_context(drm_r128_private_t * dev_priv)
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(13);
 
@@ -129,7 +129,7 @@ static __inline__ void r128_emit_setup(drm_r128_private_t * dev_priv)
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(3);
 
@@ -145,7 +145,7 @@ static __inline__ void r128_emit_masks(drm_r128_private_t * dev_priv)
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(5);
 
@@ -164,7 +164,7 @@ static __inline__ void r128_emit_window(drm_r128_private_t * dev_priv)
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_r128_context_regs_t *ctx = &sarea_priv->context_state;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(2);
 
@@ -181,7 +181,7 @@ static __inline__ void r128_emit_tex0(drm_r128_private_t * dev_priv)
 	drm_r128_texture_regs_t *tex = &sarea_priv->tex_state[0];
 	int i;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(7 + R128_MAX_TEXTURE_LEVELS);
 
@@ -207,7 +207,7 @@ static __inline__ void r128_emit_tex1(drm_r128_private_t * dev_priv)
 	drm_r128_texture_regs_t *tex = &sarea_priv->tex_state[1];
 	int i;
 	RING_LOCALS;
-	DRM_DEBUG("    %s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(5 + R128_MAX_TEXTURE_LEVELS);
 
@@ -224,12 +224,12 @@ static __inline__ void r128_emit_tex1(drm_r128_private_t * dev_priv)
 	ADVANCE_RING();
 }
 
-static __inline__ void r128_emit_state(drm_r128_private_t * dev_priv)
+static void r128_emit_state(drm_r128_private_t * dev_priv)
 {
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	unsigned int dirty = sarea_priv->dirty;
 
-	DRM_DEBUG("%s: dirty=0x%08x\n", __FUNCTION__, dirty);
+	DRM_DEBUG("dirty=0x%08x\n", dirty);
 
 	if (dirty & R128_UPLOAD_CORE) {
 		r128_emit_core(dev_priv);
@@ -355,17 +355,17 @@ static void r128_print_dirty(const char *msg, unsigned int flags)
 		 (flags & R128_REQUIRE_QUIESCENCE) ? "quiescence, " : "");
 }
 
-static void r128_cce_dispatch_clear(drm_device_t * dev,
+static void r128_cce_dispatch_clear(struct drm_device * dev,
 				    drm_r128_clear_t * clear)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	unsigned int flags = clear->flags;
 	int i;
 	RING_LOCALS;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	if (dev_priv->page_flipping && dev_priv->current_page == 1) {
 		unsigned int tmp = flags;
@@ -461,15 +461,15 @@ static void r128_cce_dispatch_clear(drm_device_t * dev,
 	}
 }
 
-static void r128_cce_dispatch_swap(drm_device_t * dev)
+static void r128_cce_dispatch_swap(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	int i;
 	RING_LOCALS;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 #if R128_PERFORMANCE_BOXES
 	/* Do some trivial performance monitoring...
@@ -527,12 +527,11 @@ static void r128_cce_dispatch_swap(drm_device_t * dev)
 	ADVANCE_RING();
 }
 
-static void r128_cce_dispatch_flip(drm_device_t * dev)
+static void r128_cce_dispatch_flip(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
-	DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-		  __FUNCTION__,
+	DRM_DEBUG("page=%d pfCurrentPage=%d\n",
 		  dev_priv->current_page, dev_priv->sarea_priv->pfCurrentPage);
 
 #if R128_PERFORMANCE_BOXES
@@ -570,7 +569,7 @@ static void r128_cce_dispatch_flip(drm_device_t * dev)
 	ADVANCE_RING();
 }
 
-static void r128_cce_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
+static void r128_cce_dispatch_vertex(struct drm_device * dev, struct drm_buf * buf)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_buf_priv_t *buf_priv = buf->dev_private;
@@ -640,8 +639,8 @@ static void r128_cce_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 	sarea_priv->nbox = 0;
 }
 
-static void r128_cce_dispatch_indirect(drm_device_t * dev,
-				       drm_buf_t * buf, int start, int end)
+static void r128_cce_dispatch_indirect(struct drm_device * dev,
+				       struct drm_buf * buf, int start, int end)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_buf_priv_t *buf_priv = buf->dev_private;
@@ -695,8 +694,8 @@ static void r128_cce_dispatch_indirect(drm_device_t * dev,
 	dev_priv->sarea_priv->last_dispatch++;
 }
 
-static void r128_cce_dispatch_indices(drm_device_t * dev,
-				      drm_buf_t * buf,
+static void r128_cce_dispatch_indices(struct drm_device * dev,
+				      struct drm_buf * buf,
 				      int start, int end, int count)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
@@ -779,12 +778,13 @@ static void r128_cce_dispatch_indices(drm_device_t * dev,
 	sarea_priv->nbox = 0;
 }
 
-static int r128_cce_dispatch_blit(DRMFILE filp,
-				  drm_device_t * dev, drm_r128_blit_t * blit)
+static int r128_cce_dispatch_blit(struct drm_device * dev,
+				  struct drm_file *file_priv,
+				  drm_r128_blit_t * blit)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_r128_buf_priv_t *buf_priv;
 	u32 *data;
 	int dword_shift, dwords;
@@ -812,7 +812,7 @@ static int r128_cce_dispatch_blit(DRMFILE filp,
 		break;
 	default:
 		DRM_ERROR("invalid blit format %d\n", blit->format);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	/* Flush the pixel cache, and mark the contents as Read Invalid.
@@ -832,14 +832,14 @@ static int r128_cce_dispatch_blit(DRMFILE filp,
 	buf = dma->buflist[blit->idx];
 	buf_priv = buf->dev_private;
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
 		DRM_ERROR("sending pending buffer %d\n", blit->idx);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	buf_priv->discard = 1;
@@ -890,7 +890,7 @@ static int r128_cce_dispatch_blit(DRMFILE filp,
  * have hardware stencil support.
  */
 
-static int r128_cce_dispatch_write_span(drm_device_t * dev,
+static int r128_cce_dispatch_write_span(struct drm_device * dev,
 					drm_r128_depth_t * depth)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
@@ -903,22 +903,22 @@ static int r128_cce_dispatch_write_span(drm_device_t * dev,
 
 	count = depth->n;
 	if (count > 4096 || count <= 0)
-		return DRM_ERR(EMSGSIZE);
+		return -EMSGSIZE;
 
 	if (DRM_COPY_FROM_USER(&x, depth->x, sizeof(x))) {
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 	if (DRM_COPY_FROM_USER(&y, depth->y, sizeof(y))) {
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	buffer_size = depth->n * sizeof(u32);
 	buffer = drm_alloc(buffer_size, DRM_MEM_BUFS);
 	if (buffer == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	if (DRM_COPY_FROM_USER(buffer, depth->buffer, buffer_size)) {
 		drm_free(buffer, buffer_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	mask_size = depth->n * sizeof(u8);
@@ -926,12 +926,12 @@ static int r128_cce_dispatch_write_span(drm_device_t * dev,
 		mask = drm_alloc(mask_size, DRM_MEM_BUFS);
 		if (mask == NULL) {
 			drm_free(buffer, buffer_size, DRM_MEM_BUFS);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 		if (DRM_COPY_FROM_USER(mask, depth->mask, mask_size)) {
 			drm_free(buffer, buffer_size, DRM_MEM_BUFS);
 			drm_free(mask, mask_size, DRM_MEM_BUFS);
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		}
 
 		for (i = 0; i < count; i++, x++) {
@@ -986,7 +986,7 @@ static int r128_cce_dispatch_write_span(drm_device_t * dev,
 	return 0;
 }
 
-static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
+static int r128_cce_dispatch_write_pixels(struct drm_device * dev,
 					  drm_r128_depth_t * depth)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
@@ -999,28 +999,28 @@ static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
 
 	count = depth->n;
 	if (count > 4096 || count <= 0)
-		return DRM_ERR(EMSGSIZE);
+		return -EMSGSIZE;
 
 	xbuf_size = count * sizeof(*x);
 	ybuf_size = count * sizeof(*y);
 	x = drm_alloc(xbuf_size, DRM_MEM_BUFS);
 	if (x == NULL) {
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 	y = drm_alloc(ybuf_size, DRM_MEM_BUFS);
 	if (y == NULL) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 	if (DRM_COPY_FROM_USER(x, depth->x, xbuf_size)) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 	if (DRM_COPY_FROM_USER(y, depth->y, xbuf_size)) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	buffer_size = depth->n * sizeof(u32);
@@ -1028,13 +1028,13 @@ static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
 	if (buffer == NULL) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 	if (DRM_COPY_FROM_USER(buffer, depth->buffer, buffer_size)) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
 		drm_free(buffer, buffer_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	if (depth->mask) {
@@ -1044,14 +1044,14 @@ static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
 			drm_free(x, xbuf_size, DRM_MEM_BUFS);
 			drm_free(y, ybuf_size, DRM_MEM_BUFS);
 			drm_free(buffer, buffer_size, DRM_MEM_BUFS);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 		if (DRM_COPY_FROM_USER(mask, depth->mask, mask_size)) {
 			drm_free(x, xbuf_size, DRM_MEM_BUFS);
 			drm_free(y, ybuf_size, DRM_MEM_BUFS);
 			drm_free(buffer, buffer_size, DRM_MEM_BUFS);
 			drm_free(mask, mask_size, DRM_MEM_BUFS);
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		}
 
 		for (i = 0; i < count; i++) {
@@ -1108,7 +1108,7 @@ static int r128_cce_dispatch_write_pixels(drm_device_t * dev,
 	return 0;
 }
 
-static int r128_cce_dispatch_read_span(drm_device_t * dev,
+static int r128_cce_dispatch_read_span(struct drm_device * dev,
 				       drm_r128_depth_t * depth)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
@@ -1118,13 +1118,13 @@ static int r128_cce_dispatch_read_span(drm_device_t * dev,
 
 	count = depth->n;
 	if (count > 4096 || count <= 0)
-		return DRM_ERR(EMSGSIZE);
+		return -EMSGSIZE;
 
 	if (DRM_COPY_FROM_USER(&x, depth->x, sizeof(x))) {
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 	if (DRM_COPY_FROM_USER(&y, depth->y, sizeof(y))) {
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	BEGIN_RING(7);
@@ -1151,18 +1151,18 @@ static int r128_cce_dispatch_read_span(drm_device_t * dev,
 	return 0;
 }
 
-static int r128_cce_dispatch_read_pixels(drm_device_t * dev,
+static int r128_cce_dispatch_read_pixels(struct drm_device * dev,
 					 drm_r128_depth_t * depth)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	int count, *x, *y;
 	int i, xbuf_size, ybuf_size;
 	RING_LOCALS;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	count = depth->n;
 	if (count > 4096 || count <= 0)
-		return DRM_ERR(EMSGSIZE);
+		return -EMSGSIZE;
 
 	if (count > dev_priv->depth_pitch) {
 		count = dev_priv->depth_pitch;
@@ -1172,22 +1172,22 @@ static int r128_cce_dispatch_read_pixels(drm_device_t * dev,
 	ybuf_size = count * sizeof(*y);
 	x = drm_alloc(xbuf_size, DRM_MEM_BUFS);
 	if (x == NULL) {
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 	y = drm_alloc(ybuf_size, DRM_MEM_BUFS);
 	if (y == NULL) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 	if (DRM_COPY_FROM_USER(x, depth->x, xbuf_size)) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 	if (DRM_COPY_FROM_USER(y, depth->y, ybuf_size)) {
 		drm_free(x, xbuf_size, DRM_MEM_BUFS);
 		drm_free(y, ybuf_size, DRM_MEM_BUFS);
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	for (i = 0; i < count; i++) {
@@ -1223,12 +1223,12 @@ static int r128_cce_dispatch_read_pixels(drm_device_t * dev,
  * Polygon stipple
  */
 
-static void r128_cce_dispatch_stipple(drm_device_t * dev, u32 * stipple)
+static void r128_cce_dispatch_stipple(struct drm_device * dev, u32 * stipple)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	int i;
 	RING_LOCALS;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
 	BEGIN_RING(33);
 
@@ -1244,25 +1244,21 @@ static void r128_cce_dispatch_stipple(drm_device_t * dev, u32 * stipple)
  * IOCTL functions
  */
 
-static int r128_cce_clear(DRM_IOCTL_ARGS)
+static int r128_cce_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_r128_clear_t clear;
+	drm_r128_clear_t *clear = data;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(clear, (drm_r128_clear_t __user *) data,
-				 sizeof(clear));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
 	if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
 
-	r128_cce_dispatch_clear(dev, &clear);
+	r128_cce_dispatch_clear(dev, clear);
 	COMMIT_RING();
 
 	/* Make sure we restore the 3D state next time.
@@ -1272,7 +1268,7 @@ static int r128_cce_clear(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int r128_do_init_pageflip(drm_device_t * dev)
+static int r128_do_init_pageflip(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
@@ -1291,7 +1287,7 @@ static int r128_do_init_pageflip(drm_device_t * dev)
 	return 0;
 }
 
-static int r128_do_cleanup_pageflip(drm_device_t * dev)
+static int r128_do_cleanup_pageflip(struct drm_device * dev)
 {
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
@@ -1312,13 +1308,12 @@ static int r128_do_cleanup_pageflip(drm_device_t * dev)
  * They can & should be intermixed to support multiple 3d windows.
  */
 
-static int r128_cce_flip(DRM_IOCTL_ARGS)
+static int r128_cce_flip(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -1331,14 +1326,13 @@ static int r128_cce_flip(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int r128_cce_swap(DRM_IOCTL_ARGS)
+static int r128_cce_swap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
 	drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	DRM_DEBUG("%s\n", __FUNCTION__);
+	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -1353,58 +1347,54 @@ static int r128_cce_swap(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int r128_cce_vertex(DRM_IOCTL_ARGS)
+static int r128_cce_vertex(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_r128_buf_priv_t *buf_priv;
-	drm_r128_vertex_t vertex;
+	drm_r128_vertex_t *vertex = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(vertex, (drm_r128_vertex_t __user *) data,
-				 sizeof(vertex));
-
 	DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
-		  DRM_CURRENTPID, vertex.idx, vertex.count, vertex.discard);
+		  DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard);
 
-	if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
+	if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  vertex.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  vertex->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
-	if (vertex.prim < 0 ||
-	    vertex.prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
-		DRM_ERROR("buffer prim %d\n", vertex.prim);
-		return DRM_ERR(EINVAL);
+	if (vertex->prim < 0 ||
+	    vertex->prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
+		DRM_ERROR("buffer prim %d\n", vertex->prim);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf = dma->buflist[vertex.idx];
+	buf = dma->buflist[vertex->idx];
 	buf_priv = buf->dev_private;
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", vertex.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", vertex->idx);
+		return -EINVAL;
 	}
 
-	buf->used = vertex.count;
-	buf_priv->prim = vertex.prim;
-	buf_priv->discard = vertex.discard;
+	buf->used = vertex->count;
+	buf_priv->prim = vertex->prim;
+	buf_priv->discard = vertex->discard;
 
 	r128_cce_dispatch_vertex(dev, buf);
 
@@ -1412,134 +1402,123 @@ static int r128_cce_vertex(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int r128_cce_indices(DRM_IOCTL_ARGS)
+static int r128_cce_indices(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_r128_buf_priv_t *buf_priv;
-	drm_r128_indices_t elts;
+	drm_r128_indices_t *elts = data;
 	int count;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(elts, (drm_r128_indices_t __user *) data,
-				 sizeof(elts));
-
 	DRM_DEBUG("pid=%d buf=%d s=%d e=%d d=%d\n", DRM_CURRENTPID,
-		  elts.idx, elts.start, elts.end, elts.discard);
+		  elts->idx, elts->start, elts->end, elts->discard);
 
-	if (elts.idx < 0 || elts.idx >= dma->buf_count) {
+	if (elts->idx < 0 || elts->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  elts.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  elts->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
-	if (elts.prim < 0 || elts.prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
-		DRM_ERROR("buffer prim %d\n", elts.prim);
-		return DRM_ERR(EINVAL);
+	if (elts->prim < 0 ||
+	    elts->prim > R128_CCE_VC_CNTL_PRIM_TYPE_TRI_TYPE2) {
+		DRM_ERROR("buffer prim %d\n", elts->prim);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf = dma->buflist[elts.idx];
+	buf = dma->buflist[elts->idx];
 	buf_priv = buf->dev_private;
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", elts.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", elts->idx);
+		return -EINVAL;
 	}
 
-	count = (elts.end - elts.start) / sizeof(u16);
-	elts.start -= R128_INDEX_PRIM_OFFSET;
+	count = (elts->end - elts->start) / sizeof(u16);
+	elts->start -= R128_INDEX_PRIM_OFFSET;
 
-	if (elts.start & 0x7) {
-		DRM_ERROR("misaligned buffer 0x%x\n", elts.start);
-		return DRM_ERR(EINVAL);
+	if (elts->start & 0x7) {
+		DRM_ERROR("misaligned buffer 0x%x\n", elts->start);
+		return -EINVAL;
 	}
-	if (elts.start < buf->used) {
-		DRM_ERROR("no header 0x%x - 0x%x\n", elts.start, buf->used);
-		return DRM_ERR(EINVAL);
+	if (elts->start < buf->used) {
+		DRM_ERROR("no header 0x%x - 0x%x\n", elts->start, buf->used);
+		return -EINVAL;
 	}
 
-	buf->used = elts.end;
-	buf_priv->prim = elts.prim;
-	buf_priv->discard = elts.discard;
+	buf->used = elts->end;
+	buf_priv->prim = elts->prim;
+	buf_priv->discard = elts->discard;
 
-	r128_cce_dispatch_indices(dev, buf, elts.start, elts.end, count);
+	r128_cce_dispatch_indices(dev, buf, elts->start, elts->end, count);
 
 	COMMIT_RING();
 	return 0;
 }
 
-static int r128_cce_blit(DRM_IOCTL_ARGS)
+static int r128_cce_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_r128_blit_t blit;
+	drm_r128_blit_t *blit = data;
 	int ret;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_COPY_FROM_USER_IOCTL(blit, (drm_r128_blit_t __user *) data,
-				 sizeof(blit));
+	DRM_DEBUG("pid=%d index=%d\n", DRM_CURRENTPID, blit->idx);
 
-	DRM_DEBUG("pid=%d index=%d\n", DRM_CURRENTPID, blit.idx);
-
-	if (blit.idx < 0 || blit.idx >= dma->buf_count) {
+	if (blit->idx < 0 || blit->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  blit.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  blit->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	ret = r128_cce_dispatch_blit(filp, dev, &blit);
+	ret = r128_cce_dispatch_blit(dev, file_priv, blit);
 
 	COMMIT_RING();
 	return ret;
 }
 
-static int r128_cce_depth(DRM_IOCTL_ARGS)
+static int r128_cce_depth(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_r128_depth_t depth;
+	drm_r128_depth_t *depth = data;
 	int ret;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(depth, (drm_r128_depth_t __user *) data,
-				 sizeof(depth));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
-	ret = DRM_ERR(EINVAL);
-	switch (depth.func) {
+	ret = -EINVAL;
+	switch (depth->func) {
 	case R128_WRITE_SPAN:
-		ret = r128_cce_dispatch_write_span(dev, &depth);
+		ret = r128_cce_dispatch_write_span(dev, depth);
 		break;
 	case R128_WRITE_PIXELS:
-		ret = r128_cce_dispatch_write_pixels(dev, &depth);
+		ret = r128_cce_dispatch_write_pixels(dev, depth);
 		break;
 	case R128_READ_SPAN:
-		ret = r128_cce_dispatch_read_span(dev, &depth);
+		ret = r128_cce_dispatch_read_span(dev, depth);
 		break;
 	case R128_READ_PIXELS:
-		ret = r128_cce_dispatch_read_pixels(dev, &depth);
+		ret = r128_cce_dispatch_read_pixels(dev, depth);
 		break;
 	}
 
@@ -1547,20 +1526,16 @@ static int r128_cce_depth(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-static int r128_cce_stipple(DRM_IOCTL_ARGS)
+static int r128_cce_stipple(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_r128_stipple_t stipple;
+	drm_r128_stipple_t *stipple = data;
 	u32 mask[32];
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(stipple, (drm_r128_stipple_t __user *) data,
-				 sizeof(stipple));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (DRM_COPY_FROM_USER(&mask, stipple.mask, 32 * sizeof(u32)))
-		return DRM_ERR(EFAULT);
+	if (DRM_COPY_FROM_USER(&mask, stipple->mask, 32 * sizeof(u32)))
+		return -EFAULT;
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -1570,61 +1545,58 @@ static int r128_cce_stipple(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int r128_cce_indirect(DRM_IOCTL_ARGS)
+static int r128_cce_indirect(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_r128_buf_priv_t *buf_priv;
-	drm_r128_indirect_t indirect;
+	drm_r128_indirect_t *indirect = data;
 #if 0
 	RING_LOCALS;
 #endif
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(indirect, (drm_r128_indirect_t __user *) data,
-				 sizeof(indirect));
+	DRM_DEBUG("idx=%d s=%d e=%d d=%d\n",
+		  indirect->idx, indirect->start, indirect->end,
+		  indirect->discard);
 
-	DRM_DEBUG("indirect: idx=%d s=%d e=%d d=%d\n",
-		  indirect.idx, indirect.start, indirect.end, indirect.discard);
-
-	if (indirect.idx < 0 || indirect.idx >= dma->buf_count) {
+	if (indirect->idx < 0 || indirect->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  indirect.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  indirect->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
 
-	buf = dma->buflist[indirect.idx];
+	buf = dma->buflist[indirect->idx];
 	buf_priv = buf->dev_private;
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", indirect.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", indirect->idx);
+		return -EINVAL;
 	}
 
-	if (indirect.start < buf->used) {
+	if (indirect->start < buf->used) {
 		DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
-			  indirect.start, buf->used);
-		return DRM_ERR(EINVAL);
+			  indirect->start, buf->used);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf->used = indirect.end;
-	buf_priv->discard = indirect.discard;
+	buf->used = indirect->end;
+	buf_priv->discard = indirect->discard;
 
 #if 0
 	/* Wait for the 3D stream to idle before the indirect buffer
@@ -1639,46 +1611,42 @@ static int r128_cce_indirect(DRM_IOCTL_ARGS)
 	 * X server.  This is insecure and is thus only available to
 	 * privileged clients.
 	 */
-	r128_cce_dispatch_indirect(dev, buf, indirect.start, indirect.end);
+	r128_cce_dispatch_indirect(dev, buf, indirect->start, indirect->end);
 
 	COMMIT_RING();
 	return 0;
 }
 
-static int r128_getparam(DRM_IOCTL_ARGS)
+static int r128_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_r128_private_t *dev_priv = dev->dev_private;
-	drm_r128_getparam_t param;
+	drm_r128_getparam_t *param = data;
 	int value;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_r128_getparam_t __user *) data,
-				 sizeof(param));
-
 	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	switch (param.param) {
+	switch (param->param) {
 	case R128_PARAM_IRQ_NR:
 		value = dev->irq;
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
+	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-void r128_driver_preclose(drm_device_t * dev, DRMFILE filp)
+void r128_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
 {
 	if (dev->dev_private) {
 		drm_r128_private_t *dev_priv = dev->dev_private;
@@ -1688,29 +1656,29 @@ void r128_driver_preclose(drm_device_t * dev, DRMFILE filp)
 	}
 }
 
-void r128_driver_lastclose(drm_device_t * dev)
+void r128_driver_lastclose(struct drm_device * dev)
 {
 	r128_do_cleanup_cce(dev);
 }
 
-drm_ioctl_desc_t r128_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_R128_INIT)] = {r128_cce_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_R128_CCE_START)] = {r128_cce_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_R128_CCE_STOP)] = {r128_cce_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_R128_CCE_RESET)] = {r128_cce_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_R128_CCE_IDLE)] = {r128_cce_idle, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_RESET)] = {r128_engine_reset, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_FULLSCREEN)] = {r128_fullscreen, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_SWAP)] = {r128_cce_swap, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_FLIP)] = {r128_cce_flip, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_CLEAR)] = {r128_cce_clear, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_VERTEX)] = {r128_cce_vertex, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_INDICES)] = {r128_cce_indices, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_BLIT)] = {r128_cce_blit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_DEPTH)] = {r128_cce_depth, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_STIPPLE)] = {r128_cce_stipple, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_R128_INDIRECT)] = {r128_cce_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_R128_GETPARAM)] = {r128_getparam, DRM_AUTH},
+struct drm_ioctl_desc r128_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_R128_INIT, r128_cce_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_R128_CCE_START, r128_cce_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_R128_CCE_STOP, r128_cce_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_R128_CCE_RESET, r128_cce_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_R128_CCE_IDLE, r128_cce_idle, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_RESET, r128_engine_reset, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_FULLSCREEN, r128_fullscreen, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_SWAP, r128_cce_swap, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_FLIP, r128_cce_flip, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_CLEAR, r128_cce_clear, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_VERTEX, r128_cce_vertex, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_INDICES, r128_cce_indices, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_BLIT, r128_cce_blit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_DEPTH, r128_cce_depth, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_STIPPLE, r128_cce_stipple, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_R128_INDIRECT, r128_cce_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_R128_GETPARAM, r128_getparam, DRM_AUTH),
 };
 
 int r128_max_ioctl = DRM_ARRAY_SIZE(r128_ioctls);
diff --git a/sys/dev/drm/r300_cmdbuf.c b/sys/dev/drm/r300_cmdbuf.c
index d71f8cb..b9e9f0f 100644
--- a/sys/dev/drm/r300_cmdbuf.c
+++ b/sys/dev/drm/r300_cmdbuf.c
@@ -58,7 +58,7 @@ static const int r300_cliprect_cntl[4] = {
 static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
 			       drm_radeon_kcmd_buffer_t *cmdbuf, int n)
 {
-	drm_clip_rect_t box;
+	struct drm_clip_rect box;
 	int nr;
 	int i;
 	RING_LOCALS;
@@ -77,26 +77,37 @@ static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
 			if (DRM_COPY_FROM_USER_UNCHECKED
 			    (&box, &cmdbuf->boxes[n + i], sizeof(box))) {
 				DRM_ERROR("copy cliprect faulted\n");
-				return DRM_ERR(EFAULT);
+				return -EFAULT;
 			}
 
-			box.x1 =
-			    (box.x1 +
-			     R300_CLIPRECT_OFFSET) & R300_CLIPRECT_MASK;
-			box.y1 =
-			    (box.y1 +
-			     R300_CLIPRECT_OFFSET) & R300_CLIPRECT_MASK;
-			box.x2 =
-			    (box.x2 +
-			     R300_CLIPRECT_OFFSET) & R300_CLIPRECT_MASK;
-			box.y2 =
-			    (box.y2 +
-			     R300_CLIPRECT_OFFSET) & R300_CLIPRECT_MASK;
+			box.x2--; /* Hardware expects inclusive bottom-right corner */
+			box.y2--;
+
+			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
+				box.x1 = (box.x1) &
+					R300_CLIPRECT_MASK;
+				box.y1 = (box.y1) &
+					R300_CLIPRECT_MASK;
+				box.x2 = (box.x2) &
+					R300_CLIPRECT_MASK;
+				box.y2 = (box.y2) &
+					R300_CLIPRECT_MASK;
+			} else {
+				box.x1 = (box.x1 + R300_CLIPRECT_OFFSET) &
+					R300_CLIPRECT_MASK;
+				box.y1 = (box.y1 + R300_CLIPRECT_OFFSET) &
+					R300_CLIPRECT_MASK;
+				box.x2 = (box.x2 + R300_CLIPRECT_OFFSET) &
+					R300_CLIPRECT_MASK;
+				box.y2 = (box.y2 + R300_CLIPRECT_OFFSET) &
+					R300_CLIPRECT_MASK;
+			}
 
 			OUT_RING((box.x1 << R300_CLIPRECT_X_SHIFT) |
 				 (box.y1 << R300_CLIPRECT_Y_SHIFT));
 			OUT_RING((box.x2 << R300_CLIPRECT_X_SHIFT) |
 				 (box.y2 << R300_CLIPRECT_Y_SHIFT));
+
 		}
 
 		OUT_RING_REG(R300_RE_CLIPRECT_CNTL, r300_cliprect_cntl[nr - 1]);
@@ -131,14 +142,28 @@ static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
 		ADVANCE_RING();
 	}
 
+	/* flus cache and wait idle clean after cliprect change */
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
+	OUT_RING(R300_RB3D_DC_FLUSH);
+	ADVANCE_RING();
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
+	ADVANCE_RING();
+	/* set flush flag */
+	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
+
 	return 0;
 }
 
 static u8 r300_reg_flags[0x10000 >> 2];
 
-void r300_init_reg_flags(void)
+void r300_init_reg_flags(struct drm_device *dev)
 {
 	int i;
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
 	memset(r300_reg_flags, 0, 0x10000 >> 2);
 #define ADD_RANGE_MARK(reg, count,mark) \
 		for(i=((reg)>>2);i<((reg)>>2)+(count);i++)\
@@ -151,70 +176,63 @@ void r300_init_reg_flags(void)
 
 	/* these match cmducs() command in r300_driver/r300/r300_cmdbuf.c */
 	ADD_RANGE(R300_SE_VPORT_XSCALE, 6);
-	ADD_RANGE(0x2080, 1);
+	ADD_RANGE(R300_VAP_CNTL, 1);
 	ADD_RANGE(R300_SE_VTE_CNTL, 2);
 	ADD_RANGE(0x2134, 2);
-	ADD_RANGE(0x2140, 1);
+	ADD_RANGE(R300_VAP_CNTL_STATUS, 1);
 	ADD_RANGE(R300_VAP_INPUT_CNTL_0, 2);
 	ADD_RANGE(0x21DC, 1);
-	ADD_RANGE(0x221C, 1);
-	ADD_RANGE(0x2220, 4);
-	ADD_RANGE(0x2288, 1);
+	ADD_RANGE(R300_VAP_UNKNOWN_221C, 1);
+	ADD_RANGE(R300_VAP_CLIP_X_0, 4);
+	ADD_RANGE(R300_VAP_PVS_STATE_FLUSH_REG, 1);
+	ADD_RANGE(R300_VAP_UNKNOWN_2288, 1);
 	ADD_RANGE(R300_VAP_OUTPUT_VTX_FMT_0, 2);
 	ADD_RANGE(R300_VAP_PVS_CNTL_1, 3);
 	ADD_RANGE(R300_GB_ENABLE, 1);
 	ADD_RANGE(R300_GB_MSPOS0, 5);
-	ADD_RANGE(R300_TX_CNTL, 1);
+	ADD_RANGE(R300_TX_INVALTAGS, 1);
 	ADD_RANGE(R300_TX_ENABLE, 1);
 	ADD_RANGE(0x4200, 4);
 	ADD_RANGE(0x4214, 1);
 	ADD_RANGE(R300_RE_POINTSIZE, 1);
 	ADD_RANGE(0x4230, 3);
 	ADD_RANGE(R300_RE_LINE_CNT, 1);
-	ADD_RANGE(0x4238, 1);
+	ADD_RANGE(R300_RE_UNK4238, 1);
 	ADD_RANGE(0x4260, 3);
-	ADD_RANGE(0x4274, 4);
-	ADD_RANGE(0x4288, 5);
-	ADD_RANGE(0x42A0, 1);
+	ADD_RANGE(R300_RE_SHADE, 4);
+	ADD_RANGE(R300_RE_POLYGON_MODE, 5);
+	ADD_RANGE(R300_RE_ZBIAS_CNTL, 1);
 	ADD_RANGE(R300_RE_ZBIAS_T_FACTOR, 4);
-	ADD_RANGE(0x42B4, 1);
+	ADD_RANGE(R300_RE_OCCLUSION_CNTL, 1);
 	ADD_RANGE(R300_RE_CULL_CNTL, 1);
 	ADD_RANGE(0x42C0, 2);
 	ADD_RANGE(R300_RS_CNTL_0, 2);
-	ADD_RANGE(R300_RS_INTERP_0, 8);
-	ADD_RANGE(R300_RS_ROUTE_0, 8);
-	ADD_RANGE(0x43A4, 2);
+
+	ADD_RANGE(R300_SC_HYPERZ, 2);
 	ADD_RANGE(0x43E8, 1);
-	ADD_RANGE(R300_PFS_CNTL_0, 3);
-	ADD_RANGE(R300_PFS_NODE_0, 4);
-	ADD_RANGE(R300_PFS_TEXI_0, 64);
+
 	ADD_RANGE(0x46A4, 5);
-	ADD_RANGE(R300_PFS_INSTR0_0, 64);
-	ADD_RANGE(R300_PFS_INSTR1_0, 64);
-	ADD_RANGE(R300_PFS_INSTR2_0, 64);
-	ADD_RANGE(R300_PFS_INSTR3_0, 64);
-	ADD_RANGE(0x4BC0, 1);
-	ADD_RANGE(0x4BC8, 3);
+
+	ADD_RANGE(R300_RE_FOG_STATE, 1);
+	ADD_RANGE(R300_FOG_COLOR_R, 3);
 	ADD_RANGE(R300_PP_ALPHA_TEST, 2);
 	ADD_RANGE(0x4BD8, 1);
 	ADD_RANGE(R300_PFS_PARAM_0_X, 64);
 	ADD_RANGE(0x4E00, 1);
 	ADD_RANGE(R300_RB3D_CBLEND, 2);
 	ADD_RANGE(R300_RB3D_COLORMASK, 1);
-	ADD_RANGE(0x4E10, 3);
+	ADD_RANGE(R300_RB3D_BLEND_COLOR, 3);
 	ADD_RANGE_MARK(R300_RB3D_COLOROFFSET0, 1, MARK_CHECK_OFFSET);	/* check offset */
 	ADD_RANGE(R300_RB3D_COLORPITCH0, 1);
 	ADD_RANGE(0x4E50, 9);
 	ADD_RANGE(0x4E88, 1);
 	ADD_RANGE(0x4EA0, 2);
-	ADD_RANGE(R300_RB3D_ZSTENCIL_CNTL_0, 3);
-	ADD_RANGE(0x4F10, 4);
-	ADD_RANGE_MARK(R300_RB3D_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);	/* check offset */
-	ADD_RANGE(R300_RB3D_DEPTHPITCH, 1);
-	ADD_RANGE(0x4F28, 1);
-	ADD_RANGE(0x4F30, 2);
-	ADD_RANGE(0x4F44, 1);
-	ADD_RANGE(0x4F54, 1);
+	ADD_RANGE(R300_ZB_CNTL, 3);
+	ADD_RANGE(R300_ZB_FORMAT, 4);
+	ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);	/* check offset */
+	ADD_RANGE(R300_ZB_DEPTHPITCH, 1);
+	ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1);
+	ADD_RANGE(R300_ZB_ZMASK_OFFSET, 13);
 
 	ADD_RANGE(R300_TX_FILTER_0, 16);
 	ADD_RANGE(R300_TX_FILTER1_0, 16);
@@ -227,11 +245,33 @@ void r300_init_reg_flags(void)
 	ADD_RANGE(R300_TX_BORDER_COLOR_0, 16);
 
 	/* Sporadic registers used as primitives are emitted */
-	ADD_RANGE(0x4f18, 1);
+	ADD_RANGE(R300_ZB_ZCACHE_CTLSTAT, 1);
 	ADD_RANGE(R300_RB3D_DSTCACHE_CTLSTAT, 1);
 	ADD_RANGE(R300_VAP_INPUT_ROUTE_0_0, 8);
 	ADD_RANGE(R300_VAP_INPUT_ROUTE_1_0, 8);
 
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
+		ADD_RANGE(R500_VAP_INDEX_OFFSET, 1);
+		ADD_RANGE(R500_US_CONFIG, 2);
+		ADD_RANGE(R500_US_CODE_ADDR, 3);
+		ADD_RANGE(R500_US_FC_CTRL, 1);
+		ADD_RANGE(R500_RS_IP_0, 16);
+		ADD_RANGE(R500_RS_INST_0, 16);
+		ADD_RANGE(R500_RB3D_COLOR_CLEAR_VALUE_AR, 2);
+		ADD_RANGE(R500_RB3D_CONSTANT_COLOR_AR, 2);
+		ADD_RANGE(R500_ZB_FIFO_SIZE, 2);
+	} else {
+		ADD_RANGE(R300_PFS_CNTL_0, 3);
+		ADD_RANGE(R300_PFS_NODE_0, 4);
+		ADD_RANGE(R300_PFS_TEXI_0, 64);
+		ADD_RANGE(R300_PFS_INSTR0_0, 64);
+		ADD_RANGE(R300_PFS_INSTR1_0, 64);
+		ADD_RANGE(R300_PFS_INSTR2_0, 64);
+		ADD_RANGE(R300_PFS_INSTR3_0, 64);
+		ADD_RANGE(R300_RS_INTERP_0, 8);
+		ADD_RANGE(R300_RS_ROUTE_0, 8);
+
+	}
 }
 
 static __inline__ int r300_check_range(unsigned reg, int count)
@@ -245,26 +285,6 @@ static __inline__ int r300_check_range(unsigned reg, int count)
 	return 0;
 }
 
-/*
- * we expect offsets passed to the framebuffer to be either within video 
- * memory or within AGP space 
- */
-static __inline__ int r300_check_offset(drm_radeon_private_t *dev_priv,
-					u32 offset)
-{
-	/* we realy want to check against end of video aperture
-	   but this value is not being kept.
-	   This code is correct for now (does the same thing as the
-	   code that sets MC_FB_LOCATION) in radeon_cp.c */
-	if (offset >= dev_priv->fb_location &&
-	    offset < (dev_priv->fb_location + dev_priv->fb_size))
-		return 0;
-	if (offset >= dev_priv->gart_vm_start &&
-	    offset < (dev_priv->gart_vm_start + dev_priv->gart_size))
-		return 0;
-	return 1;
-}
-
 static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
 							  dev_priv,
 							  drm_radeon_kcmd_buffer_t
@@ -285,7 +305,7 @@ static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
 		DRM_ERROR
 		    ("Cannot emit more than 64 values at a time (reg=%04x sz=%d)\n",
 		     reg, sz);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	for (i = 0; i < sz; i++) {
 		values[i] = ((int *)cmdbuf->buf)[i];
@@ -293,17 +313,17 @@ static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
 		case MARK_SAFE:
 			break;
 		case MARK_CHECK_OFFSET:
-			if (r300_check_offset(dev_priv, (u32) values[i])) {
+			if (!radeon_check_offset(dev_priv, (u32) values[i])) {
 				DRM_ERROR
 				    ("Offset failed range check (reg=%04x sz=%d)\n",
 				     reg, sz);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 			break;
 		default:
 			DRM_ERROR("Register %04x failed check as flag=%02x\n",
 				  reg + i * 4, r300_reg_flags[(reg >> 2) + i]);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -335,16 +355,17 @@ static __inline__ int r300_emit_packet0(drm_radeon_private_t *dev_priv,
 	sz = header.packet0.count;
 	reg = (header.packet0.reghi << 8) | header.packet0.reglo;
 
+	DRM_DEBUG("R300_CMD_PACKET0: reg %04x, sz %d\n", reg, sz);
 	if (!sz)
 		return 0;
 
 	if (sz * 4 > cmdbuf->bufsz)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	if (reg + sz * 4 >= 0x10000) {
 		DRM_ERROR("No such registers in hardware reg=%04x sz=%d\n", reg,
 			  sz);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (r300_check_range(reg, sz)) {
@@ -384,17 +405,30 @@ static __inline__ int r300_emit_vpu(drm_radeon_private_t *dev_priv,
 	if (!sz)
 		return 0;
 	if (sz * 16 > cmdbuf->bufsz)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
+
+	/* VAP is very sensitive so we purge cache before we program it
+	 * and we also flush its state before & after */
+	BEGIN_RING(6);
+	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
+	OUT_RING(R300_RB3D_DC_FLUSH);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
+	OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
+	OUT_RING(0);
+	ADVANCE_RING();
+	/* set flush flag */
+	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 
-	BEGIN_RING(5 + sz * 4);
-	/* Wait for VAP to come to senses.. */
-	/* there is no need to emit it multiple times, (only once before VAP is programmed,
-	   but this optimization is for later */
-	OUT_RING_REG(R300_VAP_PVS_WAITIDLE, 0);
+	BEGIN_RING(3 + sz * 4);
 	OUT_RING_REG(R300_VAP_PVS_UPLOAD_ADDRESS, addr);
 	OUT_RING(CP_PACKET0_TABLE(R300_VAP_PVS_UPLOAD_DATA, sz * 4 - 1));
 	OUT_RING_TABLE((int *)cmdbuf->buf, sz * 4);
+	ADVANCE_RING();
 
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
+	OUT_RING(0);
 	ADVANCE_RING();
 
 	cmdbuf->buf += sz * 16;
@@ -413,7 +447,7 @@ static __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv,
 	RING_LOCALS;
 
 	if (8 * 4 > cmdbuf->bufsz)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	BEGIN_RING(10);
 	OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 8));
@@ -422,6 +456,15 @@ static __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv,
 	OUT_RING_TABLE((int *)cmdbuf->buf, 8);
 	ADVANCE_RING();
 
+	BEGIN_RING(4);
+	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
+	OUT_RING(R300_RB3D_DC_FLUSH);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
+	ADVANCE_RING();
+	/* set flush flag */
+	dev_priv->track_flush |= RADEON_FLUSH_EMITED;
+
 	cmdbuf->buf += 8 * 4;
 	cmdbuf->bufsz -= 8 * 4;
 
@@ -443,7 +486,7 @@ static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
 	if ((count + 1) > MAX_ARRAY_PACKET) {
 		DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
 			  count);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	memset(payload, 0, MAX_ARRAY_PACKET * 4);
 	memcpy(payload, cmdbuf->buf + 4, (count + 1) * 4);
@@ -455,22 +498,22 @@ static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
 	i = 1;
 	while ((k < narrays) && (i < (count + 1))) {
 		i++;		/* skip attribute field */
-		if (r300_check_offset(dev_priv, payload[i])) {
+		if (!radeon_check_offset(dev_priv, payload[i])) {
 			DRM_ERROR
 			    ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 			     k, i);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		k++;
 		i++;
 		if (k == narrays)
 			break;
 		/* have one more to process, they come in pairs */
-		if (r300_check_offset(dev_priv, payload[i])) {
+		if (!radeon_check_offset(dev_priv, payload[i])) {
 			DRM_ERROR
 			    ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 			     k, i);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		k++;
 		i++;
@@ -480,7 +523,7 @@ static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
 		DRM_ERROR
 		    ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
 		     k, i, narrays, count + 1);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	/* all clear, output packet */
@@ -508,25 +551,25 @@ static __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv,
 	if (cmd[0] & 0x8000) {
 		u32 offset;
 
-		if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL 
+		if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 			      | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 			offset = cmd[2] << 10;
-			ret = r300_check_offset(dev_priv, offset);
+			ret = !radeon_check_offset(dev_priv, offset);
 			if (ret) {
 				DRM_ERROR("Invalid bitblt first offset is %08X\n", offset);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 		}
 
 		if ((cmd[1] & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
 		    (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 			offset = cmd[3] << 10;
-			ret = r300_check_offset(dev_priv, offset);
+			ret = !radeon_check_offset(dev_priv, offset);
 			if (ret) {
 				DRM_ERROR("Invalid bitblt second offset is %08X\n", offset);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
-			
+
 		}
 	}
 
@@ -541,6 +584,81 @@ static __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv,
 	return 0;
 }
 
+static __inline__ int r300_emit_draw_indx_2(drm_radeon_private_t *dev_priv,
+					    drm_radeon_kcmd_buffer_t *cmdbuf)
+{
+	u32 *cmd;
+	int count;
+	int expected_count;
+	RING_LOCALS;
+
+	cmd = (u32 *) cmdbuf->buf;
+	count = (cmd[0]>>16) & 0x3fff;
+	expected_count = cmd[1] >> 16;
+	if (!(cmd[1] & R300_VAP_VF_CNTL__INDEX_SIZE_32bit))
+		expected_count = (expected_count+1)/2;
+
+	if (count && count != expected_count) {
+		DRM_ERROR("3D_DRAW_INDX_2: packet size %i, expected %i\n",
+			count, expected_count);
+		return -EINVAL;
+	}
+
+	BEGIN_RING(count+2);
+	OUT_RING(cmd[0]);
+	OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
+	ADVANCE_RING();
+
+	cmdbuf->buf += (count+2)*4;
+	cmdbuf->bufsz -= (count+2)*4;
+
+	if (!count) {
+		drm_r300_cmd_header_t header;
+
+		if (cmdbuf->bufsz < 4*4 + sizeof(header)) {
+			DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER, but stream is too short.\n");
+			return -EINVAL;
+		}
+
+		header.u = *(unsigned int *)cmdbuf->buf;
+
+		cmdbuf->buf += sizeof(header);
+		cmdbuf->bufsz -= sizeof(header);
+		cmd = (u32 *) cmdbuf->buf;
+
+		if (header.header.cmd_type != R300_CMD_PACKET3 ||
+		    header.packet3.packet != R300_CMD_PACKET3_RAW ||
+		    cmd[0] != CP_PACKET3(RADEON_CP_INDX_BUFFER, 2)) {
+			DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER.\n");
+			return -EINVAL;
+		}
+
+		if ((cmd[1] & 0x8000ffff) != 0x80000810) {
+			DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
+			return -EINVAL;
+		}
+		if (!radeon_check_offset(dev_priv, cmd[2])) {
+			DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
+			return -EINVAL;
+		}
+		if (cmd[3] != expected_count) {
+			DRM_ERROR("INDX_BUFFER: buffer size %i, expected %i\n",
+				cmd[3], expected_count);
+			return -EINVAL;
+		}
+
+		BEGIN_RING(4);
+		OUT_RING(cmd[0]);
+		OUT_RING_TABLE((int *)(cmdbuf->buf + 4), 3);
+		ADVANCE_RING();
+
+		cmdbuf->buf += 4*4;
+		cmdbuf->bufsz -= 4*4;
+	}
+
+	return 0;
+}
+
 static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 					    drm_radeon_kcmd_buffer_t *cmdbuf)
 {
@@ -549,7 +667,7 @@ static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 	RING_LOCALS;
 
 	if (4 > cmdbuf->bufsz)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	/* Fixme !! This simply emits a packet without much checking.
 	   We need to be smarter. */
@@ -560,7 +678,7 @@ static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 	/* Is it packet 3 ? */
 	if ((header >> 30) != 0x3) {
 		DRM_ERROR("Not a packet3 header (0x%08x)\n", header);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	count = (header >> 16) & 0x3fff;
@@ -570,7 +688,7 @@ static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 		DRM_ERROR
 		    ("Expected packet3 of length %d but have only %d bytes left\n",
 		     (count + 2) * 4, cmdbuf->bufsz);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	/* Is it a packet type we know about ? */
@@ -581,17 +699,29 @@ static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 	case RADEON_CNTL_BITBLT_MULTI:
 		return r300_emit_bitblt_multi(dev_priv, cmdbuf);
 
-	case RADEON_CP_3D_DRAW_IMMD_2:	/* triggers drawing using in-packet vertex data */
-	case RADEON_CP_3D_DRAW_VBUF_2:	/* triggers drawing of vertex buffers setup elsewhere */
-	case RADEON_CP_3D_DRAW_INDX_2:	/* triggers drawing using indices to vertex buffer */
-	case RADEON_CP_INDX_BUFFER:	/* DRAW_INDX_2 without INDX_BUFFER seems to lock up the gpu */
+	case RADEON_CP_INDX_BUFFER:
+		DRM_ERROR("packet3 INDX_BUFFER without preceding 3D_DRAW_INDX_2 is illegal.\n");
+		return -EINVAL;
+	case RADEON_CP_3D_DRAW_IMMD_2:
+		/* triggers drawing using in-packet vertex data */
+	case RADEON_CP_3D_DRAW_VBUF_2:
+		/* triggers drawing of vertex buffers setup elsewhere */
+		dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
+					   RADEON_PURGE_EMITED);
+		break;
+	case RADEON_CP_3D_DRAW_INDX_2:
+		/* triggers drawing using indices to vertex buffer */
+		/* whenever we send vertex we clear flush & purge */
+		dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
+					   RADEON_PURGE_EMITED);
+		return r300_emit_draw_indx_2(dev_priv, cmdbuf);
 	case RADEON_WAIT_FOR_IDLE:
 	case RADEON_CP_NOP:
 		/* these packets are safe */
 		break;
 	default:
 		DRM_ERROR("Unknown packet3 header (0x%08x)\n", header);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	BEGIN_RING(count + 2);
@@ -655,7 +785,7 @@ static __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv,
 			DRM_ERROR("bad packet3 type %i at %p\n",
 				  header.packet3.packet,
 				  cmdbuf->buf - sizeof(header));
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		n += R300_SIMULTANEOUS_CLIPRECTS;
@@ -680,16 +810,53 @@ static __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv,
  */
 static __inline__ void r300_pacify(drm_radeon_private_t *dev_priv)
 {
+	uint32_t cache_z, cache_3d, cache_2d;
 	RING_LOCALS;
 
-	BEGIN_RING(6);
+	cache_z = R300_ZC_FLUSH;
+	cache_2d = R300_RB2D_DC_FLUSH;
+	cache_3d = R300_RB3D_DC_FLUSH;
+	if (!(dev_priv->track_flush & RADEON_PURGE_EMITED)) {
+		/* we can purge, primitive where draw since last purge */
+		cache_z |= R300_ZC_FREE;
+		cache_2d |= R300_RB2D_DC_FREE;
+		cache_3d |= R300_RB3D_DC_FREE;
+	}
+
+	/* flush & purge zbuffer */
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));
+	OUT_RING(cache_z);
+	ADVANCE_RING();
+	/* flush & purge 3d */
+	BEGIN_RING(2);
 	OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
-	OUT_RING(0xa);
-	OUT_RING(CP_PACKET0(0x4f18, 0));
-	OUT_RING(0x3);
-	OUT_RING(CP_PACKET3(RADEON_CP_NOP, 0));
-	OUT_RING(0x0);
+	OUT_RING(cache_3d);
 	ADVANCE_RING();
+	/* flush & purge texture */
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(R300_TX_INVALTAGS, 0));
+	OUT_RING(0);
+	ADVANCE_RING();
+	/* FIXME: is this one really needed ? */
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(R300_RB3D_AARESOLVE_CTL, 0));
+	OUT_RING(0);
+	ADVANCE_RING();
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
+	ADVANCE_RING();
+	/* flush & purge 2d through E2 as RB2D will trigger lockup */
+	BEGIN_RING(4);
+	OUT_RING(CP_PACKET0(R300_DSTCACHE_CTLSTAT, 0));
+	OUT_RING(cache_2d);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(RADEON_WAIT_2D_IDLECLEAN |
+		 RADEON_WAIT_HOST_IDLECLEAN);
+	ADVANCE_RING();
+	/* set flush & purge flags */
+	dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 }
 
 /**
@@ -697,7 +864,7 @@ static __inline__ void r300_pacify(drm_radeon_private_t *dev_priv)
  * The actual age emit is done by r300_do_cp_cmdbuf, which is why you must
  * be careful about how this function is called.
  */
-static void r300_discard_buffer(drm_device_t * dev, drm_buf_t * buf)
+static void r300_discard_buffer(struct drm_device * dev, struct drm_buf * buf)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
@@ -707,6 +874,47 @@ static void r300_discard_buffer(drm_device_t * dev, drm_buf_t * buf)
 	buf->used = 0;
 }
 
+static void r300_cmd_wait(drm_radeon_private_t * dev_priv,
+			  drm_r300_cmd_header_t header)
+{
+	u32 wait_until;
+	RING_LOCALS;
+
+	if (!header.wait.flags)
+		return;
+
+	wait_until = 0;
+
+	switch(header.wait.flags) {
+	case R300_WAIT_2D:
+		wait_until = RADEON_WAIT_2D_IDLE;
+		break;
+	case R300_WAIT_3D:
+		wait_until = RADEON_WAIT_3D_IDLE;
+		break;
+	case R300_NEW_WAIT_2D_3D:
+		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_3D_IDLE;
+		break;
+	case R300_NEW_WAIT_2D_2D_CLEAN:
+		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
+		break;
+	case R300_NEW_WAIT_3D_3D_CLEAN:
+		wait_until = RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
+		break;
+	case R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN:
+		wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
+		wait_until |= RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
+		break;
+	default:
+		return;
+	}
+
+	BEGIN_RING(2);
+	OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
+	OUT_RING(wait_until);
+	ADVANCE_RING();
+}
+
 static int r300_scratch(drm_radeon_private_t *dev_priv,
 			drm_radeon_kcmd_buffer_t *cmdbuf,
 			drm_r300_cmd_header_t header)
@@ -714,76 +922,122 @@ static int r300_scratch(drm_radeon_private_t *dev_priv,
 	u32 *ref_age_base;
 	u32 i, buf_idx, h_pending;
 	RING_LOCALS;
-	
+
 	if (cmdbuf->bufsz < sizeof(uint64_t) + header.scratch.n_bufs * sizeof(buf_idx) ) {
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
-	
+
 	if (header.scratch.reg >= 5) {
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
-	
+
 	dev_priv->scratch_ages[header.scratch.reg] ++;
-	
+
 	ref_age_base = (u32 *)(unsigned long)*((uint64_t *)cmdbuf->buf);
-	
+
 	cmdbuf->buf += sizeof(uint64_t);
 	cmdbuf->bufsz -= sizeof(uint64_t);
-	
+
 	for (i=0; i < header.scratch.n_bufs; i++) {
 		buf_idx = *(u32 *)cmdbuf->buf;
 		buf_idx *= 2; /* 8 bytes per buf */
-		
+
 		if (DRM_COPY_TO_USER(ref_age_base + buf_idx, &dev_priv->scratch_ages[header.scratch.reg], sizeof(u32))) {
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
-					
+
 		if (DRM_COPY_FROM_USER(&h_pending, ref_age_base + buf_idx + 1, sizeof(u32))) {
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
-					
+
 		if (h_pending == 0) {
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
-					
+
 		h_pending--;
-						
+
 		if (DRM_COPY_TO_USER(ref_age_base + buf_idx + 1, &h_pending, sizeof(u32))) {
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
-					
+
 		cmdbuf->buf += sizeof(buf_idx);
 		cmdbuf->bufsz -= sizeof(buf_idx);
 	}
-	
+
 	BEGIN_RING(2);
 	OUT_RING( CP_PACKET0( RADEON_SCRATCH_REG0 + header.scratch.reg * 4, 0 ) );
 	OUT_RING( dev_priv->scratch_ages[header.scratch.reg] );
 	ADVANCE_RING();
-	
+
 	return 0;
 }
 
 /**
+ * Uploads user-supplied vertex program instructions or parameters onto
+ * the graphics card.
+ * Called by r300_do_cp_cmdbuf.
+ */
+static __inline__ int r300_emit_r500fp(drm_radeon_private_t *dev_priv,
+				       drm_radeon_kcmd_buffer_t *cmdbuf,
+				       drm_r300_cmd_header_t header)
+{
+	int sz;
+	int addr;
+	int type;
+	int clamp;
+	int stride;
+	RING_LOCALS;
+
+	sz = header.r500fp.count;
+	/* address is 9 bits 0 - 8, bit 1 of flags is part of address */
+	addr = ((header.r500fp.adrhi_flags & 1) << 8) | header.r500fp.adrlo;
+
+	type = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_TYPE);
+	clamp = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_CLAMP);
+
+	addr |= (type << 16);
+	addr |= (clamp << 17);
+
+	stride = type ? 4 : 6;
+
+	DRM_DEBUG("r500fp %d %d type: %d\n", sz, addr, type);
+	if (!sz)
+		return 0;
+	if (sz * stride * 4 > cmdbuf->bufsz)
+		return -EINVAL;
+
+	BEGIN_RING(3 + sz * stride);
+	OUT_RING_REG(R500_GA_US_VECTOR_INDEX, addr);
+	OUT_RING(CP_PACKET0_TABLE(R500_GA_US_VECTOR_DATA, sz * stride - 1));
+	OUT_RING_TABLE((int *)cmdbuf->buf, sz * stride);
+
+	ADVANCE_RING();
+
+	cmdbuf->buf += sz * stride * 4;
+	cmdbuf->bufsz -= sz * stride * 4;
+
+	return 0;
+}
+
+
+/**
  * Parses and validates a user-supplied command buffer and emits appropriate
  * commands on the DMA ring buffer.
  * Called by the ioctl handler function radeon_cp_cmdbuf.
  */
-int r300_do_cp_cmdbuf(drm_device_t *dev,
-		      DRMFILE filp,
-		      drm_file_t *filp_priv,
+int r300_do_cp_cmdbuf(struct drm_device *dev,
+		      struct drm_file *file_priv,
 		      drm_radeon_kcmd_buffer_t *cmdbuf)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf = NULL;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf = NULL;
 	int emit_dispatch_age = 0;
 	int ret = 0;
 
 	DRM_DEBUG("\n");
 
-	/* See the comment above r300_emit_begin3d for why this call must be here,
-	 * and what the cleanup gotos are for. */
+	/* pacify */
 	r300_pacify(dev_priv);
 
 	if (cmdbuf->nbox <= R300_SIMULTANEOUS_CLIPRECTS) {
@@ -803,7 +1057,6 @@ int r300_do_cp_cmdbuf(drm_device_t *dev,
 
 		switch (header.header.cmd_type) {
 		case R300_CMD_PACKET0:
-			DRM_DEBUG("R300_CMD_PACKET0\n");
 			ret = r300_emit_packet0(dev_priv, cmdbuf, header);
 			if (ret) {
 				DRM_ERROR("r300_emit_packet0 failed\n");
@@ -869,15 +1122,16 @@ int r300_do_cp_cmdbuf(drm_device_t *dev,
 			if (idx < 0 || idx >= dma->buf_count) {
 				DRM_ERROR("buffer index %d (of %d max)\n",
 					  idx, dma->buf_count - 1);
-				ret = DRM_ERR(EINVAL);
+				ret = -EINVAL;
 				goto cleanup;
 			}
 
 			buf = dma->buflist[idx];
-			if (buf->filp != filp || buf->pending) {
+			if (buf->file_priv != file_priv || buf->pending) {
 				DRM_ERROR("bad buffer %p %p %d\n",
-					  buf->filp, filp, buf->pending);
-				ret = DRM_ERR(EINVAL);
+					  buf->file_priv, file_priv,
+					  buf->pending);
+				ret = -EINVAL;
 				goto cleanup;
 			}
 
@@ -886,19 +1140,8 @@ int r300_do_cp_cmdbuf(drm_device_t *dev,
 			break;
 
 		case R300_CMD_WAIT:
-			/* simple enough, we can do it here */
 			DRM_DEBUG("R300_CMD_WAIT\n");
-			if (header.wait.flags == 0)
-				break;	/* nothing to do */
-
-			{
-				RING_LOCALS;
-
-				BEGIN_RING(2);
-				OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
-				OUT_RING((header.wait.flags & 0xf) << 14);
-				ADVANCE_RING();
-			}
+			r300_cmd_wait(dev_priv, header);
 			break;
 
 		case R300_CMD_SCRATCH:
@@ -909,12 +1152,25 @@ int r300_do_cp_cmdbuf(drm_device_t *dev,
 				goto cleanup;
 			}
 			break;
-			
+
+		case R300_CMD_R500FP:
+			if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV515) {
+				DRM_ERROR("Calling r500 command on r300 card\n");
+				ret = -EINVAL;
+				goto cleanup;
+			}
+			DRM_DEBUG("R300_CMD_R500FP\n");
+			ret = r300_emit_r500fp(dev_priv, cmdbuf, header);
+			if (ret) {
+				DRM_ERROR("r300_emit_r500fp failed\n");
+				goto cleanup;
+			}
+			break;
 		default:
 			DRM_ERROR("bad cmd_type %i at %p\n",
 				  header.header.cmd_type,
 				  cmdbuf->buf - sizeof(header));
-			ret = DRM_ERR(EINVAL);
+			ret = -EINVAL;
 			goto cleanup;
 		}
 	}
diff --git a/sys/dev/drm/r300_reg.h b/sys/dev/drm/r300_reg.h
index 0108406..c92779e 100644
--- a/sys/dev/drm/r300_reg.h
+++ b/sys/dev/drm/r300_reg.h
@@ -26,6 +26,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+/* *INDENT-OFF* */
+
 #ifndef _R300_REG_H
 #define _R300_REG_H
 
@@ -51,12 +53,12 @@ __FBSDID("$FreeBSD$");
 #	define R300_MC_MISC__MC_GLOBW_FULL_LAT_SHIFT	28
 
 /*
-This file contains registers and constants for the R300. They have been
-found mostly by examining command buffers captured using glxtest, as well
-as by extrapolating some known registers and constants from the R200.
-
-I am fairly certain that they are correct unless stated otherwise in comments.
-*/
+ * This file contains registers and constants for the R300. They have been
+ * found mostly by examining command buffers captured using glxtest, as well
+ * as by extrapolating some known registers and constants from the R200.
+ * I am fairly certain that they are correct unless stated otherwise
+ * in comments.
+ */
 
 #define R300_SE_VPORT_XSCALE                0x1D98
 #define R300_SE_VPORT_XOFFSET               0x1D9C
@@ -66,49 +68,59 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_SE_VPORT_ZOFFSET               0x1DAC
 
 
-/* This register is written directly and also starts data section in many 3d CP_PACKET3's */
-#define R300_VAP_VF_CNTL	0x2084
+/*
+ * Vertex Array Processing (VAP) Control
+ * Stolen from r200 code from Christoph Brill (It's a guess!)
+ */
+#define R300_VAP_CNTL	0x2080
 
-#	define	R300_VAP_VF_CNTL__PRIM_TYPE__SHIFT                       0
-#	define  R300_VAP_VF_CNTL__PRIM_NONE				 (0<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_POINTS				 (1<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_LINES				 (2<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_LINE_STRIP			 (3<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLES			 (4<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLE_FAN			 (5<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLE_STRIP			 (6<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_LINE_LOOP			 (12<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_QUADS			 	 (13<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_QUAD_STRIP			 (14<<0)
-#	define  R300_VAP_VF_CNTL__PRIM_POLYGON			 	 (15<<0)
-
-#	define	R300_VAP_VF_CNTL__PRIM_WALK__SHIFT                       4
-	/* State based - direct writes to registers trigger vertex generation */
-#	define	R300_VAP_VF_CNTL__PRIM_WALK_STATE_BASED                      (0<<4)
-#	define	R300_VAP_VF_CNTL__PRIM_WALK_INDICES                          (1<<4)
-#	define	R300_VAP_VF_CNTL__PRIM_WALK_VERTEX_LIST                      (2<<4)
-#	define	R300_VAP_VF_CNTL__PRIM_WALK_VERTEX_EMBEDDED                  (3<<4)
-
-		/* I don't think I saw these three used.. */
-#	define	R300_VAP_VF_CNTL__COLOR_ORDER__SHIFT                     6
-#	define	R300_VAP_VF_CNTL__TCL_OUTPUT_CTL_ENA__SHIFT              9
-#	define	R300_VAP_VF_CNTL__PROG_STREAM_ENA__SHIFT                 10
-
-		/* index size - when not set the indices are assumed to be 16 bit */
-#	define	R300_VAP_VF_CNTL__INDEX_SIZE_32bit                      (1<<11)
-                /* number of vertices */
-#	define	R300_VAP_VF_CNTL__NUM_VERTICES__SHIFT                    16
+/* This register is written directly and also starts data section
+ * in many 3d CP_PACKET3's
+ */
+#define R300_VAP_VF_CNTL	0x2084
+#	define	R300_VAP_VF_CNTL__PRIM_TYPE__SHIFT              0
+#	define  R300_VAP_VF_CNTL__PRIM_NONE                     (0<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_POINTS                   (1<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_LINES                    (2<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_LINE_STRIP               (3<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLES                (4<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLE_FAN             (5<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_TRIANGLE_STRIP           (6<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_LINE_LOOP                (12<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_QUADS                    (13<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_QUAD_STRIP               (14<<0)
+#	define  R300_VAP_VF_CNTL__PRIM_POLYGON                  (15<<0)
+
+#	define	R300_VAP_VF_CNTL__PRIM_WALK__SHIFT              4
+	/* State based - direct writes to registers trigger vertex
+           generation */
+#	define	R300_VAP_VF_CNTL__PRIM_WALK_STATE_BASED         (0<<4)
+#	define	R300_VAP_VF_CNTL__PRIM_WALK_INDICES             (1<<4)
+#	define	R300_VAP_VF_CNTL__PRIM_WALK_VERTEX_LIST         (2<<4)
+#	define	R300_VAP_VF_CNTL__PRIM_WALK_VERTEX_EMBEDDED     (3<<4)
+
+	/* I don't think I saw these three used.. */
+#	define	R300_VAP_VF_CNTL__COLOR_ORDER__SHIFT            6
+#	define	R300_VAP_VF_CNTL__TCL_OUTPUT_CTL_ENA__SHIFT     9
+#	define	R300_VAP_VF_CNTL__PROG_STREAM_ENA__SHIFT        10
+
+	/* index size - when not set the indices are assumed to be 16 bit */
+#	define	R300_VAP_VF_CNTL__INDEX_SIZE_32bit              (1<<11)
+	/* number of vertices */
+#	define	R300_VAP_VF_CNTL__NUM_VERTICES__SHIFT           16
 
 /* BEGIN: Wild guesses */
 #define R300_VAP_OUTPUT_VTX_FMT_0           0x2090
 #       define R300_VAP_OUTPUT_VTX_FMT_0__POS_PRESENT     (1<<0)
 #       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_PRESENT   (1<<1)
-#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_1_PRESENT (1<<2) /* GUESS */
-#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_2_PRESENT (1<<3) /* GUESS */
-#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_3_PRESENT (1<<4) /* GUESS */
+#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_1_PRESENT (1<<2)  /* GUESS */
+#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_2_PRESENT (1<<3)  /* GUESS */
+#       define R300_VAP_OUTPUT_VTX_FMT_0__COLOR_3_PRESENT (1<<4)  /* GUESS */
 #       define R300_VAP_OUTPUT_VTX_FMT_0__PT_SIZE_PRESENT (1<<16) /* GUESS */
 
 #define R300_VAP_OUTPUT_VTX_FMT_1           0x2094
+	/* each of the following is 3 bits wide, specifies number
+	   of components */
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_0_COMP_CNT_SHIFT 0
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_1_COMP_CNT_SHIFT 3
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_2_COMP_CNT_SHIFT 6
@@ -117,7 +129,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_5_COMP_CNT_SHIFT 15
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_6_COMP_CNT_SHIFT 18
 #       define R300_VAP_OUTPUT_VTX_FMT_1__TEX_7_COMP_CNT_SHIFT 21
-/* END */
+/* END: Wild guesses */
 
 #define R300_SE_VTE_CNTL                  0x20b0
 #	define     R300_VPORT_X_SCALE_ENA                0x00000001
@@ -133,29 +145,40 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define     R300_VTX_ST_DENORMALIZED              0x00001000
 
 /* BEGIN: Vertex data assembly - lots of uncertainties */
+
 /* gap */
+
+#define R300_VAP_CNTL_STATUS              0x2140
+#	define R300_VC_NO_SWAP                  (0 << 0)
+#	define R300_VC_16BIT_SWAP               (1 << 0)
+#	define R300_VC_32BIT_SWAP               (2 << 0)
+#	define R300_VAP_TCL_BYPASS		(1 << 8)
+
+/* gap */
+
 /* Where do we get our vertex data?
-//
-// Vertex data either comes either from immediate mode registers or from
-// vertex arrays.
-// There appears to be no mixed mode (though we can force the pitch of
-// vertex arrays to 0, effectively reusing the same element over and over
-// again).
-//
-// Immediate mode is controlled by the INPUT_CNTL registers. I am not sure
-// if these registers influence vertex array processing.
-//
-// Vertex arrays are controlled via the 3D_LOAD_VBPNTR packet3.
-//
-// In both cases, vertex attributes are then passed through INPUT_ROUTE.
-
-// Beginning with INPUT_ROUTE_0_0 is a list of WORDs that route vertex data
-// into the vertex processor's input registers.
-// The first word routes the first input, the second word the second, etc.
-// The corresponding input is routed into the register with the given index.
-// The list is ended by a word with INPUT_ROUTE_END set.
-//
-// Always set COMPONENTS_4 in immediate mode. */
+ *
+ * Vertex data either comes either from immediate mode registers or from
+ * vertex arrays.
+ * There appears to be no mixed mode (though we can force the pitch of
+ * vertex arrays to 0, effectively reusing the same element over and over
+ * again).
+ *
+ * Immediate mode is controlled by the INPUT_CNTL registers. I am not sure
+ * if these registers influence vertex array processing.
+ *
+ * Vertex arrays are controlled via the 3D_LOAD_VBPNTR packet3.
+ *
+ * In both cases, vertex attributes are then passed through INPUT_ROUTE.
+ *
+ * Beginning with INPUT_ROUTE_0_0 is a list of WORDs that route vertex data
+ * into the vertex processor's input registers.
+ * The first word routes the first input, the second word the second, etc.
+ * The corresponding input is routed into the register with the given index.
+ * The list is ended by a word with INPUT_ROUTE_END set.
+ *
+ * Always set COMPONENTS_4 in immediate mode.
+ */
 
 #define R300_VAP_INPUT_ROUTE_0_0            0x2150
 #       define R300_INPUT_ROUTE_COMPONENTS_1     (0 << 0)
@@ -179,10 +202,12 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VAP_INPUT_ROUTE_0_7            0x216C
 
 /* gap */
+
 /* Notes:
-//  - always set up to produce at least two attributes:
-//    if vertex program uses only position, fglrx will set normal, too
-//  - INPUT_CNTL_0_COLOR and INPUT_CNTL_COLOR bits are always equal */
+ *  - always set up to produce at least two attributes:
+ *    if vertex program uses only position, fglrx will set normal, too
+ *  - INPUT_CNTL_0_COLOR and INPUT_CNTL_COLOR bits are always equal.
+ */
 #define R300_VAP_INPUT_CNTL_0               0x2180
 #       define R300_INPUT_CNTL_0_COLOR           0x00000001
 #define R300_VAP_INPUT_CNTL_1               0x2184
@@ -199,12 +224,14 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_INPUT_CNTL_TC7               0x00020000 /* GUESS */
 
 /* gap */
+
 /* Words parallel to INPUT_ROUTE_0; All words that are active in INPUT_ROUTE_0
-// are set to a swizzling bit pattern, other words are 0.
-//
-// In immediate mode, the pattern is always set to xyzw. In vertex array
-// mode, the swizzling pattern is e.g. used to set zw components in texture
-// coordinates with only tweo components. */
+ * are set to a swizzling bit pattern, other words are 0.
+ *
+ * In immediate mode, the pattern is always set to xyzw. In vertex array
+ * mode, the swizzling pattern is e.g. used to set zw components in texture
+ * coordinates with only tweo components.
+ */
 #define R300_VAP_INPUT_ROUTE_1_0            0x21E0
 #       define R300_INPUT_ROUTE_SELECT_X    0
 #       define R300_INPUT_ROUTE_SELECT_Y    1
@@ -213,11 +240,11 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_INPUT_ROUTE_SELECT_ZERO 4
 #       define R300_INPUT_ROUTE_SELECT_ONE  5
 #       define R300_INPUT_ROUTE_SELECT_MASK 7
-#       define R300_INPUT_ROUTE_X_SHIFT          0
-#       define R300_INPUT_ROUTE_Y_SHIFT          3
-#       define R300_INPUT_ROUTE_Z_SHIFT          6
-#       define R300_INPUT_ROUTE_W_SHIFT          9
-#       define R300_INPUT_ROUTE_ENABLE           (15 << 12)
+#       define R300_INPUT_ROUTE_X_SHIFT     0
+#       define R300_INPUT_ROUTE_Y_SHIFT     3
+#       define R300_INPUT_ROUTE_Z_SHIFT     6
+#       define R300_INPUT_ROUTE_W_SHIFT     9
+#       define R300_INPUT_ROUTE_ENABLE      (15 << 12)
 #define R300_VAP_INPUT_ROUTE_1_1            0x21E4
 #define R300_VAP_INPUT_ROUTE_1_2            0x21E8
 #define R300_VAP_INPUT_ROUTE_1_3            0x21EC
@@ -226,54 +253,77 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VAP_INPUT_ROUTE_1_6            0x21F8
 #define R300_VAP_INPUT_ROUTE_1_7            0x21FC
 
-/* END */
+/* END: Vertex data assembly */
 
 /* gap */
-/* BEGIN: Upload vertex program and data
-// The programmable vertex shader unit has a memory bank of unknown size
-// that can be written to in 16 byte units by writing the address into
-// UPLOAD_ADDRESS, followed by data in UPLOAD_DATA (multiples of 4 DWORDs).
-//
-// Pointers into the memory bank are always in multiples of 16 bytes.
-//
-// The memory bank is divided into areas with fixed meaning.
-//
-// Starting at address UPLOAD_PROGRAM: Vertex program instructions.
-// Native limits reported by drivers from ATI suggest size 256 (i.e. 4KB),
-// whereas the difference between known addresses suggests size 512.
-//
-// Starting at address UPLOAD_PARAMETERS: Vertex program parameters.
-// Native reported limits and the VPI layout suggest size 256, whereas
-// difference between known addresses suggests size 512.
-//
-// At address UPLOAD_POINTSIZE is a vector (0, 0, ps, 0), where ps is the
-// floating point pointsize. The exact purpose of this state is uncertain,
-// as there is also the R300_RE_POINTSIZE register.
-//
-// Multiple vertex programs and parameter sets can be loaded at once,
-// which could explain the size discrepancy. */
+
+/* BEGIN: Upload vertex program and data */
+
+/*
+ * The programmable vertex shader unit has a memory bank of unknown size
+ * that can be written to in 16 byte units by writing the address into
+ * UPLOAD_ADDRESS, followed by data in UPLOAD_DATA (multiples of 4 DWORDs).
+ *
+ * Pointers into the memory bank are always in multiples of 16 bytes.
+ *
+ * The memory bank is divided into areas with fixed meaning.
+ *
+ * Starting at address UPLOAD_PROGRAM: Vertex program instructions.
+ * Native limits reported by drivers from ATI suggest size 256 (i.e. 4KB),
+ * whereas the difference between known addresses suggests size 512.
+ *
+ * Starting at address UPLOAD_PARAMETERS: Vertex program parameters.
+ * Native reported limits and the VPI layout suggest size 256, whereas
+ * difference between known addresses suggests size 512.
+ *
+ * At address UPLOAD_POINTSIZE is a vector (0, 0, ps, 0), where ps is the
+ * floating point pointsize. The exact purpose of this state is uncertain,
+ * as there is also the R300_RE_POINTSIZE register.
+ *
+ * Multiple vertex programs and parameter sets can be loaded at once,
+ * which could explain the size discrepancy.
+ */
 #define R300_VAP_PVS_UPLOAD_ADDRESS         0x2200
 #       define R300_PVS_UPLOAD_PROGRAM           0x00000000
 #       define R300_PVS_UPLOAD_PARAMETERS        0x00000200
 #       define R300_PVS_UPLOAD_POINTSIZE         0x00000406
+
 /* gap */
+
 #define R300_VAP_PVS_UPLOAD_DATA            0x2208
-/* END */
+
+/* END: Upload vertex program and data */
 
 /* gap */
+
 /* I do not know the purpose of this register. However, I do know that
-// it is set to 221C_CLEAR for clear operations and to 221C_NORMAL
-// for normal rendering. */
+ * it is set to 221C_CLEAR for clear operations and to 221C_NORMAL
+ * for normal rendering.
+ */
 #define R300_VAP_UNKNOWN_221C               0x221C
 #       define R300_221C_NORMAL                  0x00000000
 #       define R300_221C_CLEAR                   0x0001C000
 
+/* These seem to be per-pixel and per-vertex X and Y clipping planes. The first
+ * plane is per-pixel and the second plane is per-vertex.
+ *
+ * This was determined by experimentation alone but I believe it is correct.
+ *
+ * These registers are called X_QUAD0_1_FL to X_QUAD0_4_FL by glxtest.
+ */
+#define R300_VAP_CLIP_X_0                   0x2220
+#define R300_VAP_CLIP_X_1                   0x2224
+#define R300_VAP_CLIP_Y_0                   0x2228
+#define R300_VAP_CLIP_Y_1                   0x2230
+
 /* gap */
+
 /* Sometimes, END_OF_PKT and 0x2284=0 are the only commands sent between
-// rendering commands and overwriting vertex program parameters.
-// Therefore, I suspect writing zero to 0x2284 synchronizes the engine and
-// avoids bugs caused by still running shaders reading bad data from memory. */
-#define R300_VAP_PVS_WAITIDLE               0x2284 /* GUESS */
+ * rendering commands and overwriting vertex program parameters.
+ * Therefore, I suspect writing zero to 0x2284 synchronizes the engine and
+ * avoids bugs caused by still running shaders reading bad data from memory.
+ */
+#define R300_VAP_PVS_STATE_FLUSH_REG        0x2284
 
 /* Absolutely no clue what this register is about. */
 #define R300_VAP_UNKNOWN_2288               0x2288
@@ -281,19 +331,24 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_2288_RV350                   0x0000FFFF /* -- Vladimir */
 
 /* gap */
+
 /* Addresses are relative to the vertex program instruction area of the
-// memory bank. PROGRAM_END points to the last instruction of the active
-// program
-//
-// The meaning of the two UNKNOWN fields is obviously not known. However,
-// experiments so far have shown that both *must* point to an instruction
-// inside the vertex program, otherwise the GPU locks up.
-// fglrx usually sets CNTL_3_UNKNOWN to the end of the program and
-// CNTL_1_UNKNOWN points to instruction where last write to position takes place. 
-// Most likely this is used to ignore rest of the program in cases where group of verts arent visible.
-// For some reason this "section" is sometimes accepted other instruction that have
-// no relationship with position calculations. 
-*/
+ * memory bank. PROGRAM_END points to the last instruction of the active
+ * program
+ *
+ * The meaning of the two UNKNOWN fields is obviously not known. However,
+ * experiments so far have shown that both *must* point to an instruction
+ * inside the vertex program, otherwise the GPU locks up.
+ *
+ * fglrx usually sets CNTL_3_UNKNOWN to the end of the program and
+ * R300_PVS_CNTL_1_POS_END_SHIFT points to instruction where last write to
+ * position takes place.
+ *
+ * Most likely this is used to ignore rest of the program in cases
+ * where group of verts arent visible. For some reason this "section"
+ * is sometimes accepted other instruction that have no relationship with
+ * position calculations.
+ */
 #define R300_VAP_PVS_CNTL_1                 0x22D0
 #       define R300_PVS_CNTL_1_PROGRAM_START_SHIFT   0
 #       define R300_PVS_CNTL_1_POS_END_SHIFT         10
@@ -307,7 +362,8 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_PVS_CNTL_3_PROGRAM_UNKNOWN2_SHIFT 0
 
 /* The entire range from 0x2300 to 0x2AC inclusive seems to be used for
-// immediate vertices */
+ * immediate vertices
+ */
 #define R300_VAP_VTX_COLOR_R                0x2464
 #define R300_VAP_VTX_COLOR_G                0x2468
 #define R300_VAP_VTX_COLOR_B                0x246C
@@ -317,13 +373,15 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VAP_VTX_POS_0_X_2              0x24A0 /* used for glVertex3*() */
 #define R300_VAP_VTX_POS_0_Y_2              0x24A4
 #define R300_VAP_VTX_POS_0_Z_2              0x24A8
-#define R300_VAP_VTX_END_OF_PKT             0x24AC /* write 0 to indicate end of packet? */
+/* write 0 to indicate end of packet? */
+#define R300_VAP_VTX_END_OF_PKT             0x24AC
 
 /* gap */
 
 /* These are values from r300_reg/r300_reg.h - they are known to be correct
-   and are here so we can use one register file instead of several
-   - Vladimir */
+ * and are here so we can use one register file instead of several
+ * - Vladimir
+ */
 #define R300_GB_VAP_RASTER_VTX_FMT_0	0x4000
 #	define R300_GB_VAP_RASTER_VTX_FMT_0__POS_PRESENT	(1<<0)
 #	define R300_GB_VAP_RASTER_VTX_FMT_0__COLOR_0_PRESENT	(1<<1)
@@ -346,14 +404,16 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_GB_VAP_RASTER_VTX_FMT_1__TEX_7_COMP_CNT_SHIFT	21
 
 /* UNK30 seems to enables point to quad transformation on textures
-   (or something closely related to that).
-   This bit is rather fatal at the time being due to lackings at pixel shader side */
+ * (or something closely related to that).
+ * This bit is rather fatal at the time being due to lackings at pixel
+ * shader side
+ */
 #define R300_GB_ENABLE	0x4008
 #	define R300_GB_POINT_STUFF_ENABLE	(1<<0)
 #	define R300_GB_LINE_STUFF_ENABLE	(1<<1)
 #	define R300_GB_TRIANGLE_STUFF_ENABLE	(1<<2)
 #	define R300_GB_STENCIL_AUTO_ENABLE	(1<<4)
-#	define R300_GB_UNK30			(1<<30)
+#	define R300_GB_UNK31			(1<<31)
 	/* each of the following is 2 bits wide */
 #define R300_GB_TEX_REPLICATE	0
 #define R300_GB_TEX_ST		1
@@ -394,6 +454,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_GB_TILE_PIPE_COUNT_RV300	0
 #	define R300_GB_TILE_PIPE_COUNT_R300	(3<<1)
 #	define R300_GB_TILE_PIPE_COUNT_R420	(7<<1)
+#	define R300_GB_TILE_PIPE_COUNT_RV410	(3<<1)
 #	define R300_GB_TILE_SIZE_8		0
 #	define R300_GB_TILE_SIZE_16		(1<<4)
 #	define R300_GB_TILE_SIZE_32		(2<<4)
@@ -448,17 +509,18 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_GB_W_SELECT_1		(1<<4)
 
 #define R300_GB_AA_CONFIG		0x4020
+#	define R300_AA_DISABLE			0x00
 #	define R300_AA_ENABLE			0x01
 #	define R300_AA_SUBSAMPLES_2		0
 #	define R300_AA_SUBSAMPLES_3		(1<<1)
 #	define R300_AA_SUBSAMPLES_4		(2<<1)
 #	define R300_AA_SUBSAMPLES_6		(3<<1)
 
-/* END */
-
 /* gap */
+
 /* Zero to flush caches. */
-#define R300_TX_CNTL                        0x4100
+#define R300_TX_INVALTAGS                   0x4100
+#define R300_TX_FLUSH                       0x0
 
 /* The upper enable bits are guessed, based on fglrx reported limits. */
 #define R300_TX_ENABLE                      0x4104
@@ -480,8 +542,9 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_TX_ENABLE_15                 (1 << 15)
 
 /* The pointsize is given in multiples of 6. The pointsize can be
-// enormous: Clear() renders a single point that fills the entire
-// framebuffer. */
+ * enormous: Clear() renders a single point that fills the entire
+ * framebuffer.
+ */
 #define R300_RE_POINTSIZE                   0x421C
 #       define R300_POINTSIZE_Y_SHIFT            0
 #       define R300_POINTSIZE_Y_MASK             (0xFFFF << 0) /* GUESS */
@@ -490,11 +553,11 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_POINTSIZE_MAX             (R300_POINTSIZE_Y_MASK / 6)
 
 /* The line width is given in multiples of 6.
-   In default mode lines are classified as vertical lines.
-   HO: horizontal
-   VE: vertical or horizontal
-   HO & VE: no classification
-*/
+ * In default mode lines are classified as vertical lines.
+ * HO: horizontal
+ * VE: vertical or horizontal
+ * HO & VE: no classification
+ */
 #define R300_RE_LINE_CNT                      0x4234
 #       define R300_LINESIZE_SHIFT            0
 #       define R300_LINESIZE_MASK             (0xFFFF << 0) /* GUESS */
@@ -505,6 +568,9 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 /* Some sort of scale or clamp value for texcoordless textures. */
 #define R300_RE_UNK4238                       0x4238
 
+/* Something shade related */
+#define R300_RE_SHADE                         0x4274
+
 #define R300_RE_SHADE_MODEL                   0x4278
 #	define R300_RE_SHADE_MODEL_SMOOTH     0x3aaaa
 #	define R300_RE_SHADE_MODEL_FLAT       0x39595
@@ -519,24 +585,31 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_PM_BACK_LINE              (1 << 7)
 #	define R300_PM_BACK_FILL              (1 << 8)
 
-/* Not sure why there are duplicate of factor and constant values. 
-   My best guess so far is that there are seperate zbiases for test and write. 
-   Ordering might be wrong.
-   Some of the tests indicate that fgl has a fallback implementation of zbias
-   via pixel shaders. */
+/* Fog parameters */
+#define R300_RE_FOG_SCALE                     0x4294
+#define R300_RE_FOG_START                     0x4298
+
+/* Not sure why there are duplicate of factor and constant values.
+ * My best guess so far is that there are seperate zbiases for test and write.
+ * Ordering might be wrong.
+ * Some of the tests indicate that fgl has a fallback implementation of zbias
+ * via pixel shaders.
+ */
+#define R300_RE_ZBIAS_CNTL                    0x42A0 /* GUESS */
 #define R300_RE_ZBIAS_T_FACTOR                0x42A4
 #define R300_RE_ZBIAS_T_CONSTANT              0x42A8
 #define R300_RE_ZBIAS_W_FACTOR                0x42AC
 #define R300_RE_ZBIAS_W_CONSTANT              0x42B0
 
 /* This register needs to be set to (1<<1) for RV350 to correctly
-   perform depth test (see --vb-triangles in r300_demo)
-   Don't know about other chips. - Vladimir
-   This is set to 3 when GL_POLYGON_OFFSET_FILL is on.
-   My guess is that there are two bits for each zbias primitive (FILL, LINE, POINT).
-   One to enable depth test and one for depth write.
-   Yet this doesnt explain why depth writes work ...
-    */
+ * perform depth test (see --vb-triangles in r300_demo)
+ * Don't know about other chips. - Vladimir
+ * This is set to 3 when GL_POLYGON_OFFSET_FILL is on.
+ * My guess is that there are two bits for each zbias primitive
+ * (FILL, LINE, POINT).
+ *  One to enable depth test and one for depth write.
+ * Yet this doesnt explain why depth writes work ...
+ */
 #define R300_RE_OCCLUSION_CNTL		    0x42B4
 #	define R300_OCCLUSION_ON		(1<<1)
 
@@ -547,30 +620,37 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FRONT_FACE_CW                (1 << 2)
 
 
-/* BEGIN: Rasterization / Interpolators - many guesses
-// 0_UNKNOWN_18 has always been set except for clear operations.
-// TC_CNT is the number of incoming texture coordinate sets (i.e. it depends
-// on the vertex program, *not* the fragment program) */
+/* BEGIN: Rasterization / Interpolators - many guesses */
+
+/* 0_UNKNOWN_18 has always been set except for clear operations.
+ * TC_CNT is the number of incoming texture coordinate sets (i.e. it depends
+ * on the vertex program, *not* the fragment program)
+ */
 #define R300_RS_CNTL_0                      0x4300
 #       define R300_RS_CNTL_TC_CNT_SHIFT         2
 #       define R300_RS_CNTL_TC_CNT_MASK          (7 << 2)
-#		define R300_RS_CNTL_CI_CNT_SHIFT         7 /* number of color interpolators used */
+	/* number of color interpolators used */
+#	define R300_RS_CNTL_CI_CNT_SHIFT         7
 #       define R300_RS_CNTL_0_UNKNOWN_18         (1 << 18)
-/* Guess: RS_CNTL_1 holds the index of the highest used RS_ROUTE_n register. */
+	/* Guess: RS_CNTL_1 holds the index of the highest used RS_ROUTE_n
+	   register. */
 #define R300_RS_CNTL_1                      0x4304
 
 /* gap */
+
 /* Only used for texture coordinates.
-// Use the source field to route texture coordinate input from the vertex program
-// to the desired interpolator. Note that the source field is relative to the
-// outputs the vertex program *actually* writes. If a vertex program only writes
-// texcoord[1], this will be source index 0.
-// Set INTERP_USED on all interpolators that produce data used by the
-// fragment program. INTERP_USED looks like a swizzling mask, but
-// I haven't seen it used that way.
-//
-// Note: The _UNKNOWN constants are always set in their respective register.
-// I don't know if this is necessary. */
+ * Use the source field to route texture coordinate input from the
+ * vertex program to the desired interpolator. Note that the source
+ * field is relative to the outputs the vertex program *actually*
+ * writes. If a vertex program only writes texcoord[1], this will
+ * be source index 0.
+ * Set INTERP_USED on all interpolators that produce data used by
+ * the fragment program. INTERP_USED looks like a swizzling mask,
+ * but I haven't seen it used that way.
+ *
+ * Note: The _UNKNOWN constants are always set in their respective
+ * register. I don't know if this is necessary.
+ */
 #define R300_RS_INTERP_0                    0x4310
 #define R300_RS_INTERP_1                    0x4314
 #       define R300_RS_INTERP_1_UNKNOWN          0x40
@@ -587,7 +667,8 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_RS_INTERP_USED               0x00D10000
 
 /* These DWORDs control how vertex data is routed into fragment program
-// registers, after interpolators. */
+ * registers, after interpolators.
+ */
 #define R300_RS_ROUTE_0                     0x4330
 #define R300_RS_ROUTE_1                     0x4334
 #define R300_RS_ROUTE_2                     0x4338
@@ -609,8 +690,14 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_RS_ROUTE_DEST_MASK           (31 << 6) /* GUESS */
 
 /* Special handling for color: When the fragment program uses color,
-// the ROUTE_0_COLOR bit is set and ROUTE_0_COLOR_DEST contains the
-// color register index. */
+ * the ROUTE_0_COLOR bit is set and ROUTE_0_COLOR_DEST contains the
+ * color register index.
+ *
+ * Apperently you may set the R300_RS_ROUTE_0_COLOR bit, but not provide any
+ * R300_RS_ROUTE_0_COLOR_DEST value; this setup is used for clearing the state.
+ * See r300_ioctl.c:r300EmitClearState. I'm not sure if this setup is strictly
+ * correct or not. - Oliver.
+ */
 #       define R300_RS_ROUTE_0_COLOR             (1 << 14)
 #       define R300_RS_ROUTE_0_COLOR_DEST_SHIFT  17
 #       define R300_RS_ROUTE_0_COLOR_DEST_MASK   (31 << 17) /* GUESS */
@@ -619,22 +706,45 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #		define R300_RS_ROUTE_1_COLOR1_DEST_SHIFT 17
 #		define R300_RS_ROUTE_1_COLOR1_DEST_MASK  (31 << 17)
 #		define R300_RS_ROUTE_1_UNKNOWN11         (1 << 11)
-/* END */
-
-/* BEGIN: Scissors and cliprects
-// There are four clipping rectangles. Their corner coordinates are inclusive.
-// Every pixel is assigned a number from 0 and 15 by setting bits 0-3 depending
-// on whether the pixel is inside cliprects 0-3, respectively. For example,
-// if a pixel is inside cliprects 0 and 1, but outside 2 and 3, it is assigned
-// the number 3 (binary 0011).
-// Iff the bit corresponding to the pixel's number in RE_CLIPRECT_CNTL is set,
-// the pixel is rasterized.
-//
-// In addition to this, there is a scissors rectangle. Only pixels inside the
-// scissors rectangle are drawn. (coordinates are inclusive)
-//
-// For some reason, the top-left corner of the framebuffer is at (1440, 1440)
-// for the purpose of clipping and scissors. */
+/* END: Rasterization / Interpolators - many guesses */
+
+/* Hierarchical Z Enable */
+#define R300_SC_HYPERZ                   0x43a4
+#	define R300_SC_HYPERZ_DISABLE     (0 << 0)
+#	define R300_SC_HYPERZ_ENABLE      (1 << 0)
+#	define R300_SC_HYPERZ_MIN         (0 << 1)
+#	define R300_SC_HYPERZ_MAX         (1 << 1)
+#	define R300_SC_HYPERZ_ADJ_256     (0 << 2)
+#	define R300_SC_HYPERZ_ADJ_128     (1 << 2)
+#	define R300_SC_HYPERZ_ADJ_64      (2 << 2)
+#	define R300_SC_HYPERZ_ADJ_32      (3 << 2)
+#	define R300_SC_HYPERZ_ADJ_16      (4 << 2)
+#	define R300_SC_HYPERZ_ADJ_8       (5 << 2)
+#	define R300_SC_HYPERZ_ADJ_4       (6 << 2)
+#	define R300_SC_HYPERZ_ADJ_2       (7 << 2)
+#	define R300_SC_HYPERZ_HZ_Z0MIN_NO (0 << 5)
+#	define R300_SC_HYPERZ_HZ_Z0MIN    (1 << 5)
+#	define R300_SC_HYPERZ_HZ_Z0MAX_NO (0 << 6)
+#	define R300_SC_HYPERZ_HZ_Z0MAX    (1 << 6)
+
+#define R300_SC_EDGERULE                 0x43a8
+
+/* BEGIN: Scissors and cliprects */
+
+/* There are four clipping rectangles. Their corner coordinates are inclusive.
+ * Every pixel is assigned a number from 0 and 15 by setting bits 0-3 depending
+ * on whether the pixel is inside cliprects 0-3, respectively. For example,
+ * if a pixel is inside cliprects 0 and 1, but outside 2 and 3, it is assigned
+ * the number 3 (binary 0011).
+ * Iff the bit corresponding to the pixel's number in RE_CLIPRECT_CNTL is set,
+ * the pixel is rasterized.
+ *
+ * In addition to this, there is a scissors rectangle. Only pixels inside the
+ * scissors rectangle are drawn. (coordinates are inclusive)
+ *
+ * For some reason, the top-left corner of the framebuffer is at (1440, 1440)
+ * for the purpose of clipping and scissors.
+ */
 #define R300_RE_CLIPRECT_TL_0               0x43B0
 #define R300_RE_CLIPRECT_BR_0               0x43B4
 #define R300_RE_CLIPRECT_TL_1               0x43B8
@@ -668,6 +778,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_CLIP_3210                    (1 << 15)
 
 /* gap */
+
 #define R300_RE_SCISSORS_TL                 0x43E0
 #define R300_RE_SCISSORS_BR                 0x43E4
 #       define R300_SCISSORS_OFFSET              1440
@@ -675,12 +786,15 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_SCISSORS_X_MASK              (0x1FFF << 0)
 #       define R300_SCISSORS_Y_SHIFT             13
 #       define R300_SCISSORS_Y_MASK              (0x1FFF << 13)
-/* END */
+/* END: Scissors and cliprects */
+
+/* BEGIN: Texture specification */
 
-/* BEGIN: Texture specification
-// The texture specification dwords are grouped by meaning and not by texture unit.
-// This means that e.g. the offset for texture image unit N is found in register
-// TX_OFFSET_0 + (4*N) */
+/*
+ * The texture specification dwords are grouped by meaning and not by texture
+ * unit. This means that e.g. the offset for texture image unit N is found in
+ * register TX_OFFSET_0 + (4*N)
+ */
 #define R300_TX_FILTER_0                    0x4400
 #       define R300_TX_REPEAT                    0
 #       define R300_TX_MIRRORED                  1
@@ -704,13 +818,14 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_TX_MIN_FILTER_LINEAR_MIP_LINEAR         (10 <<  11)
 
 /* NOTE: NEAREST doesnt seem to exist.
-   Im not seting MAG_FILTER_MASK and (3 << 11) on for all
-   anisotropy modes because that would void selected mag filter */
-#	define R300_TX_MIN_FILTER_ANISO_NEAREST             ((0 << 13) /*|R300_TX_MAG_FILTER_MASK|(3<<11)*/)
-#	define R300_TX_MIN_FILTER_ANISO_LINEAR              ((0 << 13) /*|R300_TX_MAG_FILTER_MASK|(3<<11)*/)
-#	define R300_TX_MIN_FILTER_ANISO_NEAREST_MIP_NEAREST ((1 << 13) /*|R300_TX_MAG_FILTER_MASK|(3<<11)*/)
-#	define R300_TX_MIN_FILTER_ANISO_NEAREST_MIP_LINEAR  ((2 << 13) /*|R300_TX_MAG_FILTER_MASK|(3<<11)*/)
-#       define R300_TX_MIN_FILTER_MASK           ( (15 << 11) | (3 << 13) )
+ * Im not seting MAG_FILTER_MASK and (3 << 11) on for all
+ * anisotropy modes because that would void selected mag filter
+ */
+#	define R300_TX_MIN_FILTER_ANISO_NEAREST             (0 << 13)
+#	define R300_TX_MIN_FILTER_ANISO_LINEAR              (0 << 13)
+#	define R300_TX_MIN_FILTER_ANISO_NEAREST_MIP_NEAREST (1 << 13)
+#	define R300_TX_MIN_FILTER_ANISO_NEAREST_MIP_LINEAR  (2 << 13)
+#       define R300_TX_MIN_FILTER_MASK   ( (15 << 11) | (3 << 13) )
 #	define R300_TX_MAX_ANISO_1_TO_1  (0 << 21)
 #	define R300_TX_MAX_ANISO_2_TO_1  (2 << 21)
 #	define R300_TX_MAX_ANISO_4_TO_1  (4 << 21)
@@ -741,10 +856,10 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_TX_HEIGHTMASK_SHIFT          11
 #       define R300_TX_HEIGHTMASK_MASK           (2047 << 11)
 #       define R300_TX_UNK23                     (1 << 23)
-#       define R300_TX_SIZE_SHIFT                26 /* largest of width, height */
-#       define R300_TX_SIZE_MASK                 (15 << 26)
-#       define R300_TX_SIZE_PROJECTED                     (1<<30)
-#       define R300_TX_SIZE_TXPITCH_EN                     (1<<31)
+#       define R300_TX_MAX_MIP_LEVEL_SHIFT       26
+#       define R300_TX_MAX_MIP_LEVEL_MASK        (0xf << 26)
+#       define R300_TX_SIZE_PROJECTED            (1<<30)
+#       define R300_TX_SIZE_TXPITCH_EN           (1<<31)
 #define R300_TX_FORMAT_0                    0x44C0
 	/* The interpretation of the format word by Wladimir van der Laan */
 	/* The X, Y, Z and W refer to the layout of the components.
@@ -765,33 +880,33 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_TX_FORMAT_W8Z8Y8X8		    0xC
 #	define R300_TX_FORMAT_W2Z10Y10X10	    0xD
 #	define R300_TX_FORMAT_W16Z16Y16X16	    0xE
-#	define R300_TX_FORMAT_DXT1	    	    0xF
-#	define R300_TX_FORMAT_DXT3	    	    0x10
-#	define R300_TX_FORMAT_DXT5	    	    0x11
+#	define R300_TX_FORMAT_DXT1		    0xF
+#	define R300_TX_FORMAT_DXT3		    0x10
+#	define R300_TX_FORMAT_DXT5		    0x11
 #	define R300_TX_FORMAT_D3DMFT_CxV8U8	    0x12     /* no swizzle */
-#	define R300_TX_FORMAT_A8R8G8B8	    	    0x13     /* no swizzle */
-#	define R300_TX_FORMAT_B8G8_B8G8	    	    0x14     /* no swizzle */
-#	define R300_TX_FORMAT_G8R8_G8B8	    	    0x15     /* no swizzle */
-						  /* 0x16 - some 16 bit green format.. ?? */
+#	define R300_TX_FORMAT_A8R8G8B8		    0x13     /* no swizzle */
+#	define R300_TX_FORMAT_B8G8_B8G8		    0x14     /* no swizzle */
+#	define R300_TX_FORMAT_G8R8_G8B8		    0x15     /* no swizzle */
+	/* 0x16 - some 16 bit green format.. ?? */
 #	define R300_TX_FORMAT_UNK25		   (1 << 25) /* no swizzle */
 #	define R300_TX_FORMAT_CUBIC_MAP		   (1 << 26)
 
 	/* gap */
 	/* Floating point formats */
 	/* Note - hardware supports both 16 and 32 bit floating point */
-#	define R300_TX_FORMAT_FL_I16	    	    0x18
-#	define R300_TX_FORMAT_FL_I16A16	    	    0x19
+#	define R300_TX_FORMAT_FL_I16		    0x18
+#	define R300_TX_FORMAT_FL_I16A16		    0x19
 #	define R300_TX_FORMAT_FL_R16G16B16A16	    0x1A
-#	define R300_TX_FORMAT_FL_I32	    	    0x1B
-#	define R300_TX_FORMAT_FL_I32A32	    	    0x1C
+#	define R300_TX_FORMAT_FL_I32		    0x1B
+#	define R300_TX_FORMAT_FL_I32A32		    0x1C
 #	define R300_TX_FORMAT_FL_R32G32B32A32	    0x1D
 	/* alpha modes, convenience mostly */
 	/* if you have alpha, pick constant appropriate to the
 	   number of channels (1 for I8, 2 for I8A8, 4 for R8G8B8A8, etc */
-# 	define R300_TX_FORMAT_ALPHA_1CH		    0x000
-# 	define R300_TX_FORMAT_ALPHA_2CH		    0x200
-# 	define R300_TX_FORMAT_ALPHA_4CH		    0x600
-# 	define R300_TX_FORMAT_ALPHA_NONE	    0xA00
+#	define R300_TX_FORMAT_ALPHA_1CH		    0x000
+#	define R300_TX_FORMAT_ALPHA_2CH		    0x200
+#	define R300_TX_FORMAT_ALPHA_4CH		    0x600
+#	define R300_TX_FORMAT_ALPHA_NONE	    0xA00
 	/* Swizzling */
 	/* constants */
 #	define R300_TX_FORMAT_X		0
@@ -800,23 +915,26 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_TX_FORMAT_W		3
 #	define R300_TX_FORMAT_ZERO	4
 #	define R300_TX_FORMAT_ONE	5
-#	define R300_TX_FORMAT_CUT_Z	6		/* 2.0*Z, everything above 1.0 is set to 0.0 */
-#	define R300_TX_FORMAT_CUT_W	7		/* 2.0*W, everything above 1.0 is set to 0.0 */
+	/* 2.0*Z, everything above 1.0 is set to 0.0 */
+#	define R300_TX_FORMAT_CUT_Z	6
+	/* 2.0*W, everything above 1.0 is set to 0.0 */
+#	define R300_TX_FORMAT_CUT_W	7
 
 #	define R300_TX_FORMAT_B_SHIFT	18
 #	define R300_TX_FORMAT_G_SHIFT	15
 #	define R300_TX_FORMAT_R_SHIFT	12
 #	define R300_TX_FORMAT_A_SHIFT	9
 	/* Convenience macro to take care of layout and swizzling */
-#	define R300_EASY_TX_FORMAT(B, G, R, A, FMT)	(\
-	  ((R300_TX_FORMAT_##B)<<R300_TX_FORMAT_B_SHIFT) \
-	| ((R300_TX_FORMAT_##G)<<R300_TX_FORMAT_G_SHIFT) \
-	| ((R300_TX_FORMAT_##R)<<R300_TX_FORMAT_R_SHIFT) \
-	| ((R300_TX_FORMAT_##A)<<R300_TX_FORMAT_A_SHIFT) \
-	| (R300_TX_FORMAT_##FMT) \
-	  )
-	/* These can be ORed with result of R300_EASY_TX_FORMAT() */
-	/* We don't really know what they do. Take values from a constant color ? */
+#	define R300_EASY_TX_FORMAT(B, G, R, A, FMT)	(		\
+		((R300_TX_FORMAT_##B)<<R300_TX_FORMAT_B_SHIFT)		\
+		| ((R300_TX_FORMAT_##G)<<R300_TX_FORMAT_G_SHIFT)	\
+		| ((R300_TX_FORMAT_##R)<<R300_TX_FORMAT_R_SHIFT)	\
+		| ((R300_TX_FORMAT_##A)<<R300_TX_FORMAT_A_SHIFT)	\
+		| (R300_TX_FORMAT_##FMT)				\
+		)
+	/* These can be ORed with result of R300_EASY_TX_FORMAT()
+	   We don't really know what they do. Take values from a
+           constant color ? */
 #	define R300_TX_FORMAT_CONST_X		(1<<5)
 #	define R300_TX_FORMAT_CONST_Y		(2<<5)
 #	define R300_TX_FORMAT_CONST_Z		(4<<5)
@@ -826,7 +944,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 
 #define R300_TX_PITCH_0			    0x4500 /* obvious missing in gap */
 #define R300_TX_OFFSET_0                    0x4540
-/* BEGIN: Guess from R200 */
+	/* BEGIN: Guess from R200 */
 #       define R300_TXO_ENDIAN_NO_SWAP           (0 << 0)
 #       define R300_TXO_ENDIAN_BYTE_SWAP         (1 << 0)
 #       define R300_TXO_ENDIAN_WORD_SWAP         (2 << 0)
@@ -835,53 +953,61 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_TXO_MICRO_TILE               (1 << 3)
 #       define R300_TXO_OFFSET_MASK              0xffffffe0
 #       define R300_TXO_OFFSET_SHIFT             5
-/* END */
-#define R300_TX_CHROMA_KEY_0                      0x4580 /* 32 bit chroma key */
-#define R300_TX_BORDER_COLOR_0              0x45C0 //ff00ff00 == { 0, 1.0, 0, 1.0 }
-
-/* END */
-
-/* BEGIN: Fragment program instruction set
-// Fragment programs are written directly into register space.
-// There are separate instruction streams for texture instructions and ALU
-// instructions.
-// In order to synchronize these streams, the program is divided into up
-// to 4 nodes. Each node begins with a number of TEX operations, followed
-// by a number of ALU operations.
-// The first node can have zero TEX ops, all subsequent nodes must have at least
-// one TEX ops.
-// All nodes must have at least one ALU op.
-//
-// The index of the last node is stored in PFS_CNTL_0: A value of 0 means
-// 1 node, a value of 3 means 4 nodes.
-// The total amount of instructions is defined in PFS_CNTL_2. The offsets are
-// offsets into the respective instruction streams, while *_END points to the
-// last instruction relative to this offset. */
+	/* END: Guess from R200 */
+
+/* 32 bit chroma key */
+#define R300_TX_CHROMA_KEY_0                      0x4580
+/* ff00ff00 == { 0, 1.0, 0, 1.0 } */
+#define R300_TX_BORDER_COLOR_0              0x45C0
+
+/* END: Texture specification */
+
+/* BEGIN: Fragment program instruction set */
+
+/* Fragment programs are written directly into register space.
+ * There are separate instruction streams for texture instructions and ALU
+ * instructions.
+ * In order to synchronize these streams, the program is divided into up
+ * to 4 nodes. Each node begins with a number of TEX operations, followed
+ * by a number of ALU operations.
+ * The first node can have zero TEX ops, all subsequent nodes must have at
+ * least
+ * one TEX ops.
+ * All nodes must have at least one ALU op.
+ *
+ * The index of the last node is stored in PFS_CNTL_0: A value of 0 means
+ * 1 node, a value of 3 means 4 nodes.
+ * The total amount of instructions is defined in PFS_CNTL_2. The offsets are
+ * offsets into the respective instruction streams, while *_END points to the
+ * last instruction relative to this offset.
+ */
 #define R300_PFS_CNTL_0                     0x4600
 #       define R300_PFS_CNTL_LAST_NODES_SHIFT    0
 #       define R300_PFS_CNTL_LAST_NODES_MASK     (3 << 0)
 #       define R300_PFS_CNTL_FIRST_NODE_HAS_TEX  (1 << 3)
 #define R300_PFS_CNTL_1                     0x4604
 /* There is an unshifted value here which has so far always been equal to the
-// index of the highest used temporary register. */
+ * index of the highest used temporary register.
+ */
 #define R300_PFS_CNTL_2                     0x4608
 #       define R300_PFS_CNTL_ALU_OFFSET_SHIFT    0
 #       define R300_PFS_CNTL_ALU_OFFSET_MASK     (63 << 0)
 #       define R300_PFS_CNTL_ALU_END_SHIFT       6
-#       define R300_PFS_CNTL_ALU_END_MASK        (63 << 0)
+#       define R300_PFS_CNTL_ALU_END_MASK        (63 << 6)
 #       define R300_PFS_CNTL_TEX_OFFSET_SHIFT    12
 #       define R300_PFS_CNTL_TEX_OFFSET_MASK     (31 << 12) /* GUESS */
 #       define R300_PFS_CNTL_TEX_END_SHIFT       18
 #       define R300_PFS_CNTL_TEX_END_MASK        (31 << 18) /* GUESS */
 
 /* gap */
+
 /* Nodes are stored backwards. The last active node is always stored in
-// PFS_NODE_3.
-// Example: In a 2-node program, NODE_0 and NODE_1 are set to 0. The
-// first node is stored in NODE_2, the second node is stored in NODE_3.
-//
-// Offsets are relative to the master offset from PFS_CNTL_2.
-// LAST_NODE is set for the last node, and only for the last node. */
+ * PFS_NODE_3.
+ * Example: In a 2-node program, NODE_0 and NODE_1 are set to 0. The
+ * first node is stored in NODE_2, the second node is stored in NODE_3.
+ *
+ * Offsets are relative to the master offset from PFS_CNTL_2.
+ */
 #define R300_PFS_NODE_0                     0x4610
 #define R300_PFS_NODE_1                     0x4614
 #define R300_PFS_NODE_2                     0x4618
@@ -894,91 +1020,98 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_PFS_NODE_TEX_OFFSET_MASK     (31 << 12)
 #       define R300_PFS_NODE_TEX_END_SHIFT       17
 #       define R300_PFS_NODE_TEX_END_MASK        (31 << 17)
-/*#       define R300_PFS_NODE_LAST_NODE           (1 << 22) */
 #		define R300_PFS_NODE_OUTPUT_COLOR        (1 << 22)
 #		define R300_PFS_NODE_OUTPUT_DEPTH        (1 << 23)
 
 /* TEX
-// As far as I can tell, texture instructions cannot write into output
-// registers directly. A subsequent ALU instruction is always necessary,
-// even if it's just MAD o0, r0, 1, 0 */
+ * As far as I can tell, texture instructions cannot write into output
+ * registers directly. A subsequent ALU instruction is always necessary,
+ * even if it's just MAD o0, r0, 1, 0
+ */
 #define R300_PFS_TEXI_0                     0x4620
-#       define R300_FPITX_SRC_SHIFT              0
-#       define R300_FPITX_SRC_MASK               (31 << 0)
-#       define R300_FPITX_SRC_CONST              (1 << 5) /* GUESS */
-#       define R300_FPITX_DST_SHIFT              6
-#       define R300_FPITX_DST_MASK               (31 << 6)
-#       define R300_FPITX_IMAGE_SHIFT            11
-#       define R300_FPITX_IMAGE_MASK             (15 << 11) /* GUESS based on layout and native limits */
+#	define R300_FPITX_SRC_SHIFT              0
+#	define R300_FPITX_SRC_MASK               (31 << 0)
+	/* GUESS */
+#	define R300_FPITX_SRC_CONST              (1 << 5)
+#	define R300_FPITX_DST_SHIFT              6
+#	define R300_FPITX_DST_MASK               (31 << 6)
+#	define R300_FPITX_IMAGE_SHIFT            11
+	/* GUESS based on layout and native limits */
+#       define R300_FPITX_IMAGE_MASK             (15 << 11)
 /* Unsure if these are opcodes, or some kind of bitfield, but this is how
  * they were set when I checked
  */
-#		define R300_FPITX_OPCODE_SHIFT			15
-#			define R300_FPITX_OP_TEX			1
-#			define R300_FPITX_OP_KIL			2
-#			define R300_FPITX_OP_TXP			3
-#			define R300_FPITX_OP_TXB			4
+#	define R300_FPITX_OPCODE_SHIFT		15
+#		define R300_FPITX_OP_TEX	1
+#		define R300_FPITX_OP_KIL	2
+#		define R300_FPITX_OP_TXP	3
+#		define R300_FPITX_OP_TXB	4
+#	define R300_FPITX_OPCODE_MASK           (7 << 15)
 
 /* ALU
-// The ALU instructions register blocks are enumerated according to the order
-// in which fglrx. I assume there is space for 64 instructions, since
-// each block has space for a maximum of 64 DWORDs, and this matches reported
-// native limits.
-//
-// The basic functional block seems to be one MAD for each color and alpha,
-// and an adder that adds all components after the MUL.
-//  - ADD, MUL, MAD etc.: use MAD with appropriate neutral operands
-//  - DP4: Use OUTC_DP4, OUTA_DP4
-//  - DP3: Use OUTC_DP3, OUTA_DP4, appropriate alpha operands
-//  - DPH: Use OUTC_DP4, OUTA_DP4, appropriate alpha operands
-//  - CMP: If ARG2 < 0, return ARG1, else return ARG0
-//  - FLR: use FRC+MAD
-//  - XPD: use MAD+MAD
-//  - SGE, SLT: use MAD+CMP
-//  - RSQ: use ABS modifier for argument
-//  - Use OUTC_REPL_ALPHA to write results of an alpha-only operation (e.g. RCP)
-//    into color register
-//  - apparently, there's no quick DST operation
-//  - fglrx set FPI2_UNKNOWN_31 on a "MAD fragment.color, tmp0, tmp1, tmp2"
-//  - fglrx set FPI2_UNKNOWN_31 on a "MAX r2, r1, c0"
-//  - fglrx once set FPI0_UNKNOWN_31 on a "FRC r1, r1"
-//
-// Operand selection
-// First stage selects three sources from the available registers and
-// constant parameters. This is defined in INSTR1 (color) and INSTR3 (alpha).
-// fglrx sorts the three source fields: Registers before constants,
-// lower indices before higher indices; I do not know whether this is necessary.
-// fglrx fills unused sources with "read constant 0"
-// According to specs, you cannot select more than two different constants.
-//
-// Second stage selects the operands from the sources. This is defined in
-// INSTR0 (color) and INSTR2 (alpha). You can also select the special constants
-// zero and one.
-// Swizzling and negation happens in this stage, as well.
-//
-// Important: Color and alpha seem to be mostly separate, i.e. their sources
-// selection appears to be fully independent (the register storage is probably
-// physically split into a color and an alpha section).
-// However (because of the apparent physical split), there is some interaction
-// WRT swizzling. If, for example, you want to load an R component into an
-// Alpha operand, this R component is taken from a *color* source, not from
-// an alpha source. The corresponding register doesn't even have to appear in
-// the alpha sources list. (I hope this alll makes sense to you)
-//
-// Destination selection
-// The destination register index is in FPI1 (color) and FPI3 (alpha) together
-// with enable bits.
-// There are separate enable bits for writing into temporary registers
-// (DSTC_REG_* /DSTA_REG) and and program output registers (DSTC_OUTPUT_* /DSTA_OUTPUT).
-// You can write to both at once, or not write at all (the same index
-// must be used for both).
-//
-// Note: There is a special form for LRP
-//  - Argument order is the same as in ARB_fragment_program.
-//  - Operation is MAD
-//  - ARG1 is set to ARGC_SRC1C_LRP/ARGC_SRC1A_LRP
-//  - Set FPI0/FPI2_SPECIAL_LRP
-// Arbitrary LRP (including support for swizzling) requires vanilla MAD+MAD */
+ * The ALU instructions register blocks are enumerated according to the order
+ * in which fglrx. I assume there is space for 64 instructions, since
+ * each block has space for a maximum of 64 DWORDs, and this matches reported
+ * native limits.
+ *
+ * The basic functional block seems to be one MAD for each color and alpha,
+ * and an adder that adds all components after the MUL.
+ *  - ADD, MUL, MAD etc.: use MAD with appropriate neutral operands
+ *  - DP4: Use OUTC_DP4, OUTA_DP4
+ *  - DP3: Use OUTC_DP3, OUTA_DP4, appropriate alpha operands
+ *  - DPH: Use OUTC_DP4, OUTA_DP4, appropriate alpha operands
+ *  - CMPH: If ARG2 > 0.5, return ARG0, else return ARG1
+ *  - CMP: If ARG2 < 0, return ARG1, else return ARG0
+ *  - FLR: use FRC+MAD
+ *  - XPD: use MAD+MAD
+ *  - SGE, SLT: use MAD+CMP
+ *  - RSQ: use ABS modifier for argument
+ *  - Use OUTC_REPL_ALPHA to write results of an alpha-only operation
+ *    (e.g. RCP) into color register
+ *  - apparently, there's no quick DST operation
+ *  - fglrx set FPI2_UNKNOWN_31 on a "MAD fragment.color, tmp0, tmp1, tmp2"
+ *  - fglrx set FPI2_UNKNOWN_31 on a "MAX r2, r1, c0"
+ *  - fglrx once set FPI0_UNKNOWN_31 on a "FRC r1, r1"
+ *
+ * Operand selection
+ * First stage selects three sources from the available registers and
+ * constant parameters. This is defined in INSTR1 (color) and INSTR3 (alpha).
+ * fglrx sorts the three source fields: Registers before constants,
+ * lower indices before higher indices; I do not know whether this is
+ * necessary.
+ *
+ * fglrx fills unused sources with "read constant 0"
+ * According to specs, you cannot select more than two different constants.
+ *
+ * Second stage selects the operands from the sources. This is defined in
+ * INSTR0 (color) and INSTR2 (alpha). You can also select the special constants
+ * zero and one.
+ * Swizzling and negation happens in this stage, as well.
+ *
+ * Important: Color and alpha seem to be mostly separate, i.e. their sources
+ * selection appears to be fully independent (the register storage is probably
+ * physically split into a color and an alpha section).
+ * However (because of the apparent physical split), there is some interaction
+ * WRT swizzling. If, for example, you want to load an R component into an
+ * Alpha operand, this R component is taken from a *color* source, not from
+ * an alpha source. The corresponding register doesn't even have to appear in
+ * the alpha sources list. (I hope this all makes sense to you)
+ *
+ * Destination selection
+ * The destination register index is in FPI1 (color) and FPI3 (alpha)
+ * together with enable bits.
+ * There are separate enable bits for writing into temporary registers
+ * (DSTC_REG_* /DSTA_REG) and and program output registers (DSTC_OUTPUT_*
+ * /DSTA_OUTPUT). You can write to both at once, or not write at all (the
+ * same index must be used for both).
+ *
+ * Note: There is a special form for LRP
+ *  - Argument order is the same as in ARB_fragment_program.
+ *  - Operation is MAD
+ *  - ARG1 is set to ARGC_SRC1C_LRP/ARGC_SRC1A_LRP
+ *  - Set FPI0/FPI2_SPECIAL_LRP
+ * Arbitrary LRP (including support for swizzling) requires vanilla MAD+MAD
+ */
 #define R300_PFS_INSTR1_0                   0x46C0
 #       define R300_FPI1_SRC0C_SHIFT             0
 #       define R300_FPI1_SRC0C_MASK              (31 << 0)
@@ -989,6 +1122,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI1_SRC2C_SHIFT             12
 #       define R300_FPI1_SRC2C_MASK              (31 << 12)
 #       define R300_FPI1_SRC2C_CONST             (1 << 17)
+#       define R300_FPI1_SRC_MASK                0x0003ffff
 #       define R300_FPI1_DSTC_SHIFT              18
 #       define R300_FPI1_DSTC_MASK               (31 << 18)
 #		define R300_FPI1_DSTC_REG_MASK_SHIFT     23
@@ -1010,6 +1144,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI3_SRC2A_SHIFT             12
 #       define R300_FPI3_SRC2A_MASK              (31 << 12)
 #       define R300_FPI3_SRC2A_CONST             (1 << 17)
+#       define R300_FPI3_SRC_MASK                0x0003ffff
 #       define R300_FPI3_DSTA_SHIFT              18
 #       define R300_FPI3_DSTA_MASK               (31 << 18)
 #       define R300_FPI3_DSTA_REG                (1 << 23)
@@ -1035,7 +1170,8 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI0_ARGC_SRC1C_LRP          15
 #       define R300_FPI0_ARGC_ZERO               20
 #       define R300_FPI0_ARGC_ONE                21
-#       define R300_FPI0_ARGC_HALF               22 /* GUESS */
+	/* GUESS */
+#       define R300_FPI0_ARGC_HALF               22
 #       define R300_FPI0_ARGC_SRC0C_YZX          23
 #       define R300_FPI0_ARGC_SRC1C_YZX          24
 #       define R300_FPI0_ARGC_SRC2C_YZX          25
@@ -1064,6 +1200,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI0_OUTC_DP4                (2 << 23)
 #       define R300_FPI0_OUTC_MIN                (4 << 23)
 #       define R300_FPI0_OUTC_MAX                (5 << 23)
+#       define R300_FPI0_OUTC_CMPH               (7 << 23)
 #       define R300_FPI0_OUTC_CMP                (8 << 23)
 #       define R300_FPI0_OUTC_FRC                (9 << 23)
 #       define R300_FPI0_OUTC_REPL_ALPHA         (10 << 23)
@@ -1086,20 +1223,23 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI2_ARGA_SRC1A_LRP          15
 #       define R300_FPI2_ARGA_ZERO               16
 #       define R300_FPI2_ARGA_ONE                17
-#       define R300_FPI2_ARGA_HALF               18 /* GUESS */
-
+	/* GUESS */
+#       define R300_FPI2_ARGA_HALF               18
 #       define R300_FPI2_ARG0A_SHIFT             0
 #       define R300_FPI2_ARG0A_MASK              (31 << 0)
 #       define R300_FPI2_ARG0A_NEG               (1 << 5)
-#		define R300_FPI2_ARG0A_ABS				 (1 << 6) /* GUESS */
+	/* GUESS */
+#	define R300_FPI2_ARG0A_ABS		 (1 << 6)
 #       define R300_FPI2_ARG1A_SHIFT             7
 #       define R300_FPI2_ARG1A_MASK              (31 << 7)
 #       define R300_FPI2_ARG1A_NEG               (1 << 12)
-#		define R300_FPI2_ARG1A_ABS				 (1 << 13) /* GUESS */
+	/* GUESS */
+#	define R300_FPI2_ARG1A_ABS		 (1 << 13)
 #       define R300_FPI2_ARG2A_SHIFT             14
 #       define R300_FPI2_ARG2A_MASK              (31 << 14)
 #       define R300_FPI2_ARG2A_NEG               (1 << 19)
-#		define R300_FPI2_ARG2A_ABS				 (1 << 20) /* GUESS */
+	/* GUESS */
+#	define R300_FPI2_ARG2A_ABS		 (1 << 20)
 #       define R300_FPI2_SPECIAL_LRP             (1 << 21)
 #       define R300_FPI2_OUTA_MAD                (0 << 23)
 #       define R300_FPI2_OUTA_DP4                (1 << 23)
@@ -1113,9 +1253,19 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_FPI2_OUTA_RSQ                (11 << 23)
 #       define R300_FPI2_OUTA_SAT                (1 << 30)
 #       define R300_FPI2_UNKNOWN_31              (1 << 31)
-/* END */
+/* END: Fragment program instruction set */
+
+/* Fog state and color */
+#define R300_RE_FOG_STATE                   0x4BC0
+#       define R300_FOG_ENABLE                   (1 << 0)
+#	define R300_FOG_MODE_LINEAR              (0 << 1)
+#	define R300_FOG_MODE_EXP                 (1 << 1)
+#	define R300_FOG_MODE_EXP2                (2 << 1)
+#	define R300_FOG_MODE_MASK                (3 << 1)
+#define R300_FOG_COLOR_R                    0x4BC8
+#define R300_FOG_COLOR_G                    0x4BCC
+#define R300_FOG_COLOR_B                    0x4BD0
 
-/* gap */
 #define R300_PP_ALPHA_TEST                  0x4BD4
 #       define R300_REF_ALPHA_MASK               0x000000ff
 #       define R300_ALPHA_TEST_FAIL              (0 << 8)
@@ -1130,6 +1280,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_ALPHA_TEST_ENABLE            (1 << 11)
 
 /* gap */
+
 /* Fragment program parameters in 7.16 floating point */
 #define R300_PFS_PARAM_0_X                  0x4C00
 #define R300_PFS_PARAM_0_Y                  0x4C04
@@ -1142,45 +1293,48 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_PFS_PARAM_31_W                 0x4DFC
 
 /* Notes:
-// - AFAIK fglrx always sets BLEND_UNKNOWN when blending is used in the application
-// - AFAIK fglrx always sets BLEND_NO_SEPARATE when CBLEND and ABLEND are set to the same
-//   function (both registers are always set up completely in any case)
-// - Most blend flags are simply copied from R200 and not tested yet */
+ * - AFAIK fglrx always sets BLEND_UNKNOWN when blending is used in
+ *   the application
+ * - AFAIK fglrx always sets BLEND_NO_SEPARATE when CBLEND and ABLEND
+ *    are set to the same
+ *   function (both registers are always set up completely in any case)
+ * - Most blend flags are simply copied from R200 and not tested yet
+ */
 #define R300_RB3D_CBLEND                    0x4E04
 #define R300_RB3D_ABLEND                    0x4E08
- /* the following only appear in CBLEND */
+/* the following only appear in CBLEND */
 #       define R300_BLEND_ENABLE                     (1 << 0)
 #       define R300_BLEND_UNKNOWN                    (3 << 1)
 #       define R300_BLEND_NO_SEPARATE                (1 << 3)
- /* the following are shared between CBLEND and ABLEND */
+/* the following are shared between CBLEND and ABLEND */
 #       define R300_FCN_MASK                         (3  << 12)
 #       define R300_COMB_FCN_ADD_CLAMP               (0  << 12)
 #       define R300_COMB_FCN_ADD_NOCLAMP             (1  << 12)
 #       define R300_COMB_FCN_SUB_CLAMP               (2  << 12)
 #       define R300_COMB_FCN_SUB_NOCLAMP             (3  << 12)
-#       define R300_SRC_BLEND_GL_ZERO                (32 << 16)
-#       define R300_SRC_BLEND_GL_ONE                 (33 << 16)
-#       define R300_SRC_BLEND_GL_SRC_COLOR           (34 << 16)
-#       define R300_SRC_BLEND_GL_ONE_MINUS_SRC_COLOR (35 << 16)
-#       define R300_SRC_BLEND_GL_DST_COLOR           (36 << 16)
-#       define R300_SRC_BLEND_GL_ONE_MINUS_DST_COLOR (37 << 16)
-#       define R300_SRC_BLEND_GL_SRC_ALPHA           (38 << 16)
-#       define R300_SRC_BLEND_GL_ONE_MINUS_SRC_ALPHA (39 << 16)
-#       define R300_SRC_BLEND_GL_DST_ALPHA           (40 << 16)
-#       define R300_SRC_BLEND_GL_ONE_MINUS_DST_ALPHA (41 << 16)
-#       define R300_SRC_BLEND_GL_SRC_ALPHA_SATURATE  (42 << 16)
-#       define R300_SRC_BLEND_MASK                   (63 << 16)
-#       define R300_DST_BLEND_GL_ZERO                (32 << 24)
-#       define R300_DST_BLEND_GL_ONE                 (33 << 24)
-#       define R300_DST_BLEND_GL_SRC_COLOR           (34 << 24)
-#       define R300_DST_BLEND_GL_ONE_MINUS_SRC_COLOR (35 << 24)
-#       define R300_DST_BLEND_GL_DST_COLOR           (36 << 24)
-#       define R300_DST_BLEND_GL_ONE_MINUS_DST_COLOR (37 << 24)
-#       define R300_DST_BLEND_GL_SRC_ALPHA           (38 << 24)
-#       define R300_DST_BLEND_GL_ONE_MINUS_SRC_ALPHA (39 << 24)
-#       define R300_DST_BLEND_GL_DST_ALPHA           (40 << 24)
-#       define R300_DST_BLEND_GL_ONE_MINUS_DST_ALPHA (41 << 24)
-#       define R300_DST_BLEND_MASK                   (63 << 24)
+#       define R300_COMB_FCN_MIN                     (4  << 12)
+#       define R300_COMB_FCN_MAX                     (5  << 12)
+#       define R300_COMB_FCN_RSUB_CLAMP              (6  << 12)
+#       define R300_COMB_FCN_RSUB_NOCLAMP            (7  << 12)
+#       define R300_BLEND_GL_ZERO                    (32)
+#       define R300_BLEND_GL_ONE                     (33)
+#       define R300_BLEND_GL_SRC_COLOR               (34)
+#       define R300_BLEND_GL_ONE_MINUS_SRC_COLOR     (35)
+#       define R300_BLEND_GL_DST_COLOR               (36)
+#       define R300_BLEND_GL_ONE_MINUS_DST_COLOR     (37)
+#       define R300_BLEND_GL_SRC_ALPHA               (38)
+#       define R300_BLEND_GL_ONE_MINUS_SRC_ALPHA     (39)
+#       define R300_BLEND_GL_DST_ALPHA               (40)
+#       define R300_BLEND_GL_ONE_MINUS_DST_ALPHA     (41)
+#       define R300_BLEND_GL_SRC_ALPHA_SATURATE      (42)
+#       define R300_BLEND_GL_CONST_COLOR             (43)
+#       define R300_BLEND_GL_ONE_MINUS_CONST_COLOR   (44)
+#       define R300_BLEND_GL_CONST_ALPHA             (45)
+#       define R300_BLEND_GL_ONE_MINUS_CONST_ALPHA   (46)
+#       define R300_BLEND_MASK                       (63)
+#       define R300_SRC_BLEND_SHIFT                  (16)
+#       define R300_DST_BLEND_SHIFT                  (24)
+#define R300_RB3D_BLEND_COLOR               0x4E10
 #define R300_RB3D_COLORMASK                 0x4E0C
 #       define R300_COLORMASK0_B                 (1<<0)
 #       define R300_COLORMASK0_G                 (1<<1)
@@ -1188,15 +1342,19 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #       define R300_COLORMASK0_A                 (1<<3)
 
 /* gap */
+
 #define R300_RB3D_COLOROFFSET0              0x4E28
 #       define R300_COLOROFFSET_MASK             0xFFFFFFF0 /* GUESS */
 #define R300_RB3D_COLOROFFSET1              0x4E2C /* GUESS */
 #define R300_RB3D_COLOROFFSET2              0x4E30 /* GUESS */
 #define R300_RB3D_COLOROFFSET3              0x4E34 /* GUESS */
+
 /* gap */
+
 /* Bit 16: Larger tiles
-// Bit 17: 4x2 tiles
-// Bit 18: Extremely weird tile like, but some pixels duplicated? */
+ * Bit 17: 4x2 tiles
+ * Bit 18: Extremely weird tile like, but some pixels duplicated?
+ */
 #define R300_RB3D_COLORPITCH0               0x4E38
 #       define R300_COLORPITCH_MASK              0x00001FF8 /* GUESS */
 #       define R300_COLOR_TILE_ENABLE            (1 << 16) /* GUESS */
@@ -1210,30 +1368,30 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_RB3D_COLORPITCH2               0x4E40 /* GUESS */
 #define R300_RB3D_COLORPITCH3               0x4E44 /* GUESS */
 
+#define R300_RB3D_AARESOLVE_CTL             0x4E88
 /* gap */
+
 /* Guess by Vladimir.
-// Set to 0A before 3D operations, set to 02 afterwards. */
-#define R300_RB3D_DSTCACHE_CTLSTAT          0x4E4C
-#       define R300_RB3D_DSTCACHE_02             0x00000002
-#       define R300_RB3D_DSTCACHE_0A             0x0000000A
+ * Set to 0A before 3D operations, set to 02 afterwards.
+ */
+/*#define R300_RB3D_DSTCACHE_CTLSTAT          0x4E4C*/
+#       define R300_RB3D_DSTCACHE_UNKNOWN_02             0x00000002
+#       define R300_RB3D_DSTCACHE_UNKNOWN_0A             0x0000000A
 
 /* gap */
-/* There seems to be no "write only" setting, so use Z-test = ALWAYS for this. */
-/* Bit (1<<8) is the "test" bit. so plain write is 6  - vd */
-#define R300_RB3D_ZSTENCIL_CNTL_0                   0x4F00
-#       define R300_RB3D_Z_DISABLED_1            0x00000010 /* GUESS */
-#       define R300_RB3D_Z_DISABLED_2            0x00000014 /* GUESS */
-#       define R300_RB3D_Z_TEST                  0x00000012
-#       define R300_RB3D_Z_TEST_AND_WRITE        0x00000016
-#       define R300_RB3D_Z_WRITE_ONLY        	 0x00000006
-
-#       define R300_RB3D_Z_TEST                  0x00000012
-#       define R300_RB3D_Z_TEST_AND_WRITE        0x00000016
-#       define R300_RB3D_Z_WRITE_ONLY        	 0x00000006
-#	define R300_RB3D_STENCIL_ENABLE		 0x00000001
-
-#define R300_RB3D_ZSTENCIL_CNTL_1                   0x4F04
-		/* functions */
+/* There seems to be no "write only" setting, so use Z-test = ALWAYS
+ * for this.
+ * Bit (1<<8) is the "test" bit. so plain write is 6  - vd
+ */
+#define R300_ZB_CNTL                             0x4F00
+#	define R300_STENCIL_ENABLE		 (1 << 0)
+#	define R300_Z_ENABLE		         (1 << 1)
+#	define R300_Z_WRITE_ENABLE		 (1 << 2)
+#	define R300_Z_SIGNED_COMPARE		 (1 << 3)
+#	define R300_STENCIL_FRONT_BACK		 (1 << 4)
+
+#define R300_ZB_ZSTENCILCNTL                   0x4f04
+	/* functions */
 #	define R300_ZS_NEVER			0
 #	define R300_ZS_LESS			1
 #	define R300_ZS_LEQUAL			2
@@ -1243,7 +1401,7 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_ZS_NOTEQUAL			6
 #	define R300_ZS_ALWAYS			7
 #       define R300_ZS_MASK                     7
-		/* operations */
+	/* operations */
 #	define R300_ZS_KEEP			0
 #	define R300_ZS_ZERO			1
 #	define R300_ZS_REPLACE			2
@@ -1252,71 +1410,201 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #	define R300_ZS_INVERT			5
 #	define R300_ZS_INCR_WRAP		6
 #	define R300_ZS_DECR_WRAP		7
+#	define R300_Z_FUNC_SHIFT		0
+	/* front and back refer to operations done for front
+	   and back faces, i.e. separate stencil function support */
+#	define R300_S_FRONT_FUNC_SHIFT	        3
+#	define R300_S_FRONT_SFAIL_OP_SHIFT	6
+#	define R300_S_FRONT_ZPASS_OP_SHIFT	9
+#	define R300_S_FRONT_ZFAIL_OP_SHIFT      12
+#	define R300_S_BACK_FUNC_SHIFT           15
+#	define R300_S_BACK_SFAIL_OP_SHIFT       18
+#	define R300_S_BACK_ZPASS_OP_SHIFT       21
+#	define R300_S_BACK_ZFAIL_OP_SHIFT       24
+
+#define R300_ZB_STENCILREFMASK                        0x4f08
+#	define R300_STENCILREF_SHIFT       0
+#	define R300_STENCILREF_MASK        0x000000ff
+#	define R300_STENCILMASK_SHIFT      8
+#	define R300_STENCILMASK_MASK       0x0000ff00
+#	define R300_STENCILWRITEMASK_SHIFT 16
+#	define R300_STENCILWRITEMASK_MASK  0x00ff0000
 
-       /* front and back refer to operations done for front
-          and back faces, i.e. separate stencil function support */
-#	define R300_RB3D_ZS1_DEPTH_FUNC_SHIFT		0
-#	define R300_RB3D_ZS1_FRONT_FUNC_SHIFT		3
-#	define R300_RB3D_ZS1_FRONT_FAIL_OP_SHIFT	6
-#	define R300_RB3D_ZS1_FRONT_ZPASS_OP_SHIFT	9
-#	define R300_RB3D_ZS1_FRONT_ZFAIL_OP_SHIFT      12
-#	define R300_RB3D_ZS1_BACK_FUNC_SHIFT           15
-#	define R300_RB3D_ZS1_BACK_FAIL_OP_SHIFT        18
-#	define R300_RB3D_ZS1_BACK_ZPASS_OP_SHIFT       21
-#	define R300_RB3D_ZS1_BACK_ZFAIL_OP_SHIFT       24
-
+/* gap */
 
+#define R300_ZB_FORMAT                             0x4f10
+#	define R300_DEPTHFORMAT_16BIT_INT_Z   (0 << 0)
+#	define R300_DEPTHFORMAT_16BIT_13E3    (1 << 0)
+#	define R300_DEPTHFORMAT_24BIT_INT_Z_8BIT_STENCIL   (2 << 0)
+/* reserved up to (15 << 0) */
+#	define R300_INVERT_13E3_LEADING_ONES  (0 << 4)
+#	define R300_INVERT_13E3_LEADING_ZEROS (1 << 4)
 
-#define R300_RB3D_ZSTENCIL_CNTL_2                   0x4F08
-#	define R300_RB3D_ZS2_STENCIL_REF_SHIFT		0
-#	define R300_RB3D_ZS2_STENCIL_MASK		0xFF
-#	define R300_RB3D_ZS2_STENCIL_MASK_SHIFT	        8
-#	define R300_RB3D_ZS2_STENCIL_WRITE_MASK_SHIFT	16
+#define R300_ZB_ZTOP                             0x4F14
+#	define R300_ZTOP_DISABLE                 (0 << 0)
+#	define R300_ZTOP_ENABLE                  (1 << 0)
 
 /* gap */
 
-#define R300_RB3D_ZSTENCIL_FORMAT                   0x4F10
-#	define R300_DEPTH_FORMAT_16BIT_INT_Z     (0 << 0)
-#	define R300_DEPTH_FORMAT_24BIT_INT_Z     (2 << 0)
+#define R300_ZB_ZCACHE_CTLSTAT            0x4f18
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_FLUSH_NO_EFFECT      (0 << 0)
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_FLUSH_FLUSH_AND_FREE (1 << 0)
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_FREE_NO_EFFECT       (0 << 1)
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_FREE_FREE            (1 << 1)
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_BUSY_IDLE            (0 << 31)
+#       define R300_ZB_ZCACHE_CTLSTAT_ZC_BUSY_BUSY            (1 << 31)
+
+#define R300_ZB_BW_CNTL                     0x4f1c
+#	define R300_HIZ_DISABLE                              (0 << 0)
+#	define R300_HIZ_ENABLE                               (1 << 0)
+#	define R300_HIZ_MIN                                  (0 << 1)
+#	define R300_HIZ_MAX                                  (1 << 1)
+#	define R300_FAST_FILL_DISABLE                        (0 << 2)
+#	define R300_FAST_FILL_ENABLE                         (1 << 2)
+#	define R300_RD_COMP_DISABLE                          (0 << 3)
+#	define R300_RD_COMP_ENABLE                           (1 << 3)
+#	define R300_WR_COMP_DISABLE                          (0 << 4)
+#	define R300_WR_COMP_ENABLE                           (1 << 4)
+#	define R300_ZB_CB_CLEAR_RMW                          (0 << 5)
+#	define R300_ZB_CB_CLEAR_CACHE_LINEAR                 (1 << 5)
+#	define R300_FORCE_COMPRESSED_STENCIL_VALUE_DISABLE   (0 << 6)
+#	define R300_FORCE_COMPRESSED_STENCIL_VALUE_ENABLE    (1 << 6)
+
+#	define R500_ZEQUAL_OPTIMIZE_ENABLE                   (0 << 7)
+#	define R500_ZEQUAL_OPTIMIZE_DISABLE                  (1 << 7)
+#	define R500_SEQUAL_OPTIMIZE_ENABLE                   (0 << 8)
+#	define R500_SEQUAL_OPTIMIZE_DISABLE                  (1 << 8)
+
+#	define R500_BMASK_ENABLE                             (0 << 10)
+#	define R500_BMASK_DISABLE                            (1 << 10)
+#	define R500_HIZ_EQUAL_REJECT_DISABLE                 (0 << 11)
+#	define R500_HIZ_EQUAL_REJECT_ENABLE                  (1 << 11)
+#	define R500_HIZ_FP_EXP_BITS_DISABLE                  (0 << 12)
+#	define R500_HIZ_FP_EXP_BITS_1                        (1 << 12)
+#	define R500_HIZ_FP_EXP_BITS_2                        (2 << 12)
+#	define R500_HIZ_FP_EXP_BITS_3                        (3 << 12)
+#	define R500_HIZ_FP_EXP_BITS_4                        (4 << 12)
+#	define R500_HIZ_FP_EXP_BITS_5                        (5 << 12)
+#	define R500_HIZ_FP_INVERT_LEADING_ONES               (0 << 15)
+#	define R500_HIZ_FP_INVERT_LEADING_ZEROS              (1 << 15)
+#	define R500_TILE_OVERWRITE_RECOMPRESSION_ENABLE      (0 << 16)
+#	define R500_TILE_OVERWRITE_RECOMPRESSION_DISABLE     (1 << 16)
+#	define R500_CONTIGUOUS_6XAA_SAMPLES_ENABLE           (0 << 17)
+#	define R500_CONTIGUOUS_6XAA_SAMPLES_DISABLE          (1 << 17)
+#	define R500_PEQ_PACKING_DISABLE                      (0 << 18)
+#	define R500_PEQ_PACKING_ENABLE                       (1 << 18)
+#	define R500_COVERED_PTR_MASKING_DISABLE              (0 << 18)
+#	define R500_COVERED_PTR_MASKING_ENABLE               (1 << 18)
+
 
 /* gap */
-#define R300_RB3D_DEPTHOFFSET               0x4F20
-#define R300_RB3D_DEPTHPITCH                0x4F24
-#       define R300_DEPTHPITCH_MASK              0x00001FF8 /* GUESS */
-#       define R300_DEPTH_TILE_ENABLE            (1 << 16) /* GUESS */
-#       define R300_DEPTH_MICROTILE_ENABLE       (1 << 17) /* GUESS */
-#       define R300_DEPTH_ENDIAN_NO_SWAP         (0 << 18) /* GUESS */
-#       define R300_DEPTH_ENDIAN_WORD_SWAP       (1 << 18) /* GUESS */
-#       define R300_DEPTH_ENDIAN_DWORD_SWAP      (2 << 18) /* GUESS */
-
-/* BEGIN: Vertex program instruction set
-// Every instruction is four dwords long:
-//  DWORD 0: output and opcode
-//  DWORD 1: first argument
-//  DWORD 2: second argument
-//  DWORD 3: third argument
-//
-// Notes:
-//  - ABS r, a is implemented as MAX r, a, -a
-//  - MOV is implemented as ADD to zero
-//  - XPD is implemented as MUL + MAD
-//  - FLR is implemented as FRC + ADD
-//  - apparently, fglrx tries to schedule instructions so that there is at least
-//    one instruction between the write to a temporary and the first read
-//    from said temporary; however, violations of this scheduling are allowed
-//  - register indices seem to be unrelated with OpenGL aliasing to conventional state
-//  - only one attribute and one parameter can be loaded at a time; however, the
-//    same attribute/parameter can be used for more than one argument
-//  - the second software argument for POW is the third hardware argument (no idea why)
-//  - MAD with only temporaries as input seems to use VPI_OUT_SELECT_MAD_2
-//
-// There is some magic surrounding LIT:
-//  The single argument is replicated across all three inputs, but swizzled:
-//   First argument: xyzy
-//   Second argument: xyzx
-//   Third argument: xyzw
-//  Whenever the result is used later in the fragment program, fglrx forces x and w
-//  to be 1.0 in the input selection; I don't know whether this is strictly necessary */
+
+/* Z Buffer Address Offset.
+ * Bits 31 to 5 are used for aligned Z buffer address offset for macro tiles.
+ */
+#define R300_ZB_DEPTHOFFSET               0x4f20
+
+/* Z Buffer Pitch and Endian Control */
+#define R300_ZB_DEPTHPITCH                0x4f24
+#       define R300_DEPTHPITCH_MASK              0x00003FFC
+#       define R300_DEPTHMACROTILE_DISABLE      (0 << 16)
+#       define R300_DEPTHMACROTILE_ENABLE       (1 << 16)
+#       define R300_DEPTHMICROTILE_LINEAR       (0 << 17)
+#       define R300_DEPTHMICROTILE_TILED        (1 << 17)
+#       define R300_DEPTHMICROTILE_TILED_SQUARE (2 << 17)
+#       define R300_DEPTHENDIAN_NO_SWAP         (0 << 18)
+#       define R300_DEPTHENDIAN_WORD_SWAP       (1 << 18)
+#       define R300_DEPTHENDIAN_DWORD_SWAP      (2 << 18)
+#       define R300_DEPTHENDIAN_HALF_DWORD_SWAP (3 << 18)
+
+/* Z Buffer Clear Value */
+#define R300_ZB_DEPTHCLEARVALUE                  0x4f28
+
+#define R300_ZB_ZMASK_OFFSET			 0x4f30
+#define R300_ZB_ZMASK_PITCH			 0x4f34
+#define R300_ZB_ZMASK_WRINDEX			 0x4f38
+#define R300_ZB_ZMASK_DWORD			 0x4f3c
+#define R300_ZB_ZMASK_RDINDEX			 0x4f40
+
+/* Hierarchical Z Memory Offset */
+#define R300_ZB_HIZ_OFFSET                       0x4f44
+
+/* Hierarchical Z Write Index */
+#define R300_ZB_HIZ_WRINDEX                      0x4f48
+
+/* Hierarchical Z Data */
+#define R300_ZB_HIZ_DWORD                        0x4f4c
+
+/* Hierarchical Z Read Index */
+#define R300_ZB_HIZ_RDINDEX                      0x4f50
+
+/* Hierarchical Z Pitch */
+#define R300_ZB_HIZ_PITCH                        0x4f54
+
+/* Z Buffer Z Pass Counter Data */
+#define R300_ZB_ZPASS_DATA                       0x4f58
+
+/* Z Buffer Z Pass Counter Address */
+#define R300_ZB_ZPASS_ADDR                       0x4f5c
+
+/* Depth buffer X and Y coordinate offset */
+#define R300_ZB_DEPTHXY_OFFSET                   0x4f60
+#	define R300_DEPTHX_OFFSET_SHIFT  1
+#	define R300_DEPTHX_OFFSET_MASK   0x000007FE
+#	define R300_DEPTHY_OFFSET_SHIFT  17
+#	define R300_DEPTHY_OFFSET_MASK   0x07FE0000
+
+/* Sets the fifo sizes */
+#define R500_ZB_FIFO_SIZE                        0x4fd0
+#	define R500_OP_FIFO_SIZE_FULL   (0 << 0)
+#	define R500_OP_FIFO_SIZE_HALF   (1 << 0)
+#	define R500_OP_FIFO_SIZE_QUATER (2 << 0)
+#	define R500_OP_FIFO_SIZE_EIGTHS (4 << 0)
+
+/* Stencil Reference Value and Mask for backfacing quads */
+/* R300_ZB_STENCILREFMASK handles front face */
+#define R500_ZB_STENCILREFMASK_BF                0x4fd4
+#	define R500_STENCILREF_SHIFT       0
+#	define R500_STENCILREF_MASK        0x000000ff
+#	define R500_STENCILMASK_SHIFT      8
+#	define R500_STENCILMASK_MASK       0x0000ff00
+#	define R500_STENCILWRITEMASK_SHIFT 16
+#	define R500_STENCILWRITEMASK_MASK  0x00ff0000
+
+/* BEGIN: Vertex program instruction set */
+
+/* Every instruction is four dwords long:
+ *  DWORD 0: output and opcode
+ *  DWORD 1: first argument
+ *  DWORD 2: second argument
+ *  DWORD 3: third argument
+ *
+ * Notes:
+ *  - ABS r, a is implemented as MAX r, a, -a
+ *  - MOV is implemented as ADD to zero
+ *  - XPD is implemented as MUL + MAD
+ *  - FLR is implemented as FRC + ADD
+ *  - apparently, fglrx tries to schedule instructions so that there is at
+ *    least one instruction between the write to a temporary and the first
+ *    read from said temporary; however, violations of this scheduling are
+ *    allowed
+ *  - register indices seem to be unrelated with OpenGL aliasing to
+ *    conventional state
+ *  - only one attribute and one parameter can be loaded at a time; however,
+ *    the same attribute/parameter can be used for more than one argument
+ *  - the second software argument for POW is the third hardware argument
+ *    (no idea why)
+ *  - MAD with only temporaries as input seems to use VPI_OUT_SELECT_MAD_2
+ *
+ * There is some magic surrounding LIT:
+ *   The single argument is replicated across all three inputs, but swizzled:
+ *     First argument: xyzy
+ *     Second argument: xyzx
+ *     Third argument: xyzw
+ *   Whenever the result is used later in the fragment program, fglrx forces
+ *   x and w to be 1.0 in the input selection; I don't know whether this is
+ *   strictly necessary
+ */
 #define R300_VPI_OUT_OP_DOT                     (1 << 0)
 #define R300_VPI_OUT_OP_MUL                     (2 << 0)
 #define R300_VPI_OUT_OP_ADD                     (3 << 0)
@@ -1327,26 +1615,33 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VPI_OUT_OP_MIN                     (8 << 0)
 #define R300_VPI_OUT_OP_SGE                     (9 << 0)
 #define R300_VPI_OUT_OP_SLT                     (10 << 0)
-#define R300_VPI_OUT_OP_UNK12                   (12 << 0) /* Used in GL_POINT_DISTANCE_ATTENUATION_ARB, vector(scalar, vector) */
+	/* Used in GL_POINT_DISTANCE_ATTENUATION_ARB, vector(scalar, vector) */
+#define R300_VPI_OUT_OP_UNK12                   (12 << 0)
+#define R300_VPI_OUT_OP_ARL                     (13 << 0)
 #define R300_VPI_OUT_OP_EXP                     (65 << 0)
 #define R300_VPI_OUT_OP_LOG                     (66 << 0)
-#define R300_VPI_OUT_OP_UNK67                   (67 << 0) /* Used in fog computations, scalar(scalar) */
+	/* Used in fog computations, scalar(scalar) */
+#define R300_VPI_OUT_OP_UNK67                   (67 << 0)
 #define R300_VPI_OUT_OP_LIT                     (68 << 0)
 #define R300_VPI_OUT_OP_POW                     (69 << 0)
 #define R300_VPI_OUT_OP_RCP                     (70 << 0)
 #define R300_VPI_OUT_OP_RSQ                     (72 << 0)
-#define R300_VPI_OUT_OP_UNK73                   (73 << 0) /* Used in GL_POINT_DISTANCE_ATTENUATION_ARB, scalar(scalar) */
+	/* Used in GL_POINT_DISTANCE_ATTENUATION_ARB, scalar(scalar) */
+#define R300_VPI_OUT_OP_UNK73                   (73 << 0)
 #define R300_VPI_OUT_OP_EX2                     (75 << 0)
 #define R300_VPI_OUT_OP_LG2                     (76 << 0)
 #define R300_VPI_OUT_OP_MAD_2                   (128 << 0)
-#define R300_VPI_OUT_OP_UNK129                  (129 << 0) /* all temps, vector(scalar, vector, vector) */
+	/* all temps, vector(scalar, vector, vector) */
+#define R300_VPI_OUT_OP_UNK129                  (129 << 0)
 
 #define R300_VPI_OUT_REG_CLASS_TEMPORARY        (0 << 8)
+#define R300_VPI_OUT_REG_CLASS_ADDR             (1 << 8)
 #define R300_VPI_OUT_REG_CLASS_RESULT           (2 << 8)
 #define R300_VPI_OUT_REG_CLASS_MASK             (31 << 8)
 
 #define R300_VPI_OUT_REG_INDEX_SHIFT            13
-#define R300_VPI_OUT_REG_INDEX_MASK             (31 << 13) /* GUESS based on fglrx native limits */
+	/* GUESS based on fglrx native limits */
+#define R300_VPI_OUT_REG_INDEX_MASK             (31 << 13)
 
 #define R300_VPI_OUT_WRITE_X                    (1 << 20)
 #define R300_VPI_OUT_WRITE_Y                    (1 << 21)
@@ -1357,14 +1652,16 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VPI_IN_REG_CLASS_ATTRIBUTE         (1 << 0)
 #define R300_VPI_IN_REG_CLASS_PARAMETER         (2 << 0)
 #define R300_VPI_IN_REG_CLASS_NONE              (9 << 0)
-#define R300_VPI_IN_REG_CLASS_MASK              (31 << 0) /* GUESS */
+#define R300_VPI_IN_REG_CLASS_MASK              (31 << 0)
 
 #define R300_VPI_IN_REG_INDEX_SHIFT             5
-#define R300_VPI_IN_REG_INDEX_MASK              (255 << 5) /* GUESS based on fglrx native limits */
+	/* GUESS based on fglrx native limits */
+#define R300_VPI_IN_REG_INDEX_MASK              (255 << 5)
 
 /* The R300 can select components from the input register arbitrarily.
-// Use the following constants, shifted by the component shift you
-// want to select */
+ * Use the following constants, shifted by the component shift you
+ * want to select
+ */
 #define R300_VPI_IN_SELECT_X    0
 #define R300_VPI_IN_SELECT_Y    1
 #define R300_VPI_IN_SELECT_Z    2
@@ -1382,11 +1679,11 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_VPI_IN_NEG_Y                       (1 << 26)
 #define R300_VPI_IN_NEG_Z                       (1 << 27)
 #define R300_VPI_IN_NEG_W                       (1 << 28)
-/* END */
+/* END: Vertex program instruction set */
 
-//BEGIN: Packet 3 commands
+/* BEGIN: Packet 3 commands */
 
-// A primitive emission dword.
+/* A primitive emission dword. */
 #define R300_PRIM_TYPE_NONE                     (0 << 0)
 #define R300_PRIM_TYPE_POINT                    (1 << 0)
 #define R300_PRIM_TYPE_LINE                     (2 << 0)
@@ -1398,7 +1695,8 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_PRIM_TYPE_RECT_LIST                (8 << 0)
 #define R300_PRIM_TYPE_3VRT_POINT_LIST          (9 << 0)
 #define R300_PRIM_TYPE_3VRT_LINE_LIST           (10 << 0)
-#define R300_PRIM_TYPE_POINT_SPRITES            (11 << 0) // GUESS (based on r200)
+	/* GUESS (based on r200) */
+#define R300_PRIM_TYPE_POINT_SPRITES            (11 << 0)
 #define R300_PRIM_TYPE_LINE_LOOP                (12 << 0)
 #define R300_PRIM_TYPE_QUADS                    (13 << 0)
 #define R300_PRIM_TYPE_QUAD_STRIP               (14 << 0)
@@ -1408,37 +1706,76 @@ I am fairly certain that they are correct unless stated otherwise in comments.
 #define R300_PRIM_WALK_LIST                     (2 << 4)
 #define R300_PRIM_WALK_RING                     (3 << 4)
 #define R300_PRIM_WALK_MASK                     (3 << 4)
-#define R300_PRIM_COLOR_ORDER_BGRA              (0 << 6) // GUESS (based on r200)
-#define R300_PRIM_COLOR_ORDER_RGBA              (1 << 6) // GUESS
+	/* GUESS (based on r200) */
+#define R300_PRIM_COLOR_ORDER_BGRA              (0 << 6)
+#define R300_PRIM_COLOR_ORDER_RGBA              (1 << 6)
 #define R300_PRIM_NUM_VERTICES_SHIFT            16
+#define R300_PRIM_NUM_VERTICES_MASK             0xffff
 
-// Draw a primitive from vertex data in arrays loaded via 3D_LOAD_VBPNTR.
-// Two parameter dwords:
-// 0. The first parameter appears to be always 0
-// 1. The second parameter is a standard primitive emission dword.
+/* Draw a primitive from vertex data in arrays loaded via 3D_LOAD_VBPNTR.
+ * Two parameter dwords:
+ * 0. The first parameter appears to be always 0
+ * 1. The second parameter is a standard primitive emission dword.
+ */
 #define R300_PACKET3_3D_DRAW_VBUF           0x00002800
 
-// Specify the full set of vertex arrays as (address, stride).
-// The first parameter is the number of vertex arrays specified.
-// The rest of the command is a variable length list of blocks, where
-// each block is three dwords long and specifies two arrays.
-// The first dword of a block is split into two words, the lower significant
-// word refers to the first array, the more significant word to the second
-// array in the block.
-// The low byte of each word contains the size of an array entry in dwords,
-// the high byte contains the stride of the array.
-// The second dword of a block contains the pointer to the first array,
-// the third dword of a block contains the pointer to the second array.
-// Note that if the total number of arrays is odd, the third dword of
-// the last block is omitted.
+/* Specify the full set of vertex arrays as (address, stride).
+ * The first parameter is the number of vertex arrays specified.
+ * The rest of the command is a variable length list of blocks, where
+ * each block is three dwords long and specifies two arrays.
+ * The first dword of a block is split into two words, the lower significant
+ * word refers to the first array, the more significant word to the second
+ * array in the block.
+ * The low byte of each word contains the size of an array entry in dwords,
+ * the high byte contains the stride of the array.
+ * The second dword of a block contains the pointer to the first array,
+ * the third dword of a block contains the pointer to the second array.
+ * Note that if the total number of arrays is odd, the third dword of
+ * the last block is omitted.
+ */
 #define R300_PACKET3_3D_LOAD_VBPNTR         0x00002F00
 
 #define R300_PACKET3_INDX_BUFFER            0x00003300
 #    define R300_EB_UNK1_SHIFT                      24
 #    define R300_EB_UNK1                    (0x80<<24)
 #    define R300_EB_UNK2                        0x0810
+#define R300_PACKET3_3D_DRAW_VBUF_2         0x00003400
 #define R300_PACKET3_3D_DRAW_INDX_2         0x00003600
 
-//END
+/* END: Packet 3 commands */
+
+
+/* Color formats for 2d packets
+ */
+#define R300_CP_COLOR_FORMAT_CI8	2
+#define R300_CP_COLOR_FORMAT_ARGB1555	3
+#define R300_CP_COLOR_FORMAT_RGB565	4
+#define R300_CP_COLOR_FORMAT_ARGB8888	6
+#define R300_CP_COLOR_FORMAT_RGB332	7
+#define R300_CP_COLOR_FORMAT_RGB8	9
+#define R300_CP_COLOR_FORMAT_ARGB4444	15
+
+/*
+ * CP type-3 packets
+ */
+#define R300_CP_CMD_BITBLT_MULTI	0xC0009B00
+
+#define R500_VAP_INDEX_OFFSET		0x208c
+
+#define R500_GA_US_VECTOR_INDEX         0x4250
+#define R500_GA_US_VECTOR_DATA          0x4254
+
+#define R500_RS_IP_0                    0x4074
+#define R500_RS_INST_0                  0x4320
+
+#define R500_US_CONFIG                  0x4600
+
+#define R500_US_FC_CTRL			0x4624
+#define R500_US_CODE_ADDR		0x4630
+
+#define R500_RB3D_COLOR_CLEAR_VALUE_AR  0x46c0
+#define R500_RB3D_CONSTANT_COLOR_AR     0x4ef8
 
 #endif /* _R300_REG_H */
+
+/* *INDENT-ON* */
diff --git a/sys/dev/drm/radeon_cp.c b/sys/dev/drm/radeon_cp.c
index af2fdcf..401a094 100644
--- a/sys/dev/drm/radeon_cp.c
+++ b/sys/dev/drm/radeon_cp.c
@@ -2,6 +2,7 @@
 /*-
  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
+ * Copyright 2007 Advanced Micro Devices, Inc.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -37,789 +38,110 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/radeon_drv.h"
 #include "dev/drm/r300_reg.h"
 
+#include "dev/drm/radeon_microcode.h"
 #define RADEON_FIFO_DEBUG	0
 
-static int radeon_do_cleanup_cp(drm_device_t * dev);
-
-/* CP microcode (from ATI) */
-static const u32 R200_cp_microcode[][2] = {
-	{0x21007000, 0000000000},
-	{0x20007000, 0000000000},
-	{0x000000ab, 0x00000004},
-	{0x000000af, 0x00000004},
-	{0x66544a49, 0000000000},
-	{0x49494174, 0000000000},
-	{0x54517d83, 0000000000},
-	{0x498d8b64, 0000000000},
-	{0x49494949, 0000000000},
-	{0x49da493c, 0000000000},
-	{0x49989898, 0000000000},
-	{0xd34949d5, 0000000000},
-	{0x9dc90e11, 0000000000},
-	{0xce9b9b9b, 0000000000},
-	{0x000f0000, 0x00000016},
-	{0x352e232c, 0000000000},
-	{0x00000013, 0x00000004},
-	{0x000f0000, 0x00000016},
-	{0x352e272c, 0000000000},
-	{0x000f0001, 0x00000016},
-	{0x3239362f, 0000000000},
-	{0x000077ef, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x00000020, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00061000, 0x00000002},
-	{0x00000020, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00061000, 0x00000002},
-	{0x00000020, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00000016, 0x00000004},
-	{0x0003802a, 0x00000002},
-	{0x040067e0, 0x00000002},
-	{0x00000016, 0x00000004},
-	{0x000077e0, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x000037e1, 0x00000002},
-	{0x040067e1, 0x00000006},
-	{0x000077e0, 0x00000002},
-	{0x000077e1, 0x00000002},
-	{0x000077e1, 0x00000006},
-	{0xffffffff, 0000000000},
-	{0x10000000, 0000000000},
-	{0x0003802a, 0x00000002},
-	{0x040067e0, 0x00000006},
-	{0x00007675, 0x00000002},
-	{0x00007676, 0x00000002},
-	{0x00007677, 0x00000002},
-	{0x00007678, 0x00000006},
-	{0x0003802b, 0x00000002},
-	{0x04002676, 0x00000002},
-	{0x00007677, 0x00000002},
-	{0x00007678, 0x00000006},
-	{0x0000002e, 0x00000018},
-	{0x0000002e, 0x00000018},
-	{0000000000, 0x00000006},
-	{0x0000002f, 0x00000018},
-	{0x0000002f, 0x00000018},
-	{0000000000, 0x00000006},
-	{0x01605000, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x00098000, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x64c0603d, 0x00000004},
-	{0x00080000, 0x00000016},
-	{0000000000, 0000000000},
-	{0x0400251d, 0x00000002},
-	{0x00007580, 0x00000002},
-	{0x00067581, 0x00000002},
-	{0x04002580, 0x00000002},
-	{0x00067581, 0x00000002},
-	{0x00000046, 0x00000004},
-	{0x00005000, 0000000000},
-	{0x00061000, 0x00000002},
-	{0x0000750e, 0x00000002},
-	{0x00019000, 0x00000002},
-	{0x00011055, 0x00000014},
-	{0x00000055, 0x00000012},
-	{0x0400250f, 0x00000002},
-	{0x0000504a, 0x00000004},
-	{0x00007565, 0x00000002},
-	{0x00007566, 0x00000002},
-	{0x00000051, 0x00000004},
-	{0x01e655b4, 0x00000002},
-	{0x4401b0dc, 0x00000002},
-	{0x01c110dc, 0x00000002},
-	{0x2666705d, 0x00000018},
-	{0x040c2565, 0x00000002},
-	{0x0000005d, 0x00000018},
-	{0x04002564, 0x00000002},
-	{0x00007566, 0x00000002},
-	{0x00000054, 0x00000004},
-	{0x00401060, 0x00000008},
-	{0x00101000, 0x00000002},
-	{0x000d80ff, 0x00000002},
-	{0x00800063, 0x00000008},
-	{0x000f9000, 0x00000002},
-	{0x000e00ff, 0x00000002},
-	{0000000000, 0x00000006},
-	{0x00000080, 0x00000018},
-	{0x00000054, 0x00000004},
-	{0x00007576, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x00009000, 0x00000002},
-	{0x00041000, 0x00000002},
-	{0x0c00350e, 0x00000002},
-	{0x00049000, 0x00000002},
-	{0x00051000, 0x00000002},
-	{0x01e785f8, 0x00000002},
-	{0x00200000, 0x00000002},
-	{0x00600073, 0x0000000c},
-	{0x00007563, 0x00000002},
-	{0x006075f0, 0x00000021},
-	{0x20007068, 0x00000004},
-	{0x00005068, 0x00000004},
-	{0x00007576, 0x00000002},
-	{0x00007577, 0x00000002},
-	{0x0000750e, 0x00000002},
-	{0x0000750f, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00600076, 0x0000000c},
-	{0x006075f0, 0x00000021},
-	{0x000075f8, 0x00000002},
-	{0x00000076, 0x00000004},
-	{0x000a750e, 0x00000002},
-	{0x0020750f, 0x00000002},
-	{0x00600079, 0x00000004},
-	{0x00007570, 0x00000002},
-	{0x00007571, 0x00000002},
-	{0x00007572, 0x00000006},
-	{0x00005000, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00007568, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x00000084, 0x0000000c},
-	{0x00058000, 0x00000002},
-	{0x0c607562, 0x00000002},
-	{0x00000086, 0x00000004},
-	{0x00600085, 0x00000004},
-	{0x400070dd, 0000000000},
-	{0x000380dd, 0x00000002},
-	{0x00000093, 0x0000001c},
-	{0x00065095, 0x00000018},
-	{0x040025bb, 0x00000002},
-	{0x00061096, 0x00000018},
-	{0x040075bc, 0000000000},
-	{0x000075bb, 0x00000002},
-	{0x000075bc, 0000000000},
-	{0x00090000, 0x00000006},
-	{0x00090000, 0x00000002},
-	{0x000d8002, 0x00000006},
-	{0x00005000, 0x00000002},
-	{0x00007821, 0x00000002},
-	{0x00007800, 0000000000},
-	{0x00007821, 0x00000002},
-	{0x00007800, 0000000000},
-	{0x01665000, 0x00000002},
-	{0x000a0000, 0x00000002},
-	{0x000671cc, 0x00000002},
-	{0x0286f1cd, 0x00000002},
-	{0x000000a3, 0x00000010},
-	{0x21007000, 0000000000},
-	{0x000000aa, 0x0000001c},
-	{0x00065000, 0x00000002},
-	{0x000a0000, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x000b0000, 0x00000002},
-	{0x38067000, 0x00000002},
-	{0x000a00a6, 0x00000004},
-	{0x20007000, 0000000000},
-	{0x01200000, 0x00000002},
-	{0x20077000, 0x00000002},
-	{0x01200000, 0x00000002},
-	{0x20007000, 0000000000},
-	{0x00061000, 0x00000002},
-	{0x0120751b, 0x00000002},
-	{0x8040750a, 0x00000002},
-	{0x8040750b, 0x00000002},
-	{0x00110000, 0x00000002},
-	{0x000380dd, 0x00000002},
-	{0x000000bd, 0x0000001c},
-	{0x00061096, 0x00000018},
-	{0x844075bd, 0x00000002},
-	{0x00061095, 0x00000018},
-	{0x840075bb, 0x00000002},
-	{0x00061096, 0x00000018},
-	{0x844075bc, 0x00000002},
-	{0x000000c0, 0x00000004},
-	{0x804075bd, 0x00000002},
-	{0x800075bb, 0x00000002},
-	{0x804075bc, 0x00000002},
-	{0x00108000, 0x00000002},
-	{0x01400000, 0x00000002},
-	{0x006000c4, 0x0000000c},
-	{0x20c07000, 0x00000020},
-	{0x000000c6, 0x00000012},
-	{0x00800000, 0x00000006},
-	{0x0080751d, 0x00000006},
-	{0x000025bb, 0x00000002},
-	{0x000040c0, 0x00000004},
-	{0x0000775c, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00661000, 0x00000002},
-	{0x0460275d, 0x00000020},
-	{0x00004000, 0000000000},
-	{0x00007999, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00661000, 0x00000002},
-	{0x0460299b, 0x00000020},
-	{0x00004000, 0000000000},
-	{0x01e00830, 0x00000002},
-	{0x21007000, 0000000000},
-	{0x00005000, 0x00000002},
-	{0x00038042, 0x00000002},
-	{0x040025e0, 0x00000002},
-	{0x000075e1, 0000000000},
-	{0x00000001, 0000000000},
-	{0x000380d9, 0x00000002},
-	{0x04007394, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-};
-
-static const u32 radeon_cp_microcode[][2] = {
-	{0x21007000, 0000000000},
-	{0x20007000, 0000000000},
-	{0x000000b4, 0x00000004},
-	{0x000000b8, 0x00000004},
-	{0x6f5b4d4c, 0000000000},
-	{0x4c4c427f, 0000000000},
-	{0x5b568a92, 0000000000},
-	{0x4ca09c6d, 0000000000},
-	{0xad4c4c4c, 0000000000},
-	{0x4ce1af3d, 0000000000},
-	{0xd8afafaf, 0000000000},
-	{0xd64c4cdc, 0000000000},
-	{0x4cd10d10, 0000000000},
-	{0x000f0000, 0x00000016},
-	{0x362f242d, 0000000000},
-	{0x00000012, 0x00000004},
-	{0x000f0000, 0x00000016},
-	{0x362f282d, 0000000000},
-	{0x000380e7, 0x00000002},
-	{0x04002c97, 0x00000002},
-	{0x000f0001, 0x00000016},
-	{0x333a3730, 0000000000},
-	{0x000077ef, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x00000021, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00061000, 0x00000002},
-	{0x00000021, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00061000, 0x00000002},
-	{0x00000021, 0x0000001a},
-	{0x00004000, 0x0000001e},
-	{0x00000017, 0x00000004},
-	{0x0003802b, 0x00000002},
-	{0x040067e0, 0x00000002},
-	{0x00000017, 0x00000004},
-	{0x000077e0, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x000037e1, 0x00000002},
-	{0x040067e1, 0x00000006},
-	{0x000077e0, 0x00000002},
-	{0x000077e1, 0x00000002},
-	{0x000077e1, 0x00000006},
-	{0xffffffff, 0000000000},
-	{0x10000000, 0000000000},
-	{0x0003802b, 0x00000002},
-	{0x040067e0, 0x00000006},
-	{0x00007675, 0x00000002},
-	{0x00007676, 0x00000002},
-	{0x00007677, 0x00000002},
-	{0x00007678, 0x00000006},
-	{0x0003802c, 0x00000002},
-	{0x04002676, 0x00000002},
-	{0x00007677, 0x00000002},
-	{0x00007678, 0x00000006},
-	{0x0000002f, 0x00000018},
-	{0x0000002f, 0x00000018},
-	{0000000000, 0x00000006},
-	{0x00000030, 0x00000018},
-	{0x00000030, 0x00000018},
-	{0000000000, 0x00000006},
-	{0x01605000, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x00098000, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x64c0603e, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00080000, 0x00000016},
-	{0000000000, 0000000000},
-	{0x0400251d, 0x00000002},
-	{0x00007580, 0x00000002},
-	{0x00067581, 0x00000002},
-	{0x04002580, 0x00000002},
-	{0x00067581, 0x00000002},
-	{0x00000049, 0x00000004},
-	{0x00005000, 0000000000},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x0000750e, 0x00000002},
-	{0x00019000, 0x00000002},
-	{0x00011055, 0x00000014},
-	{0x00000055, 0x00000012},
-	{0x0400250f, 0x00000002},
-	{0x0000504f, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00007565, 0x00000002},
-	{0x00007566, 0x00000002},
-	{0x00000058, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x01e655b4, 0x00000002},
-	{0x4401b0e4, 0x00000002},
-	{0x01c110e4, 0x00000002},
-	{0x26667066, 0x00000018},
-	{0x040c2565, 0x00000002},
-	{0x00000066, 0x00000018},
-	{0x04002564, 0x00000002},
-	{0x00007566, 0x00000002},
-	{0x0000005d, 0x00000004},
-	{0x00401069, 0x00000008},
-	{0x00101000, 0x00000002},
-	{0x000d80ff, 0x00000002},
-	{0x0080006c, 0x00000008},
-	{0x000f9000, 0x00000002},
-	{0x000e00ff, 0x00000002},
-	{0000000000, 0x00000006},
-	{0x0000008f, 0x00000018},
-	{0x0000005b, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00007576, 0x00000002},
-	{0x00065000, 0x00000002},
-	{0x00009000, 0x00000002},
-	{0x00041000, 0x00000002},
-	{0x0c00350e, 0x00000002},
-	{0x00049000, 0x00000002},
-	{0x00051000, 0x00000002},
-	{0x01e785f8, 0x00000002},
-	{0x00200000, 0x00000002},
-	{0x0060007e, 0x0000000c},
-	{0x00007563, 0x00000002},
-	{0x006075f0, 0x00000021},
-	{0x20007073, 0x00000004},
-	{0x00005073, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00007576, 0x00000002},
-	{0x00007577, 0x00000002},
-	{0x0000750e, 0x00000002},
-	{0x0000750f, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00600083, 0x0000000c},
-	{0x006075f0, 0x00000021},
-	{0x000075f8, 0x00000002},
-	{0x00000083, 0x00000004},
-	{0x000a750e, 0x00000002},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x0020750f, 0x00000002},
-	{0x00600086, 0x00000004},
-	{0x00007570, 0x00000002},
-	{0x00007571, 0x00000002},
-	{0x00007572, 0x00000006},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00005000, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00007568, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x00000095, 0x0000000c},
-	{0x00058000, 0x00000002},
-	{0x0c607562, 0x00000002},
-	{0x00000097, 0x00000004},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x00600096, 0x00000004},
-	{0x400070e5, 0000000000},
-	{0x000380e6, 0x00000002},
-	{0x040025c5, 0x00000002},
-	{0x000380e5, 0x00000002},
-	{0x000000a8, 0x0000001c},
-	{0x000650aa, 0x00000018},
-	{0x040025bb, 0x00000002},
-	{0x000610ab, 0x00000018},
-	{0x040075bc, 0000000000},
-	{0x000075bb, 0x00000002},
-	{0x000075bc, 0000000000},
-	{0x00090000, 0x00000006},
-	{0x00090000, 0x00000002},
-	{0x000d8002, 0x00000006},
-	{0x00007832, 0x00000002},
-	{0x00005000, 0x00000002},
-	{0x000380e7, 0x00000002},
-	{0x04002c97, 0x00000002},
-	{0x00007820, 0x00000002},
-	{0x00007821, 0x00000002},
-	{0x00007800, 0000000000},
-	{0x01200000, 0x00000002},
-	{0x20077000, 0x00000002},
-	{0x01200000, 0x00000002},
-	{0x20007000, 0x00000002},
-	{0x00061000, 0x00000002},
-	{0x0120751b, 0x00000002},
-	{0x8040750a, 0x00000002},
-	{0x8040750b, 0x00000002},
-	{0x00110000, 0x00000002},
-	{0x000380e5, 0x00000002},
-	{0x000000c6, 0x0000001c},
-	{0x000610ab, 0x00000018},
-	{0x844075bd, 0x00000002},
-	{0x000610aa, 0x00000018},
-	{0x840075bb, 0x00000002},
-	{0x000610ab, 0x00000018},
-	{0x844075bc, 0x00000002},
-	{0x000000c9, 0x00000004},
-	{0x804075bd, 0x00000002},
-	{0x800075bb, 0x00000002},
-	{0x804075bc, 0x00000002},
-	{0x00108000, 0x00000002},
-	{0x01400000, 0x00000002},
-	{0x006000cd, 0x0000000c},
-	{0x20c07000, 0x00000020},
-	{0x000000cf, 0x00000012},
-	{0x00800000, 0x00000006},
-	{0x0080751d, 0x00000006},
-	{0000000000, 0000000000},
-	{0x0000775c, 0x00000002},
-	{0x00a05000, 0x00000002},
-	{0x00661000, 0x00000002},
-	{0x0460275d, 0x00000020},
-	{0x00004000, 0000000000},
-	{0x01e00830, 0x00000002},
-	{0x21007000, 0000000000},
-	{0x6464614d, 0000000000},
-	{0x69687420, 0000000000},
-	{0x00000073, 0000000000},
-	{0000000000, 0000000000},
-	{0x00005000, 0x00000002},
-	{0x000380d0, 0x00000002},
-	{0x040025e0, 0x00000002},
-	{0x000075e1, 0000000000},
-	{0x00000001, 0000000000},
-	{0x000380e0, 0x00000002},
-	{0x04002394, 0x00000002},
-	{0x00005000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0x00000008, 0000000000},
-	{0x00000004, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-	{0000000000, 0000000000},
-};
-
-static const u32 R300_cp_microcode[][2] = {
-	{ 0x4200e000, 0000000000 },
-	{ 0x4000e000, 0000000000 },
-	{ 0x000000af, 0x00000008 },
-	{ 0x000000b3, 0x00000008 },
-	{ 0x6c5a504f, 0000000000 },
-	{ 0x4f4f497a, 0000000000 },
-	{ 0x5a578288, 0000000000 },
-	{ 0x4f91906a, 0000000000 },
-	{ 0x4f4f4f4f, 0000000000 },
-	{ 0x4fe24f44, 0000000000 },
-	{ 0x4f9c9c9c, 0000000000 },
-	{ 0xdc4f4fde, 0000000000 },
-	{ 0xa1cd4f4f, 0000000000 },
-	{ 0xd29d9d9d, 0000000000 },
-	{ 0x4f0f9fd7, 0000000000 },
-	{ 0x000ca000, 0x00000004 },
-	{ 0x000d0012, 0x00000038 },
-	{ 0x0000e8b4, 0x00000004 },
-	{ 0x000d0014, 0x00000038 },
-	{ 0x0000e8b6, 0x00000004 },
-	{ 0x000d0016, 0x00000038 },
-	{ 0x0000e854, 0x00000004 },
-	{ 0x000d0018, 0x00000038 },
-	{ 0x0000e855, 0x00000004 },
-	{ 0x000d001a, 0x00000038 },
-	{ 0x0000e856, 0x00000004 },
-	{ 0x000d001c, 0x00000038 },
-	{ 0x0000e857, 0x00000004 },
-	{ 0x000d001e, 0x00000038 },
-	{ 0x0000e824, 0x00000004 },
-	{ 0x000d0020, 0x00000038 },
-	{ 0x0000e825, 0x00000004 },
-	{ 0x000d0022, 0x00000038 },
-	{ 0x0000e830, 0x00000004 },
-	{ 0x000d0024, 0x00000038 },
-	{ 0x0000f0c0, 0x00000004 },
-	{ 0x000d0026, 0x00000038 },
-	{ 0x0000f0c1, 0x00000004 },
-	{ 0x000d0028, 0x00000038 },
-	{ 0x0000f041, 0x00000004 },
-	{ 0x000d002a, 0x00000038 },
-	{ 0x0000f184, 0x00000004 },
-	{ 0x000d002c, 0x00000038 },
-	{ 0x0000f185, 0x00000004 },
-	{ 0x000d002e, 0x00000038 },
-	{ 0x0000f186, 0x00000004 },
-	{ 0x000d0030, 0x00000038 },
-	{ 0x0000f187, 0x00000004 },
-	{ 0x000d0032, 0x00000038 },
-	{ 0x0000f180, 0x00000004 },
-	{ 0x000d0034, 0x00000038 },
-	{ 0x0000f393, 0x00000004 },
-	{ 0x000d0036, 0x00000038 },
-	{ 0x0000f38a, 0x00000004 },
-	{ 0x000d0038, 0x00000038 },
-	{ 0x0000f38e, 0x00000004 },
-	{ 0x0000e821, 0x00000004 },
-	{ 0x0140a000, 0x00000004 },
-	{ 0x00000043, 0x00000018 },
-	{ 0x00cce800, 0x00000004 },
-	{ 0x001b0001, 0x00000004 },
-	{ 0x08004800, 0x00000004 },
-	{ 0x001b0001, 0x00000004 },
-	{ 0x08004800, 0x00000004 },
-	{ 0x001b0001, 0x00000004 },
-	{ 0x08004800, 0x00000004 },
-	{ 0x0000003a, 0x00000008 },
-	{ 0x0000a000, 0000000000 },
-	{ 0x02c0a000, 0x00000004 },
-	{ 0x000ca000, 0x00000004 },
-	{ 0x00130000, 0x00000004 },
-	{ 0x000c2000, 0x00000004 },
-	{ 0xc980c045, 0x00000008 },
-	{ 0x2000451d, 0x00000004 },
-	{ 0x0000e580, 0x00000004 },
-	{ 0x000ce581, 0x00000004 },
-	{ 0x08004580, 0x00000004 },
-	{ 0x000ce581, 0x00000004 },
-	{ 0x0000004c, 0x00000008 },
-	{ 0x0000a000, 0000000000 },
-	{ 0x000c2000, 0x00000004 },
-	{ 0x0000e50e, 0x00000004 },
-	{ 0x00032000, 0x00000004 },
-	{ 0x00022056, 0x00000028 },
-	{ 0x00000056, 0x00000024 },
-	{ 0x0800450f, 0x00000004 },
-	{ 0x0000a050, 0x00000008 },
-	{ 0x0000e565, 0x00000004 },
-	{ 0x0000e566, 0x00000004 },
-	{ 0x00000057, 0x00000008 },
-	{ 0x03cca5b4, 0x00000004 },
-	{ 0x05432000, 0x00000004 },
-	{ 0x00022000, 0x00000004 },
-	{ 0x4ccce063, 0x00000030 },
-	{ 0x08274565, 0x00000004 },
-	{ 0x00000063, 0x00000030 },
-	{ 0x08004564, 0x00000004 },
-	{ 0x0000e566, 0x00000004 },
-	{ 0x0000005a, 0x00000008 },
-	{ 0x00802066, 0x00000010 },
-	{ 0x00202000, 0x00000004 },
-	{ 0x001b00ff, 0x00000004 },
-	{ 0x01000069, 0x00000010 },
-	{ 0x001f2000, 0x00000004 },
-	{ 0x001c00ff, 0x00000004 },
-	{ 0000000000, 0x0000000c },
-	{ 0x00000085, 0x00000030 },
-	{ 0x0000005a, 0x00000008 },
-	{ 0x0000e576, 0x00000004 },
-	{ 0x000ca000, 0x00000004 },
-	{ 0x00012000, 0x00000004 },
-	{ 0x00082000, 0x00000004 },
-	{ 0x1800650e, 0x00000004 },
-	{ 0x00092000, 0x00000004 },
-	{ 0x000a2000, 0x00000004 },
-	{ 0x000f0000, 0x00000004 },
-	{ 0x00400000, 0x00000004 },
-	{ 0x00000079, 0x00000018 },
-	{ 0x0000e563, 0x00000004 },
-	{ 0x00c0e5f9, 0x000000c2 },
-	{ 0x0000006e, 0x00000008 },
-	{ 0x0000a06e, 0x00000008 },
-	{ 0x0000e576, 0x00000004 },
-	{ 0x0000e577, 0x00000004 },
-	{ 0x0000e50e, 0x00000004 },
-	{ 0x0000e50f, 0x00000004 },
-	{ 0x0140a000, 0x00000004 },
-	{ 0x0000007c, 0x00000018 },
-	{ 0x00c0e5f9, 0x000000c2 },
-	{ 0x0000007c, 0x00000008 },
-	{ 0x0014e50e, 0x00000004 },
-	{ 0x0040e50f, 0x00000004 },
-	{ 0x00c0007f, 0x00000008 },
-	{ 0x0000e570, 0x00000004 },
-	{ 0x0000e571, 0x00000004 },
-	{ 0x0000e572, 0x0000000c },
-	{ 0x0000a000, 0x00000004 },
-	{ 0x0140a000, 0x00000004 }, 
-	{ 0x0000e568, 0x00000004 },
-	{ 0x000c2000, 0x00000004 },
-	{ 0x00000089, 0x00000018 },
-	{ 0x000b0000, 0x00000004 },
-	{ 0x18c0e562, 0x00000004 },
-	{ 0x0000008b, 0x00000008 },
-	{ 0x00c0008a, 0x00000008 },
-	{ 0x000700e4, 0x00000004 },
-	{ 0x00000097, 0x00000038 },
-	{ 0x000ca099, 0x00000030 },
-	{ 0x080045bb, 0x00000004 },
-	{ 0x000c209a, 0x00000030 },
-	{ 0x0800e5bc, 0000000000 },
-	{ 0x0000e5bb, 0x00000004 },
-	{ 0x0000e5bc, 0000000000 },
-	{ 0x00120000, 0x0000000c },
-	{ 0x00120000, 0x00000004 },
-	{ 0x001b0002, 0x0000000c },
-	{ 0x0000a000, 0x00000004 },
-	{ 0x0000e821, 0x00000004 },
-	{ 0x0000e800, 0000000000 },
-	{ 0x0000e821, 0x00000004 },
-	{ 0x0000e82e, 0000000000 },
-	{ 0x02cca000, 0x00000004 },
-	{ 0x00140000, 0x00000004 },
-	{ 0x000ce1cc, 0x00000004 },
-	{ 0x050de1cd, 0x00000004 },
-	{ 0x000000a7, 0x00000020 },
-	{ 0x4200e000, 0000000000 },
-	{ 0x000000ae, 0x00000038 },
-	{ 0x000ca000, 0x00000004 },
-	{ 0x00140000, 0x00000004 },
-	{ 0x000c2000, 0x00000004 },
-	{ 0x00160000, 0x00000004 },
-	{ 0x700ce000, 0x00000004 },
-	{ 0x001400aa, 0x00000008 },
-	{ 0x4000e000, 0000000000 },
-	{ 0x02400000, 0x00000004 },
-	{ 0x400ee000, 0x00000004 },
-	{ 0x02400000, 0x00000004 },
-	{ 0x4000e000, 0000000000 },
-	{ 0x000c2000, 0x00000004 },
-	{ 0x0240e51b, 0x00000004 },
-	{ 0x0080e50a, 0x00000005 },
-	{ 0x0080e50b, 0x00000005 },
-	{ 0x00220000, 0x00000004 },
-	{ 0x000700e4, 0x00000004 },
-	{ 0x000000c1, 0x00000038 },
-	{ 0x000c209a, 0x00000030 },
-	{ 0x0880e5bd, 0x00000005 },
-	{ 0x000c2099, 0x00000030 },
-	{ 0x0800e5bb, 0x00000005 },
-	{ 0x000c209a, 0x00000030 },
-	{ 0x0880e5bc, 0x00000005 },
-	{ 0x000000c4, 0x00000008 },
-	{ 0x0080e5bd, 0x00000005 },
-	{ 0x0000e5bb, 0x00000005 },
-	{ 0x0080e5bc, 0x00000005 },
-	{ 0x00210000, 0x00000004 },
-	{ 0x02800000, 0x00000004 },
-	{ 0x00c000c8, 0x00000018 },
-	{ 0x4180e000, 0x00000040 },
-	{ 0x000000ca, 0x00000024 },
-	{ 0x01000000, 0x0000000c },
-	{ 0x0100e51d, 0x0000000c },
-	{ 0x000045bb, 0x00000004 },
-	{ 0x000080c4, 0x00000008 },
-	{ 0x0000f3ce, 0x00000004 },
-	{ 0x0140a000, 0x00000004 },
-	{ 0x00cc2000, 0x00000004 },
-	{ 0x08c053cf, 0x00000040 },
-	{ 0x00008000, 0000000000 },
-	{ 0x0000f3d2, 0x00000004 },
-	{ 0x0140a000, 0x00000004 },
-	{ 0x00cc2000, 0x00000004 },
-	{ 0x08c053d3, 0x00000040 },
-	{ 0x00008000, 0000000000 },
-	{ 0x0000f39d, 0x00000004 },
-	{ 0x0140a000, 0x00000004 },
-	{ 0x00cc2000, 0x00000004 },
-	{ 0x08c0539e, 0x00000040 },
-	{ 0x00008000, 0000000000 },
-	{ 0x03c00830, 0x00000004 },
-	{ 0x4200e000, 0000000000 },
-	{ 0x0000a000, 0x00000004 },
-	{ 0x200045e0, 0x00000004 },
-	{ 0x0000e5e1, 0000000000 },
-	{ 0x00000001, 0000000000 },
-	{ 0x000700e1, 0x00000004 },
-	{ 0x0800e394, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-	{ 0000000000, 0000000000 },
-};
-
-static int RADEON_READ_PLL(drm_device_t * dev, int addr)
+static int radeon_do_cleanup_cp(struct drm_device * dev);
+static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
+
+static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
+{
+	u32 ret;
+	RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
+	ret = RADEON_READ(R520_MC_IND_DATA);
+	RADEON_WRITE(R520_MC_IND_INDEX, 0);
+	return ret;
+}
+
+static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
+{
+	u32 ret;
+	RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
+	ret = RADEON_READ(RS480_NB_MC_DATA);
+	RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
+	return ret;
+}
+
+static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
+{
+	u32 ret;
+	RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
+	ret = RADEON_READ(RS690_MC_DATA);
+	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
+	return ret;
+}
+
+static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
+{
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
+	    return RS690_READ_MCIND(dev_priv, addr);
+	else
+	    return RS480_READ_MCIND(dev_priv, addr);
+}
+
+u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
+{
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
+		return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
+		return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
+		return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
+	else
+		return RADEON_READ(RADEON_MC_FB_LOCATION);
+}
+
+static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
+{
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
+		R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
+		RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
+		R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
+	else
+		RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
+}
+
+static void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
+{
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
+		R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
+		RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
+	else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
+		R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
+	else
+		RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
+}
+
+static void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
+{
+	u32 agp_base_hi = upper_32_bits(agp_base);
+	u32 agp_base_lo = agp_base & 0xffffffff;
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
+		R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
+		R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
+	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
+		RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
+		RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
+	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
+		R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
+		R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
+	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
+		RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
+		RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
+	} else {
+		RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
+		if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
+			RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
+	}
+}
+
+static int RADEON_READ_PLL(struct drm_device * dev, int addr)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 
@@ -827,7 +149,7 @@ static int RADEON_READ_PLL(drm_device_t * dev, int addr)
 	return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 }
 
-static int RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
+static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 {
 	RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 	return RADEON_READ(RADEON_PCIE_DATA);
@@ -867,23 +189,28 @@ static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 
 	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 
-	tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
-	tmp |= RADEON_RB3D_DC_FLUSH_ALL;
-	RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
+		tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
+		tmp |= RADEON_RB3D_DC_FLUSH_ALL;
+		RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 
-	for (i = 0; i < dev_priv->usec_timeout; i++) {
-		if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
-		      & RADEON_RB3D_DC_BUSY)) {
-			return 0;
+		for (i = 0; i < dev_priv->usec_timeout; i++) {
+			if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
+			      & RADEON_RB3D_DC_BUSY)) {
+				return 0;
+			}
+			DRM_UDELAY(1);
 		}
-		DRM_UDELAY(1);
+	} else {
+		/* don't flush or purge cache here or lockup */
+		return 0;
 	}
 
 #if RADEON_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 	radeon_status(dev_priv);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
@@ -899,12 +226,15 @@ static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 			return 0;
 		DRM_UDELAY(1);
 	}
+	DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
+		 RADEON_READ(RADEON_RBBM_STATUS),
+		 RADEON_READ(R300_VAP_CNTL_STATUS));
 
 #if RADEON_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 	radeon_status(dev_priv);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
@@ -925,12 +255,59 @@ static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 		}
 		DRM_UDELAY(1);
 	}
+	DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
+		 RADEON_READ(RADEON_RBBM_STATUS),
+		 RADEON_READ(R300_VAP_CNTL_STATUS));
 
 #if RADEON_FIFO_DEBUG
 	DRM_ERROR("failed!\n");
 	radeon_status(dev_priv);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
+}
+
+static void radeon_init_pipes(drm_radeon_private_t * dev_priv)
+{
+	uint32_t gb_tile_config, gb_pipe_sel = 0;
+
+	/* RS4xx/RS6xx/R4xx/R5xx */
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
+		gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
+		dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
+	} else {
+		/* R3xx */
+		if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
+		    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
+			dev_priv->num_gb_pipes = 2;
+		} else {
+			/* R3Vxx */
+			dev_priv->num_gb_pipes = 1;
+		}
+	}
+	DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
+
+	gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
+
+	switch(dev_priv->num_gb_pipes) {
+	case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
+	case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
+	case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
+	default:
+	case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
+	}
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
+		RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
+		RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
+	}
+	RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
+	radeon_do_wait_for_idle(dev_priv);
+	RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
+	RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
+					       R300_DC_AUTOFLUSH_ENABLE |
+					       R300_DC_DC_DISABLE_IGNORE_PE));
+
+
 }
 
 /* ================================================================
@@ -947,7 +324,22 @@ static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 
 	RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 
-	if (dev_priv->microcode_version == UCODE_R200) {
+	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
+	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
+	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
+	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
+	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
+		DRM_INFO("Loading R100 Microcode\n");
+		for (i = 0; i < 256; i++) {
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
+				     R100_cp_microcode[i][1]);
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
+				     R100_cp_microcode[i][0]);
+		}
+	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 		DRM_INFO("Loading R200 Microcode\n");
 		for (i = 0; i < 256; i++) {
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
@@ -955,7 +347,12 @@ static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 				     R200_cp_microcode[i][0]);
 		}
-	} else if (dev_priv->microcode_version == UCODE_R300) {
+	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 		DRM_INFO("Loading R300 Microcode\n");
 		for (i = 0; i < 256; i++) {
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
@@ -963,12 +360,35 @@ static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 				     R300_cp_microcode[i][0]);
 		}
-	} else {
+	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
+		DRM_INFO("Loading R400 Microcode\n");
+		for (i = 0; i < 256; i++) {
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
+				     R420_cp_microcode[i][1]);
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
+				     R420_cp_microcode[i][0]);
+		}
+	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) {
+		DRM_INFO("Loading RS690 Microcode\n");
+		for (i = 0; i < 256; i++) {
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
+				     RS690_cp_microcode[i][1]);
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
+				     RS690_cp_microcode[i][0]);
+		}
+	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
+		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
+		DRM_INFO("Loading R500 Microcode\n");
 		for (i = 0; i < 256; i++) {
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     radeon_cp_microcode[i][1]);
+				     R520_cp_microcode[i][1]);
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     radeon_cp_microcode[i][0]);
+				     R520_cp_microcode[i][0]);
 		}
 	}
 }
@@ -1020,14 +440,20 @@ static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 
 	dev_priv->cp_running = 1;
 
-	BEGIN_RING(6);
-
+	BEGIN_RING(8);
+	/* isync can only be written through cp on r5xx write it here */
+	OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
+	OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
+		 RADEON_ISYNC_ANY3D_IDLE2D |
+		 RADEON_ISYNC_WAIT_IDLEGUI |
+		 RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 	RADEON_PURGE_CACHE();
 	RADEON_PURGE_ZCACHE();
 	RADEON_WAIT_UNTIL_IDLE();
-
 	ADVANCE_RING();
 	COMMIT_RING();
+
+	dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 }
 
 /* Reset the Command Processor.  This will not flush any pending
@@ -1060,24 +486,27 @@ static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 
 /* Reset the engine.  This will stop the CP if it is running.
  */
-static int radeon_do_engine_reset(drm_device_t * dev)
+static int radeon_do_engine_reset(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	u32 clock_cntl_index, mclk_cntl, rbbm_soft_reset;
+	u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 	DRM_DEBUG("\n");
 
 	radeon_do_pixcache_flush(dev_priv);
 
-	clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
-	mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
-
-	RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
-					    RADEON_FORCEON_MCLKA |
-					    RADEON_FORCEON_MCLKB |
-					    RADEON_FORCEON_YCLKA |
-					    RADEON_FORCEON_YCLKB |
-					    RADEON_FORCEON_MC |
-					    RADEON_FORCEON_AIC));
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
+	        /* may need something similar for newer chips */
+		clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
+		mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
+
+		RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
+						    RADEON_FORCEON_MCLKA |
+						    RADEON_FORCEON_MCLKB |
+						    RADEON_FORCEON_YCLKA |
+						    RADEON_FORCEON_YCLKB |
+						    RADEON_FORCEON_MC |
+						    RADEON_FORCEON_AIC));
+	}
 
 	rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 
@@ -1100,9 +529,15 @@ static int radeon_do_engine_reset(drm_device_t * dev)
 						RADEON_SOFT_RESET_RB)));
 	RADEON_READ(RADEON_RBBM_SOFT_RESET);
 
-	RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
-	RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
-	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
+		RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
+		RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
+		RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
+	}
+
+	/* setup the raster pipes */
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
+	    radeon_init_pipes(dev_priv);
 
 	/* Reset the CP ring */
 	radeon_do_cp_reset(dev_priv);
@@ -1116,26 +551,27 @@ static int radeon_do_engine_reset(drm_device_t * dev)
 	return 0;
 }
 
-static void radeon_cp_init_ring_buffer(drm_device_t * dev,
+static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 				       drm_radeon_private_t * dev_priv)
 {
 	u32 ring_start, cur_read_ptr;
 	u32 tmp;
-	
+
 	/* Initialize the memory controller. With new memory map, the fb location
 	 * is not changed, it should have been properly initialized already. Part
 	 * of the problem is that the code below is bogus, assuming the GART is
 	 * always appended to the fb which is not necessarily the case
 	 */
 	if (!dev_priv->new_memmap)
-		RADEON_WRITE(RADEON_MC_FB_LOCATION,
+		radeon_write_fb_location(dev_priv,
 			     ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 			     | (dev_priv->fb_location >> 16));
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
-		RADEON_WRITE(RADEON_AGP_BASE, (unsigned int)dev->agp->base);
-		RADEON_WRITE(RADEON_MC_AGP_LOCATION,
+	if (dev_priv->flags & RADEON_IS_AGP) {
+		radeon_write_agp_base(dev_priv, dev->agp->base);
+
+		radeon_write_agp_location(dev_priv,
 			     (((dev_priv->gart_vm_start - 1 +
 				dev_priv->gart_size) & 0xffff0000) |
 			      (dev_priv->gart_vm_start >> 16)));
@@ -1161,14 +597,14 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 	dev_priv->ring.tail = cur_read_ptr;
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
+	if (dev_priv->flags & RADEON_IS_AGP) {
 		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 			     dev_priv->ring_rptr->offset
 			     - dev->agp->base + dev_priv->gart_vm_start);
 	} else
 #endif
 	{
-		drm_sg_mem_t *entry = dev->sg;
+		struct drm_sg_mem *entry = dev->sg;
 		unsigned long tmp_ofs, page_ofs;
 
 		tmp_ofs = dev_priv->ring_rptr->offset -
@@ -1184,9 +620,15 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 	/* Set ring buffer size */
 #ifdef __BIG_ENDIAN
 	RADEON_WRITE(RADEON_CP_RB_CNTL,
-		     dev_priv->ring.size_l2qw | RADEON_BUF_SWAP_32BIT);
+		     RADEON_BUF_SWAP_32BIT |
+		     (dev_priv->ring.fetch_size_l2ow << 18) |
+		     (dev_priv->ring.rptr_update_l2qw << 8) |
+		     dev_priv->ring.size_l2qw);
 #else
-	RADEON_WRITE(RADEON_CP_RB_CNTL, dev_priv->ring.size_l2qw);
+	RADEON_WRITE(RADEON_CP_RB_CNTL,
+		     (dev_priv->ring.fetch_size_l2ow << 18) |
+		     (dev_priv->ring.rptr_update_l2qw << 8) |
+		     dev_priv->ring.size_l2qw);
 #endif
 
 	/* Start with assuming that writeback doesn't work */
@@ -1269,7 +711,77 @@ static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 	}
 }
 
-/* Enable or disable PCI-E GART on the chip */
+/* Enable or disable IGP GART on the chip */
+static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
+{
+	u32 temp;
+
+	if (on) {
+		DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
+			 dev_priv->gart_vm_start,
+			 (long)dev_priv->gart_info.bus_addr,
+			 dev_priv->gart_size);
+
+		temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
+
+		if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)
+			IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
+							     RS690_BLOCK_GFX_D3_EN));
+		else
+			IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
+
+		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
+							       RS480_VA_SIZE_32MB));
+
+		temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
+		IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
+							RS480_TLB_ENABLE |
+							RS480_GTW_LAC_EN |
+							RS480_1LEVEL_GART));
+
+		temp = dev_priv->gart_info.bus_addr & 0xfffff000;
+		temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
+		IGP_WRITE_MCIND(RS480_GART_BASE, temp);
+
+		temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
+		IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
+						      RS480_REQ_TYPE_SNOOP_DIS));
+
+		radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
+
+		dev_priv->gart_size = 32*1024*1024;
+		temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) & 
+			0xffff0000) | (dev_priv->gart_vm_start >> 16));
+
+		radeon_write_agp_location(dev_priv, temp);
+
+		temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
+		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
+							       RS480_VA_SIZE_32MB));
+
+		do {
+			temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
+			if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
+				break;
+			DRM_UDELAY(1);
+		} while(1);
+
+		IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
+				RS480_GART_CACHE_INVALIDATE);
+
+		do {
+			temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
+			if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
+				break;
+			DRM_UDELAY(1);
+		} while(1);
+
+		IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
+	} else {
+		IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
+	}
+}
+
 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 {
 	u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
@@ -1289,7 +801,7 @@ static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 				  dev_priv->gart_vm_start +
 				  dev_priv->gart_size - 1);
 
-		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
+		radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
 
 		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
 				  RADEON_PCIE_TX_GART_EN);
@@ -1304,12 +816,18 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 {
 	u32 tmp;
 
-	if (dev_priv->flags & CHIP_IS_PCIE) {
+	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
+	    (dev_priv->flags & RADEON_IS_IGPGART)) {
+		radeon_set_igpgart(dev_priv, on);
+		return;
+	}
+
+	if (dev_priv->flags & RADEON_IS_PCIE) {
 		radeon_set_pciegart(dev_priv, on);
 		return;
 	}
 
- 	tmp = RADEON_READ(RADEON_AIC_CNTL);
+	tmp = RADEON_READ(RADEON_AIC_CNTL);
 
 	if (on) {
 		RADEON_WRITE(RADEON_AIC_CNTL,
@@ -1327,7 +845,7 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 
 		/* Turn off AGP aperture -- is this required for PCI GART?
 		 */
-		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
+		radeon_write_agp_location(dev_priv, 0xffffffc0);
 		RADEON_WRITE(RADEON_AGP_COMMAND, 0);	/* clear AGP_COMMAND */
 	} else {
 		RADEON_WRITE(RADEON_AIC_CNTL,
@@ -1335,36 +853,35 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 	}
 }
 
-static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
+static int radeon_do_init_cp(struct drm_device * dev, drm_radeon_init_t * init)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 
 	DRM_DEBUG("\n");
 
 	/* if we require new memory map but we don't have it fail */
-	if ((dev_priv->flags & CHIP_NEW_MEMMAP) && !dev_priv->new_memmap)
-	{
+	if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
 		DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (init->is_pci && (dev_priv->flags & CHIP_IS_AGP))
+	if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP))
 	{
 		DRM_DEBUG("Forcing AGP card to PCI mode\n");
-		dev_priv->flags &= ~CHIP_IS_AGP;
+		dev_priv->flags &= ~RADEON_IS_AGP;
 	}
-	else if (!(dev_priv->flags & (CHIP_IS_AGP | CHIP_IS_PCI | CHIP_IS_PCIE))
+	else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
 		 && !init->is_pci)
 	{
 		DRM_DEBUG("Restoring AGP flag\n");
-		dev_priv->flags |= CHIP_IS_AGP;
+		dev_priv->flags |= RADEON_IS_AGP;
 	}
 
-	if ((!(dev_priv->flags & CHIP_IS_AGP)) && !dev->sg) {
+	if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
 		DRM_ERROR("PCI GART memory not allocated!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->usec_timeout = init->usec_timeout;
@@ -1372,19 +889,12 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	    dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
 		DRM_DEBUG("TIMEOUT problem!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	switch(init->func) {
-	case RADEON_INIT_R200_CP:
-		dev_priv->microcode_version = UCODE_R200;
-		break;
-	case RADEON_INIT_R300_CP:
-		dev_priv->microcode_version = UCODE_R300;
-		break;
-	default:
-		dev_priv->microcode_version = UCODE_R100;
-	}
+	/* Enable vblank on CRTC1 for older X servers
+	 */
+	dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
 
 	dev_priv->do_boxes = 0;
 	dev_priv->cp_mode = init->cp_mode;
@@ -1397,7 +907,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	    (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
 		DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	switch (init->fb_bpp) {
@@ -1433,8 +943,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	 */
 	dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
 					   (dev_priv->color_fmt << 10) |
-					   (dev_priv->microcode_version ==
-					    UCODE_R100 ? RADEON_ZBLOCK16 : 0));
+					   (dev_priv->chip_family < CHIP_R200 ? RADEON_ZBLOCK16 : 0));
 
 	dev_priv->depth_clear.rb3d_zstencilcntl =
 	    (dev_priv->depth_fmt |
@@ -1456,37 +965,37 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 					 RADEON_ROUND_MODE_TRUNC |
 					 RADEON_ROUND_PREC_8TH_PIX);
 
-	DRM_GETSAREA();
 
 	dev_priv->ring_offset = init->ring_offset;
 	dev_priv->ring_rptr_offset = init->ring_rptr_offset;
 	dev_priv->buffers_offset = init->buffers_offset;
 	dev_priv->gart_textures_offset = init->gart_textures_offset;
 
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("could not find sarea!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
 	if (!dev_priv->cp_ring) {
 		DRM_ERROR("could not find cp ring region!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
 	if (!dev_priv->ring_rptr) {
 		DRM_ERROR("could not find ring read pointer!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	dev->agp_buffer_token = init->buffers_offset;
 	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
 	if (!dev->agp_buffer_map) {
 		DRM_ERROR("could not find dma buffer region!\n");
 		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (init->gart_textures_offset) {
@@ -1495,7 +1004,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		if (!dev_priv->gart_textures) {
 			DRM_ERROR("could not find GART texture region!\n");
 			radeon_do_cleanup_cp(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -1504,7 +1013,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 				    init->sarea_priv_offset);
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
+	if (dev_priv->flags & RADEON_IS_AGP) {
 		drm_core_ioremap(dev_priv->cp_ring, dev);
 		drm_core_ioremap(dev_priv->ring_rptr, dev);
 		drm_core_ioremap(dev->agp_buffer_map, dev);
@@ -1513,7 +1022,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		    !dev->agp_buffer_map->handle) {
 			DRM_ERROR("could not find ioremap agp regions!\n");
 			radeon_do_cleanup_cp(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else
 #endif
@@ -1532,10 +1041,9 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 			  dev->agp_buffer_map->handle);
 	}
 
-	dev_priv->fb_location = (RADEON_READ(RADEON_MC_FB_LOCATION)
-				 & 0xffff) << 16;
-	dev_priv->fb_size = 
-		((RADEON_READ(RADEON_MC_FB_LOCATION) & 0xffff0000u) + 0x10000)
+	dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
+	dev_priv->fb_size =
+		((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
 		- dev_priv->fb_location;
 
 	dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
@@ -1563,11 +1071,11 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		 * align it down.
 		 */
 #if __OS_HAS_AGP
-		if (dev_priv->flags & CHIP_IS_AGP) {
+		if (dev_priv->flags & RADEON_IS_AGP) {
 			base = dev->agp->base;
 			/* Check if valid */
-			if ((base + dev_priv->gart_size) > dev_priv->fb_location &&
-			    base < (dev_priv->fb_location + dev_priv->fb_size)) {
+			if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
+			    base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
 				DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
 					 dev->agp->base);
 				base = 0;
@@ -1577,11 +1085,11 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		/* If not or if AGP is at 0 (Macs), try to put it elsewhere */
 		if (base == 0) {
 			base = dev_priv->fb_location + dev_priv->fb_size;
-			if (((base + dev_priv->gart_size) & 0xfffffffful)
-			    < base)
+			if (base < dev_priv->fb_location ||
+			    ((base + dev_priv->gart_size) & 0xfffffffful) < base)
 				base = dev_priv->fb_location
 					- dev_priv->gart_size;
-		}		
+		}
 		dev_priv->gart_vm_start = base & 0xffc00000u;
 		if (dev_priv->gart_vm_start != base)
 			DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
@@ -1593,7 +1101,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	}
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP)
+	if (dev_priv->flags & RADEON_IS_AGP)
 		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
 						 - dev->agp->base
 						 + dev_priv->gart_vm_start);
@@ -1614,32 +1122,41 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	dev_priv->ring.size = init->ring_size;
 	dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
 
+	dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
+	dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
+
+	dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
+	dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
+
 	dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
 
 	dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
+	if (dev_priv->flags & RADEON_IS_AGP) {
 		/* Turn off PCI GART */
 		radeon_set_pcigart(dev_priv, 0);
 	} else
 #endif
 	{
+		dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
 		/* if we have an offset set from userspace */
-		if (dev_priv->pcigart_offset) {
+		if (dev_priv->pcigart_offset_set) {
 			dev_priv->gart_info.bus_addr =
 			    dev_priv->pcigart_offset + dev_priv->fb_location;
 			dev_priv->gart_info.mapping.offset =
-			    dev_priv->gart_info.bus_addr;
+			    dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
 			dev_priv->gart_info.mapping.size =
-			    RADEON_PCIGART_TABLE_SIZE;
+			    dev_priv->gart_info.table_size;
 
-			drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
+			drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
 			dev_priv->gart_info.addr =
 			    dev_priv->gart_info.mapping.handle;
 
-			dev_priv->gart_info.is_pcie =
-			    !!(dev_priv->flags & CHIP_IS_PCIE);
+			if (dev_priv->flags & RADEON_IS_PCIE)
+				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
+			else
+				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 			dev_priv->gart_info.gart_table_location =
 			    DRM_ATI_GART_FB;
 
@@ -1647,22 +1164,26 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 				  dev_priv->gart_info.addr,
 				  dev_priv->pcigart_offset);
 		} else {
+			if (dev_priv->flags & RADEON_IS_IGPGART)
+				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
+			else
+				dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
 			dev_priv->gart_info.gart_table_location =
 			    DRM_ATI_GART_MAIN;
 			dev_priv->gart_info.addr = NULL;
 			dev_priv->gart_info.bus_addr = 0;
-			if (dev_priv->flags & CHIP_IS_PCIE) {
+			if (dev_priv->flags & RADEON_IS_PCIE) {
 				DRM_ERROR
 				    ("Cannot use PCI Express without GART in FB memory\n");
 				radeon_do_cleanup_cp(dev);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 		}
 
 		if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 			DRM_ERROR("failed to init PCI GART!\n");
 			radeon_do_cleanup_cp(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 
 		/* Turn on PCI GART */
@@ -1680,7 +1201,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	return 0;
 }
 
-static int radeon_do_cleanup_cp(drm_device_t * dev)
+static int radeon_do_cleanup_cp(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
@@ -1693,7 +1214,7 @@ static int radeon_do_cleanup_cp(drm_device_t * dev)
 		drm_irq_uninstall(dev);
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
+	if (dev_priv->flags & RADEON_IS_AGP) {
 		if (dev_priv->cp_ring != NULL) {
 			drm_core_ioremapfree(dev_priv->cp_ring, dev);
 			dev_priv->cp_ring = NULL;
@@ -1736,19 +1257,19 @@ static int radeon_do_cleanup_cp(drm_device_t * dev)
  *
  * Charl P. Botha <http://cpbotha.net>
  */
-static int radeon_do_resume_cp(drm_device_t * dev)
+static int radeon_do_resume_cp(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 
 	if (!dev_priv) {
 		DRM_ERROR("Called with no initialization\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	DRM_DEBUG("Starting radeon_do_resume_cp()\n");
 
 #if __OS_HAS_AGP
-	if (dev_priv->flags & CHIP_IS_AGP) {
+	if (dev_priv->flags & RADEON_IS_AGP) {
 		/* Turn off PCI GART */
 		radeon_set_pcigart(dev_priv, 0);
 	} else
@@ -1762,52 +1283,48 @@ static int radeon_do_resume_cp(drm_device_t * dev)
 	radeon_cp_init_ring_buffer(dev, dev_priv);
 
 	radeon_do_engine_reset(dev);
+	radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
 
 	DRM_DEBUG("radeon_do_resume_cp() complete\n");
 
 	return 0;
 }
 
-int radeon_cp_init(DRM_IOCTL_ARGS)
+int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_radeon_init_t init;
+	drm_radeon_init_t *init = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_radeon_init_t __user *) data,
-				 sizeof(init));
+	if (init->func == RADEON_INIT_R300_CP)
+		r300_init_reg_flags(dev);
 
-	if (init.func == RADEON_INIT_R300_CP)
-		r300_init_reg_flags();
-
-	switch (init.func) {
+	switch (init->func) {
 	case RADEON_INIT_CP:
 	case RADEON_INIT_R200_CP:
 	case RADEON_INIT_R300_CP:
-		return radeon_do_init_cp(dev, &init);
+		return radeon_do_init_cp(dev, init);
 	case RADEON_CLEANUP_CP:
 		return radeon_do_cleanup_cp(dev);
 	}
 
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
-int radeon_cp_start(DRM_IOCTL_ARGS)
+int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (dev_priv->cp_running) {
-		DRM_DEBUG("%s while CP running\n", __FUNCTION__);
+		DRM_DEBUG("while CP running\n");
 		return 0;
 	}
 	if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
-		DRM_DEBUG("%s called with bogus CP mode (%d)\n",
-			  __FUNCTION__, dev_priv->cp_mode);
+		DRM_DEBUG("called with bogus CP mode (%d)\n",
+			  dev_priv->cp_mode);
 		return 0;
 	}
 
@@ -1819,18 +1336,14 @@ int radeon_cp_start(DRM_IOCTL_ARGS)
 /* Stop the CP.  The engine must have been idled before calling this
  * routine.
  */
-int radeon_cp_stop(DRM_IOCTL_ARGS)
+int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_cp_stop_t stop;
+	drm_radeon_cp_stop_t *stop = data;
 	int ret;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(stop, (drm_radeon_cp_stop_t __user *) data,
-				 sizeof(stop));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv->cp_running)
 		return 0;
@@ -1838,14 +1351,14 @@ int radeon_cp_stop(DRM_IOCTL_ARGS)
 	/* Flush any pending CP commands.  This ensures any outstanding
 	 * commands are exectuted by the engine before we turn it off.
 	 */
-	if (stop.flush) {
+	if (stop->flush) {
 		radeon_do_cp_flush(dev_priv);
 	}
 
 	/* If we fail to make the engine go idle, we return an error
 	 * code so that the DRM ioctl wrapper can try again.
 	 */
-	if (stop.idle) {
+	if (stop->idle) {
 		ret = radeon_do_cp_idle(dev_priv);
 		if (ret)
 			return ret;
@@ -1863,7 +1376,7 @@ int radeon_cp_stop(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-void radeon_do_release(drm_device_t * dev)
+void radeon_do_release(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	int i, ret;
@@ -1877,7 +1390,7 @@ void radeon_do_release(drm_device_t * dev)
 				schedule();
 #else
 #if defined(__FreeBSD__) && __FreeBSD_version > 500000
-				msleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
+				mtx_sleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
 				       1);
 #else
 				tsleep(&ret, PZERO, "rdnrel", 1);
@@ -1913,17 +1426,16 @@ void radeon_do_release(drm_device_t * dev)
 
 /* Just reset the CP ring.  Called as part of an X Server engine reset.
  */
-int radeon_cp_reset(DRM_IOCTL_ARGS)
+int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_DEBUG("%s called before init done\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_DEBUG("called before init done\n");
+		return -EINVAL;
 	}
 
 	radeon_do_cp_reset(dev_priv);
@@ -1934,32 +1446,29 @@ int radeon_cp_reset(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-int radeon_cp_idle(DRM_IOCTL_ARGS)
+int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return radeon_do_cp_idle(dev_priv);
 }
 
 /* Added by Charl P. Botha to call radeon_do_resume_cp().
  */
-int radeon_cp_resume(DRM_IOCTL_ARGS)
+int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 
 	return radeon_do_resume_cp(dev);
 }
 
-int radeon_engine_reset(DRM_IOCTL_ARGS)
+int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	return radeon_do_engine_reset(dev);
 }
@@ -1970,7 +1479,7 @@ int radeon_engine_reset(DRM_IOCTL_ARGS)
 
 /* KW: Deprecated to say the least:
  */
-int radeon_fullscreen(DRM_IOCTL_ARGS)
+int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	return 0;
 }
@@ -1996,12 +1505,12 @@ int radeon_fullscreen(DRM_IOCTL_ARGS)
  * they can't get the lock.
  */
 
-drm_buf_t *radeon_freelist_get(drm_device_t * dev)
+struct drm_buf *radeon_freelist_get(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_buf_priv_t *buf_priv;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	int i, t;
 	int start;
 
@@ -2016,8 +1525,9 @@ drm_buf_t *radeon_freelist_get(drm_device_t * dev)
 		for (i = start; i < dma->buf_count; i++) {
 			buf = dma->buflist[i];
 			buf_priv = buf->dev_private;
-			if (buf->filp == 0 || (buf->pending &&
-					       buf_priv->age <= done_age)) {
+			if (buf->file_priv == NULL || (buf->pending &&
+						       buf_priv->age <=
+						       done_age)) {
 				dev_priv->stats.requested_bufs++;
 				buf->pending = 0;
 				return buf;
@@ -2036,12 +1546,12 @@ drm_buf_t *radeon_freelist_get(drm_device_t * dev)
 }
 
 #if 0
-drm_buf_t *radeon_freelist_get(drm_device_t * dev)
+struct drm_buf *radeon_freelist_get(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_buf_priv_t *buf_priv;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	int i, t;
 	int start;
 	u32 done_age = DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1));
@@ -2056,8 +1566,9 @@ drm_buf_t *radeon_freelist_get(drm_device_t * dev)
 		for (i = start; i < dma->buf_count; i++) {
 			buf = dma->buflist[i];
 			buf_priv = buf->dev_private;
-			if (buf->filp == 0 || (buf->pending &&
-					       buf_priv->age <= done_age)) {
+			if (buf->file_priv == 0 || (buf->pending &&
+						    buf_priv->age <=
+						    done_age)) {
 				dev_priv->stats.requested_bufs++;
 				buf->pending = 0;
 				return buf;
@@ -2070,15 +1581,15 @@ drm_buf_t *radeon_freelist_get(drm_device_t * dev)
 }
 #endif
 
-void radeon_freelist_reset(drm_device_t * dev)
+void radeon_freelist_reset(struct drm_device * dev)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	int i;
 
 	dev_priv->last_buf = 0;
 	for (i = 0; i < dma->buf_count; i++) {
-		drm_buf_t *buf = dma->buflist[i];
+		struct drm_buf *buf = dma->buflist[i];
 		drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 		buf_priv->age = 0;
 	}
@@ -2117,70 +1628,65 @@ int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
 	radeon_status(dev_priv);
 	DRM_ERROR("failed!\n");
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
-static int radeon_cp_get_buffers(DRMFILE filp, drm_device_t * dev,
-				 drm_dma_t * d)
+static int radeon_cp_get_buffers(struct drm_device *dev,
+				 struct drm_file *file_priv,
+				 struct drm_dma * d)
 {
 	int i;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 
 	for (i = d->granted_count; i < d->request_count; i++) {
 		buf = radeon_freelist_get(dev);
 		if (!buf)
-			return DRM_ERR(EBUSY);	/* NOTE: broken client */
+			return -EBUSY;	/* NOTE: broken client */
 
-		buf->filp = filp;
+		buf->file_priv = file_priv;
 
 		if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
 				     sizeof(buf->idx)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
 				     sizeof(buf->total)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 
 		d->granted_count++;
 	}
 	return 0;
 }
 
-int radeon_cp_buffers(DRM_IOCTL_ARGS)
+int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	int ret = 0;
-	drm_dma_t __user *argp = (void __user *)data;
-	drm_dma_t d;
+	struct drm_dma *d = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(d, argp, sizeof(d));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Please don't send us buffers.
 	 */
-	if (d.send_count != 0) {
+	if (d->send_count != 0) {
 		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d.send_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->send_count);
+		return -EINVAL;
 	}
 
 	/* We'll send you buffers.
 	 */
-	if (d.request_count < 0 || d.request_count > dma->buf_count) {
+	if (d->request_count < 0 || d->request_count > dma->buf_count) {
 		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d.request_count, dma->buf_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->request_count, dma->buf_count);
+		return -EINVAL;
 	}
 
-	d.granted_count = 0;
+	d->granted_count = 0;
 
-	if (d.request_count) {
-		ret = radeon_cp_get_buffers(filp, dev, &d);
+	if (d->request_count) {
+		ret = radeon_cp_get_buffers(dev, file_priv, d);
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, d, sizeof(d));
-
 	return ret;
 }
 
@@ -2191,13 +1697,13 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 
 	dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
 	if (dev_priv == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv, 0, sizeof(drm_radeon_private_t));
 	dev->dev_private = (void *)dev_priv;
 	dev_priv->flags = flags;
 
-	switch (flags & CHIP_FAMILY_MASK) {
+	switch (flags & RADEON_FAMILY_MASK) {
 	case CHIP_R100:
 	case CHIP_RV200:
 	case CHIP_R200:
@@ -2205,22 +1711,27 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 	case CHIP_R350:
 	case CHIP_R420:
 	case CHIP_RV410:
-		dev_priv->flags |= CHIP_HAS_HIERZ;
+	case CHIP_RV515:
+	case CHIP_R520:
+	case CHIP_RV570:
+	case CHIP_R580:
+		dev_priv->flags |= RADEON_HAS_HIERZ;
 		break;
 	default:
 		/* all other chips have no hierarchical z buffer */
 		break;
 	}
 
+	dev_priv->chip_family = flags & RADEON_FAMILY_MASK;
 	if (drm_device_is_agp(dev))
-		dev_priv->flags |= CHIP_IS_AGP;
+		dev_priv->flags |= RADEON_IS_AGP;
 	else if (drm_device_is_pcie(dev))
-		dev_priv->flags |= CHIP_IS_PCIE;
+		dev_priv->flags |= RADEON_IS_PCIE;
 	else
-		dev_priv->flags |= CHIP_IS_PCI;
+		dev_priv->flags |= RADEON_IS_PCI;
 
 	DRM_DEBUG("%s card detected\n",
-		  ((dev_priv->flags & CHIP_IS_AGP) ? "AGP" : (((dev_priv->flags & CHIP_IS_PCIE) ? "PCIE" : "PCI"))));
+		  ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
 	return ret;
 }
 
@@ -2233,13 +1744,16 @@ int radeon_driver_firstopen(struct drm_device *dev)
 	drm_local_map_t *map;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 
+	dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
+
 	ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
 			 drm_get_resource_len(dev, 2), _DRM_REGISTERS,
 			 _DRM_READ_ONLY, &dev_priv->mmio);
 	if (ret != 0)
 		return ret;
 
-	ret = drm_addmap(dev, drm_get_resource_start(dev, 0),
+	dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
+	ret = drm_addmap(dev, dev_priv->fb_aper_offset,
 			 drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
 			 _DRM_WRITE_COMBINING, &map);
 	if (ret != 0)
diff --git a/sys/dev/drm/radeon_drm.h b/sys/dev/drm/radeon_drm.h
index b40b43c..acea7d5 100644
--- a/sys/dev/drm/radeon_drm.h
+++ b/sys/dev/drm/radeon_drm.h
@@ -226,11 +226,24 @@ typedef union {
 #define R300_CMD_CP_DELAY		5
 #define R300_CMD_DMA_DISCARD		6
 #define R300_CMD_WAIT			7
-#	define R300_WAIT_2D  		0x1
-#	define R300_WAIT_3D  		0x2
-#	define R300_WAIT_2D_CLEAN  	0x3
-#	define R300_WAIT_3D_CLEAN  	0x4
+#	define R300_WAIT_2D		0x1
+#	define R300_WAIT_3D		0x2
+/* these two defines are DOING IT WRONG - however
+ * we have userspace which relies on using these.
+ * The wait interface is backwards compat new 
+ * code should use the NEW_WAIT defines below
+ * THESE ARE NOT BIT FIELDS
+ */
+#	define R300_WAIT_2D_CLEAN	0x3
+#	define R300_WAIT_3D_CLEAN	0x4
+
+#	define R300_NEW_WAIT_2D_3D	0x3
+#	define R300_NEW_WAIT_2D_2D_CLEAN	0x4
+#	define R300_NEW_WAIT_3D_3D_CLEAN	0x6
+#	define R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN	0x8
+
 #define R300_CMD_SCRATCH		8
+#define R300_CMD_R500FP                 9
 
 typedef union {
 	unsigned int u;
@@ -259,6 +272,9 @@ typedef union {
 	struct {
 		unsigned char cmd_type, reg, n_bufs, flags;
 	} scratch;
+	struct {
+		unsigned char cmd_type, count, adrlo, adrhi_flags;
+	} r500fp;
 } drm_r300_cmd_header_t;
 
 #define RADEON_FRONT			0x1
@@ -269,6 +285,9 @@ typedef union {
 #define RADEON_USE_HIERZ		0x40000000
 #define RADEON_USE_COMP_ZBUF		0x20000000
 
+#define R500FP_CONSTANT_TYPE  (1 << 1)
+#define R500FP_CONSTANT_CLAMP (1 << 2)
+
 /* Primitive types
  */
 #define RADEON_POINTS			0x1
@@ -420,7 +439,7 @@ typedef struct {
 
 	/* The current cliprects, or a subset thereof.
 	 */
-	drm_clip_rect_t boxes[RADEON_NR_SAREA_CLIPRECTS];
+	struct drm_clip_rect boxes[RADEON_NR_SAREA_CLIPRECTS];
 	unsigned int nbox;
 
 	/* Counters for client-side throttling of rendering clients.
@@ -429,7 +448,7 @@ typedef struct {
 	unsigned int last_dispatch;
 	unsigned int last_clear;
 
-	drm_tex_region_t tex_list[RADEON_NR_TEX_HEAPS][RADEON_NR_TEX_REGIONS +
+	struct drm_tex_region tex_list[RADEON_NR_TEX_HEAPS][RADEON_NR_TEX_REGIONS +
 						       1];
 	unsigned int tex_age[RADEON_NR_TEX_HEAPS];
 	int ctx_owner;
@@ -607,7 +626,7 @@ typedef struct drm_radeon_cmd_buffer {
 	int bufsz;
 	char __user *buf;
 	int nbox;
-	drm_clip_rect_t __user *boxes;
+	struct drm_clip_rect __user *boxes;
 } drm_radeon_cmd_buffer_t;
 
 typedef struct drm_radeon_tex_image {
@@ -658,6 +677,9 @@ typedef struct drm_radeon_indirect {
 #define RADEON_PARAM_GART_TEX_HANDLE       10
 #define RADEON_PARAM_SCRATCH_OFFSET        11
 #define RADEON_PARAM_CARD_TYPE             12
+#define RADEON_PARAM_VBLANK_CRTC           13   /* VBLANK CRTC */
+#define RADEON_PARAM_FB_LOCATION           14   /* FB location */
+#define RADEON_PARAM_NUM_GB_PIPES          15   /* num GB pipes */
 
 typedef struct drm_radeon_getparam {
 	int param;
@@ -711,7 +733,8 @@ typedef struct drm_radeon_setparam {
 #define RADEON_SETPARAM_PCIGART_LOCATION 3	/* PCI Gart Location */
 
 #define RADEON_SETPARAM_NEW_MEMMAP 4		/* Use new memory map */
-
+#define RADEON_SETPARAM_PCIGART_TABLE_SIZE 5    /* PCI GART Table Size */
+#define RADEON_SETPARAM_VBLANK_CRTC 6           /* VBLANK CRTC */
 /* 1.14: Clients can allocate/free a surface
  */
 typedef struct drm_radeon_surface_alloc {
@@ -724,5 +747,7 @@ typedef struct drm_radeon_surface_free {
 	unsigned int address;
 } drm_radeon_surface_free_t;
 
+#define	DRM_RADEON_VBLANK_CRTC1		1
+#define	DRM_RADEON_VBLANK_CRTC2		2
 
 #endif
diff --git a/sys/dev/drm/radeon_drv.c b/sys/dev/drm/radeon_drv.c
index af12d3d..96107e5 100644
--- a/sys/dev/drm/radeon_drv.c
+++ b/sys/dev/drm/radeon_drv.c
@@ -45,7 +45,7 @@ static drm_pci_id_list_t radeon_pciidlist[] = {
 	radeon_PCI_IDS
 };
 
-static void radeon_configure(drm_device_t *dev)
+static void radeon_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= sizeof(drm_radeon_buf_priv_t);
 	dev->driver.load		= radeon_driver_load;
@@ -55,7 +55,9 @@ static void radeon_configure(drm_device_t *dev)
 	dev->driver.preclose		= radeon_driver_preclose;
 	dev->driver.postclose		= radeon_driver_postclose;
 	dev->driver.lastclose		= radeon_driver_lastclose;
-	dev->driver.vblank_wait		= radeon_driver_vblank_wait;
+	dev->driver.get_vblank_counter	= radeon_get_vblank_counter;
+	dev->driver.enable_vblank	= radeon_enable_vblank;
+	dev->driver.disable_vblank	= radeon_disable_vblank;
 	dev->driver.irq_preinstall	= radeon_driver_irq_preinstall;
 	dev->driver.irq_postinstall	= radeon_driver_irq_postinstall;
 	dev->driver.irq_uninstall	= radeon_driver_irq_uninstall;
@@ -79,6 +81,7 @@ static void radeon_configure(drm_device_t *dev)
 	dev->driver.use_dma		= 1;
 	dev->driver.use_irq		= 1;
 	dev->driver.use_vbl_irq		= 1;
+	dev->driver.use_vbl_irq2	= 1;
 }
 
 #ifdef __FreeBSD__
@@ -91,9 +94,9 @@ radeon_probe(device_t dev)
 static int
 radeon_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	radeon_configure(dev);
 	return drm_attach(nbdev, radeon_pciidlist);
 }
@@ -110,7 +113,7 @@ static device_method_t radeon_methods[] = {
 static driver_t radeon_driver = {
 	"drm",
 	radeon_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
@@ -125,7 +128,7 @@ MODULE_DEPEND(radeon, drm, 1, 1, 1);
 #ifdef _LKM
 CFDRIVER_DECL(radeon, DV_TTY, NULL);
 #else
-CFATTACH_DECL(radeon, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
-    drm_activate);
+CFATTACH_DECL(radeon, sizeof(struct drm_device), drm_probe, drm_attach,
+    drm_detach, drm_activate);
 #endif
 #endif /* __FreeBSD__ */
diff --git a/sys/dev/drm/radeon_drv.h b/sys/dev/drm/radeon_drv.h
index 2183279..7d40a6f 100644
--- a/sys/dev/drm/radeon_drv.h
+++ b/sys/dev/drm/radeon_drv.h
@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
 
 #define DRIVER_NAME		"radeon"
 #define DRIVER_DESC		"ATI Radeon"
-#define DRIVER_DATE		"20060524"
+#define DRIVER_DATE		"20080613"
 
 /* Interface history:
  *
@@ -98,10 +98,14 @@ __FBSDID("$FreeBSD$");
  * 1.24- Add general-purpose packet for manipulating scratch registers (r300)
  * 1.25- Add support for r200 vertex programs (R200_EMIT_VAP_PVS_CNTL,
  *       new packet type)
+ * 1.26- Add support for variable size PCI(E) gart aperture
+ * 1.27- Add support for IGP GART
+ * 1.28- Add support for VBL on CRTC2
+ * 1.29- R500 3D cmd buffer support
  */
 
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		25
+#define DRIVER_MINOR		29
 #define DRIVER_PATCHLEVEL	0
 
 /*
@@ -124,29 +128,32 @@ enum radeon_family {
 	CHIP_R420,
 	CHIP_RV410,
 	CHIP_RS400,
+	CHIP_RS480,
+	CHIP_RS690,
+	CHIP_RV515,
+	CHIP_R520,
+	CHIP_RV530,
+	CHIP_RV560,
+	CHIP_RV570,
+	CHIP_R580,
 	CHIP_LAST,
 };
 
-enum radeon_cp_microcode_version {
-	UCODE_R100,
-	UCODE_R200,
-	UCODE_R300,
-};
-
 /*
  * Chip flags
  */
 enum radeon_chip_flags {
-	CHIP_FAMILY_MASK = 0x0000ffffUL,
-	CHIP_FLAGS_MASK = 0xffff0000UL,
-	CHIP_IS_MOBILITY = 0x00010000UL,
-	CHIP_IS_IGP = 0x00020000UL,
-	CHIP_SINGLE_CRTC = 0x00040000UL,
-	CHIP_IS_AGP = 0x00080000UL,
-	CHIP_HAS_HIERZ = 0x00100000UL,
-	CHIP_IS_PCIE = 0x00200000UL,
-	CHIP_NEW_MEMMAP = 0x00400000UL,
-	CHIP_IS_PCI = 0x00800000UL,
+	RADEON_FAMILY_MASK = 0x0000ffffUL,
+	RADEON_FLAGS_MASK = 0xffff0000UL,
+	RADEON_IS_MOBILITY = 0x00010000UL,
+	RADEON_IS_IGP = 0x00020000UL,
+	RADEON_SINGLE_CRTC = 0x00040000UL,
+	RADEON_IS_AGP = 0x00080000UL,
+	RADEON_HAS_HIERZ = 0x00100000UL,
+	RADEON_IS_PCIE = 0x00200000UL,
+	RADEON_NEW_MEMMAP = 0x00400000UL,
+	RADEON_IS_PCI = 0x00800000UL,
+	RADEON_IS_IGPGART = 0x01000000UL,
 };
 
 #define GET_RING_HEAD(dev_priv)	(dev_priv->writeback_works ? \
@@ -155,7 +162,7 @@ enum radeon_chip_flags {
 
 typedef struct drm_radeon_freelist {
 	unsigned int age;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	struct drm_radeon_freelist *next;
 	struct drm_radeon_freelist *prev;
 } drm_radeon_freelist_t;
@@ -163,8 +170,14 @@ typedef struct drm_radeon_freelist {
 typedef struct drm_radeon_ring_buffer {
 	u32 *start;
 	u32 *end;
-	int size;
-	int size_l2qw;
+	int size; /* Double Words */
+	int size_l2qw; /* log2 Quad Words */
+
+	int rptr_update; /* Double Words */
+	int rptr_update_l2qw; /* log2 Quad Words */
+
+	int fetch_size; /* Double Words */
+	int fetch_size_l2ow; /* log2 Oct Words */
 
 	u32 tail;
 	u32 tail_mask;
@@ -188,7 +201,7 @@ struct mem_block {
 	struct mem_block *prev;
 	int start;
 	int size;
-	DRMFILE filp;		/* 0: free, -1: heap, other: real files */
+	struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */
 };
 
 struct radeon_surface {
@@ -203,9 +216,12 @@ struct radeon_virt_surface {
 	u32 lower;
 	u32 upper;
 	u32 flags;
-	DRMFILE filp;
+	struct drm_file *file_priv;
 };
 
+#define RADEON_FLUSH_EMITED	(1 < 0)
+#define RADEON_PURGE_EMITED	(1 < 1)
+
 typedef struct drm_radeon_private {
 
 	drm_radeon_ring_buffer_t ring;
@@ -230,8 +246,6 @@ typedef struct drm_radeon_private {
 
 	int usec_timeout;
 
-	int microcode_version;
-
 	struct {
 		u32 boxes;
 		int freelist_timeouts;
@@ -245,7 +259,6 @@ typedef struct drm_radeon_private {
 
 	int do_boxes;
 	int page_flipping;
-	int current_page;
 
 	u32 color_fmt;
 	unsigned int front_offset;
@@ -280,18 +293,30 @@ typedef struct drm_radeon_private {
 	/* SW interrupt */
 	wait_queue_head_t swi_queue;
 	atomic_t swi_emitted;
+	int vblank_crtc;
+	uint32_t irq_enable_reg;
+	int irq_enabled;
+	uint32_t r500_disp_irq_reg;
 
 	struct radeon_surface surfaces[RADEON_MAX_SURFACES];
-	struct radeon_virt_surface virt_surfaces[2*RADEON_MAX_SURFACES];
+	struct radeon_virt_surface virt_surfaces[2 * RADEON_MAX_SURFACES];
 
 	unsigned long pcigart_offset;
-	drm_ati_pcigart_info gart_info;
+	unsigned int pcigart_offset_set;
+	struct drm_ati_pcigart_info gart_info;
 
 	u32 scratch_ages[5];
 
+	unsigned int crtc_last_cnt;
+	unsigned int crtc2_last_cnt;
+
 	/* starting from here on, data is preserved accross an open */
 	uint32_t flags;		/* see radeon_chip_flags */
+	unsigned long fb_aper_offset;
 
+	int num_gb_pipes;
+	int track_flush;
+	uint32_t chip_family; /* extract from flags */
 } drm_radeon_private_t;
 
 typedef struct drm_radeon_buf_priv {
@@ -302,65 +327,89 @@ typedef struct drm_radeon_kcmd_buffer {
 	int bufsz;
 	char *buf;
 	int nbox;
-	drm_clip_rect_t __user *boxes;
+	struct drm_clip_rect __user *boxes;
 } drm_radeon_kcmd_buffer_t;
 
 extern int radeon_no_wb;
-extern drm_ioctl_desc_t radeon_ioctls[];
+extern struct drm_ioctl_desc radeon_ioctls[];
 extern int radeon_max_ioctl;
 
+/* Check whether the given hardware address is inside the framebuffer or the
+ * GART area.
+ */
+static __inline__ int radeon_check_offset(drm_radeon_private_t *dev_priv,
+					  u64 off)
+{
+	u32 fb_start = dev_priv->fb_location;
+	u32 fb_end = fb_start + dev_priv->fb_size - 1;
+	u32 gart_start = dev_priv->gart_vm_start;
+	u32 gart_end = gart_start + dev_priv->gart_size - 1;
+
+	return ((off >= fb_start && off <= fb_end) ||
+		(off >= gart_start && off <= gart_end));
+}
+
 				/* radeon_cp.c */
-extern int radeon_cp_init(DRM_IOCTL_ARGS);
-extern int radeon_cp_start(DRM_IOCTL_ARGS);
-extern int radeon_cp_stop(DRM_IOCTL_ARGS);
-extern int radeon_cp_reset(DRM_IOCTL_ARGS);
-extern int radeon_cp_idle(DRM_IOCTL_ARGS);
-extern int radeon_cp_resume(DRM_IOCTL_ARGS);
-extern int radeon_engine_reset(DRM_IOCTL_ARGS);
-extern int radeon_fullscreen(DRM_IOCTL_ARGS);
-extern int radeon_cp_buffers(DRM_IOCTL_ARGS);
-
-extern void radeon_freelist_reset(drm_device_t * dev);
-extern drm_buf_t *radeon_freelist_get(drm_device_t * dev);
+extern int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv);
+
+extern void radeon_freelist_reset(struct drm_device * dev);
+extern struct drm_buf *radeon_freelist_get(struct drm_device * dev);
 
 extern int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n);
 
 extern int radeon_do_cp_idle(drm_radeon_private_t * dev_priv);
 
-extern int radeon_mem_alloc(DRM_IOCTL_ARGS);
-extern int radeon_mem_free(DRM_IOCTL_ARGS);
-extern int radeon_mem_init_heap(DRM_IOCTL_ARGS);
+extern int radeon_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_mem_init_heap(struct drm_device *dev, void *data, struct drm_file *file_priv);
 extern void radeon_mem_takedown(struct mem_block **heap);
-extern void radeon_mem_release(DRMFILE filp, struct mem_block *heap);
+extern void radeon_mem_release(struct drm_file *file_priv,
+			       struct mem_block *heap);
 
 				/* radeon_irq.c */
-extern int radeon_irq_emit(DRM_IOCTL_ARGS);
-extern int radeon_irq_wait(DRM_IOCTL_ARGS);
-
-extern void radeon_do_release(drm_device_t * dev);
-extern int radeon_driver_vblank_wait(drm_device_t * dev,
-				     unsigned int *sequence);
+extern void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state);
+extern int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv);
+
+extern void radeon_do_release(struct drm_device * dev);
+extern u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc);
+extern int radeon_enable_vblank(struct drm_device *dev, int crtc);
+extern void radeon_disable_vblank(struct drm_device *dev, int crtc);
 extern irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS);
-extern void radeon_driver_irq_preinstall(drm_device_t * dev);
-extern void radeon_driver_irq_postinstall(drm_device_t * dev);
-extern void radeon_driver_irq_uninstall(drm_device_t * dev);
+extern void radeon_driver_irq_preinstall(struct drm_device * dev);
+extern int radeon_driver_irq_postinstall(struct drm_device * dev);
+extern void radeon_driver_irq_uninstall(struct drm_device * dev);
+extern int radeon_vblank_crtc_get(struct drm_device *dev);
+extern int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value);
 
 extern int radeon_driver_load(struct drm_device *dev, unsigned long flags);
 extern int radeon_driver_unload(struct drm_device *dev);
 extern int radeon_driver_firstopen(struct drm_device *dev);
-extern void radeon_driver_preclose(drm_device_t * dev, DRMFILE filp);
-extern void radeon_driver_postclose(drm_device_t * dev, drm_file_t * filp);
-extern void radeon_driver_lastclose(drm_device_t * dev);
-extern int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv);
+extern void radeon_driver_preclose(struct drm_device * dev,
+				   struct drm_file *file_priv);
+extern void radeon_driver_postclose(struct drm_device * dev,
+				    struct drm_file *file_priv);
+extern void radeon_driver_lastclose(struct drm_device * dev);
+extern int radeon_driver_open(struct drm_device * dev,
+			      struct drm_file * file_priv);
 extern long radeon_compat_ioctl(struct file *filp, unsigned int cmd,
 					 unsigned long arg);
 
 /* r300_cmdbuf.c */
-extern void r300_init_reg_flags(void);
+extern void r300_init_reg_flags(struct drm_device *dev);
 
-extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
-			     drm_file_t* filp_priv,
-			     drm_radeon_kcmd_buffer_t* cmdbuf);
+extern int r300_do_cp_cmdbuf(struct drm_device *dev,
+			     struct drm_file *file_priv,
+			     drm_radeon_kcmd_buffer_t *cmdbuf);
 
 /* Flags for stats.boxes
  */
@@ -402,30 +451,122 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #define RADEON_PCIE_INDEX               0x0030
 #define RADEON_PCIE_DATA                0x0034
 #define RADEON_PCIE_TX_GART_CNTL	0x10
-#	define RADEON_PCIE_TX_GART_EN   	(1 << 0)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_PASS_THRU (0<<1)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_CLAMP_LO  (1<<1)
-#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD   (3<<1)
-#	define RADEON_PCIE_TX_GART_MODE_32_128_CACHE	(0<<3)
-#	define RADEON_PCIE_TX_GART_MODE_8_4_128_CACHE	(1<<3)
-#	define RADEON_PCIE_TX_GART_CHK_RW_VALID_EN      (1<<5)
-#	define RADEON_PCIE_TX_GART_INVALIDATE_TLB	(1<<8)
+#	define RADEON_PCIE_TX_GART_EN		(1 << 0)
+#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_PASS_THRU (0 << 1)
+#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_CLAMP_LO  (1 << 1)
+#	define RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD   (3 << 1)
+#	define RADEON_PCIE_TX_GART_MODE_32_128_CACHE	(0 << 3)
+#	define RADEON_PCIE_TX_GART_MODE_8_4_128_CACHE	(1 << 3)
+#	define RADEON_PCIE_TX_GART_CHK_RW_VALID_EN      (1 << 5)
+#	define RADEON_PCIE_TX_GART_INVALIDATE_TLB	(1 << 8)
 #define RADEON_PCIE_TX_DISCARD_RD_ADDR_LO 0x11
 #define RADEON_PCIE_TX_DISCARD_RD_ADDR_HI 0x12
-#define RADEON_PCIE_TX_GART_BASE  	0x13
+#define RADEON_PCIE_TX_GART_BASE	0x13
 #define RADEON_PCIE_TX_GART_START_LO	0x14
 #define RADEON_PCIE_TX_GART_START_HI	0x15
 #define RADEON_PCIE_TX_GART_END_LO	0x16
 #define RADEON_PCIE_TX_GART_END_HI	0x17
 
+#define RS480_NB_MC_INDEX               0x168
+#	define RS480_NB_MC_IND_WR_EN	(1 << 8)
+#define RS480_NB_MC_DATA                0x16c
+
+#define RS690_MC_INDEX                  0x78
+#   define RS690_MC_INDEX_MASK          0x1ff
+#   define RS690_MC_INDEX_WR_EN         (1 << 9)
+#   define RS690_MC_INDEX_WR_ACK        0x7f
+#define RS690_MC_DATA                   0x7c
+
+/* MC indirect registers */
+#define RS480_MC_MISC_CNTL              0x18
+#	define RS480_DISABLE_GTW	(1 << 1)
+/* switch between MCIND GART and MM GART registers. 0 = mmgart, 1 = mcind gart */
+#	define RS480_GART_INDEX_REG_EN	(1 << 12)
+#	define RS690_BLOCK_GFX_D3_EN	(1 << 14)
+#define RS480_K8_FB_LOCATION            0x1e
+#define RS480_GART_FEATURE_ID           0x2b
+#	define RS480_HANG_EN	        (1 << 11)
+#	define RS480_TLB_ENABLE	        (1 << 18)
+#	define RS480_P2P_ENABLE	        (1 << 19)
+#	define RS480_GTW_LAC_EN	        (1 << 25)
+#	define RS480_2LEVEL_GART	(0 << 30)
+#	define RS480_1LEVEL_GART	(1 << 30)
+#	define RS480_PDC_EN	        (1 << 31)
+#define RS480_GART_BASE                 0x2c
+#define RS480_GART_CACHE_CNTRL          0x2e
+#	define RS480_GART_CACHE_INVALIDATE (1 << 0) /* wait for it to clear */
+#define RS480_AGP_ADDRESS_SPACE_SIZE    0x38
+#	define RS480_GART_EN	        (1 << 0)
+#	define RS480_VA_SIZE_32MB	(0 << 1)
+#	define RS480_VA_SIZE_64MB	(1 << 1)
+#	define RS480_VA_SIZE_128MB	(2 << 1)
+#	define RS480_VA_SIZE_256MB	(3 << 1)
+#	define RS480_VA_SIZE_512MB	(4 << 1)
+#	define RS480_VA_SIZE_1GB	(5 << 1)
+#	define RS480_VA_SIZE_2GB	(6 << 1)
+#define RS480_AGP_MODE_CNTL             0x39
+#	define RS480_POST_GART_Q_SIZE	(1 << 18)
+#	define RS480_NONGART_SNOOP	(1 << 19)
+#	define RS480_AGP_RD_BUF_SIZE	(1 << 20)
+#	define RS480_REQ_TYPE_SNOOP_SHIFT 22
+#	define RS480_REQ_TYPE_SNOOP_MASK  0x3
+#	define RS480_REQ_TYPE_SNOOP_DIS	(1 << 24)
+#define RS480_MC_MISC_UMA_CNTL          0x5f
+#define RS480_MC_MCLK_CNTL              0x7a
+#define RS480_MC_UMA_DUALCH_CNTL        0x86
+
+#define RS690_MC_FB_LOCATION            0x100
+#define RS690_MC_AGP_LOCATION           0x101
+#define RS690_MC_AGP_BASE               0x102
+#define RS690_MC_AGP_BASE_2             0x103
+
+#define R520_MC_IND_INDEX 0x70
+#define R520_MC_IND_WR_EN (1 << 24)
+#define R520_MC_IND_DATA  0x74
+
+#define RV515_MC_FB_LOCATION 0x01
+#define RV515_MC_AGP_LOCATION 0x02
+#define RV515_MC_AGP_BASE     0x03
+#define RV515_MC_AGP_BASE_2   0x04
+
+#define R520_MC_FB_LOCATION 0x04
+#define R520_MC_AGP_LOCATION 0x05
+#define R520_MC_AGP_BASE     0x06
+#define R520_MC_AGP_BASE_2   0x07
+
 #define RADEON_MPP_TB_CONFIG		0x01c0
 #define RADEON_MEM_CNTL			0x0140
 #define RADEON_MEM_SDRAM_MODE_REG	0x0158
+#define RADEON_AGP_BASE_2		0x015c /* r200+ only */
+#define RS480_AGP_BASE_2		0x0164
 #define RADEON_AGP_BASE			0x0170
 
+/* pipe config regs */
+#define R400_GB_PIPE_SELECT             0x402c
+#define R500_DYN_SCLK_PWMEM_PIPE        0x000d /* PLL */
+#define R500_SU_REG_DEST                0x42c8
+#define R300_GB_TILE_CONFIG             0x4018
+#       define R300_ENABLE_TILING       (1 << 0)
+#       define R300_PIPE_COUNT_RV350    (0 << 1)
+#       define R300_PIPE_COUNT_R300     (3 << 1)
+#       define R300_PIPE_COUNT_R420_3P  (6 << 1)
+#       define R300_PIPE_COUNT_R420     (7 << 1)
+#       define R300_TILE_SIZE_8         (0 << 4)
+#       define R300_TILE_SIZE_16        (1 << 4)
+#       define R300_TILE_SIZE_32        (2 << 4)
+#       define R300_SUBPIXEL_1_12       (0 << 16)
+#       define R300_SUBPIXEL_1_16       (1 << 16)
+#define R300_DST_PIPE_CONFIG            0x170c
+#       define R300_PIPE_AUTO_CONFIG    (1 << 31)
+#define R300_RB2D_DSTCACHE_MODE         0x3428
+#       define R300_DC_AUTOFLUSH_ENABLE (1 << 8)
+#       define R300_DC_DC_DISABLE_IGNORE_PE (1 << 17)
+
 #define RADEON_RB3D_COLOROFFSET		0x1c40
 #define RADEON_RB3D_COLORPITCH		0x1c48
 
+#define	RADEON_SRC_X_Y			0x1590
+
 #define RADEON_DP_GUI_MASTER_CNTL	0x146c
 #	define RADEON_GMC_SRC_PITCH_OFFSET_CNTL	(1 << 0)
 #	define RADEON_GMC_DST_PITCH_OFFSET_CNTL	(1 << 1)
@@ -443,6 +584,7 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #	define RADEON_ROP3_S			0x00cc0000
 #	define RADEON_ROP3_P			0x00f00000
 #define RADEON_DP_WRITE_MASK		0x16cc
+#define RADEON_SRC_PITCH_OFFSET		0x1428
 #define RADEON_DST_PITCH_OFFSET		0x142c
 #define RADEON_DST_PITCH_OFFSET_C	0x1c80
 #	define RADEON_DST_TILE_LINEAR		(0 << 30)
@@ -465,18 +607,29 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 				? DRM_READ32( dev_priv->ring_rptr, RADEON_SCRATCHOFF(x) ) \
 				: RADEON_READ( RADEON_SCRATCH_REG0 + 4*(x) ) )
 
+#define RADEON_CRTC_CRNT_FRAME 0x0214
+#define RADEON_CRTC2_CRNT_FRAME 0x0314
+
+#define RADEON_CRTC_STATUS		0x005c
+#define RADEON_CRTC2_STATUS		0x03fc
+
 #define RADEON_GEN_INT_CNTL		0x0040
 #	define RADEON_CRTC_VBLANK_MASK		(1 << 0)
+#	define RADEON_CRTC2_VBLANK_MASK		(1 << 9)
 #	define RADEON_GUI_IDLE_INT_ENABLE	(1 << 19)
 #	define RADEON_SW_INT_ENABLE		(1 << 25)
 
 #define RADEON_GEN_INT_STATUS		0x0044
 #	define RADEON_CRTC_VBLANK_STAT		(1 << 0)
-#	define RADEON_CRTC_VBLANK_STAT_ACK   	(1 << 0)
+#	define RADEON_CRTC_VBLANK_STAT_ACK	(1 << 0)
+#	define RADEON_CRTC2_VBLANK_STAT		(1 << 9)
+#	define RADEON_CRTC2_VBLANK_STAT_ACK	(1 << 9)
 #	define RADEON_GUI_IDLE_INT_TEST_ACK     (1 << 19)
 #	define RADEON_SW_INT_TEST		(1 << 25)
-#	define RADEON_SW_INT_TEST_ACK   	(1 << 25)
+#	define RADEON_SW_INT_TEST_ACK		(1 << 25)
 #	define RADEON_SW_INT_FIRE		(1 << 26)
+#       define R500_DISPLAY_INT_STATUS          (1 << 0)
+
 
 #define RADEON_HOST_PATH_CNTL		0x0130
 #	define RADEON_HDP_SOFT_RESET		(1 << 26)
@@ -520,11 +673,12 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #define RADEON_PP_TXFILTER_1		0x1c6c
 #define RADEON_PP_TXFILTER_2		0x1c84
 
-#define RADEON_RB2D_DSTCACHE_CTLSTAT	0x342c
-#	define RADEON_RB2D_DC_FLUSH		(3 << 0)
-#	define RADEON_RB2D_DC_FREE		(3 << 2)
-#	define RADEON_RB2D_DC_FLUSH_ALL		0xf
-#	define RADEON_RB2D_DC_BUSY		(1 << 31)
+#define R300_RB2D_DSTCACHE_CTLSTAT	0x342c /* use R300_DSTCACHE_CTLSTAT */
+#define R300_DSTCACHE_CTLSTAT		0x1714
+#	define R300_RB2D_DC_FLUSH		(3 << 0)
+#	define R300_RB2D_DC_FREE		(3 << 2)
+#	define R300_RB2D_DC_FLUSH_ALL		0xf
+#	define R300_RB2D_DC_BUSY		(1 << 31)
 #define RADEON_RB3D_CNTL		0x1c3c
 #	define RADEON_ALPHA_BLEND_ENABLE	(1 << 0)
 #	define RADEON_PLANE_MASK_ENABLE		(1 << 1)
@@ -547,11 +701,19 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #	define RADEON_RB3D_ZC_FREE		(1 << 2)
 #	define RADEON_RB3D_ZC_FLUSH_ALL		0x5
 #	define RADEON_RB3D_ZC_BUSY		(1 << 31)
+#define R300_ZB_ZCACHE_CTLSTAT                  0x4f18
+#	define R300_ZC_FLUSH		        (1 << 0)
+#	define R300_ZC_FREE		        (1 << 1)
+#	define R300_ZC_BUSY		        (1 << 31)
 #define RADEON_RB3D_DSTCACHE_CTLSTAT            0x325c
 #	define RADEON_RB3D_DC_FLUSH		(3 << 0)
 #	define RADEON_RB3D_DC_FREE		(3 << 2)
 #	define RADEON_RB3D_DC_FLUSH_ALL		0xf
 #	define RADEON_RB3D_DC_BUSY		(1 << 31)
+#define R300_RB3D_DSTCACHE_CTLSTAT              0x4e4c
+#	define R300_RB3D_DC_FLUSH		(2 << 0)
+#	define R300_RB3D_DC_FREE		(2 << 2)
+#	define R300_RB3D_DC_FINISH		(1 << 4)
 #define RADEON_RB3D_ZSTENCILCNTL	0x1c2c
 #	define RADEON_Z_TEST_MASK		(7 << 4)
 #	define RADEON_Z_TEST_ALWAYS		(7 << 4)
@@ -573,9 +735,51 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #	define RADEON_SOFT_RESET_E2		(1 <<  5)
 #	define RADEON_SOFT_RESET_RB		(1 <<  6)
 #	define RADEON_SOFT_RESET_HDP		(1 <<  7)
+/*
+ *   6:0  Available slots in the FIFO
+ *   8    Host Interface active
+ *   9    CP request active
+ *   10   FIFO request active
+ *   11   Host Interface retry active
+ *   12   CP retry active
+ *   13   FIFO retry active
+ *   14   FIFO pipeline busy
+ *   15   Event engine busy
+ *   16   CP command stream busy
+ *   17   2D engine busy
+ *   18   2D portion of render backend busy
+ *   20   3D setup engine busy
+ *   26   GA engine busy
+ *   27   CBA 2D engine busy
+ *   31   2D engine busy or 3D engine busy or FIFO not empty or CP busy or
+ *           command stream queue not empty or Ring Buffer not empty
+ */
 #define RADEON_RBBM_STATUS		0x0e40
+/* Same as the previous RADEON_RBBM_STATUS; this is a mirror of that register.  */
+/* #define RADEON_RBBM_STATUS		0x1740 */
+/* bits 6:0 are dword slots available in the cmd fifo */
 #	define RADEON_RBBM_FIFOCNT_MASK		0x007f
-#	define RADEON_RBBM_ACTIVE		(1 << 31)
+#	define RADEON_HIRQ_ON_RBB	(1 <<  8)
+#	define RADEON_CPRQ_ON_RBB	(1 <<  9)
+#	define RADEON_CFRQ_ON_RBB	(1 << 10)
+#	define RADEON_HIRQ_IN_RTBUF	(1 << 11)
+#	define RADEON_CPRQ_IN_RTBUF	(1 << 12)
+#	define RADEON_CFRQ_IN_RTBUF	(1 << 13)
+#	define RADEON_PIPE_BUSY		(1 << 14)
+#	define RADEON_ENG_EV_BUSY	(1 << 15)
+#	define RADEON_CP_CMDSTRM_BUSY	(1 << 16)
+#	define RADEON_E2_BUSY		(1 << 17)
+#	define RADEON_RB2D_BUSY		(1 << 18)
+#	define RADEON_RB3D_BUSY		(1 << 19) /* not used on r300 */
+#	define RADEON_VAP_BUSY		(1 << 20)
+#	define RADEON_RE_BUSY		(1 << 21) /* not used on r300 */
+#	define RADEON_TAM_BUSY		(1 << 22) /* not used on r300 */
+#	define RADEON_TDM_BUSY		(1 << 23) /* not used on r300 */
+#	define RADEON_PB_BUSY		(1 << 24) /* not used on r300 */
+#	define RADEON_TIM_BUSY		(1 << 25) /* not used on r300 */
+#	define RADEON_GA_BUSY		(1 << 26)
+#	define RADEON_CBA2D_BUSY	(1 << 27)
+#	define RADEON_RBBM_ACTIVE	(1 << 31)
 #define RADEON_RE_LINE_PATTERN		0x1cd0
 #define RADEON_RE_MISC			0x26c4
 #define RADEON_RE_TOP_LEFT		0x26c0
@@ -947,7 +1151,30 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 #define RADEON_VHA_BACKFRAME0_OFF_V_2            0x1894
 #define RADEON_VHA_BACKFRAME1_OFF_PITCH_V_2      0x1898
 
+#define R500_D1CRTC_STATUS 0x609c
+#define R500_D2CRTC_STATUS 0x689c
+#define R500_CRTC_V_BLANK (1<<0)
+
+#define R500_D1CRTC_FRAME_COUNT 0x60a4
+#define R500_D2CRTC_FRAME_COUNT 0x68a4
+
+#define R500_D1MODE_V_COUNTER 0x6530
+#define R500_D2MODE_V_COUNTER 0x6d30
 
+#define R500_D1MODE_VBLANK_STATUS 0x6534
+#define R500_D2MODE_VBLANK_STATUS 0x6d34
+#define R500_VBLANK_OCCURED (1<<0)
+#define R500_VBLANK_ACK     (1<<4)
+#define R500_VBLANK_STAT    (1<<12)
+#define R500_VBLANK_INT     (1<<16)
+
+#define R500_DxMODE_INT_MASK 0x6540
+#define R500_D1MODE_INT_MASK (1<<0)
+#define R500_D2MODE_INT_MASK (1<<8)
+
+#define R500_DISP_INTERRUPT_STATUS 0x7edc
+#define R500_D1_VBLANK_INTERRUPT (1 << 4)
+#define R500_D2_VBLANK_INTERRUPT (1 << 5)
 
 /* Constants */
 #define RADEON_MAX_USEC_TIMEOUT		100000	/* 100 ms */
@@ -965,8 +1192,8 @@ extern int r300_do_cp_cmdbuf(drm_device_t *dev, DRMFILE filp,
 
 #define RADEON_PCIGART_TABLE_SIZE      (32*1024)
 
-#define RADEON_READ(reg)	DRM_READ32(  dev_priv->mmio, (reg) )
-#define RADEON_WRITE(reg,val)	DRM_WRITE32( dev_priv->mmio, (reg), (val) )
+#define RADEON_READ(reg)    DRM_READ32(  dev_priv->mmio, (reg) )
+#define RADEON_WRITE(reg,val)  DRM_WRITE32( dev_priv->mmio, (reg), (val) )
 #define RADEON_READ8(reg)	DRM_READ8(  dev_priv->mmio, (reg) )
 #define RADEON_WRITE8(reg,val)	DRM_WRITE8( dev_priv->mmio, (reg), (val) )
 
@@ -984,6 +1211,36 @@ do {									\
 	RADEON_WRITE( RADEON_PCIE_DATA, (val) );			\
 } while (0)
 
+#define R500_WRITE_MCIND( addr, val )					\
+do {								\
+	RADEON_WRITE(R520_MC_IND_INDEX, 0xff0000 | ((addr) & 0xff));	\
+	RADEON_WRITE(R520_MC_IND_DATA, (val));			\
+	RADEON_WRITE(R520_MC_IND_INDEX, 0);	\
+} while (0)
+
+#define RS480_WRITE_MCIND( addr, val )				\
+do {									\
+	RADEON_WRITE( RS480_NB_MC_INDEX,				\
+			((addr) & 0xff) | RS480_NB_MC_IND_WR_EN);	\
+	RADEON_WRITE( RS480_NB_MC_DATA, (val) );			\
+	RADEON_WRITE( RS480_NB_MC_INDEX, 0xff );			\
+} while (0)
+
+#define RS690_WRITE_MCIND( addr, val )					\
+do {								\
+	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_EN | ((addr) & RS690_MC_INDEX_MASK));	\
+	RADEON_WRITE(RS690_MC_DATA, val);			\
+	RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_WR_ACK);	\
+} while (0)
+
+#define IGP_WRITE_MCIND( addr, val )				\
+do {									\
+        if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690)       \
+	        RS690_WRITE_MCIND( addr, val );                         \
+	else                                                            \
+	        RS480_WRITE_MCIND( addr, val );                         \
+} while (0)
+
 #define CP_PACKET0( reg, n )						\
 	(RADEON_CP_PACKET0 | ((n) << 16) | ((reg) >> 2))
 #define CP_PACKET0_TABLE( reg, n )					\
@@ -1024,23 +1281,43 @@ do {									\
 } while (0)
 
 #define RADEON_FLUSH_CACHE() do {					\
-	OUT_RING( CP_PACKET0( RADEON_RB3D_DSTCACHE_CTLSTAT, 0 ) );	\
-	OUT_RING( RADEON_RB3D_DC_FLUSH );				\
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
+	        OUT_RING(CP_PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0));	\
+	        OUT_RING(RADEON_RB3D_DC_FLUSH);				\
+	} else {                                                        \
+	        OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));	\
+	        OUT_RING(R300_RB3D_DC_FLUSH);				\
+        }                                                               \
 } while (0)
 
 #define RADEON_PURGE_CACHE() do {					\
-	OUT_RING( CP_PACKET0( RADEON_RB3D_DSTCACHE_CTLSTAT, 0 ) );	\
-	OUT_RING( RADEON_RB3D_DC_FLUSH_ALL );				\
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
+	        OUT_RING(CP_PACKET0( RADEON_RB3D_DSTCACHE_CTLSTAT, 0));	\
+	        OUT_RING(RADEON_RB3D_DC_FLUSH | RADEON_RB3D_DC_FREE);	\
+	} else {                                                        \
+	        OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));	\
+	        OUT_RING(R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE );	\
+        }                                                               \
 } while (0)
 
 #define RADEON_FLUSH_ZCACHE() do {					\
-	OUT_RING( CP_PACKET0( RADEON_RB3D_ZCACHE_CTLSTAT, 0 ) );	\
-	OUT_RING( RADEON_RB3D_ZC_FLUSH );				\
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
+	        OUT_RING( CP_PACKET0( RADEON_RB3D_ZCACHE_CTLSTAT, 0 ) ); \
+	        OUT_RING( RADEON_RB3D_ZC_FLUSH );			\
+	} else {                                                        \
+	        OUT_RING( CP_PACKET0( R300_ZB_ZCACHE_CTLSTAT, 0 ) );	\
+	        OUT_RING( R300_ZC_FLUSH );				\
+        }                                                               \
 } while (0)
 
 #define RADEON_PURGE_ZCACHE() do {					\
-	OUT_RING( CP_PACKET0( RADEON_RB3D_ZCACHE_CTLSTAT, 0 ) );	\
-	OUT_RING( RADEON_RB3D_ZC_FLUSH_ALL );				\
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {	\
+	        OUT_RING(CP_PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0));	\
+	        OUT_RING(RADEON_RB3D_ZC_FLUSH | RADEON_RB3D_ZC_FREE);	\
+	} else {                                                        \
+	        OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));	\
+	        OUT_RING(R300_ZC_FLUSH | R300_ZC_FREE);			\
+        }                                                               \
 } while (0)
 
 /* ================================================================
@@ -1094,8 +1371,7 @@ do {									\
 
 #define BEGIN_RING( n ) do {						\
 	if ( RADEON_VERBOSE ) {						\
-		DRM_INFO( "BEGIN_RING( %d ) in %s\n",			\
-			   n, __FUNCTION__ );				\
+		DRM_INFO( "BEGIN_RING( %d )\n", (n));			\
 	}								\
 	if ( dev_priv->ring.space <= (n) * sizeof(u32) ) {		\
 		COMMIT_RING();						\
@@ -1113,7 +1389,7 @@ do {									\
 			  write, dev_priv->ring.tail );			\
 	}								\
 	if (((dev_priv->ring.tail + _nr) & mask) != write) {		\
-		DRM_ERROR( 						\
+		DRM_ERROR(						\
 			"ADVANCE_RING(): mismatch: nr: %x write: %x line: %d\n",	\
 			((dev_priv->ring.tail + _nr) & mask),		\
 			write, __LINE__);						\
diff --git a/sys/dev/drm/radeon_irq.c b/sys/dev/drm/radeon_irq.c
index 420fc84..ac3b195 100644
--- a/sys/dev/drm/radeon_irq.c
+++ b/sys/dev/drm/radeon_irq.c
@@ -38,12 +38,130 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/radeon_drm.h"
 #include "dev/drm/radeon_drv.h"
 
-static __inline__ u32 radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv,
-					      u32 mask)
+void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state)
 {
-	u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS) & mask;
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
+	if (state)
+		dev_priv->irq_enable_reg |= mask;
+	else
+		dev_priv->irq_enable_reg &= ~mask;
+
+	RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
+}
+
+static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state)
+{
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
+	if (state)
+		dev_priv->r500_disp_irq_reg |= mask;
+	else
+		dev_priv->r500_disp_irq_reg &= ~mask;
+
+	RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
+}
+
+int radeon_enable_vblank(struct drm_device *dev, int crtc)
+{
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {	
+		switch (crtc) {
+		case 0:
+			r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 1);
+			break;
+		case 1:
+			r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 1);
+			break;
+		default:
+			DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+				  crtc);
+			return EINVAL;
+		}
+	} else {
+		switch (crtc) {
+		case 0:
+			radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 1);
+			break;
+		case 1:
+			radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 1);
+			break;
+		default:
+			DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+				  crtc);
+			return EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+void radeon_disable_vblank(struct drm_device *dev, int crtc)
+{
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {	
+		switch (crtc) {
+		case 0:
+			r500_vbl_irq_set_state(dev, R500_D1MODE_INT_MASK, 0);
+			break;
+		case 1:
+			r500_vbl_irq_set_state(dev, R500_D2MODE_INT_MASK, 0);
+			break;
+		default:
+			DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+				  crtc);
+			break;
+		}
+	} else {
+		switch (crtc) {
+		case 0:
+			radeon_irq_set_state(dev, RADEON_CRTC_VBLANK_MASK, 0);
+			break;
+		case 1:
+			radeon_irq_set_state(dev, RADEON_CRTC2_VBLANK_MASK, 0);
+			break;
+		default:
+			DRM_ERROR("tried to enable vblank on non-existent crtc %d\n",
+				  crtc);
+			break;
+		}
+	}
+}
+
+static __inline__ u32 radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv, u32 *r500_disp_int)
+{
+	u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS);
+	u32 irq_mask = RADEON_SW_INT_TEST;
+
+	*r500_disp_int = 0;
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+		/* vbl interrupts in a different place */
+
+		if (irqs & R500_DISPLAY_INT_STATUS) {
+			/* if a display interrupt */
+			u32 disp_irq;
+
+			disp_irq = RADEON_READ(R500_DISP_INTERRUPT_STATUS);
+
+			*r500_disp_int = disp_irq;
+			if (disp_irq & R500_D1_VBLANK_INTERRUPT) {
+				RADEON_WRITE(R500_D1MODE_VBLANK_STATUS, R500_VBLANK_ACK);
+			}
+			if (disp_irq & R500_D2_VBLANK_INTERRUPT) {
+				RADEON_WRITE(R500_D2MODE_VBLANK_STATUS, R500_VBLANK_ACK);
+			}
+		}
+		irq_mask |= R500_DISPLAY_INT_STATUS;
+	} else
+		irq_mask |= RADEON_CRTC_VBLANK_STAT | RADEON_CRTC2_VBLANK_STAT;
+
+	irqs &=	irq_mask;
+
 	if (irqs)
 		RADEON_WRITE(RADEON_GEN_INT_STATUS, irqs);
+	
 	return irqs;
 }
 
@@ -67,35 +185,41 @@ static __inline__ u32 radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv,
 
 irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS)
 {
-	drm_device_t *dev = (drm_device_t *) arg;
+	struct drm_device *dev = (struct drm_device *) arg;
 	drm_radeon_private_t *dev_priv =
 	    (drm_radeon_private_t *) dev->dev_private;
 	u32 stat;
+	u32 r500_disp_int;
 
 	/* Only consider the bits we're interested in - others could be used
 	 * outside the DRM
 	 */
-	stat = radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
-						  RADEON_CRTC_VBLANK_STAT));
+	stat = radeon_acknowledge_irqs(dev_priv, &r500_disp_int);
 	if (!stat)
 		return IRQ_NONE;
 
+	stat &= dev_priv->irq_enable_reg;
+
 	/* SW interrupt */
-	if (stat & RADEON_SW_INT_TEST) {
+	if (stat & RADEON_SW_INT_TEST)
 		DRM_WAKEUP(&dev_priv->swi_queue);
-	}
 
 	/* VBLANK interrupt */
-	if (stat & RADEON_CRTC_VBLANK_STAT) {
-		atomic_inc(&dev->vbl_received);
-		DRM_WAKEUP(&dev->vbl_queue);
-		drm_vbl_send_signals(dev);
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+		if (r500_disp_int & R500_D1_VBLANK_INTERRUPT)
+			drm_handle_vblank(dev, 0);
+		if (r500_disp_int & R500_D2_VBLANK_INTERRUPT)
+			drm_handle_vblank(dev, 1);
+	} else {
+		if (stat & RADEON_CRTC_VBLANK_STAT)
+			drm_handle_vblank(dev, 0);
+		if (stat & RADEON_CRTC2_VBLANK_STAT)
+			drm_handle_vblank(dev, 1);
 	}
-
 	return IRQ_HANDLED;
 }
 
-static int radeon_emit_irq(drm_device_t * dev)
+static int radeon_emit_irq(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	unsigned int ret;
@@ -113,7 +237,7 @@ static int radeon_emit_irq(drm_device_t * dev)
 	return ret;
 }
 
-static int radeon_wait_irq(drm_device_t * dev, int swi_nr)
+static int radeon_wait_irq(struct drm_device * dev, int swi_nr)
 {
 	drm_radeon_private_t *dev_priv =
 	    (drm_radeon_private_t *) dev->dev_private;
@@ -130,59 +254,53 @@ static int radeon_wait_irq(drm_device_t * dev, int swi_nr)
 	return ret;
 }
 
-int radeon_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
+u32 radeon_get_vblank_counter(struct drm_device *dev, int crtc)
 {
-	drm_radeon_private_t *dev_priv =
-	    (drm_radeon_private_t *) dev->dev_private;
-	unsigned int cur_vblank;
-	int ret = 0;
+	drm_radeon_private_t *dev_priv = dev->dev_private;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	radeon_acknowledge_irqs(dev_priv, RADEON_CRTC_VBLANK_STAT);
-
-	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
-
-	/* Assume that the user has missed the current sequence number
-	 * by about a day rather than she wants to wait for years
-	 * using vertical blanks...
-	 */
-	DRM_WAIT_ON(ret, dev->vbl_queue, 3 * DRM_HZ,
-		    (((cur_vblank = atomic_read(&dev->vbl_received))
-		      - *sequence) <= (1 << 23)));
-
-	*sequence = cur_vblank;
+	if (crtc < 0 || crtc > 1) {
+		DRM_ERROR("Invalid crtc %d\n", crtc);
+		return -EINVAL;
+	}
 
-	return ret;
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) {
+		if (crtc == 0)
+			return RADEON_READ(R500_D1CRTC_FRAME_COUNT);
+		else
+			return RADEON_READ(R500_D2CRTC_FRAME_COUNT);
+	} else {
+		if (crtc == 0)
+			return RADEON_READ(RADEON_CRTC_CRNT_FRAME);
+		else
+			return RADEON_READ(RADEON_CRTC2_CRNT_FRAME);
+	}
 }
 
 /* Needs the lock as it touches the ring.
  */
-int radeon_irq_emit(DRM_IOCTL_ARGS)
+int radeon_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_irq_emit_t emit;
+	drm_radeon_irq_emit_t *emit = data;
 	int result;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(emit, (drm_radeon_irq_emit_t __user *) data,
-				 sizeof(emit));
-
 	result = radeon_emit_irq(dev);
 
-	if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
+	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
@@ -190,58 +308,86 @@ int radeon_irq_emit(DRM_IOCTL_ARGS)
 
 /* Doesn't need the hardware lock.
  */
-int radeon_irq_wait(DRM_IOCTL_ARGS)
+int radeon_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_irq_wait_t irqwait;
+	drm_radeon_irq_wait_t *irqwait = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(irqwait, (drm_radeon_irq_wait_t __user *) data,
-				 sizeof(irqwait));
-
-	return radeon_wait_irq(dev, irqwait.irq_seq);
+	return radeon_wait_irq(dev, irqwait->irq_seq);
 }
 
 /* drm_dma.h hooks
 */
-void radeon_driver_irq_preinstall(drm_device_t * dev)
+void radeon_driver_irq_preinstall(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv =
 	    (drm_radeon_private_t *) dev->dev_private;
+	u32 dummy;
 
 	/* Disable *all* interrupts */
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+		RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
 	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 
 	/* Clear bits if they're already high */
-	radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
-					   RADEON_CRTC_VBLANK_STAT));
+	radeon_acknowledge_irqs(dev_priv, &dummy);
 }
 
-void radeon_driver_irq_postinstall(drm_device_t * dev)
+int radeon_driver_irq_postinstall(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv =
 	    (drm_radeon_private_t *) dev->dev_private;
+	int ret;
 
 	atomic_set(&dev_priv->swi_emitted, 0);
 	DRM_INIT_WAITQUEUE(&dev_priv->swi_queue);
 
-	/* Turn on SW and VBL ints */
-	RADEON_WRITE(RADEON_GEN_INT_CNTL,
-		     RADEON_CRTC_VBLANK_MASK | RADEON_SW_INT_ENABLE);
+	ret = drm_vblank_init(dev, 2);
+	if (ret)
+		return ret;
+
+	dev->max_vblank_count = 0x001fffff;
+
+	radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
+
+	return 0;
 }
 
-void radeon_driver_irq_uninstall(drm_device_t * dev)
+void radeon_driver_irq_uninstall(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv =
 	    (drm_radeon_private_t *) dev->dev_private;
 	if (!dev_priv)
 		return;
 
+	dev_priv->irq_enabled = 0;
+
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
+		RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
 	/* Disable *all* interrupts */
 	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 }
+
+
+int radeon_vblank_crtc_get(struct drm_device *dev)
+{
+	drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
+
+	return dev_priv->vblank_crtc;
+}
+
+int radeon_vblank_crtc_set(struct drm_device *dev, int64_t value)
+{
+	drm_radeon_private_t *dev_priv = (drm_radeon_private_t *) dev->dev_private;
+	if (value & ~(DRM_RADEON_VBLANK_CRTC1 | DRM_RADEON_VBLANK_CRTC2)) {
+		DRM_ERROR("called with invalid crtc 0x%x\n", (unsigned int)value);
+		return -EINVAL;
+	}
+	dev_priv->vblank_crtc = (unsigned int)value;
+	return 0;
+}
diff --git a/sys/dev/drm/radeon_mem.c b/sys/dev/drm/radeon_mem.c
index bb9ff30..b39175e 100644
--- a/sys/dev/drm/radeon_mem.c
+++ b/sys/dev/drm/radeon_mem.c
@@ -42,7 +42,7 @@ __FBSDID("$FreeBSD$");
  */
 
 static struct mem_block *split_block(struct mem_block *p, int start, int size,
-				     DRMFILE filp)
+				     struct drm_file *file_priv)
 {
 	/* Maybe cut off the start of an existing block */
 	if (start > p->start) {
@@ -52,7 +52,7 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 			goto out;
 		newblock->start = start;
 		newblock->size = p->size - (start - p->start);
-		newblock->filp = NULL;
+		newblock->file_priv = NULL;
 		newblock->next = p->next;
 		newblock->prev = p;
 		p->next->prev = newblock;
@@ -69,7 +69,7 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 			goto out;
 		newblock->start = start + size;
 		newblock->size = p->size - size;
-		newblock->filp = NULL;
+		newblock->file_priv = NULL;
 		newblock->next = p->next;
 		newblock->prev = p;
 		p->next->prev = newblock;
@@ -79,20 +79,20 @@ static struct mem_block *split_block(struct mem_block *p, int start, int size,
 
       out:
 	/* Our block is in the middle */
-	p->filp = filp;
+	p->file_priv = file_priv;
 	return p;
 }
 
 static struct mem_block *alloc_block(struct mem_block *heap, int size,
-				     int align2, DRMFILE filp)
+				     int align2, struct drm_file *file_priv)
 {
 	struct mem_block *p;
 	int mask = (1 << align2) - 1;
 
 	list_for_each(p, heap) {
 		int start = (p->start + mask) & ~mask;
-		if (p->filp == 0 && start + size <= p->start + p->size)
-			return split_block(p, start, size, filp);
+		if (p->file_priv == NULL && start + size <= p->start + p->size)
+			return split_block(p, start, size, file_priv);
 	}
 
 	return NULL;
@@ -111,12 +111,12 @@ static struct mem_block *find_block(struct mem_block *heap, int start)
 
 static void free_block(struct mem_block *p)
 {
-	p->filp = NULL;
+	p->file_priv = NULL;
 
-	/* Assumes a single contiguous range.  Needs a special filp in
+	/* Assumes a single contiguous range.  Needs a special file_priv in
 	 * 'heap' to stop it being subsumed.
 	 */
-	if (p->next->filp == 0) {
+	if (p->next->file_priv == NULL) {
 		struct mem_block *q = p->next;
 		p->size += q->size;
 		p->next = q->next;
@@ -124,7 +124,7 @@ static void free_block(struct mem_block *p)
 		drm_free(q, sizeof(*q), DRM_MEM_BUFS);
 	}
 
-	if (p->prev->filp == 0) {
+	if (p->prev->file_priv == NULL) {
 		struct mem_block *q = p->prev;
 		q->size += p->size;
 		q->next = p->next;
@@ -140,28 +140,28 @@ static int init_heap(struct mem_block **heap, int start, int size)
 	struct mem_block *blocks = drm_alloc(sizeof(*blocks), DRM_MEM_BUFS);
 
 	if (!blocks)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	*heap = drm_alloc(sizeof(**heap), DRM_MEM_BUFS);
 	if (!*heap) {
 		drm_free(blocks, sizeof(*blocks), DRM_MEM_BUFS);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	blocks->start = start;
 	blocks->size = size;
-	blocks->filp = NULL;
+	blocks->file_priv = NULL;
 	blocks->next = blocks->prev = *heap;
 
 	memset(*heap, 0, sizeof(**heap));
-	(*heap)->filp = (DRMFILE) - 1;
+	(*heap)->file_priv = (struct drm_file *) - 1;
 	(*heap)->next = (*heap)->prev = blocks;
 	return 0;
 }
 
 /* Free all blocks associated with the releasing file.
  */
-void radeon_mem_release(DRMFILE filp, struct mem_block *heap)
+void radeon_mem_release(struct drm_file *file_priv, struct mem_block *heap)
 {
 	struct mem_block *p;
 
@@ -169,15 +169,15 @@ void radeon_mem_release(DRMFILE filp, struct mem_block *heap)
 		return;
 
 	list_for_each(p, heap) {
-		if (p->filp == filp)
-			p->filp = NULL;
+		if (p->file_priv == file_priv)
+			p->file_priv = NULL;
 	}
 
-	/* Assumes a single contiguous range.  Needs a special filp in
+	/* Assumes a single contiguous range.  Needs a special file_priv in
 	 * 'heap' to stop it being subsumed.
 	 */
 	list_for_each(p, heap) {
-		while (p->filp == 0 && p->next->filp == 0) {
+		while (p->file_priv == NULL && p->next->file_priv == NULL) {
 			struct mem_block *q = p->next;
 			p->size += q->size;
 			p->next = q->next;
@@ -220,98 +220,86 @@ static struct mem_block **get_heap(drm_radeon_private_t * dev_priv, int region)
 	}
 }
 
-int radeon_mem_alloc(DRM_IOCTL_ARGS)
+int radeon_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_alloc_t alloc;
+	drm_radeon_mem_alloc_t *alloc = data;
 	struct mem_block *block, **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(alloc, (drm_radeon_mem_alloc_t __user *) data,
-				 sizeof(alloc));
-
-	heap = get_heap(dev_priv, alloc.region);
+	heap = get_heap(dev_priv, alloc->region);
 	if (!heap || !*heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
 	/* Make things easier on ourselves: all allocations at least
 	 * 4k aligned.
 	 */
-	if (alloc.alignment < 12)
-		alloc.alignment = 12;
+	if (alloc->alignment < 12)
+		alloc->alignment = 12;
 
-	block = alloc_block(*heap, alloc.size, alloc.alignment, filp);
+	block = alloc_block(*heap, alloc->size, alloc->alignment, file_priv);
 
 	if (!block)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
-	if (DRM_COPY_TO_USER(alloc.region_offset, &block->start, sizeof(int))) {
+	if (DRM_COPY_TO_USER(alloc->region_offset, &block->start,
+			     sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-int radeon_mem_free(DRM_IOCTL_ARGS)
+int radeon_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_free_t memfree;
+	drm_radeon_mem_free_t *memfree = data;
 	struct mem_block *block, **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(memfree, (drm_radeon_mem_free_t __user *) data,
-				 sizeof(memfree));
-
-	heap = get_heap(dev_priv, memfree.region);
+	heap = get_heap(dev_priv, memfree->region);
 	if (!heap || !*heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
-	block = find_block(*heap, memfree.region_offset);
+	block = find_block(*heap, memfree->region_offset);
 	if (!block)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
-	if (block->filp != filp)
-		return DRM_ERR(EPERM);
+	if (block->file_priv != file_priv)
+		return -EPERM;
 
 	free_block(block);
 	return 0;
 }
 
-int radeon_mem_init_heap(DRM_IOCTL_ARGS)
+int radeon_mem_init_heap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_mem_init_heap_t initheap;
+	drm_radeon_mem_init_heap_t *initheap = data;
 	struct mem_block **heap;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(initheap,
-				 (drm_radeon_mem_init_heap_t __user *) data,
-				 sizeof(initheap));
-
-	heap = get_heap(dev_priv, initheap.region);
+	heap = get_heap(dev_priv, initheap->region);
 	if (!heap)
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
 	if (*heap) {
 		DRM_ERROR("heap already initialized?");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
-	return init_heap(heap, initheap.start, initheap.size);
+	return init_heap(heap, initheap->start, initheap->size);
 }
diff --git a/sys/dev/drm/radeon_microcode.h b/sys/dev/drm/radeon_microcode.h
new file mode 100644
index 0000000..e5665aa
--- /dev/null
+++ b/sys/dev/drm/radeon_microcode.h
@@ -0,0 +1,1847 @@
+/*-
+ * Copyright 2007 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#ifndef RADEON_MICROCODE_H
+#define RADEON_MICROCODE_H
+
+/* production radeon ucode r1xx-r6xx */
+static const u32 R100_cp_microcode[][2]={
+    { 0x21007000, 0000000000 },
+    { 0x20007000, 0000000000 },
+    { 0x000000b4, 0x00000004 },
+    { 0x000000b8, 0x00000004 },
+    { 0x6f5b4d4c, 0000000000 },
+    { 0x4c4c427f, 0000000000 },
+    { 0x5b568a92, 0000000000 },
+    { 0x4ca09c6d, 0000000000 },
+    { 0xad4c4c4c, 0000000000 },
+    { 0x4ce1af3d, 0000000000 },
+    { 0xd8afafaf, 0000000000 },
+    { 0xd64c4cdc, 0000000000 },
+    { 0x4cd10d10, 0000000000 },
+    { 0x000f0000, 0x00000016 },
+    { 0x362f242d, 0000000000 },
+    { 0x00000012, 0x00000004 },
+    { 0x000f0000, 0x00000016 },
+    { 0x362f282d, 0000000000 },
+    { 0x000380e7, 0x00000002 },
+    { 0x04002c97, 0x00000002 },
+    { 0x000f0001, 0x00000016 },
+    { 0x333a3730, 0000000000 },
+    { 0x000077ef, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x00000021, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00061000, 0x00000002 },
+    { 0x00000021, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00061000, 0x00000002 },
+    { 0x00000021, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00000017, 0x00000004 },
+    { 0x0003802b, 0x00000002 },
+    { 0x040067e0, 0x00000002 },
+    { 0x00000017, 0x00000004 },
+    { 0x000077e0, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x000037e1, 0x00000002 },
+    { 0x040067e1, 0x00000006 },
+    { 0x000077e0, 0x00000002 },
+    { 0x000077e1, 0x00000002 },
+    { 0x000077e1, 0x00000006 },
+    { 0xffffffff, 0000000000 },
+    { 0x10000000, 0000000000 },
+    { 0x0003802b, 0x00000002 },
+    { 0x040067e0, 0x00000006 },
+    { 0x00007675, 0x00000002 },
+    { 0x00007676, 0x00000002 },
+    { 0x00007677, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0003802c, 0x00000002 },
+    { 0x04002676, 0x00000002 },
+    { 0x00007677, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0000002f, 0x00000018 },
+    { 0x0000002f, 0x00000018 },
+    { 0000000000, 0x00000006 },
+    { 0x00000030, 0x00000018 },
+    { 0x00000030, 0x00000018 },
+    { 0000000000, 0x00000006 },
+    { 0x01605000, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x00098000, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x64c0603e, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00080000, 0x00000016 },
+    { 0000000000, 0000000000 },
+    { 0x0400251d, 0x00000002 },
+    { 0x00007580, 0x00000002 },
+    { 0x00067581, 0x00000002 },
+    { 0x04002580, 0x00000002 },
+    { 0x00067581, 0x00000002 },
+    { 0x00000049, 0x00000004 },
+    { 0x00005000, 0000000000 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x0000750e, 0x00000002 },
+    { 0x00019000, 0x00000002 },
+    { 0x00011055, 0x00000014 },
+    { 0x00000055, 0x00000012 },
+    { 0x0400250f, 0x00000002 },
+    { 0x0000504f, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00007565, 0x00000002 },
+    { 0x00007566, 0x00000002 },
+    { 0x00000058, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x01e655b4, 0x00000002 },
+    { 0x4401b0e4, 0x00000002 },
+    { 0x01c110e4, 0x00000002 },
+    { 0x26667066, 0x00000018 },
+    { 0x040c2565, 0x00000002 },
+    { 0x00000066, 0x00000018 },
+    { 0x04002564, 0x00000002 },
+    { 0x00007566, 0x00000002 },
+    { 0x0000005d, 0x00000004 },
+    { 0x00401069, 0x00000008 },
+    { 0x00101000, 0x00000002 },
+    { 0x000d80ff, 0x00000002 },
+    { 0x0080006c, 0x00000008 },
+    { 0x000f9000, 0x00000002 },
+    { 0x000e00ff, 0x00000002 },
+    { 0000000000, 0x00000006 },
+    { 0x0000008f, 0x00000018 },
+    { 0x0000005b, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00007576, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x00009000, 0x00000002 },
+    { 0x00041000, 0x00000002 },
+    { 0x0c00350e, 0x00000002 },
+    { 0x00049000, 0x00000002 },
+    { 0x00051000, 0x00000002 },
+    { 0x01e785f8, 0x00000002 },
+    { 0x00200000, 0x00000002 },
+    { 0x0060007e, 0x0000000c },
+    { 0x00007563, 0x00000002 },
+    { 0x006075f0, 0x00000021 },
+    { 0x20007073, 0x00000004 },
+    { 0x00005073, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00007576, 0x00000002 },
+    { 0x00007577, 0x00000002 },
+    { 0x0000750e, 0x00000002 },
+    { 0x0000750f, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00600083, 0x0000000c },
+    { 0x006075f0, 0x00000021 },
+    { 0x000075f8, 0x00000002 },
+    { 0x00000083, 0x00000004 },
+    { 0x000a750e, 0x00000002 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x0020750f, 0x00000002 },
+    { 0x00600086, 0x00000004 },
+    { 0x00007570, 0x00000002 },
+    { 0x00007571, 0x00000002 },
+    { 0x00007572, 0x00000006 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00005000, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00007568, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x00000095, 0x0000000c },
+    { 0x00058000, 0x00000002 },
+    { 0x0c607562, 0x00000002 },
+    { 0x00000097, 0x00000004 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x00600096, 0x00000004 },
+    { 0x400070e5, 0000000000 },
+    { 0x000380e6, 0x00000002 },
+    { 0x040025c5, 0x00000002 },
+    { 0x000380e5, 0x00000002 },
+    { 0x000000a8, 0x0000001c },
+    { 0x000650aa, 0x00000018 },
+    { 0x040025bb, 0x00000002 },
+    { 0x000610ab, 0x00000018 },
+    { 0x040075bc, 0000000000 },
+    { 0x000075bb, 0x00000002 },
+    { 0x000075bc, 0000000000 },
+    { 0x00090000, 0x00000006 },
+    { 0x00090000, 0x00000002 },
+    { 0x000d8002, 0x00000006 },
+    { 0x00007832, 0x00000002 },
+    { 0x00005000, 0x00000002 },
+    { 0x000380e7, 0x00000002 },
+    { 0x04002c97, 0x00000002 },
+    { 0x00007820, 0x00000002 },
+    { 0x00007821, 0x00000002 },
+    { 0x00007800, 0000000000 },
+    { 0x01200000, 0x00000002 },
+    { 0x20077000, 0x00000002 },
+    { 0x01200000, 0x00000002 },
+    { 0x20007000, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x0120751b, 0x00000002 },
+    { 0x8040750a, 0x00000002 },
+    { 0x8040750b, 0x00000002 },
+    { 0x00110000, 0x00000002 },
+    { 0x000380e5, 0x00000002 },
+    { 0x000000c6, 0x0000001c },
+    { 0x000610ab, 0x00000018 },
+    { 0x844075bd, 0x00000002 },
+    { 0x000610aa, 0x00000018 },
+    { 0x840075bb, 0x00000002 },
+    { 0x000610ab, 0x00000018 },
+    { 0x844075bc, 0x00000002 },
+    { 0x000000c9, 0x00000004 },
+    { 0x804075bd, 0x00000002 },
+    { 0x800075bb, 0x00000002 },
+    { 0x804075bc, 0x00000002 },
+    { 0x00108000, 0x00000002 },
+    { 0x01400000, 0x00000002 },
+    { 0x006000cd, 0x0000000c },
+    { 0x20c07000, 0x00000020 },
+    { 0x000000cf, 0x00000012 },
+    { 0x00800000, 0x00000006 },
+    { 0x0080751d, 0x00000006 },
+    { 0000000000, 0000000000 },
+    { 0x0000775c, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00661000, 0x00000002 },
+    { 0x0460275d, 0x00000020 },
+    { 0x00004000, 0000000000 },
+    { 0x01e00830, 0x00000002 },
+    { 0x21007000, 0000000000 },
+    { 0x6464614d, 0000000000 },
+    { 0x69687420, 0000000000 },
+    { 0x00000073, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x00005000, 0x00000002 },
+    { 0x000380d0, 0x00000002 },
+    { 0x040025e0, 0x00000002 },
+    { 0x000075e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000380e0, 0x00000002 },
+    { 0x04002394, 0x00000002 },
+    { 0x00005000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x00000008, 0000000000 },
+    { 0x00000004, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 R200_cp_microcode[][2]={
+    { 0x21007000, 0000000000 },
+    { 0x20007000, 0000000000 },
+    { 0x000000bf, 0x00000004 },
+    { 0x000000c3, 0x00000004 },
+    { 0x7a685e5d, 0000000000 },
+    { 0x5d5d5588, 0000000000 },
+    { 0x68659197, 0000000000 },
+    { 0x5da19f78, 0000000000 },
+    { 0x5d5d5d5d, 0000000000 },
+    { 0x5dee5d50, 0000000000 },
+    { 0xf2acacac, 0000000000 },
+    { 0xe75df9e9, 0000000000 },
+    { 0xb1dd0e11, 0000000000 },
+    { 0xe2afafaf, 0000000000 },
+    { 0x000f0000, 0x00000016 },
+    { 0x452f232d, 0000000000 },
+    { 0x00000013, 0x00000004 },
+    { 0x000f0000, 0x00000016 },
+    { 0x452f272d, 0000000000 },
+    { 0x000f0001, 0x00000016 },
+    { 0x3e4d4a37, 0000000000 },
+    { 0x000077ef, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x00000020, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00061000, 0x00000002 },
+    { 0x00000020, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00061000, 0x00000002 },
+    { 0x00000020, 0x0000001a },
+    { 0x00004000, 0x0000001e },
+    { 0x00000016, 0x00000004 },
+    { 0x0003802a, 0x00000002 },
+    { 0x040067e0, 0x00000002 },
+    { 0x00000016, 0x00000004 },
+    { 0x000077e0, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x000037e1, 0x00000002 },
+    { 0x040067e1, 0x00000006 },
+    { 0x000077e0, 0x00000002 },
+    { 0x000077e1, 0x00000002 },
+    { 0x000077e1, 0x00000006 },
+    { 0xffffffff, 0000000000 },
+    { 0x10000000, 0000000000 },
+    { 0x07f007f0, 0000000000 },
+    { 0x0003802a, 0x00000002 },
+    { 0x040067e0, 0x00000006 },
+    { 0x0003802c, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002743, 0x00000002 },
+    { 0x00007675, 0x00000002 },
+    { 0x00007676, 0x00000002 },
+    { 0x00007677, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0003802c, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002743, 0x00000002 },
+    { 0x00007676, 0x00000002 },
+    { 0x00007677, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0003802b, 0x00000002 },
+    { 0x04002676, 0x00000002 },
+    { 0x00007677, 0x00000002 },
+    { 0x0003802c, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002743, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0003802c, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002741, 0x00000002 },
+    { 0x04002743, 0x00000002 },
+    { 0x00007678, 0x00000006 },
+    { 0x0000002f, 0x00000018 },
+    { 0x0000002f, 0x00000018 },
+    { 0000000000, 0x00000006 },
+    { 0x00000037, 0x00000018 },
+    { 0x00000037, 0x00000018 },
+    { 0000000000, 0x00000006 },
+    { 0x01605000, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x00098000, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x64c06051, 0x00000004 },
+    { 0x00080000, 0x00000016 },
+    { 0000000000, 0000000000 },
+    { 0x0400251d, 0x00000002 },
+    { 0x00007580, 0x00000002 },
+    { 0x00067581, 0x00000002 },
+    { 0x04002580, 0x00000002 },
+    { 0x00067581, 0x00000002 },
+    { 0x0000005a, 0x00000004 },
+    { 0x00005000, 0000000000 },
+    { 0x00061000, 0x00000002 },
+    { 0x0000750e, 0x00000002 },
+    { 0x00019000, 0x00000002 },
+    { 0x00011064, 0x00000014 },
+    { 0x00000064, 0x00000012 },
+    { 0x0400250f, 0x00000002 },
+    { 0x0000505e, 0x00000004 },
+    { 0x00007565, 0x00000002 },
+    { 0x00007566, 0x00000002 },
+    { 0x00000065, 0x00000004 },
+    { 0x01e655b4, 0x00000002 },
+    { 0x4401b0f0, 0x00000002 },
+    { 0x01c110f0, 0x00000002 },
+    { 0x26667071, 0x00000018 },
+    { 0x040c2565, 0x00000002 },
+    { 0x00000071, 0x00000018 },
+    { 0x04002564, 0x00000002 },
+    { 0x00007566, 0x00000002 },
+    { 0x00000068, 0x00000004 },
+    { 0x00401074, 0x00000008 },
+    { 0x00101000, 0x00000002 },
+    { 0x000d80ff, 0x00000002 },
+    { 0x00800077, 0x00000008 },
+    { 0x000f9000, 0x00000002 },
+    { 0x000e00ff, 0x00000002 },
+    { 0000000000, 0x00000006 },
+    { 0x00000094, 0x00000018 },
+    { 0x00000068, 0x00000004 },
+    { 0x00007576, 0x00000002 },
+    { 0x00065000, 0x00000002 },
+    { 0x00009000, 0x00000002 },
+    { 0x00041000, 0x00000002 },
+    { 0x0c00350e, 0x00000002 },
+    { 0x00049000, 0x00000002 },
+    { 0x00051000, 0x00000002 },
+    { 0x01e785f8, 0x00000002 },
+    { 0x00200000, 0x00000002 },
+    { 0x00600087, 0x0000000c },
+    { 0x00007563, 0x00000002 },
+    { 0x006075f0, 0x00000021 },
+    { 0x2000707c, 0x00000004 },
+    { 0x0000507c, 0x00000004 },
+    { 0x00007576, 0x00000002 },
+    { 0x00007577, 0x00000002 },
+    { 0x0000750e, 0x00000002 },
+    { 0x0000750f, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x0060008a, 0x0000000c },
+    { 0x006075f0, 0x00000021 },
+    { 0x000075f8, 0x00000002 },
+    { 0x0000008a, 0x00000004 },
+    { 0x000a750e, 0x00000002 },
+    { 0x0020750f, 0x00000002 },
+    { 0x0060008d, 0x00000004 },
+    { 0x00007570, 0x00000002 },
+    { 0x00007571, 0x00000002 },
+    { 0x00007572, 0x00000006 },
+    { 0x00005000, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00007568, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x00000098, 0x0000000c },
+    { 0x00058000, 0x00000002 },
+    { 0x0c607562, 0x00000002 },
+    { 0x0000009a, 0x00000004 },
+    { 0x00600099, 0x00000004 },
+    { 0x400070f1, 0000000000 },
+    { 0x000380f1, 0x00000002 },
+    { 0x000000a7, 0x0000001c },
+    { 0x000650a9, 0x00000018 },
+    { 0x040025bb, 0x00000002 },
+    { 0x000610aa, 0x00000018 },
+    { 0x040075bc, 0000000000 },
+    { 0x000075bb, 0x00000002 },
+    { 0x000075bc, 0000000000 },
+    { 0x00090000, 0x00000006 },
+    { 0x00090000, 0x00000002 },
+    { 0x000d8002, 0x00000006 },
+    { 0x00005000, 0x00000002 },
+    { 0x00007821, 0x00000002 },
+    { 0x00007800, 0000000000 },
+    { 0x00007821, 0x00000002 },
+    { 0x00007800, 0000000000 },
+    { 0x01665000, 0x00000002 },
+    { 0x000a0000, 0x00000002 },
+    { 0x000671cc, 0x00000002 },
+    { 0x0286f1cd, 0x00000002 },
+    { 0x000000b7, 0x00000010 },
+    { 0x21007000, 0000000000 },
+    { 0x000000be, 0x0000001c },
+    { 0x00065000, 0x00000002 },
+    { 0x000a0000, 0x00000002 },
+    { 0x00061000, 0x00000002 },
+    { 0x000b0000, 0x00000002 },
+    { 0x38067000, 0x00000002 },
+    { 0x000a00ba, 0x00000004 },
+    { 0x20007000, 0000000000 },
+    { 0x01200000, 0x00000002 },
+    { 0x20077000, 0x00000002 },
+    { 0x01200000, 0x00000002 },
+    { 0x20007000, 0000000000 },
+    { 0x00061000, 0x00000002 },
+    { 0x0120751b, 0x00000002 },
+    { 0x8040750a, 0x00000002 },
+    { 0x8040750b, 0x00000002 },
+    { 0x00110000, 0x00000002 },
+    { 0x000380f1, 0x00000002 },
+    { 0x000000d1, 0x0000001c },
+    { 0x000610aa, 0x00000018 },
+    { 0x844075bd, 0x00000002 },
+    { 0x000610a9, 0x00000018 },
+    { 0x840075bb, 0x00000002 },
+    { 0x000610aa, 0x00000018 },
+    { 0x844075bc, 0x00000002 },
+    { 0x000000d4, 0x00000004 },
+    { 0x804075bd, 0x00000002 },
+    { 0x800075bb, 0x00000002 },
+    { 0x804075bc, 0x00000002 },
+    { 0x00108000, 0x00000002 },
+    { 0x01400000, 0x00000002 },
+    { 0x006000d8, 0x0000000c },
+    { 0x20c07000, 0x00000020 },
+    { 0x000000da, 0x00000012 },
+    { 0x00800000, 0x00000006 },
+    { 0x0080751d, 0x00000006 },
+    { 0x000025bb, 0x00000002 },
+    { 0x000040d4, 0x00000004 },
+    { 0x0000775c, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00661000, 0x00000002 },
+    { 0x0460275d, 0x00000020 },
+    { 0x00004000, 0000000000 },
+    { 0x00007999, 0x00000002 },
+    { 0x00a05000, 0x00000002 },
+    { 0x00661000, 0x00000002 },
+    { 0x0460299b, 0x00000020 },
+    { 0x00004000, 0000000000 },
+    { 0x01e00830, 0x00000002 },
+    { 0x21007000, 0000000000 },
+    { 0x00005000, 0x00000002 },
+    { 0x00038056, 0x00000002 },
+    { 0x040025e0, 0x00000002 },
+    { 0x000075e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000380ed, 0x00000002 },
+    { 0x04007394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000078c4, 0x00000002 },
+    { 0x000078c5, 0x00000002 },
+    { 0x000078c6, 0x00000002 },
+    { 0x00007924, 0x00000002 },
+    { 0x00007925, 0x00000002 },
+    { 0x00007926, 0x00000002 },
+    { 0x000000f2, 0x00000004 },
+    { 0x00007924, 0x00000002 },
+    { 0x00007925, 0x00000002 },
+    { 0x00007926, 0x00000002 },
+    { 0x000000f9, 0x00000004 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 R300_cp_microcode[][2]={
+    { 0x4200e000, 0000000000 },
+    { 0x4000e000, 0000000000 },
+    { 0x000000ae, 0x00000008 },
+    { 0x000000b2, 0x00000008 },
+    { 0x67554b4a, 0000000000 },
+    { 0x4a4a4475, 0000000000 },
+    { 0x55527d83, 0000000000 },
+    { 0x4a8c8b65, 0000000000 },
+    { 0x4aef4af6, 0000000000 },
+    { 0x4ae14a4a, 0000000000 },
+    { 0xe4979797, 0000000000 },
+    { 0xdb4aebdd, 0000000000 },
+    { 0x9ccc4a4a, 0000000000 },
+    { 0xd1989898, 0000000000 },
+    { 0x4a0f9ad6, 0000000000 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000d0012, 0x00000038 },
+    { 0x0000e8b4, 0x00000004 },
+    { 0x000d0014, 0x00000038 },
+    { 0x0000e8b6, 0x00000004 },
+    { 0x000d0016, 0x00000038 },
+    { 0x0000e854, 0x00000004 },
+    { 0x000d0018, 0x00000038 },
+    { 0x0000e855, 0x00000004 },
+    { 0x000d001a, 0x00000038 },
+    { 0x0000e856, 0x00000004 },
+    { 0x000d001c, 0x00000038 },
+    { 0x0000e857, 0x00000004 },
+    { 0x000d001e, 0x00000038 },
+    { 0x0000e824, 0x00000004 },
+    { 0x000d0020, 0x00000038 },
+    { 0x0000e825, 0x00000004 },
+    { 0x000d0022, 0x00000038 },
+    { 0x0000e830, 0x00000004 },
+    { 0x000d0024, 0x00000038 },
+    { 0x0000f0c0, 0x00000004 },
+    { 0x000d0026, 0x00000038 },
+    { 0x0000f0c1, 0x00000004 },
+    { 0x000d0028, 0x00000038 },
+    { 0x0000f041, 0x00000004 },
+    { 0x000d002a, 0x00000038 },
+    { 0x0000f184, 0x00000004 },
+    { 0x000d002c, 0x00000038 },
+    { 0x0000f185, 0x00000004 },
+    { 0x000d002e, 0x00000038 },
+    { 0x0000f186, 0x00000004 },
+    { 0x000d0030, 0x00000038 },
+    { 0x0000f187, 0x00000004 },
+    { 0x000d0032, 0x00000038 },
+    { 0x0000f180, 0x00000004 },
+    { 0x000d0034, 0x00000038 },
+    { 0x0000f393, 0x00000004 },
+    { 0x000d0036, 0x00000038 },
+    { 0x0000f38a, 0x00000004 },
+    { 0x000d0038, 0x00000038 },
+    { 0x0000f38e, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000043, 0x00000018 },
+    { 0x00cce800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x0000003a, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x2000451d, 0x00000004 },
+    { 0x0000e580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x08004580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x00000047, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x00032000, 0x00000004 },
+    { 0x00022051, 0x00000028 },
+    { 0x00000051, 0x00000024 },
+    { 0x0800450f, 0x00000004 },
+    { 0x0000a04b, 0x00000008 },
+    { 0x0000e565, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000052, 0x00000008 },
+    { 0x03cca5b4, 0x00000004 },
+    { 0x05432000, 0x00000004 },
+    { 0x00022000, 0x00000004 },
+    { 0x4ccce05e, 0x00000030 },
+    { 0x08274565, 0x00000004 },
+    { 0x0000005e, 0x00000030 },
+    { 0x08004564, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000055, 0x00000008 },
+    { 0x00802061, 0x00000010 },
+    { 0x00202000, 0x00000004 },
+    { 0x001b00ff, 0x00000004 },
+    { 0x01000064, 0x00000010 },
+    { 0x001f2000, 0x00000004 },
+    { 0x001c00ff, 0x00000004 },
+    { 0000000000, 0x0000000c },
+    { 0x00000080, 0x00000030 },
+    { 0x00000055, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00012000, 0x00000004 },
+    { 0x00082000, 0x00000004 },
+    { 0x1800650e, 0x00000004 },
+    { 0x00092000, 0x00000004 },
+    { 0x000a2000, 0x00000004 },
+    { 0x000f0000, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x00000074, 0x00000018 },
+    { 0x0000e563, 0x00000004 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000069, 0x00000008 },
+    { 0x0000a069, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x0000e577, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x0000e50f, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000077, 0x00000018 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000077, 0x00000008 },
+    { 0x0014e50e, 0x00000004 },
+    { 0x0040e50f, 0x00000004 },
+    { 0x00c0007a, 0x00000008 },
+    { 0x0000e570, 0x00000004 },
+    { 0x0000e571, 0x00000004 },
+    { 0x0000e572, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x0000e568, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00000084, 0x00000018 },
+    { 0x000b0000, 0x00000004 },
+    { 0x18c0e562, 0x00000004 },
+    { 0x00000086, 0x00000008 },
+    { 0x00c00085, 0x00000008 },
+    { 0x000700e3, 0x00000004 },
+    { 0x00000092, 0x00000038 },
+    { 0x000ca094, 0x00000030 },
+    { 0x080045bb, 0x00000004 },
+    { 0x000c2095, 0x00000030 },
+    { 0x0800e5bc, 0000000000 },
+    { 0x0000e5bb, 0x00000004 },
+    { 0x0000e5bc, 0000000000 },
+    { 0x00120000, 0x0000000c },
+    { 0x00120000, 0x00000004 },
+    { 0x001b0002, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e800, 0000000000 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e82e, 0000000000 },
+    { 0x02cca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000ce1cc, 0x00000004 },
+    { 0x050de1cd, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x000000a4, 0x00000018 },
+    { 0x00c0a000, 0x00000004 },
+    { 0x000000a1, 0x00000008 },
+    { 0x000000a6, 0x00000020 },
+    { 0x4200e000, 0000000000 },
+    { 0x000000ad, 0x00000038 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00160000, 0x00000004 },
+    { 0x700ce000, 0x00000004 },
+    { 0x001400a9, 0x00000008 },
+    { 0x4000e000, 0000000000 },
+    { 0x02400000, 0x00000004 },
+    { 0x400ee000, 0x00000004 },
+    { 0x02400000, 0x00000004 },
+    { 0x4000e000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0240e51b, 0x00000004 },
+    { 0x0080e50a, 0x00000005 },
+    { 0x0080e50b, 0x00000005 },
+    { 0x00220000, 0x00000004 },
+    { 0x000700e3, 0x00000004 },
+    { 0x000000c0, 0x00000038 },
+    { 0x000c2095, 0x00000030 },
+    { 0x0880e5bd, 0x00000005 },
+    { 0x000c2094, 0x00000030 },
+    { 0x0800e5bb, 0x00000005 },
+    { 0x000c2095, 0x00000030 },
+    { 0x0880e5bc, 0x00000005 },
+    { 0x000000c3, 0x00000008 },
+    { 0x0080e5bd, 0x00000005 },
+    { 0x0000e5bb, 0x00000005 },
+    { 0x0080e5bc, 0x00000005 },
+    { 0x00210000, 0x00000004 },
+    { 0x02800000, 0x00000004 },
+    { 0x00c000c7, 0x00000018 },
+    { 0x4180e000, 0x00000040 },
+    { 0x000000c9, 0x00000024 },
+    { 0x01000000, 0x0000000c },
+    { 0x0100e51d, 0x0000000c },
+    { 0x000045bb, 0x00000004 },
+    { 0x000080c3, 0x00000008 },
+    { 0x0000f3ce, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053cf, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f3d2, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053d3, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f39d, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c0539e, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x03c00830, 0x00000004 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x200045e0, 0x00000004 },
+    { 0x0000e5e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000700e0, 0x00000004 },
+    { 0x0800e394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x0000e8c4, 0x00000004 },
+    { 0x0000e8c5, 0x00000004 },
+    { 0x0000e8c6, 0x00000004 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000e4, 0x00000008 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000eb, 0x00000008 },
+    { 0x02c02000, 0x00000004 },
+    { 0x00060000, 0x00000004 },
+    { 0x000000f3, 0x00000034 },
+    { 0x000000f0, 0x00000008 },
+    { 0x00008000, 0x00000004 },
+    { 0xc000e000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x001d0018, 0x00000004 },
+    { 0x001a0001, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0x0500a04a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 R420_cp_microcode[][2]={
+    { 0x4200e000, 0000000000 },
+    { 0x4000e000, 0000000000 },
+    { 0x00000099, 0x00000008 },
+    { 0x0000009d, 0x00000008 },
+    { 0x4a554b4a, 0000000000 },
+    { 0x4a4a4467, 0000000000 },
+    { 0x55526f75, 0000000000 },
+    { 0x4a7e7d65, 0000000000 },
+    { 0xd9d3dff6, 0000000000 },
+    { 0x4ac54a4a, 0000000000 },
+    { 0xc8828282, 0000000000 },
+    { 0xbf4acfc1, 0000000000 },
+    { 0x87b04a4a, 0000000000 },
+    { 0xb5838383, 0000000000 },
+    { 0x4a0f85ba, 0000000000 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000d0012, 0x00000038 },
+    { 0x0000e8b4, 0x00000004 },
+    { 0x000d0014, 0x00000038 },
+    { 0x0000e8b6, 0x00000004 },
+    { 0x000d0016, 0x00000038 },
+    { 0x0000e854, 0x00000004 },
+    { 0x000d0018, 0x00000038 },
+    { 0x0000e855, 0x00000004 },
+    { 0x000d001a, 0x00000038 },
+    { 0x0000e856, 0x00000004 },
+    { 0x000d001c, 0x00000038 },
+    { 0x0000e857, 0x00000004 },
+    { 0x000d001e, 0x00000038 },
+    { 0x0000e824, 0x00000004 },
+    { 0x000d0020, 0x00000038 },
+    { 0x0000e825, 0x00000004 },
+    { 0x000d0022, 0x00000038 },
+    { 0x0000e830, 0x00000004 },
+    { 0x000d0024, 0x00000038 },
+    { 0x0000f0c0, 0x00000004 },
+    { 0x000d0026, 0x00000038 },
+    { 0x0000f0c1, 0x00000004 },
+    { 0x000d0028, 0x00000038 },
+    { 0x0000f041, 0x00000004 },
+    { 0x000d002a, 0x00000038 },
+    { 0x0000f184, 0x00000004 },
+    { 0x000d002c, 0x00000038 },
+    { 0x0000f185, 0x00000004 },
+    { 0x000d002e, 0x00000038 },
+    { 0x0000f186, 0x00000004 },
+    { 0x000d0030, 0x00000038 },
+    { 0x0000f187, 0x00000004 },
+    { 0x000d0032, 0x00000038 },
+    { 0x0000f180, 0x00000004 },
+    { 0x000d0034, 0x00000038 },
+    { 0x0000f393, 0x00000004 },
+    { 0x000d0036, 0x00000038 },
+    { 0x0000f38a, 0x00000004 },
+    { 0x000d0038, 0x00000038 },
+    { 0x0000f38e, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000043, 0x00000018 },
+    { 0x00cce800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x0000003a, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x2000451d, 0x00000004 },
+    { 0x0000e580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x08004580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x00000047, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x00032000, 0x00000004 },
+    { 0x00022051, 0x00000028 },
+    { 0x00000051, 0x00000024 },
+    { 0x0800450f, 0x00000004 },
+    { 0x0000a04b, 0x00000008 },
+    { 0x0000e565, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000052, 0x00000008 },
+    { 0x03cca5b4, 0x00000004 },
+    { 0x05432000, 0x00000004 },
+    { 0x00022000, 0x00000004 },
+    { 0x4ccce05e, 0x00000030 },
+    { 0x08274565, 0x00000004 },
+    { 0x0000005e, 0x00000030 },
+    { 0x08004564, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000055, 0x00000008 },
+    { 0x00802061, 0x00000010 },
+    { 0x00202000, 0x00000004 },
+    { 0x001b00ff, 0x00000004 },
+    { 0x01000064, 0x00000010 },
+    { 0x001f2000, 0x00000004 },
+    { 0x001c00ff, 0x00000004 },
+    { 0000000000, 0x0000000c },
+    { 0x00000072, 0x00000030 },
+    { 0x00000055, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x0000e577, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x0000e50f, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000069, 0x00000018 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000069, 0x00000008 },
+    { 0x0014e50e, 0x00000004 },
+    { 0x0040e50f, 0x00000004 },
+    { 0x00c0006c, 0x00000008 },
+    { 0x0000e570, 0x00000004 },
+    { 0x0000e571, 0x00000004 },
+    { 0x0000e572, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x0000e568, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00000076, 0x00000018 },
+    { 0x000b0000, 0x00000004 },
+    { 0x18c0e562, 0x00000004 },
+    { 0x00000078, 0x00000008 },
+    { 0x00c00077, 0x00000008 },
+    { 0x000700c7, 0x00000004 },
+    { 0x00000080, 0x00000038 },
+    { 0x0000e5bb, 0x00000004 },
+    { 0x0000e5bc, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e800, 0000000000 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e82e, 0000000000 },
+    { 0x02cca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000ce1cc, 0x00000004 },
+    { 0x050de1cd, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x0000008f, 0x00000018 },
+    { 0x00c0a000, 0x00000004 },
+    { 0x0000008c, 0x00000008 },
+    { 0x00000091, 0x00000020 },
+    { 0x4200e000, 0000000000 },
+    { 0x00000098, 0x00000038 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00160000, 0x00000004 },
+    { 0x700ce000, 0x00000004 },
+    { 0x00140094, 0x00000008 },
+    { 0x4000e000, 0000000000 },
+    { 0x02400000, 0x00000004 },
+    { 0x400ee000, 0x00000004 },
+    { 0x02400000, 0x00000004 },
+    { 0x4000e000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0240e51b, 0x00000004 },
+    { 0x0080e50a, 0x00000005 },
+    { 0x0080e50b, 0x00000005 },
+    { 0x00220000, 0x00000004 },
+    { 0x000700c7, 0x00000004 },
+    { 0x000000a4, 0x00000038 },
+    { 0x0080e5bd, 0x00000005 },
+    { 0x0000e5bb, 0x00000005 },
+    { 0x0080e5bc, 0x00000005 },
+    { 0x00210000, 0x00000004 },
+    { 0x02800000, 0x00000004 },
+    { 0x00c000ab, 0x00000018 },
+    { 0x4180e000, 0x00000040 },
+    { 0x000000ad, 0x00000024 },
+    { 0x01000000, 0x0000000c },
+    { 0x0100e51d, 0x0000000c },
+    { 0x000045bb, 0x00000004 },
+    { 0x000080a7, 0x00000008 },
+    { 0x0000f3ce, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053cf, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f3d2, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053d3, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f39d, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c0539e, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x03c00830, 0x00000004 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x200045e0, 0x00000004 },
+    { 0x0000e5e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000700c4, 0x00000004 },
+    { 0x0800e394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x0000e8c4, 0x00000004 },
+    { 0x0000e8c5, 0x00000004 },
+    { 0x0000e8c6, 0x00000004 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000c8, 0x00000008 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000cf, 0x00000008 },
+    { 0x02c02000, 0x00000004 },
+    { 0x00060000, 0x00000004 },
+    { 0x000000d7, 0x00000034 },
+    { 0x000000d4, 0x00000008 },
+    { 0x00008000, 0x00000004 },
+    { 0xc000e000, 0000000000 },
+    { 0x0000e1cc, 0x00000004 },
+    { 0x0500e1cd, 0x00000004 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000000de, 0x00000034 },
+    { 0x000000da, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x0019e1cc, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x0500a000, 0x00000004 },
+    { 0x080041cd, 0x00000004 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x001d0018, 0x00000004 },
+    { 0x001a0001, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0x0500a04a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 RS600_cp_microcode[][2]={
+    { 0x4200e000, 0000000000 },
+    { 0x4000e000, 0000000000 },
+    { 0x000000a0, 0x00000008 },
+    { 0x000000a4, 0x00000008 },
+    { 0x4a554b4a, 0000000000 },
+    { 0x4a4a4467, 0000000000 },
+    { 0x55526f75, 0000000000 },
+    { 0x4a7e7d65, 0000000000 },
+    { 0x4ae74af6, 0000000000 },
+    { 0x4ad34a4a, 0000000000 },
+    { 0xd6898989, 0000000000 },
+    { 0xcd4addcf, 0000000000 },
+    { 0x8ebe4ae2, 0000000000 },
+    { 0xc38a8a8a, 0000000000 },
+    { 0x4a0f8cc8, 0000000000 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000d0012, 0x00000038 },
+    { 0x0000e8b4, 0x00000004 },
+    { 0x000d0014, 0x00000038 },
+    { 0x0000e8b6, 0x00000004 },
+    { 0x000d0016, 0x00000038 },
+    { 0x0000e854, 0x00000004 },
+    { 0x000d0018, 0x00000038 },
+    { 0x0000e855, 0x00000004 },
+    { 0x000d001a, 0x00000038 },
+    { 0x0000e856, 0x00000004 },
+    { 0x000d001c, 0x00000038 },
+    { 0x0000e857, 0x00000004 },
+    { 0x000d001e, 0x00000038 },
+    { 0x0000e824, 0x00000004 },
+    { 0x000d0020, 0x00000038 },
+    { 0x0000e825, 0x00000004 },
+    { 0x000d0022, 0x00000038 },
+    { 0x0000e830, 0x00000004 },
+    { 0x000d0024, 0x00000038 },
+    { 0x0000f0c0, 0x00000004 },
+    { 0x000d0026, 0x00000038 },
+    { 0x0000f0c1, 0x00000004 },
+    { 0x000d0028, 0x00000038 },
+    { 0x0000f041, 0x00000004 },
+    { 0x000d002a, 0x00000038 },
+    { 0x0000f184, 0x00000004 },
+    { 0x000d002c, 0x00000038 },
+    { 0x0000f185, 0x00000004 },
+    { 0x000d002e, 0x00000038 },
+    { 0x0000f186, 0x00000004 },
+    { 0x000d0030, 0x00000038 },
+    { 0x0000f187, 0x00000004 },
+    { 0x000d0032, 0x00000038 },
+    { 0x0000f180, 0x00000004 },
+    { 0x000d0034, 0x00000038 },
+    { 0x0000f393, 0x00000004 },
+    { 0x000d0036, 0x00000038 },
+    { 0x0000f38a, 0x00000004 },
+    { 0x000d0038, 0x00000038 },
+    { 0x0000f38e, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000043, 0x00000018 },
+    { 0x00cce800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x0000003a, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x2000451d, 0x00000004 },
+    { 0x0000e580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x08004580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x00000047, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x00032000, 0x00000004 },
+    { 0x00022051, 0x00000028 },
+    { 0x00000051, 0x00000024 },
+    { 0x0800450f, 0x00000004 },
+    { 0x0000a04b, 0x00000008 },
+    { 0x0000e565, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000052, 0x00000008 },
+    { 0x03cca5b4, 0x00000004 },
+    { 0x05432000, 0x00000004 },
+    { 0x00022000, 0x00000004 },
+    { 0x4ccce05e, 0x00000030 },
+    { 0x08274565, 0x00000004 },
+    { 0x0000005e, 0x00000030 },
+    { 0x08004564, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000055, 0x00000008 },
+    { 0x00802061, 0x00000010 },
+    { 0x00202000, 0x00000004 },
+    { 0x001b00ff, 0x00000004 },
+    { 0x01000064, 0x00000010 },
+    { 0x001f2000, 0x00000004 },
+    { 0x001c00ff, 0x00000004 },
+    { 0000000000, 0x0000000c },
+    { 0x00000072, 0x00000030 },
+    { 0x00000055, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x0000e577, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x0000e50f, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000069, 0x00000018 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000069, 0x00000008 },
+    { 0x0014e50e, 0x00000004 },
+    { 0x0040e50f, 0x00000004 },
+    { 0x00c0006c, 0x00000008 },
+    { 0x0000e570, 0x00000004 },
+    { 0x0000e571, 0x00000004 },
+    { 0x0000e572, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x0000e568, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00000076, 0x00000018 },
+    { 0x000b0000, 0x00000004 },
+    { 0x18c0e562, 0x00000004 },
+    { 0x00000078, 0x00000008 },
+    { 0x00c00077, 0x00000008 },
+    { 0x000700d5, 0x00000004 },
+    { 0x00000084, 0x00000038 },
+    { 0x000ca086, 0x00000030 },
+    { 0x080045bb, 0x00000004 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0800e5bc, 0000000000 },
+    { 0x0000e5bb, 0x00000004 },
+    { 0x0000e5bc, 0000000000 },
+    { 0x00120000, 0x0000000c },
+    { 0x00120000, 0x00000004 },
+    { 0x001b0002, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e800, 0000000000 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e82e, 0000000000 },
+    { 0x02cca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000ce1cc, 0x00000004 },
+    { 0x050de1cd, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x00000096, 0x00000018 },
+    { 0x00c0a000, 0x00000004 },
+    { 0x00000093, 0x00000008 },
+    { 0x00000098, 0x00000020 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000009f, 0x00000038 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00160000, 0x00000004 },
+    { 0x700ce000, 0x00000004 },
+    { 0x0014009b, 0x00000008 },
+    { 0x4000e000, 0000000000 },
+    { 0x02400000, 0x00000004 },
+    { 0x400ee000, 0x00000004 },
+    { 0x02400000, 0x00000004 },
+    { 0x4000e000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0240e51b, 0x00000004 },
+    { 0x0080e50a, 0x00000005 },
+    { 0x0080e50b, 0x00000005 },
+    { 0x00220000, 0x00000004 },
+    { 0x000700d5, 0x00000004 },
+    { 0x000000b2, 0x00000038 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0880e5bd, 0x00000005 },
+    { 0x000c2086, 0x00000030 },
+    { 0x0800e5bb, 0x00000005 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0880e5bc, 0x00000005 },
+    { 0x000000b5, 0x00000008 },
+    { 0x0080e5bd, 0x00000005 },
+    { 0x0000e5bb, 0x00000005 },
+    { 0x0080e5bc, 0x00000005 },
+    { 0x00210000, 0x00000004 },
+    { 0x02800000, 0x00000004 },
+    { 0x00c000b9, 0x00000018 },
+    { 0x4180e000, 0x00000040 },
+    { 0x000000bb, 0x00000024 },
+    { 0x01000000, 0x0000000c },
+    { 0x0100e51d, 0x0000000c },
+    { 0x000045bb, 0x00000004 },
+    { 0x000080b5, 0x00000008 },
+    { 0x0000f3ce, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053cf, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f3d2, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053d3, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f39d, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c0539e, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x03c00830, 0x00000004 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x200045e0, 0x00000004 },
+    { 0x0000e5e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000700d2, 0x00000004 },
+    { 0x0800e394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x0000e8c4, 0x00000004 },
+    { 0x0000e8c5, 0x00000004 },
+    { 0x0000e8c6, 0x00000004 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000d6, 0x00000008 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000dd, 0x00000008 },
+    { 0x00e00116, 0000000000 },
+    { 0x000700e1, 0x00000004 },
+    { 0x0800401c, 0x00000004 },
+    { 0x200050e7, 0x00000004 },
+    { 0x0000e01d, 0x00000004 },
+    { 0x000000e4, 0x00000008 },
+    { 0x02c02000, 0x00000004 },
+    { 0x00060000, 0x00000004 },
+    { 0x000000eb, 0x00000034 },
+    { 0x000000e8, 0x00000008 },
+    { 0x00008000, 0x00000004 },
+    { 0xc000e000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x001d0018, 0x00000004 },
+    { 0x001a0001, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0x0500a04a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 RS690_cp_microcode[][2]={
+    { 0x000000dd, 0x00000008 },
+    { 0x000000df, 0x00000008 },
+    { 0x000000a0, 0x00000008 },
+    { 0x000000a4, 0x00000008 },
+    { 0x4a554b4a, 0000000000 },
+    { 0x4a4a4467, 0000000000 },
+    { 0x55526f75, 0000000000 },
+    { 0x4a7e7d65, 0000000000 },
+    { 0x4ad74af6, 0000000000 },
+    { 0x4ac94a4a, 0000000000 },
+    { 0xcc898989, 0000000000 },
+    { 0xc34ad3c5, 0000000000 },
+    { 0x8e4a4a4a, 0000000000 },
+    { 0x4a8a8a8a, 0000000000 },
+    { 0x4a0f8c4a, 0000000000 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000d0012, 0x00000038 },
+    { 0x0000e8b4, 0x00000004 },
+    { 0x000d0014, 0x00000038 },
+    { 0x0000e8b6, 0x00000004 },
+    { 0x000d0016, 0x00000038 },
+    { 0x0000e854, 0x00000004 },
+    { 0x000d0018, 0x00000038 },
+    { 0x0000e855, 0x00000004 },
+    { 0x000d001a, 0x00000038 },
+    { 0x0000e856, 0x00000004 },
+    { 0x000d001c, 0x00000038 },
+    { 0x0000e857, 0x00000004 },
+    { 0x000d001e, 0x00000038 },
+    { 0x0000e824, 0x00000004 },
+    { 0x000d0020, 0x00000038 },
+    { 0x0000e825, 0x00000004 },
+    { 0x000d0022, 0x00000038 },
+    { 0x0000e830, 0x00000004 },
+    { 0x000d0024, 0x00000038 },
+    { 0x0000f0c0, 0x00000004 },
+    { 0x000d0026, 0x00000038 },
+    { 0x0000f0c1, 0x00000004 },
+    { 0x000d0028, 0x00000038 },
+    { 0x0000f041, 0x00000004 },
+    { 0x000d002a, 0x00000038 },
+    { 0x0000f184, 0x00000004 },
+    { 0x000d002c, 0x00000038 },
+    { 0x0000f185, 0x00000004 },
+    { 0x000d002e, 0x00000038 },
+    { 0x0000f186, 0x00000004 },
+    { 0x000d0030, 0x00000038 },
+    { 0x0000f187, 0x00000004 },
+    { 0x000d0032, 0x00000038 },
+    { 0x0000f180, 0x00000004 },
+    { 0x000d0034, 0x00000038 },
+    { 0x0000f393, 0x00000004 },
+    { 0x000d0036, 0x00000038 },
+    { 0x0000f38a, 0x00000004 },
+    { 0x000d0038, 0x00000038 },
+    { 0x0000f38e, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000043, 0x00000018 },
+    { 0x00cce800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x0000003a, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x2000451d, 0x00000004 },
+    { 0x0000e580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x08004580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x00000047, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x00032000, 0x00000004 },
+    { 0x00022051, 0x00000028 },
+    { 0x00000051, 0x00000024 },
+    { 0x0800450f, 0x00000004 },
+    { 0x0000a04b, 0x00000008 },
+    { 0x0000e565, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000052, 0x00000008 },
+    { 0x03cca5b4, 0x00000004 },
+    { 0x05432000, 0x00000004 },
+    { 0x00022000, 0x00000004 },
+    { 0x4ccce05e, 0x00000030 },
+    { 0x08274565, 0x00000004 },
+    { 0x0000005e, 0x00000030 },
+    { 0x08004564, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000055, 0x00000008 },
+    { 0x00802061, 0x00000010 },
+    { 0x00202000, 0x00000004 },
+    { 0x001b00ff, 0x00000004 },
+    { 0x01000064, 0x00000010 },
+    { 0x001f2000, 0x00000004 },
+    { 0x001c00ff, 0x00000004 },
+    { 0000000000, 0x0000000c },
+    { 0x00000072, 0x00000030 },
+    { 0x00000055, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x0000e577, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x0000e50f, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000069, 0x00000018 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000069, 0x00000008 },
+    { 0x0014e50e, 0x00000004 },
+    { 0x0040e50f, 0x00000004 },
+    { 0x00c0006c, 0x00000008 },
+    { 0x0000e570, 0x00000004 },
+    { 0x0000e571, 0x00000004 },
+    { 0x0000e572, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x0000e568, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00000076, 0x00000018 },
+    { 0x000b0000, 0x00000004 },
+    { 0x18c0e562, 0x00000004 },
+    { 0x00000078, 0x00000008 },
+    { 0x00c00077, 0x00000008 },
+    { 0x000700cb, 0x00000004 },
+    { 0x00000084, 0x00000038 },
+    { 0x000ca086, 0x00000030 },
+    { 0x080045bb, 0x00000004 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0800e5bc, 0000000000 },
+    { 0x0000e5bb, 0x00000004 },
+    { 0x0000e5bc, 0000000000 },
+    { 0x00120000, 0x0000000c },
+    { 0x00120000, 0x00000004 },
+    { 0x001b0002, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e800, 0000000000 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e82e, 0000000000 },
+    { 0x02cca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000ce1cc, 0x00000004 },
+    { 0x050de1cd, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x00000096, 0x00000018 },
+    { 0x00c0a000, 0x00000004 },
+    { 0x00000093, 0x00000008 },
+    { 0x00000098, 0x00000020 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000009f, 0x00000038 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00160000, 0x00000004 },
+    { 0x700ce000, 0x00000004 },
+    { 0x0014009b, 0x00000008 },
+    { 0x4000e000, 0000000000 },
+    { 0x02400000, 0x00000004 },
+    { 0x400ee000, 0x00000004 },
+    { 0x02400000, 0x00000004 },
+    { 0x4000e000, 0000000000 },
+    { 0x00100000, 0x0000002c },
+    { 0x00004000, 0000000000 },
+    { 0x080045c8, 0x00000004 },
+    { 0x00240005, 0x00000004 },
+    { 0x08004d0b, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0240e51b, 0x00000004 },
+    { 0x0080e50a, 0x00000005 },
+    { 0x0080e50b, 0x00000005 },
+    { 0x00220000, 0x00000004 },
+    { 0x000700cb, 0x00000004 },
+    { 0x000000b7, 0x00000038 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0880e5bd, 0x00000005 },
+    { 0x000c2086, 0x00000030 },
+    { 0x0800e5bb, 0x00000005 },
+    { 0x000c2087, 0x00000030 },
+    { 0x0880e5bc, 0x00000005 },
+    { 0x000000ba, 0x00000008 },
+    { 0x0080e5bd, 0x00000005 },
+    { 0x0000e5bb, 0x00000005 },
+    { 0x0080e5bc, 0x00000005 },
+    { 0x00210000, 0x00000004 },
+    { 0x02800000, 0x00000004 },
+    { 0x00c000be, 0x00000018 },
+    { 0x4180e000, 0x00000040 },
+    { 0x000000c0, 0x00000024 },
+    { 0x01000000, 0x0000000c },
+    { 0x0100e51d, 0x0000000c },
+    { 0x000045bb, 0x00000004 },
+    { 0x000080ba, 0x00000008 },
+    { 0x03c00830, 0x00000004 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x200045e0, 0x00000004 },
+    { 0x0000e5e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000700c8, 0x00000004 },
+    { 0x0800e394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x0000e8c4, 0x00000004 },
+    { 0x0000e8c5, 0x00000004 },
+    { 0x0000e8c6, 0x00000004 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000cc, 0x00000008 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000d3, 0x00000008 },
+    { 0x02c02000, 0x00000004 },
+    { 0x00060000, 0x00000004 },
+    { 0x000000db, 0x00000034 },
+    { 0x000000d8, 0x00000008 },
+    { 0x00008000, 0x00000004 },
+    { 0xc000e000, 0000000000 },
+    { 0x000000e1, 0x00000030 },
+    { 0x4200e000, 0000000000 },
+    { 0x000000e1, 0x00000030 },
+    { 0x4000e000, 0000000000 },
+    { 0x0025001b, 0x00000004 },
+    { 0x00230000, 0x00000004 },
+    { 0x00250005, 0x00000004 },
+    { 0x000000e6, 0x00000034 },
+    { 0000000000, 0x0000000c },
+    { 0x00244000, 0x00000004 },
+    { 0x080045c8, 0x00000004 },
+    { 0x00240005, 0x00000004 },
+    { 0x08004d0b, 0x0000000c },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x001d0018, 0x00000004 },
+    { 0x001a0001, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0x0500a04a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+static const u32 R520_cp_microcode[][2]={
+    { 0x4200e000, 0000000000 },
+    { 0x4000e000, 0000000000 },
+    { 0x00000099, 0x00000008 },
+    { 0x0000009d, 0x00000008 },
+    { 0x4a554b4a, 0000000000 },
+    { 0x4a4a4467, 0000000000 },
+    { 0x55526f75, 0000000000 },
+    { 0x4a7e7d65, 0000000000 },
+    { 0xe0dae6f6, 0000000000 },
+    { 0x4ac54a4a, 0000000000 },
+    { 0xc8828282, 0000000000 },
+    { 0xbf4acfc1, 0000000000 },
+    { 0x87b04ad5, 0000000000 },
+    { 0xb5838383, 0000000000 },
+    { 0x4a0f85ba, 0000000000 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000d0012, 0x00000038 },
+    { 0x0000e8b4, 0x00000004 },
+    { 0x000d0014, 0x00000038 },
+    { 0x0000e8b6, 0x00000004 },
+    { 0x000d0016, 0x00000038 },
+    { 0x0000e854, 0x00000004 },
+    { 0x000d0018, 0x00000038 },
+    { 0x0000e855, 0x00000004 },
+    { 0x000d001a, 0x00000038 },
+    { 0x0000e856, 0x00000004 },
+    { 0x000d001c, 0x00000038 },
+    { 0x0000e857, 0x00000004 },
+    { 0x000d001e, 0x00000038 },
+    { 0x0000e824, 0x00000004 },
+    { 0x000d0020, 0x00000038 },
+    { 0x0000e825, 0x00000004 },
+    { 0x000d0022, 0x00000038 },
+    { 0x0000e830, 0x00000004 },
+    { 0x000d0024, 0x00000038 },
+    { 0x0000f0c0, 0x00000004 },
+    { 0x000d0026, 0x00000038 },
+    { 0x0000f0c1, 0x00000004 },
+    { 0x000d0028, 0x00000038 },
+    { 0x0000e000, 0x00000004 },
+    { 0x000d002a, 0x00000038 },
+    { 0x0000e000, 0x00000004 },
+    { 0x000d002c, 0x00000038 },
+    { 0x0000e000, 0x00000004 },
+    { 0x000d002e, 0x00000038 },
+    { 0x0000e000, 0x00000004 },
+    { 0x000d0030, 0x00000038 },
+    { 0x0000e000, 0x00000004 },
+    { 0x000d0032, 0x00000038 },
+    { 0x0000f180, 0x00000004 },
+    { 0x000d0034, 0x00000038 },
+    { 0x0000f393, 0x00000004 },
+    { 0x000d0036, 0x00000038 },
+    { 0x0000f38a, 0x00000004 },
+    { 0x000d0038, 0x00000038 },
+    { 0x0000f38e, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000043, 0x00000018 },
+    { 0x00cce800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x08004800, 0x00000004 },
+    { 0x0000003a, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x2000451d, 0x00000004 },
+    { 0x0000e580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x08004580, 0x00000004 },
+    { 0x000ce581, 0x00000004 },
+    { 0x00000047, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x00032000, 0x00000004 },
+    { 0x00022051, 0x00000028 },
+    { 0x00000051, 0x00000024 },
+    { 0x0800450f, 0x00000004 },
+    { 0x0000a04b, 0x00000008 },
+    { 0x0000e565, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000052, 0x00000008 },
+    { 0x03cca5b4, 0x00000004 },
+    { 0x05432000, 0x00000004 },
+    { 0x00022000, 0x00000004 },
+    { 0x4ccce05e, 0x00000030 },
+    { 0x08274565, 0x00000004 },
+    { 0x0000005e, 0x00000030 },
+    { 0x08004564, 0x00000004 },
+    { 0x0000e566, 0x00000004 },
+    { 0x00000055, 0x00000008 },
+    { 0x00802061, 0x00000010 },
+    { 0x00202000, 0x00000004 },
+    { 0x001b00ff, 0x00000004 },
+    { 0x01000064, 0x00000010 },
+    { 0x001f2000, 0x00000004 },
+    { 0x001c00ff, 0x00000004 },
+    { 0000000000, 0x0000000c },
+    { 0x00000072, 0x00000030 },
+    { 0x00000055, 0x00000008 },
+    { 0x0000e576, 0x00000004 },
+    { 0x0000e577, 0x00000004 },
+    { 0x0000e50e, 0x00000004 },
+    { 0x0000e50f, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00000069, 0x00000018 },
+    { 0x00c0e5f9, 0x000000c2 },
+    { 0x00000069, 0x00000008 },
+    { 0x0014e50e, 0x00000004 },
+    { 0x0040e50f, 0x00000004 },
+    { 0x00c0006c, 0x00000008 },
+    { 0x0000e570, 0x00000004 },
+    { 0x0000e571, 0x00000004 },
+    { 0x0000e572, 0x0000000c },
+    { 0x0000a000, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x0000e568, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00000076, 0x00000018 },
+    { 0x000b0000, 0x00000004 },
+    { 0x18c0e562, 0x00000004 },
+    { 0x00000078, 0x00000008 },
+    { 0x00c00077, 0x00000008 },
+    { 0x000700c7, 0x00000004 },
+    { 0x00000080, 0x00000038 },
+    { 0x0000e5bb, 0x00000004 },
+    { 0x0000e5bc, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e800, 0000000000 },
+    { 0x0000e821, 0x00000004 },
+    { 0x0000e82e, 0000000000 },
+    { 0x02cca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000ce1cc, 0x00000004 },
+    { 0x050de1cd, 0x00000004 },
+    { 0x00400000, 0x00000004 },
+    { 0x0000008f, 0x00000018 },
+    { 0x00c0a000, 0x00000004 },
+    { 0x0000008c, 0x00000008 },
+    { 0x00000091, 0x00000020 },
+    { 0x4200e000, 0000000000 },
+    { 0x00000098, 0x00000038 },
+    { 0x000ca000, 0x00000004 },
+    { 0x00140000, 0x00000004 },
+    { 0x000c2000, 0x00000004 },
+    { 0x00160000, 0x00000004 },
+    { 0x700ce000, 0x00000004 },
+    { 0x00140094, 0x00000008 },
+    { 0x4000e000, 0000000000 },
+    { 0x02400000, 0x00000004 },
+    { 0x400ee000, 0x00000004 },
+    { 0x02400000, 0x00000004 },
+    { 0x4000e000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x0240e51b, 0x00000004 },
+    { 0x0080e50a, 0x00000005 },
+    { 0x0080e50b, 0x00000005 },
+    { 0x00220000, 0x00000004 },
+    { 0x000700c7, 0x00000004 },
+    { 0x000000a4, 0x00000038 },
+    { 0x0080e5bd, 0x00000005 },
+    { 0x0000e5bb, 0x00000005 },
+    { 0x0080e5bc, 0x00000005 },
+    { 0x00210000, 0x00000004 },
+    { 0x02800000, 0x00000004 },
+    { 0x00c000ab, 0x00000018 },
+    { 0x4180e000, 0x00000040 },
+    { 0x000000ad, 0x00000024 },
+    { 0x01000000, 0x0000000c },
+    { 0x0100e51d, 0x0000000c },
+    { 0x000045bb, 0x00000004 },
+    { 0x000080a7, 0x00000008 },
+    { 0x0000f3ce, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053cf, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f3d2, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c053d3, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x0000f39d, 0x00000004 },
+    { 0x0140a000, 0x00000004 },
+    { 0x00cc2000, 0x00000004 },
+    { 0x08c0539e, 0x00000040 },
+    { 0x00008000, 0000000000 },
+    { 0x03c00830, 0x00000004 },
+    { 0x4200e000, 0000000000 },
+    { 0x0000a000, 0x00000004 },
+    { 0x200045e0, 0x00000004 },
+    { 0x0000e5e1, 0000000000 },
+    { 0x00000001, 0000000000 },
+    { 0x000700c4, 0x00000004 },
+    { 0x0800e394, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x0000e8c4, 0x00000004 },
+    { 0x0000e8c5, 0x00000004 },
+    { 0x0000e8c6, 0x00000004 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000c8, 0x00000008 },
+    { 0x0000e928, 0x00000004 },
+    { 0x0000e929, 0x00000004 },
+    { 0x0000e92a, 0x00000004 },
+    { 0x000000cf, 0x00000008 },
+    { 0xdeadbeef, 0000000000 },
+    { 0x00000116, 0000000000 },
+    { 0x000700d3, 0x00000004 },
+    { 0x080050e7, 0x00000004 },
+    { 0x000700d4, 0x00000004 },
+    { 0x0800401c, 0x00000004 },
+    { 0x0000e01d, 0000000000 },
+    { 0x02c02000, 0x00000004 },
+    { 0x00060000, 0x00000004 },
+    { 0x000000de, 0x00000034 },
+    { 0x000000db, 0x00000008 },
+    { 0x00008000, 0x00000004 },
+    { 0xc000e000, 0000000000 },
+    { 0x0000e1cc, 0x00000004 },
+    { 0x0500e1cd, 0x00000004 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000000e5, 0x00000034 },
+    { 0x000000e1, 0x00000008 },
+    { 0x0000a000, 0000000000 },
+    { 0x0019e1cc, 0x00000004 },
+    { 0x001b0001, 0x00000004 },
+    { 0x0500a000, 0x00000004 },
+    { 0x080041cd, 0x00000004 },
+    { 0x000ca000, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0x000c2000, 0x00000004 },
+    { 0x001d0018, 0x00000004 },
+    { 0x001a0001, 0x00000004 },
+    { 0x000000fb, 0x00000034 },
+    { 0x0000004a, 0x00000008 },
+    { 0x0500a04a, 0x00000008 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+    { 0000000000, 0000000000 },
+};
+
+
+#endif
diff --git a/sys/dev/drm/radeon_state.c b/sys/dev/drm/radeon_state.c
index 868b5ca..d06748d 100644
--- a/sys/dev/drm/radeon_state.c
+++ b/sys/dev/drm/radeon_state.c
@@ -42,14 +42,11 @@ __FBSDID("$FreeBSD$");
 
 static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
 						    dev_priv,
-						    drm_file_t * filp_priv,
+						    struct drm_file *file_priv,
 						    u32 * offset)
 {
 	u64 off = *offset;
-	u32 fb_start = dev_priv->fb_location;
-	u32 fb_end = fb_start + dev_priv->fb_size - 1;
-	u32 gart_start = dev_priv->gart_vm_start;
-	u32 gart_end = gart_start + dev_priv->gart_size - 1;
+	u32 fb_end = dev_priv->fb_location + dev_priv->fb_size - 1;
 	struct drm_radeon_driver_file_fields *radeon_priv;
 
 	/* Hrm ... the story of the offset ... So this function converts
@@ -69,8 +66,7 @@ static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
 	/* First, the best case, the offset already lands in either the
 	 * framebuffer or the GART mapped space
 	 */
-	if ((off >= fb_start && off <= fb_end) ||
-	    (off >= gart_start && off <= gart_end))
+	if (radeon_check_offset(dev_priv, off))
 		return 0;
 
 	/* Ok, that didn't happen... now check if we have a zero based
@@ -78,44 +74,43 @@ static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
 	 * magic offset we get from SETPARAM or calculated from fb_location
 	 */
 	if (off < (dev_priv->fb_size + dev_priv->gart_size)) {
-		radeon_priv = filp_priv->driver_priv;
+		radeon_priv = file_priv->driver_priv;
 		off += radeon_priv->radeon_fb_delta;
 	}
 
 	/* Finally, assume we aimed at a GART offset if beyond the fb */
 	if (off > fb_end)
-		off = off - fb_end - 1 + gart_start;
+		off = off - fb_end - 1 + dev_priv->gart_vm_start;
 
 	/* Now recheck and fail if out of bounds */
-	if ((off >= fb_start && off <= fb_end) ||
-	    (off >= gart_start && off <= gart_end)) {
+	if (radeon_check_offset(dev_priv, off)) {
 		DRM_DEBUG("offset fixed up to 0x%x\n", (unsigned int)off);
 		*offset = off;
 		return 0;
 	}
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
 static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 						     dev_priv,
-						     drm_file_t * filp_priv,
+						     struct drm_file *file_priv,
 						     int id, u32 *data)
 {
 	switch (id) {
 
 	case RADEON_EMIT_PP_MISC:
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 		    &data[(RADEON_RB3D_DEPTHOFFSET - RADEON_PP_MISC) / 4])) {
 			DRM_ERROR("Invalid depth buffer offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 
 	case RADEON_EMIT_PP_CNTL:
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 		    &data[(RADEON_RB3D_COLOROFFSET - RADEON_PP_CNTL) / 4])) {
 			DRM_ERROR("Invalid colour buffer offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 
@@ -125,20 +120,20 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 	case R200_EMIT_PP_TXOFFSET_3:
 	case R200_EMIT_PP_TXOFFSET_4:
 	case R200_EMIT_PP_TXOFFSET_5:
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &data[0])) {
 			DRM_ERROR("Invalid R200 texture offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 
 	case RADEON_EMIT_PP_TXFILTER_0:
 	case RADEON_EMIT_PP_TXFILTER_1:
 	case RADEON_EMIT_PP_TXFILTER_2:
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 		    &data[(RADEON_PP_TXOFFSET_0 - RADEON_PP_TXFILTER_0) / 4])) {
 			DRM_ERROR("Invalid R100 texture offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 
@@ -151,11 +146,11 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 			int i;
 			for (i = 0; i < 5; i++) {
 				if (radeon_check_and_fixup_offset(dev_priv,
-								  filp_priv,
+								  file_priv,
 								  &data[i])) {
 					DRM_ERROR
 					    ("Invalid R200 cubic texture offset\n");
-					return DRM_ERR(EINVAL);
+					return -EINVAL;
 				}
 			}
 			break;
@@ -167,16 +162,24 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 			int i;
 			for (i = 0; i < 5; i++) {
 				if (radeon_check_and_fixup_offset(dev_priv,
-								  filp_priv,
+								  file_priv,
 								  &data[i])) {
 					DRM_ERROR
 					    ("Invalid R100 cubic texture offset\n");
-					return DRM_ERR(EINVAL);
+					return -EINVAL;
 				}
 			}
 		}
 		break;
 
+	case R200_EMIT_VAP_CTL: {
+			RING_LOCALS;
+			BEGIN_RING(2);
+			OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
+			ADVANCE_RING();
+		}
+		break;
+
 	case RADEON_EMIT_RB3D_COLORPITCH:
 	case RADEON_EMIT_RE_LINE_PATTERN:
 	case RADEON_EMIT_SE_LINE_WIDTH:
@@ -204,7 +207,6 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 	case R200_EMIT_TCL_LIGHT_MODEL_CTL_0:
 	case R200_EMIT_TFACTOR_0:
 	case R200_EMIT_VTX_FMT_0:
-	case R200_EMIT_VAP_CTL:
 	case R200_EMIT_MATRIX_SELECT_0:
 	case R200_EMIT_TEX_PROC_CTL_2:
 	case R200_EMIT_TCL_UCP_VERT_BLEND_CTL:
@@ -257,7 +259,7 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 
 	default:
 		DRM_ERROR("Unknown state packet ID %d\n", id);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -265,36 +267,140 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 
 static __inline__ int radeon_check_and_fixup_packet3(drm_radeon_private_t *
 						     dev_priv,
-						     drm_file_t *filp_priv,
+						     struct drm_file *file_priv,
 						     drm_radeon_kcmd_buffer_t *
 						     cmdbuf,
 						     unsigned int *cmdsz)
 {
 	u32 *cmd = (u32 *) cmdbuf->buf;
+	u32 offset, narrays;
+	int count, i, k;
 
 	*cmdsz = 2 + ((cmd[0] & RADEON_CP_PACKET_COUNT_MASK) >> 16);
 
 	if ((cmd[0] & 0xc0000000) != RADEON_CP_PACKET3) {
 		DRM_ERROR("Not a type 3 packet\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (4 * *cmdsz > cmdbuf->bufsz) {
 		DRM_ERROR("Packet size larger than size of data provided\n");
-		return DRM_ERR(EINVAL);
-	}
+		return -EINVAL;
+	}
+
+	switch(cmd[0] & 0xff00) {
+	/* XXX Are there old drivers needing other packets? */
+
+	case RADEON_3D_DRAW_IMMD:
+	case RADEON_3D_DRAW_VBUF:
+	case RADEON_3D_DRAW_INDX:
+	case RADEON_WAIT_FOR_IDLE:
+	case RADEON_CP_NOP:
+	case RADEON_3D_CLEAR_ZMASK:
+/*	case RADEON_CP_NEXT_CHAR:
+	case RADEON_CP_PLY_NEXTSCAN:
+	case RADEON_CP_SET_SCISSORS: */ /* probably safe but will never need them? */
+		/* these packets are safe */
+		break;
+
+	case RADEON_CP_3D_DRAW_IMMD_2:
+	case RADEON_CP_3D_DRAW_VBUF_2:
+	case RADEON_CP_3D_DRAW_INDX_2:
+	case RADEON_3D_CLEAR_HIZ:
+		/* safe but r200 only */
+		if ((dev_priv->chip_family < CHIP_R200) ||
+		    (dev_priv->chip_family > CHIP_RV280)) {
+			DRM_ERROR("Invalid 3d packet for non r200-class chip\n");
+			return -EINVAL;
+		}
+		break;
+
+	case RADEON_3D_LOAD_VBPNTR:
+		count = (cmd[0] >> 16) & 0x3fff;
+
+		if (count > 18) { /* 12 arrays max */
+			DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
+				  count);
+			return -EINVAL;
+		}
+
+		/* carefully check packet contents */
+		narrays = cmd[1] & ~0xc000;
+		k = 0;
+		i = 2;
+		while ((k < narrays) && (i < (count + 2))) {
+			i++;		/* skip attribute field */
+			if (radeon_check_and_fixup_offset(dev_priv, file_priv,
+							  &cmd[i])) {
+				DRM_ERROR
+				    ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
+				     k, i);
+				return -EINVAL;
+			}
+			k++;
+			i++;
+			if (k == narrays)
+				break;
+			/* have one more to process, they come in pairs */
+			if (radeon_check_and_fixup_offset(dev_priv,
+							  file_priv, &cmd[i]))
+			{
+				DRM_ERROR
+				    ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
+				     k, i);
+				return -EINVAL;
+			}
+			k++;
+			i++;
+		}
+		/* do the counts match what we expect ? */
+		if ((k != narrays) || (i != (count + 2))) {
+			DRM_ERROR
+			    ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
+			      k, i, narrays, count + 1);
+			return -EINVAL;
+		}
+		break;
+
+	case RADEON_3D_RNDR_GEN_INDX_PRIM:
+		if (dev_priv->chip_family > CHIP_RS200) {
+			DRM_ERROR("Invalid 3d packet for non-r100-class chip\n");
+			return -EINVAL;
+		}
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv, &cmd[1])) {
+				DRM_ERROR("Invalid rndr_gen_indx offset\n");
+				return -EINVAL;
+		}
+		break;
 
-	/* Check client state and fix it up if necessary */
-	if (cmd[0] & 0x8000) {	/* MSB of opcode: next DWORD GUI_CNTL */
-		u32 offset;
+	case RADEON_CP_INDX_BUFFER:
+		/* safe but r200 only */
+		if ((dev_priv->chip_family < CHIP_R200) ||
+		    (dev_priv->chip_family > CHIP_RV280)) {
+			DRM_ERROR("Invalid 3d packet for non-r200-class chip\n");
+			return -EINVAL;
+		}
+		if ((cmd[1] & 0x8000ffff) != 0x80000810) {
+			DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
+			return -EINVAL;
+		}
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv, &cmd[2])) {
+			DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
+			return -EINVAL;
+		}
+		break;
 
+	case RADEON_CNTL_HOSTDATA_BLT:
+	case RADEON_CNTL_PAINT_MULTI:
+	case RADEON_CNTL_BITBLT_MULTI:
+		/* MSB of opcode: next DWORD GUI_CNTL */
 		if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 			      | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 			offset = cmd[2] << 10;
 			if (radeon_check_and_fixup_offset
-			    (dev_priv, filp_priv, &offset)) {
+			    (dev_priv, file_priv, &offset)) {
 				DRM_ERROR("Invalid first packet offset\n");
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 			cmd[2] = (cmd[2] & 0xffc00000) | offset >> 10;
 		}
@@ -303,12 +409,17 @@ static __inline__ int radeon_check_and_fixup_packet3(drm_radeon_private_t *
 		    (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 			offset = cmd[3] << 10;
 			if (radeon_check_and_fixup_offset
-			    (dev_priv, filp_priv, &offset)) {
+			    (dev_priv, file_priv, &offset)) {
 				DRM_ERROR("Invalid second packet offset\n");
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 			cmd[3] = (cmd[3] & 0xffc00000) | offset >> 10;
 		}
+		break;
+
+	default:
+		DRM_ERROR("Invalid packet type %x\n", cmd[0] & 0xff00);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -319,7 +430,7 @@ static __inline__ int radeon_check_and_fixup_packet3(drm_radeon_private_t *
  */
 
 static __inline__ void radeon_emit_clip_rect(drm_radeon_private_t * dev_priv,
-					     drm_clip_rect_t * box)
+					     struct drm_clip_rect * box)
 {
 	RING_LOCALS;
 
@@ -337,7 +448,7 @@ static __inline__ void radeon_emit_clip_rect(drm_radeon_private_t * dev_priv,
 /* Emit 1.1 state
  */
 static int radeon_emit_state(drm_radeon_private_t * dev_priv,
-			     drm_file_t * filp_priv,
+			     struct drm_file *file_priv,
 			     drm_radeon_context_regs_t * ctx,
 			     drm_radeon_texture_regs_t * tex,
 			     unsigned int dirty)
@@ -346,16 +457,16 @@ static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 	DRM_DEBUG("dirty=0x%08x\n", dirty);
 
 	if (dirty & RADEON_UPLOAD_CONTEXT) {
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &ctx->rb3d_depthoffset)) {
 			DRM_ERROR("Invalid depth buffer offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &ctx->rb3d_coloroffset)) {
 			DRM_ERROR("Invalid depth buffer offset\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		BEGIN_RING(14);
@@ -441,10 +552,10 @@ static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 	}
 
 	if (dirty & RADEON_UPLOAD_TEX0) {
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &tex[0].pp_txoffset)) {
 			DRM_ERROR("Invalid texture offset for unit 0\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		BEGIN_RING(9);
@@ -461,10 +572,10 @@ static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 	}
 
 	if (dirty & RADEON_UPLOAD_TEX1) {
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &tex[1].pp_txoffset)) {
 			DRM_ERROR("Invalid texture offset for unit 1\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		BEGIN_RING(9);
@@ -481,10 +592,10 @@ static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 	}
 
 	if (dirty & RADEON_UPLOAD_TEX2) {
-		if (radeon_check_and_fixup_offset(dev_priv, filp_priv,
+		if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 						  &tex[2].pp_txoffset)) {
 			DRM_ERROR("Invalid texture offset for unit 2\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		BEGIN_RING(9);
@@ -506,7 +617,7 @@ static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 /* Emit 1.2 state
  */
 static int radeon_emit_state2(drm_radeon_private_t * dev_priv,
-			      drm_file_t * filp_priv,
+			      struct drm_file *file_priv,
 			      drm_radeon_state_t * state)
 {
 	RING_LOCALS;
@@ -519,7 +630,7 @@ static int radeon_emit_state2(drm_radeon_private_t * dev_priv,
 		ADVANCE_RING();
 	}
 
-	return radeon_emit_state(dev_priv, filp_priv, &state->context,
+	return radeon_emit_state(dev_priv, file_priv, &state->context,
 				 state->tex, state->dirty);
 }
 
@@ -671,7 +782,7 @@ static void radeon_clear_box(drm_radeon_private_t * dev_priv,
 		 RADEON_GMC_SRC_DATATYPE_COLOR |
 		 RADEON_ROP3_P | RADEON_GMC_CLR_CMP_CNTL_DIS);
 
-	if (dev_priv->page_flipping && dev_priv->current_page == 1) {
+	if (dev_priv->sarea_priv->pfCurrentPage == 1) {
 		OUT_RING(dev_priv->front_pitch_offset);
 	} else {
 		OUT_RING(dev_priv->back_pitch_offset);
@@ -742,7 +853,7 @@ static void radeon_cp_performance_boxes(drm_radeon_private_t * dev_priv)
  * CP command dispatch functions
  */
 
-static void radeon_cp_dispatch_clear(drm_device_t * dev,
+static void radeon_cp_dispatch_clear(struct drm_device * dev,
 				     drm_radeon_clear_t * clear,
 				     drm_radeon_clear_rect_t * depth_boxes)
 {
@@ -750,7 +861,7 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_radeon_depth_clear_t *depth_clear = &dev_priv->depth_clear;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	unsigned int flags = clear->flags;
 	u32 rb3d_cntl = 0, rb3d_stencilrefmask = 0;
 	int i;
@@ -759,7 +870,7 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 
 	dev_priv->stats.clears++;
 
-	if (dev_priv->page_flipping && dev_priv->current_page == 1) {
+	if (dev_priv->sarea_priv->pfCurrentPage == 1) {
 		unsigned int tmp = flags;
 
 		flags &= ~(RADEON_FRONT | RADEON_BACK);
@@ -793,7 +904,7 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 			int w = pbox[i].x2 - x;
 			int h = pbox[i].y2 - y;
 
-			DRM_DEBUG("dispatch clear %d,%d-%d,%d flags 0x%x\n",
+			DRM_DEBUG("%d,%d-%d,%d flags 0x%x\n",
 				  x, y, w, h, flags);
 
 			if (flags & RADEON_FRONT) {
@@ -864,7 +975,7 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 		 */
 		dev_priv->sarea_priv->ctx_owner = 0;
 
-		if ((dev_priv->flags & CHIP_HAS_HIERZ)
+		if ((dev_priv->flags & RADEON_HAS_HIERZ)
 		    && (flags & RADEON_USE_HIERZ)) {
 			/* FIXME : reverse engineer that for Rx00 cards */
 			/* FIXME : the mask supposedly contains low-res z values. So can't set
@@ -909,8 +1020,8 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 		for (i = 0; i < nbox; i++) {
 			int tileoffset, nrtilesx, nrtilesy, j;
 			/* it looks like r200 needs rv-style clears, at least if hierz is not enabled? */
-			if ((dev_priv->flags & CHIP_HAS_HIERZ)
-			    && !(dev_priv->microcode_version == UCODE_R200)) {
+			if ((dev_priv->flags & RADEON_HAS_HIERZ)
+			    && (dev_priv->chip_family < CHIP_R200)) {
 				/* FIXME : figure this out for r200 (when hierz is enabled). Or
 				   maybe r200 actually doesn't need to put the low-res z value into
 				   the tile cache like r100, but just needs to clear the hi-level z-buffer?
@@ -939,7 +1050,8 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 					ADVANCE_RING();
 					tileoffset += depthpixperline >> 6;
 				}
-			} else if (dev_priv->microcode_version == UCODE_R200) {
+			} else if ((dev_priv->chip_family >= CHIP_R200) &&
+				   (dev_priv->chip_family <= CHIP_RV280)) {
 				/* works for rv250. */
 				/* find first macro tile (8x2 4x4 z-pixels on rv250) */
 				tileoffset =
@@ -993,8 +1105,9 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 		}
 
 		/* TODO don't always clear all hi-level z tiles */
-		if ((dev_priv->flags & CHIP_HAS_HIERZ)
-		    && (dev_priv->microcode_version == UCODE_R200)
+		if ((dev_priv->flags & RADEON_HAS_HIERZ)
+		    && ((dev_priv->chip_family >= CHIP_R200) &&
+			(dev_priv->chip_family <= CHIP_RV280))
 		    && (flags & RADEON_USE_HIERZ))
 			/* r100 and cards without hierarchical z-buffer have no high-level z-buffer */
 			/* FIXME : the mask supposedly contains low-res z values. So can't set
@@ -1014,8 +1127,9 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 	 * rendering a quad into just those buffers.  Thus, we have to
 	 * make sure the 3D engine is configured correctly.
 	 */
-	else if ((dev_priv->microcode_version == UCODE_R200) &&
-		(flags & (RADEON_DEPTH | RADEON_STENCIL))) {
+	else if ((dev_priv->chip_family >= CHIP_R200) &&
+		 (dev_priv->chip_family <= CHIP_RV280) &&
+		 (flags & (RADEON_DEPTH | RADEON_STENCIL))) {
 
 		int tempPP_CNTL;
 		int tempRE_CNTL;
@@ -1233,12 +1347,12 @@ static void radeon_cp_dispatch_clear(drm_device_t * dev,
 	ADVANCE_RING();
 }
 
-static void radeon_cp_dispatch_swap(drm_device_t * dev)
+static void radeon_cp_dispatch_swap(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	int nbox = sarea_priv->nbox;
-	drm_clip_rect_t *pbox = sarea_priv->boxes;
+	struct drm_clip_rect *pbox = sarea_priv->boxes;
 	int i;
 	RING_LOCALS;
 	DRM_DEBUG("\n");
@@ -1263,11 +1377,11 @@ static void radeon_cp_dispatch_swap(drm_device_t * dev)
 		int w = pbox[i].x2 - x;
 		int h = pbox[i].y2 - y;
 
-		DRM_DEBUG("dispatch swap %d,%d-%d,%d\n", x, y, w, h);
+		DRM_DEBUG("%d,%d-%d,%d\n", x, y, w, h);
 
-		BEGIN_RING(7);
+		BEGIN_RING(9);
 
-		OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 5));
+		OUT_RING(CP_PACKET0(RADEON_DP_GUI_MASTER_CNTL, 0));
 		OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
 			 RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 			 RADEON_GMC_BRUSH_NONE |
@@ -1279,7 +1393,8 @@ static void radeon_cp_dispatch_swap(drm_device_t * dev)
 
 		/* Make this work even if front & back are flipped:
 		 */
-		if (dev_priv->current_page == 0) {
+		OUT_RING(CP_PACKET0(RADEON_SRC_PITCH_OFFSET, 1));
+		if (dev_priv->sarea_priv->pfCurrentPage == 0) {
 			OUT_RING(dev_priv->back_pitch_offset);
 			OUT_RING(dev_priv->front_pitch_offset);
 		} else {
@@ -1287,6 +1402,7 @@ static void radeon_cp_dispatch_swap(drm_device_t * dev)
 			OUT_RING(dev_priv->back_pitch_offset);
 		}
 
+		OUT_RING(CP_PACKET0(RADEON_SRC_X_Y, 2));
 		OUT_RING((x << 16) | y);
 		OUT_RING((x << 16) | y);
 		OUT_RING((w << 16) | h);
@@ -1308,16 +1424,15 @@ static void radeon_cp_dispatch_swap(drm_device_t * dev)
 	ADVANCE_RING();
 }
 
-static void radeon_cp_dispatch_flip(drm_device_t * dev)
+static void radeon_cp_dispatch_flip(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_sarea_t *sarea = (drm_sarea_t *) dev_priv->sarea->handle;
-	int offset = (dev_priv->current_page == 1)
+	struct drm_sarea *sarea = (struct drm_sarea *) dev_priv->sarea->handle;
+	int offset = (dev_priv->sarea_priv->pfCurrentPage == 1)
 	    ? dev_priv->front_offset : dev_priv->back_offset;
 	RING_LOCALS;
-	DRM_DEBUG("%s: page=%d pfCurrentPage=%d\n",
-		  __FUNCTION__,
-		  dev_priv->current_page, dev_priv->sarea_priv->pfCurrentPage);
+	DRM_DEBUG("pfCurrentPage=%d\n",
+		  dev_priv->sarea_priv->pfCurrentPage);
 
 	/* Do some trivial performance monitoring...
 	 */
@@ -1345,8 +1460,8 @@ static void radeon_cp_dispatch_flip(drm_device_t * dev)
 	 * performing the swapbuffer ioctl.
 	 */
 	dev_priv->sarea_priv->last_frame++;
-	dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page =
-	    1 - dev_priv->current_page;
+	dev_priv->sarea_priv->pfCurrentPage =
+		1 - dev_priv->sarea_priv->pfCurrentPage;
 
 	BEGIN_RING(2);
 
@@ -1387,8 +1502,8 @@ typedef struct {
 	unsigned int vc_format;
 } drm_radeon_tcl_prim_t;
 
-static void radeon_cp_dispatch_vertex(drm_device_t * dev,
-				      drm_buf_t * buf,
+static void radeon_cp_dispatch_vertex(struct drm_device * dev,
+				      struct drm_buf * buf,
 				      drm_radeon_tcl_prim_t * prim)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -1433,7 +1548,7 @@ static void radeon_cp_dispatch_vertex(drm_device_t * dev,
 	} while (i < nbox);
 }
 
-static void radeon_cp_discard_buffer(drm_device_t * dev, drm_buf_t * buf)
+static void radeon_cp_discard_buffer(struct drm_device * dev, struct drm_buf * buf)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
@@ -1450,12 +1565,12 @@ static void radeon_cp_discard_buffer(drm_device_t * dev, drm_buf_t * buf)
 	buf->used = 0;
 }
 
-static void radeon_cp_dispatch_indirect(drm_device_t * dev,
-					drm_buf_t * buf, int start, int end)
+static void radeon_cp_dispatch_indirect(struct drm_device * dev,
+					struct drm_buf * buf, int start, int end)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
-	DRM_DEBUG("indirect: buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
+	DRM_DEBUG("buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
 
 	if (start != end) {
 		int offset = (dev_priv->gart_buffers_offset
@@ -1484,8 +1599,8 @@ static void radeon_cp_dispatch_indirect(drm_device_t * dev,
 	}
 }
 
-static void radeon_cp_dispatch_indices(drm_device_t * dev,
-				       drm_buf_t * elt_buf,
+static void radeon_cp_dispatch_indices(struct drm_device * dev,
+				       struct drm_buf * elt_buf,
 				       drm_radeon_tcl_prim_t * prim)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -1542,14 +1657,13 @@ static void radeon_cp_dispatch_indices(drm_device_t * dev,
 
 #define RADEON_MAX_TEXTURE_SIZE RADEON_BUFFER_SIZE
 
-static int radeon_cp_dispatch_texture(DRMFILE filp,
-				      drm_device_t * dev,
+static int radeon_cp_dispatch_texture(struct drm_device * dev,
+				      struct drm_file *file_priv,
 				      drm_radeon_texture_t * tex,
 				      drm_radeon_tex_image_t * image)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	u32 format;
 	u32 *buffer;
 	const u8 __user *data;
@@ -1557,14 +1671,12 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 	u32 height;
 	int i;
 	u32 texpitch, microtile;
-	u32 offset;
+	u32 offset, byte_offset;
 	RING_LOCALS;
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	if (radeon_check_and_fixup_offset(dev_priv, filp_priv, &tex->offset)) {
+	if (radeon_check_and_fixup_offset(dev_priv, file_priv, &tex->offset)) {
 		DRM_ERROR("Invalid destination offset\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->stats.boxes |= RADEON_BOX_TEXTURE_LOAD;
@@ -1607,11 +1719,11 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 		break;
 	default:
 		DRM_ERROR("invalid texture format %d\n", tex->format);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	spitch = blit_width >> 6;
 	if (spitch == 0 && image->height > 1)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	texpitch = tex->pitch;
 	if ((texpitch << 22) & RADEON_DST_TILE_MICRO) {
@@ -1624,6 +1736,13 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 	} else
 		microtile = 0;
 
+	/* this might fail for zero-sized uploads - are those illegal? */
+	if (!radeon_check_offset(dev_priv, tex->offset + image->height *
+				blit_width - 1)) {
+		DRM_ERROR("Invalid final destination offset\n");
+		return -EINVAL;
+	}
+
 	DRM_DEBUG("tex=%dx%d blit=%d\n", tex_width, tex->height, blit_width);
 
 	do {
@@ -1654,10 +1773,10 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 			buf = radeon_freelist_get(dev);
 		}
 		if (!buf) {
-			DRM_DEBUG("radeon_cp_dispatch_texture: EAGAIN\n");
+			DRM_DEBUG("EAGAIN\n");
 			if (DRM_COPY_TO_USER(tex->image, image, sizeof(*image)))
-				return DRM_ERR(EFAULT);
-			return DRM_ERR(EAGAIN);
+				return -EFAULT;
+			return -EAGAIN;
 		}
 
 		/* Dispatch the indirect buffer.
@@ -1670,7 +1789,7 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 	do { \
 		if (DRM_COPY_FROM_USER(_buf, _data, (_width))) {\
 			DRM_ERROR("EFAULT on pad, %d bytes\n", (_width)); \
-			return DRM_ERR(EFAULT); \
+			return -EFAULT; \
 		} \
 	} while(0)
 
@@ -1737,7 +1856,8 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 		}
 
 #undef RADEON_COPY_MT
-		buf->filp = filp;
+		byte_offset = (image->y & ~2047) * blit_width;
+		buf->file_priv = file_priv;
 		buf->used = size;
 		offset = dev_priv->gart_buffers_offset + buf->offset;
 		BEGIN_RING(9);
@@ -1751,12 +1871,13 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 			 RADEON_DP_SRC_SOURCE_MEMORY |
 			 RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
 		OUT_RING((spitch << 22) | (offset >> 10));
-		OUT_RING((texpitch << 22) | (tex->offset >> 10));
+		OUT_RING((texpitch << 22) | ((tex->offset >> 10) + (byte_offset >> 10)));
 		OUT_RING(0);
-		OUT_RING((image->x << 16) | image->y);
+		OUT_RING((image->x << 16) | (image->y % 2048));
 		OUT_RING((image->width << 16) | height);
 		RADEON_WAIT_UNTIL_2D_IDLE();
 		ADVANCE_RING();
+		COMMIT_RING();
 
 		radeon_cp_discard_buffer(dev, buf);
 
@@ -1774,10 +1895,12 @@ static int radeon_cp_dispatch_texture(DRMFILE filp,
 	RADEON_FLUSH_CACHE();
 	RADEON_WAIT_UNTIL_2D_IDLE();
 	ADVANCE_RING();
+	COMMIT_RING();
+
 	return 0;
 }
 
-static void radeon_cp_dispatch_stipple(drm_device_t * dev, u32 * stipple)
+static void radeon_cp_dispatch_stipple(struct drm_device * dev, u32 * stipple)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	int i;
@@ -1825,7 +1948,8 @@ static void radeon_apply_surface_regs(int surf_index,
  * not always be available.
  */
 static int alloc_surface(drm_radeon_surface_alloc_t *new,
-			 drm_radeon_private_t *dev_priv, DRMFILE filp)
+			 drm_radeon_private_t *dev_priv,
+			 struct drm_file *file_priv)
 {
 	struct radeon_virt_surface *s;
 	int i;
@@ -1855,7 +1979,7 @@ static int alloc_surface(drm_radeon_surface_alloc_t *new,
 
 	/* find a virtual surface */
 	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++)
-		if (dev_priv->virt_surfaces[i].filp == 0)
+		if (dev_priv->virt_surfaces[i].file_priv == 0)
 			break;
 	if (i == 2 * RADEON_MAX_SURFACES) {
 		return -1;
@@ -1873,7 +1997,7 @@ static int alloc_surface(drm_radeon_surface_alloc_t *new,
 			s->lower = new_lower;
 			s->upper = new_upper;
 			s->flags = new->flags;
-			s->filp = filp;
+			s->file_priv = file_priv;
 			dev_priv->surfaces[i].refcount++;
 			dev_priv->surfaces[i].lower = s->lower;
 			radeon_apply_surface_regs(s->surface_index, dev_priv);
@@ -1889,7 +2013,7 @@ static int alloc_surface(drm_radeon_surface_alloc_t *new,
 			s->lower = new_lower;
 			s->upper = new_upper;
 			s->flags = new->flags;
-			s->filp = filp;
+			s->file_priv = file_priv;
 			dev_priv->surfaces[i].refcount++;
 			dev_priv->surfaces[i].upper = s->upper;
 			radeon_apply_surface_regs(s->surface_index, dev_priv);
@@ -1905,7 +2029,7 @@ static int alloc_surface(drm_radeon_surface_alloc_t *new,
 			s->lower = new_lower;
 			s->upper = new_upper;
 			s->flags = new->flags;
-			s->filp = filp;
+			s->file_priv = file_priv;
 			dev_priv->surfaces[i].refcount = 1;
 			dev_priv->surfaces[i].lower = s->lower;
 			dev_priv->surfaces[i].upper = s->upper;
@@ -1919,7 +2043,8 @@ static int alloc_surface(drm_radeon_surface_alloc_t *new,
 	return -1;
 }
 
-static int free_surface(DRMFILE filp, drm_radeon_private_t * dev_priv,
+static int free_surface(struct drm_file *file_priv,
+			drm_radeon_private_t * dev_priv,
 			int lower)
 {
 	struct radeon_virt_surface *s;
@@ -1927,8 +2052,9 @@ static int free_surface(DRMFILE filp, drm_radeon_private_t * dev_priv,
 	/* find the virtual surface */
 	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
 		s = &(dev_priv->virt_surfaces[i]);
-		if (s->filp) {
-			if ((lower == s->lower) && (filp == s->filp)) {
+		if (s->file_priv) {
+			if ((lower == s->lower) && (file_priv == s->file_priv))
+			{
 				if (dev_priv->surfaces[s->surface_index].
 				    lower == s->lower)
 					dev_priv->surfaces[s->surface_index].
@@ -1944,7 +2070,7 @@ static int free_surface(DRMFILE filp, drm_radeon_private_t * dev_priv,
 				    refcount == 0)
 					dev_priv->surfaces[s->surface_index].
 					    flags = 0;
-				s->filp = NULL;
+				s->file_priv = NULL;
 				radeon_apply_surface_regs(s->surface_index,
 							  dev_priv);
 				return 0;
@@ -1954,13 +2080,13 @@ static int free_surface(DRMFILE filp, drm_radeon_private_t * dev_priv,
 	return 1;
 }
 
-static void radeon_surfaces_release(DRMFILE filp,
+static void radeon_surfaces_release(struct drm_file *file_priv,
 				    drm_radeon_private_t * dev_priv)
 {
 	int i;
 	for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
-		if (dev_priv->virt_surfaces[i].filp == filp)
-			free_surface(filp, dev_priv,
+		if (dev_priv->virt_surfaces[i].file_priv == file_priv)
+			free_surface(file_priv, dev_priv,
 				     dev_priv->virt_surfaces[i].lower);
 	}
 }
@@ -1968,71 +2094,58 @@ static void radeon_surfaces_release(DRMFILE filp,
 /* ================================================================
  * IOCTL functions
  */
-static int radeon_surface_alloc(DRM_IOCTL_ARGS)
+static int radeon_surface_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_surface_alloc_t alloc;
+	drm_radeon_surface_alloc_t *alloc = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(alloc,
-				 (drm_radeon_surface_alloc_t __user *) data,
-				 sizeof(alloc));
-
-	if (alloc_surface(&alloc, dev_priv, filp) == -1)
-		return DRM_ERR(EINVAL);
+	if (alloc_surface(alloc, dev_priv, file_priv) == -1)
+		return -EINVAL;
 	else
 		return 0;
 }
 
-static int radeon_surface_free(DRM_IOCTL_ARGS)
+static int radeon_surface_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_surface_free_t memfree;
+	drm_radeon_surface_free_t *memfree = data;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(memfree, (drm_radeon_surface_free_t __user *) data,
-				 sizeof(memfree));
-
-	if (free_surface(filp, dev_priv, memfree.address))
-		return DRM_ERR(EINVAL);
+	if (free_surface(file_priv, dev_priv, memfree->address))
+		return -EINVAL;
 	else
 		return 0;
 }
 
-static int radeon_cp_clear(DRM_IOCTL_ARGS)
+static int radeon_cp_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	drm_radeon_clear_t clear;
+	drm_radeon_clear_t *clear = data;
 	drm_radeon_clear_rect_t depth_boxes[RADEON_NR_SAREA_CLIPRECTS];
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(clear, (drm_radeon_clear_t __user *) data,
-				 sizeof(clear));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
 	if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
 		sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
 
-	if (DRM_COPY_FROM_USER(&depth_boxes, clear.depth_boxes,
+	if (DRM_COPY_FROM_USER(&depth_boxes, clear->depth_boxes,
 			       sarea_priv->nbox * sizeof(depth_boxes[0])))
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
-	radeon_cp_dispatch_clear(dev, &clear, depth_boxes);
+	radeon_cp_dispatch_clear(dev, clear, depth_boxes);
 
 	COMMIT_RING();
 	return 0;
@@ -2040,7 +2153,7 @@ static int radeon_cp_clear(DRM_IOCTL_ARGS)
 
 /* Not sure why this isn't set all the time:
  */
-static int radeon_do_init_pageflip(drm_device_t * dev)
+static int radeon_do_init_pageflip(struct drm_device * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
@@ -2058,37 +2171,22 @@ static int radeon_do_init_pageflip(drm_device_t * dev)
 	ADVANCE_RING();
 
 	dev_priv->page_flipping = 1;
-	dev_priv->current_page = 0;
-	dev_priv->sarea_priv->pfCurrentPage = dev_priv->current_page;
-
-	return 0;
-}
-
-/* Called whenever a client dies, from drm_release.
- * NOTE:  Lock isn't necessarily held when this is called!
- */
-static int radeon_do_cleanup_pageflip(drm_device_t * dev)
-{
-	drm_radeon_private_t *dev_priv = dev->dev_private;
-	DRM_DEBUG("\n");
 
-	if (dev_priv->current_page != 0)
-		radeon_cp_dispatch_flip(dev);
+	if (dev_priv->sarea_priv->pfCurrentPage != 1)
+		dev_priv->sarea_priv->pfCurrentPage = 0;
 
-	dev_priv->page_flipping = 0;
 	return 0;
 }
 
 /* Swapping and flipping are different operations, need different ioctls.
  * They can & should be intermixed to support multiple 3d windows.
  */
-static int radeon_cp_flip(DRM_IOCTL_ARGS)
+static int radeon_cp_flip(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -2101,14 +2199,13 @@ static int radeon_cp_flip(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_swap(DRM_IOCTL_ARGS)
+static int radeon_cp_swap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -2122,71 +2219,64 @@ static int radeon_cp_swap(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_vertex(DRM_IOCTL_ARGS)
+static int radeon_cp_vertex(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
 	drm_radeon_sarea_t *sarea_priv;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
-	drm_radeon_vertex_t vertex;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
+	drm_radeon_vertex_t *vertex = data;
 	drm_radeon_tcl_prim_t prim;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
 	sarea_priv = dev_priv->sarea_priv;
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(vertex, (drm_radeon_vertex_t __user *) data,
-				 sizeof(vertex));
-
 	DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
-		  DRM_CURRENTPID, vertex.idx, vertex.count, vertex.discard);
+		  DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard);
 
-	if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
+	if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  vertex.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  vertex->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
-	if (vertex.prim < 0 || vertex.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
-		DRM_ERROR("buffer prim %d\n", vertex.prim);
-		return DRM_ERR(EINVAL);
+	if (vertex->prim < 0 || vertex->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
+		DRM_ERROR("buffer prim %d\n", vertex->prim);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf = dma->buflist[vertex.idx];
+	buf = dma->buflist[vertex->idx];
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", vertex.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", vertex->idx);
+		return -EINVAL;
 	}
 
 	/* Build up a prim_t record:
 	 */
-	if (vertex.count) {
-		buf->used = vertex.count;	/* not used? */
+	if (vertex->count) {
+		buf->used = vertex->count;	/* not used? */
 
 		if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
-			if (radeon_emit_state(dev_priv, filp_priv,
+			if (radeon_emit_state(dev_priv, file_priv,
 					      &sarea_priv->context_state,
 					      sarea_priv->tex_state,
 					      sarea_priv->dirty)) {
 				DRM_ERROR("radeon_emit_state failed\n");
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 
 			sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
@@ -2196,15 +2286,15 @@ static int radeon_cp_vertex(DRM_IOCTL_ARGS)
 		}
 
 		prim.start = 0;
-		prim.finish = vertex.count;	/* unused */
-		prim.prim = vertex.prim;
-		prim.numverts = vertex.count;
+		prim.finish = vertex->count;	/* unused */
+		prim.prim = vertex->prim;
+		prim.numverts = vertex->count;
 		prim.vc_format = dev_priv->sarea_priv->vc_format;
 
 		radeon_cp_dispatch_vertex(dev, buf, &prim);
 	}
 
-	if (vertex.discard) {
+	if (vertex->discard) {
 		radeon_cp_discard_buffer(dev, buf);
 	}
 
@@ -2212,80 +2302,74 @@ static int radeon_cp_vertex(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_indices(DRM_IOCTL_ARGS)
+static int radeon_cp_indices(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
 	drm_radeon_sarea_t *sarea_priv;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
-	drm_radeon_indices_t elts;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
+	drm_radeon_indices_t *elts = data;
 	drm_radeon_tcl_prim_t prim;
 	int count;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 	sarea_priv = dev_priv->sarea_priv;
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(elts, (drm_radeon_indices_t __user *) data,
-				 sizeof(elts));
-
 	DRM_DEBUG("pid=%d index=%d start=%d end=%d discard=%d\n",
-		  DRM_CURRENTPID, elts.idx, elts.start, elts.end, elts.discard);
+		  DRM_CURRENTPID, elts->idx, elts->start, elts->end,
+		  elts->discard);
 
-	if (elts.idx < 0 || elts.idx >= dma->buf_count) {
+	if (elts->idx < 0 || elts->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  elts.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  elts->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
-	if (elts.prim < 0 || elts.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
-		DRM_ERROR("buffer prim %d\n", elts.prim);
-		return DRM_ERR(EINVAL);
+	if (elts->prim < 0 || elts->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
+		DRM_ERROR("buffer prim %d\n", elts->prim);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf = dma->buflist[elts.idx];
+	buf = dma->buflist[elts->idx];
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", elts.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", elts->idx);
+		return -EINVAL;
 	}
 
-	count = (elts.end - elts.start) / sizeof(u16);
-	elts.start -= RADEON_INDEX_PRIM_OFFSET;
+	count = (elts->end - elts->start) / sizeof(u16);
+	elts->start -= RADEON_INDEX_PRIM_OFFSET;
 
-	if (elts.start & 0x7) {
-		DRM_ERROR("misaligned buffer 0x%x\n", elts.start);
-		return DRM_ERR(EINVAL);
+	if (elts->start & 0x7) {
+		DRM_ERROR("misaligned buffer 0x%x\n", elts->start);
+		return -EINVAL;
 	}
-	if (elts.start < buf->used) {
-		DRM_ERROR("no header 0x%x - 0x%x\n", elts.start, buf->used);
-		return DRM_ERR(EINVAL);
+	if (elts->start < buf->used) {
+		DRM_ERROR("no header 0x%x - 0x%x\n", elts->start, buf->used);
+		return -EINVAL;
 	}
 
-	buf->used = elts.end;
+	buf->used = elts->end;
 
 	if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
-		if (radeon_emit_state(dev_priv, filp_priv,
+		if (radeon_emit_state(dev_priv, file_priv,
 				      &sarea_priv->context_state,
 				      sarea_priv->tex_state,
 				      sarea_priv->dirty)) {
 			DRM_ERROR("radeon_emit_state failed\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 
 		sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
@@ -2296,15 +2380,15 @@ static int radeon_cp_indices(DRM_IOCTL_ARGS)
 
 	/* Build up a prim_t record:
 	 */
-	prim.start = elts.start;
-	prim.finish = elts.end;
-	prim.prim = elts.prim;
+	prim.start = elts->start;
+	prim.finish = elts->end;
+	prim.prim = elts->prim;
 	prim.offset = 0;	/* offset from start of dma buffers */
 	prim.numverts = RADEON_MAX_VB_VERTS;	/* duh */
 	prim.vc_format = dev_priv->sarea_priv->vc_format;
 
 	radeon_cp_dispatch_indices(dev, buf, &prim);
-	if (elts.discard) {
+	if (elts->discard) {
 		radeon_cp_discard_buffer(dev, buf);
 	}
 
@@ -2312,52 +2396,43 @@ static int radeon_cp_indices(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_texture(DRM_IOCTL_ARGS)
+static int radeon_cp_texture(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_texture_t tex;
+	drm_radeon_texture_t *tex = data;
 	drm_radeon_tex_image_t image;
 	int ret;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(tex, (drm_radeon_texture_t __user *) data,
-				 sizeof(tex));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (tex.image == NULL) {
+	if (tex->image == NULL) {
 		DRM_ERROR("null texture image!\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (DRM_COPY_FROM_USER(&image,
-			       (drm_radeon_tex_image_t __user *) tex.image,
+			       (drm_radeon_tex_image_t __user *) tex->image,
 			       sizeof(image)))
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	ret = radeon_cp_dispatch_texture(filp, dev, &tex, &image);
+	ret = radeon_cp_dispatch_texture(dev, file_priv, tex, &image);
 
-	COMMIT_RING();
 	return ret;
 }
 
-static int radeon_cp_stipple(DRM_IOCTL_ARGS)
+static int radeon_cp_stipple(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_stipple_t stipple;
+	drm_radeon_stipple_t *stipple = data;
 	u32 mask[32];
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(stipple, (drm_radeon_stipple_t __user *) data,
-				 sizeof(stipple));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	if (DRM_COPY_FROM_USER(&mask, stipple.mask, 32 * sizeof(u32)))
-		return DRM_ERR(EFAULT);
+	if (DRM_COPY_FROM_USER(&mask, stipple->mask, 32 * sizeof(u32)))
+		return -EFAULT;
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 
@@ -2367,57 +2442,53 @@ static int radeon_cp_stipple(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_indirect(DRM_IOCTL_ARGS)
+static int radeon_cp_indirect(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
-	drm_radeon_indirect_t indirect;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
+	drm_radeon_indirect_t *indirect = data;
 	RING_LOCALS;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(indirect,
-				 (drm_radeon_indirect_t __user *) data,
-				 sizeof(indirect));
+	DRM_DEBUG("idx=%d s=%d e=%d d=%d\n",
+		  indirect->idx, indirect->start, indirect->end,
+		  indirect->discard);
 
-	DRM_DEBUG("indirect: idx=%d s=%d e=%d d=%d\n",
-		  indirect.idx, indirect.start, indirect.end, indirect.discard);
-
-	if (indirect.idx < 0 || indirect.idx >= dma->buf_count) {
+	if (indirect->idx < 0 || indirect->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  indirect.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  indirect->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
 
-	buf = dma->buflist[indirect.idx];
+	buf = dma->buflist[indirect->idx];
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", indirect.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", indirect->idx);
+		return -EINVAL;
 	}
 
-	if (indirect.start < buf->used) {
+	if (indirect->start < buf->used) {
 		DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
-			  indirect.start, buf->used);
-		return DRM_ERR(EINVAL);
+			  indirect->start, buf->used);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf->used = indirect.end;
+	buf->used = indirect->end;
 
 	/* Wait for the 3D stream to idle before the indirect buffer
 	 * containing 2D acceleration commands is processed.
@@ -2432,8 +2503,8 @@ static int radeon_cp_indirect(DRM_IOCTL_ARGS)
 	 * X server.  This is insecure and is thus only available to
 	 * privileged clients.
 	 */
-	radeon_cp_dispatch_indirect(dev, buf, indirect.start, indirect.end);
-	if (indirect.discard) {
+	radeon_cp_dispatch_indirect(dev, buf, indirect->start, indirect->end);
+	if (indirect->discard) {
 		radeon_cp_discard_buffer(dev, buf);
 	}
 
@@ -2441,78 +2512,71 @@ static int radeon_cp_indirect(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
+static int radeon_cp_vertex2(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
 	drm_radeon_sarea_t *sarea_priv;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
-	drm_radeon_vertex2_t vertex;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
+	drm_radeon_vertex2_t *vertex = data;
 	int i;
 	unsigned char laststate;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
 	sarea_priv = dev_priv->sarea_priv;
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(vertex, (drm_radeon_vertex2_t __user *) data,
-				 sizeof(vertex));
-
 	DRM_DEBUG("pid=%d index=%d discard=%d\n",
-		  DRM_CURRENTPID, vertex.idx, vertex.discard);
+		  DRM_CURRENTPID, vertex->idx, vertex->discard);
 
-	if (vertex.idx < 0 || vertex.idx >= dma->buf_count) {
+	if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
 		DRM_ERROR("buffer index %d (of %d max)\n",
-			  vertex.idx, dma->buf_count - 1);
-		return DRM_ERR(EINVAL);
+			  vertex->idx, dma->buf_count - 1);
+		return -EINVAL;
 	}
 
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	buf = dma->buflist[vertex.idx];
+	buf = dma->buflist[vertex->idx];
 
-	if (buf->filp != filp) {
+	if (buf->file_priv != file_priv) {
 		DRM_ERROR("process %d using buffer owned by %p\n",
-			  DRM_CURRENTPID, buf->filp);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, buf->file_priv);
+		return -EINVAL;
 	}
 
 	if (buf->pending) {
-		DRM_ERROR("sending pending buffer %d\n", vertex.idx);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("sending pending buffer %d\n", vertex->idx);
+		return -EINVAL;
 	}
 
 	if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
-	for (laststate = 0xff, i = 0; i < vertex.nr_prims; i++) {
+	for (laststate = 0xff, i = 0; i < vertex->nr_prims; i++) {
 		drm_radeon_prim_t prim;
 		drm_radeon_tcl_prim_t tclprim;
 
-		if (DRM_COPY_FROM_USER(&prim, &vertex.prim[i], sizeof(prim)))
-			return DRM_ERR(EFAULT);
+		if (DRM_COPY_FROM_USER(&prim, &vertex->prim[i], sizeof(prim)))
+			return -EFAULT;
 
 		if (prim.stateidx != laststate) {
 			drm_radeon_state_t state;
 
 			if (DRM_COPY_FROM_USER(&state,
-					       &vertex.state[prim.stateidx],
+					       &vertex->state[prim.stateidx],
 					       sizeof(state)))
-				return DRM_ERR(EFAULT);
+				return -EFAULT;
 
-			if (radeon_emit_state2(dev_priv, filp_priv, &state)) {
+			if (radeon_emit_state2(dev_priv, file_priv, &state)) {
 				DRM_ERROR("radeon_emit_state2 failed\n");
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 
 			laststate = prim.stateidx;
@@ -2539,7 +2603,7 @@ static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
 			sarea_priv->nbox = 0;
 	}
 
-	if (vertex.discard) {
+	if (vertex->discard) {
 		radeon_cp_discard_buffer(dev, buf);
 	}
 
@@ -2548,7 +2612,7 @@ static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
 }
 
 static int radeon_emit_packets(drm_radeon_private_t * dev_priv,
-			       drm_file_t * filp_priv,
+			       struct drm_file *file_priv,
 			       drm_radeon_cmd_header_t header,
 			       drm_radeon_kcmd_buffer_t *cmdbuf)
 {
@@ -2558,19 +2622,19 @@ static int radeon_emit_packets(drm_radeon_private_t * dev_priv,
 	RING_LOCALS;
 
 	if (id >= RADEON_MAX_STATE_PACKETS)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	sz = packet[id].len;
 	reg = packet[id].start;
 
 	if (sz * sizeof(int) > cmdbuf->bufsz) {
 		DRM_ERROR("Packet size provided larger than data provided\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (radeon_check_and_fixup_packets(dev_priv, filp_priv, id, data)) {
+	if (radeon_check_and_fixup_packets(dev_priv, file_priv, id, data)) {
 		DRM_ERROR("Packet verification failed\n");
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	BEGIN_RING(sz + 1);
@@ -2658,7 +2722,7 @@ static __inline__ int radeon_emit_veclinear(drm_radeon_private_t *dev_priv,
 	if (!sz)
 		return 0;
 	if (sz * 4 > cmdbuf->bufsz)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
 	BEGIN_RING(5 + sz);
 	OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
@@ -2673,8 +2737,8 @@ static __inline__ int radeon_emit_veclinear(drm_radeon_private_t *dev_priv,
 	return 0;
 }
 
-static int radeon_emit_packet3(drm_device_t * dev,
-			       drm_file_t * filp_priv,
+static int radeon_emit_packet3(struct drm_device * dev,
+			       struct drm_file *file_priv,
 			       drm_radeon_kcmd_buffer_t *cmdbuf)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -2684,7 +2748,7 @@ static int radeon_emit_packet3(drm_device_t * dev,
 
 	DRM_DEBUG("\n");
 
-	if ((ret = radeon_check_and_fixup_packet3(dev_priv, filp_priv,
+	if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
 						  cmdbuf, &cmdsz))) {
 		DRM_ERROR("Packet verification failed\n");
 		return ret;
@@ -2699,22 +2763,22 @@ static int radeon_emit_packet3(drm_device_t * dev,
 	return 0;
 }
 
-static int radeon_emit_packet3_cliprect(drm_device_t *dev,
-					drm_file_t *filp_priv,
+static int radeon_emit_packet3_cliprect(struct drm_device *dev,
+					struct drm_file *file_priv,
 					drm_radeon_kcmd_buffer_t *cmdbuf,
 					int orig_nbox)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_clip_rect_t box;
+	struct drm_clip_rect box;
 	unsigned int cmdsz;
 	int ret;
-	drm_clip_rect_t __user *boxes = cmdbuf->boxes;
+	struct drm_clip_rect __user *boxes = cmdbuf->boxes;
 	int i = 0;
 	RING_LOCALS;
 
 	DRM_DEBUG("\n");
 
-	if ((ret = radeon_check_and_fixup_packet3(dev_priv, filp_priv,
+	if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
 						  cmdbuf, &cmdsz))) {
 		DRM_ERROR("Packet verification failed\n");
 		return ret;
@@ -2726,7 +2790,7 @@ static int radeon_emit_packet3_cliprect(drm_device_t *dev,
 	do {
 		if (i < cmdbuf->nbox) {
 			if (DRM_COPY_FROM_USER(&box, &boxes[i], sizeof(box)))
-				return DRM_ERR(EFAULT);
+				return -EFAULT;
 			/* FIXME The second and subsequent times round
 			 * this loop, send a WAIT_UNTIL_3D_IDLE before
 			 * calling emit_clip_rect(). This fixes a
@@ -2761,12 +2825,12 @@ static int radeon_emit_packet3_cliprect(drm_device_t *dev,
 	return 0;
 }
 
-static int radeon_emit_wait(drm_device_t * dev, int flags)
+static int radeon_emit_wait(struct drm_device * dev, int flags)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
-	DRM_DEBUG("%s: %x\n", __FUNCTION__, flags);
+	DRM_DEBUG("%x\n", flags);
 	switch (flags) {
 	case RADEON_WAIT_2D:
 		BEGIN_RING(2);
@@ -2784,67 +2848,59 @@ static int radeon_emit_wait(drm_device_t * dev, int flags)
 		ADVANCE_RING();
 		break;
 	default:
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	return 0;
 }
 
-static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
+static int radeon_cp_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf = NULL;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf = NULL;
 	int idx;
-	drm_radeon_kcmd_buffer_t cmdbuf;
+	drm_radeon_kcmd_buffer_t *cmdbuf = data;
 	drm_radeon_cmd_header_t header;
 	int orig_nbox, orig_bufsz;
 	char *kbuf = NULL;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(cmdbuf,
-				 (drm_radeon_kcmd_buffer_t __user *) data,
-				 sizeof(cmdbuf));
-
 	RING_SPACE_TEST_WITH_RETURN(dev_priv);
 	VB_AGE_TEST_WITH_RETURN(dev_priv);
 
-	if (cmdbuf.bufsz > 64 * 1024 || cmdbuf.bufsz < 0) {
-		return DRM_ERR(EINVAL);
+	if (cmdbuf->bufsz > 64 * 1024 || cmdbuf->bufsz < 0) {
+		return -EINVAL;
 	}
 
 	/* Allocate an in-kernel area and copy in the cmdbuf.  Do this to avoid
 	 * races between checking values and using those values in other code,
 	 * and simply to avoid a lot of function calls to copy in data.
 	 */
-	orig_bufsz = cmdbuf.bufsz;
+	orig_bufsz = cmdbuf->bufsz;
 	if (orig_bufsz != 0) {
-		kbuf = drm_alloc(cmdbuf.bufsz, DRM_MEM_DRIVER);
+		kbuf = drm_alloc(cmdbuf->bufsz, DRM_MEM_DRIVER);
 		if (kbuf == NULL)
-			return DRM_ERR(ENOMEM);
-		if (DRM_COPY_FROM_USER(kbuf, (void __user *)cmdbuf.buf,
-				       cmdbuf.bufsz)) {
+			return -ENOMEM;
+		if (DRM_COPY_FROM_USER(kbuf, (void __user *)cmdbuf->buf,
+				       cmdbuf->bufsz)) {
 			drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		}
-		cmdbuf.buf = kbuf;
+		cmdbuf->buf = kbuf;
 	}
 
-	orig_nbox = cmdbuf.nbox;
+	orig_nbox = cmdbuf->nbox;
 
-	if (dev_priv->microcode_version == UCODE_R300) {
+	if (dev_priv->chip_family >= CHIP_R300) {
 		int temp;
-		temp = r300_do_cp_cmdbuf(dev, filp, filp_priv, &cmdbuf);
+		temp = r300_do_cp_cmdbuf(dev, file_priv, cmdbuf);
 
 		if (orig_bufsz != 0)
 			drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
@@ -2853,17 +2909,17 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 	}
 
 	/* microcode_version != r300 */
-	while (cmdbuf.bufsz >= sizeof(header)) {
+	while (cmdbuf->bufsz >= sizeof(header)) {
 
-		header.i = *(int *)cmdbuf.buf;
-		cmdbuf.buf += sizeof(header);
-		cmdbuf.bufsz -= sizeof(header);
+		header.i = *(int *)cmdbuf->buf;
+		cmdbuf->buf += sizeof(header);
+		cmdbuf->bufsz -= sizeof(header);
 
 		switch (header.header.cmd_type) {
 		case RADEON_CMD_PACKET:
 			DRM_DEBUG("RADEON_CMD_PACKET\n");
 			if (radeon_emit_packets
-			    (dev_priv, filp_priv, header, &cmdbuf)) {
+			    (dev_priv, file_priv, header, cmdbuf)) {
 				DRM_ERROR("radeon_emit_packets failed\n");
 				goto err;
 			}
@@ -2871,7 +2927,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 
 		case RADEON_CMD_SCALARS:
 			DRM_DEBUG("RADEON_CMD_SCALARS\n");
-			if (radeon_emit_scalars(dev_priv, header, &cmdbuf)) {
+			if (radeon_emit_scalars(dev_priv, header, cmdbuf)) {
 				DRM_ERROR("radeon_emit_scalars failed\n");
 				goto err;
 			}
@@ -2879,7 +2935,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 
 		case RADEON_CMD_VECTORS:
 			DRM_DEBUG("RADEON_CMD_VECTORS\n");
-			if (radeon_emit_vectors(dev_priv, header, &cmdbuf)) {
+			if (radeon_emit_vectors(dev_priv, header, cmdbuf)) {
 				DRM_ERROR("radeon_emit_vectors failed\n");
 				goto err;
 			}
@@ -2895,9 +2951,10 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 			}
 
 			buf = dma->buflist[idx];
-			if (buf->filp != filp || buf->pending) {
+			if (buf->file_priv != file_priv || buf->pending) {
 				DRM_ERROR("bad buffer %p %p %d\n",
-					  buf->filp, filp, buf->pending);
+					  buf->file_priv, file_priv,
+					  buf->pending);
 				goto err;
 			}
 
@@ -2906,7 +2963,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 
 		case RADEON_CMD_PACKET3:
 			DRM_DEBUG("RADEON_CMD_PACKET3\n");
-			if (radeon_emit_packet3(dev, filp_priv, &cmdbuf)) {
+			if (radeon_emit_packet3(dev, file_priv, cmdbuf)) {
 				DRM_ERROR("radeon_emit_packet3 failed\n");
 				goto err;
 			}
@@ -2915,7 +2972,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 		case RADEON_CMD_PACKET3_CLIP:
 			DRM_DEBUG("RADEON_CMD_PACKET3_CLIP\n");
 			if (radeon_emit_packet3_cliprect
-			    (dev, filp_priv, &cmdbuf, orig_nbox)) {
+			    (dev, file_priv, cmdbuf, orig_nbox)) {
 				DRM_ERROR("radeon_emit_packet3_clip failed\n");
 				goto err;
 			}
@@ -2923,7 +2980,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 
 		case RADEON_CMD_SCALARS2:
 			DRM_DEBUG("RADEON_CMD_SCALARS2\n");
-			if (radeon_emit_scalars2(dev_priv, header, &cmdbuf)) {
+			if (radeon_emit_scalars2(dev_priv, header, cmdbuf)) {
 				DRM_ERROR("radeon_emit_scalars2 failed\n");
 				goto err;
 			}
@@ -2938,7 +2995,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 			break;
 		case RADEON_CMD_VECLINEAR:
 			DRM_DEBUG("RADEON_CMD_VECLINEAR\n");
-			if (radeon_emit_veclinear(dev_priv, header, &cmdbuf)) {
+			if (radeon_emit_veclinear(dev_priv, header, cmdbuf)) {
 				DRM_ERROR("radeon_emit_veclinear failed\n");
 				goto err;
 			}
@@ -2947,7 +3004,7 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
 		default:
 			DRM_ERROR("bad cmd_type %d at %p\n",
 				  header.header.cmd_type,
-				  cmdbuf.buf - sizeof(header));
+				  cmdbuf->buf - sizeof(header));
 			goto err;
 		}
 	}
@@ -2962,27 +3019,23 @@ static int radeon_cp_cmdbuf(DRM_IOCTL_ARGS)
       err:
 	if (orig_bufsz != 0)
 		drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
-static int radeon_cp_getparam(DRM_IOCTL_ARGS)
+static int radeon_cp_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_radeon_getparam_t param;
+	drm_radeon_getparam_t *param = data;
 	int value;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_COPY_FROM_USER_IOCTL(param, (drm_radeon_getparam_t __user *) data,
-				 sizeof(param));
-
 	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
 
-	switch (param.param) {
+	switch (param->param) {
 	case RADEON_PARAM_GART_BUFFER_OFFSET:
 		value = dev_priv->gart_buffers_offset;
 		break;
@@ -3029,76 +3082,90 @@ static int radeon_cp_getparam(DRM_IOCTL_ARGS)
 		break;
 	case RADEON_PARAM_SCRATCH_OFFSET:
 		if (!dev_priv->writeback_works)
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		value = RADEON_SCRATCH_REG_OFFSET;
 		break;
-	
+
 	case RADEON_PARAM_CARD_TYPE:
-		if (dev_priv->flags & CHIP_IS_PCIE)
+		if (dev_priv->flags & RADEON_IS_PCIE)
 			value = RADEON_CARD_PCIE;
-		else if (dev_priv->flags & CHIP_IS_AGP)
+		else if (dev_priv->flags & RADEON_IS_AGP)
 			value = RADEON_CARD_AGP;
 		else
 			value = RADEON_CARD_PCI;
 		break;
+	case RADEON_PARAM_VBLANK_CRTC:
+		value = radeon_vblank_crtc_get(dev);
+		break;
+	case RADEON_PARAM_FB_LOCATION:
+		value = radeon_read_fb_location(dev_priv);
+		break;
+	case RADEON_PARAM_NUM_GB_PIPES:
+		value = dev_priv->num_gb_pipes;
+		break;
 	default:
-		DRM_DEBUG( "Invalid parameter %d\n", param.param );
-		return DRM_ERR(EINVAL);
+		DRM_DEBUG( "Invalid parameter %d\n", param->param );
+		return -EINVAL;
 	}
 
-	if (DRM_COPY_TO_USER(param.value, &value, sizeof(int))) {
+	if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
-		return DRM_ERR(EFAULT);
+		return -EFAULT;
 	}
 
 	return 0;
 }
 
-static int radeon_cp_setparam(DRM_IOCTL_ARGS)
+static int radeon_cp_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
-	drm_file_t *filp_priv;
-	drm_radeon_setparam_t sp;
+	drm_radeon_setparam_t *sp = data;
 	struct drm_radeon_driver_file_fields *radeon_priv;
 
 	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
-		return DRM_ERR(EINVAL);
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
 	}
 
-	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(sp, (drm_radeon_setparam_t __user *) data,
-				 sizeof(sp));
-
-	switch (sp.param) {
+	switch (sp->param) {
 	case RADEON_SETPARAM_FB_LOCATION:
-		radeon_priv = filp_priv->driver_priv;
-		radeon_priv->radeon_fb_delta = dev_priv->fb_location - sp.value;
+		radeon_priv = file_priv->driver_priv;
+		radeon_priv->radeon_fb_delta = dev_priv->fb_location -
+		    sp->value;
 		break;
 	case RADEON_SETPARAM_SWITCH_TILING:
-		if (sp.value == 0) {
+		if (sp->value == 0) {
 			DRM_DEBUG("color tiling disabled\n");
 			dev_priv->front_pitch_offset &= ~RADEON_DST_TILE_MACRO;
 			dev_priv->back_pitch_offset &= ~RADEON_DST_TILE_MACRO;
-			dev_priv->sarea_priv->tiling_enabled = 0;
-		} else if (sp.value == 1) {
+			if (dev_priv->sarea_priv)
+				dev_priv->sarea_priv->tiling_enabled = 0;
+		} else if (sp->value == 1) {
 			DRM_DEBUG("color tiling enabled\n");
 			dev_priv->front_pitch_offset |= RADEON_DST_TILE_MACRO;
 			dev_priv->back_pitch_offset |= RADEON_DST_TILE_MACRO;
-			dev_priv->sarea_priv->tiling_enabled = 1;
+			if (dev_priv->sarea_priv)
+				dev_priv->sarea_priv->tiling_enabled = 1;
 		}
 		break;
 	case RADEON_SETPARAM_PCIGART_LOCATION:
-		dev_priv->pcigart_offset = sp.value;
+		dev_priv->pcigart_offset = sp->value;
+		dev_priv->pcigart_offset_set = 1;
 		break;
 	case RADEON_SETPARAM_NEW_MEMMAP:
-		dev_priv->new_memmap = sp.value;
+		dev_priv->new_memmap = sp->value;
+		break;
+	case RADEON_SETPARAM_PCIGART_TABLE_SIZE:
+		dev_priv->gart_info.table_size = sp->value;
+		if (dev_priv->gart_info.table_size < RADEON_PCIGART_TABLE_SIZE)
+			dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
+		break;
+	case RADEON_SETPARAM_VBLANK_CRTC:
+		return radeon_vblank_crtc_set(dev, sp->value);
 		break;
 	default:
-		DRM_DEBUG("Invalid parameter %d\n", sp.param);
-		return DRM_ERR(EINVAL);
+		DRM_DEBUG("Invalid parameter %d\n", sp->param);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -3111,25 +3178,32 @@ static int radeon_cp_setparam(DRM_IOCTL_ARGS)
  *
  * DRM infrastructure takes care of reclaiming dma buffers.
  */
-void radeon_driver_preclose(drm_device_t * dev, DRMFILE filp)
+void radeon_driver_preclose(struct drm_device *dev,
+			    struct drm_file *file_priv)
 {
 	if (dev->dev_private) {
 		drm_radeon_private_t *dev_priv = dev->dev_private;
-		if (dev_priv->page_flipping) {
-			radeon_do_cleanup_pageflip(dev);
-		}
-		radeon_mem_release(filp, dev_priv->gart_heap);
-		radeon_mem_release(filp, dev_priv->fb_heap);
-		radeon_surfaces_release(filp, dev_priv);
+		dev_priv->page_flipping = 0;
+		radeon_mem_release(file_priv, dev_priv->gart_heap);
+		radeon_mem_release(file_priv, dev_priv->fb_heap);
+		radeon_surfaces_release(file_priv, dev_priv);
 	}
 }
 
-void radeon_driver_lastclose(drm_device_t * dev)
+void radeon_driver_lastclose(struct drm_device *dev)
 {
+	if (dev->dev_private) {
+		drm_radeon_private_t *dev_priv = dev->dev_private;
+
+		if (dev_priv->sarea_priv &&
+		    dev_priv->sarea_priv->pfCurrentPage != 0)
+			radeon_cp_dispatch_flip(dev);
+	}
+
 	radeon_do_release(dev);
 }
 
-int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv)
+int radeon_driver_open(struct drm_device *dev, struct drm_file *file_priv)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	struct drm_radeon_driver_file_fields *radeon_priv;
@@ -3142,7 +3216,7 @@ int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv)
 	if (!radeon_priv)
 		return -ENOMEM;
 
-	filp_priv->driver_priv = radeon_priv;
+	file_priv->driver_priv = radeon_priv;
 
 	if (dev_priv)
 		radeon_priv->radeon_fb_delta = dev_priv->fb_location;
@@ -3151,42 +3225,42 @@ int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv)
 	return 0;
 }
 
-void radeon_driver_postclose(drm_device_t * dev, drm_file_t * filp_priv)
+void radeon_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
 {
 	struct drm_radeon_driver_file_fields *radeon_priv =
-	    filp_priv->driver_priv;
+	    file_priv->driver_priv;
 
 	drm_free(radeon_priv, sizeof(*radeon_priv), DRM_MEM_FILES);
 }
 
-drm_ioctl_desc_t radeon_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_RADEON_CP_INIT)] = {radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_START)] = {radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_STOP)] = {radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_RESET)] = {radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_IDLE)] = {radeon_cp_idle, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_RESUME)] = {radeon_cp_resume, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_RESET)] = {radeon_engine_reset, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_FULLSCREEN)] = {radeon_fullscreen, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_SWAP)] = {radeon_cp_swap, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_CLEAR)] = {radeon_cp_clear, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_VERTEX)] = {radeon_cp_vertex, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_INDICES)] = {radeon_cp_indices, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_TEXTURE)] = {radeon_cp_texture, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_STIPPLE)] = {radeon_cp_stipple, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_INDIRECT)] = {radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_VERTEX2)] = {radeon_cp_vertex2, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_CMDBUF)] = {radeon_cp_cmdbuf, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_GETPARAM)] = {radeon_cp_getparam, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_FLIP)] = {radeon_cp_flip, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_ALLOC)] = {radeon_mem_alloc, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_FREE)] = {radeon_mem_free, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_INIT_HEAP)] = {radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_RADEON_IRQ_EMIT)] = {radeon_irq_emit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_IRQ_WAIT)] = {radeon_irq_wait, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_SETPARAM)] = {radeon_cp_setparam, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_SURF_ALLOC)] = {radeon_surface_alloc, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_RADEON_SURF_FREE)] = {radeon_surface_free, DRM_AUTH}
+struct drm_ioctl_desc radeon_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_RADEON_CP_INIT, radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_CP_START, radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_CP_STOP, radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_CP_RESET, radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_CP_IDLE, radeon_cp_idle, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_CP_RESUME, radeon_cp_resume, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_RESET, radeon_engine_reset, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_FULLSCREEN, radeon_fullscreen, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_SWAP, radeon_cp_swap, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_CLEAR, radeon_cp_clear, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_VERTEX, radeon_cp_vertex, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_INDICES, radeon_cp_indices, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_TEXTURE, radeon_cp_texture, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_STIPPLE, radeon_cp_stipple, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_INDIRECT, radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_VERTEX2, radeon_cp_vertex2, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_CMDBUF, radeon_cp_cmdbuf, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_GETPARAM, radeon_cp_getparam, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_FLIP, radeon_cp_flip, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_ALLOC, radeon_mem_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_FREE, radeon_mem_free, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_INIT_HEAP, radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_RADEON_IRQ_EMIT, radeon_irq_emit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_IRQ_WAIT, radeon_irq_wait, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_SETPARAM, radeon_cp_setparam, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_SURF_ALLOC, radeon_surface_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_RADEON_SURF_FREE, radeon_surface_free, DRM_AUTH)
 };
 
 int radeon_max_ioctl = DRM_ARRAY_SIZE(radeon_ioctls);
diff --git a/sys/dev/drm/savage_bci.c b/sys/dev/drm/savage_bci.c
index bbd45fe..4168ddf 100644
--- a/sys/dev/drm/savage_bci.c
+++ b/sys/dev/drm/savage_bci.c
@@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$");
 #define SAVAGE_EVENT_USEC_TIMEOUT	5000000 /* 5s */
 #define SAVAGE_FREELIST_DEBUG		0
 
+static int savage_do_cleanup_bci(struct drm_device *dev);
+
 static int
 savage_bci_wait_fifo_shadow(drm_savage_private_t *dev_priv, unsigned int n)
 {
@@ -61,7 +63,7 @@ savage_bci_wait_fifo_shadow(drm_savage_private_t *dev_priv, unsigned int n)
 	DRM_ERROR("failed!\n");
 	DRM_INFO("   status=0x%08x, threshold=0x%08x\n", status, threshold);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int
@@ -82,7 +84,7 @@ savage_bci_wait_fifo_s3d(drm_savage_private_t *dev_priv, unsigned int n)
 	DRM_ERROR("failed!\n");
 	DRM_INFO("   status=0x%08x\n", status);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int
@@ -103,7 +105,7 @@ savage_bci_wait_fifo_s4(drm_savage_private_t *dev_priv, unsigned int n)
 	DRM_ERROR("failed!\n");
 	DRM_INFO("   status=0x%08x\n", status);
 #endif
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 /*
@@ -137,7 +139,7 @@ savage_bci_wait_event_shadow(drm_savage_private_t *dev_priv, uint16_t e)
 	DRM_INFO("   status=0x%08x, e=0x%04x\n", status, e);
 #endif
 
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 static int
@@ -159,7 +161,7 @@ savage_bci_wait_event_reg(drm_savage_private_t *dev_priv, uint16_t e)
 	DRM_INFO("   status=0x%08x, e=0x%04x\n", status, e);
 #endif
 
-	return DRM_ERR(EBUSY);
+	return -EBUSY;
 }
 
 uint16_t savage_bci_emit_event(drm_savage_private_t *dev_priv,
@@ -204,11 +206,11 @@ uint16_t savage_bci_emit_event(drm_savage_private_t *dev_priv,
 /*
  * Freelist management
  */
-static int savage_freelist_init(drm_device_t *dev)
+static int savage_freelist_init(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *buf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *buf;
 	drm_savage_buf_priv_t *entry;
 	int i;
 	DRM_DEBUG("count=%d\n", dma->buf_count);
@@ -237,7 +239,7 @@ static int savage_freelist_init(drm_device_t *dev)
 	return 0;
 }
 
-static drm_buf_t *savage_freelist_get(drm_device_t *dev)
+static struct drm_buf *savage_freelist_get(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	drm_savage_buf_priv_t *tail = dev_priv->tail.prev;
@@ -270,7 +272,7 @@ static drm_buf_t *savage_freelist_get(drm_device_t *dev)
 	return NULL;
 }
 
-void savage_freelist_put(drm_device_t *dev, drm_buf_t *buf)
+void savage_freelist_put(struct drm_device *dev, struct drm_buf *buf)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	drm_savage_buf_priv_t *entry = buf->dev_private, *prev, *next;
@@ -302,7 +304,7 @@ static int savage_dma_init(drm_savage_private_t *dev_priv)
 	dev_priv->dma_pages = drm_alloc(sizeof(drm_savage_dma_page_t) *
 					dev_priv->nr_dma_pages, DRM_MEM_DRIVER);
 	if (dev_priv->dma_pages == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	for (i = 0; i < dev_priv->nr_dma_pages; ++i) {
 		SET_AGE(&dev_priv->dma_pages[i].age, 0, 0);
@@ -365,7 +367,7 @@ uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv, unsigned int n)
 	unsigned int cur = dev_priv->current_dma_page;
 	unsigned int rest = SAVAGE_DMA_PAGE_SIZE -
 		dev_priv->dma_pages[cur].used;
-	unsigned int nr_pages = (n - rest + SAVAGE_DMA_PAGE_SIZE-1) /
+	unsigned int nr_pages = (n - rest + SAVAGE_DMA_PAGE_SIZE - 1) /
 		SAVAGE_DMA_PAGE_SIZE;
 	uint32_t *dma_ptr;
 	unsigned int i;
@@ -375,7 +377,7 @@ uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv, unsigned int n)
 
 	if (cur + nr_pages < dev_priv->nr_dma_pages) {
 		dma_ptr = (uint32_t *)dev_priv->cmd_dma->handle +
-		    cur*SAVAGE_DMA_PAGE_SIZE + dev_priv->dma_pages[cur].used;
+		    cur * SAVAGE_DMA_PAGE_SIZE + dev_priv->dma_pages[cur].used;
 		if (n < rest)
 			rest = n;
 		dev_priv->dma_pages[cur].used += rest;
@@ -384,7 +386,7 @@ uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv, unsigned int n)
 	} else {
 		dev_priv->dma_flush(dev_priv);
 		nr_pages =
-		    (n + SAVAGE_DMA_PAGE_SIZE-1) / SAVAGE_DMA_PAGE_SIZE;
+		    (n + SAVAGE_DMA_PAGE_SIZE - 1) / SAVAGE_DMA_PAGE_SIZE;
 		for (i = cur; i < dev_priv->nr_dma_pages; ++i) {
 			dev_priv->dma_pages[i].age = dev_priv->last_dma_age;
 			dev_priv->dma_pages[i].used = 0;
@@ -444,7 +446,7 @@ static void savage_dma_flush(drm_savage_private_t *dev_priv)
 		uint32_t *dma_ptr = (uint32_t *)dev_priv->cmd_dma->handle +
 		    cur * SAVAGE_DMA_PAGE_SIZE + dev_priv->dma_pages[cur].used;
 		dev_priv->dma_pages[cur].used += pad;
-		while(pad != 0) {
+		while (pad != 0) {
 			*dma_ptr++ = BCI_CMD_WAIT;
 			pad--;
 		}
@@ -536,13 +538,13 @@ static void savage_fake_dma_flush(drm_savage_private_t *dev_priv)
 	dev_priv->first_dma_page = dev_priv->current_dma_page = 0;
 }
 
-int savage_driver_load(drm_device_t *dev, unsigned long chipset)
+int savage_driver_load(struct drm_device *dev, unsigned long chipset)
 {
 	drm_savage_private_t *dev_priv;
 
 	dev_priv = drm_alloc(sizeof(drm_savage_private_t), DRM_MEM_DRIVER);
 	if (dev_priv == NULL)
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 
 	memset(dev_priv, 0, sizeof(drm_savage_private_t));
 	dev->dev_private = (void *)dev_priv;
@@ -558,7 +560,7 @@ int savage_driver_load(drm_device_t *dev, unsigned long chipset)
  * in drm_addmap. Therefore we add them manually before the maps are
  * initialized, and tear them down on last close.
  */
-int savage_driver_firstopen(drm_device_t *dev)
+int savage_driver_firstopen(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	unsigned long mmio_base, fb_base, fb_size, aperture_base;
@@ -585,18 +587,18 @@ int savage_driver_firstopen(drm_device_t *dev)
 			 * MTRRs. */
 			dev_priv->mtrr[0].base = fb_base;
 			dev_priv->mtrr[0].size = 0x01000000;
-			dev_priv->mtrr[0].handle = 
+			dev_priv->mtrr[0].handle =
 			    drm_mtrr_add(dev_priv->mtrr[0].base,
 					 dev_priv->mtrr[0].size, DRM_MTRR_WC);
-			dev_priv->mtrr[1].base = fb_base+0x02000000;
+			dev_priv->mtrr[1].base = fb_base + 0x02000000;
 			dev_priv->mtrr[1].size = 0x02000000;
 			dev_priv->mtrr[1].handle =
 			    drm_mtrr_add(dev_priv->mtrr[1].base,
 					 dev_priv->mtrr[1].size, DRM_MTRR_WC);
-			dev_priv->mtrr[2].base = fb_base+0x04000000;
+			dev_priv->mtrr[2].base = fb_base + 0x04000000;
 			dev_priv->mtrr[2].size = 0x04000000;
 			dev_priv->mtrr[2].handle =
-			    drm_mtrr_add(dev_priv->mtrr[2].base, 
+			    drm_mtrr_add(dev_priv->mtrr[2].base,
 				         dev_priv->mtrr[2].size, DRM_MTRR_WC);
 		} else {
 			DRM_ERROR("strange pci_resource_len %08lx\n",
@@ -616,7 +618,7 @@ int savage_driver_firstopen(drm_device_t *dev)
 			 * aperture. */
 			dev_priv->mtrr[0].base = fb_base;
 			dev_priv->mtrr[0].size = 0x08000000;
-			dev_priv->mtrr[0].handle = 
+			dev_priv->mtrr[0].handle =
 			    drm_mtrr_add(dev_priv->mtrr[0].base,
 					 dev_priv->mtrr[0].size, DRM_MTRR_WC);
 		} else {
@@ -655,7 +657,7 @@ int savage_driver_firstopen(drm_device_t *dev)
 /*
  * Delete MTRRs and free device-private data.
  */
-void savage_driver_lastclose(drm_device_t *dev)
+void savage_driver_lastclose(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	int i;
@@ -667,7 +669,7 @@ void savage_driver_lastclose(drm_device_t *dev)
 				     dev_priv->mtrr[i].size, DRM_MTRR_WC);
 }
 
-int savage_driver_unload(drm_device_t *dev)
+int savage_driver_unload(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 
@@ -676,22 +678,22 @@ int savage_driver_unload(drm_device_t *dev)
 	return 0;
 }
 
-static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
+static int savage_do_init_bci(struct drm_device *dev, drm_savage_init_t *init)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 
 	if (init->fb_bpp != 16 && init->fb_bpp != 32) {
 		DRM_ERROR("invalid frame buffer bpp %d!\n", init->fb_bpp);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	if (init->depth_bpp != 16 && init->depth_bpp != 32) {
 		DRM_ERROR("invalid depth buffer bpp %d!\n", init->fb_bpp);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	if (init->dma_type != SAVAGE_DMA_AGP &&
 	    init->dma_type != SAVAGE_DMA_PCI) {
 		DRM_ERROR("invalid dma memory type %d!\n", init->dma_type);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	dev_priv->cob_size = init->cob_size;
@@ -711,35 +713,36 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 	dev_priv->texture_offset = init->texture_offset;
 	dev_priv->texture_size = init->texture_size;
 
-	DRM_GETSAREA();
+	dev_priv->sarea = drm_getsarea(dev);
 	if (!dev_priv->sarea) {
 		DRM_ERROR("could not find sarea!\n");
 		savage_do_cleanup_bci(dev);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	if (init->status_offset != 0) {
 		dev_priv->status = drm_core_findmap(dev, init->status_offset);
 		if (!dev_priv->status) {
 			DRM_ERROR("could not find shadow status region!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else {
 		dev_priv->status = NULL;
 	}
 	if (dev_priv->dma_type == SAVAGE_DMA_AGP && init->buffers_offset) {
+		dev->agp_buffer_token = init->buffers_offset;
 		dev->agp_buffer_map = drm_core_findmap(dev,
 						       init->buffers_offset);
 		if (!dev->agp_buffer_map) {
 			DRM_ERROR("could not find DMA buffer region!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		drm_core_ioremap(dev->agp_buffer_map, dev);
 		if (!dev->agp_buffer_map) {
 			DRM_ERROR("failed to ioremap DMA buffer region!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 	}
 	if (init->agp_textures_offset) {
@@ -748,7 +751,7 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 		if (!dev_priv->agp_textures) {
 			DRM_ERROR("could not find agp texture region!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else {
 		dev_priv->agp_textures = NULL;
@@ -759,39 +762,39 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 			DRM_ERROR("command DMA not supported on "
 				  "Savage3D/MX/IX.\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		if (dev->dma && dev->dma->buflist) {
 			DRM_ERROR("command and vertex DMA not supported "
 				  "at the same time.\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		dev_priv->cmd_dma = drm_core_findmap(dev, init->cmd_dma_offset);
 		if (!dev_priv->cmd_dma) {
 			DRM_ERROR("could not find command DMA region!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		if (dev_priv->dma_type == SAVAGE_DMA_AGP) {
 			if (dev_priv->cmd_dma->type != _DRM_AGP) {
 				DRM_ERROR("AGP command DMA region is not a "
 					  "_DRM_AGP map!\n");
 				savage_do_cleanup_bci(dev);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 			drm_core_ioremap(dev_priv->cmd_dma, dev);
 			if (!dev_priv->cmd_dma->handle) {
 				DRM_ERROR("failed to ioremap command "
 					  "DMA region!\n");
 				savage_do_cleanup_bci(dev);
-				return DRM_ERR(ENOMEM);
+				return -ENOMEM;
 			}
 		} else if (dev_priv->cmd_dma->type != _DRM_CONSISTENT) {
 			DRM_ERROR("PCI command DMA region is not a "
 				  "_DRM_CONSISTENT map!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else {
 		dev_priv->cmd_dma = NULL;
@@ -808,7 +811,7 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 		if (!dev_priv->fake_dma.handle) {
 			DRM_ERROR("could not allocate faked DMA buffer!\n");
 			savage_do_cleanup_bci(dev);
-			return DRM_ERR(ENOMEM);
+			return -ENOMEM;
 		}
 		dev_priv->cmd_dma = &dev_priv->fake_dma;
 		dev_priv->dma_flush = savage_fake_dma_flush;
@@ -833,8 +836,8 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 			depth_tile_format = SAVAGE_BD_TILE_DEST;
 		}
 		front_stride = dev_priv->front_pitch / (dev_priv->fb_bpp / 8);
-		back_stride  = dev_priv->back_pitch / (dev_priv->fb_bpp / 8);
-		depth_stride = 
+		back_stride = dev_priv->back_pitch / (dev_priv->fb_bpp / 8);
+		depth_stride =
 		    dev_priv->depth_pitch / (dev_priv->depth_bpp / 8);
 
 		dev_priv->front_bd = front_stride | SAVAGE_BD_BW_DISABLE |
@@ -885,19 +888,19 @@ static int savage_do_init_bci(drm_device_t *dev, drm_savage_init_t *init)
 	if (savage_freelist_init(dev) < 0) {
 		DRM_ERROR("could not initialize freelist\n");
 		savage_do_cleanup_bci(dev);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
-	if (savage_dma_init(dev_priv) <  0) {
+	if (savage_dma_init(dev_priv) < 0) {
 		DRM_ERROR("could not initialize command DMA\n");
 		savage_do_cleanup_bci(dev);
-		return DRM_ERR(ENOMEM);
+		return -ENOMEM;
 	}
 
 	return 0;
 }
 
-int savage_do_cleanup_bci(drm_device_t *dev)
+static int savage_do_cleanup_bci(struct drm_device *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 
@@ -927,59 +930,46 @@ int savage_do_cleanup_bci(drm_device_t *dev)
 	return 0;
 }
 
-static int savage_bci_init(DRM_IOCTL_ARGS)
+static int savage_bci_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_savage_init_t init;
+	drm_savage_init_t *init = data;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_COPY_FROM_USER_IOCTL(init, (drm_savage_init_t __user *)data,
-				 sizeof(init));
-
-	switch (init.func) {
+	switch (init->func) {
 	case SAVAGE_INIT_BCI:
-		return savage_do_init_bci(dev, &init);
+		return savage_do_init_bci(dev, init);
 	case SAVAGE_CLEANUP_BCI:
 		return savage_do_cleanup_bci(dev);
 	}
 
-	return DRM_ERR(EINVAL);
+	return -EINVAL;
 }
 
-static int savage_bci_event_emit(DRM_IOCTL_ARGS)
+static int savage_bci_event_emit(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_savage_private_t *dev_priv = dev->dev_private;
-	drm_savage_event_emit_t event;
+	drm_savage_event_emit_t *event = data;
 
 	DRM_DEBUG("\n");
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_COPY_FROM_USER_IOCTL(event, (drm_savage_event_emit_t __user *)data,
-				 sizeof(event));
+	event->count = savage_bci_emit_event(dev_priv, event->flags);
+	event->count |= dev_priv->event_wrap << 16;
 
-	event.count = savage_bci_emit_event(dev_priv, event.flags);
-	event.count |= dev_priv->event_wrap << 16;
-	DRM_COPY_TO_USER_IOCTL((drm_savage_event_emit_t __user *)data,
-			       event, sizeof(event));
 	return 0;
 }
 
-static int savage_bci_event_wait(DRM_IOCTL_ARGS)
+static int savage_bci_event_wait(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_savage_private_t *dev_priv = dev->dev_private;
-	drm_savage_event_wait_t event;
+	drm_savage_event_wait_t *event = data;
 	unsigned int event_e, hw_e;
 	unsigned int event_w, hw_w;
 
 	DRM_DEBUG("\n");
 
-	DRM_COPY_FROM_USER_IOCTL(event, (drm_savage_event_wait_t __user *)data,
-				 sizeof(event));
-
 	UPDATE_EVENT_COUNTER();
 	if (dev_priv->status_ptr)
 		hw_e = dev_priv->status_ptr[1] & 0xffff;
@@ -989,14 +979,14 @@ static int savage_bci_event_wait(DRM_IOCTL_ARGS)
 	if (hw_e > dev_priv->event_counter)
 		hw_w--; /* hardware hasn't passed the last wrap yet */
 
-	event_e = event.count & 0xffff;
-	event_w = event.count >> 16;
+	event_e = event->count & 0xffff;
+	event_w = event->count >> 16;
 
 	/* Don't need to wait if
 	 * - event counter wrapped since the event was emitted or
 	 * - the hardware has advanced up to or over the event to wait for.
 	 */
-	if (event_w < hw_w || (event_w == hw_w && event_e <= hw_e) )
+	if (event_w < hw_w || (event_w == hw_w && event_e <= hw_e))
 		return 0;
 	else
 		return dev_priv->wait_evnt(dev_priv, event_e);
@@ -1006,71 +996,68 @@ static int savage_bci_event_wait(DRM_IOCTL_ARGS)
  * DMA buffer management
  */
 
-static int savage_bci_get_buffers(DRMFILE filp, drm_device_t *dev, drm_dma_t *d)
+static int savage_bci_get_buffers(struct drm_device *dev,
+				  struct drm_file *file_priv,
+				  struct drm_dma *d)
 {
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 	int i;
 
 	for (i = d->granted_count; i < d->request_count; i++) {
 		buf = savage_freelist_get(dev);
 		if (!buf)
-			return DRM_ERR(EAGAIN);
+			return -EAGAIN;
 
-		buf->filp = filp;
+		buf->file_priv = file_priv;
 
 		if (DRM_COPY_TO_USER(&d->request_indices[i],
 				     &buf->idx, sizeof(buf->idx)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 		if (DRM_COPY_TO_USER(&d->request_sizes[i],
 				     &buf->total, sizeof(buf->total)))
-			return DRM_ERR(EFAULT);
+			return -EFAULT;
 
 		d->granted_count++;
 	}
 	return 0;
 }
 
-int savage_bci_buffers(DRM_IOCTL_ARGS)
+int savage_bci_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
-	drm_device_dma_t *dma = dev->dma;
-	drm_dma_t d;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_dma *d = data;
 	int ret = 0;
 
-	LOCK_TEST_WITH_RETURN(dev, filp);
-
-	DRM_COPY_FROM_USER_IOCTL(d, (drm_dma_t __user *)data, sizeof(d));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	/* Please don't send us buffers.
 	 */
-	if (d.send_count != 0) {
+	if (d->send_count != 0) {
 		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
-			  DRM_CURRENTPID, d.send_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->send_count);
+		return -EINVAL;
 	}
 
 	/* We'll send you buffers.
 	 */
-	if (d.request_count < 0 || d.request_count > dma->buf_count) {
+	if (d->request_count < 0 || d->request_count > dma->buf_count) {
 		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
-			  DRM_CURRENTPID, d.request_count, dma->buf_count);
-		return DRM_ERR(EINVAL);
+			  DRM_CURRENTPID, d->request_count, dma->buf_count);
+		return -EINVAL;
 	}
 
-	d.granted_count = 0;
+	d->granted_count = 0;
 
-	if (d.request_count) {
-		ret = savage_bci_get_buffers(filp, dev, &d);
+	if (d->request_count) {
+		ret = savage_bci_get_buffers(dev, file_priv, d);
 	}
 
-	DRM_COPY_TO_USER_IOCTL((drm_dma_t __user *)data, d, sizeof(d));
-
 	return ret;
 }
 
-void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp)
+void savage_reclaim_buffers(struct drm_device *dev, struct drm_file *file_priv)
 {
-	drm_device_dma_t *dma = dev->dma;
+	struct drm_device_dma *dma = dev->dma;
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	int i;
 
@@ -1081,13 +1068,11 @@ void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp)
 	if (!dma->buflist)
 		return;
 
-	/*i830_flush_queue(dev);*/
-
 	for (i = 0; i < dma->buf_count; i++) {
-		drm_buf_t *buf = dma->buflist[i];
+		struct drm_buf *buf = dma->buflist[i];
 		drm_savage_buf_priv_t *buf_priv = buf->dev_private;
 
-		if (buf->filp == filp && buf_priv &&
+		if (buf->file_priv == file_priv && buf_priv &&
 		    buf_priv->next == NULL && buf_priv->prev == NULL) {
 			uint16_t event;
 			DRM_DEBUG("reclaimed from client\n");
@@ -1097,15 +1082,14 @@ void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp)
 		}
 	}
 
-	drm_core_reclaim_buffers(dev, filp);
+	drm_core_reclaim_buffers(dev, file_priv);
 }
 
-drm_ioctl_desc_t savage_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_INIT)] = {savage_bci_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_CMDBUF)] = {savage_bci_cmdbuf, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_EVENT_EMIT)] = {savage_bci_event_emit, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_EVENT_WAIT)] = {savage_bci_event_wait, DRM_AUTH},
+struct drm_ioctl_desc savage_ioctls[] = {
+	DRM_IOCTL_DEF(DRM_SAVAGE_BCI_INIT, savage_bci_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_SAVAGE_BCI_CMDBUF, savage_bci_cmdbuf, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SAVAGE_BCI_EVENT_EMIT, savage_bci_event_emit, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SAVAGE_BCI_EVENT_WAIT, savage_bci_event_wait, DRM_AUTH),
 };
 
 int savage_max_ioctl = DRM_ARRAY_SIZE(savage_ioctls);
-
diff --git a/sys/dev/drm/savage_drm.h b/sys/dev/drm/savage_drm.h
index b0e0a04..c54d2d4 100644
--- a/sys/dev/drm/savage_drm.h
+++ b/sys/dev/drm/savage_drm.h
@@ -50,7 +50,7 @@ __FBSDID("$FreeBSD$");
 typedef struct _drm_savage_sarea {
 	/* LRU lists for texture memory in agp space and on the card.
 	 */
-	drm_tex_region_t texList[SAVAGE_NR_TEX_HEAPS][SAVAGE_NR_TEX_REGIONS+1];
+	struct drm_tex_region texList[SAVAGE_NR_TEX_HEAPS][SAVAGE_NR_TEX_REGIONS+1];
 	unsigned int texAge[SAVAGE_NR_TEX_HEAPS];
 
 	/* Mechanism to validate card state.
@@ -115,7 +115,7 @@ typedef struct drm_savage_cmdbuf {
 	unsigned int vb_size;	/* size of client vertex buffer in bytes */
 	unsigned int vb_stride;	/* stride of vertices in 32bit words */
 				/* boxes in client's address space */
-	drm_clip_rect_t __user *box_addr;
+	struct drm_clip_rect __user *box_addr;
 	unsigned int nbox;	/* number of clipping boxes */
 } drm_savage_cmdbuf_t;
 
diff --git a/sys/dev/drm/savage_drv.c b/sys/dev/drm/savage_drv.c
index 1d046c4..25c75db 100644
--- a/sys/dev/drm/savage_drv.c
+++ b/sys/dev/drm/savage_drv.c
@@ -40,7 +40,7 @@ static drm_pci_id_list_t savage_pciidlist[] = {
 	savage_PCI_IDS
 };
 
-static void savage_configure(drm_device_t *dev)
+static void savage_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= sizeof(drm_savage_buf_priv_t);
 	dev->driver.load		= savage_driver_load;
@@ -76,9 +76,9 @@ savage_probe(device_t dev)
 static int
 savage_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	savage_configure(dev);
 	return drm_attach(nbdev, savage_pciidlist);
 }
@@ -95,7 +95,7 @@ static device_method_t savage_methods[] = {
 static driver_t savage_driver = {
 	"drm",
 	savage_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
diff --git a/sys/dev/drm/savage_drv.h b/sys/dev/drm/savage_drv.h
index af3ccc8..e1e7b25 100644
--- a/sys/dev/drm/savage_drv.h
+++ b/sys/dev/drm/savage_drv.h
@@ -61,7 +61,7 @@ typedef struct drm_savage_buf_priv {
 	struct drm_savage_buf_priv *next;
 	struct drm_savage_buf_priv *prev;
 	drm_savage_age_t age;
-	drm_buf_t *buf;
+	struct drm_buf *buf;
 } drm_savage_buf_priv_t;
 
 typedef struct drm_savage_dma_page {
@@ -107,7 +107,7 @@ enum savage_family {
 	S3_LAST
 };
 
-extern drm_ioctl_desc_t savage_ioctls[];
+extern struct drm_ioctl_desc savage_ioctls[];
 extern int savage_max_ioctl;
 
 #define S3_SAVAGE3D_SERIES(chip)  ((chip>=S3_SAVAGE3D) && (chip<=S3_SAVAGE_MX))
@@ -195,34 +195,34 @@ typedef struct drm_savage_private {
 	/* Err, there is a macro wait_event in include/linux/wait.h.
 	 * Avoid unwanted macro expansion. */
 	void (*emit_clip_rect)(struct drm_savage_private *dev_priv,
-			       const drm_clip_rect_t *pbox);
+			       const struct drm_clip_rect *pbox);
 	void (*dma_flush)(struct drm_savage_private *dev_priv);
 } drm_savage_private_t;
 
 /* ioctls */
-extern int savage_bci_cmdbuf(DRM_IOCTL_ARGS);
-extern int savage_bci_buffers(DRM_IOCTL_ARGS);
+extern int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_priv);
+extern int savage_bci_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv);
 
 /* BCI functions */
 extern uint16_t savage_bci_emit_event(drm_savage_private_t *dev_priv,
 				      unsigned int flags);
-extern void savage_freelist_put(drm_device_t *dev, drm_buf_t *buf);
+extern void savage_freelist_put(struct drm_device *dev, struct drm_buf *buf);
 extern void savage_dma_reset(drm_savage_private_t *dev_priv);
 extern void savage_dma_wait(drm_savage_private_t *dev_priv, unsigned int page);
 extern uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv,
 				  unsigned int n);
-extern int savage_driver_load(drm_device_t *dev, unsigned long chipset);
-extern int savage_driver_firstopen(drm_device_t *dev);
-extern void savage_driver_lastclose(drm_device_t *dev);
-extern int savage_driver_unload(drm_device_t *dev);
-extern int savage_do_cleanup_bci(drm_device_t *dev);
-extern void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp);
+extern int savage_driver_load(struct drm_device *dev, unsigned long chipset);
+extern int savage_driver_firstopen(struct drm_device *dev);
+extern void savage_driver_lastclose(struct drm_device *dev);
+extern int savage_driver_unload(struct drm_device *dev);
+extern void savage_reclaim_buffers(struct drm_device *dev,
+				   struct drm_file *file_priv);
 
 /* state functions */
 extern void savage_emit_clip_rect_s3d(drm_savage_private_t *dev_priv,
-				      const drm_clip_rect_t *pbox);
+				      const struct drm_clip_rect *pbox);
 extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
-				     const drm_clip_rect_t *pbox);
+				     const struct drm_clip_rect *pbox);
 
 #define SAVAGE_FB_SIZE_S3	0x01000000	/*  16MB */
 #define SAVAGE_FB_SIZE_S4	0x02000000	/*  32MB */
@@ -240,7 +240,7 @@ extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
  */
 #define SAVAGE_STATUS_WORD0		0x48C00
 #define SAVAGE_STATUS_WORD1		0x48C04
-#define SAVAGE_ALT_STATUS_WORD0 	0x48C60
+#define SAVAGE_ALT_STATUS_WORD0		0x48C60
 
 #define SAVAGE_FIFO_USED_MASK_S3D	0x0001ffff
 #define SAVAGE_FIFO_USED_MASK_S4	0x001fffff
@@ -313,7 +313,7 @@ extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
 #define SAVAGE_DESTCTRL_S3D		0x34
 #define SAVAGE_SCSTART_S3D		0x35
 #define SAVAGE_SCEND_S3D		0x36
-#define SAVAGE_ZWATERMARK_S3D		0x37 
+#define SAVAGE_ZWATERMARK_S3D		0x37
 #define SAVAGE_DESTTEXRWWATERMARK_S3D	0x38
 /* common stuff */
 #define SAVAGE_VERTBUFADDR		0x3e
diff --git a/sys/dev/drm/savage_state.c b/sys/dev/drm/savage_state.c
index d2fc0db..b3e7fe1 100644
--- a/sys/dev/drm/savage_state.c
+++ b/sys/dev/drm/savage_state.c
@@ -30,33 +30,33 @@ __FBSDID("$FreeBSD$");
 #include "dev/drm/savage_drv.h"
 
 void savage_emit_clip_rect_s3d(drm_savage_private_t *dev_priv,
-			       const drm_clip_rect_t *pbox)
+			       const struct drm_clip_rect *pbox)
 {
 	uint32_t scstart = dev_priv->state.s3d.new_scstart;
-	uint32_t scend   = dev_priv->state.s3d.new_scend;
+	uint32_t scend = dev_priv->state.s3d.new_scend;
 	scstart = (scstart & ~SAVAGE_SCISSOR_MASK_S3D) |
-		((uint32_t)pbox->x1 & 0x000007ff) | 
+		((uint32_t)pbox->x1 & 0x000007ff) |
 		(((uint32_t)pbox->y1 << 16) & 0x07ff0000);
-	scend   = (scend   & ~SAVAGE_SCISSOR_MASK_S3D) |
-		(((uint32_t)pbox->x2-1) & 0x000007ff) |
-		((((uint32_t)pbox->y2-1) << 16) & 0x07ff0000);
+	scend   = (scend & ~SAVAGE_SCISSOR_MASK_S3D) |
+		(((uint32_t)pbox->x2 - 1) & 0x000007ff) |
+		((((uint32_t)pbox->y2 - 1) << 16) & 0x07ff0000);
 	if (scstart != dev_priv->state.s3d.scstart ||
 	    scend   != dev_priv->state.s3d.scend) {
 		DMA_LOCALS;
 		BEGIN_DMA(4);
-		DMA_WRITE(BCI_CMD_WAIT|BCI_CMD_WAIT_3D);
+		DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
 		DMA_SET_REGISTERS(SAVAGE_SCSTART_S3D, 2);
 		DMA_WRITE(scstart);
 		DMA_WRITE(scend);
 		dev_priv->state.s3d.scstart = scstart;
-		dev_priv->state.s3d.scend   = scend;
+		dev_priv->state.s3d.scend = scend;
 		dev_priv->waiting = 1;
 		DMA_COMMIT();
 	}
 }
 
 void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
-			      const drm_clip_rect_t *pbox)
+			      const struct drm_clip_rect *pbox)
 {
 	uint32_t drawctrl0 = dev_priv->state.s4.new_drawctrl0;
 	uint32_t drawctrl1 = dev_priv->state.s4.new_drawctrl1;
@@ -64,13 +64,13 @@ void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
 		((uint32_t)pbox->x1 & 0x000007ff) |
 		(((uint32_t)pbox->y1 << 12) & 0x00fff000);
 	drawctrl1 = (drawctrl1 & ~SAVAGE_SCISSOR_MASK_S4) |
-		(((uint32_t)pbox->x2-1) & 0x000007ff) |
-		((((uint32_t)pbox->y2-1) << 12) & 0x00fff000);
+		(((uint32_t)pbox->x2 - 1) & 0x000007ff) |
+		((((uint32_t)pbox->y2 - 1) << 12) & 0x00fff000);
 	if (drawctrl0 != dev_priv->state.s4.drawctrl0 ||
 	    drawctrl1 != dev_priv->state.s4.drawctrl1) {
 		DMA_LOCALS;
 		BEGIN_DMA(4);
-		DMA_WRITE(BCI_CMD_WAIT|BCI_CMD_WAIT_3D);
+		DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
 		DMA_SET_REGISTERS(SAVAGE_DRAWCTRL0_S4, 2);
 		DMA_WRITE(drawctrl0);
 		DMA_WRITE(drawctrl1);
@@ -86,22 +86,22 @@ static int savage_verify_texaddr(drm_savage_private_t *dev_priv, int unit,
 {
 	if ((addr & 6) != 2) { /* reserved bits */
 		DRM_ERROR("bad texAddr%d %08x (reserved bits)\n", unit, addr);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 	if (!(addr & 1)) { /* local */
 		addr &= ~7;
-		if (addr <  dev_priv->texture_offset ||
-		    addr >= dev_priv->texture_offset+dev_priv->texture_size) {
+		if (addr < dev_priv->texture_offset ||
+		    addr >= dev_priv->texture_offset + dev_priv->texture_size) {
 			DRM_ERROR
 			    ("bad texAddr%d %08x (local addr out of range)\n",
 			     unit, addr);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else { /* AGP */
 		if (!dev_priv->agp_textures) {
 			DRM_ERROR("bad texAddr%d %08x (AGP not available)\n",
 				  unit, addr);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		addr &= ~7;
 		if (addr < dev_priv->agp_textures->offset ||
@@ -110,17 +110,17 @@ static int savage_verify_texaddr(drm_savage_private_t *dev_priv, int unit,
 			DRM_ERROR
 			    ("bad texAddr%d %08x (AGP addr out of range)\n",
 			     unit, addr);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 	return 0;
 }
 
 #define SAVE_STATE(reg,where)			\
-	if(start <= reg && start+count > reg)	\
+	if(start <= reg && start + count > reg)	\
 		dev_priv->state.where = regs[reg - start]
 #define SAVE_STATE_MASK(reg,where,mask) do {			\
-	if(start <= reg && start+count > reg) {			\
+	if(start <= reg && start + count > reg) {			\
 		uint32_t tmp;					\
 		tmp = regs[reg - start];			\
 		dev_priv->state.where = (tmp & (mask)) |	\
@@ -132,10 +132,10 @@ static int savage_verify_state_s3d(drm_savage_private_t *dev_priv,
 				   const uint32_t *regs)
 {
 	if (start < SAVAGE_TEXPALADDR_S3D ||
-	    start+count-1 > SAVAGE_DESTTEXRWWATERMARK_S3D) {
+	    start + count - 1 > SAVAGE_DESTTEXRWWATERMARK_S3D) {
 		DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
-			  start, start+count-1);
-		return DRM_ERR(EINVAL);
+			  start, start + count - 1);
+		return -EINVAL;
 	}
 
 	SAVE_STATE_MASK(SAVAGE_SCSTART_S3D, s3d.new_scstart,
@@ -145,7 +145,7 @@ static int savage_verify_state_s3d(drm_savage_private_t *dev_priv,
 
 	/* if any texture regs were changed ... */
 	if (start <= SAVAGE_TEXCTRL_S3D &&
-	    start+count > SAVAGE_TEXPALADDR_S3D) {
+	    start + count > SAVAGE_TEXPALADDR_S3D) {
 		/* ... check texture state */
 		SAVE_STATE(SAVAGE_TEXCTRL_S3D, s3d.texctrl);
 		SAVE_STATE(SAVAGE_TEXADDR_S3D, s3d.texaddr);
@@ -164,10 +164,10 @@ static int savage_verify_state_s4(drm_savage_private_t *dev_priv,
 	int ret = 0;
 
 	if (start < SAVAGE_DRAWLOCALCTRL_S4 ||
-	    start+count-1 > SAVAGE_TEXBLENDCOLOR_S4) {
+	    start + count - 1 > SAVAGE_TEXBLENDCOLOR_S4) {
 		DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
-			  start, start+count-1);
-		return DRM_ERR(EINVAL);
+			  start, start + count - 1);
+		return -EINVAL;
 	}
 
 	SAVE_STATE_MASK(SAVAGE_DRAWCTRL0_S4, s4.new_drawctrl0,
@@ -215,14 +215,14 @@ static int savage_dispatch_state(drm_savage_private_t *dev_priv,
 			return ret;
 		/* scissor regs are emitted in savage_dispatch_draw */
 		if (start < SAVAGE_SCSTART_S3D) {
-			if (start+count > SAVAGE_SCEND_S3D+1)
-				count2 = count - (SAVAGE_SCEND_S3D+1 - start);
-			if (start+count > SAVAGE_SCSTART_S3D)
+			if (start + count > SAVAGE_SCEND_S3D + 1)
+				count2 = count - (SAVAGE_SCEND_S3D + 1 - start);
+			if (start + count > SAVAGE_SCSTART_S3D)
 				count = SAVAGE_SCSTART_S3D - start;
 		} else if (start <= SAVAGE_SCEND_S3D) {
-			if (start+count > SAVAGE_SCEND_S3D+1) {
-				count -= SAVAGE_SCEND_S3D+1 - start;
-				start = SAVAGE_SCEND_S3D+1;
+			if (start + count > SAVAGE_SCEND_S3D + 1) {
+				count -= SAVAGE_SCEND_S3D + 1 - start;
+				start = SAVAGE_SCEND_S3D + 1;
 			} else
 				return 0;
 		}
@@ -232,24 +232,24 @@ static int savage_dispatch_state(drm_savage_private_t *dev_priv,
 			return ret;
 		/* scissor regs are emitted in savage_dispatch_draw */
 		if (start < SAVAGE_DRAWCTRL0_S4) {
-			if (start+count > SAVAGE_DRAWCTRL1_S4+1)
+			if (start + count > SAVAGE_DRAWCTRL1_S4 + 1)
 				count2 = count -
 					 (SAVAGE_DRAWCTRL1_S4 + 1 - start);
-			if (start+count > SAVAGE_DRAWCTRL0_S4)
+			if (start + count > SAVAGE_DRAWCTRL0_S4)
 				count = SAVAGE_DRAWCTRL0_S4 - start;
 		} else if (start <= SAVAGE_DRAWCTRL1_S4) {
-			if (start+count > SAVAGE_DRAWCTRL1_S4+1) {
-				count -= SAVAGE_DRAWCTRL1_S4+1 - start;
-				start = SAVAGE_DRAWCTRL1_S4+1;
+			if (start + count > SAVAGE_DRAWCTRL1_S4 + 1) {
+				count -= SAVAGE_DRAWCTRL1_S4 + 1 - start;
+				start = SAVAGE_DRAWCTRL1_S4 + 1;
 			} else
 				return 0;
 		}
 	}
 
-	bci_size = count + (count+254)/255 + count2 + (count2+254)/255;
+	bci_size = count + (count + 254) / 255 + count2 + (count2 + 254) / 255;
 
 	if (cmd_header->state.global) {
-		BEGIN_DMA(bci_size+1);
+		BEGIN_DMA(bci_size + 1);
 		DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
 		dev_priv->waiting = 1;
 	} else {
@@ -278,7 +278,7 @@ static int savage_dispatch_state(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 				    const drm_savage_cmd_header_t *cmd_header,
-				    const drm_buf_t *dmabuf)
+				    const struct drm_buf *dmabuf)
 {
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->prim.prim;
@@ -289,8 +289,8 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 	BCI_LOCALS;
 
 	if (!dmabuf) {
-	    DRM_ERROR("called without dma buffers!\n");
-	    return DRM_ERR(EINVAL);
+		DRM_ERROR("called without dma buffers!\n");
+		return -EINVAL;
 	}
 
 	if (!n)
@@ -304,7 +304,7 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 		if (n % 3 != 0) {
 			DRM_ERROR("wrong number of vertices %u in TRILIST\n",
 				  n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	case SAVAGE_PRIM_TRISTRIP:
@@ -313,18 +313,18 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 			DRM_ERROR
 			   ("wrong number of vertices %u in TRIFAN/STRIP\n",
 			    n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	default:
 		DRM_ERROR("invalid primitive type %u\n", prim);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 		if (skip != 0) {
 			DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else {
 		unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
@@ -332,18 +332,18 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 			(skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
 		if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
 			DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		if (reorder) {
 			DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
-	if (start + n > dmabuf->total/32) {
+	if (start + n > dmabuf->total / 32) {
 		DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
-			  start, start + n - 1, dmabuf->total/32);
-		return DRM_ERR(EINVAL);
+			  start, start + n - 1, dmabuf->total / 32);
+		return -EINVAL;
 	}
 
 	/* Vertex DMA doesn't work with command DMA at the same time,
@@ -377,33 +377,33 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 			/* Need to reorder indices for correct flat
 			 * shading while preserving the clock sense
 			 * for correct culling. Only on Savage3D. */
-			int reorder[3] = {-1, -1, -1};
-			reorder[start%3] = 2;
+			int reorder[3] = { -1, -1, -1 };
+			reorder[start % 3] = 2;
 
-			BEGIN_BCI((count+1+1)/2);
-			BCI_DRAW_INDICES_S3D(count, prim, start+2);
+			BEGIN_BCI((count + 1 + 1) / 2);
+			BCI_DRAW_INDICES_S3D(count, prim, start + 2);
 
-			for (i = start+1; i+1 < start+count; i += 2)
+			for (i = start + 1; i + 1 < start + count; i += 2)
 				BCI_WRITE((i + reorder[i % 3]) |
 					  ((i + 1 +
 					    reorder[(i + 1) % 3]) << 16));
-			if (i < start+count)
-				BCI_WRITE(i + reorder[i%3]);
+			if (i < start + count)
+				BCI_WRITE(i + reorder[i % 3]);
 		} else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
-			BEGIN_BCI((count+1+1)/2);
+			BEGIN_BCI((count + 1 + 1) / 2);
 			BCI_DRAW_INDICES_S3D(count, prim, start);
 
-			for (i = start+1; i+1 < start+count; i += 2)
-				BCI_WRITE(i | ((i+1) << 16));
-			if (i < start+count)
+			for (i = start + 1; i + 1 < start + count; i += 2)
+				BCI_WRITE(i | ((i + 1) << 16));
+			if (i < start + count)
 				BCI_WRITE(i);
 		} else {
-			BEGIN_BCI((count+2+1)/2);
+			BEGIN_BCI((count + 2 + 1) / 2);
 			BCI_DRAW_INDICES_S4(count, prim, skip);
 
-			for (i = start; i+1 < start+count; i += 2)
-				BCI_WRITE(i | ((i+1) << 16));
-			if (i < start+count)
+			for (i = start; i + 1 < start + count; i += 2)
+				BCI_WRITE(i | ((i + 1) << 16));
+			if (i < start + count)
 				BCI_WRITE(i);
 		}
 
@@ -441,7 +441,7 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 		if (n % 3 != 0) {
 			DRM_ERROR("wrong number of vertices %u in TRILIST\n",
 				  n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	case SAVAGE_PRIM_TRISTRIP:
@@ -450,24 +450,24 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 			DRM_ERROR
 			    ("wrong number of vertices %u in TRIFAN/STRIP\n",
 			     n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	default:
 		DRM_ERROR("invalid primitive type %u\n", prim);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 		if (skip > SAVAGE_SKIP_ALL_S3D) {
 			DRM_ERROR("invalid skip flags 0x%04x\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		vtx_size = 8; /* full vertex */
 	} else {
 		if (skip > SAVAGE_SKIP_ALL_S4) {
 			DRM_ERROR("invalid skip flags 0x%04x\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		vtx_size = 10; /* full vertex */
 	}
@@ -479,13 +479,13 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 	if (vtx_size > vb_stride) {
 		DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
 			  vtx_size, vb_stride);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
-	if (start + n > vb_size / (vb_stride*4)) {
+	if (start + n > vb_size / (vb_stride * 4)) {
 		DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
-			  start, start + n - 1, vb_size / (vb_stride*4));
-		return DRM_ERR(EINVAL);
+			  start, start + n - 1, vb_size / (vb_stride * 4));
+		return -EINVAL;
 	}
 
 	prim <<= 25;
@@ -496,28 +496,28 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 			/* Need to reorder vertices for correct flat
 			 * shading while preserving the clock sense
 			 * for correct culling. Only on Savage3D. */
-			int reorder[3] = {-1, -1, -1};
-			reorder[start%3] = 2;
+			int reorder[3] = { -1, -1, -1 };
+			reorder[start % 3] = 2;
 
-			BEGIN_DMA(count*vtx_size+1);
+			BEGIN_DMA(count * vtx_size + 1);
 			DMA_DRAW_PRIMITIVE(count, prim, skip);
 
-			for (i = start; i < start+count; ++i) {
+			for (i = start; i < start + count; ++i) {
 				unsigned int j = i + reorder[i % 3];
-				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 			}
 
 			DMA_COMMIT();
 		} else {
-			BEGIN_DMA(count*vtx_size+1);
+			BEGIN_DMA(count * vtx_size + 1);
 			DMA_DRAW_PRIMITIVE(count, prim, skip);
 
 			if (vb_stride == vtx_size) {
-				DMA_COPY(&vtxbuf[vb_stride*start],
-					 vtx_size*count);
+				DMA_COPY(&vtxbuf[vb_stride * start],
+					 vtx_size * count);
 			} else {
-				for (i = start; i < start+count; ++i) {
-					DMA_COPY(&vtxbuf[vb_stride*i],
+				for (i = start; i < start + count; ++i) {
+					DMA_COPY(&vtxbuf[vb_stride * i],
 						 vtx_size);
 				}
 			}
@@ -537,7 +537,7 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 				   const drm_savage_cmd_header_t *cmd_header,
 				   const uint16_t *idx,
-				   const drm_buf_t *dmabuf)
+				   const struct drm_buf *dmabuf)
 {
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->idx.prim;
@@ -547,8 +547,8 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 	BCI_LOCALS;
 
 	if (!dmabuf) {
-	    DRM_ERROR("called without dma buffers!\n");
-	    return DRM_ERR(EINVAL);
+		DRM_ERROR("called without dma buffers!\n");
+		return -EINVAL;
 	}
 
 	if (!n)
@@ -561,7 +561,7 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 	case SAVAGE_PRIM_TRILIST:
 		if (n % 3 != 0) {
 			DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	case SAVAGE_PRIM_TRISTRIP:
@@ -569,18 +569,18 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 		if (n < 3) {
 			DRM_ERROR
 			    ("wrong number of indices %u in TRIFAN/STRIP\n", n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	default:
 		DRM_ERROR("invalid primitive type %u\n", prim);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 		if (skip != 0) {
 			DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	} else {
 		unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
@@ -588,11 +588,11 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 			(skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
 		if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
 			DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		if (reorder) {
 			DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 	}
 
@@ -626,10 +626,10 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 
 		/* check indices */
 		for (i = 0; i < count; ++i) {
-			if (idx[i] > dmabuf->total/32) {
+			if (idx[i] > dmabuf->total / 32) {
 				DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
-					  i, idx[i], dmabuf->total/32);
-				return DRM_ERR(EINVAL);
+					  i, idx[i], dmabuf->total / 32);
+				return -EINVAL;
 			}
 		}
 
@@ -637,31 +637,31 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 			/* Need to reorder indices for correct flat
 			 * shading while preserving the clock sense
 			 * for correct culling. Only on Savage3D. */
-			int reorder[3] = {2, -1, -1};
+			int reorder[3] = { 2, -1, -1 };
 
-			BEGIN_BCI((count+1+1)/2);
+			BEGIN_BCI((count + 1 + 1) / 2);
 			BCI_DRAW_INDICES_S3D(count, prim, idx[2]);
 
-			for (i = 1; i+1 < count; i += 2)
+			for (i = 1; i + 1 < count; i += 2)
 				BCI_WRITE(idx[i + reorder[i % 3]] |
 					  (idx[i + 1 +
 					   reorder[(i + 1) % 3]] << 16));
 			if (i < count)
-				BCI_WRITE(idx[i + reorder[i%3]]);
+				BCI_WRITE(idx[i + reorder[i % 3]]);
 		} else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
-			BEGIN_BCI((count+1+1)/2);
+			BEGIN_BCI((count + 1 + 1) / 2);
 			BCI_DRAW_INDICES_S3D(count, prim, idx[0]);
 
-			for (i = 1; i+1 < count; i += 2)
-				BCI_WRITE(idx[i] | (idx[i+1] << 16));
+			for (i = 1; i + 1 < count; i += 2)
+				BCI_WRITE(idx[i] | (idx[i + 1] << 16));
 			if (i < count)
 				BCI_WRITE(idx[i]);
 		} else {
-			BEGIN_BCI((count+2+1)/2);
+			BEGIN_BCI((count + 2 + 1) / 2);
 			BCI_DRAW_INDICES_S4(count, prim, skip);
 
-			for (i = 0; i+1 < count; i += 2)
-				BCI_WRITE(idx[i] | (idx[i+1] << 16));
+			for (i = 0; i + 1 < count; i += 2)
+				BCI_WRITE(idx[i] | (idx[i + 1] << 16));
 			if (i < count)
 				BCI_WRITE(idx[i]);
 		}
@@ -699,7 +699,7 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 	case SAVAGE_PRIM_TRILIST:
 		if (n % 3 != 0) {
 			DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	case SAVAGE_PRIM_TRISTRIP:
@@ -707,24 +707,24 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 		if (n < 3) {
 			DRM_ERROR
 			    ("wrong number of indices %u in TRIFAN/STRIP\n", n);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		break;
 	default:
 		DRM_ERROR("invalid primitive type %u\n", prim);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 		if (skip > SAVAGE_SKIP_ALL_S3D) {
 			DRM_ERROR("invalid skip flags 0x%04x\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		vtx_size = 8; /* full vertex */
 	} else {
 		if (skip > SAVAGE_SKIP_ALL_S4) {
 			DRM_ERROR("invalid skip flags 0x%04x\n", skip);
-			return DRM_ERR(EINVAL);
+			return -EINVAL;
 		}
 		vtx_size = 10; /* full vertex */
 	}
@@ -736,7 +736,7 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 	if (vtx_size > vb_stride) {
 		DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
 			  vtx_size, vb_stride);
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 	}
 
 	prim <<= 25;
@@ -746,10 +746,10 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 
 		/* Check indices */
 		for (i = 0; i < count; ++i) {
-			if (idx[i] > vb_size / (vb_stride*4)) {
+			if (idx[i] > vb_size / (vb_stride * 4)) {
 				DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
-					  i, idx[i],  vb_size / (vb_stride*4));
-				return DRM_ERR(EINVAL);
+					  i, idx[i],  vb_size / (vb_stride * 4));
+				return -EINVAL;
 			}
 		}
 
@@ -757,24 +757,24 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 			/* Need to reorder vertices for correct flat
 			 * shading while preserving the clock sense
 			 * for correct culling. Only on Savage3D. */
-			int reorder[3] = {2, -1, -1};
+			int reorder[3] = { 2, -1, -1 };
 
-			BEGIN_DMA(count*vtx_size+1);
+			BEGIN_DMA(count * vtx_size + 1);
 			DMA_DRAW_PRIMITIVE(count, prim, skip);
 
 			for (i = 0; i < count; ++i) {
 				unsigned int j = idx[i + reorder[i % 3]];
-				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 			}
 
 			DMA_COMMIT();
 		} else {
-			BEGIN_DMA(count*vtx_size+1);
+			BEGIN_DMA(count * vtx_size + 1);
 			DMA_DRAW_PRIMITIVE(count, prim, skip);
 
 			for (i = 0; i < count; ++i) {
 				unsigned int j = idx[i];
-				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride * j], vtx_size);
 			}
 
 			DMA_COMMIT();
@@ -793,7 +793,7 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 				 const drm_savage_cmd_header_t *cmd_header,
 				 const drm_savage_cmd_header_t *data,
 				 unsigned int nbox,
-				 const drm_clip_rect_t *boxes)
+				 const struct drm_clip_rect *boxes)
 {
 	unsigned int flags = cmd_header->clear0.flags;
 	unsigned int clear_cmd;
@@ -826,12 +826,12 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 		x = boxes[i].x1, y = boxes[i].y1;
 		w = boxes[i].x2 - boxes[i].x1;
 		h = boxes[i].y2 - boxes[i].y1;
-		BEGIN_DMA(nbufs*6);
+		BEGIN_DMA(nbufs * 6);
 		for (buf = SAVAGE_FRONT; buf <= SAVAGE_DEPTH; buf <<= 1) {
 			if (!(flags & buf))
 				continue;
 			DMA_WRITE(clear_cmd);
-			switch(buf) {
+			switch (buf) {
 			case SAVAGE_FRONT:
 				DMA_WRITE(dev_priv->front_offset);
 				DMA_WRITE(dev_priv->front_bd);
@@ -863,7 +863,7 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 }
 
 static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
-				unsigned int nbox, const drm_clip_rect_t *boxes)
+				unsigned int nbox, const struct drm_clip_rect *boxes)
 {
 	unsigned int swap_cmd;
 	unsigned int i;
@@ -883,8 +883,8 @@ static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
 		DMA_WRITE(dev_priv->back_bd);
 		DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
 		DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
-		DMA_WRITE(BCI_W_H(boxes[i].x2-boxes[i].x1,
-				  boxes[i].y2-boxes[i].y1));
+		DMA_WRITE(BCI_W_H(boxes[i].x2 - boxes[i].x1,
+				  boxes[i].y2 - boxes[i].y1));
 		DMA_COMMIT();
 	}
 
@@ -894,11 +894,11 @@ static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
 static int savage_dispatch_draw(drm_savage_private_t *dev_priv,
 				const drm_savage_cmd_header_t *start,
 				const drm_savage_cmd_header_t *end,
-				const drm_buf_t *dmabuf,
+				const struct drm_buf *dmabuf,
 				const unsigned int *vtxbuf,
 				unsigned int vb_size, unsigned int vb_stride,
 				unsigned int nbox,
-				const drm_clip_rect_t *boxes)
+				const struct drm_clip_rect *boxes)
 {
 	unsigned int i, j;
 	int ret;
@@ -944,7 +944,7 @@ static int savage_dispatch_draw(drm_savage_private_t *dev_priv,
 				DRM_ERROR("IMPLEMENTATION ERROR: "
 					  "non-drawing-command %d\n",
 					  cmd_header.cmd.cmd);
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 
 			if (ret != 0)
@@ -955,35 +955,31 @@ static int savage_dispatch_draw(drm_savage_private_t *dev_priv,
 	return 0;
 }
 
-int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
+int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_savage_private_t *dev_priv = dev->dev_private;
-	drm_device_dma_t *dma = dev->dma;
-	drm_buf_t *dmabuf;
-	drm_savage_cmdbuf_t cmdbuf;
+	struct drm_device_dma *dma = dev->dma;
+	struct drm_buf *dmabuf;
+	drm_savage_cmdbuf_t *cmdbuf = data;
 	drm_savage_cmd_header_t *kcmd_addr = NULL;
 	drm_savage_cmd_header_t *first_draw_cmd;
 	unsigned int *kvb_addr = NULL;
-	drm_clip_rect_t *kbox_addr = NULL;
+	struct drm_clip_rect *kbox_addr = NULL;
 	unsigned int i, j;
 	int ret = 0;
 
 	DRM_DEBUG("\n");
-	
-	LOCK_TEST_WITH_RETURN(dev, filp);
 
-	DRM_COPY_FROM_USER_IOCTL(cmdbuf, (drm_savage_cmdbuf_t __user *)data,
-				 sizeof(cmdbuf));
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	if (dma && dma->buflist) {
-		if (cmdbuf.dma_idx > dma->buf_count) {
+		if (cmdbuf->dma_idx > dma->buf_count) {
 			DRM_ERROR
 			    ("vertex buffer index %u out of range (0-%u)\n",
-			     cmdbuf.dma_idx, dma->buf_count-1);
-			return DRM_ERR(EINVAL);
+			     cmdbuf->dma_idx, dma->buf_count - 1);
+			return -EINVAL;
 		}
-		dmabuf = dma->buflist[cmdbuf.dma_idx];
+		dmabuf = dma->buflist[cmdbuf->dma_idx];
 	} else {
 		dmabuf = NULL;
 	}
@@ -993,47 +989,49 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 	 * COPY_FROM_USER_UNCHECKED when done in other drivers, and is correct
 	 * for locking on FreeBSD.
 	 */
-	if (cmdbuf.size) {
-		kcmd_addr = drm_alloc(cmdbuf.size * 8, DRM_MEM_DRIVER);
+	if (cmdbuf->size) {
+		kcmd_addr = drm_alloc(cmdbuf->size * 8, DRM_MEM_DRIVER);
 		if (kcmd_addr == NULL)
-			return ENOMEM;
+			return -ENOMEM;
 
-		if (DRM_COPY_FROM_USER(kcmd_addr, cmdbuf.cmd_addr,
-				       cmdbuf.size * 8))
+		if (DRM_COPY_FROM_USER(kcmd_addr, cmdbuf->cmd_addr,
+				       cmdbuf->size * 8))
 		{
-			drm_free(kcmd_addr, cmdbuf.size * 8, DRM_MEM_DRIVER);
-			return DRM_ERR(EFAULT);
+			drm_free(kcmd_addr, cmdbuf->size * 8, DRM_MEM_DRIVER);
+			return -EFAULT;
 		}
-		cmdbuf.cmd_addr = kcmd_addr;
+		cmdbuf->cmd_addr = kcmd_addr;
 	}
-	if (cmdbuf.vb_size) {
-		kvb_addr = drm_alloc(cmdbuf.vb_size, DRM_MEM_DRIVER);
+	if (cmdbuf->vb_size) {
+		kvb_addr = drm_alloc(cmdbuf->vb_size, DRM_MEM_DRIVER);
 		if (kvb_addr == NULL) {
-			ret = DRM_ERR(ENOMEM);
+			ret = -ENOMEM;
 			goto done;
 		}
 
-		if (DRM_COPY_FROM_USER(kvb_addr, cmdbuf.vb_addr,
-				       cmdbuf.vb_size)) {
-			ret = DRM_ERR(EFAULT);
+		if (DRM_COPY_FROM_USER(kvb_addr, cmdbuf->vb_addr,
+				       cmdbuf->vb_size)) {
+			ret = -EFAULT;
 			goto done;
 		}
-		cmdbuf.vb_addr = kvb_addr;
+		cmdbuf->vb_addr = kvb_addr;
 	}
-	if (cmdbuf.nbox) {
-		kbox_addr = drm_alloc(cmdbuf.nbox * sizeof(drm_clip_rect_t),
-				       DRM_MEM_DRIVER);
+	if (cmdbuf->nbox) {
+		kbox_addr = drm_alloc(cmdbuf->nbox *
+				      sizeof(struct drm_clip_rect),
+				      DRM_MEM_DRIVER);
 		if (kbox_addr == NULL) {
-			ret = DRM_ERR(ENOMEM);
+			ret = -ENOMEM;
 			goto done;
 		}
 
-		if (DRM_COPY_FROM_USER(kbox_addr, cmdbuf.box_addr,
-				       cmdbuf.nbox * sizeof(drm_clip_rect_t))) {
-			ret = DRM_ERR(EFAULT);
+		if (DRM_COPY_FROM_USER(kbox_addr, cmdbuf->box_addr,
+				       cmdbuf->nbox *
+				       sizeof(struct drm_clip_rect))) {
+			ret = -EFAULT;
 			goto done;
 		}
-		cmdbuf.box_addr = kbox_addr;
+		cmdbuf->box_addr = kbox_addr;
 	}
 
 	/* Make sure writes to DMA buffers are finished before sending
@@ -1046,10 +1044,10 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 
 	i = 0;
 	first_draw_cmd = NULL;
-	while (i < cmdbuf.size) {
+	while (i < cmdbuf->size) {
 		drm_savage_cmd_header_t cmd_header;
-		cmd_header = *(drm_savage_cmd_header_t *)cmdbuf.cmd_addr;
-		cmdbuf.cmd_addr++;
+		cmd_header = *(drm_savage_cmd_header_t *)cmdbuf->cmd_addr;
+		cmdbuf->cmd_addr++;
 		i++;
 
 		/* Group drawing commands with same state to minimize
@@ -1059,28 +1057,29 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 		case SAVAGE_CMD_DMA_IDX:
 		case SAVAGE_CMD_VB_IDX:
 			j = (cmd_header.idx.count + 3) / 4;
-			if (i + j > cmdbuf.size) {
+			if (i + j > cmdbuf->size) {
 				DRM_ERROR("indexed drawing command extends "
 					  "beyond end of command buffer\n");
 				DMA_FLUSH();
-				return DRM_ERR(EINVAL);
+				return -EINVAL;
 			}
 			/* fall through */
 		case SAVAGE_CMD_DMA_PRIM:
 		case SAVAGE_CMD_VB_PRIM:
 			if (!first_draw_cmd)
-				first_draw_cmd = cmdbuf.cmd_addr-1;
-			cmdbuf.cmd_addr += j;
+				first_draw_cmd = cmdbuf->cmd_addr - 1;
+			cmdbuf->cmd_addr += j;
 			i += j;
 			break;
 		default:
 			if (first_draw_cmd) {
-				ret = savage_dispatch_draw (
+				ret = savage_dispatch_draw(
 					dev_priv, first_draw_cmd,
-					cmdbuf.cmd_addr-1,
-					dmabuf, cmdbuf.vb_addr, cmdbuf.vb_size,
-					cmdbuf.vb_stride,
-					cmdbuf.nbox, cmdbuf.box_addr);
+					cmdbuf->cmd_addr - 1,
+					dmabuf, cmdbuf->vb_addr,
+					cmdbuf->vb_size,
+					cmdbuf->vb_stride,
+					cmdbuf->nbox, cmdbuf->box_addr);
 				if (ret != 0)
 					return ret;
 				first_draw_cmd = NULL;
@@ -1092,40 +1091,42 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 		switch (cmd_header.cmd.cmd) {
 		case SAVAGE_CMD_STATE:
 			j = (cmd_header.state.count + 1) / 2;
-			if (i + j > cmdbuf.size) {
+			if (i + j > cmdbuf->size) {
 				DRM_ERROR("command SAVAGE_CMD_STATE extends "
 					  "beyond end of command buffer\n");
 				DMA_FLUSH();
-				ret = DRM_ERR(EINVAL);
+				ret = -EINVAL;
 				goto done;
 			}
 			ret = savage_dispatch_state(dev_priv, &cmd_header,
-				(const uint32_t *)cmdbuf.cmd_addr);
-			cmdbuf.cmd_addr += j;
+				(const uint32_t *)cmdbuf->cmd_addr);
+			cmdbuf->cmd_addr += j;
 			i += j;
 			break;
 		case SAVAGE_CMD_CLEAR:
-			if (i + 1 > cmdbuf.size) {
+			if (i + 1 > cmdbuf->size) {
 				DRM_ERROR("command SAVAGE_CMD_CLEAR extends "
 					  "beyond end of command buffer\n");
 				DMA_FLUSH();
-				ret = DRM_ERR(EINVAL);
+				ret = -EINVAL;
 				goto done;
 			}
 			ret = savage_dispatch_clear(dev_priv, &cmd_header,
-						    cmdbuf.cmd_addr,
-						    cmdbuf.nbox, cmdbuf.box_addr);
-			cmdbuf.cmd_addr++;
+						    cmdbuf->cmd_addr,
+						    cmdbuf->nbox,
+						    cmdbuf->box_addr);
+			cmdbuf->cmd_addr++;
 			i++;
 			break;
 		case SAVAGE_CMD_SWAP:
-			ret = savage_dispatch_swap(dev_priv, cmdbuf.nbox,
-						   cmdbuf.box_addr);
+			ret = savage_dispatch_swap(dev_priv, cmdbuf->nbox,
+						   cmdbuf->box_addr);
 			break;
 		default:
-			DRM_ERROR("invalid command 0x%x\n", cmd_header.cmd.cmd);
+			DRM_ERROR("invalid command 0x%x\n",
+				  cmd_header.cmd.cmd);
 			DMA_FLUSH();
-			ret = DRM_ERR(EINVAL);
+			ret = -EINVAL;
 			goto done;
 		}
 
@@ -1136,10 +1137,10 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 	}
 
 	if (first_draw_cmd) {
-		ret = savage_dispatch_draw (
-			dev_priv, first_draw_cmd, cmdbuf.cmd_addr, dmabuf,
-			cmdbuf.vb_addr, cmdbuf.vb_size, cmdbuf.vb_stride,
-			cmdbuf.nbox, cmdbuf.box_addr);
+		ret = savage_dispatch_draw(
+			dev_priv, first_draw_cmd, cmdbuf->cmd_addr, dmabuf,
+			cmdbuf->vb_addr, cmdbuf->vb_size, cmdbuf->vb_stride,
+			cmdbuf->nbox, cmdbuf->box_addr);
 		if (ret != 0) {
 			DMA_FLUSH();
 			goto done;
@@ -1148,7 +1149,7 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 
 	DMA_FLUSH();
 
-	if (dmabuf && cmdbuf.discard) {
+	if (dmabuf && cmdbuf->discard) {
 		drm_savage_buf_priv_t *buf_priv = dmabuf->dev_private;
 		uint16_t event;
 		event = savage_bci_emit_event(dev_priv, SAVAGE_WAIT_3D);
@@ -1158,9 +1159,9 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 
 done:
 	/* If we didn't need to allocate them, these'll be NULL */
-	drm_free(kcmd_addr, cmdbuf.size * 8, DRM_MEM_DRIVER);
-	drm_free(kvb_addr, cmdbuf.vb_size, DRM_MEM_DRIVER);
-	drm_free(kbox_addr, cmdbuf.nbox * sizeof(drm_clip_rect_t),
+	drm_free(kcmd_addr, cmdbuf->size * 8, DRM_MEM_DRIVER);
+	drm_free(kvb_addr, cmdbuf->vb_size, DRM_MEM_DRIVER);
+	drm_free(kbox_addr, cmdbuf->nbox * sizeof(struct drm_clip_rect),
 		 DRM_MEM_DRIVER);
 
 	return ret;
diff --git a/sys/dev/drm/sis_drv.c b/sys/dev/drm/sis_drv.c
index 04d1451..a07f60a 100644
--- a/sys/dev/drm/sis_drv.c
+++ b/sys/dev/drm/sis_drv.c
@@ -39,7 +39,7 @@ static drm_pci_id_list_t sis_pciidlist[] = {
 	sis_PCI_IDS
 };
 
-static void sis_configure(drm_device_t *dev)
+static void sis_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= 1; /* No dev_priv */
 	dev->driver.context_ctor	= sis_init_context;
@@ -69,9 +69,9 @@ sis_probe(device_t dev)
 static int
 sis_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	sis_configure(dev);
 	return drm_attach(nbdev, sis_pciidlist);
 }
@@ -88,7 +88,7 @@ static device_method_t sis_methods[] = {
 static driver_t sis_driver = {
 	"drm",
 	sis_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
@@ -103,7 +103,7 @@ MODULE_DEPEND(sisdrm, drm, 1, 1, 1);
 #ifdef _LKM
 CFDRIVER_DECL(sis, DV_TTY, NULL);
 #else
-CFATTACH_DECL(sis, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+CFATTACH_DECL(sis, sizeof(struct drm_device), drm_probe, drm_attach, drm_detach,
     drm_activate);
 #endif
 #endif
diff --git a/sys/dev/drm/sis_drv.h b/sys/dev/drm/sis_drv.h
index 7fc3771..18b1503 100644
--- a/sys/dev/drm/sis_drv.h
+++ b/sys/dev/drm/sis_drv.h
@@ -34,14 +34,47 @@ __FBSDID("$FreeBSD$");
 /* General customization:
  */
 
-#define DRIVER_AUTHOR		"SIS"
+#define DRIVER_AUTHOR		"SIS, Tungsten Graphics"
 #define DRIVER_NAME		"sis"
-#define DRIVER_DESC		"SIS 300/630/540"
-#define DRIVER_DATE		"20030826"
+#define DRIVER_DESC		"SIS 300/630/540 and XGI V3XE/V5/V8"
+#define DRIVER_DATE		"20070626"
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		1
+#define DRIVER_MINOR		3
 #define DRIVER_PATCHLEVEL	0
 
+enum sis_family {
+	SIS_OTHER = 0,
+	SIS_CHIP_315 = 1,
+};
+
+#if defined(__linux__)
+#define SIS_HAVE_CORE_MM
+#endif
+
+#ifdef SIS_HAVE_CORE_MM
+#include "dev/drm/drm_sman.h"
+
+#define SIS_BASE (dev_priv->mmio)
+#define SIS_READ(reg)	 DRM_READ32(SIS_BASE, reg);
+#define SIS_WRITE(reg, val)   DRM_WRITE32(SIS_BASE, reg, val);
+
+typedef struct drm_sis_private {
+	drm_local_map_t *mmio;
+	unsigned int idle_fault;
+	struct drm_sman sman;
+	unsigned int chipset;
+	int vram_initialized;
+	int agp_initialized;
+	unsigned long vram_offset;
+	unsigned long agp_offset;
+} drm_sis_private_t;
+
+extern int sis_idle(struct drm_device *dev);
+extern void sis_reclaim_buffers_locked(struct drm_device *dev,
+				       struct drm_file *file_priv);
+extern void sis_lastclose(struct drm_device *dev);
+
+#else
 #include "dev/drm/sis_ds.h"
 
 typedef struct drm_sis_private {
@@ -49,10 +82,12 @@ typedef struct drm_sis_private {
 	memHeap_t *FBHeap;
 } drm_sis_private_t;
 
-extern int sis_init_context(drm_device_t * dev, int context);
-extern int sis_final_context(drm_device_t * dev, int context);
+extern int sis_init_context(struct drm_device * dev, int context);
+extern int sis_final_context(struct drm_device * dev, int context);
+
+#endif
 
-extern drm_ioctl_desc_t sis_ioctls[];
+extern struct drm_ioctl_desc sis_ioctls[];
 extern int sis_max_ioctl;
 
 #endif
diff --git a/sys/dev/drm/sis_mm.c b/sys/dev/drm/sis_mm.c
index 188fede..ea04eaa 100644
--- a/sys/dev/drm/sis_mm.c
+++ b/sys/dev/drm/sis_mm.c
@@ -84,59 +84,52 @@ static int del_alloc_set(int context, int type, unsigned int val)
 /* fb management via fb device */
 #if defined(__linux__) && defined(CONFIG_FB_SIS)
 
-static int sis_fb_init(DRM_IOCTL_ARGS)
+static int sis_fb_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	return 0;
 }
 
-static int sis_fb_alloc(DRM_IOCTL_ARGS)
+static int sis_fb_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	drm_sis_mem_t fb;
+	drm_sis_mem_t *fb = data;
 	struct sis_memreq req;
-	drm_sis_mem_t __user *argp = (drm_sis_mem_t __user *)data;
 	int retval = 0;
 
-	DRM_COPY_FROM_USER_IOCTL(fb, argp, sizeof(fb));
-
-	req.size = fb.size;
+	req.size = fb->size;
 	sis_malloc(&req);
 	if (req.offset) {
 		/* TODO */
-		fb.offset = req.offset;
-		fb.free = req.offset;
-		if (!add_alloc_set(fb.context, VIDEO_TYPE, fb.free)) {
+		fb->offset = req.offset;
+		fb->free = req.offset;
+		if (!add_alloc_set(fb->context, VIDEO_TYPE, fb->free)) {
 			DRM_DEBUG("adding to allocation set fails\n");
 			sis_free(req.offset);
-			retval = DRM_ERR(EINVAL);
+			retval = -EINVAL;
 		}
 	} else {
-		fb.offset = 0;
-		fb.size = 0;
-		fb.free = 0;
+		fb->offset = 0;
+		fb->size = 0;
+		fb->free = 0;
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, fb, sizeof(fb));
-
-	DRM_DEBUG("alloc fb, size = %d, offset = %ld\n", fb.size, req.offset);
+	DRM_DEBUG("alloc fb, size = %d, offset = %ld\n", fb->size, req.offset);
 
 	return retval;
 }
 
-static int sis_fb_free(DRM_IOCTL_ARGS)
+static int sis_fb_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	drm_sis_mem_t fb;
 	int retval = 0;
 
-	DRM_COPY_FROM_USER_IOCTL(fb, (drm_sis_mem_t __user *) data, sizeof(fb));
-
-	if (!fb.free)
-		return DRM_ERR(EINVAL);
+	if (!fb->free)
+		return -EINVAL;
 
-	if (!del_alloc_set(fb.context, VIDEO_TYPE, fb.free))
-		retval = DRM_ERR(EINVAL);
-	sis_free(fb.free);
+	if (!del_alloc_set(fb->context, VIDEO_TYPE, fb->free))
+		retval = -EINVAL;
+	sis_free(fb->free);
 
-	DRM_DEBUG("free fb, offset = 0x%lx\n", fb.free);
+	DRM_DEBUG("free fb, offset = 0x%lx\n", fb->free);
 
 	return retval;
 }
@@ -153,13 +146,10 @@ static int sis_fb_free(DRM_IOCTL_ARGS)
  *    X driver/sisfb                                  HW-   Command-
  *  framebuffer memory           DRI heap           Cursor   queue
  */
-static int sis_fb_init(DRM_IOCTL_ARGS)
+static int sis_fb_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_fb_t fb;
-
-	DRM_COPY_FROM_USER_IOCTL(fb, (drm_sis_fb_t __user *) data, sizeof(fb));
+	drm_sis_fb_t *fb = data;
 
 	if (dev_priv == NULL) {
 		dev->dev_private = drm_calloc(1, sizeof(drm_sis_private_t),
@@ -170,71 +160,62 @@ static int sis_fb_init(DRM_IOCTL_ARGS)
 	}
 
 	if (dev_priv->FBHeap != NULL)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
-	dev_priv->FBHeap = mmInit(fb.offset, fb.size);
+	dev_priv->FBHeap = mmInit(fb->offset, fb->size);
 
-	DRM_DEBUG("offset = %u, size = %u", fb.offset, fb.size);
+	DRM_DEBUG("offset = %u, size = %u", fb->offset, fb->size);
 
 	return 0;
 }
 
-static int sis_fb_alloc(DRM_IOCTL_ARGS)
+static int sis_fb_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_mem_t __user *argp = (drm_sis_mem_t __user *)data;
-	drm_sis_mem_t fb;
+	drm_sis_mem_t *fb = data;
 	PMemBlock block;
 	int retval = 0;
 
 	if (dev_priv == NULL || dev_priv->FBHeap == NULL)
-		return DRM_ERR(EINVAL);
-
-	DRM_COPY_FROM_USER_IOCTL(fb, argp, sizeof(fb));
+		return -EINVAL;
 
-	block = mmAllocMem(dev_priv->FBHeap, fb.size, 0, 0);
+	block = mmAllocMem(dev_priv->FBHeap, fb->size, 0, 0);
 	if (block) {
 		/* TODO */
-		fb.offset = block->ofs;
-		fb.free = (unsigned long)block;
-		if (!add_alloc_set(fb.context, VIDEO_TYPE, fb.free)) {
+		fb->offset = block->ofs;
+		fb->free = (unsigned long)block;
+		if (!add_alloc_set(fb->context, VIDEO_TYPE, fb->free)) {
 			DRM_DEBUG("adding to allocation set fails\n");
-			mmFreeMem((PMemBlock) fb.free);
-			retval = DRM_ERR(EINVAL);
+			mmFreeMem((PMemBlock) fb->free);
+			retval = -EINVAL;
 		}
 	} else {
-		fb.offset = 0;
-		fb.size = 0;
-		fb.free = 0;
+		fb->offset = 0;
+		fb->size = 0;
+		fb->free = 0;
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, fb, sizeof(fb));
-
-	DRM_DEBUG("alloc fb, size = %d, offset = %d\n", fb.size, fb.offset);
+	DRM_DEBUG("alloc fb, size = %d, offset = %d\n", fb->size, fb->offset);
 
 	return retval;
 }
 
-static int sis_fb_free(DRM_IOCTL_ARGS)
+static int sis_fb_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_mem_t fb;
+	drm_sis_mem_t *fb = data;
 
 	if (dev_priv == NULL || dev_priv->FBHeap == NULL)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
-	DRM_COPY_FROM_USER_IOCTL(fb, (drm_sis_mem_t __user *) data, sizeof(fb));
+	if (!mmBlockInHeap(dev_priv->FBHeap, (PMemBlock) fb->free))
+		return -EINVAL;
 
-	if (!mmBlockInHeap(dev_priv->FBHeap, (PMemBlock) fb.free))
-		return DRM_ERR(EINVAL);
+	if (!del_alloc_set(fb->context, VIDEO_TYPE, fb->free))
+		return -EINVAL;
+	mmFreeMem((PMemBlock) fb->free);
 
-	if (!del_alloc_set(fb.context, VIDEO_TYPE, fb.free))
-		return DRM_ERR(EINVAL);
-	mmFreeMem((PMemBlock) fb.free);
-
-	DRM_DEBUG("free fb, free = 0x%lx\n", fb.free);
+	DRM_DEBUG("free fb, free = 0x%lx\n", fb->free);
 
 	return 0;
 }
@@ -243,11 +224,10 @@ static int sis_fb_free(DRM_IOCTL_ARGS)
 
 /* agp memory management */
 
-static int sis_ioctl_agp_init(DRM_IOCTL_ARGS)
+static int sis_ioctl_agp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_agp_t agp;
+	drm_sis_agp_t *agp = data;
 
 	if (dev_priv == NULL) {
 		dev->dev_private = drm_calloc(1, sizeof(drm_sis_private_t),
@@ -258,75 +238,63 @@ static int sis_ioctl_agp_init(DRM_IOCTL_ARGS)
 	}
 
 	if (dev_priv->AGPHeap != NULL)
-		return DRM_ERR(EINVAL);
-
-	DRM_COPY_FROM_USER_IOCTL(agp, (drm_sis_agp_t __user *) data,
-				 sizeof(agp));
+		return -EINVAL;
 
-	dev_priv->AGPHeap = mmInit(agp.offset, agp.size);
+	dev_priv->AGPHeap = mmInit(agp->offset, agp->size);
 
-	DRM_DEBUG("offset = %u, size = %u", agp.offset, agp.size);
+	DRM_DEBUG("offset = %u, size = %u", agp->offset, agp->size);
 
 	return 0;
 }
 
-static int sis_ioctl_agp_alloc(DRM_IOCTL_ARGS)
+static int sis_ioctl_agp_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_mem_t __user *argp = (drm_sis_mem_t __user *)data;
-	drm_sis_mem_t agp;
+	drm_sis_mem_t *agp = data;
 	PMemBlock block;
 	int retval = 0;
 
 	if (dev_priv == NULL || dev_priv->AGPHeap == NULL)
-		return DRM_ERR(EINVAL);
+		return -EINVAL;
 
-	DRM_COPY_FROM_USER_IOCTL(agp, argp, sizeof(agp));
-
-	block = mmAllocMem(dev_priv->AGPHeap, agp.size, 0, 0);
+	block = mmAllocMem(dev_priv->AGPHeap, agp->size, 0, 0);
 	if (block) {
 		/* TODO */
-		agp.offset = block->ofs;
-		agp.free = (unsigned long)block;
-		if (!add_alloc_set(agp.context, AGP_TYPE, agp.free)) {
+		agp->offset = block->ofs;
+		agp->free = (unsigned long)block;
+		if (!add_alloc_set(agp->context, AGP_TYPE, agp->free)) {
 			DRM_DEBUG("adding to allocation set fails\n");
-			mmFreeMem((PMemBlock) agp.free);
+			mmFreeMem((PMemBlock) agp->free);
 			retval = -1;
 		}
 	} else {
-		agp.offset = 0;
-		agp.size = 0;
-		agp.free = 0;
+		agp->offset = 0;
+		agp->size = 0;
+		agp->free = 0;
 	}
 
-	DRM_COPY_TO_USER_IOCTL(argp, agp, sizeof(agp));
-
-	DRM_DEBUG("alloc agp, size = %d, offset = %d\n", agp.size, agp.offset);
+	DRM_DEBUG("alloc agp, size = %d, offset = %d\n", agp->size,
+	    agp->offset);
 
 	return retval;
 }
 
-static int sis_ioctl_agp_free(DRM_IOCTL_ARGS)
+static int sis_ioctl_agp_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
-	DRM_DEVICE;
 	drm_sis_private_t *dev_priv = dev->dev_private;
-	drm_sis_mem_t agp;
+	drm_sis_mem_t *agp = data;
 
 	if (dev_priv == NULL || dev_priv->AGPHeap == NULL)
-		return DRM_ERR(EINVAL);
-
-	DRM_COPY_FROM_USER_IOCTL(agp, (drm_sis_mem_t __user *) data,
-				 sizeof(agp));
+		return -EINVAL;
 
-	if (!mmBlockInHeap(dev_priv->AGPHeap, (PMemBlock) agp.free))
-		return DRM_ERR(EINVAL);
+	if (!mmBlockInHeap(dev_priv->AGPHeap, (PMemBlock) agp->free))
+		return -EINVAL;
 
-	mmFreeMem((PMemBlock) agp.free);
-	if (!del_alloc_set(agp.context, AGP_TYPE, agp.free))
-		return DRM_ERR(EINVAL);
+	mmFreeMem((PMemBlock) agp->free);
+	if (!del_alloc_set(agp->context, AGP_TYPE, agp->free))
+		return -EINVAL;
 
-	DRM_DEBUG("free agp, free = 0x%lx\n", agp.free);
+	DRM_DEBUG("free agp, free = 0x%lx\n", agp->free);
 
 	return 0;
 }
@@ -410,12 +378,12 @@ int sis_final_context(struct drm_device *dev, int context)
 }
 
 drm_ioctl_desc_t sis_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_SIS_FB_ALLOC)] = {sis_fb_alloc, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SIS_FB_FREE)] = {sis_fb_free, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_INIT)] = {sis_ioctl_agp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_ALLOC)] = {sis_ioctl_agp_alloc, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_FREE)] = {sis_ioctl_agp_free, DRM_AUTH},
-	[DRM_IOCTL_NR(DRM_SIS_FB_INIT)] = {sis_fb_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY}
+	DRM_IOCTL_DEF(DRM_SIS_FB_ALLOC, sis_fb_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SIS_FB_FREE, sis_fb_free, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SIS_AGP_INIT, sis_ioctl_agp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_SIS_AGP_ALLOC, sis_ioctl_agp_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SIS_AGP_FREE, sis_ioctl_agp_free, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_SIS_FB_INIT, sis_fb_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY)
 };
 
 int sis_max_ioctl = DRM_ARRAY_SIZE(sis_ioctls);
diff --git a/sys/dev/drm/tdfx_drv.c b/sys/dev/drm/tdfx_drv.c
index 87ecf3f..5910cdc 100644
--- a/sys/dev/drm/tdfx_drv.c
+++ b/sys/dev/drm/tdfx_drv.c
@@ -44,7 +44,7 @@ static drm_pci_id_list_t tdfx_pciidlist[] = {
 	tdfx_PCI_IDS
 };
 
-static void tdfx_configure(drm_device_t *dev)
+static void tdfx_configure(struct drm_device *dev)
 {
 	dev->driver.buf_priv_size	= 1; /* No dev_priv */
 
@@ -70,9 +70,9 @@ tdfx_probe(device_t dev)
 static int
 tdfx_attach(device_t nbdev)
 {
-	drm_device_t *dev = device_get_softc(nbdev);
+	struct drm_device *dev = device_get_softc(nbdev);
 
-	bzero(dev, sizeof(drm_device_t));
+	bzero(dev, sizeof(struct drm_device));
 	tdfx_configure(dev);
 	return drm_attach(nbdev, tdfx_pciidlist);
 }
@@ -89,7 +89,7 @@ static device_method_t tdfx_methods[] = {
 static driver_t tdfx_driver = {
 	"drm",
 	tdfx_methods,
-	sizeof(drm_device_t)
+	sizeof(struct drm_device)
 };
 
 extern devclass_t drm_devclass;
@@ -104,7 +104,7 @@ MODULE_DEPEND(tdfx, drm, 1, 1, 1);
 #ifdef _LKM
 CFDRIVER_DECL(tdfx, DV_TTY, NULL);
 #else
-CFATTACH_DECL(tdfx, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
-    drm_activate);
+CFATTACH_DECL(tdfx, sizeof(struct drm_device), drm_probe, drm_attach,
+    drm_detach, drm_activate);
 #endif
 #endif
diff --git a/sys/modules/drm/i915/Makefile b/sys/modules/drm/i915/Makefile
index a34695a..b64539e 100644
--- a/sys/modules/drm/i915/Makefile
+++ b/sys/modules/drm/i915/Makefile
@@ -2,7 +2,7 @@
 
 .PATH:	${.CURDIR}/../../../dev/drm
 KMOD	= i915
-SRCS	= i915_dma.c i915_drv.c i915_irq.c i915_mem.c
+SRCS	= i915_dma.c i915_drv.c i915_irq.c i915_mem.c i915_suspend.c
 SRCS	+=device_if.h bus_if.h pci_if.h opt_drm.h
 
 .include <bsd.kmod.mk>