79 files changed, 3509 insertions, 1722 deletions

diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
index bfcd13049b92..2a90df515b7c 100644
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -208,6 +208,7 @@ device		mach64drm	# ATI Rage Pro, Rage Mobility P/M, Rage XL
 device		mgadrm		# AGP Matrox G200, G400, G450, G550
 device		r128drm		# ATI Rage 128
 device		radeondrm	# ATI Radeon
+device		savagedrm	# S3 Savage3D, Savage4
 device		sisdrm		# SiS 300/305, 540, 630
 device		tdfxdrm		# 3dfx Voodoo 3/4/5 and Banshee
 options 	DRM_DEBUG	# Include debug printfs (slow)
diff --git a/sys/conf/files b/sys/conf/files
index 53b27cd4d679..6e6832cc8476 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -556,6 +556,9 @@ dev/drm/radeon_drv.c		optional radeondrm
 dev/drm/radeon_irq.c		optional radeondrm
 dev/drm/radeon_mem.c		optional radeondrm
 dev/drm/radeon_state.c		optional radeondrm
+dev/drm/savage_bci.c		optional savagedrm
+dev/drm/savage_drv.c		optional savagedrm
+dev/drm/savage_state.c		optional savagedrm
 dev/drm/sis_drv.c		optional sisdrm
 dev/drm/sis_ds.c		optional sisdrm
 dev/drm/sis_mm.c		optional sisdrm
diff --git a/sys/dev/drm/ati_pcigart.c b/sys/dev/drm/ati_pcigart.c
index 1442247e9852..386702c6ae1a 100644
--- a/sys/dev/drm/ati_pcigart.c
+++ b/sys/dev/drm/ati_pcigart.c
@@ -27,66 +27,58 @@
  * Authors:
  *   Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
-#include "dev/drm/drmP.h"
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#if PAGE_SIZE == 8192
-# define ATI_PCIGART_TABLE_ORDER 	2
-# define ATI_PCIGART_TABLE_PAGES 	(1 << 2)
-#elif PAGE_SIZE == 4096
-# define ATI_PCIGART_TABLE_ORDER 	3
-# define ATI_PCIGART_TABLE_PAGES 	(1 << 3)
-#elif
-# error - PAGE_SIZE not 8K or 4K
-#endif
+#include "dev/drm/drmP.h"
 
-# define ATI_MAX_PCIGART_PAGES		8192	/* 32 MB aperture, 4K pages */
-# define ATI_PCIGART_PAGE_SIZE		4096	/* PCI GART page size */
+#define ATI_PCIGART_PAGE_SIZE		4096	/* PCI GART page size */
+#define ATI_MAX_PCIGART_PAGES		8192	/* 32 MB aperture, 4K pages */
+#define ATI_PCIGART_TABLE_SIZE		32768
 
-int drm_ati_pcigart_init(drm_device_t *dev, unsigned long *addr,
-			 dma_addr_t *bus_addr, int is_pcie)
+int drm_ati_pcigart_init(drm_device_t *dev, drm_ati_pcigart_info *gart_info)
 {
-	drm_sg_mem_t *entry = dev->sg;
-	unsigned long address = 0;
 	unsigned long pages;
-	u32 *pci_gart=0, page_base, bus_address = 0;
-	int i, j, ret = 0;
+	u32 *pci_gart = NULL, page_base;
+	int i, j;
 
-	if ( !entry ) {
+	if (dev->sg == NULL) {
 		DRM_ERROR( "no scatter/gather memory!\n" );
-		goto done;
+		return 0;
 	}
 
-	address = (long)contigmalloc((1 << ATI_PCIGART_TABLE_ORDER) * PAGE_SIZE, 
-	    M_DRM, M_NOWAIT, 0ul, 0xfffffffful, PAGE_SIZE, 0);
-	if ( !address ) {
-		DRM_ERROR( "cannot allocate PCI GART page!\n" );
-		goto done;
+	if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) {
+		/* GART table in system memory */
+		dev->sg->dmah = drm_pci_alloc(dev, ATI_PCIGART_TABLE_SIZE, 0,
+		    0xfffffffful);
+		if (dev->sg->dmah == NULL) {
+			DRM_ERROR("cannot allocate PCI GART table!\n");
+			return 0;
+		}
+	
+		gart_info->addr = (void *)dev->sg->dmah->vaddr;
+		gart_info->bus_addr = dev->sg->dmah->busaddr;
+		pci_gart = (u32 *)dev->sg->dmah->vaddr;
+	} else {
+		/* GART table in framebuffer memory */
+		pci_gart = gart_info->addr;
 	}
+	
+	pages = DRM_MIN(dev->sg->pages, ATI_MAX_PCIGART_PAGES);
 
-	/* XXX: we need to busdma this */
-	bus_address = vtophys( address );
-
-	pci_gart = (u32 *)address;
+	bzero(pci_gart, ATI_PCIGART_TABLE_SIZE);
 
-	pages = ( entry->pages <= ATI_MAX_PCIGART_PAGES )
-		? entry->pages : ATI_MAX_PCIGART_PAGES;
-
-	bzero( pci_gart, ATI_MAX_PCIGART_PAGES * sizeof(u32) );
+	KASSERT(PAGE_SIZE >= ATI_PCIGART_PAGE_SIZE, ("page size too small"));
 
 	for ( i = 0 ; i < pages ; i++ ) {
-		entry->busaddr[i] = vtophys( entry->handle + (i*PAGE_SIZE) );
-		page_base = (u32) entry->busaddr[i];
+		page_base = (u32) dev->sg->busaddr[i];
 
 		for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
-			if (is_pcie) {
-				*pci_gart = (cpu_to_le32(page_base)>>8) | 0xc;
-				DRM_DEBUG("PCIE: %d %08X %08X to %p\n", i,
-				    page_base, (cpu_to_le32(page_base)>>8)|0xc,
-				    pci_gart);
-			} else
+			if (gart_info->is_pcie)
+				*pci_gart = (cpu_to_le32(page_base) >> 8) | 0xc;
+			else
 				*pci_gart = cpu_to_le32(page_base);
 			pci_gart++;
 			page_base += ATI_PCIGART_PAGE_SIZE;
@@ -95,29 +87,17 @@ int drm_ati_pcigart_init(drm_device_t *dev, unsigned long *addr,
 
 	DRM_MEMORYBARRIER();
 
-	ret = 1;
-
-done:
-	*addr = address;
-	*bus_addr = bus_address;
-	return ret;
+	return 1;
 }
 
-int drm_ati_pcigart_cleanup(drm_device_t *dev, unsigned long addr,
-			    dma_addr_t bus_addr)
+int drm_ati_pcigart_cleanup(drm_device_t *dev, drm_ati_pcigart_info *gart_info)
 {
-	drm_sg_mem_t *entry = dev->sg;
-
-	/* we need to support large memory configurations */
-	if ( !entry ) {
+	if (dev->sg == NULL) {
 		DRM_ERROR( "no scatter/gather memory!\n" );
 		return 0;
 	}
 
-#if __FreeBSD_version > 500000
-	/* Not available on 4.x */
-	contigfree((void *)addr, (1 << ATI_PCIGART_TABLE_ORDER) * PAGE_SIZE,
-		   M_DRM);
-#endif
+	drm_pci_free(dev, dev->sg->dmah);
+
 	return 1;
 }
diff --git a/sys/dev/drm/drm-preprocess.sh b/sys/dev/drm/drm-preprocess.sh
new file mode 100644
index 000000000000..b636e2bb722f
--- /dev/null
+++ b/sys/dev/drm/drm-preprocess.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# $FreeBSD$
+
+cp /usr/src/drm/bsd-core/*.[ch] .
+rm i810*.[ch]
+rm via*.[ch]
+
+# Replace drm_pciids.h with one with a $FreeBSD$
+rm -f drm_pciids.h
+echo "/*" >> drm_pciids.h
+echo " * \$FreeBSD\$" >> drm_pciids.h
+echo " */" >> drm_pciids.h
+cat /usr/src/drm/bsd-core/drm_pciids.h >> drm_pciids.h
+
+for i in `ls *.[ch]`; do
+	mv $i $i.orig
+	perl drm-subprocess.pl < $i.orig > $i
+done
+
diff --git a/sys/dev/drm/drm-subprocess.pl b/sys/dev/drm/drm-subprocess.pl
new file mode 100644
index 000000000000..d053c595c12c
--- /dev/null
+++ b/sys/dev/drm/drm-subprocess.pl
@@ -0,0 +1,50 @@
+# $FreeBSD$
+#
+# Props to Daniel Stone for starting this script for me.  I hate perl.
+
+my $lastline = "";
+my $foundopening = 0;
+my $foundclosing = 0;
+
+while (<>) {
+	$curline = $_;
+	if (!$foundopening) {
+		if (/Copyright/) {
+			$foundopening = 1;
+			# print the previous line we buffered, but with /*-
+			if ($lastline !~ /\/\*-/) {
+				$lastline =~ s/\/\*/\/\*-/;
+			}
+			print $lastline;
+			# now, print the current line.
+			print $curline;
+		} else {
+			# print the previous line and continue on
+			print $lastline;
+		}
+	} elsif ($foundopening && !$foundclosing && /\*\//) {
+		# print the $FreeBSD$ bits after the end of the license block
+		$foundclosing = 1;
+		print;
+		print "\n";
+		print "#include <sys/cdefs.h>\n";
+		print "__FBSDID(\"\$FreeBSD\$\");\n";
+	} elsif ($foundopening) {
+		# Replace headers with the system's paths.  the headers we're
+		# concerned with are drm*.h, *_drm.h and *_drv.h
+		# 
+		s/#include "(.*)_drv.h/#include "dev\/drm\/\1_drv.h/;
+		s/#include "(.*)_drm.h/#include "dev\/drm\/\1_drm.h/;
+		s/#include "mga_ucode.h/#include "dev\/drm\/mga_ucode.h/;
+		s/#include "r300_reg.h/#include "dev\/drm\/r300_reg.h/;
+		s/#include "sis_ds.h/#include "dev\/drm\/sis_ds.h/;
+		s/#include "drm/#include "dev\/drm\/drm/;
+		print;
+	}
+	$lastline = $curline;
+}
+
+# if we never found the copyright header, then we're still a line behind.
+if (!$foundopening) {
+	print $lastline;
+}
\ No newline at end of file
diff --git a/sys/dev/drm/drm.h b/sys/dev/drm/drm.h
index 00a79bfe941e..fc46d5403cdc 100644
--- a/sys/dev/drm/drm.h
+++ b/sys/dev/drm/drm.h
@@ -31,10 +31,11 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 /**
  * \mainpage
  *
@@ -64,8 +65,16 @@
 #define __user
 #endif
 
+#ifdef __GNUC__
+# define DEPRECATED  __attribute__ ((deprecated))
+#else
+# define DEPRECATED
+#endif
+
 #if defined(__linux__)
+#if defined(__KERNEL__)
 #include <linux/config.h>
+#endif
 #include <asm/ioctl.h>		/* For _IO* macros */
 #define DRM_IOCTL_NR(n)		_IOC_NR(n)
 #define DRM_IOC_VOID		_IOC_NONE
@@ -73,7 +82,7 @@
 #define DRM_IOC_WRITE		_IOC_WRITE
 #define DRM_IOC_READWRITE	_IOC_READ|_IOC_WRITE
 #define DRM_IOC(dir, group, nr, size) _IOC(dir, group, nr, size)
-#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
+#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
 #if defined(__FreeBSD__) && defined(IN_MODULE)
 /* Prevent name collision when including sys/ioccom.h */
 #undef ioctl
@@ -127,7 +136,11 @@
 #define _DRM_LOCK_IS_CONT(lock)	   ((lock) & _DRM_LOCK_CONT)
 #define _DRM_LOCKING_CONTEXT(lock) ((lock) & ~(_DRM_LOCK_HELD|_DRM_LOCK_CONT))
 
+#if defined(__linux__)
+typedef unsigned int drm_handle_t;
+#else
 typedef unsigned long drm_handle_t;	/**< To mapped regions */
+#endif
 typedef unsigned int drm_context_t;	/**< GLXContext handle */
 typedef unsigned int drm_drawable_t;
 typedef unsigned int drm_magic_t;	/**< Magic for authentication */
@@ -439,7 +452,11 @@ typedef struct drm_buf_pub {
  */
 typedef struct drm_buf_map {
 	int count;		/**< Length of the buffer list */
+#if defined(__cplusplus)
+	void __user *c_virtual;
+#else
 	void __user *virtual;		/**< Mmap'd area in user-virtual */
+#endif
 	drm_buf_pub_t __user *list;	/**< Buffer information */
 } drm_buf_map_t;
 
diff --git a/sys/dev/drm/drmP.h b/sys/dev/drm/drmP.h
index d17cabf98b4a..081d98ce5de0 100644
--- a/sys/dev/drm/drmP.h
+++ b/sys/dev/drm/drmP.h
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef _DRM_P_H_
 #define _DRM_P_H_
 
@@ -216,6 +218,13 @@ typedef void			irqreturn_t;
 #define IRQ_HANDLED		/* nothing */
 #define IRQ_NONE		/* nothing */
 
+enum {
+	DRM_IS_NOT_AGP,
+	DRM_MIGHT_BE_AGP,
+	DRM_IS_AGP
+};
+#define DRM_AGP_MEM		struct agp_memory_info
+
 #if defined(__FreeBSD__)
 #define DRM_DEVICE							\
 	drm_device_t *dev = kdev->si_drv1
@@ -223,7 +232,8 @@ typedef void			irqreturn_t;
 				int flags, DRM_STRUCTPROC *p, DRMFILE filp
 
 #define PAGE_ALIGN(addr) round_page(addr)
-#define DRM_SUSER(p)		suser(p)
+/* DRM_SUSER returns true if the user is superuser */
+#define DRM_SUSER(p)		(suser(p) == 0)
 #define DRM_AGP_FIND_DEVICE()	agp_find_device()
 #define DRM_MTRR_WC		MDF_WRITECOMBINE
 #define jiffies			ticks
@@ -243,7 +253,8 @@ typedef void			irqreturn_t;
 
 #define CDEV_MAJOR		34
 #define PAGE_ALIGN(addr)	(((addr) + PAGE_SIZE - 1) & PAGE_MASK)
-#define DRM_SUSER(p)		suser(p->p_ucred, &p->p_acflag)
+/* DRM_SUSER returns true if the user is superuser */
+#define DRM_SUSER(p)		(suser(p->p_ucred, &p->p_acflag) == 0)
 #define DRM_AGP_FIND_DEVICE()	agp_find_device(0)
 #define DRM_MTRR_WC		MTRR_TYPE_WC
 #define jiffies			hardclock_ticks
@@ -440,10 +451,12 @@ typedef struct drm_pci_id_list
 	char *name;
 } drm_pci_id_list_t;
 
+#define DRM_AUTH	0x1
+#define DRM_MASTER	0x2
+#define DRM_ROOT_ONLY	0x4
 typedef struct drm_ioctl_desc {
 	int		     (*func)(DRM_IOCTL_ARGS);
-	int		     auth_needed;
-	int		     root_only;
+	int flags;
 } drm_ioctl_desc_t;
 
 typedef struct drm_magic_entry {
@@ -491,14 +504,24 @@ typedef struct drm_freelist {
 	int		  high_mark;   /* High water mark		   */
 } drm_freelist_t;
 
+typedef struct drm_dma_handle {
+	void *vaddr;
+	bus_addr_t busaddr;
+#if defined(__FreeBSD__)
+	bus_dma_tag_t tag;
+	bus_dmamap_t map;
+#elif defined(__NetBSD__)
+	bus_dma_segment_t seg;
+#endif
+} drm_dma_handle_t;
+
 typedef struct drm_buf_entry {
 	int		  buf_size;
 	int		  buf_count;
 	drm_buf_t	  *buflist;
 	int		  seg_count;
+	drm_dma_handle_t  **seglist;
 	int		  page_order;
-	vm_offset_t	  *seglist;
-	dma_addr_t	  *seglist_bus;
 
 	drm_freelist_t	  freelist;
 } drm_buf_entry_t;
@@ -507,6 +530,7 @@ typedef TAILQ_HEAD(drm_file_list, drm_file) drm_file_list_t;
 struct drm_file {
 	TAILQ_ENTRY(drm_file) link;
 	int		  authenticated;
+	int		  master;
 	int		  minor;
 	pid_t		  pid;
 	uid_t		  uid;
@@ -571,6 +595,7 @@ typedef struct drm_sg_mem {
 	void            *virtual;
 	int             pages;
 	dma_addr_t	*busaddr;
+	drm_dma_handle_t *dmah;	/* Handle to PCI memory for ATI PCIGART table */
 } drm_sg_mem_t;
 
 typedef TAILQ_HEAD(drm_map_list, drm_local_map) drm_map_list_t;
@@ -585,10 +610,10 @@ typedef struct drm_local_map {
 	int		mtrr;	 /* Boolean: MTRR used */
 				 /* Private data			    */
 	int		rid;	 /* PCI resource ID for bus_space */
-	int		kernel_owned; /* Boolean: 1 = initmapped, 0 = addmapped */
 	struct resource *bsr;
 	bus_space_tag_t bst;
 	bus_space_handle_t bsh;
+	drm_dma_handle_t *dmah;
 	TAILQ_ENTRY(drm_local_map) link;
 } drm_local_map_t;
 
@@ -600,25 +625,27 @@ typedef struct drm_vbl_sig {
 	int		pid;
 } drm_vbl_sig_t;
 
-/** 
- * DRM device functions structure
- */
-struct drm_device {
-#if defined(__NetBSD__) || defined(__OpenBSD__)
-	struct device	  device; /* softc is an extension of struct device */
-#endif
-
-	/* Beginning of driver-config section */
-	int	(*preinit)(struct drm_device *, unsigned long flags);
-	int	(*postinit)(struct drm_device *, unsigned long flags);
-	void	(*prerelease)(struct drm_device *, void *filp);
-	void	(*pretakedown)(struct drm_device *);
-	int	(*postcleanup)(struct drm_device *);
-	int	(*presetup)(struct drm_device *);
-	int	(*postsetup)(struct drm_device *);
-	int	(*open_helper)(struct drm_device *, drm_file_t *);
-	void	(*free_filp_priv)(struct drm_device *, drm_file_t *);
-	void	(*release)(struct drm_device *, void *filp);
+/* location of GART table */
+#define DRM_ATI_GART_MAIN 1
+#define DRM_ATI_GART_FB   2
+
+typedef struct ati_pcigart_info {
+	int gart_table_location;
+	int is_pcie;
+	void *addr;
+	dma_addr_t bus_addr;
+	drm_local_map_t mapping;
+} drm_ati_pcigart_info;
+
+struct drm_driver_info {
+	int	(*load)(struct drm_device *, unsigned long flags);
+	int	(*firstopen)(struct drm_device *);
+	int	(*open)(struct drm_device *, drm_file_t *);
+	void	(*preclose)(struct drm_device *, void *filp);
+	void	(*postclose)(struct drm_device *, drm_file_t *);
+	void	(*lastclose)(struct drm_device *);
+	int	(*unload)(struct drm_device *);
+	void	(*reclaim_buffers_locked)(struct drm_device *, void *filp);
 	int	(*dma_ioctl)(DRM_IOCTL_ARGS);
 	void	(*dma_ready)(struct drm_device *);
 	int	(*dma_quiescent)(struct drm_device *);
@@ -637,17 +664,32 @@ struct drm_device {
 	void	(*irq_handler)(DRM_IRQ_ARGS);
 	int	(*vblank_wait)(drm_device_t *dev, unsigned int *sequence);
 
-	drm_ioctl_desc_t *driver_ioctls;
-	int	max_driver_ioctl;
-
-	int	dev_priv_size;
-
-	int	driver_major;
-	int	driver_minor;
-	int	driver_patchlevel;
-	const char *driver_name;	/* Simple driver name		   */
-	const char *driver_desc;	/* Longer driver name		   */
-	const char *driver_date;	/* Date of last major changes.	   */
+	drm_pci_id_list_t *id_entry;	/* PCI ID, name, and chipset private */
+
+	/**
+	 * Called by \c drm_device_is_agp.  Typically used to determine if a
+	 * card is really attached to AGP or not.
+	 *
+	 * \param dev  DRM device handle
+	 *
+	 * \returns 
+	 * One of three values is returned depending on whether or not the
+	 * card is absolutely \b not AGP (return of 0), absolutely \b is AGP
+	 * (return of 1), or may or may not be AGP (return of 2).
+	 */
+	int	(*device_is_agp) (struct drm_device * dev);
+
+	drm_ioctl_desc_t *ioctls;
+	int	max_ioctl;
+
+	int	buf_priv_size;
+
+	int	major;
+	int	minor;
+	int	patchlevel;
+	const char *name;		/* Simple driver name		   */
+	const char *desc;		/* Longer driver name		   */
+	const char *date;		/* Date of last major changes.	   */
 
 	unsigned use_agp :1;
 	unsigned require_agp :1;
@@ -658,7 +700,21 @@ struct drm_device {
 	unsigned use_irq :1;
 	unsigned use_vbl_irq :1;
 	unsigned use_mtrr :1;
-	/* End of driver-config section */
+};
+
+/* Length for the array of resource pointers for drm_get_resource_*. */
+#define DRM_MAX_PCI_RESOURCE	3
+
+/** 
+ * DRM device functions structure
+ */
+struct drm_device {
+#if defined(__NetBSD__) || defined(__OpenBSD__)
+	struct device	  device; /* softc is an extension of struct device */
+#endif
+
+	struct drm_driver_info driver;
+	drm_pci_id_list_t *id_entry;	/* PCI ID, name, and chipset private */
 
 	char		  *unique;	/* Unique identifier: e.g., busid  */
 	int		  unique_len;	/* Length of unique field	   */
@@ -708,10 +764,13 @@ struct drm_device {
 	struct resource   *irqr;	/* Resource for interrupt used by board	   */
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
 	struct pci_attach_args  pa;
-	pci_intr_handle_t	ih;
 #endif
 	void		  *irqh;	/* Handle from bus_setup_intr      */
 
+	/* Storage of resource pointers for drm_get_resource_* */
+	struct resource   *pcir[DRM_MAX_PCI_RESOURCE];
+	int		  pcirid[DRM_MAX_PCI_RESOURCE];
+
 	int		  pci_domain;
 	int		  pci_bus;
 	int		  pci_slot;
@@ -735,6 +794,7 @@ struct drm_device {
 	drm_sg_mem_t      *sg;  /* Scatter gather memory */
 	atomic_t          *ctx_bitmap;
 	void		  *dev_private;
+	unsigned int	  agp_buffer_token;
 	drm_local_map_t   *agp_buffer_map;
 };
 
@@ -786,7 +846,7 @@ void	drm_free(void *pt, size_t size, int area);
 void	*drm_ioremap(drm_device_t *dev, drm_local_map_t *map);
 void	drm_ioremapfree(drm_local_map_t *map);
 int	drm_mtrr_add(unsigned long offset, size_t size, int flags);
-int	drm_mtrr_del(unsigned long offset, size_t size, int flags);
+int	drm_mtrr_del(int handle, unsigned long offset, size_t size, int flags);
 
 int	drm_context_switch(drm_device_t *dev, int old, int new);
 int	drm_context_switch_complete(drm_device_t *dev, int new);
@@ -809,16 +869,21 @@ int	drm_lock_free(drm_device_t *dev,
 /* Buffer management support (drm_bufs.c) */
 unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource);
 unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource);
-int	drm_initmap(drm_device_t *dev, unsigned long start, unsigned long len,
-		    unsigned int resource, int type, int flags);
-void	drm_remove_map(drm_device_t *dev, drm_local_map_t *map);
+void	drm_rmmap(drm_device_t *dev, drm_local_map_t *map);
 int	drm_order(unsigned long size);
+int	drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
+		   drm_map_type_t type, drm_map_flags_t flags,
+		   drm_local_map_t **map_ptr);
+int	drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request);
+int	drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request);
+int	drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request);
 
 /* DMA support (drm_dma.c) */
 int	drm_dma_setup(drm_device_t *dev);
 void	drm_dma_takedown(drm_device_t *dev);
 void	drm_free_buffer(drm_device_t *dev, drm_buf_t *buf);
 void	drm_reclaim_buffers(drm_device_t *dev, DRMFILE filp);
+#define drm_core_reclaim_buffers drm_reclaim_buffers
 
 /* IRQ support (drm_irq.c) */
 int	drm_irq_install(drm_device_t *dev);
@@ -834,12 +899,18 @@ void	drm_vbl_send_signals(drm_device_t *dev);
 int	drm_device_is_agp(drm_device_t *dev);
 int	drm_device_is_pcie(drm_device_t *dev);
 drm_agp_head_t *drm_agp_init(void);
-void	drm_agp_uninit(void);
-void	drm_agp_do_release(void);
+int	drm_agp_acquire(drm_device_t *dev);
+int	drm_agp_release(drm_device_t *dev);
+int	drm_agp_info(drm_device_t * dev, drm_agp_info_t *info);
+int	drm_agp_enable(drm_device_t *dev, drm_agp_mode_t mode);
 void	*drm_agp_allocate_memory(size_t pages, u32 type);
 int	drm_agp_free_memory(void *handle);
 int	drm_agp_bind_memory(void *handle, off_t start);
 int	drm_agp_unbind_memory(void *handle);
+int	drm_agp_alloc(drm_device_t *dev, drm_agp_buffer_t *request);
+int	drm_agp_free(drm_device_t *dev, drm_agp_buffer_t *request);
+int	drm_agp_bind(drm_device_t *dev, drm_agp_binding_t *request);
+int	drm_agp_unbind(drm_device_t *dev, drm_agp_binding_t *request);
 
 /* Scatter Gather Support (drm_scatter.c) */
 void	drm_sg_cleanup(drm_sg_mem_t *entry);
@@ -851,10 +922,10 @@ extern int		drm_sysctl_cleanup(drm_device_t *dev);
 #endif /* __FreeBSD__ */
 
 /* ATI PCIGART support (ati_pcigart.c) */
-int	drm_ati_pcigart_init(drm_device_t *dev, unsigned long *addr,
-			     dma_addr_t *bus_addr, int is_pcie);
-int	drm_ati_pcigart_cleanup(drm_device_t *dev, unsigned long addr,
-				dma_addr_t bus_addr);
+int	drm_ati_pcigart_init(drm_device_t *dev,
+			     drm_ati_pcigart_info *gart_info);
+int	drm_ati_pcigart_cleanup(drm_device_t *dev,
+				drm_ati_pcigart_info *gart_info);
 
 /* Locking IOCTL support (drm_drv.c) */
 int	drm_lock(DRM_IOCTL_ARGS);
@@ -891,9 +962,9 @@ int	drm_getmagic(DRM_IOCTL_ARGS);
 int	drm_authmagic(DRM_IOCTL_ARGS);
 
 /* Buffer management support (drm_bufs.c) */
-int	drm_addmap(DRM_IOCTL_ARGS);
-int	drm_rmmap(DRM_IOCTL_ARGS);
-int	drm_addbufs(DRM_IOCTL_ARGS);
+int	drm_addmap_ioctl(DRM_IOCTL_ARGS);
+int	drm_rmmap_ioctl(DRM_IOCTL_ARGS);
+int	drm_addbufs_ioctl(DRM_IOCTL_ARGS);
 int	drm_infobufs(DRM_IOCTL_ARGS);
 int	drm_markbufs(DRM_IOCTL_ARGS);
 int	drm_freebufs(DRM_IOCTL_ARGS);
@@ -907,24 +978,23 @@ int	drm_control(DRM_IOCTL_ARGS);
 int	drm_wait_vblank(DRM_IOCTL_ARGS);
 
 /* AGP/GART support (drm_agpsupport.c) */
-int	drm_agp_acquire(DRM_IOCTL_ARGS);
-int	drm_agp_release(DRM_IOCTL_ARGS);
-int	drm_agp_enable(DRM_IOCTL_ARGS);
-int	drm_agp_info(DRM_IOCTL_ARGS);
-int	drm_agp_alloc(DRM_IOCTL_ARGS);
-int	drm_agp_free(DRM_IOCTL_ARGS);
-int	drm_agp_unbind(DRM_IOCTL_ARGS);
-int	drm_agp_bind(DRM_IOCTL_ARGS);
+int	drm_agp_acquire_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_release_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_enable_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_info_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_alloc_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_free_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_unbind_ioctl(DRM_IOCTL_ARGS);
+int	drm_agp_bind_ioctl(DRM_IOCTL_ARGS);
 
 /* Scatter Gather Support (drm_scatter.c) */
 int	drm_sg_alloc(DRM_IOCTL_ARGS);
 int	drm_sg_free(DRM_IOCTL_ARGS);
 
 /* consistent PCI memory functions (drm_pci.c) */
-void	*drm_pci_alloc(drm_device_t *dev, size_t size, size_t align,
-		       dma_addr_t maxaddr, dma_addr_t *busaddr);
-void	drm_pci_free(drm_device_t *dev, size_t size, void *vaddr,
-		     dma_addr_t busaddr);
+drm_dma_handle_t *drm_pci_alloc(drm_device_t *dev, size_t size, size_t align,
+				dma_addr_t maxaddr);
+void	drm_pci_free(drm_device_t *dev, drm_dma_handle_t *dmah);
 
 /* Inline replacements for DRM_IOREMAP macros */
 static __inline__ void drm_core_ioremap(struct drm_local_map *map, struct drm_device *dev)
diff --git a/sys/dev/drm/drm_agpsupport.c b/sys/dev/drm/drm_agpsupport.c
index 8507be5cebad..91ead3b4ad39 100644
--- a/sys/dev/drm/drm_agpsupport.c
+++ b/sys/dev/drm/drm_agpsupport.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 #ifdef __FreeBSD__
@@ -39,9 +41,20 @@
 #include <dev/pci/pcireg.h>
 #endif
 
+/* Returns 1 if AGP or 0 if not. */
 static int
 drm_device_find_capability(drm_device_t *dev, int cap)
 {
+	int ret;
+
+	if (dev->driver.device_is_agp != NULL) {
+		ret = (*dev->driver.device_is_agp)(dev);
+		
+		if (ret != DRM_MIGHT_BE_AGP) {
+			return ret == 2;
+		}
+	}
+
 #ifdef __FreeBSD__
 	/* Code taken from agp.c.  IWBNI that was a public interface. */
 	u_int32_t status;
@@ -77,87 +90,106 @@ drm_device_find_capability(drm_device_t *dev, int cap)
 #endif
 }
 
-int
-drm_device_is_agp(drm_device_t *dev)
+int drm_device_is_agp(drm_device_t *dev)
 {
+	if (dev->driver.device_is_agp != NULL) {
+		int ret;
+
+		/* device_is_agp returns a tristate, 0 = not AGP, 1 = definitely
+		 * AGP, 2 = fall back to PCI capability
+		 */
+		ret = (*dev->driver.device_is_agp)(dev);
+		if (ret != 2)
+			return ret;
+	}
+
 	return (drm_device_find_capability(dev, PCIY_AGP));
 }
 
-int
-drm_device_is_pcie(drm_device_t *dev)
+int drm_device_is_pcie(drm_device_t *dev)
 {
 	return (drm_device_find_capability(dev, PCIY_EXPRESS));
 }
 
-int drm_agp_info(DRM_IOCTL_ARGS)
+int drm_agp_info(drm_device_t * dev, drm_agp_info_t *info)
 {
-	DRM_DEVICE;
 	struct agp_info *kern;
-	drm_agp_info_t   info;
 
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
 
 	kern                   = &dev->agp->info;
 	agp_get_info(dev->agp->agpdev, kern);
-	info.agp_version_major = 1;
-	info.agp_version_minor = 0;
-	info.mode              = kern->ai_mode;
-	info.aperture_base     = kern->ai_aperture_base;
-	info.aperture_size     = kern->ai_aperture_size;
-	info.memory_allowed    = kern->ai_memory_allowed;
-	info.memory_used       = kern->ai_memory_used;
-	info.id_vendor         = kern->ai_devid & 0xffff;
-	info.id_device         = kern->ai_devid >> 16;
+	info->agp_version_major = 1;
+	info->agp_version_minor = 0;
+	info->mode              = kern->ai_mode;
+	info->aperture_base     = kern->ai_aperture_base;
+	info->aperture_size     = kern->ai_aperture_size;
+	info->memory_allowed    = kern->ai_memory_allowed;
+	info->memory_used       = kern->ai_memory_used;
+	info->id_vendor         = kern->ai_devid & 0xffff;
+	info->id_device         = kern->ai_devid >> 16;
+
+	return 0;
+}
+
+int drm_agp_info_ioctl(DRM_IOCTL_ARGS)
+{
+	int err;
+	drm_agp_info_t info;
+	DRM_DEVICE;
+
+	err = drm_agp_info(dev, &info);
+	if (err != 0)
+		return err;
 
 	*(drm_agp_info_t *) data = info;
 	return 0;
 }
 
-int drm_agp_acquire(DRM_IOCTL_ARGS)
+int drm_agp_acquire_ioctl(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
-	int          retcode;
+
+	return drm_agp_acquire(dev);
+}
+
+int drm_agp_acquire(drm_device_t *dev)
+{
+	int retcode;
 
 	if (!dev->agp || dev->agp->acquired)
 		return EINVAL;
+
 	retcode = agp_acquire(dev->agp->agpdev);
 	if (retcode)
 		return retcode;
+
 	dev->agp->acquired = 1;
 	return 0;
 }
 
-int drm_agp_release(DRM_IOCTL_ARGS)
+int drm_agp_release_ioctl(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 
+	return drm_agp_release(dev);
+}
+
+int drm_agp_release(drm_device_t * dev)
+{
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
 	agp_release(dev->agp->agpdev);
 	dev->agp->acquired = 0;
 	return 0;
-	
 }
 
-void drm_agp_do_release(void)
+int drm_agp_enable(drm_device_t *dev, drm_agp_mode_t mode)
 {
-	device_t agpdev;
-
-	agpdev = DRM_AGP_FIND_DEVICE();
-	if (agpdev)
-		agp_release(agpdev);
-}
-
-int drm_agp_enable(DRM_IOCTL_ARGS)
-{
-	DRM_DEVICE;
-	drm_agp_mode_t mode;
 
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
-
-	mode = *(drm_agp_mode_t *) data;
 	
 	dev->agp->mode    = mode.mode;
 	agp_enable(dev->agp->agpdev, mode.mode);
@@ -166,10 +198,18 @@ int drm_agp_enable(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-int drm_agp_alloc(DRM_IOCTL_ARGS)
+int drm_agp_enable_ioctl(DRM_IOCTL_ARGS)
 {
+	drm_agp_mode_t mode;
 	DRM_DEVICE;
-	drm_agp_buffer_t request;
+
+	mode = *(drm_agp_mode_t *) data;
+
+	return drm_agp_enable(dev, mode);
+}
+
+int drm_agp_alloc(drm_device_t *dev, drm_agp_buffer_t *request)
+{
 	drm_agp_mem_t    *entry;
 	void	         *handle;
 	unsigned long    pages;
@@ -179,16 +219,17 @@ int drm_agp_alloc(DRM_IOCTL_ARGS)
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
 
-	request = *(drm_agp_buffer_t *) data;
-
 	entry = malloc(sizeof(*entry), M_DRM, M_NOWAIT | M_ZERO);
 	if (entry == NULL)
 		return ENOMEM;
 
-	pages = (request.size + PAGE_SIZE - 1) / PAGE_SIZE;
-	type = (u_int32_t) request.type;
+	pages = (request->size + PAGE_SIZE - 1) / PAGE_SIZE;
+	type = (u_int32_t) request->type;
 
-	if (!(handle = drm_agp_allocate_memory(pages, type))) {
+	DRM_UNLOCK();
+	handle = drm_agp_allocate_memory(pages, type);
+	DRM_LOCK();
+	if (handle == NULL) {
 		free(entry, M_DRM);
 		return ENOMEM;
 	}
@@ -204,12 +245,27 @@ int drm_agp_alloc(DRM_IOCTL_ARGS)
 
 	agp_memory_info(dev->agp->agpdev, entry->handle, &info);
 
-	request.handle   = (unsigned long) entry->handle;
-        request.physical = info.ami_physical;
+	request->handle   = (unsigned long) entry->handle;
+        request->physical = info.ami_physical;
+
+	return 0;
+}
+
+int drm_agp_alloc_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_agp_buffer_t request;
+	int retcode;
+
+	request = *(drm_agp_buffer_t *) data;
+
+	DRM_LOCK();
+	retcode = drm_agp_alloc(dev, &request);
+	DRM_UNLOCK();
 
 	*(drm_agp_buffer_t *) data = request;
 
-	return 0;
+	return retcode;
 }
 
 static drm_agp_mem_t * drm_agp_lookup_entry(drm_device_t *dev, void *handle)
@@ -222,64 +278,94 @@ static drm_agp_mem_t * drm_agp_lookup_entry(drm_device_t *dev, void *handle)
 	return NULL;
 }
 
-int drm_agp_unbind(DRM_IOCTL_ARGS)
+int drm_agp_unbind(drm_device_t *dev, drm_agp_binding_t *request)
 {
-	DRM_DEVICE;
-	drm_agp_binding_t request;
 	drm_agp_mem_t     *entry;
 	int retcode;
 
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
-	request = *(drm_agp_binding_t *) data;
-	if (!(entry = drm_agp_lookup_entry(dev, (void *)request.handle)))
+
+	entry = drm_agp_lookup_entry(dev, (void *)request->handle);
+	if (entry == NULL || !entry->bound)
 		return EINVAL;
-	if (!entry->bound) return EINVAL;
+
+	DRM_UNLOCK();
 	retcode = drm_agp_unbind_memory(entry->handle);
-	if (!retcode)
-	{
-		entry->bound=0;
-		return 0;
-	}
-	else
-		return retcode;
+	DRM_LOCK();
+
+	if (retcode == 0)
+		entry->bound = 0;
+
+	return retcode;
 }
 
-int drm_agp_bind(DRM_IOCTL_ARGS)
+int drm_agp_unbind_ioctl(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_agp_binding_t request;
+	int retcode;
+
+	request = *(drm_agp_binding_t *) data;
+
+	DRM_LOCK();
+	retcode = drm_agp_unbind(dev, &request);
+	DRM_UNLOCK();
+
+	return retcode;
+}
+
+int drm_agp_bind(drm_device_t *dev, drm_agp_binding_t *request)
+{
 	drm_agp_mem_t     *entry;
 	int               retcode;
 	int               page;
 	
-	DRM_DEBUG("agp_bind, page_size=%x\n", PAGE_SIZE);
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
-	request = *(drm_agp_binding_t *) data;
-	if (!(entry = drm_agp_lookup_entry(dev, (void *)request.handle)))
+
+	DRM_DEBUG("agp_bind, page_size=%x\n", PAGE_SIZE);
+
+	entry = drm_agp_lookup_entry(dev, (void *)request->handle);
+	if (entry == NULL || entry->bound)
 		return EINVAL;
-	if (entry->bound) return EINVAL;
-	page = (request.offset + PAGE_SIZE - 1) / PAGE_SIZE;
-	if ((retcode = drm_agp_bind_memory(entry->handle, page)))
-		return retcode;
-	entry->bound = dev->agp->base + (page << PAGE_SHIFT);
-	return 0;
+
+	page = (request->offset + PAGE_SIZE - 1) / PAGE_SIZE;
+
+	DRM_UNLOCK();
+	retcode = drm_agp_bind_memory(entry->handle, page);
+	DRM_LOCK();
+	if (retcode == 0)
+		entry->bound = dev->agp->base + (page << PAGE_SHIFT);
+
+	return retcode;
 }
 
-int drm_agp_free(DRM_IOCTL_ARGS)
+int drm_agp_bind_ioctl(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
-	drm_agp_buffer_t request;
+	drm_agp_binding_t request;
+	int retcode;
+
+	request = *(drm_agp_binding_t *) data;
+
+	DRM_LOCK();
+	retcode = drm_agp_bind(dev, &request);
+	DRM_UNLOCK();
+
+	return retcode;
+}
+
+int drm_agp_free(drm_device_t *dev, drm_agp_buffer_t *request)
+{
 	drm_agp_mem_t    *entry;
 	
 	if (!dev->agp || !dev->agp->acquired)
 		return EINVAL;
-	request = *(drm_agp_buffer_t *) data;
-	if (!(entry = drm_agp_lookup_entry(dev, (void*)request.handle)))
+
+	entry = drm_agp_lookup_entry(dev, (void*)request->handle);
+	if (entry == NULL)
 		return EINVAL;
-	if (entry->bound)
-		drm_agp_unbind_memory(entry->handle);
    
 	if (entry->prev)
 		entry->prev->next = entry->next;
@@ -287,9 +373,32 @@ int drm_agp_free(DRM_IOCTL_ARGS)
 		dev->agp->memory  = entry->next;
 	if (entry->next)
 		entry->next->prev = entry->prev;
+
+	DRM_UNLOCK();
+	if (entry->bound)
+		drm_agp_unbind_memory(entry->handle);
 	drm_agp_free_memory(entry->handle);
+	DRM_LOCK();
+
 	free(entry, M_DRM);
+
 	return 0;
+
+}
+
+int drm_agp_free_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_agp_buffer_t request;
+	int retcode;
+
+	request = *(drm_agp_buffer_t *) data;
+
+	DRM_LOCK();
+	retcode = drm_agp_free(dev, &request);
+	DRM_UNLOCK();
+
+	return retcode;
 }
 
 drm_agp_head_t *drm_agp_init(void)
@@ -318,12 +427,6 @@ drm_agp_head_t *drm_agp_init(void)
 	return head;
 }
 
-void drm_agp_uninit(void)
-{
-/* FIXME: What goes here */
-}
-
-
 void *drm_agp_allocate_memory(size_t pages, u32 type)
 {
 	device_t agpdev;
diff --git a/sys/dev/drm/drm_atomic.h b/sys/dev/drm/drm_atomic.h
index 8cdabc7ffe69..cf9d2068ceb7 100644
--- a/sys/dev/drm/drm_atomic.h
+++ b/sys/dev/drm/drm_atomic.h
@@ -27,10 +27,11 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 /* Many of these implementations are rather fake, but good enough. */
 
 typedef u_int32_t atomic_t;
diff --git a/sys/dev/drm/drm_auth.c b/sys/dev/drm/drm_auth.c
index 910c0b03d961..03b3a4705d18 100644
--- a/sys/dev/drm/drm_auth.c
+++ b/sys/dev/drm/drm_auth.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 static int drm_hash_magic(drm_magic_t magic)
diff --git a/sys/dev/drm/drm_bufs.c b/sys/dev/drm/drm_bufs.c
index 1601b36996de..c8ef0e410032 100644
--- a/sys/dev/drm/drm_bufs.c
+++ b/sys/dev/drm/drm_bufs.c
@@ -29,9 +29,13 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "dev/pci/pcireg.h"
+
 #include "dev/drm/drmP.h"
 
 /*
@@ -50,146 +54,98 @@ int drm_order(unsigned long size)
 	return order;
 }
 
-unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource)
+/* Allocation of PCI memory resources (framebuffer, registers, etc.) for
+ * drm_get_resource_*.  Note that they are not RF_ACTIVE, so there's no virtual
+ * address for accessing them.  Cleaned up at unload.
+ */
+static int drm_alloc_resource(drm_device_t *dev, int resource)
 {
-	struct resource *bsr;
-	unsigned long offset;
-
-	resource = resource * 4 + 0x10;
+	if (resource >= DRM_MAX_PCI_RESOURCE) {
+		DRM_ERROR("Resource %d too large\n", resource);
+		return 1;
+	}
 
-	bsr = bus_alloc_resource_any(dev->device, SYS_RES_MEMORY, &resource,
-	    RF_ACTIVE | RF_SHAREABLE);
-	if (bsr == NULL) {
-		DRM_ERROR("Couldn't find resource 0x%x\n", resource);
+	DRM_UNLOCK();
+	if (dev->pcir[resource] != NULL) {
+		DRM_LOCK();
 		return 0;
 	}
 
-	offset = rman_get_start(bsr);
-
-	bus_release_resource(dev->device, SYS_RES_MEMORY, resource, bsr);
-
-	return offset;
-}
-
-unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource)
-{
-	struct resource *bsr;
-	unsigned long len;
-
-	resource = resource * 4 + 0x10;
+	dev->pcirid[resource] = PCIR_BAR(resource);
+	dev->pcir[resource] = bus_alloc_resource_any(dev->device,
+	    SYS_RES_MEMORY, &dev->pcirid[resource], RF_SHAREABLE);
+	DRM_LOCK();
 
-	bsr = bus_alloc_resource_any(dev->device, SYS_RES_MEMORY, &resource,
-	    RF_ACTIVE | RF_SHAREABLE);
-	if (bsr == NULL) {
+	if (dev->pcir[resource] == NULL) {
 		DRM_ERROR("Couldn't find resource 0x%x\n", resource);
-		return ENOMEM;
+		return 1;
 	}
 
-	len = rman_get_size(bsr);
-
-	bus_release_resource(dev->device, SYS_RES_MEMORY, resource, bsr);
-
-	return len;
+	return 0;
 }
 
-int drm_initmap(drm_device_t *dev, unsigned long start, unsigned long len,
-		unsigned int resource, int type, int flags)
+unsigned long drm_get_resource_start(drm_device_t *dev, unsigned int resource)
 {
-	drm_local_map_t *map;
-	struct resource *bsr;
-
-	if (type != _DRM_REGISTERS && type != _DRM_FRAME_BUFFER)
-		return EINVAL;
-	if (len == 0)
-		return EINVAL;
-
-	map = malloc(sizeof(*map), M_DRM, M_ZERO | M_NOWAIT);
-	if (map == NULL)
-		return ENOMEM;
-
-	map->rid = resource * 4 + 0x10;
-	bsr = bus_alloc_resource_any(dev->device, SYS_RES_MEMORY, &map->rid,
-	    RF_ACTIVE | RF_SHAREABLE);
-	if (bsr == NULL) {
-		DRM_ERROR("Couldn't allocate %s resource\n",
-		    ((type == _DRM_REGISTERS) ? "mmio" : "framebuffer"));
-		free(map, M_DRM);
-		return ENOMEM;
-	}
-
-	map->kernel_owned = 1;
-	map->type = type;
-	map->flags = flags;
-	map->bsr = bsr;
-	map->bst = rman_get_bustag(bsr);
-	map->bsh = rman_get_bushandle(bsr);
-	map->offset = start;
-	map->size = len;
-
-	if (type == _DRM_REGISTERS)
-		map->handle = rman_get_virtual(bsr);
-
-	DRM_DEBUG("initmap %d,0x%x@0x%lx/0x%lx\n", map->type, map->flags,
-	    map->offset, map->size);
-
-	if (map->flags & _DRM_WRITE_COMBINING) {
-		int err;
+	if (drm_alloc_resource(dev, resource) != 0)
+		return 0;
 
-		err = drm_mtrr_add(map->offset, map->size, DRM_MTRR_WC);
-		if (err == 0)
-			map->mtrr = 1;
-	}
+	return rman_get_start(dev->pcir[resource]);
+}
 
-	DRM_LOCK();
-	TAILQ_INSERT_TAIL(&dev->maplist, map, link);
-	DRM_UNLOCK();
+unsigned long drm_get_resource_len(drm_device_t *dev, unsigned int resource)
+{
+	if (drm_alloc_resource(dev, resource) != 0)
+		return 0;
 
-	return 0;
+	return rman_get_size(dev->pcir[resource]);
 }
 
-int drm_addmap(DRM_IOCTL_ARGS)
+int drm_addmap(drm_device_t * dev, unsigned long offset, unsigned long size,
+    drm_map_type_t type, drm_map_flags_t flags, drm_local_map_t **map_ptr)
 {
-	DRM_DEVICE;
-	drm_map_t request;
 	drm_local_map_t *map;
-	dma_addr_t bus_addr;
-	
-	if (!(dev->flags & (FREAD|FWRITE)))
-		return DRM_ERR(EACCES); /* Require read/write */
-
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_map_t *)data, sizeof(drm_map_t) );
+	int align;
+	/*drm_agp_mem_t *entry;
+	int valid;*/
 
 	/* Only allow shared memory to be removable since we only keep enough
 	 * book keeping information about shared memory to allow for removal
 	 * when processes fork.
 	 */
-	if ((request.flags & _DRM_REMOVABLE) && request.type != _DRM_SHM)
+	if ((flags & _DRM_REMOVABLE) && type != _DRM_SHM) {
+		DRM_ERROR("Requested removable map for non-DRM_SHM\n");
 		return EINVAL;
-	if ((request.offset & PAGE_MASK) || (request.size & PAGE_MASK))
+	}
+	if ((offset & PAGE_MASK) || (size & PAGE_MASK)) {
+		DRM_ERROR("offset/size not page aligned: 0x%lx/0x%lx\n",
+		    offset, size);
 		return EINVAL;
-	if (request.offset + request.size < request.offset)
+	}
+	if (offset + size < offset) {
+		DRM_ERROR("offset and size wrap around: 0x%lx/0x%lx\n",
+		    offset, size);
 		return EINVAL;
+	}
 
-	DRM_DEBUG("offset = 0x%08lx, size = 0x%08lx, type = %d\n",
-	    request.offset, request.size, request.type);
+	DRM_DEBUG("offset = 0x%08lx, size = 0x%08lx, type = %d\n", offset,
+	    size, type);
 
 	/* Check if this is just another version of a kernel-allocated map, and
 	 * just hand that back if so.
 	 */
-	if (request.type == _DRM_REGISTERS || request.type == _DRM_FRAME_BUFFER)
-	{
-		DRM_LOCK();
+	if (type == _DRM_REGISTERS || type == _DRM_FRAME_BUFFER ||
+	    type == _DRM_SHM) {
 		TAILQ_FOREACH(map, &dev->maplist, link) {
-			if (map->kernel_owned && map->type == request.type &&
-			    map->offset == request.offset) {
-				/* XXX: this size setting is questionable. */
-				map->size = request.size;
-				DRM_DEBUG("Found kernel map %d\n", request.type);
+			if (map->type == type && (map->offset == offset ||
+			    (map->type == _DRM_SHM &&
+			    map->flags == _DRM_CONTAINS_LOCK))) {
+				map->size = size;
+				DRM_DEBUG("Found kernel map %d\n", type);
 				goto done;
 			}
 		}
-		DRM_UNLOCK();
 	}
+	DRM_UNLOCK();
 
 	/* Allocate a new map structure, fill it in, and do any type-specific
 	 * initialization necessary.
@@ -198,10 +154,10 @@ int drm_addmap(DRM_IOCTL_ARGS)
 	if ( !map )
 		return DRM_ERR(ENOMEM);
 
-	map->offset = request.offset;
-	map->size = request.size;
-	map->type = request.type;
-	map->flags = request.flags;
+	map->offset = offset;
+	map->size = size;
+	map->type = type;
+	map->flags = flags;
 
 	switch ( map->type ) {
 	case _DRM_REGISTERS:
@@ -236,8 +192,21 @@ int drm_addmap(DRM_IOCTL_ARGS)
 		}
 		break;
 	case _DRM_AGP:
+		/*valid = 0;*/
 		map->offset += dev->agp->base;
 		map->mtrr   = dev->agp->mtrr; /* for getmap */
+		/*for (entry = dev->agp->memory; entry; entry = entry->next) {
+			if ((map->offset >= entry->bound) &&
+			    (map->offset + map->size <=
+			    entry->bound + entry->pages * PAGE_SIZE)) {
+				valid = 1;
+				break;
+			}
+		}
+		if (!valid) {
+			free(map, M_DRM);
+			return DRM_ERR(EACCES);
+		}*/
 		break;
 	case _DRM_SCATTER_GATHER:
 		if (!dev->sg) {
@@ -247,15 +216,25 @@ int drm_addmap(DRM_IOCTL_ARGS)
 		map->offset = map->offset + dev->sg->handle;
 		break;
 	case _DRM_CONSISTENT:
-		map->handle = drm_pci_alloc(dev, map->size, map->size,
-		    0xfffffffful, &bus_addr);
-		if (map->handle == NULL) {
+		/* Unfortunately, we don't get any alignment specification from
+		 * the caller, so we have to guess.  drm_pci_alloc requires
+		 * a power-of-two alignment, so try to align the bus address of
+		 * the map to it size if possible, otherwise just assume
+		 * PAGE_SIZE alignment.
+		 */
+		align = map->size;
+		if ((align & (align - 1)) != 0)
+			align = PAGE_SIZE;
+		map->dmah = drm_pci_alloc(dev, map->size, align, 0xfffffffful);
+		if (map->dmah == NULL) {
 			free(map, M_DRM);
-			return ENOMEM;
+			return DRM_ERR(ENOMEM);
 		}
-		map->offset = (unsigned long)bus_addr;
+		map->handle = map->dmah->vaddr;
+		map->offset = map->dmah->busaddr;
 		break;
 	default:
+		DRM_ERROR("Bad map type %d\n", map->type);
 		free(map, M_DRM);
 		return DRM_ERR(EINVAL);
 	}
@@ -265,26 +244,53 @@ int drm_addmap(DRM_IOCTL_ARGS)
 
 done:
 	/* Jumped to, with lock held, when a kernel map is found. */
+
+	DRM_DEBUG("Added map %d 0x%lx/0x%lx\n", map->type, map->offset,
+	    map->size);
+
+	*map_ptr = map;
+
+	return 0;
+}
+
+int drm_addmap_ioctl(DRM_IOCTL_ARGS)
+{
+	drm_map_t request;
+	drm_local_map_t *map;
+	int err;
+	DRM_DEVICE;
+
+	if (!(dev->flags & (FREAD|FWRITE)))
+		return DRM_ERR(EACCES); /* Require read/write */
+
+	DRM_COPY_FROM_USER_IOCTL(request, (drm_map_t *)data, sizeof(drm_map_t));
+
+	if (!DRM_SUSER(p) && request.type != _DRM_AGP)
+		return DRM_ERR(EACCES);
+
+	DRM_LOCK();
+	err = drm_addmap(dev, request.offset, request.size, request.type,
+	    request.flags, &map);
+	DRM_UNLOCK();
+	if (err != 0)
+		return err;
+
 	request.offset = map->offset;
 	request.size = map->size;
 	request.type = map->type;
 	request.flags = map->flags;
 	request.mtrr   = map->mtrr;
 	request.handle = map->handle;
-	DRM_UNLOCK();
 
-	DRM_DEBUG("Added map %d 0x%lx/0x%lx\n", request.type, request.offset, request.size);
-
-	if ( request.type != _DRM_SHM ) {
+	if (request.type != _DRM_SHM) {
 		request.handle = (void *)request.offset;
 	}
-
-	DRM_COPY_TO_USER_IOCTL( (drm_map_t *)data, request, sizeof(drm_map_t) );
+	DRM_COPY_TO_USER_IOCTL((drm_map_t *)data, request, sizeof(drm_map_t));
 
 	return 0;
 }
 
-void drm_remove_map(drm_device_t *dev, drm_local_map_t *map)
+void drm_rmmap(drm_device_t *dev, drm_local_map_t *map)
 {
 	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
 
@@ -299,7 +305,7 @@ void drm_remove_map(drm_device_t *dev, drm_local_map_t *map)
 		if (map->mtrr) {
 			int __unused retcode;
 			
-			retcode = drm_mtrr_del(map->offset, map->size,
+			retcode = drm_mtrr_del(0, map->offset, map->size,
 			    DRM_MTRR_WC);
 			DRM_DEBUG("mtrr_del = %d\n", retcode);
 		}
@@ -311,7 +317,7 @@ void drm_remove_map(drm_device_t *dev, drm_local_map_t *map)
 	case _DRM_SCATTER_GATHER:
 		break;
 	case _DRM_CONSISTENT:
-		drm_pci_free(dev, map->size, map->handle, map->offset);
+		drm_pci_free(dev, map->dmah);
 		break;
 	}
 
@@ -327,7 +333,7 @@ void drm_remove_map(drm_device_t *dev, drm_local_map_t *map)
  * isn't in use.
  */
 
-int drm_rmmap(DRM_IOCTL_ARGS)
+int drm_rmmap_ioctl(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_local_map_t *map;
@@ -348,7 +354,7 @@ int drm_rmmap(DRM_IOCTL_ARGS)
 		return DRM_ERR(EINVAL);
 	}
 
-	drm_remove_map(dev, map);
+	drm_rmmap(dev, map);
 
 	DRM_UNLOCK();
 
@@ -362,12 +368,9 @@ static void drm_cleanup_buf_error(drm_device_t *dev, drm_buf_entry_t *entry)
 
 	if (entry->seg_count) {
 		for (i = 0; i < entry->seg_count; i++) {
-			drm_pci_free(dev, entry->buf_size,
-			    (void *)entry->seglist[i],
-			    entry->seglist_bus[i]);
+			drm_pci_free(dev, entry->seglist[i]);
 		}
 		free(entry->seglist, M_DRM);
-		free(entry->seglist_bus, M_DRM);
 
 		entry->seg_count = 0;
 	}
@@ -382,10 +385,12 @@ static void drm_cleanup_buf_error(drm_device_t *dev, drm_buf_entry_t *entry)
 	}
 }
 
-static int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_entry_t *entry;
+	/*drm_agp_mem_t *agp_entry;
+	int valid*/
 	drm_buf_t *buf;
 	unsigned long offset;
 	unsigned long agp_offset;
@@ -419,6 +424,26 @@ static int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	DRM_DEBUG( "page_order: %d\n",  page_order );
 	DRM_DEBUG( "total:      %d\n",  total );
 
+	/* Make sure buffers are located in AGP memory that we own */
+	/* Breaks MGA due to drm_alloc_agp not setting up entries for the
+	 * memory.  Safe to ignore for now because these ioctls are still
+	 * root-only.
+	 */
+	/*valid = 0;
+	for (agp_entry = dev->agp->memory; agp_entry;
+	    agp_entry = agp_entry->next) {
+		if ((agp_offset >= agp_entry->bound) &&
+		    (agp_offset + total * count <=
+		    agp_entry->bound + agp_entry->pages * PAGE_SIZE)) {
+			valid = 1;
+			break;
+		}
+	}
+	if (!valid) {
+		DRM_DEBUG("zone invalid\n");
+		return DRM_ERR(EINVAL);
+	}*/
+
 	entry = &dma->bufs[order];
 
 	entry->buflist = malloc(count * sizeof(*entry->buflist), M_DRM,
@@ -446,7 +471,7 @@ static int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 		buf->pending = 0;
 		buf->filp    = NULL;
 
-		buf->dev_priv_size = dev->dev_priv_size;
+		buf->dev_priv_size = dev->driver.buf_priv_size;
 		buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
 		    M_NOWAIT | M_ZERO);
 		if (buf->dev_private == NULL) {
@@ -491,7 +516,7 @@ static int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 	return 0;
 }
 
-static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	int count;
@@ -500,7 +525,6 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	int total;
 	int page_order;
 	drm_buf_entry_t *entry;
-	vm_offset_t vaddr;
 	drm_buf_t *buf;
 	int alignment;
 	unsigned long offset;
@@ -509,7 +533,6 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	int page_count;
 	unsigned long *temp_pagelist;
 	drm_buf_t **temp_buflist;
-	dma_addr_t bus_addr;
 
 	count = request->count;
 	order = drm_order(request->size);
@@ -529,8 +552,6 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	    M_NOWAIT | M_ZERO);
 	entry->seglist = malloc(count * sizeof(*entry->seglist), M_DRM,
 	    M_NOWAIT | M_ZERO);
-	entry->seglist_bus = malloc(count * sizeof(*entry->seglist_bus), M_DRM,
-	    M_NOWAIT | M_ZERO);
 
 	/* Keep the original pagelist until we know all the allocations
 	 * have succeeded
@@ -539,10 +560,9 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	    sizeof(*dma->pagelist), M_DRM, M_NOWAIT);
 
 	if (entry->buflist == NULL || entry->seglist == NULL || 
-	    entry->seglist_bus == NULL || temp_pagelist == NULL) {
+	    temp_pagelist == NULL) {
 		free(entry->buflist, M_DRM);
 		free(entry->seglist, M_DRM);
-		free(entry->seglist_bus, M_DRM);
 		return DRM_ERR(ENOMEM);
 	}
 	
@@ -558,9 +578,9 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 	page_count = 0;
 
 	while ( entry->buf_count < count ) {
-		vaddr = (vm_offset_t)drm_pci_alloc(dev, size, alignment,
-		    0xfffffffful, &bus_addr);
-		if (vaddr == 0) {
+		drm_dma_handle_t *dmah = drm_pci_alloc(dev, size, alignment,
+		    0xfffffffful);
+		if (dmah == NULL) {
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
 			entry->seg_count = count;
@@ -568,15 +588,14 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 			free(temp_pagelist, M_DRM);
 			return DRM_ERR(ENOMEM);
 		}
-	
-		entry->seglist_bus[entry->seg_count] = bus_addr;
-		entry->seglist[entry->seg_count++] = vaddr;
+
+		entry->seglist[entry->seg_count++] = dmah;
 		for ( i = 0 ; i < (1 << page_order) ; i++ ) {
-			DRM_DEBUG( "page %d @ 0x%08lx\n",
+			DRM_DEBUG( "page %d @ %p\n",
 				   dma->page_count + page_count,
-				   (long)vaddr + PAGE_SIZE * i );
+				   (char *)dmah->vaddr + PAGE_SIZE * i );
 			temp_pagelist[dma->page_count + page_count++] = 
-			    vaddr + PAGE_SIZE * i;
+			    (long)dmah->vaddr + PAGE_SIZE * i;
 		}
 		for ( offset = 0 ;
 		      offset + size <= total && entry->buf_count < count ;
@@ -587,13 +606,13 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 			buf->order   = order;
 			buf->used    = 0;
 			buf->offset  = (dma->byte_count + byte_count + offset);
-			buf->address = (void *)(vaddr + offset);
-			buf->bus_address = bus_addr + offset;
+			buf->address = ((char *)dmah->vaddr + offset);
+			buf->bus_address = dmah->busaddr + offset;
 			buf->next    = NULL;
 			buf->pending = 0;
 			buf->filp    = NULL;
 
-			buf->dev_priv_size = dev->dev_priv_size;
+			buf->dev_priv_size = dev->driver.buf_priv_size;
 			buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
 			    M_NOWAIT | M_ZERO);
 			if (buf->dev_private == NULL) {
@@ -644,7 +663,7 @@ static int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
 
 }
 
-static int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
+static int drm_do_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 {
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_entry_t *entry;
@@ -707,7 +726,7 @@ static int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 		buf->pending = 0;
 		buf->filp    = NULL;
 
-		buf->dev_priv_size = dev->dev_priv_size;
+		buf->dev_priv_size = dev->driver.buf_priv_size;
 		buf->dev_private = malloc(buf->dev_priv_size, M_DRM,
 		    M_NOWAIT | M_ZERO);
 		if (buf->dev_private == NULL) {
@@ -755,23 +774,87 @@ static int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
 	return 0;
 }
 
-int drm_addbufs(DRM_IOCTL_ARGS)
+int drm_addbufs_agp(drm_device_t *dev, drm_buf_desc_t *request)
 {
-	DRM_DEVICE;
-	drm_buf_desc_t request;
-	int err;
-	int order;
+	int order, ret;
 
-	DRM_COPY_FROM_USER_IOCTL( request, (drm_buf_desc_t *)data, sizeof(request) );
+	DRM_SPINLOCK(&dev->dma_lock);
 
-	if (request.count < 0 || request.count > 4096)
+	if (request->count < 0 || request->count > 4096)
+		return DRM_ERR(EINVAL);
+	
+	order = drm_order(request->size);
+	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
 		return DRM_ERR(EINVAL);
 
-	order = drm_order(request.size);
+	/* No more allocations after first buffer-using ioctl. */
+	if (dev->buf_use != 0) {
+		DRM_SPINUNLOCK(&dev->dma_lock);
+		return DRM_ERR(EBUSY);
+	}
+	/* No more than one allocation per order */
+	if (dev->dma->bufs[order].buf_count != 0) {
+		DRM_SPINUNLOCK(&dev->dma_lock);
+		return DRM_ERR(ENOMEM);
+	}
+
+	ret = drm_do_addbufs_agp(dev, request);
+
+	DRM_SPINUNLOCK(&dev->dma_lock);
+
+	return ret;
+}
+
+int drm_addbufs_sg(drm_device_t *dev, drm_buf_desc_t *request)
+{
+	int order, ret;
+
+	DRM_SPINLOCK(&dev->dma_lock);
+
+	if (!DRM_SUSER(DRM_CURPROC))
+		return DRM_ERR(EACCES);
+
+	if (request->count < 0 || request->count > 4096)
+		return DRM_ERR(EINVAL);
+	
+	order = drm_order(request->size);
 	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
 		return DRM_ERR(EINVAL);
 
+	/* No more allocations after first buffer-using ioctl. */
+	if (dev->buf_use != 0) {
+		DRM_SPINUNLOCK(&dev->dma_lock);
+		return DRM_ERR(EBUSY);
+	}
+	/* No more than one allocation per order */
+	if (dev->dma->bufs[order].buf_count != 0) {
+		DRM_SPINUNLOCK(&dev->dma_lock);
+		return DRM_ERR(ENOMEM);
+	}
+
+	ret = drm_do_addbufs_sg(dev, request);
+
+	DRM_SPINUNLOCK(&dev->dma_lock);
+
+	return ret;
+}
+
+int drm_addbufs_pci(drm_device_t *dev, drm_buf_desc_t *request)
+{
+	int order, ret;
+
 	DRM_SPINLOCK(&dev->dma_lock);
+
+	if (!DRM_SUSER(DRM_CURPROC))
+		return DRM_ERR(EACCES);
+
+	if (request->count < 0 || request->count > 4096)
+		return DRM_ERR(EINVAL);
+	
+	order = drm_order(request->size);
+	if (order < DRM_MIN_ORDER || order > DRM_MAX_ORDER)
+		return DRM_ERR(EINVAL);
+
 	/* No more allocations after first buffer-using ioctl. */
 	if (dev->buf_use != 0) {
 		DRM_SPINUNLOCK(&dev->dma_lock);
@@ -783,16 +866,31 @@ int drm_addbufs(DRM_IOCTL_ARGS)
 		return DRM_ERR(ENOMEM);
 	}
 
-	if ( request.flags & _DRM_AGP_BUFFER )
+	ret = drm_do_addbufs_pci(dev, request);
+
+	DRM_SPINUNLOCK(&dev->dma_lock);
+
+	return ret;
+}
+
+int drm_addbufs_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_buf_desc_t request;
+	int err;
+
+	DRM_COPY_FROM_USER_IOCTL(request, (drm_buf_desc_t *)data,
+	    sizeof(request));
+
+	if (request.flags & _DRM_AGP_BUFFER)
 		err = drm_addbufs_agp(dev, &request);
-	else
-	if ( request.flags & _DRM_SG_BUFFER )
+	else if (request.flags & _DRM_SG_BUFFER)
 		err = drm_addbufs_sg(dev, &request);
 	else
 		err = drm_addbufs_pci(dev, &request);
-	DRM_SPINUNLOCK(&dev->dma_lock);
 
-	DRM_COPY_TO_USER_IOCTL((drm_buf_desc_t *)data, request, sizeof(request));
+	DRM_COPY_TO_USER_IOCTL((drm_buf_desc_t *)data, request,
+	    sizeof(request));
 
 	return err;
 }
@@ -937,7 +1035,8 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	vm_offset_t vaddr;
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
 	struct vnode *vn;
-	vm_size_t size;
+	voff_t foff;
+	vsize_t size;
 	vaddr_t vaddr;
 #endif /* __NetBSD__ || __OpenBSD__ */
 
@@ -964,8 +1063,8 @@ int drm_mapbufs(DRM_IOCTL_ARGS)
 	if (request.count < dma->buf_count)
 		goto done;
 
-	if ((dev->use_agp && (dma->flags & _DRM_DMA_USE_AGP)) ||
-	    (dev->use_sg && (dma->flags & _DRM_DMA_USE_SG))) {
+	if ((dev->driver.use_agp && (dma->flags & _DRM_DMA_USE_AGP)) ||
+	    (dev->driver.use_sg && (dma->flags & _DRM_DMA_USE_SG))) {
 		drm_local_map_t *map = dev->agp_buffer_map;
 
 		if (map == NULL) {
diff --git a/sys/dev/drm/drm_context.c b/sys/dev/drm/drm_context.c
index 7c02d92b710a..cf5bc15bcf6b 100644
--- a/sys/dev/drm/drm_context.c
+++ b/sys/dev/drm/drm_context.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 /* ================================================================
@@ -229,14 +231,17 @@ int drm_context_switch_complete(drm_device_t *dev, int new)
 int drm_resctx(DRM_IOCTL_ARGS)
 {
 	drm_ctx_res_t res;
+	drm_ctx_t ctx;
 	int i;
 
 	DRM_COPY_FROM_USER_IOCTL( res, (drm_ctx_res_t *)data, sizeof(res) );
 
 	if ( res.count >= DRM_RESERVED_CONTEXTS ) {
+		bzero(&ctx, sizeof(ctx));
 		for ( i = 0 ; i < DRM_RESERVED_CONTEXTS ; i++ ) {
+			ctx.handle = i;
 			if ( DRM_COPY_TO_USER( &res.contexts[i],
-					   &i, sizeof(i) ) )
+					   &ctx, sizeof(ctx) ) )
 				return DRM_ERR(EFAULT);
 		}
 	}
@@ -266,9 +271,9 @@ int drm_addctx(DRM_IOCTL_ARGS)
 		return DRM_ERR(ENOMEM);
 	}
 
-	if (dev->context_ctor && ctx.handle != DRM_KERNEL_CONTEXT) {
+	if (dev->driver.context_ctor && ctx.handle != DRM_KERNEL_CONTEXT) {
 		DRM_LOCK();
-		dev->context_ctor(dev, ctx.handle);
+		dev->driver.context_ctor(dev, ctx.handle);
 		DRM_UNLOCK();
 	}
 
@@ -330,9 +335,9 @@ int drm_rmctx(DRM_IOCTL_ARGS)
 
 	DRM_DEBUG( "%d\n", ctx.handle );
 	if ( ctx.handle != DRM_KERNEL_CONTEXT ) {
-		if (dev->context_dtor) {
+		if (dev->driver.context_dtor) {
 			DRM_LOCK();
-			dev->context_dtor(dev, ctx.handle);
+			dev->driver.context_dtor(dev, ctx.handle);
 			DRM_UNLOCK();
 		}
 
diff --git a/sys/dev/drm/drm_dma.c b/sys/dev/drm/drm_dma.c
index 21073b676df8..a8661ca22d3d 100644
--- a/sys/dev/drm/drm_dma.c
+++ b/sys/dev/drm/drm_dma.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 int drm_dma_setup(drm_device_t *dev)
@@ -63,12 +65,9 @@ void drm_dma_takedown(drm_device_t *dev)
 				  dma->bufs[i].buf_count,
 				  dma->bufs[i].seg_count);
 			for (j = 0; j < dma->bufs[i].seg_count; j++) {
-				drm_pci_free(dev, dma->bufs[i].buf_size,
-				    (void *)dma->bufs[i].seglist[j],
-				    dma->bufs[i].seglist_bus[j]);
+				drm_pci_free(dev, dma->bufs[i].seglist[j]);
 			}
 			free(dma->bufs[i].seglist, M_DRM);
-			free(dma->bufs[i].seglist_bus, M_DRM);
 		}
 
 	   	if (dma->bufs[i].buf_count) {
@@ -125,8 +124,8 @@ int drm_dma(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 
-	if (dev->dma_ioctl) {
-		return dev->dma_ioctl(kdev, cmd, data, flags, p, filp);
+	if (dev->driver.dma_ioctl) {
+		return dev->driver.dma_ioctl(kdev, cmd, data, flags, p, filp);
 	} else {
 		DRM_DEBUG("DMA ioctl on driver with no dma handler\n");
 		return EINVAL;
diff --git a/sys/dev/drm/drm_drawable.c b/sys/dev/drm/drm_drawable.c
index 43d71e8791f6..8de8095432cb 100644
--- a/sys/dev/drm/drm_drawable.c
+++ b/sys/dev/drm/drm_drawable.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 int drm_adddraw(DRM_IOCTL_ARGS)
diff --git a/sys/dev/drm/drm_drv.c b/sys/dev/drm/drm_drv.c
index 4f5831c607bc..0e941a4e08b3 100644
--- a/sys/dev/drm/drm_drv.c
+++ b/sys/dev/drm/drm_drv.c
@@ -29,16 +29,21 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
+#include "dev/drm/drm_sarea.h"
 
 int drm_debug_flag = 0;
 
-static int drm_init(device_t nbdev);
-static void drm_cleanup(drm_device_t *dev);
+static int drm_load(drm_device_t *dev);
+static void drm_unload(drm_device_t *dev);
+static drm_pci_id_list_t *drm_find_description(int vendor, int device,
+    drm_pci_id_list_t *idlist);
 
 #ifdef __FreeBSD__
 #define DRIVER_SOFTC(unit) \
@@ -58,67 +63,65 @@ MODULE_DEPEND(drm, mem, 1, 1, 1);
 #endif /* __NetBSD__ || __OpenBSD__ */
 
 static drm_ioctl_desc_t		  drm_ioctls[256] = {
-	[DRM_IOCTL_NR(DRM_IOCTL_VERSION)]       = { drm_version,     0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_UNIQUE)]    = { drm_getunique,   0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAGIC)]     = { drm_getmagic,    0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_IRQ_BUSID)]     = { drm_irq_by_busid, 0, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAP)]       = { drm_getmap,      0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_CLIENT)]    = { drm_getclient,   0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_STATS)]     = { drm_getstats,    0, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_VERSION)]   = { drm_setversion,  0, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_UNIQUE)]    = { drm_setunique,   1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_BLOCK)]         = { drm_noop,        1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_UNBLOCK)]       = { drm_noop,        1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AUTH_MAGIC)]    = { drm_authmagic,   1, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_MAP)]       = { drm_addmap,      1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_MAP)]        = { drm_rmmap,       1, 0 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SET_SAREA_CTX)] = { drm_setsareactx, 1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_SAREA_CTX)] = { drm_getsareactx, 1, 0 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_CTX)]       = { drm_addctx,      1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_CTX)]        = { drm_rmctx,       1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_MOD_CTX)]       = { drm_modctx,      1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_GET_CTX)]       = { drm_getctx,      1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_SWITCH_CTX)]    = { drm_switchctx,   1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_NEW_CTX)]       = { drm_newctx,      1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_RES_CTX)]       = { drm_resctx,      1, 0 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_DRAW)]      = { drm_adddraw,     1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_RM_DRAW)]       = { drm_rmdraw,      1, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_LOCK)]	        = { drm_lock,        1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_UNLOCK)]        = { drm_unlock,      1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_FINISH)]        = { drm_noop,        1, 0 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_ADD_BUFS)]      = { drm_addbufs,     1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_MARK_BUFS)]     = { drm_markbufs,    1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_INFO_BUFS)]     = { drm_infobufs,    1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_MAP_BUFS)]      = { drm_mapbufs,     1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_FREE_BUFS)]     = { drm_freebufs,    1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_DMA)]           = { drm_dma,         1, 0 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_CONTROL)]       = { drm_control,     1, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ACQUIRE)]   = { drm_agp_acquire, 1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_RELEASE)]   = { drm_agp_release, 1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ENABLE)]    = { drm_agp_enable,  1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_INFO)]      = { drm_agp_info,    1, 0 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ALLOC)]     = { drm_agp_alloc,   1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_FREE)]      = { drm_agp_free,    1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_BIND)]      = { drm_agp_bind,    1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_AGP_UNBIND)]    = { drm_agp_unbind,  1, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_SG_ALLOC)]      = { drm_sg_alloc,    1, 1 },
-	[DRM_IOCTL_NR(DRM_IOCTL_SG_FREE)]       = { drm_sg_free,     1, 1 },
-
-	[DRM_IOCTL_NR(DRM_IOCTL_WAIT_VBLANK)]   = { drm_wait_vblank, 0, 0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_VERSION)]       = { drm_version,     0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_UNIQUE)]    = { drm_getunique,   0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAGIC)]     = { drm_getmagic,    0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_IRQ_BUSID)]     = { drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_MAP)]       = { drm_getmap,      0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_CLIENT)]    = { drm_getclient,   0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_STATS)]     = { drm_getstats,    0 },
+	[DRM_IOCTL_NR(DRM_IOCTL_SET_VERSION)]   = { drm_setversion,  DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_SET_UNIQUE)]    = { drm_setunique,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_BLOCK)]         = { drm_noop,        DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_UNBLOCK)]       = { drm_noop,        DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AUTH_MAGIC)]    = { drm_authmagic,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_ADD_MAP)]       = { drm_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_RM_MAP)]        = { drm_rmmap_ioctl, DRM_AUTH },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_SET_SAREA_CTX)] = { drm_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_SAREA_CTX)] = { drm_getsareactx, DRM_AUTH },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_ADD_CTX)]       = { drm_addctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_RM_CTX)]        = { drm_rmctx,       DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_MOD_CTX)]       = { drm_modctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_GET_CTX)]       = { drm_getctx,      DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_SWITCH_CTX)]    = { drm_switchctx,   DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_NEW_CTX)]       = { drm_newctx,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_RES_CTX)]       = { drm_resctx,      DRM_AUTH },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_ADD_DRAW)]      = { drm_adddraw,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_RM_DRAW)]       = { drm_rmdraw,      DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_LOCK)]	        = { drm_lock,        DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_UNLOCK)]        = { drm_unlock,      DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_FINISH)]        = { drm_noop,        DRM_AUTH },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_ADD_BUFS)]      = { drm_addbufs_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_MARK_BUFS)]     = { drm_markbufs,    DRM_AUTH|DRM_MASTER },
+	[DRM_IOCTL_NR(DRM_IOCTL_INFO_BUFS)]     = { drm_infobufs,    DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_MAP_BUFS)]      = { drm_mapbufs,     DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_FREE_BUFS)]     = { drm_freebufs,    DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_DMA)]           = { drm_dma,         DRM_AUTH },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_CONTROL)]       = { drm_control,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ACQUIRE)]   = { drm_agp_acquire_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_RELEASE)]   = { drm_agp_release_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ENABLE)]    = { drm_agp_enable_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_INFO)]      = { drm_agp_info_ioctl, DRM_AUTH },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_ALLOC)]     = { drm_agp_alloc_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_FREE)]      = { drm_agp_free_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_BIND)]      = { drm_agp_bind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_AGP_UNBIND)]    = { drm_agp_unbind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_SG_ALLOC)]      = { drm_sg_alloc,    DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+	[DRM_IOCTL_NR(DRM_IOCTL_SG_FREE)]       = { drm_sg_free,     DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY },
+
+	[DRM_IOCTL_NR(DRM_IOCTL_WAIT_VBLANK)]   = { drm_wait_vblank, 0 },
 };
 
-const char *drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist);
-
 #ifdef __FreeBSD__
 static struct cdevsw drm_cdevsw = {
 #if __FreeBSD_version >= 502103
@@ -144,29 +147,55 @@ static struct cdevsw drm_cdevsw = {
 
 int drm_probe(device_t dev, drm_pci_id_list_t *idlist)
 {
-	const char *s = NULL;
+	drm_pci_id_list_t *id_entry;
 	int vendor, device;
 
 	vendor = pci_get_vendor(dev);
 	device = pci_get_device(dev);
 
-	s = drm_find_description(vendor, device, idlist);
-	if (s != NULL) {
-		device_set_desc(dev, s);
+	id_entry = drm_find_description(vendor, device, idlist);
+	if (id_entry != NULL) {
+		device_set_desc(dev, id_entry->name);
 		return 0;
 	}
 
 	return ENXIO;
 }
 
-int drm_attach(device_t dev, drm_pci_id_list_t *idlist)
+int drm_attach(device_t nbdev, drm_pci_id_list_t *idlist)
 {
-	return drm_init(dev);
+	drm_device_t *dev;
+	drm_pci_id_list_t *id_entry;
+	int unit;
+
+	unit = device_get_unit(nbdev);
+	dev = device_get_softc(nbdev);
+
+	if (!strcmp(device_get_name(nbdev), "drmsub"))
+		dev->device = device_get_parent(nbdev);
+	else
+		dev->device = nbdev;
+
+	dev->devnode = make_dev(&drm_cdevsw,
+			unit,
+			DRM_DEV_UID,
+			DRM_DEV_GID,
+			DRM_DEV_MODE,
+			"dri/card%d", unit);
+#if __FreeBSD_version >= 500000
+	mtx_init(&dev->dev_lock, "drm device", NULL, MTX_DEF);
+#endif
+
+	id_entry = drm_find_description(pci_get_vendor(nbdev),
+	    pci_get_device(nbdev), idlist);
+	dev->id_entry = id_entry;
+
+	return drm_load(dev);
 }
 
 int drm_detach(device_t dev)
 {
-	drm_cleanup(device_get_softc(dev));
+	drm_unload(device_get_softc(dev));
 	return 0;
 }
 
@@ -203,7 +232,7 @@ MOD_DEV("drm", LM_DT_CHAR, CDEV_MAJOR, &drm_cdevsw);
 int drm_lkmentry(struct lkm_table *lkmtp, int cmd, int ver);
 static int drm_lkmhandle(struct lkm_table *lkmtp, int cmd);
 
-int drm_modprobe();
+int drm_modprobe(void);
 int drm_probe(struct pci_attach_args *pa);
 void drm_attach(struct pci_attach_args *pa, dev_t kdev);
 
@@ -213,10 +242,7 @@ int drm_lkmentry(struct lkm_table *lkmtp, int cmd, int ver) {
 
 static int drm_lkmhandle(struct lkm_table *lkmtp, int cmd)
 {
-	int j, error = 0;
-#if defined(__NetBSD__) && (__NetBSD_Version__ > 106080000)
-	struct lkm_dev *args = lkmtp->private.lkm_dev;
-#endif
+	int error = 0;
 
 	switch(cmd) {
 	case LKM_E_LOAD:
@@ -243,7 +269,8 @@ static int drm_lkmhandle(struct lkm_table *lkmtp, int cmd)
 	return error;
 }
 
-int drm_modprobe() {
+int drm_modprobe(void)
+{
 	struct pci_attach_args pa;
 	int error;
 
@@ -257,10 +284,11 @@ int drm_modprobe() {
 int drm_probe(struct pci_attach_args *pa, drm_pci_id_list_t idlist)
 {
 	const char *desc;
+	drm_pci_id_list_t *id_entry;
 
-	desc = drm_find_description(PCI_VENDOR(pa->pa_id),
+	id_entry = drm_find_description(PCI_VENDOR(pa->pa_id),
 	    PCI_PRODUCT(pa->pa_id), idlist);
-	if (desc != NULL) {
+	if (id_entry != NULL) {
 		return 1;
 	}
 
@@ -272,6 +300,7 @@ void drm_attach(struct pci_attach_args *pa, dev_t kdev,
 {
 	int i;
 	drm_device_t *dev;
+	drm_pci_id_list_t *id_entry;
 
 	config_makeroom(kdev, &drm_cd);
 	drm_cd.cd_devs[(kdev)] = malloc(sizeof(drm_device_t), M_DRM, M_WAITOK);
@@ -280,13 +309,24 @@ void drm_attach(struct pci_attach_args *pa, dev_t kdev,
 	memset(dev, 0, sizeof(drm_device_t));
 	memcpy(&dev->pa, pa, sizeof(dev->pa));
 
-	DRM_INFO("%s", drm_find_description(PCI_VENDOR(pa->pa_id), PCI_PRODUCT(pa->pa_id), idlist));
-	drm_init(dev);
+	dev->irq = pa->pa_intrline;
+	dev->pci_domain = 0;
+	dev->pci_bus = pa->pa_bus;
+	dev->pci_slot = pa->pa_device;
+	dev->pci_func = pa->pa_function;
+	dev->dma_tag = pa->pa_dmat;
+
+	id_entry = drm_find_description(PCI_VENDOR(pa->pa_id),
+	    PCI_PRODUCT(pa->pa_id), idlist);
+	dev->driver.pci_id_entry = id_entry;
+
+	DRM_INFO("%s", id_entry->name);
+	drm_load(dev);
 }
 
 int drm_detach(struct device *self, int flags)
 {
-	drm_cleanup((drm_device_t *)self);
+	drm_unload((drm_device_t *)self);
 	return 0;
 }
 
@@ -305,31 +345,39 @@ int drm_activate(struct device *self, enum devact act)
 }
 #endif /* __NetBSD__ || __OpenBSD__ */
 
-const char *drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist) {
+drm_pci_id_list_t *drm_find_description(int vendor, int device,
+    drm_pci_id_list_t *idlist)
+{
 	int i = 0;
 	
 	for (i = 0; idlist[i].vendor != 0; i++) {
 		if ((idlist[i].vendor == vendor) &&
 		    (idlist[i].device == device)) {
-			return idlist[i].name;
+			return &idlist[i];
 		}
 	}
 	return NULL;
 }
 
-/* Initialize the DRM on first open. */
-static int drm_setup(drm_device_t *dev)
+static int drm_firstopen(drm_device_t *dev)
 {
+	drm_local_map_t *map;
 	int i;
 
 	DRM_SPINLOCK_ASSERT(&dev->dev_lock);
 
-	if (dev->presetup)
-		dev->presetup(dev);
+	/* prebuild the SAREA */
+	i = drm_addmap(dev, 0, SAREA_MAX, _DRM_SHM,
+		       _DRM_CONTAINS_LOCK, &map);
+	if (i != 0)
+		return i;
+
+	if (dev->driver.firstopen)
+		dev->driver.firstopen(dev);
 
 	dev->buf_use = 0;
 
-	if (dev->use_dma) {
+	if (dev->driver.use_dma) {
 		i = drm_dma_setup(dev);
 		if (i != 0)
 			return i;
@@ -351,7 +399,6 @@ static int drm_setup(drm_device_t *dev)
 		dev->magiclist[i].tail = NULL;
 	}
 
-	dev->lock.hw_lock = NULL;
 	dev->lock.lock_queue = 0;
 	dev->irq_enabled = 0;
 	dev->context_flag = 0;
@@ -366,14 +413,10 @@ static int drm_setup(drm_device_t *dev)
 
 	DRM_DEBUG( "\n" );
 
-	if (dev->postsetup)
-		dev->postsetup(dev);
-
 	return 0;
 }
 
-/* Free resources associated with the DRM on the last close. */
-static int drm_takedown(drm_device_t *dev)
+static int drm_lastclose(drm_device_t *dev)
 {
 	drm_magic_entry_t *pt, *next;
 	drm_local_map_t *map, *mapsave;
@@ -383,8 +426,8 @@ static int drm_takedown(drm_device_t *dev)
 
 	DRM_DEBUG( "\n" );
 
-	if (dev->pretakedown != NULL)
-		dev->pretakedown(dev);
+	if (dev->driver.lastclose != NULL)
+		dev->driver.lastclose(dev);
 
 	if (dev->irq_enabled)
 		drm_irq_uninstall(dev);
@@ -408,8 +451,9 @@ static int drm_takedown(drm_device_t *dev)
 		drm_agp_mem_t *entry;
 		drm_agp_mem_t *nexte;
 
-				/* Remove AGP resources, but leave dev->agp
-                                   intact until drm_cleanup is called. */
+		/* Remove AGP resources, but leave dev->agp intact until
+		 * drm_unload is called.
+		 */
 		for ( entry = dev->agp->memory ; entry ; entry = nexte ) {
 			nexte = entry->next;
 			if ( entry->bound )
@@ -420,7 +464,7 @@ static int drm_takedown(drm_device_t *dev)
 		dev->agp->memory = NULL;
 
 		if (dev->agp->acquired)
-			drm_agp_do_release();
+			drm_agp_release(dev);
 
 		dev->agp->acquired = 0;
 		dev->agp->enabled  = 0;
@@ -430,12 +474,11 @@ static int drm_takedown(drm_device_t *dev)
 		dev->sg = NULL;
 	}
 
-	/* Clean up maps that weren't set up by the driver. */
 	TAILQ_FOREACH_SAFE(map, &dev->maplist, link, mapsave) {
-		if (!map->kernel_owned)
-			drm_remove_map(dev, map);
+		drm_rmmap(dev, map);
 	}
 
+
 	drm_dma_takedown(dev);
 	if ( dev->lock.hw_lock ) {
 		dev->lock.hw_lock = NULL; /* SHM removed */
@@ -446,39 +489,11 @@ static int drm_takedown(drm_device_t *dev)
 	return 0;
 }
 
-/* linux: drm_init is called via init_module at module load time, or via
- *        linux/init/main.c (this is not currently supported).
- * bsd:   drm_init is called via the attach function per device.
- */
-static int drm_init(device_t nbdev)
+static int drm_load(drm_device_t *dev)
 {
-	int unit;
-	drm_device_t *dev;
 	int retcode;
-	DRM_DEBUG( "\n" );
-
-#ifdef __FreeBSD__
-	unit = device_get_unit(nbdev);
-	dev = device_get_softc(nbdev);
 
-	if (!strcmp(device_get_name(nbdev), "drmsub"))
-		dev->device = device_get_parent(nbdev);
-	else
-		dev->device = nbdev;
-
-	dev->devnode = make_dev(&drm_cdevsw,
-			unit,
-			DRM_DEV_UID,
-			DRM_DEV_GID,
-			DRM_DEV_MODE,
-			"dri/card%d", unit);
-#if __FreeBSD_version >= 500000
-	mtx_init(&dev->dev_lock, "drm device", NULL, MTX_DEF);
-#endif
-#elif defined(__NetBSD__) || defined(__OpenBSD__)
-	dev = nbdev;
-	unit = minor(dev->device.dv_unit);
-#endif
+	DRM_DEBUG( "\n" );
 
 	dev->irq = pci_get_irq(dev->device);
 	/* XXX Fix domain number (alpha hoses) */
@@ -495,16 +510,18 @@ static int drm_init(device_t nbdev)
 #endif
 	TAILQ_INIT(&dev->files);
 
-	if (dev->preinit != NULL) {
-		retcode = dev->preinit(dev, 0);
+	if (dev->driver.load != NULL) {
+		DRM_LOCK();
+		retcode = dev->driver.load(dev, dev->id_entry->driver_private);
+		DRM_UNLOCK();
 		if (retcode != 0)
 			goto error;
 	}
 
-	if (dev->use_agp) {
+	if (dev->driver.use_agp) {
 		if (drm_device_is_agp(dev))
 			dev->agp = drm_agp_init();
-		if (dev->require_agp && dev->agp == NULL) {
+		if (dev->driver.require_agp && dev->agp == NULL) {
 			DRM_ERROR("Card isn't AGP, or couldn't initialize "
 			    "AGP.\n");
 			retcode = DRM_ERR(ENOMEM);
@@ -523,16 +540,12 @@ static int drm_init(device_t nbdev)
 		goto error;
 	}
 	
-	DRM_INFO( "Initialized %s %d.%d.%d %s on minor %d\n",
-	  	dev->driver_name,
-	  	dev->driver_major,
-	  	dev->driver_minor,
-	  	dev->driver_patchlevel,
-	  	dev->driver_date,
-	  	unit );
-
-	if (dev->postinit != NULL)
-		dev->postinit(dev, 0);
+	DRM_INFO("Initialized %s %d.%d.%d %s\n",
+	  	dev->driver.name,
+	  	dev->driver.major,
+	  	dev->driver.minor,
+	  	dev->driver.patchlevel,
+	  	dev->driver.date);
 
 	return 0;
 
@@ -541,7 +554,7 @@ error:
 	drm_sysctl_cleanup(dev);
 #endif
 	DRM_LOCK();
-	drm_takedown(dev);
+	drm_lastclose(dev);
 	DRM_UNLOCK();
 #ifdef __FreeBSD__
 	destroy_dev(dev->devnode);
@@ -552,13 +565,9 @@ error:
 	return retcode;
 }
 
-/* linux: drm_cleanup is called via cleanup_module at module unload time.
- * bsd:   drm_cleanup is called per device at module unload time.
- * FIXME: NetBSD
- */
-static void drm_cleanup(drm_device_t *dev)
+static void drm_unload(drm_device_t *dev)
 {
-	drm_local_map_t *map;
+	int i;
 
 	DRM_DEBUG( "\n" );
 
@@ -572,28 +581,35 @@ static void drm_cleanup(drm_device_t *dev)
 	if (dev->agp && dev->agp->mtrr) {
 		int __unused retcode;
 
-		retcode = drm_mtrr_del(dev->agp->info.ai_aperture_base,
+		retcode = drm_mtrr_del(0, dev->agp->info.ai_aperture_base,
 		    dev->agp->info.ai_aperture_size, DRM_MTRR_WC);
 		DRM_DEBUG("mtrr_del = %d", retcode);
 	}
 
 	DRM_LOCK();
-	drm_takedown(dev);
+	drm_lastclose(dev);
 	DRM_UNLOCK();
 
-	/* Clean up any maps left over that had been allocated by the driver. */
-	while ((map = TAILQ_FIRST(&dev->maplist)) != NULL) {
-		drm_remove_map(dev, map);
+	/* Clean up PCI resources allocated by drm_bufs.c.  We're not really
+	 * worried about resource consumption while the DRM is inactive (between
+	 * lastclose and firstopen or unload) because these aren't actually
+	 * taking up KVA, just keeping the PCI resource allocated.
+	 */
+	for (i = 0; i < DRM_MAX_PCI_RESOURCE; i++) {
+		if (dev->pcir[i] == NULL)
+			continue;
+		bus_release_resource(dev->device, SYS_RES_MEMORY,
+		    dev->pcirid[i], dev->pcir[i]);
+		dev->pcir[i] = NULL;
 	}
 
 	if ( dev->agp ) {
-		drm_agp_uninit();
 		free(dev->agp, M_DRM);
 		dev->agp = NULL;
 	}
 
-	if (dev->postcleanup != NULL)
-		dev->postcleanup(dev);
+	if (dev->driver.unload != NULL)
+		dev->driver.unload(dev);
 
 	drm_mem_uninit();
 #if defined(__FreeBSD__) &&  __FreeBSD_version >= 500000
@@ -619,13 +635,13 @@ int drm_version(DRM_IOCTL_ARGS)
 			return DRM_ERR(EFAULT);				\
 	}
 
-	version.version_major = dev->driver_major;
-	version.version_minor = dev->driver_minor;
-	version.version_patchlevel = dev->driver_patchlevel;
+	version.version_major		= dev->driver.major;
+	version.version_minor		= dev->driver.minor;
+	version.version_patchlevel	= dev->driver.patchlevel;
 
-	DRM_COPY(version.name, dev->driver_name);
-	DRM_COPY(version.date, dev->driver_date);
-	DRM_COPY(version.desc, dev->driver_desc);
+	DRM_COPY(version.name, dev->driver.name);
+	DRM_COPY(version.date, dev->driver.date);
+	DRM_COPY(version.desc, dev->driver.desc);
 
 	DRM_COPY_TO_USER_IOCTL( (drm_version_t *)data, version, sizeof(version) );
 
@@ -650,7 +666,7 @@ int drm_open(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 		device_busy(dev->device);
 #endif
 		if ( !dev->open_count++ )
-			retcode = drm_setup(dev);
+			retcode = drm_firstopen(dev);
 		DRM_UNLOCK();
 	}
 
@@ -675,8 +691,8 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 		return EINVAL;
 	}
 
-	if (dev->prerelease != NULL)
-		dev->prerelease(dev, filp);
+	if (dev->driver.preclose != NULL)
+		dev->driver.preclose(dev, filp);
 
 	/* ========================================================
 	 * Begin inline drm_release
@@ -695,8 +711,8 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 		DRM_DEBUG("Process %d dead, freeing lock for context %d\n",
 			  DRM_CURRENTPID,
 			  _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock));
-		if (dev->release != NULL)
-			dev->release(dev, filp);
+		if (dev->driver.reclaim_buffers_locked != NULL)
+			dev->driver.reclaim_buffers_locked(dev, filp);
 
 		drm_lock_free(dev, &dev->lock.hw_lock->lock,
 		    _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock));
@@ -705,7 +721,8 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
                                    hardware at this point, possibly
                                    processed via a callback to the X
                                    server. */
-	} else if (dev->release != NULL && dev->lock.hw_lock != NULL) {
+	} else if (dev->driver.reclaim_buffers_locked != NULL &&
+	    dev->lock.hw_lock != NULL) {
 		/* The lock is required to reclaim buffers */
 		for (;;) {
 			if ( !dev->lock.hw_lock ) {
@@ -732,14 +749,14 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 				break;
 		}
 		if (retcode == 0) {
-			dev->release(dev, filp);
+			dev->driver.reclaim_buffers_locked(dev, filp);
 			drm_lock_free(dev, &dev->lock.hw_lock->lock,
 			    DRM_KERNEL_CONTEXT);
 		}
 	}
 
-	if (dev->use_dma)
-		drm_reclaim_buffers(dev, (void *)(uintptr_t)priv->pid);
+	if (dev->driver.use_dma && !dev->driver.reclaim_buffers_locked)
+		drm_reclaim_buffers(dev, filp);
 
 #if defined (__FreeBSD__) && (__FreeBSD_version >= 500000)
 	funsetown(&dev->buf_sigio);
@@ -750,8 +767,8 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 #endif /* __NetBSD__  || __OpenBSD__ */
 
 	if (--priv->refs == 0) {
-		if (dev->free_filp_priv != NULL)
-			dev->free_filp_priv(dev, priv);
+		if (dev->driver.postclose != NULL)
+			dev->driver.postclose(dev, priv);
 		TAILQ_REMOVE(&dev->files, priv, link);
 		free(priv, M_DRM);
 	}
@@ -765,7 +782,7 @@ int drm_close(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p)
 	device_unbusy(dev->device);
 #endif
 	if (--dev->open_count == 0) {
-		retcode = drm_takedown(dev);
+		retcode = drm_lastclose(dev);
 	}
 
 	DRM_UNLOCK();
@@ -844,12 +861,12 @@ int drm_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int flags,
 	if (ioctl->func == NULL && nr >= DRM_COMMAND_BASE) {
 		/* The array entries begin at DRM_COMMAND_BASE ioctl nr */
 		nr -= DRM_COMMAND_BASE;
-		if (nr > dev->max_driver_ioctl) {
+		if (nr > dev->driver.max_ioctl) {
 			DRM_DEBUG("Bad driver ioctl number, 0x%x (of 0x%x)\n",
-			    nr, dev->max_driver_ioctl);
+			    nr, dev->driver.max_ioctl);
 			return EINVAL;
 		}
-		ioctl = &dev->driver_ioctls[nr];
+		ioctl = &dev->driver.ioctls[nr];
 		is_driver_ioctl = 1;
 	}
 	func = ioctl->func;
@@ -858,8 +875,12 @@ int drm_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int flags,
 		DRM_DEBUG( "no function\n" );
 		return EINVAL;
 	}
-	if ((ioctl->root_only && DRM_SUSER(p)) || (ioctl->auth_needed &&
-	    !priv->authenticated))
+	/* ioctl->master check should be against something in the filp set up
+	 * for the first opener, but it doesn't matter yet.
+	 */
+	if (((ioctl->flags & DRM_ROOT_ONLY) && !DRM_SUSER(p)) ||
+	    ((ioctl->flags & DRM_AUTH) && !priv->authenticated) ||
+	    ((ioctl->flags & DRM_MASTER) && !priv->master))
 		return EACCES;
 
 	if (is_driver_ioctl)
diff --git a/sys/dev/drm/drm_fops.c b/sys/dev/drm/drm_fops.c
index 9f051159faff..361f5a701264 100644
--- a/sys/dev/drm/drm_fops.c
+++ b/sys/dev/drm/drm_fops.c
@@ -30,9 +30,11 @@
  *    Daryll Strauss <daryll@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 drm_file_t *drm_find_file_by_proc(drm_device_t *dev, DRM_STRUCTPROC *p)
@@ -89,10 +91,12 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		priv->refs		= 1;
 		priv->minor		= m;
 		priv->ioctl_count 	= 0;
-		priv->authenticated	= !DRM_SUSER(p);
 
-		if (dev->open_helper) {
-			retcode = dev->open_helper(dev, priv);
+		/* for compatibility root is always authenticated */
+		priv->authenticated	= DRM_SUSER(p);
+
+		if (dev->driver.open) {
+			retcode = dev->driver.open(dev, priv);
 			if (retcode != 0) {
 				free(priv, M_DRM);
 				DRM_UNLOCK();
@@ -100,6 +104,9 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 			}
 		}
 
+		/* first opener automatically becomes master */
+		priv->master = TAILQ_EMPTY(&dev->files);
+
 		TAILQ_INSERT_TAIL(&dev->files, priv, link);
 	}
 	DRM_UNLOCK();
diff --git a/sys/dev/drm/drm_ioctl.c b/sys/dev/drm/drm_ioctl.c
index 6a5646518bb1..0ea2197287ee 100644
--- a/sys/dev/drm/drm_ioctl.c
+++ b/sys/dev/drm/drm_ioctl.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 /*
@@ -262,8 +264,8 @@ int drm_setversion(DRM_IOCTL_ARGS)
 
 	retv.drm_di_major = DRM_IF_MAJOR;
 	retv.drm_di_minor = DRM_IF_MINOR;
-	retv.drm_dd_major = dev->driver_major;
-	retv.drm_dd_minor = dev->driver_minor;
+	retv.drm_dd_major = dev->driver.major;
+	retv.drm_dd_minor = dev->driver.minor;
 
 	DRM_COPY_TO_USER_IOCTL((drm_set_version_t *)data, retv, sizeof(sv));
 
@@ -282,8 +284,8 @@ int drm_setversion(DRM_IOCTL_ARGS)
 	}
 
 	if (sv.drm_dd_major != -1) {
-		if (sv.drm_dd_major != dev->driver_major ||
-		    sv.drm_dd_minor < 0 || sv.drm_dd_minor > dev->driver_minor)
+		if (sv.drm_dd_major != dev->driver.major ||
+		    sv.drm_dd_minor < 0 || sv.drm_dd_minor > dev->driver.minor)
 			return EINVAL;
 	}
 	return 0;
diff --git a/sys/dev/drm/drm_irq.c b/sys/dev/drm/drm_irq.c
index c7a9981431f1..791f11cc287f 100644
--- a/sys/dev/drm/drm_irq.c
+++ b/sys/dev/drm/drm_irq.c
@@ -1,4 +1,4 @@
-/* drm_dma.c -- DMA IOCTL and function support
+/* drm_irq.c -- IRQ IOCTL and function support
  * Created: Fri Oct 18 2003 by anholt@FreeBSD.org
  */
 /*-
@@ -26,9 +26,11 @@
  * Authors:
  *    Eric Anholt <anholt@FreeBSD.org>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
@@ -62,7 +64,7 @@ drm_irq_handler_wrap(DRM_IRQ_ARGS)
 	drm_device_t *dev = (drm_device_t *)arg;
 
 	DRM_SPINLOCK(&dev->irq_lock);
-	dev->irq_handler(arg);
+	dev->driver.irq_handler(arg);
 	DRM_SPINUNLOCK(&dev->irq_lock);
 }
 #endif
@@ -70,6 +72,9 @@ drm_irq_handler_wrap(DRM_IRQ_ARGS)
 int drm_irq_install(drm_device_t *dev)
 {
 	int retcode;
+#ifdef __NetBSD__
+	pci_intr_handle_t ih;
+#endif
 
 	if (dev->irq == 0 || dev->dev_private == NULL)
 		return DRM_ERR(EINVAL);
@@ -88,7 +93,7 @@ int drm_irq_install(drm_device_t *dev)
 	DRM_SPININIT(dev->irq_lock, "DRM IRQ lock");
 
 				/* Before installing handler */
-	dev->irq_preinstall(dev);
+	dev->driver.irq_preinstall(dev);
 	DRM_UNLOCK();
 
 				/* Install handler */
@@ -110,12 +115,12 @@ int drm_irq_install(drm_device_t *dev)
 	if (retcode != 0)
 		goto err;
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
-	if (pci_intr_map(&dev->pa, &dev->ih) != 0) {
+	if (pci_intr_map(&dev->pa, &ih) != 0) {
 		retcode = ENOENT;
 		goto err;
 	}
-	dev->irqh = pci_intr_establish(&dev->pa.pa_pc, dev->ih, IPL_TTY,
-	    (irqreturn_t (*)(DRM_IRQ_ARGS))dev->irq_handler, dev);
+	dev->irqh = pci_intr_establish(&dev->pa.pa_pc, ih, IPL_TTY,
+	    (irqreturn_t (*)(void *))dev->irq_handler, dev);
 	if (!dev->irqh) {
 		retcode = ENOENT;
 		goto err;
@@ -124,7 +129,7 @@ int drm_irq_install(drm_device_t *dev)
 
 				/* After installing handler */
 	DRM_LOCK();
-	dev->irq_postinstall(dev);
+	dev->driver.irq_postinstall(dev);
 	DRM_UNLOCK();
 
 	return 0;
@@ -145,18 +150,22 @@ err:
 
 int drm_irq_uninstall(drm_device_t *dev)
 {
+#ifdef __FreeBSD__
 	int irqrid;
+#endif
 
 	if (!dev->irq_enabled)
 		return DRM_ERR(EINVAL);
 
 	dev->irq_enabled = 0;
+#ifdef __FreeBSD__
 	irqrid = dev->irqrid;
 	dev->irqrid = 0;
+#endif
 
 	DRM_DEBUG( "%s: irq=%d\n", __FUNCTION__, dev->irq );
 
-	dev->irq_uninstall(dev);
+	dev->driver.irq_uninstall(dev);
 
 #ifdef __FreeBSD__
 	DRM_UNLOCK();
@@ -184,14 +193,14 @@ int drm_control(DRM_IOCTL_ARGS)
 		/* Handle drivers whose DRM used to require IRQ setup but the
 		 * no longer does.
 		 */
-		if (!dev->use_irq)
+		if (!dev->driver.use_irq)
 			return 0;
 		if (dev->if_version < DRM_IF_VERSION(1, 2) &&
 		    ctl.irq != dev->irq)
 			return DRM_ERR(EINVAL);
 		return drm_irq_install(dev);
 	case DRM_UNINST_HANDLER:
-		if (!dev->use_irq)
+		if (!dev->driver.use_irq)
 			return 0;
 		DRM_LOCK();
 		err = drm_irq_uninstall(dev);
@@ -242,7 +251,7 @@ int drm_wait_vblank(DRM_IOCTL_ARGS)
 		ret = EINVAL;
 	} else {
 		DRM_LOCK();
-		ret = dev->vblank_wait(dev, &vblwait.request.sequence);
+		ret = dev->driver.vblank_wait(dev, &vblwait.request.sequence);
 		DRM_UNLOCK();
 
 		microtime(&now);
diff --git a/sys/dev/drm/drm_linux_list.h b/sys/dev/drm/drm_linux_list.h
index f3d07447d292..d0a5c7008a7c 100644
--- a/sys/dev/drm/drm_linux_list.h
+++ b/sys/dev/drm/drm_linux_list.h
@@ -27,9 +27,11 @@
  * Authors:
  *    Eric Anholt <anholt@FreeBSD.org>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 struct list_head {
 	struct list_head *next, *prev;
 };
diff --git a/sys/dev/drm/drm_lock.c b/sys/dev/drm/drm_lock.c
index 6bfec3f88a36..b075fb4bf5f4 100644
--- a/sys/dev/drm/drm_lock.c
+++ b/sys/dev/drm/drm_lock.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 int drm_lock_take(__volatile__ unsigned int *lock, unsigned int context)
@@ -113,7 +115,7 @@ int drm_lock(DRM_IOCTL_ARGS)
         DRM_DEBUG("%d (pid %d) requests lock (0x%08x), flags = 0x%08x\n",
 	    lock.context, DRM_CURRENTPID, dev->lock.hw_lock->lock, lock.flags);
 
-        if (dev->use_dma_queue && lock.context < 0)
+        if (dev->driver.use_dma_queue && lock.context < 0)
                 return EINVAL;
 
 	DRM_LOCK();
@@ -144,8 +146,9 @@ int drm_lock(DRM_IOCTL_ARGS)
 
 	/* XXX: Add signal blocking here */
 
-	if (dev->dma_quiescent != NULL && (lock.flags & _DRM_LOCK_QUIESCENT))
-		dev->dma_quiescent(dev);
+	if (dev->driver.dma_quiescent != NULL &&
+	    (lock.flags & _DRM_LOCK_QUIESCENT))
+		dev->driver.dma_quiescent(dev);
 
 	return 0;
 }
diff --git a/sys/dev/drm/drm_memory.c b/sys/dev/drm/drm_memory.c
index 7d0ab5ae54d7..6f035387c81f 100644
--- a/sys/dev/drm/drm_memory.c
+++ b/sys/dev/drm/drm_memory.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 MALLOC_DEFINE(M_DRM, "drm", "DRM Data Structures");
@@ -114,7 +116,7 @@ drm_mtrr_add(unsigned long offset, size_t size, int flags)
 }
 
 int
-drm_mtrr_del(unsigned long offset, size_t size, int flags)
+drm_mtrr_del(int __unused handle, unsigned long offset, size_t size, int flags)
 {
 	int act;
 	struct mem_range_desc mrdesc;
diff --git a/sys/dev/drm/drm_pci.c b/sys/dev/drm/drm_pci.c
index 0569e9315328..0dca37b075e3 100644
--- a/sys/dev/drm/drm_pci.c
+++ b/sys/dev/drm/drm_pci.c
@@ -26,44 +26,119 @@
  * AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 /**********************************************************************/
 /** \name PCI memory */
 /*@{*/
 
+#if defined(__FreeBSD__)
+static void
+drm_pci_busdma_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+	drm_dma_handle_t *dmah = arg;
+
+	if (error != 0)
+		return;
+
+	KASSERT(nsegs == 1, ("drm_pci_busdma_callback: bad dma segment count"));
+	dmah->busaddr = segs[0].ds_addr;
+}
+#endif
+
 /**
  * \brief Allocate a physically contiguous DMA-accessible consistent 
  * memory block.
  */
-void *
-drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr,
-    dma_addr_t *busaddr)
+drm_dma_handle_t *
+drm_pci_alloc(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr)
 {
-	void *vaddr;
+	drm_dma_handle_t *dmah;
+	int ret;
+
+	/* Need power-of-two alignment, so fail the allocation if it isn't. */
+	if ((align & (align - 1)) != 0) {
+		DRM_ERROR("drm_pci_alloc with non-power-of-two alignment %d\n",
+		    (int)align);
+		return NULL;
+	}
 
-	vaddr = contigmalloc(size, M_DRM, M_NOWAIT, 0ul, maxaddr, align,
-	    0);
-	*busaddr = vtophys(vaddr);
-	
-	return vaddr;
+	dmah = malloc(sizeof(drm_dma_handle_t), M_DRM, M_ZERO | M_NOWAIT);
+	if (dmah == NULL)
+		return NULL;
+
+#ifdef __FreeBSD__
+	ret = bus_dma_tag_create(NULL, align, 0, /* tag, align, boundary */
+	    maxaddr, BUS_SPACE_MAXADDR, /* lowaddr, highaddr */
+	    NULL, NULL, /* filtfunc, filtfuncargs */
+	    size, 1, size, /* maxsize, nsegs, maxsegsize */
+	    BUS_DMA_ALLOCNOW, NULL, NULL, /* flags, lockfunc, lockfuncargs */
+	    &dmah->tag);
+	if (ret != 0) {
+		free(dmah, M_DRM);
+		return NULL;
+	}
+
+	ret = bus_dmamem_alloc(dmah->tag, &dmah->vaddr, BUS_DMA_NOWAIT,
+	    &dmah->map);
+	if (ret != 0) {
+		bus_dma_tag_destroy(dmah->tag);
+		free(dmah, M_DRM);
+		return NULL;
+	}
+
+	ret = bus_dmamap_load(dmah->tag, dmah->map, dmah->vaddr, size,
+	    drm_pci_busdma_callback, dmah, 0);
+	if (ret != 0) {
+		bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
+		bus_dma_tag_destroy(dmah->tag);
+		free(dmah, M_DRM);
+		return NULL;
+	}
+#elif defined(__NetBSD__)
+	ret = bus_dmamem_alloc(dev->dma_tag, size, align, PAGE_SIZE,
+	    &dmah->seg, 1, &nsegs, BUS_DMA_NOWAIT);
+	if ((ret != 0) || (nsegs != 1)) {
+		free(dmah, M_DRM);
+		return NULL;
+	}
+
+	ret = bus_dmamem_map(dev->dma_tag, &dmah->seg, 1, size, &dmah->addr,
+	    BUS_DMA_NOWAIT);
+	if (ret != 0) {
+		bus_dmamem_free(dev->dma_tag, &dmah->seg, 1);
+		free(dmah, M_DRM);
+		return NULL;
+	}
+
+	dmah->dmaaddr = h->seg.ds_addr;
+#endif
+
+	return dmah;
 }
 
 /**
  * \brief Free a DMA-accessible consistent memory block.
  */
 void
-drm_pci_free(drm_device_t *dev, size_t size, void *vaddr, dma_addr_t busaddr)
+drm_pci_free(drm_device_t *dev, drm_dma_handle_t *dmah)
 {
-#if __FreeBSD_version > 500000
-	if (vaddr == NULL)
+	if (dmah == NULL)
 		return;
-	contigfree(vaddr, size, M_DRM);	/* Not available on 4.x */
+
+#if defined(__FreeBSD__)
+	bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
+	bus_dma_tag_destroy(dmah->tag);
+#elif defined(__NetBSD__)
+	bus_dmamem_free(dev->dma_tag, &dmah->seg, 1);
 #endif
+
+	free(dmah, M_DRM);
 }
 
 /*@}*/
diff --git a/sys/dev/drm/drm_pciids.h b/sys/dev/drm/drm_pciids.h
index 8a5e8f7f9210..d0868a5bed72 100644
--- a/sys/dev/drm/drm_pciids.h
+++ b/sys/dev/drm/drm_pciids.h
@@ -1,7 +1,6 @@
 /*
  * $FreeBSD$
  */
-
 /*
    This file is auto-generated from the drm_pciids.txt in the DRM CVS
    Please contact dri-devel@lists.sf.net to add new cards to this list
@@ -50,6 +49,7 @@
 	{0x1002, 0x4E50, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV300 Mobility 9600 M10"}, \
 	{0x1002, 0x4E51, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon RV350 Mobility 9600 M10 NQ"}, \
 	{0x1002, 0x4E54, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon FireGL T2 128"}, \
+	{0x1002, 0x4E56, CHIP_RV350|CHIP_IS_MOBILITY, "ATI Radeon FireGL Mobility T2e"}, \
 	{0x1002, 0x5144, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QD R100"}, \
 	{0x1002, 0x5145, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QE R100"}, \
 	{0x1002, 0x5146, CHIP_R100|CHIP_SINGLE_CRTC, "ATI Radeon QF R100"}, \
@@ -66,12 +66,14 @@
 	{0x1002, 0x5158, CHIP_RV200, "ATI Radeon QX RV200 7500"}, \
 	{0x1002, 0x5159, CHIP_RV100, "ATI Radeon QY RV100 7000/VE"}, \
 	{0x1002, 0x515A, CHIP_RV100, "ATI Radeon QZ RV100 7000/VE"}, \
+	{0x1002, 0x515E, CHIP_RV100, "ATI ES1000 RN50"}, \
 	{0x1002, 0x5168, CHIP_R200, "ATI Radeon Qh R200"}, \
 	{0x1002, 0x5169, CHIP_R200, "ATI Radeon Qi R200"}, \
 	{0x1002, 0x516A, CHIP_R200, "ATI Radeon Qj R200"}, \
 	{0x1002, 0x516B, CHIP_R200, "ATI Radeon Qk R200"}, \
 	{0x1002, 0x516C, CHIP_R200, "ATI Radeon Ql R200"}, \
 	{0x1002, 0x5460, CHIP_RV350, "ATI Radeon X300"}, \
+	{0x1002, 0x554F, CHIP_R350, "ATI Radeon X800"}, \
 	{0x1002, 0x5834, CHIP_RS300|CHIP_IS_IGP, "ATI Radeon RS300 IGP"}, \
 	{0x1002, 0x5835, CHIP_RS300|CHIP_IS_IGP|CHIP_IS_MOBILITY, "ATI Radeon RS300 Mobility IGP"}, \
 	{0x1002, 0x5836, CHIP_RS300|CHIP_IS_IGP, "ATI Radeon RS300 IGP"}, \
@@ -85,10 +87,12 @@
 	{0x1002, 0x5969, CHIP_RV100, "ATI ES1000 RN50"}, \
 	{0x1002, 0x596A, CHIP_RV280, "ATI Radeon RV280 9200"}, \
 	{0x1002, 0x596B, CHIP_RV280, "ATI Radeon RV280 9200"}, \
+	{0x1002, 0x5b60, CHIP_RV350, "ATI Radeon RV370 X300SE"}, \
 	{0x1002, 0x5c61, CHIP_RV280|CHIP_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
 	{0x1002, 0x5c62, CHIP_RV280, "ATI Radeon RV280"}, \
 	{0x1002, 0x5c63, CHIP_RV280|CHIP_IS_MOBILITY, "ATI Radeon RV280 Mobility"}, \
 	{0x1002, 0x5c64, CHIP_RV280, "ATI Radeon RV280"}, \
+	{0x1002, 0x5d4d, CHIP_R350, "ATI Radeon R480"}, \
 	{0, 0, 0, NULL}
 
 #define r128_PCI_IDS \
@@ -132,9 +136,10 @@
 	{0, 0, 0, NULL}
 
 #define mga_PCI_IDS \
-	{0x102b, 0x0521, 0, "Matrox G200 (AGP)"}, \
-	{0x102b, 0x0525, 0, "Matrox G400/G450 (AGP)"}, \
-	{0x102b, 0x2527, 0, "Matrox G550 (AGP)"}, \
+	{0x102b, 0x0520, MGA_CARD_TYPE_G200, "Matrox G200 (PCI)"}, \
+	{0x102b, 0x0521, MGA_CARD_TYPE_G200, "Matrox G200 (AGP)"}, \
+	{0x102b, 0x0525, MGA_CARD_TYPE_G400, "Matrox G400/G450 (AGP)"}, \
+	{0x102b, 0x2527, MGA_CARD_TYPE_G550, "Matrox G550 (AGP)"}, \
 	{0, 0, 0, NULL}
 
 #define mach64_PCI_IDS \
@@ -178,10 +183,10 @@
 
 #define viadrv_PCI_IDS \
 	{0x1106, 0x3022, 0, "VIA CLE266 3022"}, \
-	{0x1106, 0x3118, 0, "VIA CN400"}, \
+	{0x1106, 0x3118, VIA_PRO_GROUP_A, "VIA CN400 / PM8X0"}, \
 	{0x1106, 0x3122, 0, "VIA CLE266"}, \
 	{0x1106, 0x7205, 0, "VIA KM400"}, \
-	{0x1106, 0x7204, 0, "VIA K8M800"}, \
+	{0x1106, 0x3108, 0, "VIA K8M800"}, \
 	{0, 0, 0, NULL}
 
 #define i810_PCI_IDS \
@@ -237,6 +242,194 @@
 	{0x8086, 0x3582, 0, "Intel i852GM/i855GM GMCH"}, \
 	{0x8086, 0x2572, 0, "Intel i865G GMCH"}, \
 	{0x8086, 0x2582, 0, "Intel i915G"}, \
-	{0x8086, 0x2982, 0, "Intel i915GM"}, \
+	{0x8086, 0x2592, 0, "Intel i915GM"}, \
+	{0x8086, 0x2772, 0, "Intel i945G"}, \
+	{0, 0, 0, NULL}
+
+#define imagine_PCI_IDS \
+	{0x105d, 0x2309, IMAGINE_128, "Imagine 128"}, \
+	{0x105d, 0x2339, IMAGINE_128_2, "Imagine 128-II"}, \
+	{0x105d, 0x493d, IMAGINE_T2R, "Ticket to Ride"}, \
+	{0x105d, 0x5348, IMAGINE_REV4, "Revolution IV"}, \
 	{0, 0, 0, NULL}
 
+#define nv_PCI_IDS \
+	{0x10DE, 0x0020, NV04, "NVidia RIVA TNT"}, \
+	{0x10DE, 0x0028, NV04, "NVidia RIVA TNT2"}, \
+	{0x10DE, 0x002A, NV04, "NVidia Unknown TNT2"}, \
+	{0x10DE, 0x002C, NV04, "NVidia Vanta"}, \
+	{0x10DE, 0x0029, NV04, "NVidia RIVA TNT2 Ultra"}, \
+	{0x10DE, 0x002D, NV04, "NVidia RIVA TNT2 Model 64"}, \
+	{0x10DE, 0x00A0, NV04, "NVidia Aladdin TNT2"}, \
+	{0x10DE, 0x0100, NV10, "NVidia GeForce 256"}, \
+	{0x10DE, 0x0101, NV10, "NVidia GeForce DDR"}, \
+	{0x10DE, 0x0103, NV10, "NVidia Quadro"}, \
+	{0x10DE, 0x0110, NV10, "NVidia GeForce2 MX/MX 400"}, \
+	{0x10DE, 0x0111, NV10, "NVidia GeForce2 MX 100/200"}, \
+	{0x10DE, 0x0112, NV10, "NVidia GeForce2 Go"}, \
+	{0x10DE, 0x0113, NV10, "NVidia Quadro2 MXR/EX/Go"}, \
+	{0x10DE, 0x0150, NV10, "NVidia GeForce2 GTS"}, \
+	{0x10DE, 0x0151, NV10, "NVidia GeForce2 Ti"}, \
+	{0x10DE, 0x0152, NV10, "NVidia GeForce2 Ultra"}, \
+	{0x10DE, 0x0153, NV10, "NVidia Quadro2 Pro"}, \
+	{0x10DE, 0x0170, NV10, "NVidia GeForce4 MX 460"}, \
+	{0x10DE, 0x0171, NV10, "NVidia GeForce4 MX 440"}, \
+	{0x10DE, 0x0172, NV10, "NVidia GeForce4 MX 420"}, \
+	{0x10DE, 0x0173, NV10, "NVidia GeForce4 MX 440-SE"}, \
+	{0x10DE, 0x0174, NV10, "NVidia GeForce4 440 Go"}, \
+	{0x10DE, 0x0175, NV10, "NVidia GeForce4 420 Go"}, \
+	{0x10DE, 0x0176, NV10, "NVidia GeForce4 420 Go 32M"}, \
+	{0x10DE, 0x0177, NV10, "NVidia GeForce4 460 Go"}, \
+	{0x10DE, 0x0178, NV10, "NVidia Quadro4 550 XGL"}, \
+	{0x10DE, 0x0179, NV10, "NVidia GeForce4"}, \
+	{0x10DE, 0x017A, NV10, "NVidia Quadro4 NVS"}, \
+	{0x10DE, 0x017C, NV10, "NVidia Quadro4 500 GoGL"}, \
+	{0x10DE, 0x017D, NV10, "NVidia GeForce4 410 Go 16M"}, \
+	{0x10DE, 0x0181, NV10, "NVidia GeForce4 MX 440 with AGP8X"}, \
+	{0x10DE, 0x0182, NV10, "NVidia GeForce4 MX 440SE with AGP8X"}, \
+	{0x10DE, 0x0183, NV10, "NVidia GeForce4 MX 420 with AGP8X"}, \
+	{0x10DE, 0x0185, NV10, "NVidia GeForce4 MX 4000"}, \
+	{0x10DE, 0x0186, NV10, "NVidia GeForce4 448 Go"}, \
+	{0x10DE, 0x0187, NV10, "NVidia GeForce4 488 Go"}, \
+	{0x10DE, 0x0188, NV10, "NVidia Quadro4 580 XGL"}, \
+	{0x10DE, 0x0189, NV10, "NVidia GeForce4 MX with AGP8X (Mac)"}, \
+	{0x10DE, 0x018A, NV10, "NVidia Quadro4 280 NVS"}, \
+	{0x10DE, 0x018B, NV10, "NVidia Quadro4 380 XGL"}, \
+	{0x10DE, 0x018C, NV10, "NVidia Quadro NVS 50 PCI"}, \
+	{0x10DE, 0x018D, NV10, "NVidia GeForce4 448 Go"}, \
+	{0x10DE, 0x01A0, NV10, "NVidia GeForce2 Integrated GPU"}, \
+	{0x10DE, 0x01F0, NV10, "NVidia GeForce4 MX Integrated GPU"}, \
+	{0x10DE, 0x0200, NV20, "NVidia GeForce3"}, \
+	{0x10DE, 0x0201, NV20, "NVidia GeForce3 Ti 200"}, \
+	{0x10DE, 0x0202, NV20, "NVidia GeForce3 Ti 500"}, \
+	{0x10DE, 0x0203, NV20, "NVidia Quadro DCC"}, \
+	{0x10DE, 0x0250, NV20, "NVidia GeForce4 Ti 4600"}, \
+	{0x10DE, 0x0251, NV20, "NVidia GeForce4 Ti 4400"}, \
+	{0x10DE, 0x0252, NV20, "NVidia 0x0252"}, \
+	{0x10DE, 0x0253, NV20, "NVidia GeForce4 Ti 4200"}, \
+	{0x10DE, 0x0258, NV20, "NVidia Quadro4 900 XGL"}, \
+	{0x10DE, 0x0259, NV20, "NVidia Quadro4 750 XGL"}, \
+	{0x10DE, 0x025B, NV20, "NVidia Quadro4 700 XGL"}, \
+	{0x10DE, 0x0280, NV20, "NVidia GeForce4 Ti 4800"}, \
+	{0x10DE, 0x0281, NV20, "NVidia GeForce4 Ti 4200 with AGP8X"}, \
+	{0x10DE, 0x0282, NV20, "NVidia GeForce4 Ti 4800 SE"}, \
+	{0x10DE, 0x0286, NV20, "NVidia GeForce4 4200 Go"}, \
+	{0x10DE, 0x028C, NV20, "NVidia Quadro4 700 GoGL"}, \
+	{0x10DE, 0x0288, NV20, "NVidia Quadro4 980 XGL"}, \
+	{0x10DE, 0x0289, NV20, "NVidia Quadro4 780 XGL"}, \
+	{0x10DE, 0x0301, NV30, "NVidia GeForce FX 5800 Ultra"}, \
+	{0x10DE, 0x0302, NV30, "NVidia GeForce FX 5800"}, \
+	{0x10DE, 0x0308, NV30, "NVidia Quadro FX 2000"}, \
+	{0x10DE, 0x0309, NV30, "NVidia Quadro FX 1000"}, \
+	{0x10DE, 0x0311, NV30, "NVidia GeForce FX 5600 Ultra"}, \
+	{0x10DE, 0x0312, NV30, "NVidia GeForce FX 5600"}, \
+	{0x10DE, 0x0313, NV30, "NVidia 0x0313"},}, \
+	{0x10DE, 0x0314, NV30, "NVidia GeForce FX 5600SE"}, \
+	{0x10DE, 0x0316, NV30, "NVidia 0x0316"}, \
+	{0x10DE, 0x0317, NV30, "NVidia 0x0317"}, \
+	{0x10DE, 0x031A, NV30, "NVidia GeForce FX Go5600"}, \
+	{0x10DE, 0x031B, NV30, "NVidia GeForce FX Go5650"}, \
+	{0x10DE, 0x031C, NV30, "NVidia Quadro FX Go700"}, \
+	{0x10DE, 0x031D, NV30, "NVidia 0x031D"}, \
+	{0x10DE, 0x031E, NV30, "NVidia 0x031E"}, \
+	{0x10DE, 0x031F, NV30, "NVidia 0x031F"}, \
+	{0x10DE, 0x0320, NV30, "NVidia GeForce FX 5200"}, \
+	{0x10DE, 0x0321, NV30, "NVidia GeForce FX 5200 Ultra"}, \
+	{0x10DE, 0x0322, NV30, "NVidia GeForce FX 5200"}, \
+	{0x10DE, 0x0323, NV30, "NVidia GeForce FX 5200SE"}, \
+	{0x10DE, 0x0324, NV30, "NVidia GeForce FX Go5200"}, \
+	{0x10DE, 0x0325, NV30, "NVidia GeForce FX Go5250"}, \
+	{0x10DE, 0x0326, NV30, "NVidia GeForce FX 5500"}, \
+	{0x10DE, 0x0327, NV30, "NVidia GeForce FX 5100"}, \
+	{0x10DE, 0x0328, NV30, "NVidia GeForce FX Go5200 32M/64M"}, \
+	{0x10DE, 0x0329, NV30, "NVidia GeForce FX 5200 (Mac)"}, \
+	{0x10DE, 0x032A, NV30, "NVidia Quadro NVS 280 PCI"}, \
+	{0x10DE, 0x032B, NV30, "NVidia Quadro FX 500/600 PCI"}, \
+	{0x10DE, 0x032C, NV30, "NVidia GeForce FX Go53xx Series"}, \
+	{0x10DE, 0x032D, NV30, "NVidia GeForce FX Go5100"}, \
+	{0x10DE, 0x032F, NV30, "NVidia 0x032F"}, \
+	{0x10DE, 0x0330, NV30, "NVidia GeForce FX 5900 Ultra"}, \
+	{0x10DE, 0x0331, NV30, "NVidia GeForce FX 5900"}, \
+	{0x10DE, 0x0332, NV30, "NVidia GeForce FX 5900XT"}, \
+	{0x10DE, 0x0333, NV30, "NVidia GeForce FX 5950 Ultra"}, \
+	{0x10DE, 0x033F, NV30, "NVidia Quadro FX 700"}, \
+	{0x10DE, 0x0334, NV30, "NVidia GeForce FX 5900ZT"}, \
+	{0x10DE, 0x0338, NV30, "NVidia Quadro FX 3000"}, \
+	{0x10DE, 0x0341, NV30, "NVidia GeForce FX 5700 Ultra"}, \
+	{0x10DE, 0x0342, NV30, "NVidia GeForce FX 5700"}, \
+	{0x10DE, 0x0343, NV30, "NVidia GeForce FX 5700LE"}, \
+	{0x10DE, 0x0344, NV30, "NVidia GeForce FX 5700VE"}, \
+	{0x10DE, 0x0345, NV30, "NVidia 0x0345"}, \
+	{0x10DE, 0x0347, NV30, "NVidia GeForce FX Go5700"}, \
+	{0x10DE, 0x0348, NV30, "NVidia GeForce FX Go5700"}, \
+	{0x10DE, 0x0349, NV30, "NVidia 0x0349"}, \
+	{0x10DE, 0x034B, NV30, "NVidia 0x034B"}, \
+	{0x10DE, 0x034C, NV30, "NVidia Quadro FX Go1000"}, \
+	{0x10DE, 0x034E, NV30, "NVidia Quadro FX 1100"}, \
+	{0x10DE, 0x034F, NV30, "NVidia 0x034F"}, \
+	{0x10DE, 0x0040, NV40, "NVidia GeForce 6800 Ultra"}, \
+	{0x10DE, 0x0041, NV40, "NVidia GeForce 6800"}, \
+	{0x10DE, 0x0042, NV40, "NVidia GeForce 6800 LE"}, \
+	{0x10DE, 0x0043, NV40, "NVidia 0x0043"}, \
+	{0x10DE, 0x0045, NV40, "NVidia GeForce 6800 GT"}, \
+	{0x10DE, 0x0046, NV40, "NVidia GeForce 6800 GT"}, \
+	{0x10DE, 0x0049, NV40, "NVidia 0x0049"}, \
+	{0x10DE, 0x004E, NV40, "NVidia Quadro FX 4000"}, \
+	{0x10DE, 0x00C0, NV40, "NVidia 0x00C0"}, \
+	{0x10DE, 0x00C1, NV40, "NVidia GeForce 6800"}, \
+	{0x10DE, 0x00C2, NV40, "NVidia GeForce 6800 LE"}, \
+	{0x10DE, 0x00C8, NV40, "NVidia GeForce Go 6800"}, \
+	{0x10DE, 0x00C9, NV40, "NVidia GeForce Go 6800 Ultra"}, \
+	{0x10DE, 0x00CC, NV40, "NVidia Quadro FX Go1400"}, \
+	{0x10DE, 0x00CD, NV40, "NVidia Quadro FX 3450/4000 SDI"}, \
+	{0x10DE, 0x00CE, NV40, "NVidia Quadro FX 1400"}, \
+	{0x10de, 0x00f0, NV40, "Nvidia GeForce 6600 GT"}, \
+	{0x10de, 0x00f1, NV40, "Nvidia GeForce 6600 GT"}, \
+	{0x10DE, 0x0140, NV40, "NVidia GeForce 6600 GT"}, \
+	{0x10DE, 0x0141, NV40, "NVidia GeForce 6600"}, \
+	{0x10DE, 0x0142, NV40, "NVidia GeForce 6600 LE"}, \
+	{0x10DE, 0x0143, NV40, "NVidia 0x0143"}, \
+	{0x10DE, 0x0144, NV40, "NVidia GeForce Go 6600"}, \
+	{0x10DE, 0x0145, NV40, "NVidia GeForce 6610 XL"}, \
+	{0x10DE, 0x0146, NV40, "NVidia GeForce Go 6600 TE/6200 TE"}, \
+	{0x10DE, 0x0147, NV40, "NVidia GeForce 6700 XL"}, \
+	{0x10DE, 0x0148, NV40, "NVidia GeForce Go 6600"}, \
+	{0x10DE, 0x0149, NV40, "NVidia GeForce Go 6600 GT"}, \
+	{0x10DE, 0x014B, NV40, "NVidia 0x014B"}, \
+	{0x10DE, 0x014C, NV40, "NVidia 0x014C"}, \
+	{0x10DE, 0x014D, NV40, "NVidia 0x014D"}, \
+	{0x10DE, 0x014E, NV40, "NVidia Quadro FX 540"}, \
+	{0x10DE, 0x014F, NV40, "NVidia GeForce 6200"}, \
+	{0x10DE, 0x0160, NV40, "NVidia 0x0160"}, \
+	{0x10DE, 0x0161, NV40, "NVidia GeForce 6200 TurboCache(TM)"}, \
+	{0x10DE, 0x0162, NV40, "NVidia GeForce 6200SE TurboCache(TM)"}, \
+	{0x10DE, 0x0163, NV40, "NVidia 0x0163"}, \
+	{0x10DE, 0x0164, NV40, "NVidia GeForce Go 6200"}, \
+	{0x10DE, 0x0165, NV40, "NVidia Quadro NVS 285"}, \
+	{0x10DE, 0x0166, NV40, "NVidia GeForce Go 6400"}, \
+	{0x10DE, 0x0167, NV40, "NVidia GeForce Go 6200"}, \
+	{0x10DE, 0x0168, NV40, "NVidia GeForce Go 6400"}, \
+	{0x10DE, 0x0169, NV40, "NVidia 0x0169"}, \
+	{0x10DE, 0x016B, NV40, "NVidia 0x016B"}, \
+	{0x10DE, 0x016C, NV40, "NVidia 0x016C"}, \
+	{0x10DE, 0x016D, NV40, "NVidia 0x016D"}, \
+	{0x10DE, 0x016E, NV40, "NVidia 0x016E"}, \
+	{0x10DE, 0x0210, NV40, "NVidia 0x0210"}, \
+	{0x10DE, 0x0211, NV40, "NVidia GeForce 6800"}, \
+	{0x10DE, 0x0212, NV40, "NVidia GeForce 6800 LE"}, \
+	{0x10DE, 0x0215, NV40, "NVidia GeForce 6800 GT"}, \
+	{0x10DE, 0x0220, NV40, "NVidia 0x0220"}, \
+	{0x10DE, 0x0221, NV40, "NVidia GeForce 6200"}, \
+	{0x10DE, 0x0222, NV40, "NVidia 0x0222"}, \
+	{0x10DE, 0x0228, NV40, "NVidia 0x0228"}, \
+	{0x10DE, 0x0090, NV40, "NVidia 0x0090"}, \
+	{0x10DE, 0x0091, NV40, "NVidia GeForce 7800 GTX"}, \
+	{0x10DE, 0x0092, NV40, "NVidia 0x0092"}, \
+	{0x10DE, 0x0093, NV40, "NVidia 0x0093"}, \
+	{0x10DE, 0x0094, NV40, "NVidia 0x0094"}, \
+	{0x10DE, 0x0098, NV40, "NVidia 0x0098"}, \
+	{0x10DE, 0x0099, NV40, "NVidia GeForce Go 7800 GTX"}, \
+	{0x10DE, 0x009C, NV40, "NVidia 0x009C"}, \
+	{0x10DE, 0x009D, NV40, "NVidia Quadro FX 4500"}, \
+	{0x10DE, 0x009E, NV40, "NVidia 0x009E"}, \
+	{0, 0, 0, NULL}
diff --git a/sys/dev/drm/drm_sarea.h b/sys/dev/drm/drm_sarea.h
index 5215d2e33af1..842d3ac05563 100644
--- a/sys/dev/drm/drm_sarea.h
+++ b/sys/dev/drm/drm_sarea.h
@@ -27,10 +27,11 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef _DRM_SAREA_H_
 #define _DRM_SAREA_H_
 
diff --git a/sys/dev/drm/drm_scatter.c b/sys/dev/drm/drm_scatter.c
index fe8bb4b25b81..98e02f65f81a 100644
--- a/sys/dev/drm/drm_scatter.c
+++ b/sys/dev/drm/drm_scatter.c
@@ -27,16 +27,18 @@
  *   Gareth Hughes <gareth@valinux.com>
  *   Eric Anholt <anholt@FreeBSD.org>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 
 #define DEBUG_SCATTER 0
 
 void drm_sg_cleanup(drm_sg_mem_t *entry)
 {
-	free(entry->virtual, M_DRM);
+	free((void *)entry->handle, M_DRM);
 	free(entry->busaddr, M_DRM);
 	free(entry, M_DRM);
 }
@@ -47,6 +49,7 @@ int drm_sg_alloc(DRM_IOCTL_ARGS)
 	drm_scatter_gather_t request;
 	drm_sg_mem_t *entry;
 	unsigned long pages;
+	int i;
 
 	DRM_DEBUG( "%s\n", __FUNCTION__ );
 
@@ -72,16 +75,18 @@ int drm_sg_alloc(DRM_IOCTL_ARGS)
 		return ENOMEM;
 	}
 
-	entry->virtual = malloc(pages << PAGE_SHIFT, M_DRM, M_WAITOK | M_ZERO);
-	if ( !entry->virtual ) {
+	entry->handle = (long)malloc(pages << PAGE_SHIFT, M_DRM,
+	    M_WAITOK | M_ZERO);
+	if (entry->handle == 0) {
 		drm_sg_cleanup(entry);
 		return ENOMEM;
 	}
 
-	entry->handle = (unsigned long)entry->virtual;
+	for (i = 0; i < pages; i++) {
+		entry->busaddr[i] = vtophys(entry->handle + i * PAGE_SIZE);
+	}
 
 	DRM_DEBUG( "sg alloc handle  = %08lx\n", entry->handle );
-	DRM_DEBUG( "sg alloc virtual = %p\n", entry->virtual );
 
 	request.handle = entry->handle;
 
@@ -118,7 +123,7 @@ int drm_sg_free(DRM_IOCTL_ARGS)
 	if ( !entry || entry->handle != request.handle )
 		return EINVAL;
 
-	DRM_DEBUG( "sg free virtual  = %p\n", entry->virtual );
+	DRM_DEBUG( "sg free virtual  = 0x%lx\n", entry->handle );
 
 	drm_sg_cleanup(entry);
 
diff --git a/sys/dev/drm/drm_sysctl.c b/sys/dev/drm/drm_sysctl.c
index 5a9e35b556b6..eb5bbe2c8613 100644
--- a/sys/dev/drm/drm_sysctl.c
+++ b/sys/dev/drm/drm_sysctl.c
@@ -19,10 +19,11 @@
  * ERIC ANHOLT BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
@@ -129,7 +130,7 @@ static int drm_name_info DRM_SYSCTL_HANDLER_ARGS
 	int retcode;
 	int hasunique = 0;
 
-	DRM_SYSCTL_PRINT("%s 0x%x", dev->driver_name, dev2udev(dev->devnode));
+	DRM_SYSCTL_PRINT("%s 0x%x", dev->driver.name, dev2udev(dev->devnode));
 	
 	DRM_LOCK();
 	if (dev->unique) {
diff --git a/sys/dev/drm/drm_vm.c b/sys/dev/drm/drm_vm.c
index 81c00c26a7ae..e01257a37d8d 100644
--- a/sys/dev/drm/drm_vm.c
+++ b/sys/dev/drm/drm_vm.c
@@ -19,10 +19,11 @@
  * ERIC ANHOLT BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 
@@ -39,6 +40,11 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 	drm_local_map_t *map;
 	drm_file_t *priv;
 	drm_map_type_t type;
+#ifdef __FreeBSD__
+	vm_paddr_t phys;
+#else
+	paddr_t phys;
+#endif
 
 	DRM_LOCK();
 	priv = drm_find_file_by_proc(dev, DRM_CURPROC);
@@ -51,10 +57,11 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 	if (!priv->authenticated)
 		return DRM_ERR(EACCES);
 
-	DRM_SPINLOCK(&dev->dma_lock);
 	if (dev->dma && offset >= 0 && offset < ptoa(dev->dma->page_count)) {
 		drm_device_dma_t *dma = dev->dma;
 
+		DRM_SPINLOCK(&dev->dma_lock);
+
 		if (dma->pagelist != NULL) {
 			unsigned long page = offset >> PAGE_SHIFT;
 			unsigned long phys = dma->pagelist[page];
@@ -70,8 +77,8 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 			DRM_SPINUNLOCK(&dev->dma_lock);
 			return -1;
 		}
+		DRM_SPINUNLOCK(&dev->dma_lock);
 	}
-	DRM_SPINUNLOCK(&dev->dma_lock);
 
 				/* A sequential search of a linked list is
 				   fine here because: 1) there will only be
@@ -91,7 +98,7 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 		DRM_DEBUG("can't find map\n");
 		return -1;
 	}
-	if (((map->flags&_DRM_RESTRICTED) && DRM_SUSER(DRM_CURPROC))) {
+	if (((map->flags&_DRM_RESTRICTED) && !DRM_SUSER(DRM_CURPROC))) {
 		DRM_UNLOCK();
 		DRM_DEBUG("restricted map\n");
 		return -1;
@@ -103,25 +110,25 @@ paddr_t drm_mmap(dev_t kdev, off_t offset, int prot)
 	case _DRM_FRAME_BUFFER:
 	case _DRM_REGISTERS:
 	case _DRM_AGP:
-#if defined(__FreeBSD__) && __FreeBSD_version >= 500102
-		*paddr = offset;
-		return 0;
-#else
-		return atop(offset);
-#endif
+		phys = offset;
+		break;
+	case _DRM_CONSISTENT:
+		phys = vtophys((char *)map->handle + (offset - map->offset));
+		break;
 	case _DRM_SCATTER_GATHER:
 	case _DRM_SHM:
-#if defined(__FreeBSD__) && __FreeBSD_version >= 500102
-		*paddr = vtophys(offset);
-		return 0;
-#else
-		return atop(vtophys(offset));
-#endif
+		phys = vtophys(offset);
+		break;
 	default:
+		DRM_ERROR("bad map type %d\n", type);
 		return -1;	/* This should never happen. */
 	}
-	DRM_DEBUG("bailing out\n");
 
-	return -1;
+#if defined(__FreeBSD__) && __FreeBSD_version >= 500102
+	*paddr = phys;
+	return 0;
+#else
+	return atop(phys);
+#endif
 }
 
diff --git a/sys/dev/drm/i915_dma.c b/sys/dev/drm/i915_dma.c
index 035bf204b3dc..7cee9d4af0e0 100644
--- a/sys/dev/drm/i915_dma.c
+++ b/sys/dev/drm/i915_dma.c
@@ -1,18 +1,38 @@
 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
- *
- * $FreeBSD$
  */
-/**************************************************************************
- *
+/*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- *
- **************************************************************************/
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
-#include "i915_drv.h"
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/i915_drm.h"
+#include "dev/drm/i915_drv.h"
 
 /* Really want an OS-independent resettable timer.  Would like to have
  * this loop run for (eg) 3 sec, but have the timer reset every time
@@ -77,9 +97,8 @@ static int i915_dma_cleanup(drm_device_t * dev)
 			drm_core_ioremapfree(&dev_priv->ring.map, dev);
 		}
 
-		if (dev_priv->hw_status_page) {
-			drm_pci_free(dev, PAGE_SIZE, dev_priv->hw_status_page,
-				     dev_priv->dma_status_page);
+		if (dev_priv->status_page_dmah) {
+			drm_pci_free(dev, dev_priv->status_page_dmah);
 			/* Need to rewrite hardware status page */
 			I915_WRITE(0x02080, 0x1ffff000);
 		}
@@ -156,15 +175,18 @@ static int i915_initialize(drm_device_t * dev,
 	dev_priv->allow_batchbuffer = 1;
 
 	/* Program Hardware Status Page */
-	dev_priv->hw_status_page = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 
-	    0xffffffff, &dev_priv->dma_status_page);
+	dev_priv->status_page_dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 
+	    0xffffffff);
 
-	if (!dev_priv->hw_status_page) {
+	if (!dev_priv->status_page_dmah) {
 		dev->dev_private = (void *)dev_priv;
 		i915_dma_cleanup(dev);
 		DRM_ERROR("Can not allocate hardware status page\n");
 		return DRM_ERR(ENOMEM);
 	}
+	dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr;
+	dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
+	
 	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 	DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
 
@@ -690,7 +712,19 @@ static int i915_setparam(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-void i915_driver_pretakedown(drm_device_t * dev)
+int i915_driver_load(drm_device_t *dev, unsigned long flags)
+{
+	/* i915 has 4 more counters */
+	dev->counters += 4;
+	dev->types[6] = _DRM_STAT_IRQ;
+	dev->types[7] = _DRM_STAT_PRIMARY;
+	dev->types[8] = _DRM_STAT_SECONDARY;
+	dev->types[9] = _DRM_STAT_DMA;
+
+	return 0;
+}
+
+void i915_driver_lastclose(drm_device_t * dev)
 {
 	if (dev->dev_private) {
 		drm_i915_private_t *dev_priv = dev->dev_private;
@@ -699,7 +733,7 @@ void i915_driver_pretakedown(drm_device_t * dev)
 	i915_dma_cleanup(dev);
 }
 
-void i915_driver_prerelease(drm_device_t * dev, DRMFILE filp)
+void i915_driver_preclose(drm_device_t * dev, DRMFILE filp)
 {
 	if (dev->dev_private) {
 		drm_i915_private_t *dev_priv = dev->dev_private;
@@ -711,18 +745,34 @@ void i915_driver_prerelease(drm_device_t * dev, DRMFILE filp)
 }
 
 drm_ioctl_desc_t i915_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_I915_INIT)] = {i915_dma_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_I915_FLUSH)] = {i915_flush_ioctl, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_FLIP)] = {i915_flip_bufs, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_BATCHBUFFER)] = {i915_batchbuffer, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_IRQ_EMIT)] = {i915_irq_emit, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_IRQ_WAIT)] = {i915_irq_wait, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_GETPARAM)] = {i915_getparam, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_SETPARAM)] = {i915_setparam, 1, 1},
-	[DRM_IOCTL_NR(DRM_I915_ALLOC)] = {i915_mem_alloc, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_FREE)] = {i915_mem_free, 1, 0},
-	[DRM_IOCTL_NR(DRM_I915_INIT_HEAP)] = {i915_mem_init_heap, 1, 1},
-	[DRM_IOCTL_NR(DRM_I915_CMDBUFFER)] = {i915_cmdbuffer, 1, 0}
+	[DRM_IOCTL_NR(DRM_I915_INIT)] = {i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_I915_FLUSH)] = {i915_flush_ioctl, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_FLIP)] = {i915_flip_bufs, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_BATCHBUFFER)] = {i915_batchbuffer, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_IRQ_EMIT)] = {i915_irq_emit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_IRQ_WAIT)] = {i915_irq_wait, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_GETPARAM)] = {i915_getparam, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_SETPARAM)] = {i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_I915_ALLOC)] = {i915_mem_alloc, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_FREE)] = {i915_mem_free, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_I915_INIT_HEAP)] = {i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_I915_CMDBUFFER)] = {i915_cmdbuffer, DRM_AUTH}
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
+
+/**
+ * Determine if the device really is AGP or not.
+ *
+ * All Intel graphics chipsets are treated as AGP, even if they are really
+ * PCI-e.
+ *
+ * \param dev   The device to be tested.
+ *
+ * \returns
+ * A value of 1 is always retured to indictate every i9x5 is AGP.
+ */
+int i915_driver_device_is_agp(drm_device_t * dev)
+{
+	return 1;
+}
diff --git a/sys/dev/drm/i915_drm.h b/sys/dev/drm/i915_drm.h
index 8d54d9c4fadb..d11e7af15709 100644
--- a/sys/dev/drm/i915_drm.h
+++ b/sys/dev/drm/i915_drm.h
@@ -1,4 +1,31 @@
-/* $FreeBSD$ */
+/*-
+ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
 #ifndef _I915_DRM_H_
 #define _I915_DRM_H_
@@ -7,7 +34,7 @@
  * subject to backwards-compatibility constraints.
  */
 
-#include "drm.h"
+#include "dev/drm/drm.h"
 
 /* Each region is a minimum of 16k, and there are at most 255 of them.
  */
diff --git a/sys/dev/drm/i915_drv.c b/sys/dev/drm/i915_drv.c
index 126fff136946..3a632d4b8eff 100644
--- a/sys/dev/drm/i915_drv.c
+++ b/sys/dev/drm/i915_drv.c
@@ -27,47 +27,48 @@
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
-#include "i915_drv.h"
-#include "drm_pciids.h"
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/i915_drm.h"
+#include "dev/drm/i915_drv.h"
+#include "dev/drm/drm_pciids.h"
 
 /* drv_PCI_IDs comes from drm_pciids.h, generated from drm_pciids.txt. */
 static drm_pci_id_list_t i915_pciidlist[] = {
 	i915_PCI_IDS
 };
 
-extern drm_ioctl_desc_t i915_ioctls[];
-extern int i915_max_ioctl;
-
 static void i915_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = 1;	/* No dev_priv */
-	dev->prerelease = i915_driver_prerelease;
-	dev->pretakedown = i915_driver_pretakedown;
-	dev->irq_preinstall = i915_driver_irq_preinstall;
-	dev->irq_postinstall = i915_driver_irq_postinstall;
-	dev->irq_uninstall = i915_driver_irq_uninstall;
-	dev->irq_handler = i915_driver_irq_handler;
+	dev->driver.buf_priv_size	= 1;	/* No dev_priv */
+	dev->driver.load		= i915_driver_load;
+	dev->driver.preclose		= i915_driver_preclose;
+	dev->driver.lastclose		= i915_driver_lastclose;
+	dev->driver.device_is_agp	= i915_driver_device_is_agp,
+	dev->driver.irq_preinstall	= i915_driver_irq_preinstall;
+	dev->driver.irq_postinstall	= i915_driver_irq_postinstall;
+	dev->driver.irq_uninstall	= i915_driver_irq_uninstall;
+	dev->driver.irq_handler		= i915_driver_irq_handler;
 
-	dev->driver_ioctls = i915_ioctls;
-	dev->max_driver_ioctl = i915_max_ioctl;
+	dev->driver.ioctls		= i915_ioctls;
+	dev->driver.max_ioctl		= i915_max_ioctl;
 
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
 
-	dev->use_agp = 1;
-	dev->require_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_irq = 1;
+	dev->driver.use_agp		= 1;
+	dev->driver.require_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_irq		= 1;
 }
 
 #ifdef __FreeBSD__
diff --git a/sys/dev/drm/i915_drv.h b/sys/dev/drm/i915_drv.h
index df4dda38ea04..50d8437aac0a 100644
--- a/sys/dev/drm/i915_drv.h
+++ b/sys/dev/drm/i915_drv.h
@@ -1,13 +1,34 @@
 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*-
- *
- * $FreeBSD$
  */
-/**************************************************************************
- *
+/*
+ * 
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- *
- **************************************************************************/
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
 #ifndef _I915_DRV_H_
 #define _I915_DRV_H_
@@ -57,9 +78,10 @@ typedef struct drm_i915_private {
 	drm_i915_sarea_t *sarea_priv;
 	drm_i915_ring_buffer_t ring;
 
+	drm_dma_handle_t *status_page_dmah;
 	void *hw_status_page;
-	unsigned long counter;
 	dma_addr_t dma_status_page;
+	unsigned long counter;
 
 	int back_offset;
 	int front_offset;
@@ -77,10 +99,17 @@ typedef struct drm_i915_private {
 	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
 } drm_i915_private_t;
 
+extern drm_ioctl_desc_t i915_ioctls[];
+extern int i915_max_ioctl;
+
 				/* i915_dma.c */
 extern void i915_kernel_lost_context(drm_device_t * dev);
-extern void i915_driver_pretakedown(drm_device_t * dev);
-extern void i915_driver_prerelease(drm_device_t * dev, DRMFILE filp);
+extern int i915_driver_load(struct drm_device *, unsigned long flags);
+extern void i915_driver_lastclose(drm_device_t * dev);
+extern void i915_driver_preclose(drm_device_t * dev, DRMFILE filp);
+extern int i915_driver_device_is_agp(drm_device_t * dev);
+extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
+			      unsigned long arg);
 
 /* i915_irq.c */
 extern int i915_irq_emit(DRM_IOCTL_ARGS);
diff --git a/sys/dev/drm/i915_irq.c b/sys/dev/drm/i915_irq.c
index 64a5913fe74d..123ea739c1f0 100644
--- a/sys/dev/drm/i915_irq.c
+++ b/sys/dev/drm/i915_irq.c
@@ -1,18 +1,38 @@
-/* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
- *
- * $FreeBSD$
+/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
  */
-/**************************************************************************
- *
+/*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- *
- **************************************************************************/
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
-#include "i915_drv.h"
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/i915_drm.h"
+#include "dev/drm/i915_drv.h"
 
 #define USER_INT_FLAG 0x2
 #define MAX_NOPID ((u32)~0)
diff --git a/sys/dev/drm/i915_mem.c b/sys/dev/drm/i915_mem.c
index 2ef6d77e3b10..8a817ca93fca 100644
--- a/sys/dev/drm/i915_mem.c
+++ b/sys/dev/drm/i915_mem.c
@@ -1,18 +1,38 @@
 /* i915_mem.c -- Simple agp/fb memory manager for i915 -*- linux-c -*-
- *
- * $FreeBSD$
  */
-/**************************************************************************
- *
+/*-
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
  *
- **************************************************************************/
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
-#include "i915_drv.h"
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/i915_drm.h"
+#include "dev/drm/i915_drv.h"
 
 /* This memory manager is integrated into the global/local lru
  * mechanisms used by the clients.  Specifically, it operates by
diff --git a/sys/dev/drm/mach64_dma.c b/sys/dev/drm/mach64_dma.c
index f9951670d3e8..77966a125e77 100644
--- a/sys/dev/drm/mach64_dma.c
+++ b/sys/dev/drm/mach64_dma.c
@@ -9,7 +9,7 @@
  * \author Jose Fonseca <j_r_fonseca@yahoo.co.uk>
  */
 
-/*
+/*-
  * Copyright 2000 Gareth Hughes
  * Copyright 2002 Frank C. Earl
  * Copyright 2002-2003 Leif Delgass
@@ -32,10 +32,11 @@
  * THE COPYRIGHT OWNER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mach64_drm.h"
@@ -577,8 +578,7 @@ void mach64_dump_ring_info(drm_mach64_private_t * dev_priv)
 static int mach64_bm_dma_test(drm_device_t * dev)
 {
 	drm_mach64_private_t *dev_priv = dev->dev_private;
-	dma_addr_t data_handle;
-	void *cpu_addr_data;
+	drm_dma_handle_t *cpu_addr_dmah;
 	u32 data_addr;
 	u32 *table, *data;
 	u32 expected[2];
@@ -591,14 +591,14 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 
 	/* FIXME: get a dma buffer from the freelist here */
 	DRM_DEBUG("Allocating data memory ...\n");
-	cpu_addr_data =
-	    drm_pci_alloc(dev, 0x1000, 0x1000, 0xfffffffful, &data_handle);
-	if (!cpu_addr_data || !data_handle) {
+	cpu_addr_dmah =
+	    drm_pci_alloc(dev, 0x1000, 0x1000, 0xfffffffful);
+	if (!cpu_addr_dmah) {
 		DRM_INFO("data-memory allocation failed!\n");
 		return DRM_ERR(ENOMEM);
 	} else {
-		data = (u32 *) cpu_addr_data;
-		data_addr = (u32) data_handle;
+		data = (u32 *) cpu_addr_dmah->vaddr;
+		data_addr = (u32) cpu_addr_dmah->busaddr;
 	}
 
 	/* Save the X server's value for SRC_CNTL and restore it
@@ -626,7 +626,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 			DRM_INFO("resetting engine ...\n");
 			mach64_do_engine_reset(dev_priv);
 			DRM_INFO("freeing data buffer memory.\n");
-			drm_pci_free(dev, 0x1000, cpu_addr_data, data_handle);
+			drm_pci_free(dev, cpu_addr_dmah);
 			return DRM_ERR(EIO);
 		}
 	}
@@ -681,7 +681,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 		MACH64_WRITE(MACH64_PAT_REG0, pat_reg0);
 		MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
 		DRM_INFO("freeing data buffer memory.\n");
-		drm_pci_free(dev, 0x1000, cpu_addr_data, data_handle);
+		drm_pci_free(dev, cpu_addr_dmah);
 		return i;
 	}
 	DRM_DEBUG("waiting for idle...done\n");
@@ -717,7 +717,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 		MACH64_WRITE(MACH64_PAT_REG0, pat_reg0);
 		MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
 		DRM_INFO("freeing data buffer memory.\n");
-		drm_pci_free(dev, 0x1000, cpu_addr_data, data_handle);
+		drm_pci_free(dev, cpu_addr_dmah);
 		return i;
 	}
 
@@ -745,7 +745,7 @@ static int mach64_bm_dma_test(drm_device_t * dev)
 	MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
 
 	DRM_DEBUG("freeing data buffer memory.\n");
-	drm_pci_free(dev, 0x1000, cpu_addr_data, data_handle);
+	drm_pci_free(dev, cpu_addr_dmah);
 	DRM_DEBUG("returning ...\n");
 
 	return failed;
@@ -898,16 +898,17 @@ static int mach64_do_dma_init(drm_device_t * dev, drm_mach64_init_t * init)
 	dev_priv->ring.size = 0x4000;	/* 16KB */
 
 	if (dev_priv->is_pci) {
-		dev_priv->ring.start = drm_pci_alloc(dev, dev_priv->ring.size,
+		dev_priv->ring.dmah = drm_pci_alloc(dev, dev_priv->ring.size,
 						     dev_priv->ring.size,
-						     0xfffffffful,
-						     &dev_priv->ring.handle);
+						     0xfffffffful);
 
-		if (!dev_priv->ring.start || !dev_priv->ring.handle) {
+		if (!dev_priv->ring.dmah) {
 			DRM_ERROR("Allocating dma descriptor ring failed\n");
 			return DRM_ERR(ENOMEM);
 		} else {
-			dev_priv->ring.start_addr = (u32) dev_priv->ring.handle;
+			dev_priv->ring.start = dev_priv->ring.dmah->vaddr;
+			dev_priv->ring.start_addr =
+			    (u32) dev_priv->ring.dmah->busaddr;
 		}
 	} else {
 		dev_priv->ring.start = dev_priv->ring_map->handle;
@@ -1151,11 +1152,8 @@ int mach64_do_cleanup_dma(drm_device_t * dev)
 		drm_mach64_private_t *dev_priv = dev->dev_private;
 
 		if (dev_priv->is_pci) {
-			if ((dev_priv->ring.start != NULL)
-			    && dev_priv->ring.handle) {
-				drm_pci_free(dev, dev_priv->ring.size,
-					     dev_priv->ring.start,
-					     dev_priv->ring.handle);
+			if (dev_priv->ring.dmah) {
+				drm_pci_free(dev, dev_priv->ring.dmah);
 			}
 		} else {
 			if (dev_priv->ring_map)
@@ -1524,7 +1522,7 @@ int mach64_dma_buffers(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-void mach64_driver_pretakedown(drm_device_t * dev)
+void mach64_driver_lastclose(drm_device_t * dev)
 {
 	mach64_do_cleanup_dma(dev);
 }
diff --git a/sys/dev/drm/mach64_drm.h b/sys/dev/drm/mach64_drm.h
index 43d0f7f3b47b..7321d0dd305d 100644
--- a/sys/dev/drm/mach64_drm.h
+++ b/sys/dev/drm/mach64_drm.h
@@ -1,6 +1,7 @@
 /* mach64_drm.h -- Public header for the mach64 driver -*- linux-c -*-
  * Created: Thu Nov 30 20:04:32 2000 by gareth@valinux.com
- *
+ */
+/*-
  * Copyright 2000 Gareth Hughes
  * Copyright 2002 Frank C. Earl
  * Copyright 2002-2003 Leif Delgass
@@ -28,10 +29,11 @@
  *    Gareth Hughes <gareth@valinux.com>
  *    Frank C. Earl <fearl@airmail.net>
  *    Leif Delgass <ldelgass@retinalburn.net>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __MACH64_DRM_H__
 #define __MACH64_DRM_H__
 
diff --git a/sys/dev/drm/mach64_drv.c b/sys/dev/drm/mach64_drv.c
index 9c5b483424a8..36fe0aed6e32 100644
--- a/sys/dev/drm/mach64_drv.c
+++ b/sys/dev/drm/mach64_drv.c
@@ -28,10 +28,11 @@
  * Authors:
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 
 #include <sys/types.h>
 
@@ -46,36 +47,33 @@ static drm_pci_id_list_t mach64_pciidlist[] = {
 	mach64_PCI_IDS
 };
 
-extern drm_ioctl_desc_t mach64_ioctls[];
-extern int mach64_max_ioctl;
-
 static void mach64_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = 1; /* No dev_priv */
-	dev->pretakedown = mach64_driver_pretakedown;
-	dev->vblank_wait = mach64_driver_vblank_wait;
-	dev->irq_preinstall = mach64_driver_irq_preinstall;
-	dev->irq_postinstall = mach64_driver_irq_postinstall;
-	dev->irq_uninstall = mach64_driver_irq_uninstall;
-	dev->irq_handler = mach64_driver_irq_handler;
-	dev->dma_ioctl = mach64_dma_buffers;
-
-	dev->driver_ioctls = mach64_ioctls;
-	dev->max_driver_ioctl = mach64_max_ioctl;
-
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
-
-	dev->use_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_pci_dma = 1;
-	dev->use_dma = 1;
-	dev->use_irq = 1;
-	dev->use_vbl_irq = 1;
+	dev->driver.buf_priv_size	= 1; /* No dev_priv */
+	dev->driver.lastclose		= mach64_driver_lastclose;
+	dev->driver.vblank_wait		= mach64_driver_vblank_wait;
+	dev->driver.irq_preinstall	= mach64_driver_irq_preinstall;
+	dev->driver.irq_postinstall	= mach64_driver_irq_postinstall;
+	dev->driver.irq_uninstall	= mach64_driver_irq_uninstall;
+	dev->driver.irq_handler		= mach64_driver_irq_handler;
+	dev->driver.dma_ioctl		= mach64_dma_buffers;
+
+	dev->driver.ioctls		= mach64_ioctls;
+	dev->driver.max_ioctl		= mach64_max_ioctl;
+
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
+
+	dev->driver.use_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_pci_dma		= 1;
+	dev->driver.use_dma		= 1;
+	dev->driver.use_irq		= 1;
+	dev->driver.use_vbl_irq		= 1;
 }
 
 #ifdef __FreeBSD__
diff --git a/sys/dev/drm/mach64_drv.h b/sys/dev/drm/mach64_drv.h
index 9c33af20a530..24e524ea9db8 100644
--- a/sys/dev/drm/mach64_drv.h
+++ b/sys/dev/drm/mach64_drv.h
@@ -1,6 +1,7 @@
 /* mach64_drv.h -- Private header for mach64 driver -*- linux-c -*-
  * Created: Fri Nov 24 22:07:58 2000 by gareth@valinux.com
- *
+ */
+/*-
  * Copyright 2000 Gareth Hughes
  * Copyright 2002 Frank C. Earl
  * Copyright 2002-2003 Leif Delgass
@@ -29,10 +30,11 @@
  *    Frank C. Earl <fearl@airmail.net>
  *    Leif Delgass <ldelgass@retinalburn.net>
  *    Jos�Fonseca <j_r_fonseca@yahoo.co.uk>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __MACH64_DRV_H__
 #define __MACH64_DRV_H__
 
@@ -62,7 +64,7 @@ typedef struct drm_mach64_freelist {
 } drm_mach64_freelist_t;
 
 typedef struct drm_mach64_descriptor_ring {
-	dma_addr_t handle;	/* handle (bus address) of ring returned by pci_alloc_consistent() */
+	drm_dma_handle_t *dmah;	/* Handle to pci dma memory */
 	void *start;		/* write pointer (cpu address) to start of descriptor ring */
 	u32 start_addr;		/* bus address of beginning of descriptor ring */
 	int size;		/* size of ring in bytes */
@@ -110,13 +112,16 @@ typedef struct drm_mach64_private {
 	drm_local_map_t *agp_textures;
 } drm_mach64_private_t;
 
+extern drm_ioctl_desc_t mach64_ioctls[];
+extern int mach64_max_ioctl;
+
 				/* mach64_dma.c */
 extern int mach64_dma_init(DRM_IOCTL_ARGS);
 extern int mach64_dma_idle(DRM_IOCTL_ARGS);
 extern int mach64_dma_flush(DRM_IOCTL_ARGS);
 extern int mach64_engine_reset(DRM_IOCTL_ARGS);
 extern int mach64_dma_buffers(DRM_IOCTL_ARGS);
-extern void mach64_driver_pretakedown(drm_device_t * dev);
+extern void mach64_driver_lastclose(drm_device_t * dev);
 
 extern int mach64_init_freelist(drm_device_t * dev);
 extern void mach64_destroy_freelist(drm_device_t * dev);
diff --git a/sys/dev/drm/mach64_irq.c b/sys/dev/drm/mach64_irq.c
index 912372978d4b..c6563ce9f5e1 100644
--- a/sys/dev/drm/mach64_irq.c
+++ b/sys/dev/drm/mach64_irq.c
@@ -1,6 +1,7 @@
 /* mach64_irq.c -- IRQ handling for ATI Mach64 -*- linux-c -*-
  * Created: Tue Feb 25, 2003 by Leif Delgass, based on radeon_irq.c/r128_irq.c
- *
+ */
+/*-
  * Copyright (C) The Weather Channel, Inc.  2002.
  * Copyright 2003 Leif Delgass
  * All Rights Reserved.
@@ -32,10 +33,11 @@
  *    Keith Whitwell <keith@tungstengraphics.com>
  *    Eric Anholt <anholt@FreeBSD.org>
  *    Leif Delgass <ldelgass@retinalburn.net>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mach64_drm.h"
diff --git a/sys/dev/drm/mach64_state.c b/sys/dev/drm/mach64_state.c
index 81c5481388d3..9d5ab63b2258 100644
--- a/sys/dev/drm/mach64_state.c
+++ b/sys/dev/drm/mach64_state.c
@@ -1,6 +1,7 @@
 /* mach64_state.c -- State support for mach64 (Rage Pro) driver -*- linux-c -*-
  * Created: Sun Dec 03 19:20:26 2000 by gareth@valinux.com
- *
+ */
+/*-
  * Copyright 2000 Gareth Hughes
  * Copyright 2002-2003 Leif Delgass
  * All Rights Reserved.
@@ -27,10 +28,11 @@
  *    Gareth Hughes <gareth@valinux.com>
  *    Leif Delgass <ldelgass@retinalburn.net>
  *    Jos�Fonseca <j_r_fonseca@yahoo.co.uk>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mach64_drm.h"
@@ -42,15 +44,15 @@
  *
  */
 drm_ioctl_desc_t mach64_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_MACH64_INIT)] = {mach64_dma_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_MACH64_CLEAR)] = {mach64_dma_clear, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_SWAP)] = {mach64_dma_swap, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_IDLE)] = {mach64_dma_idle, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_RESET)] = {mach64_engine_reset, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_VERTEX)] = {mach64_dma_vertex, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_BLIT)] = {mach64_dma_blit, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_FLUSH)] = {mach64_dma_flush, 1, 0},
-	[DRM_IOCTL_NR(DRM_MACH64_GETPARAM)] = {mach64_get_param, 1, 0},
+	[DRM_IOCTL_NR(DRM_MACH64_INIT)] = {mach64_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_MACH64_CLEAR)] = {mach64_dma_clear, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_SWAP)] = {mach64_dma_swap, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_IDLE)] = {mach64_dma_idle, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_RESET)] = {mach64_engine_reset, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_VERTEX)] = {mach64_dma_vertex, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_BLIT)] = {mach64_dma_blit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_FLUSH)] = {mach64_dma_flush, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MACH64_GETPARAM)] = {mach64_get_param, DRM_AUTH},
 };
 
 int mach64_max_ioctl = DRM_ARRAY_SIZE(mach64_ioctls);
@@ -481,25 +483,29 @@ static int mach64_do_get_frames_queued(drm_mach64_private_t * dev_priv)
 /* Copy and verify a client submited buffer.
  * FIXME: Make an assembly optimized version
  */
-static __inline__ int copy_and_verify_from_user(u32 * to, const u32 * from,
+static __inline__ int copy_and_verify_from_user(u32 *to,
+						const u32 __user *ufrom,
 						unsigned long bytes)
 {
 	unsigned long n = bytes;	/* dwords remaining in buffer */
+	u32 *from, *orig_from;
+
+	from = drm_alloc(bytes, DRM_MEM_DRIVER);
+	if (from == NULL)
+		return ENOMEM;
 
-	if (DRM_VERIFYAREA_READ(from, n)) {
-		DRM_ERROR("%s: verify_area\n", __FUNCTION__);
+	if (DRM_COPY_FROM_USER(from, ufrom, bytes)) {
+		drm_free(from, bytes, DRM_MEM_DRIVER);
 		return DRM_ERR(EFAULT);
 	}
+	orig_from = from; /* we'll be modifying the "from" ptr, so save it */
 
 	n >>= 2;
 
 	while (n > 1) {
 		u32 data, reg, count;
 
-		if (DRM_GET_USER_UNCHECKED(data, from++)) {
-			DRM_ERROR("%s: get_user\n", __FUNCTION__);
-			return DRM_ERR(EFAULT);
-		}
+		data = *from++;
 
 		n--;
 
@@ -515,28 +521,25 @@ static __inline__ int copy_and_verify_from_user(u32 * to, const u32 * from,
 			if ((reg >= 0x0190 && reg < 0x01c1) ||
 			    (reg >= 0x01ca && reg <= 0x01cf)) {
 				*to++ = data;
-				if (DRM_COPY_FROM_USER_UNCHECKED
-				    (to, from, count << 2)) {
-					DRM_ERROR("%s: copy_from_user\n",
-						  __FUNCTION__);
-					return DRM_ERR(EFAULT);
-				}
+				memcpy(to, from, count << 2);
+				from += count;
 				to += count;
 			} else {
 				DRM_ERROR("%s: Got bad command: 0x%04x\n",
 					  __FUNCTION__, reg);
+				drm_free(orig_from, bytes, DRM_MEM_DRIVER);
 				return DRM_ERR(EACCES);
 			}
-
-			from += count;
 		} else {
 			DRM_ERROR
 			    ("%s: Got bad command count(=%u) dwords remaining=%lu\n",
 			     __FUNCTION__, count, n);
+			drm_free(orig_from, bytes, DRM_MEM_DRIVER);
 			return DRM_ERR(EINVAL);
 		}
 	}
 
+	drm_free(orig_from, bytes, DRM_MEM_DRIVER);
 	if (n == 0)
 		return 0;
 	else {
diff --git a/sys/dev/drm/mga_dma.c b/sys/dev/drm/mga_dma.c
index 9feac053bb1a..093dd25868fc 100644
--- a/sys/dev/drm/mga_dma.c
+++ b/sys/dev/drm/mga_dma.c
@@ -1,7 +1,7 @@
 /* mga_dma.c -- DMA support for mga g200/g400 -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com */
-/*-
- * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
+ * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
+ */
+/* Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
  *
@@ -23,27 +23,33 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Rickard E. (Rik) Faith <faith@valinux.com>
- *    Jeff Hartmann <jhartmann@valinux.com>
- *    Keith Whitwell <keith@tungstengraphics.com>
- *
- * Rewritten by:
- *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/**
+ * \file mga_dma.c
+ * DMA support for MGA G200 / G400.
+ * 
+ * \author Rickard E. (Rik) Faith <faith@valinux.com>
+ * \author Jeff Hartmann <jhartmann@valinux.com>
+ * \author Keith Whitwell <keith@tungstengraphics.com>
+ * \author Gareth Hughes <gareth@valinux.com>
  */
 
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
+#include "dev/drm/drm_sarea.h"
 #include "dev/drm/mga_drm.h"
 #include "dev/drm/mga_drv.h"
 
 #define MGA_DEFAULT_USEC_TIMEOUT	10000
 #define MGA_FREELIST_DEBUG		0
 
-static int mga_do_cleanup_dma(drm_device_t * dev);
+#define MINIMAL_CLEANUP    0
+#define FULL_CLEANUP       1
+static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup);
 
 /* ================================================================
  * Engine control
@@ -150,7 +156,7 @@ void mga_do_dma_flush(drm_mga_private_t * dev_priv)
 	DRM_DEBUG("  space = 0x%06x\n", primary->space);
 
 	mga_flush_write_combine();
-	MGA_WRITE(MGA_PRIMEND, tail | MGA_PAGPXFER);
+	MGA_WRITE(MGA_PRIMEND, tail | dev_priv->dma_access);
 
 	DRM_DEBUG("done.\n");
 }
@@ -190,7 +196,7 @@ void mga_do_dma_wrap_start(drm_mga_private_t * dev_priv)
 	DRM_DEBUG("  space = 0x%06x\n", primary->space);
 
 	mga_flush_write_combine();
-	MGA_WRITE(MGA_PRIMEND, tail | MGA_PAGPXFER);
+	MGA_WRITE(MGA_PRIMEND, tail | dev_priv->dma_access);
 
 	set_bit(0, &primary->wrapped);
 	DRM_DEBUG("done.\n");
@@ -390,21 +396,420 @@ int mga_freelist_put(drm_device_t * dev, drm_buf_t * buf)
  * DMA initialization, cleanup
  */
 
-static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
+int mga_driver_load(drm_device_t *dev, unsigned long flags)
 {
-	drm_mga_private_t *dev_priv;
-	int ret;
-	DRM_DEBUG("\n");
+	drm_mga_private_t * dev_priv;
 
 	dev_priv = drm_alloc(sizeof(drm_mga_private_t), DRM_MEM_DRIVER);
 	if (!dev_priv)
 		return DRM_ERR(ENOMEM);
 
+	dev->dev_private = (void *)dev_priv;
 	memset(dev_priv, 0, sizeof(drm_mga_private_t));
 
-	dev_priv->chipset = init->chipset;
-
 	dev_priv->usec_timeout = MGA_DEFAULT_USEC_TIMEOUT;
+	dev_priv->chipset = flags;
+
+	dev_priv->mmio_base = drm_get_resource_start(dev, 1);
+	dev_priv->mmio_size = drm_get_resource_len(dev, 1);
+
+	dev->counters += 3;
+	dev->types[6] = _DRM_STAT_IRQ;
+	dev->types[7] = _DRM_STAT_PRIMARY;
+	dev->types[8] = _DRM_STAT_SECONDARY;
+
+	return 0;
+}
+
+/**
+ * Bootstrap the driver for AGP DMA.
+ * 
+ * \todo
+ * Investigate whether there is any benifit to storing the WARP microcode in
+ * AGP memory.  If not, the microcode may as well always be put in PCI
+ * memory.
+ *
+ * \todo
+ * This routine needs to set dma_bs->agp_mode to the mode actually configured
+ * in the hardware.  Looking just at the Linux AGP driver code, I don't see
+ * an easy way to determine this.
+ *
+ * \sa mga_do_dma_bootstrap, mga_do_pci_dma_bootstrap
+ */
+static int mga_do_agp_dma_bootstrap(drm_device_t * dev,
+				    drm_mga_dma_bootstrap_t * dma_bs)
+{
+	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	unsigned int warp_size = mga_warp_microcode_size(dev_priv);
+	int err;
+	unsigned  offset;
+	const unsigned secondary_size = dma_bs->secondary_bin_count
+		* dma_bs->secondary_bin_size;
+	const unsigned agp_size = (dma_bs->agp_size << 20);
+	drm_buf_desc_t req;
+	drm_agp_mode_t mode;
+	drm_agp_info_t info;
+	drm_agp_buffer_t agp_req;
+	drm_agp_binding_t bind_req;
+
+	/* Acquire AGP. */
+	err = drm_agp_acquire(dev);
+	if (err) {
+		DRM_ERROR("Unable to acquire AGP: %d\n", err);
+		return err;
+	}
+
+	err = drm_agp_info(dev, &info);
+	if (err) {
+		DRM_ERROR("Unable to get AGP info: %d\n", err);
+		return err;
+	}
+
+	mode.mode = (info.mode & ~0x07) | dma_bs->agp_mode;
+	err = drm_agp_enable(dev, mode);
+	if (err) {
+		DRM_ERROR("Unable to enable AGP (mode = 0x%lx)\n", mode.mode);
+		return err;
+	}
+
+	/* In addition to the usual AGP mode configuration, the G200 AGP cards
+	 * need to have the AGP mode "manually" set.
+	 */
+
+	if (dev_priv->chipset == MGA_CARD_TYPE_G200) {
+		if (mode.mode & 0x02) {
+			MGA_WRITE(MGA_AGP_PLL, MGA_AGP2XPLL_ENABLE);
+		} else {
+			MGA_WRITE(MGA_AGP_PLL, MGA_AGP2XPLL_DISABLE);
+		}
+	}
+
+
+	/* Allocate and bind AGP memory. */
+	agp_req.size = agp_size;
+	agp_req.type = 0;
+	err = drm_agp_alloc( dev, & agp_req );
+	if (err) {
+		dev_priv->agp_size = 0;
+		DRM_ERROR("Unable to allocate %uMB AGP memory\n",
+			  dma_bs->agp_size);
+		return err;
+	}
+
+	dev_priv->agp_size = agp_size;
+	dev_priv->agp_handle = agp_req.handle;
+
+	bind_req.handle = agp_req.handle;
+	bind_req.offset = 0;
+	err = drm_agp_bind( dev, &bind_req );
+	if (err) {
+		DRM_ERROR("Unable to bind AGP memory: %d\n", err);
+		return err;
+	}
+
+	/* Make drm_addbufs happy by not trying to create a mapping for less
+	 * than a page.
+	 */
+	if (warp_size < PAGE_SIZE)
+		warp_size = PAGE_SIZE;
+
+	offset = 0;
+	err = drm_addmap( dev, offset, warp_size,
+			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->warp );
+	if (err) {
+		DRM_ERROR("Unable to map WARP microcode: %d\n", err);
+		return err;
+	}
+
+	offset += warp_size;
+	err = drm_addmap( dev, offset, dma_bs->primary_size,
+			  _DRM_AGP, _DRM_READ_ONLY, & dev_priv->primary );
+	if (err) {
+		DRM_ERROR("Unable to map primary DMA region: %d\n", err);
+		return err;
+	}
+
+	offset += dma_bs->primary_size;
+	err = drm_addmap( dev, offset, secondary_size,
+			  _DRM_AGP, 0, & dev->agp_buffer_map );
+	if (err) {
+		DRM_ERROR("Unable to map secondary DMA region: %d\n", err);
+		return err;
+	}
+
+	(void) memset( &req, 0, sizeof(req) );
+	req.count = dma_bs->secondary_bin_count;
+	req.size = dma_bs->secondary_bin_size;
+	req.flags = _DRM_AGP_BUFFER;
+	req.agp_start = offset;
+
+	err = drm_addbufs_agp( dev, & req );
+	if (err) {
+		DRM_ERROR("Unable to add secondary DMA buffers: %d\n", err);
+		return err;
+	}
+
+#ifdef __linux__
+	{
+		drm_map_list_t *_entry;
+		unsigned long agp_token = 0;
+
+		list_for_each_entry(_entry, &dev->maplist->head, head) {
+			if (_entry->map == dev->agp_buffer_map)
+				agp_token = _entry->user_token;
+		}
+		if (!agp_token)
+			return -EFAULT;
+
+		dev->agp_buffer_token = agp_token;
+	}
+#endif
+
+	offset += secondary_size;
+	err = drm_addmap( dev, offset, agp_size - offset,
+			  _DRM_AGP, 0, & dev_priv->agp_textures );
+	if (err) {
+		DRM_ERROR("Unable to map AGP texture region: %d\n", err);
+		return err;
+	}
+
+	drm_core_ioremap(dev_priv->warp, dev);
+	drm_core_ioremap(dev_priv->primary, dev);
+	drm_core_ioremap(dev->agp_buffer_map, dev);
+
+	if (!dev_priv->warp->handle ||
+	    !dev_priv->primary->handle || !dev->agp_buffer_map->handle) {
+		DRM_ERROR("failed to ioremap agp regions! (%p, %p, %p)\n",
+			  dev_priv->warp->handle, dev_priv->primary->handle,
+			  dev->agp_buffer_map->handle);
+		return DRM_ERR(ENOMEM);
+	}
+
+	dev_priv->dma_access = MGA_PAGPXFER;
+	dev_priv->wagp_enable = MGA_WAGP_ENABLE;
+
+	DRM_INFO("Initialized card for AGP DMA.\n");
+	return 0;
+}
+
+/**
+ * Bootstrap the driver for PCI DMA.
+ * 
+ * \todo
+ * The algorithm for decreasing the size of the primary DMA buffer could be
+ * better.  The size should be rounded up to the nearest page size, then
+ * decrease the request size by a single page each pass through the loop.
+ *
+ * \todo
+ * Determine whether the maximum address passed to drm_pci_alloc is correct.
+ * The same goes for drm_addbufs_pci.
+ * 
+ * \sa mga_do_dma_bootstrap, mga_do_agp_dma_bootstrap
+ */
+static int mga_do_pci_dma_bootstrap(drm_device_t * dev,
+				    drm_mga_dma_bootstrap_t * dma_bs)
+{
+	drm_mga_private_t * const dev_priv = (drm_mga_private_t *) dev->dev_private;
+	unsigned int warp_size = mga_warp_microcode_size(dev_priv);
+	unsigned int primary_size;
+	unsigned int bin_count;
+	int err;
+	drm_buf_desc_t req;
+
+	
+	if (dev->dma == NULL) {
+		DRM_ERROR("dev->dma is NULL\n");
+		return DRM_ERR(EFAULT);
+	}
+
+	/* Make drm_addbufs happy by not trying to create a mapping for less
+	 * than a page.
+	 */
+	if (warp_size < PAGE_SIZE)
+		warp_size = PAGE_SIZE;
+
+	/* The proper alignment is 0x100 for this mapping */
+	err = drm_addmap(dev, 0, warp_size, _DRM_CONSISTENT,
+			 _DRM_READ_ONLY, &dev_priv->warp);
+	if (err != 0) {
+		DRM_ERROR("Unable to create mapping for WARP microcode: %d\n",
+			  err);
+		return err;
+	}
+
+	/* Other than the bottom two bits being used to encode other
+	 * information, there don't appear to be any restrictions on the
+	 * alignment of the primary or secondary DMA buffers.
+	 */
+
+	for ( primary_size = dma_bs->primary_size
+	      ; primary_size != 0
+	      ; primary_size >>= 1 ) {
+		/* The proper alignment for this mapping is 0x04 */
+		err = drm_addmap(dev, 0, primary_size, _DRM_CONSISTENT,
+				 _DRM_READ_ONLY, &dev_priv->primary);
+		if (!err)
+			break;
+	}
+
+	if (err != 0) {
+		DRM_ERROR("Unable to allocate primary DMA region: %d\n", err);
+		return DRM_ERR(ENOMEM);
+	}
+
+	if (dev_priv->primary->size != dma_bs->primary_size) {
+		DRM_INFO("Primary DMA buffer size reduced from %u to %u.\n",
+			 dma_bs->primary_size, 
+			 (unsigned) dev_priv->primary->size);
+		dma_bs->primary_size = dev_priv->primary->size;
+	}
+
+	for ( bin_count = dma_bs->secondary_bin_count
+	      ; bin_count > 0 
+	      ; bin_count-- ) {
+		(void) memset( &req, 0, sizeof(req) );
+		req.count = bin_count;
+		req.size = dma_bs->secondary_bin_size;
+
+		err = drm_addbufs_pci( dev, & req );
+		if (!err) {
+			break;
+		}
+	}
+	
+	if (bin_count == 0) {
+		DRM_ERROR("Unable to add secondary DMA buffers: %d\n", err);
+		return err;
+	}
+
+	if (bin_count != dma_bs->secondary_bin_count) {
+		DRM_INFO("Secondary PCI DMA buffer bin count reduced from %u "
+			 "to %u.\n", dma_bs->secondary_bin_count, bin_count);
+
+		dma_bs->secondary_bin_count = bin_count;
+	}
+
+	dev_priv->dma_access = 0;
+	dev_priv->wagp_enable = 0;
+
+	dma_bs->agp_mode = 0;
+
+	DRM_INFO("Initialized card for PCI DMA.\n");
+	return 0;
+}
+
+
+static int mga_do_dma_bootstrap(drm_device_t * dev,
+				drm_mga_dma_bootstrap_t * dma_bs)
+{
+	const int is_agp = (dma_bs->agp_mode != 0) && drm_device_is_agp(dev);
+	int err;
+	drm_mga_private_t * const dev_priv =
+		(drm_mga_private_t *) dev->dev_private;
+
+
+	dev_priv->used_new_dma_init = 1;
+
+	/* The first steps are the same for both PCI and AGP based DMA.  Map
+	 * the cards MMIO registers and map a status page.
+	 */
+	err = drm_addmap( dev, dev_priv->mmio_base, dev_priv->mmio_size,
+			  _DRM_REGISTERS, _DRM_READ_ONLY, & dev_priv->mmio );
+	if (err) {
+		DRM_ERROR("Unable to map MMIO region: %d\n", err);
+		return err;
+	}
+
+
+	err = drm_addmap( dev, 0, SAREA_MAX, _DRM_SHM,
+			  _DRM_READ_ONLY | _DRM_LOCKED | _DRM_KERNEL,
+			  & dev_priv->status );
+	if (err) {
+		DRM_ERROR("Unable to map status region: %d\n", err);
+		return err;
+	}
+
+
+	/* The DMA initialization procedure is slightly different for PCI and
+	 * AGP cards.  AGP cards just allocate a large block of AGP memory and
+	 * carve off portions of it for internal uses.  The remaining memory
+	 * is returned to user-mode to be used for AGP textures.
+	 */
+
+	if (is_agp) {
+		err = mga_do_agp_dma_bootstrap(dev, dma_bs);
+	}
+	
+	/* If we attempted to initialize the card for AGP DMA but failed,
+	 * clean-up any mess that may have been created.
+	 */
+
+	if (err) {
+		mga_do_cleanup_dma(dev, MINIMAL_CLEANUP);
+	}
+
+
+	/* Not only do we want to try and initialized PCI cards for PCI DMA,
+	 * but we also try to initialized AGP cards that could not be
+	 * initialized for AGP DMA.  This covers the case where we have an AGP
+	 * card in a system with an unsupported AGP chipset.  In that case the
+	 * card will be detected as AGP, but we won't be able to allocate any
+	 * AGP memory, etc.
+	 */
+
+	if (!is_agp || err) {
+		err = mga_do_pci_dma_bootstrap(dev, dma_bs);
+	}
+
+
+	return err;
+}
+
+int mga_dma_bootstrap(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_dma_bootstrap_t bootstrap;
+	int err;
+	static const int modes[] = { 0, 1, 2, 2, 4, 4, 4, 4 };
+	const drm_mga_private_t * const dev_priv = 
+		(drm_mga_private_t *) dev->dev_private;
+
+
+	DRM_COPY_FROM_USER_IOCTL(bootstrap,
+				 (drm_mga_dma_bootstrap_t __user *) data,
+				 sizeof(bootstrap));
+
+	err = mga_do_dma_bootstrap(dev, & bootstrap);
+	if (err) {
+		mga_do_cleanup_dma(dev, FULL_CLEANUP);
+		return err;
+	}
+
+	if (dev_priv->agp_textures != NULL) {
+		bootstrap.texture_handle = dev_priv->agp_textures->offset;
+		bootstrap.texture_size = dev_priv->agp_textures->size;
+	} else {
+		bootstrap.texture_handle = 0;
+		bootstrap.texture_size = 0;
+	}
+
+	bootstrap.agp_mode = modes[bootstrap.agp_mode & 0x07];
+
+	DRM_COPY_TO_USER_IOCTL((drm_mga_dma_bootstrap_t __user *)data,
+			       bootstrap, sizeof(bootstrap));
+
+	return 0;
+}
+
+
+static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
+{
+	drm_mga_private_t *dev_priv;
+	int ret;
+	DRM_DEBUG("\n");
+
+
+	dev_priv = dev->dev_private;
 
 	if (init->sgram) {
 		dev_priv->clear_cmd = MGA_DWGCTL_CLEAR | MGA_ATYPE_BLK;
@@ -432,85 +837,68 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 
 	if (!dev_priv->sarea) {
 		DRM_ERROR("failed to find sarea!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
 		return DRM_ERR(EINVAL);
 	}
 
-	dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
-	if (!dev_priv->mmio) {
-		DRM_ERROR("failed to find mmio region!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->status = drm_core_findmap(dev, init->status_offset);
-	if (!dev_priv->status) {
-		DRM_ERROR("failed to find status page!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->warp = drm_core_findmap(dev, init->warp_offset);
-	if (!dev_priv->warp) {
-		DRM_ERROR("failed to find warp microcode region!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
-	}
-	dev_priv->primary = drm_core_findmap(dev, init->primary_offset);
-	if (!dev_priv->primary) {
-		DRM_ERROR("failed to find primary dma region!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
-	}
-	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
-	if (!dev->agp_buffer_map) {
-		DRM_ERROR("failed to find dma buffer region!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
-		return DRM_ERR(EINVAL);
+	if (! dev_priv->used_new_dma_init) {
+
+		dev_priv->dma_access = MGA_PAGPXFER;
+		dev_priv->wagp_enable = MGA_WAGP_ENABLE;
+
+		dev_priv->status = drm_core_findmap(dev, init->status_offset);
+		if (!dev_priv->status) {
+			DRM_ERROR("failed to find status page!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
+		if (!dev_priv->mmio) {
+			DRM_ERROR("failed to find mmio region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->warp = drm_core_findmap(dev, init->warp_offset);
+		if (!dev_priv->warp) {
+			DRM_ERROR("failed to find warp microcode region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev_priv->primary = drm_core_findmap(dev, init->primary_offset);
+		if (!dev_priv->primary) {
+			DRM_ERROR("failed to find primary dma region!\n");
+			return DRM_ERR(EINVAL);
+		}
+		dev->agp_buffer_token = init->buffers_offset;
+		dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
+		if (!dev->agp_buffer_map) {
+			DRM_ERROR("failed to find dma buffer region!\n");
+			return DRM_ERR(EINVAL);
+		}
+
+		drm_core_ioremap(dev_priv->warp, dev);
+		drm_core_ioremap(dev_priv->primary, dev);
+		drm_core_ioremap(dev->agp_buffer_map, dev);
 	}
 
 	dev_priv->sarea_priv =
 	    (drm_mga_sarea_t *) ((u8 *) dev_priv->sarea->handle +
 				 init->sarea_priv_offset);
 
-	drm_core_ioremap(dev_priv->warp, dev);
-	drm_core_ioremap(dev_priv->primary, dev);
-	drm_core_ioremap(dev->agp_buffer_map, dev);
-
 	if (!dev_priv->warp->handle ||
-	    !dev_priv->primary->handle || !dev->agp_buffer_map->handle) {
+	    !dev_priv->primary->handle ||
+	    ((dev_priv->dma_access != 0) &&
+	     ((dev->agp_buffer_map == NULL) ||
+	      (dev->agp_buffer_map->handle == NULL)))) {
 		DRM_ERROR("failed to ioremap agp regions!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
 		return DRM_ERR(ENOMEM);
 	}
 
 	ret = mga_warp_install_microcode(dev_priv);
-	if (ret < 0) {
-		DRM_ERROR("failed to install WARP ucode!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
+	if (ret != 0) {
+		DRM_ERROR("failed to install WARP ucode: %d!\n", ret);
 		return ret;
 	}
 
 	ret = mga_warp_init(dev_priv);
-	if (ret < 0) {
-		DRM_ERROR("failed to init WARP engine!\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
+	if (ret != 0) {
+		DRM_ERROR("failed to init WARP engine: %d!\n", ret);
 		return ret;
 	}
 
@@ -549,19 +937,15 @@ static int mga_do_init_dma(drm_device_t * dev, drm_mga_init_t * init)
 
 	if (mga_freelist_init(dev, dev_priv) < 0) {
 		DRM_ERROR("could not initialize freelist\n");
-		/* Assign dev_private so we can do cleanup. */
-		dev->dev_private = (void *)dev_priv;
-		mga_do_cleanup_dma(dev);
 		return DRM_ERR(ENOMEM);
 	}
 
-	/* Make dev_private visable to others. */
-	dev->dev_private = (void *)dev_priv;
 	return 0;
 }
 
-static int mga_do_cleanup_dma(drm_device_t * dev)
+static int mga_do_cleanup_dma(drm_device_t * dev, int full_cleanup)
 {
+	int err = 0;
 	DRM_DEBUG("\n");
 
 	/* Make sure interrupts are disabled here because the uninstall ioctl
@@ -574,22 +958,57 @@ static int mga_do_cleanup_dma(drm_device_t * dev)
 	if (dev->dev_private) {
 		drm_mga_private_t *dev_priv = dev->dev_private;
 
-		if (dev_priv->warp != NULL)
+		if ((dev_priv->warp != NULL)
+		    && (dev_priv->warp->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->warp, dev);
-		if (dev_priv->primary != NULL)
+
+		if ((dev_priv->primary != NULL) 
+		    && (dev_priv->primary->type != _DRM_CONSISTENT))
 			drm_core_ioremapfree(dev_priv->primary, dev);
-		if (dev->agp_buffer_map != NULL) {
+
+		if (dev->agp_buffer_map != NULL)
 			drm_core_ioremapfree(dev->agp_buffer_map, dev);
-			dev->agp_buffer_map = NULL;
+
+		if (dev_priv->used_new_dma_init) {
+			if (dev_priv->agp_handle != 0) {
+				drm_agp_binding_t unbind_req;
+				drm_agp_buffer_t free_req;
+
+				unbind_req.handle = dev_priv->agp_handle;
+				drm_agp_unbind(dev, &unbind_req);
+
+				free_req.handle = dev_priv->agp_handle;
+				drm_agp_free(dev, &free_req);
+
+				dev_priv->agp_textures = NULL;
+				dev_priv->agp_size = 0;
+				dev_priv->agp_handle = 0;
+			}
+
+			if ((dev->agp != NULL) && dev->agp->acquired) {
+				err = drm_agp_release(dev);
+			}
+		}
+
+		dev_priv->warp = NULL;
+		dev_priv->primary = NULL;
+		dev_priv->sarea = NULL;
+		dev_priv->sarea_priv = NULL;
+		dev->agp_buffer_map = NULL;
+
+		if (full_cleanup) {
+			dev_priv->mmio = NULL;
+			dev_priv->status = NULL;
+			dev_priv->used_new_dma_init = 0;
 		}
 
+		memset(&dev_priv->prim, 0, sizeof(dev_priv->prim));
+		dev_priv->warp_pipe = 0;
+		memset(dev_priv->warp_pipe_phys, 0, sizeof(dev_priv->warp_pipe_phys));
+
 		if (dev_priv->head != NULL) {
 			mga_freelist_cleanup(dev);
 		}
-
-		drm_free(dev->dev_private, sizeof(drm_mga_private_t),
-			 DRM_MEM_DRIVER);
-		dev->dev_private = NULL;
 	}
 
 	return 0;
@@ -599,6 +1018,7 @@ int mga_dma_init(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_mga_init_t init;
+	int err;
 
 	LOCK_TEST_WITH_RETURN(dev, filp);
 
@@ -607,9 +1027,13 @@ int mga_dma_init(DRM_IOCTL_ARGS)
 
 	switch (init.func) {
 	case MGA_INIT_DMA:
-		return mga_do_init_dma(dev, &init);
+		err = mga_do_init_dma(dev, &init);
+		if (err) {
+			(void) mga_do_cleanup_dma(dev, FULL_CLEANUP);
+		}
+		return err;
 	case MGA_CLEANUP_DMA:
-		return mga_do_cleanup_dma(dev);
+		return mga_do_cleanup_dma(dev, FULL_CLEANUP);
 	}
 
 	return DRM_ERR(EINVAL);
@@ -735,9 +1159,23 @@ int mga_dma_buffers(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-void mga_driver_pretakedown(drm_device_t * dev)
+/**
+ * Called just before the module is unloaded.
+ */
+int mga_driver_unload(drm_device_t * dev)
+{
+	drm_free(dev->dev_private, sizeof(drm_mga_private_t), DRM_MEM_DRIVER);
+	dev->dev_private = NULL;
+
+	return 0;
+}
+
+/**
+ * Called when the last opener of the device is closed.
+ */
+void mga_driver_lastclose(drm_device_t * dev)
 {
-	mga_do_cleanup_dma(dev);
+	mga_do_cleanup_dma(dev, FULL_CLEANUP);
 }
 
 int mga_driver_dma_quiescent(drm_device_t * dev)
diff --git a/sys/dev/drm/mga_drm.h b/sys/dev/drm/mga_drm.h
index 60b51533f79d..6c466e06af96 100644
--- a/sys/dev/drm/mga_drm.h
+++ b/sys/dev/drm/mga_drm.h
@@ -1,6 +1,6 @@
 /* mga_drm.h -- Public header for the Matrox g200/g400 driver -*- linux-c -*-
- * Created: Tue Jan 25 01:50:01 1999 by jhartmann@precisioninsight.com */
-/*-
+ * Created: Tue Jan 25 01:50:01 1999 by jhartmann@precisioninsight.com
+ *
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All rights reserved.
@@ -30,10 +30,11 @@
  *
  * Rewritten by:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __MGA_DRM_H__
 #define __MGA_DRM_H__
 
@@ -75,6 +76,8 @@
 
 #define MGA_CARD_TYPE_G200	1
 #define MGA_CARD_TYPE_G400	2
+#define MGA_CARD_TYPE_G450	3       /* not currently used */
+#define MGA_CARD_TYPE_G550	4
 
 #define MGA_FRONT		0x1
 #define MGA_BACK		0x2
@@ -224,9 +227,6 @@ typedef struct _drm_mga_sarea {
 	int ctxOwner;
 } drm_mga_sarea_t;
 
-/* WARNING: If you change any of these defines, make sure to change the
- * defines in the Xserver file (xf86drmMga.h)
- */
 
 /* MGA specific ioctls
  * The device specific ioctl range is 0x40 to 0x79.
@@ -242,6 +242,14 @@ typedef struct _drm_mga_sarea {
 #define DRM_MGA_BLIT     0x08
 #define DRM_MGA_GETPARAM 0x09
 
+/* 3.2:
+ * ioctls for operating on fences.
+ */
+#define DRM_MGA_SET_FENCE      0x0a
+#define DRM_MGA_WAIT_FENCE     0x0b
+#define DRM_MGA_DMA_BOOTSTRAP  0x0c
+
+
 #define DRM_IOCTL_MGA_INIT     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_INIT, drm_mga_init_t)
 #define DRM_IOCTL_MGA_FLUSH    DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_FLUSH, drm_lock_t)
 #define DRM_IOCTL_MGA_RESET    DRM_IO(  DRM_COMMAND_BASE + DRM_MGA_RESET)
@@ -252,6 +260,9 @@ typedef struct _drm_mga_sarea {
 #define DRM_IOCTL_MGA_ILOAD    DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_ILOAD, drm_mga_iload_t)
 #define DRM_IOCTL_MGA_BLIT     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_BLIT, drm_mga_blit_t)
 #define DRM_IOCTL_MGA_GETPARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_GETPARAM, drm_mga_getparam_t)
+#define DRM_IOCTL_MGA_SET_FENCE     DRM_IOW( DRM_COMMAND_BASE + DRM_MGA_SET_FENCE, uint32_t)
+#define DRM_IOCTL_MGA_WAIT_FENCE    DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_WAIT_FENCE, uint32_t)
+#define DRM_IOCTL_MGA_DMA_BOOTSTRAP DRM_IOWR(DRM_COMMAND_BASE + DRM_MGA_DMA_BOOTSTRAP, drm_mga_dma_bootstrap_t)
 
 typedef struct _drm_mga_warp_index {
 	int installed;
@@ -290,12 +301,73 @@ typedef struct drm_mga_init {
 	unsigned long buffers_offset;
 } drm_mga_init_t;
 
-typedef struct drm_mga_fullscreen {
-	enum {
-		MGA_INIT_FULLSCREEN = 0x01,
-		MGA_CLEANUP_FULLSCREEN = 0x02
-	} func;
-} drm_mga_fullscreen_t;
+
+typedef struct drm_mga_dma_bootstrap {
+	/**
+	 * \name AGP texture region
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, these fields will
+	 * be filled in with the actual AGP texture settings.
+	 * 
+	 * \warning
+	 * If these fields are non-zero, but dma_mga_dma_bootstrap::agp_mode
+	 * is zero, it means that PCI memory (most likely through the use of
+	 * an IOMMU) is being used for "AGP" textures.
+	 */
+	/*@{*/
+	unsigned long texture_handle;  /**< Handle used to map AGP textures. */
+	uint32_t     texture_size;    /**< Size of the AGP texture region. */
+	/*@}*/
+
+
+	/**
+	 * Requested size of the primary DMA region.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual AGP mode.  If AGP was not available
+	 */
+	uint32_t primary_size;
+
+
+	/**
+	 * Requested number of secondary DMA buffers.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual number of secondary DMA buffers
+	 * allocated.  Particularly when PCI DMA is used, this may be
+	 * (subtantially) less than the number requested.
+	 */
+	uint32_t secondary_bin_count;
+	
+	
+	/**
+	 * Requested size of each secondary DMA buffer.
+	 * 
+	 * While the kernel \b is free to reduce
+	 * dma_mga_dma_bootstrap::secondary_bin_count, it is \b not allowed
+	 * to reduce dma_mga_dma_bootstrap::secondary_bin_size.
+	 */
+	uint32_t secondary_bin_size;
+
+
+	/**
+	 * Bit-wise mask of AGPSTAT2_* values.  Currently only \c AGPSTAT2_1X,
+	 * \c AGPSTAT2_2X, and \c AGPSTAT2_4X are supported.  If this value is
+	 * zero, it means that PCI DMA should be used, even if AGP is
+	 * possible.
+	 * 
+	 * On return from the DRM_MGA_DMA_BOOTSTRAP ioctl, this field will be
+	 * filled in with the actual AGP mode.  If AGP was not available
+	 * (i.e., PCI DMA was used), this value will be zero.
+	 */
+	uint32_t agp_mode;
+
+
+	/**
+	 * Desired AGP GART size, measured in megabytes.
+	 */
+	uint8_t agp_size;
+} drm_mga_dma_bootstrap_t;
 
 typedef struct drm_mga_clear {
 	unsigned int flags;
@@ -340,6 +412,14 @@ typedef struct _drm_mga_blit {
  */
 #define MGA_PARAM_IRQ_NR            1
 
+/* 3.2: Query the actual card type.  The DDX only distinguishes between
+ * G200 chips and non-G200 chips, which it calls G400.  It turns out that
+ * there are some very sublte differences between the G4x0 chips and the G550
+ * chips.  Using this parameter query, a client-side driver can detect the
+ * difference between a G4x0 and a G550.
+ */
+#define MGA_PARAM_CARD_TYPE         2
+
 typedef struct drm_mga_getparam {
 	int param;
 	void __user *value;
diff --git a/sys/dev/drm/mga_drv.c b/sys/dev/drm/mga_drv.c
index 05569a91b34b..6df2434125c1 100644
--- a/sys/dev/drm/mga_drv.c
+++ b/sys/dev/drm/mga_drv.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mga_drm.h"
@@ -43,40 +45,76 @@ static drm_pci_id_list_t mga_pciidlist[] = {
 	mga_PCI_IDS
 };
 
-extern drm_ioctl_desc_t mga_ioctls[];
-extern int mga_max_ioctl;
+/**
+ * Determine if the device really is AGP or not.
+ *
+ * In addition to the usual tests performed by \c drm_device_is_agp, this
+ * function detects PCI G450 cards that appear to the system exactly like
+ * AGP G450 cards.
+ *
+ * \param dev   The device to be tested.
+ *
+ * \returns
+ * If the device is a PCI G450, zero is returned.  Otherwise non-zero is
+ * returned.
+ *
+ * \bug
+ * This function needs to be filled in!  The implementation in
+ * linux-core/mga_drv.c shows what needs to be done.
+ */
+static int mga_driver_device_is_agp(drm_device_t * dev)
+{
+	/* There are PCI versions of the G450.  These cards have the
+	 * same PCI ID as the AGP G450, but have an additional PCI-to-PCI
+	 * bridge chip.  We detect these cards, which are not currently
+	 * supported by this driver, by looking at the device ID of the
+	 * bus the "card" is on.  If vendor is 0x3388 (Hint Corp) and the
+	 * device is 0x0021 (HB6 Universal PCI-PCI bridge), we reject the
+	 * device.
+	 */
+	if (pci_get_device(dev->device) == 0x0525 &&
+	    pci_get_vendor(device_get_parent(dev->device)) == 0x3388 &&
+	    pci_get_device(device_get_parent(dev->device)) == 0x0021)
+		return 0;
+	else
+		return 2;
+}
 
 static void mga_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = sizeof(drm_mga_buf_priv_t);
-	/* XXX dev->prerelease = mga_driver_prerelease; */
-	dev->pretakedown = mga_driver_pretakedown;
-	dev->vblank_wait = mga_driver_vblank_wait;
-	dev->irq_preinstall = mga_driver_irq_preinstall;
-	dev->irq_postinstall = mga_driver_irq_postinstall;
-	dev->irq_uninstall = mga_driver_irq_uninstall;
-	dev->irq_handler = mga_driver_irq_handler;
-	dev->dma_ioctl = mga_dma_buffers;
-	dev->dma_quiescent = mga_driver_dma_quiescent;
-
-	dev->driver_ioctls = mga_ioctls;
-	dev->max_driver_ioctl = mga_max_ioctl;
-
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
-
-	dev->use_agp = 1;
-	dev->require_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_dma = 1;
-	dev->use_irq = 1;
-	dev->use_vbl_irq = 1;
+	dev->driver.buf_priv_size	= sizeof(drm_mga_buf_priv_t);
+	dev->driver.load		= mga_driver_load;
+	dev->driver.unload		= mga_driver_unload;
+	dev->driver.lastclose		= mga_driver_lastclose;
+	dev->driver.vblank_wait		= mga_driver_vblank_wait;
+	dev->driver.irq_preinstall	= mga_driver_irq_preinstall;
+	dev->driver.irq_postinstall	= mga_driver_irq_postinstall;
+	dev->driver.irq_uninstall	= mga_driver_irq_uninstall;
+	dev->driver.irq_handler		= mga_driver_irq_handler;
+	dev->driver.dma_ioctl		= mga_dma_buffers;
+	dev->driver.dma_quiescent	= mga_driver_dma_quiescent;
+	dev->driver.device_is_agp	= mga_driver_device_is_agp;
+
+	dev->driver.ioctls		= mga_ioctls;
+	dev->driver.max_ioctl		= mga_max_ioctl;
+
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
+
+	dev->driver.use_agp		= 1;
+	dev->driver.require_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_dma		= 1;
+	dev->driver.use_irq		= 1;
+	dev->driver.use_vbl_irq		= 1;
 }
 
+
+
 #ifdef __FreeBSD__
 static int
 mga_probe(device_t dev)
@@ -114,5 +152,10 @@ DRIVER_MODULE(mga, pci, mga_driver, drm_devclass, 0, 0);
 MODULE_DEPEND(mga, drm, 1, 1, 1);
 
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
+#ifdef _LKM
 CFDRIVER_DECL(mga, DV_TTY, NULL);
+#else
+CFATTACH_DECL(mga, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+    drm_activate);
+#endif
 #endif
diff --git a/sys/dev/drm/mga_drv.h b/sys/dev/drm/mga_drv.h
index 5bca56a0ae51..31ea2aa0bf44 100644
--- a/sys/dev/drm/mga_drv.h
+++ b/sys/dev/drm/mga_drv.h
@@ -1,6 +1,6 @@
 /* mga_drv.h -- Private header for the Matrox G200/G400 driver -*- linux-c -*-
- * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com */
-/*-
+ * Created: Mon Dec 13 01:50:01 1999 by jhartmann@precisioninsight.com
+ *
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All rights reserved.
@@ -26,10 +26,11 @@
  *
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __MGA_DRV_H__
 #define __MGA_DRV_H__
 
@@ -40,11 +41,11 @@
 
 #define DRIVER_NAME		"mga"
 #define DRIVER_DESC		"Matrox G200/G400"
-#define DRIVER_DATE		"20021029"
+#define DRIVER_DATE		"20051102"
 
 #define DRIVER_MAJOR		3
-#define DRIVER_MINOR		1
-#define DRIVER_PATCHLEVEL	0
+#define DRIVER_MINOR		2
+#define DRIVER_PATCHLEVEL	1
 
 typedef struct drm_mga_primary_buffer {
 	u8 *start;
@@ -89,9 +90,43 @@ typedef struct drm_mga_private {
 	int chipset;
 	int usec_timeout;
 
+	/**
+	 * If set, the new DMA initialization sequence was used.  This is
+	 * primarilly used to select how the driver should uninitialized its
+	 * internal DMA structures.
+	 */
+	int used_new_dma_init;
+
+	/**
+	 * If AGP memory is used for DMA buffers, this will be the value
+	 * \c MGA_PAGPXFER.  Otherwise, it will be zero (for a PCI transfer).
+	 */
+	u32 dma_access;
+
+	/**
+	 * If AGP memory is used for DMA buffers, this will be the value
+	 * \c MGA_WAGP_ENABLE.  Otherwise, it will be zero (for a PCI
+	 * transfer).
+	 */
+	u32 wagp_enable;
+
+	/**
+	 * \name MMIO region parameters.
+	 * 
+	 * \sa drm_mga_private_t::mmio
+	 */
+	/*@{*/
+	u32 mmio_base;             /**< Bus address of base of MMIO. */
+	u32 mmio_size;             /**< Size of the MMIO region. */
+	/*@}*/
+
 	u32 clear_cmd;
 	u32 maccess;
 
+	wait_queue_head_t fence_queue;
+	atomic_t last_fence_retired;
+	u32 next_fence_to_post;
+
 	unsigned int fb_cpp;
 	unsigned int front_offset;
 	unsigned int front_pitch;
@@ -110,16 +145,24 @@ typedef struct drm_mga_private {
 	drm_local_map_t *status;
 	drm_local_map_t *warp;
 	drm_local_map_t *primary;
-	drm_local_map_t *buffers;
 	drm_local_map_t *agp_textures;
+	
+	unsigned long agp_handle;
+	unsigned int agp_size;
 } drm_mga_private_t;
 
+extern drm_ioctl_desc_t mga_ioctls[];
+extern int mga_max_ioctl;
+
 				/* mga_dma.c */
+extern int mga_dma_bootstrap(DRM_IOCTL_ARGS);
 extern int mga_dma_init(DRM_IOCTL_ARGS);
 extern int mga_dma_flush(DRM_IOCTL_ARGS);
 extern int mga_dma_reset(DRM_IOCTL_ARGS);
 extern int mga_dma_buffers(DRM_IOCTL_ARGS);
-extern void mga_driver_pretakedown(drm_device_t * dev);
+extern int mga_driver_load(drm_device_t *dev, unsigned long flags);
+extern int mga_driver_unload(drm_device_t * dev);
+extern void mga_driver_lastclose(drm_device_t * dev);
 extern int mga_driver_dma_quiescent(drm_device_t * dev);
 
 extern int mga_do_wait_for_idle(drm_mga_private_t * dev_priv);
@@ -131,14 +174,19 @@ extern void mga_do_dma_wrap_end(drm_mga_private_t * dev_priv);
 extern int mga_freelist_put(drm_device_t * dev, drm_buf_t * buf);
 
 				/* mga_warp.c */
+extern unsigned int mga_warp_microcode_size(const drm_mga_private_t * dev_priv);
 extern int mga_warp_install_microcode(drm_mga_private_t * dev_priv);
 extern int mga_warp_init(drm_mga_private_t * dev_priv);
 
+				/* mga_irq.c */
+extern int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence);
 extern int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence);
 extern irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS);
 extern void mga_driver_irq_preinstall(drm_device_t * dev);
 extern void mga_driver_irq_postinstall(drm_device_t * dev);
 extern void mga_driver_irq_uninstall(drm_device_t * dev);
+extern long mga_compat_ioctl(struct file *filp, unsigned int cmd,
+			     unsigned long arg);
 
 #define mga_flush_write_combine()	DRM_WRITEMEMORYBARRIER()
 
@@ -183,7 +231,7 @@ static inline u32 _MGA_READ(u32 * addr)
 #define MGA_EMIT_STATE( dev_priv, dirty )				\
 do {									\
 	if ( (dirty) & ~MGA_UPLOAD_CLIPRECTS ) {			\
-		if ( dev_priv->chipset == MGA_CARD_TYPE_G400 ) {	\
+		if ( dev_priv->chipset >= MGA_CARD_TYPE_G400 ) {	\
 			mga_g400_emit_state( dev_priv );		\
 		} else {						\
 			mga_g200_emit_state( dev_priv );		\
@@ -520,6 +568,12 @@ do {									\
  */
 #define MGA_EXEC 			0x0100
 
+/* AGP PLL encoding (for G200 only).
+ */
+#define MGA_AGP_PLL 			0x1e4c
+#	define MGA_AGP2XPLL_DISABLE		(0 << 0)
+#	define MGA_AGP2XPLL_ENABLE		(1 << 0)
+
 /* Warp registers
  */
 #define MGA_WR0				0x2d00
diff --git a/sys/dev/drm/mga_irq.c b/sys/dev/drm/mga_irq.c
index 2aaef9cb895f..f8a88ed1553b 100644
--- a/sys/dev/drm/mga_irq.c
+++ b/sys/dev/drm/mga_irq.c
@@ -1,4 +1,5 @@
-/* mga_irq.c -- IRQ handling for radeon -*- linux-c -*- */
+/* mga_irq.c -- IRQ handling for radeon -*- linux-c -*-
+ */
 /*-
  * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
  *
@@ -28,10 +29,11 @@
  * Authors:
  *    Keith Whitwell <keith@tungstengraphics.com>
  *    Eric Anholt <anholt@FreeBSD.org>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mga_drm.h"
@@ -42,6 +44,7 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 	drm_device_t *dev = (drm_device_t *) arg;
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 	int status;
+	int handled = 0;
 
 	status = MGA_READ(MGA_STATUS);
 
@@ -51,6 +54,30 @@ irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
 		atomic_inc(&dev->vbl_received);
 		DRM_WAKEUP(&dev->vbl_queue);
 		drm_vbl_send_signals(dev);
+		handled = 1;
+	}
+
+	/* SOFTRAP interrupt */
+	if (status & MGA_SOFTRAPEN) {
+		const u32 prim_start = MGA_READ(MGA_PRIMADDRESS);
+		const u32 prim_end   = MGA_READ(MGA_PRIMEND);
+
+
+		MGA_WRITE(MGA_ICLEAR, MGA_SOFTRAPICLR);
+
+		/* In addition to clearing the interrupt-pending bit, we
+		 * have to write to MGA_PRIMEND to re-start the DMA operation.
+		 */
+		if ( (prim_start & ~0x03) != (prim_end & ~0x03) ) {
+			MGA_WRITE(MGA_PRIMEND, prim_end);
+		}
+
+		atomic_inc(&dev_priv->last_fence_retired);
+		DRM_WAKEUP(&dev_priv->fence_queue);
+		handled = 1;
+	}
+
+	if ( handled ) {
 		return IRQ_HANDLED;
 	}
 	return IRQ_NONE;
@@ -74,6 +101,25 @@ int mga_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
 	return ret;
 }
 
+int mga_driver_fence_wait(drm_device_t * dev, unsigned int *sequence)
+{
+	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
+	unsigned int cur_fence;
+	int ret = 0;
+
+	/* Assume that the user has missed the current sequence number
+	 * by about a day rather than she wants to wait for years
+	 * using fences.
+	 */
+	DRM_WAIT_ON(ret, dev_priv->fence_queue, 3 * DRM_HZ,
+		    (((cur_fence = atomic_read(&dev_priv->last_fence_retired))
+		      - *sequence) <= (1 << 23)));
+
+	*sequence = cur_fence;
+
+	return ret;
+}
+
 void mga_driver_irq_preinstall(drm_device_t * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
@@ -88,8 +134,10 @@ void mga_driver_irq_postinstall(drm_device_t * dev)
 {
 	drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
 
-	/* Turn on VBL interrupt */
-	MGA_WRITE(MGA_IEN, MGA_VLINEIEN);
+	DRM_INIT_WAITQUEUE( &dev_priv->fence_queue );
+
+	/* Turn on vertical blank interrupt and soft trap interrupt. */
+	MGA_WRITE(MGA_IEN, MGA_VLINEIEN | MGA_SOFTRAPEN);
 }
 
 void mga_driver_irq_uninstall(drm_device_t * dev)
@@ -100,4 +148,6 @@ void mga_driver_irq_uninstall(drm_device_t * dev)
 
 	/* Disable *all* interrupts */
 	MGA_WRITE(MGA_IEN, 0);
+	
+	dev->irq_enabled = 0;
 }
diff --git a/sys/dev/drm/mga_state.c b/sys/dev/drm/mga_state.c
index 136ff16c0a88..951a4c8adceb 100644
--- a/sys/dev/drm/mga_state.c
+++ b/sys/dev/drm/mga_state.c
@@ -1,5 +1,6 @@
 /* mga_state.c -- State support for MGA G200/G400 -*- linux-c -*-
- * Created: Thu Jan 27 02:53:43 2000 by jhartmann@precisioninsight.com */
+ * Created: Thu Jan 27 02:53:43 2000 by jhartmann@precisioninsight.com
+ */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -30,11 +31,11 @@
  *
  * Rewritten by:
  *    Gareth Hughes <gareth@valinux.com>
- * $FreeBSD$
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mga_drm.h"
@@ -56,7 +57,7 @@ static void mga_emit_clip_rect(drm_mga_private_t * dev_priv,
 
 	/* Force reset of DWGCTL on G400 (eliminates clip disable bit).
 	 */
-	if (dev_priv->chipset == MGA_CARD_TYPE_G400) {
+	if (dev_priv->chipset >= MGA_CARD_TYPE_G400) {
 		DMA_BLOCK(MGA_DWGCTL, ctx->dwgctl,
 			  MGA_LEN + MGA_EXEC, 0x80000000,
 			  MGA_DWGCTL, ctx->dwgctl,
@@ -267,7 +268,7 @@ static __inline__ void mga_g200_emit_pipe(drm_mga_private_t * dev_priv)
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_WIADDR, (dev_priv->warp_pipe_phys[pipe] |
-			       MGA_WMODE_START | MGA_WAGP_ENABLE));
+			       MGA_WMODE_START | dev_priv->wagp_enable));
 
 	ADVANCE_DMA();
 }
@@ -348,7 +349,7 @@ static __inline__ void mga_g400_emit_pipe(drm_mga_private_t * dev_priv)
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_DMAPAD, 0xffffffff,
 		  MGA_WIADDR2, (dev_priv->warp_pipe_phys[pipe] |
-				MGA_WMODE_START | MGA_WAGP_ENABLE));
+				MGA_WMODE_START | dev_priv->wagp_enable));
 
 	ADVANCE_DMA();
 }
@@ -458,7 +459,7 @@ static int mga_verify_state(drm_mga_private_t * dev_priv)
 	if (dirty & MGA_UPLOAD_TEX0)
 		ret |= mga_verify_tex(dev_priv, 0);
 
-	if (dev_priv->chipset == MGA_CARD_TYPE_G400) {
+	if (dev_priv->chipset >= MGA_CARD_TYPE_G400) {
 		if (dirty & MGA_UPLOAD_TEX1)
 			ret |= mga_verify_tex(dev_priv, 1);
 
@@ -682,7 +683,7 @@ static void mga_dma_dispatch_vertex(drm_device_t * dev, drm_buf_t * buf)
 				  MGA_SECADDRESS, (address |
 						   MGA_DMA_VERTEX),
 				  MGA_SECEND, ((address + length) |
-					       MGA_PAGPXFER));
+					       dev_priv->dma_access));
 
 			ADVANCE_DMA();
 		} while (++i < sarea_priv->nbox);
@@ -728,7 +729,7 @@ static void mga_dma_dispatch_indices(drm_device_t * dev, drm_buf_t * buf,
 				  MGA_DMAPAD, 0x00000000,
 				  MGA_SETUPADDRESS, address + start,
 				  MGA_SETUPEND, ((address + end) |
-						 MGA_PAGPXFER));
+						 dev_priv->dma_access));
 
 			ADVANCE_DMA();
 		} while (++i < sarea_priv->nbox);
@@ -755,7 +756,7 @@ static void mga_dma_dispatch_iload(drm_device_t * dev, drm_buf_t * buf,
 	drm_mga_private_t *dev_priv = dev->dev_private;
 	drm_mga_buf_priv_t *buf_priv = buf->dev_private;
 	drm_mga_context_regs_t *ctx = &dev_priv->sarea_priv->context_state;
-	u32 srcorg = buf->bus_address | MGA_SRCACC_AGP | MGA_SRCMAP_SYSMEM;
+	u32 srcorg = buf->bus_address | dev_priv->dma_access | MGA_SRCMAP_SYSMEM;
 	u32 y2;
 	DMA_LOCALS;
 	DRM_DEBUG("buf=%d used=%d\n", buf->idx, buf->used);
@@ -1090,6 +1091,9 @@ static int mga_getparam(DRM_IOCTL_ARGS)
 	case MGA_PARAM_IRQ_NR:
 		value = dev->irq;
 		break;
+	case MGA_PARAM_CARD_TYPE:
+		value = dev_priv->chipset;
+		break;
 	default:
 		return DRM_ERR(EINVAL);
 	}
@@ -1102,17 +1106,76 @@ static int mga_getparam(DRM_IOCTL_ARGS)
 	return 0;
 }
 
+static int mga_set_fence(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_private_t *dev_priv = dev->dev_private;
+	u32 temp;
+	DMA_LOCALS;
+
+	if (!dev_priv) {
+		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
+		return DRM_ERR(EINVAL);
+	}
+
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
+
+	/* I would normal do this assignment in the declaration of temp,
+	 * but dev_priv may be NULL.
+	 */
+
+	temp = dev_priv->next_fence_to_post;
+	dev_priv->next_fence_to_post++;
+
+	BEGIN_DMA(1);
+	DMA_BLOCK(MGA_DMAPAD, 0x00000000,
+		  MGA_DMAPAD, 0x00000000,
+		  MGA_DMAPAD, 0x00000000,
+		  MGA_SOFTRAP, 0x00000000);
+	ADVANCE_DMA();
+
+	DRM_COPY_TO_USER_IOCTL((u32 __user *)data, temp, sizeof(u32));
+
+	return 0;
+}
+
+static int mga_wait_fence(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_mga_private_t *dev_priv = dev->dev_private;
+	u32 fence;
+
+	if (!dev_priv) {
+		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
+		return DRM_ERR(EINVAL);
+	}
+
+	DRM_COPY_FROM_USER_IOCTL(fence, (u32 __user *) data, sizeof(u32));
+
+	DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
+
+	mga_driver_fence_wait(dev, & fence);
+
+	DRM_COPY_TO_USER_IOCTL((u32 __user *)data, fence, sizeof(u32));
+
+	return 0;
+}
+
 drm_ioctl_desc_t mga_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_MGA_INIT)] = {mga_dma_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_MGA_FLUSH)] = {mga_dma_flush, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_RESET)] = {mga_dma_reset, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_SWAP)] = {mga_dma_swap, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_CLEAR)] = {mga_dma_clear, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_VERTEX)] = {mga_dma_vertex, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_INDICES)] = {mga_dma_indices, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_ILOAD)] = {mga_dma_iload, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_BLIT)] = {mga_dma_blit, 1, 0},
-	[DRM_IOCTL_NR(DRM_MGA_GETPARAM)] = {mga_getparam, 1, 0},
+	[DRM_IOCTL_NR(DRM_MGA_INIT)] = {mga_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_MGA_FLUSH)] = {mga_dma_flush, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_RESET)] = {mga_dma_reset, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_SWAP)] = {mga_dma_swap, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_CLEAR)] = {mga_dma_clear, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_VERTEX)] = {mga_dma_vertex, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_INDICES)] = {mga_dma_indices, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_ILOAD)] = {mga_dma_iload, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_BLIT)] = {mga_dma_blit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_GETPARAM)] = {mga_getparam, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_SET_FENCE)] = {mga_set_fence, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_WAIT_FENCE)] = {mga_wait_fence, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_MGA_DMA_BOOTSTRAP)] = {mga_dma_bootstrap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+
 };
 
 int mga_max_ioctl = DRM_ARRAY_SIZE(mga_ioctls);
diff --git a/sys/dev/drm/mga_ucode.h b/sys/dev/drm/mga_ucode.h
index dbf9eba38021..455e25e5eb0f 100644
--- a/sys/dev/drm/mga_ucode.h
+++ b/sys/dev/drm/mga_ucode.h
@@ -1,6 +1,6 @@
 /* mga_ucode.h -- Matrox G200/G400 WARP engine microcode -*- linux-c -*-
- * Created: Thu Jan 11 21:20:43 2001 by gareth@valinux.com */
-/*-
+ * Created: Thu Jan 11 21:20:43 2001 by gareth@valinux.com
+ *
  * Copyright 1999 Matrox Graphics Inc.
  * All Rights Reserved.
  *
@@ -24,10 +24,11 @@
  *
  * Kernel-based WARP engine management:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 /*
  * WARP pipes are named according to the functions they perform, where:
  *
diff --git a/sys/dev/drm/mga_warp.c b/sys/dev/drm/mga_warp.c
index b2d5de73aced..23397397711e 100644
--- a/sys/dev/drm/mga_warp.c
+++ b/sys/dev/drm/mga_warp.c
@@ -1,5 +1,6 @@
 /* mga_warp.c -- Matrox G200/G400 WARP engine management -*- linux-c -*-
- * Created: Thu Jan 11 21:29:32 2001 by gareth@valinux.com */
+ * Created: Thu Jan 11 21:29:32 2001 by gareth@valinux.com
+ */
 /*-
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
@@ -25,10 +26,11 @@
  *
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/mga_drm.h"
@@ -49,11 +51,8 @@ do {									\
 	vcbase += WARP_UCODE_SIZE( which );				\
 } while (0)
 
-static unsigned int mga_warp_g400_microcode_size(drm_mga_private_t * dev_priv)
-{
-	unsigned int size;
-
-	size = (WARP_UCODE_SIZE(warp_g400_tgz) +
+static const unsigned int mga_warp_g400_microcode_size =
+	       (WARP_UCODE_SIZE(warp_g400_tgz) +
 		WARP_UCODE_SIZE(warp_g400_tgza) +
 		WARP_UCODE_SIZE(warp_g400_tgzaf) +
 		WARP_UCODE_SIZE(warp_g400_tgzf) +
@@ -70,17 +69,8 @@ static unsigned int mga_warp_g400_microcode_size(drm_mga_private_t * dev_priv)
 		WARP_UCODE_SIZE(warp_g400_t2gzsaf) +
 		WARP_UCODE_SIZE(warp_g400_t2gzsf));
 
-	size = PAGE_ALIGN(size);
-
-	DRM_DEBUG("G400 ucode size = %d bytes\n", size);
-	return size;
-}
-
-static unsigned int mga_warp_g200_microcode_size(drm_mga_private_t * dev_priv)
-{
-	unsigned int size;
-
-	size = (WARP_UCODE_SIZE(warp_g200_tgz) +
+static const unsigned int mga_warp_g200_microcode_size =
+	       (WARP_UCODE_SIZE(warp_g200_tgz) +
 		WARP_UCODE_SIZE(warp_g200_tgza) +
 		WARP_UCODE_SIZE(warp_g200_tgzaf) +
 		WARP_UCODE_SIZE(warp_g200_tgzf) +
@@ -89,24 +79,25 @@ static unsigned int mga_warp_g200_microcode_size(drm_mga_private_t * dev_priv)
 		WARP_UCODE_SIZE(warp_g200_tgzsaf) +
 		WARP_UCODE_SIZE(warp_g200_tgzsf));
 
-	size = PAGE_ALIGN(size);
 
-	DRM_DEBUG("G200 ucode size = %d bytes\n", size);
-	return size;
+unsigned int mga_warp_microcode_size(const drm_mga_private_t * dev_priv)
+{
+	switch (dev_priv->chipset) {
+	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
+		return PAGE_ALIGN(mga_warp_g400_microcode_size);
+	case MGA_CARD_TYPE_G200:
+		return PAGE_ALIGN(mga_warp_g200_microcode_size);
+	default:
+		DRM_ERROR("Unknown chipset value: 0x%x\n", dev_priv->chipset);
+		return 0;
+	}
 }
 
 static int mga_warp_install_g400_microcode(drm_mga_private_t * dev_priv)
 {
 	unsigned char *vcbase = dev_priv->warp->handle;
 	unsigned long pcbase = dev_priv->warp->offset;
-	unsigned int size;
-
-	size = mga_warp_g400_microcode_size(dev_priv);
-	if (size > dev_priv->warp->size) {
-		DRM_ERROR("microcode too large! (%u > %lu)\n",
-			  size, dev_priv->warp->size);
-		return DRM_ERR(ENOMEM);
-	}
 
 	memset(dev_priv->warp_pipe_phys, 0, sizeof(dev_priv->warp_pipe_phys));
 
@@ -135,14 +126,6 @@ static int mga_warp_install_g200_microcode(drm_mga_private_t * dev_priv)
 {
 	unsigned char *vcbase = dev_priv->warp->handle;
 	unsigned long pcbase = dev_priv->warp->offset;
-	unsigned int size;
-
-	size = mga_warp_g200_microcode_size(dev_priv);
-	if (size > dev_priv->warp->size) {
-		DRM_ERROR("microcode too large! (%u > %lu)\n",
-			  size, dev_priv->warp->size);
-		return DRM_ERR(ENOMEM);
-	}
 
 	memset(dev_priv->warp_pipe_phys, 0, sizeof(dev_priv->warp_pipe_phys));
 
@@ -160,8 +143,18 @@ static int mga_warp_install_g200_microcode(drm_mga_private_t * dev_priv)
 
 int mga_warp_install_microcode(drm_mga_private_t * dev_priv)
 {
+	const unsigned int size = mga_warp_microcode_size(dev_priv);
+
+	DRM_DEBUG("MGA ucode size = %d bytes\n", size);
+	if (size > dev_priv->warp->size) {
+		DRM_ERROR("microcode too large! (%u > %lu)\n",
+			  size, dev_priv->warp->size);
+		return DRM_ERR(ENOMEM);
+	}
+
 	switch (dev_priv->chipset) {
 	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
 		return mga_warp_install_g400_microcode(dev_priv);
 	case MGA_CARD_TYPE_G200:
 		return mga_warp_install_g200_microcode(dev_priv);
@@ -180,6 +173,7 @@ int mga_warp_init(drm_mga_private_t * dev_priv)
 	 */
 	switch (dev_priv->chipset) {
 	case MGA_CARD_TYPE_G400:
+	case MGA_CARD_TYPE_G550:
 		MGA_WRITE(MGA_WIADDR2, MGA_WMODE_SUSPEND);
 		MGA_WRITE(MGA_WGETMSB, 0x00000E00);
 		MGA_WRITE(MGA_WVRTXSZ, 0x00001807);
diff --git a/sys/dev/drm/r128_cce.c b/sys/dev/drm/r128_cce.c
index a7fa0f799641..9ac647eacbc7 100644
--- a/sys/dev/drm/r128_cce.c
+++ b/sys/dev/drm/r128_cce.c
@@ -1,5 +1,6 @@
 /* r128_cce.c -- ATI Rage 128 driver -*- linux-c -*-
- * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com */
+ * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
+ */
 /*-
  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -26,10 +27,11 @@
  *
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/r128_drm.h"
@@ -325,7 +327,8 @@ static void r128_cce_init_ring_buffer(drm_device_t * dev,
 		ring_start = dev_priv->cce_ring->offset - dev->agp->base;
 	else
 #endif
-		ring_start = dev_priv->cce_ring->offset - dev->sg->handle;
+		ring_start = dev_priv->cce_ring->offset - 
+				(unsigned long)dev->sg->virtual;
 
 	R128_WRITE(R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET);
 
@@ -486,6 +489,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		r128_do_cleanup_cce(dev);
 		return DRM_ERR(EINVAL);
 	}
+	dev->agp_buffer_token = init->buffers_offset;
 	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
 	if (!dev->agp_buffer_map) {
 		DRM_ERROR("could not find dma buffer region!\n");
@@ -537,7 +541,7 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 		dev_priv->cce_buffers_offset = dev->agp->base;
 	else
 #endif
-		dev_priv->cce_buffers_offset = dev->sg->handle;
+		dev_priv->cce_buffers_offset = (unsigned long)dev->sg->virtual;
 
 	dev_priv->ring.start = (u32 *) dev_priv->cce_ring->handle;
 	dev_priv->ring.end = ((u32 *) dev_priv->cce_ring->handle
@@ -558,14 +562,17 @@ static int r128_do_init_cce(drm_device_t * dev, drm_r128_init_t * init)
 #if __OS_HAS_AGP
 	if (dev_priv->is_pci) {
 #endif
-		if (!drm_ati_pcigart_init(dev, &dev_priv->phys_pci_gart,
-					  &dev_priv->bus_pci_gart, 0)) {
+		dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
+		dev_priv->gart_info.addr = NULL;
+		dev_priv->gart_info.bus_addr = 0;
+		dev_priv->gart_info.is_pcie = 0;
+		if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 			DRM_ERROR("failed to init PCI GART!\n");
 			dev->dev_private = (void *)dev_priv;
 			r128_do_cleanup_cce(dev);
 			return DRM_ERR(ENOMEM);
 		}
-		R128_WRITE(R128_PCI_GART_PAGE, dev_priv->bus_pci_gart);
+		R128_WRITE(R128_PCI_GART_PAGE, dev_priv->gart_info.bus_addr);
 #if __OS_HAS_AGP
 	}
 #endif
@@ -606,10 +613,9 @@ int r128_do_cleanup_cce(drm_device_t * dev)
 		} else
 #endif
 		{
-			if (!drm_ati_pcigart_cleanup(dev,
-						     dev_priv->phys_pci_gart,
-						     dev_priv->bus_pci_gart))
-				DRM_ERROR("failed to cleanup PCI GART!\n");
+			if (dev_priv->gart_info.bus_addr)
+				if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
+					DRM_ERROR("failed to cleanup PCI GART!\n");
 		}
 
 		drm_free(dev->dev_private, sizeof(drm_r128_private_t),
diff --git a/sys/dev/drm/r128_drm.h b/sys/dev/drm/r128_drm.h
index d5778aebe027..c9ad817d9356 100644
--- a/sys/dev/drm/r128_drm.h
+++ b/sys/dev/drm/r128_drm.h
@@ -1,5 +1,6 @@
 /* r128_drm.h -- Public header for the r128 driver -*- linux-c -*-
- * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com */
+ * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
+ */
 /*-
  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -27,10 +28,11 @@
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
  *    Kevin E. Martin <martin@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __R128_DRM_H__
 #define __R128_DRM_H__
 
@@ -215,7 +217,7 @@ typedef struct drm_r128_sarea {
 #define DRM_IOCTL_R128_INDIRECT   DRM_IOWR(DRM_COMMAND_BASE + DRM_R128_INDIRECT, drm_r128_indirect_t)
 #define DRM_IOCTL_R128_FULLSCREEN DRM_IOW( DRM_COMMAND_BASE + DRM_R128_FULLSCREEN, drm_r128_fullscreen_t)
 #define DRM_IOCTL_R128_CLEAR2     DRM_IOW( DRM_COMMAND_BASE + DRM_R128_CLEAR2, drm_r128_clear2_t)
-#define DRM_IOCTL_R128_GETPARAM   DRM_IOW( DRM_COMMAND_BASE + DRM_R128_GETPARAM, drm_r128_getparam_t)
+#define DRM_IOCTL_R128_GETPARAM   DRM_IOWR( DRM_COMMAND_BASE + DRM_R128_GETPARAM, drm_r128_getparam_t)
 #define DRM_IOCTL_R128_FLIP       DRM_IO(  DRM_COMMAND_BASE + DRM_R128_FLIP)
 
 typedef struct drm_r128_init {
diff --git a/sys/dev/drm/r128_drv.c b/sys/dev/drm/r128_drv.c
index 0a645ec694bf..e79dcbe10697 100644
--- a/sys/dev/drm/r128_drv.c
+++ b/sys/dev/drm/r128_drv.c
@@ -29,9 +29,11 @@
  *    Rickard E. (Rik) Faith <faith@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/r128_drm.h"
@@ -43,38 +45,35 @@ static drm_pci_id_list_t r128_pciidlist[] = {
 	r128_PCI_IDS
 };
 
-extern drm_ioctl_desc_t r128_ioctls[];
-extern int r128_max_ioctl;
-
 static void r128_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = sizeof(drm_r128_buf_priv_t);
-	dev->prerelease = r128_driver_prerelease;
-	dev->pretakedown = r128_driver_pretakedown;
-	dev->vblank_wait = r128_driver_vblank_wait;
-	dev->irq_preinstall = r128_driver_irq_preinstall;
-	dev->irq_postinstall = r128_driver_irq_postinstall;
-	dev->irq_uninstall = r128_driver_irq_uninstall;
-	dev->irq_handler = r128_driver_irq_handler;
-	dev->dma_ioctl = r128_cce_buffers;
+	dev->driver.buf_priv_size	= sizeof(drm_r128_buf_priv_t);
+	dev->driver.preclose		= r128_driver_preclose;
+	dev->driver.lastclose		= r128_driver_lastclose;
+	dev->driver.vblank_wait		= r128_driver_vblank_wait;
+	dev->driver.irq_preinstall	= r128_driver_irq_preinstall;
+	dev->driver.irq_postinstall	= r128_driver_irq_postinstall;
+	dev->driver.irq_uninstall	= r128_driver_irq_uninstall;
+	dev->driver.irq_handler		= r128_driver_irq_handler;
+	dev->driver.dma_ioctl		= r128_cce_buffers;
 
-	dev->driver_ioctls = r128_ioctls;
-	dev->max_driver_ioctl = r128_max_ioctl;
+	dev->driver.ioctls		= r128_ioctls;
+	dev->driver.max_ioctl		= r128_max_ioctl;
 
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
 
-	dev->use_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_pci_dma = 1;
-	dev->use_sg = 1;
-	dev->use_dma = 1;
-	dev->use_irq = 1;
-	dev->use_vbl_irq = 1;
+	dev->driver.use_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_pci_dma		= 1;
+	dev->driver.use_sg		= 1;
+	dev->driver.use_dma		= 1;
+	dev->driver.use_irq		= 1;
+	dev->driver.use_vbl_irq		= 1;
 }
 
 #ifdef __FreeBSD__
@@ -114,5 +113,10 @@ DRIVER_MODULE(r128, pci, r128_driver, drm_devclass, 0, 0);
 MODULE_DEPEND(r128, drm, 1, 1, 1);
 
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
+#ifdef _LKM
 CFDRIVER_DECL(r128, DV_TTY, NULL);
+#else
+CFATTACH_DECL(r128, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+    drm_activate);
+#endif
 #endif
diff --git a/sys/dev/drm/r128_drv.h b/sys/dev/drm/r128_drv.h
index f3caa0581dce..ba738fa3e80d 100644
--- a/sys/dev/drm/r128_drv.h
+++ b/sys/dev/drm/r128_drv.h
@@ -1,5 +1,6 @@
 /* r128_drv.h -- Private header for r128 driver -*- linux-c -*-
- * Created: Mon Dec 13 09:51:11 1999 by faith@precisioninsight.com */
+ * Created: Mon Dec 13 09:51:11 1999 by faith@precisioninsight.com
+ */
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
@@ -29,10 +30,11 @@
  *    Kevin E. Martin <martin@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *    Michel D�zer <daenzerm@student.ethz.ch>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __R128_DRV_H__
 #define __R128_DRV_H__
 
@@ -89,8 +91,6 @@ typedef struct drm_r128_private {
 
 	int usec_timeout;
 	int is_pci;
-	unsigned long phys_pci_gart;
-	dma_addr_t bus_pci_gart;
 	unsigned long cce_buffers_offset;
 
 	atomic_t idle_count;
@@ -121,6 +121,7 @@ typedef struct drm_r128_private {
 	drm_local_map_t *cce_ring;
 	drm_local_map_t *ring_rptr;
 	drm_local_map_t *agp_textures;
+	drm_ati_pcigart_info gart_info;
 } drm_r128_private_t;
 
 typedef struct drm_r128_buf_priv {
@@ -131,6 +132,9 @@ typedef struct drm_r128_buf_priv {
 	drm_r128_freelist_t *list_entry;
 } drm_r128_buf_priv_t;
 
+extern drm_ioctl_desc_t r128_ioctls[];
+extern int r128_max_ioctl;
+
 				/* r128_cce.c */
 extern int r128_cce_init(DRM_IOCTL_ARGS);
 extern int r128_cce_start(DRM_IOCTL_ARGS);
@@ -154,8 +158,11 @@ extern irqreturn_t r128_driver_irq_handler(DRM_IRQ_ARGS);
 extern void r128_driver_irq_preinstall(drm_device_t * dev);
 extern void r128_driver_irq_postinstall(drm_device_t * dev);
 extern void r128_driver_irq_uninstall(drm_device_t * dev);
-extern void r128_driver_pretakedown(drm_device_t * dev);
-extern void r128_driver_prerelease(drm_device_t * dev, DRMFILE filp);
+extern void r128_driver_lastclose(drm_device_t * dev);
+extern void r128_driver_preclose(drm_device_t * dev, DRMFILE filp);
+
+extern long r128_compat_ioctl(struct file *filp, unsigned int cmd,
+			      unsigned long arg);
 
 /* Register definitions, register access macros and drmAddMap constants
  * for Rage 128 kernel driver.
diff --git a/sys/dev/drm/r128_irq.c b/sys/dev/drm/r128_irq.c
index d40330517680..c3b4129995fa 100644
--- a/sys/dev/drm/r128_irq.c
+++ b/sys/dev/drm/r128_irq.c
@@ -1,4 +1,5 @@
-/* r128_irq.c -- IRQ handling for radeon -*- linux-c -*- */
+/* r128_irq.c -- IRQ handling for radeon -*- linux-c -*-
+ */
 /*-
  * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
  *
@@ -28,10 +29,11 @@
  * Authors:
  *    Keith Whitwell <keith@tungstengraphics.com>
  *    Eric Anholt <anholt@FreeBSD.org>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/r128_drm.h"
diff --git a/sys/dev/drm/r128_state.c b/sys/dev/drm/r128_state.c
index 90b50749e184..62f17e7b74b3 100644
--- a/sys/dev/drm/r128_state.c
+++ b/sys/dev/drm/r128_state.c
@@ -1,5 +1,6 @@
 /* r128_state.c -- State support for r128 -*- linux-c -*-
- * Created: Thu Jan 27 02:53:43 2000 by gareth@valinux.com */
+ * Created: Thu Jan 27 02:53:43 2000 by gareth@valinux.com
+ */
 /*-
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
@@ -25,10 +26,11 @@
  *
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/r128_drm.h"
@@ -1529,12 +1531,16 @@ static int r128_cce_depth(DRM_IOCTL_ARGS)
 	switch (depth.func) {
 	case R128_WRITE_SPAN:
 		ret = r128_cce_dispatch_write_span(dev, &depth);
+		break;
 	case R128_WRITE_PIXELS:
 		ret = r128_cce_dispatch_write_pixels(dev, &depth);
+		break;
 	case R128_READ_SPAN:
 		ret = r128_cce_dispatch_read_span(dev, &depth);
+		break;
 	case R128_READ_PIXELS:
 		ret = r128_cce_dispatch_read_pixels(dev, &depth);
+		break;
 	}
 
 	COMMIT_RING();
@@ -1672,7 +1678,7 @@ static int r128_getparam(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-void r128_driver_prerelease(drm_device_t * dev, DRMFILE filp)
+void r128_driver_preclose(drm_device_t * dev, DRMFILE filp)
 {
 	if (dev->dev_private) {
 		drm_r128_private_t *dev_priv = dev->dev_private;
@@ -1682,29 +1688,29 @@ void r128_driver_prerelease(drm_device_t * dev, DRMFILE filp)
 	}
 }
 
-void r128_driver_pretakedown(drm_device_t * dev)
+void r128_driver_lastclose(drm_device_t * dev)
 {
 	r128_do_cleanup_cce(dev);
 }
 
 drm_ioctl_desc_t r128_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_R128_INIT)] = {r128_cce_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_R128_CCE_START)] = {r128_cce_start, 1, 1},
-	[DRM_IOCTL_NR(DRM_R128_CCE_STOP)] = {r128_cce_stop, 1, 1},
-	[DRM_IOCTL_NR(DRM_R128_CCE_RESET)] = {r128_cce_reset, 1, 1},
-	[DRM_IOCTL_NR(DRM_R128_CCE_IDLE)] = {r128_cce_idle, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_RESET)] = {r128_engine_reset, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_FULLSCREEN)] = {r128_fullscreen, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_SWAP)] = {r128_cce_swap, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_FLIP)] = {r128_cce_flip, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_CLEAR)] = {r128_cce_clear, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_VERTEX)] = {r128_cce_vertex, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_INDICES)] = {r128_cce_indices, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_BLIT)] = {r128_cce_blit, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_DEPTH)] = {r128_cce_depth, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_STIPPLE)] = {r128_cce_stipple, 1, 0},
-	[DRM_IOCTL_NR(DRM_R128_INDIRECT)] = {r128_cce_indirect, 1, 1},
-	[DRM_IOCTL_NR(DRM_R128_GETPARAM)] = {r128_getparam, 1, 0},
+	[DRM_IOCTL_NR(DRM_R128_INIT)] = {r128_cce_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_R128_CCE_START)] = {r128_cce_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_R128_CCE_STOP)] = {r128_cce_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_R128_CCE_RESET)] = {r128_cce_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_R128_CCE_IDLE)] = {r128_cce_idle, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_RESET)] = {r128_engine_reset, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_FULLSCREEN)] = {r128_fullscreen, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_SWAP)] = {r128_cce_swap, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_FLIP)] = {r128_cce_flip, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_CLEAR)] = {r128_cce_clear, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_VERTEX)] = {r128_cce_vertex, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_INDICES)] = {r128_cce_indices, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_BLIT)] = {r128_cce_blit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_DEPTH)] = {r128_cce_depth, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_STIPPLE)] = {r128_cce_stipple, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_R128_INDIRECT)] = {r128_cce_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_R128_GETPARAM)] = {r128_getparam, DRM_AUTH},
 };
 
 int r128_max_ioctl = DRM_ARRAY_SIZE(r128_ioctls);
diff --git a/sys/dev/drm/r300_cmdbuf.c b/sys/dev/drm/r300_cmdbuf.c
index 131d7f8e7e25..c47bb3872024 100644
--- a/sys/dev/drm/r300_cmdbuf.c
+++ b/sys/dev/drm/r300_cmdbuf.c
@@ -29,10 +29,11 @@
  *
  * Authors:
  *    Nicolai Haehnle <prefect_@gmx.net>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/radeon_drm.h"
diff --git a/sys/dev/drm/r300_reg.h b/sys/dev/drm/r300_reg.h
index 64f2ab9ef073..3928495e715a 100644
--- a/sys/dev/drm/r300_reg.h
+++ b/sys/dev/drm/r300_reg.h
@@ -23,9 +23,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 **************************************************************************/
 
-/*
- * $FreeBSD$
- */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
 #ifndef _R300_REG_H
 #define _R300_REG_H
diff --git a/sys/dev/drm/radeon_cp.c b/sys/dev/drm/radeon_cp.c
index fb2c77937857..cb78288871f2 100644
--- a/sys/dev/drm/radeon_cp.c
+++ b/sys/dev/drm/radeon_cp.c
@@ -26,10 +26,11 @@
  * Authors:
  *    Kevin E. Martin <martin@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/radeon_drm.h"
@@ -1138,7 +1139,7 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 	} else
 #endif
 		ring_start = (dev_priv->cp_ring->offset
-			      - dev->sg->handle + dev_priv->gart_vm_start);
+			      - (unsigned long)dev->sg->virtual + dev_priv->gart_vm_start);
 
 
 	RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
@@ -1165,7 +1166,7 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 		drm_sg_mem_t *entry = dev->sg;
 		unsigned long tmp_ofs, page_ofs;
 
-		tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
+		tmp_ofs = dev_priv->ring_rptr->offset - (unsigned long)dev->sg->virtual;
 		page_ofs = tmp_ofs >> PAGE_SHIFT;
 
 		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
@@ -1208,6 +1209,10 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 		dev_priv->writeback_works = 0;
 		DRM_DEBUG("writeback test failed\n");
 	}
+	if (radeon_no_wb == 1) {
+		dev_priv->writeback_works = 0;
+		DRM_DEBUG("writeback forced off\n");
+	}
 
 	dev_priv->sarea_priv->last_frame = dev_priv->scratch[0] = 0;
 	RADEON_WRITE(RADEON_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
@@ -1247,25 +1252,27 @@ static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 	u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 	if (on) {
 
-		DRM_DEBUG("programming pcie %08X %08lX %08X\n", dev_priv->gart_vm_start, dev_priv->bus_pci_gart,dev_priv->gart_size);
+		DRM_DEBUG("programming pcie %08X %08lX %08X\n",
+			  dev_priv->gart_vm_start, (long)dev_priv->gart_info.bus_addr,
+			  dev_priv->gart_size);
 		RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO, dev_priv->gart_vm_start);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE, dev_priv->bus_pci_gart);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE, dev_priv->gart_info.bus_addr);
 		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO, dev_priv->gart_vm_start);
 		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO, dev_priv->gart_vm_start
 			     + dev_priv->gart_size - 1);
 
 		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
 
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, RADEON_PCIE_TX_GART_EN | RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD | RADEON_PCIE_TX_GART_CHK_RW_VALID_EN);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, RADEON_PCIE_TX_GART_EN);
 	} else {
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp & ~RADEON_PCIE_TX_GART_EN);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, (tmp & ~RADEON_PCIE_TX_GART_EN) | RADEON_PCIE_TX_GART_INVALIDATE_TLB);
 	}
 }
 
 /* Enable or disable PCI GART on the chip */
 static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 {
-	u32 tmp = RADEON_READ(RADEON_AIC_CNTL);
+	u32 tmp;
 
 	if (dev_priv->flags & CHIP_IS_PCIE)
 	{
@@ -1273,13 +1280,15 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 		return;
 	}
 
+ 	tmp = RADEON_READ(RADEON_AIC_CNTL);
+
 	if (on) {
 		RADEON_WRITE(RADEON_AIC_CNTL,
 			     tmp | RADEON_PCIGART_TRANSLATE_EN);
 
 		/* set PCI GART page-table base address
 		 */
-		RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->bus_pci_gart);
+		RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
 
 		/* set address range for PCI address translate
 		 */
@@ -1302,6 +1311,12 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	DRM_DEBUG("\n");
 
+	if (init->is_pci && (dev_priv->flags & CHIP_IS_AGP))
+	{
+		DRM_DEBUG("Forcing AGP card to PCI mode\n");
+		dev_priv->flags &= ~CHIP_IS_AGP;
+	}
+
 	if ((!(dev_priv->flags & CHIP_IS_AGP)) && !dev->sg) {
 		DRM_ERROR("PCI GART memory not allocated!\n");
 		radeon_do_cleanup_cp(dev);
@@ -1400,8 +1415,6 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 
 	DRM_GETSAREA();
 
-	dev_priv->fb_offset = init->fb_offset;
-	dev_priv->mmio_offset = init->mmio_offset;
 	dev_priv->ring_offset = init->ring_offset;
 	dev_priv->ring_rptr_offset = init->ring_rptr_offset;
 	dev_priv->buffers_offset = init->buffers_offset;
@@ -1413,12 +1426,6 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		return DRM_ERR(EINVAL);
 	}
 
-	dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
-	if (!dev_priv->mmio) {
-		DRM_ERROR("could not find mmio region!\n");
-		radeon_do_cleanup_cp(dev);
-		return DRM_ERR(EINVAL);
-	}
 	dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
 	if (!dev_priv->cp_ring) {
 		DRM_ERROR("could not find cp ring region!\n");
@@ -1431,6 +1438,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 		radeon_do_cleanup_cp(dev);
 		return DRM_ERR(EINVAL);
 	}
+	dev->agp_buffer_token = init->buffers_offset;
 	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
 	if (!dev->agp_buffer_map) {
 		DRM_ERROR("could not find dma buffer region!\n");
@@ -1508,7 +1516,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	else
 #endif
 		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-						 - dev->sg->handle
+						 - (unsigned long)dev->sg->virtual
 						 + dev_priv->gart_vm_start);
 
 	DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
@@ -1533,8 +1541,32 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	} else
 #endif
 	{
-		if (!drm_ati_pcigart_init(dev, &dev_priv->phys_pci_gart,
-					  &dev_priv->bus_pci_gart, (dev_priv->flags & CHIP_IS_PCIE))) {
+		/* if we have an offset set from userspace */
+		if (dev_priv->pcigart_offset) {
+			dev_priv->gart_info.bus_addr = dev_priv->pcigart_offset + dev_priv->fb_location;
+			dev_priv->gart_info.mapping.offset = dev_priv->gart_info.bus_addr;
+			dev_priv->gart_info.mapping.size = RADEON_PCIGART_TABLE_SIZE;
+			drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
+			dev_priv->gart_info.addr = dev_priv->gart_info.mapping.handle;
+
+			dev_priv->gart_info.is_pcie = !!(dev_priv->flags & CHIP_IS_PCIE);
+			dev_priv->gart_info.gart_table_location = DRM_ATI_GART_FB;
+			
+			DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n", dev_priv->gart_info.addr, dev_priv->pcigart_offset);
+		}
+		else {
+			dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
+			dev_priv->gart_info.addr = NULL;
+			dev_priv->gart_info.bus_addr = 0;
+			if (dev_priv->flags & CHIP_IS_PCIE)
+			{
+				DRM_ERROR("Cannot use PCI Express without GART in FB memory\n");
+				radeon_do_cleanup_cp(dev);
+				return DRM_ERR(EINVAL);
+			}
+		}
+
+		if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
 			DRM_ERROR("failed to init PCI GART!\n");
 			radeon_do_cleanup_cp(dev);
 			return DRM_ERR(ENOMEM);
@@ -1583,10 +1615,19 @@ static int radeon_do_cleanup_cp(drm_device_t * dev)
 	} else
 #endif
 	{
-		if (!drm_ati_pcigart_cleanup(dev,
-					     dev_priv->phys_pci_gart,
-					     dev_priv->bus_pci_gart))
-			DRM_ERROR("failed to cleanup PCI GART!\n");
+
+		if (dev_priv->gart_info.bus_addr) {
+			/* Turn off PCI GART */
+			radeon_set_pcigart(dev_priv, 0);
+			if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
+				DRM_ERROR("failed to cleanup PCI GART!\n");
+		}
+
+		if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
+		{
+			drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
+			dev_priv->gart_info.addr = 0;
+		}
 	}
 	/* only clear to the start of flags */
 	memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
@@ -1742,8 +1783,13 @@ void radeon_do_release(drm_device_t * dev)
 #ifdef __linux__
 				schedule();
 #else
+#if defined(__FreeBSD__) && __FreeBSD_version > 500000
+				msleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
+				       1);
+#else
 				tsleep(&ret, PZERO, "rdnrel", 1);
 #endif
+#endif
 			}
 			radeon_do_cp_stop(dev_priv);
 			radeon_do_engine_reset(dev);
@@ -2043,8 +2089,7 @@ int radeon_cp_buffers(DRM_IOCTL_ARGS)
 	return ret;
 }
 
-/* Always create a map record for MMIO and FB memory, done from DRIVER_POSTINIT */
-int radeon_preinit(struct drm_device *dev, unsigned long flags)
+int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 {
 	drm_radeon_private_t *dev_priv;
 	int ret = 0;
@@ -2070,28 +2115,6 @@ int radeon_preinit(struct drm_device *dev, unsigned long flags)
 		break;
 	}
 
-	/* Disable initmaps because it is broken on FreeBSD, and results in
-	 * crashes on startup for some.  The proper fix will involve being
-	 * smarter about allocating PCI resources.
-	 */
-	/*
-	ret = drm_initmap(dev, drm_get_resource_start(dev, 2),
-			  drm_get_resource_len(dev, 2), 2, _DRM_REGISTERS,
-			  _DRM_READ_ONLY);
-	if (ret != 0)
-		return ret;
-
-	ret = drm_initmap(dev, drm_get_resource_start(dev, 0),
-			  drm_get_resource_len(dev, 0), 0, _DRM_FRAME_BUFFER,
-			  _DRM_WRITE_COMBINING);
-	if (ret != 0)
-		return ret;
-	*/
-
-	/* The original method of detecting AGP is known to not work correctly,
-	 * according to Mike Harris.  The solution is to walk the capabilities
-	 * list, which should be done in drm_device_is_agp().
-	 */
 	if (drm_device_is_agp(dev))
 		dev_priv->flags |= CHIP_IS_AGP;
 
@@ -2101,26 +2124,38 @@ int radeon_preinit(struct drm_device *dev, unsigned long flags)
 	DRM_DEBUG("%s card detected\n",
 		  ((dev_priv->flags & CHIP_IS_AGP) ? "AGP" : (((dev_priv->flags & CHIP_IS_PCIE) ? "PCIE" : "PCI"))));
 
-#if defined(__linux__)
-	/* Check if we need a reset */
-	if (!
-	    (dev_priv->mmio =
-	     drm_core_findmap(dev, pci_resource_start(dev->pdev, 2))))
-		return DRM_ERR(ENOMEM);
-
-	ret = radeon_create_i2c_busses(dev);
-#endif
 	return ret;
 }
 
-int radeon_postcleanup(struct drm_device *dev)
+/* Create mappings for registers and framebuffer so userland doesn't necessarily
+ * have to find them.
+ */
+int radeon_driver_firstopen(struct drm_device *dev)
+{
+	int ret;
+	drm_local_map_t *map;
+	drm_radeon_private_t *dev_priv = dev->dev_private;
+
+	ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
+			 drm_get_resource_len(dev, 2), _DRM_REGISTERS,
+			 _DRM_READ_ONLY, &dev_priv->mmio);
+	if (ret != 0)
+		return ret;
+
+	ret = drm_addmap(dev, drm_get_resource_start(dev, 0),
+			 drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
+			 _DRM_WRITE_COMBINING, &map);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int radeon_driver_unload(struct drm_device *dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 
 	DRM_DEBUG("\n");
-#if defined(__linux__)
-	radeon_delete_i2c_busses(dev);
-#endif
 	drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
 
 	dev->dev_private = NULL;
diff --git a/sys/dev/drm/radeon_drm.h b/sys/dev/drm/radeon_drm.h
index 78d302d31a3b..ae0017382766 100644
--- a/sys/dev/drm/radeon_drm.h
+++ b/sys/dev/drm/radeon_drm.h
@@ -1,5 +1,5 @@
-/* radeon_drm.h -- Public header for the radeon driver -*- linux-c -*- */
-/*-
+/* radeon_drm.h -- Public header for the radeon driver -*- linux-c -*-
+ *
  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
  * Copyright 2002 Tungsten Graphics, Inc., Cedar Park, Texas.
@@ -28,10 +28,11 @@
  *    Kevin E. Martin <martin@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *    Keith Whitwell <keith@tungstengraphics.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __RADEON_DRM_H__
 #define __RADEON_DRM_H__
 
@@ -154,7 +155,16 @@
 #define RADEON_EMIT_PP_CUBIC_FACES_2                82
 #define RADEON_EMIT_PP_CUBIC_OFFSETS_T2             83
 #define R200_EMIT_PP_TRI_PERF_CNTL                  84
-#define RADEON_MAX_STATE_PACKETS                    85
+#define R200_EMIT_PP_AFS_0                          85
+#define R200_EMIT_PP_AFS_1                          86
+#define R200_EMIT_ATF_TFACTOR                       87
+#define R200_EMIT_PP_TXCTLALL_0                     88
+#define R200_EMIT_PP_TXCTLALL_1                     89
+#define R200_EMIT_PP_TXCTLALL_2                     90
+#define R200_EMIT_PP_TXCTLALL_3                     91
+#define R200_EMIT_PP_TXCTLALL_4                     92
+#define R200_EMIT_PP_TXCTLALL_5                     93
+#define RADEON_MAX_STATE_PACKETS                    94
 
 /* Commands understood by cmd_buffer ioctl.  More can be added but
  * obviously these can't be removed or changed:
@@ -494,7 +504,7 @@ typedef struct drm_radeon_init {
 		RADEON_INIT_R300_CP = 0x04
 	} func;
 	unsigned long sarea_priv_offset;
-	int is_pci;		/* not used, driver asks hardware */
+	int is_pci; /* for overriding only */
 	int cp_mode;
 	int gart_size;
 	int ring_size;
@@ -506,8 +516,8 @@ typedef struct drm_radeon_init {
 	unsigned int depth_bpp;
 	unsigned int depth_offset, depth_pitch;
 
-	unsigned long fb_offset;
-	unsigned long mmio_offset;
+	unsigned long fb_offset DEPRECATED;	/* deprecated, driver asks hardware */
+	unsigned long mmio_offset DEPRECATED;	/* deprecated, driver asks hardware */
 	unsigned long ring_offset;
 	unsigned long ring_rptr_offset;
 	unsigned long buffers_offset;
@@ -683,6 +693,7 @@ typedef struct drm_radeon_setparam {
 
 #define RADEON_SETPARAM_FB_LOCATION    1	/* determined framebuffer location */
 #define RADEON_SETPARAM_SWITCH_TILING  2	/* enable/disable color tiling */
+#define RADEON_SETPARAM_PCIGART_LOCATION 3      /* PCI Gart Location */
 
 /* 1.14: Clients can allocate/free a surface
  */
diff --git a/sys/dev/drm/radeon_drv.c b/sys/dev/drm/radeon_drv.c
index a515780988d8..8ac798eac83c 100644
--- a/sys/dev/drm/radeon_drv.c
+++ b/sys/dev/drm/radeon_drv.c
@@ -27,56 +27,58 @@
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/radeon_drm.h"
 #include "dev/drm/radeon_drv.h"
 #include "dev/drm/drm_pciids.h"
 
+int radeon_no_wb;
+
 /* drv_PCI_IDs comes from drm_pciids.h, generated from drm_pciids.txt. */
 static drm_pci_id_list_t radeon_pciidlist[] = {
 	radeon_PCI_IDS
 };
 
-extern drm_ioctl_desc_t radeon_ioctls[];
-extern int radeon_max_ioctl;
-
 static void radeon_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = sizeof(drm_radeon_buf_priv_t);
-	dev->preinit = radeon_preinit;
-	dev->postcleanup = radeon_postcleanup;
-	dev->prerelease = radeon_driver_prerelease;
-	dev->pretakedown = radeon_driver_pretakedown;
-	dev->open_helper = radeon_driver_open_helper;
-	dev->free_filp_priv = radeon_driver_free_filp_priv;
-	dev->vblank_wait = radeon_driver_vblank_wait;
-	dev->irq_preinstall = radeon_driver_irq_preinstall;
-	dev->irq_postinstall = radeon_driver_irq_postinstall;
-	dev->irq_uninstall = radeon_driver_irq_uninstall;
-	dev->irq_handler = radeon_driver_irq_handler;
-	dev->dma_ioctl = radeon_cp_buffers;
-
-	dev->driver_ioctls = radeon_ioctls;
-	dev->max_driver_ioctl = radeon_max_ioctl;
-
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
-
-	dev->use_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_pci_dma = 1;
-	dev->use_sg = 1;
-	dev->use_dma = 1;
-	dev->use_irq = 1;
-	dev->use_vbl_irq = 1;
+	dev->driver.buf_priv_size	= sizeof(drm_radeon_buf_priv_t);
+	dev->driver.load		= radeon_driver_load;
+	dev->driver.unload		= radeon_driver_unload;
+	dev->driver.firstopen		= radeon_driver_firstopen;
+	dev->driver.open		= radeon_driver_open;
+	dev->driver.preclose		= radeon_driver_preclose;
+	dev->driver.postclose		= radeon_driver_postclose;
+	dev->driver.lastclose		= radeon_driver_lastclose;
+	dev->driver.vblank_wait		= radeon_driver_vblank_wait;
+	dev->driver.irq_preinstall	= radeon_driver_irq_preinstall;
+	dev->driver.irq_postinstall	= radeon_driver_irq_postinstall;
+	dev->driver.irq_uninstall	= radeon_driver_irq_uninstall;
+	dev->driver.irq_handler		= radeon_driver_irq_handler;
+	dev->driver.dma_ioctl		= radeon_cp_buffers;
+
+	dev->driver.ioctls		= radeon_ioctls;
+	dev->driver.max_ioctl		= radeon_max_ioctl;
+
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
+
+	dev->driver.use_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_pci_dma		= 1;
+	dev->driver.use_sg		= 1;
+	dev->driver.use_dma		= 1;
+	dev->driver.use_irq		= 1;
+	dev->driver.use_vbl_irq		= 1;
 }
 
 #ifdef __FreeBSD__
@@ -116,5 +118,10 @@ DRIVER_MODULE(radeon, pci, radeon_driver, drm_devclass, 0, 0);
 MODULE_DEPEND(radeon, drm, 1, 1, 1);
 
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
+#ifdef _LKM
 CFDRIVER_DECL(radeon, DV_TTY, NULL);
+#else
+CFATTACH_DECL(radeon, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+    drm_activate);
+#endif
 #endif /* __FreeBSD__ */
diff --git a/sys/dev/drm/radeon_drv.h b/sys/dev/drm/radeon_drv.h
index 1032cab0f586..bd52c0736bdf 100644
--- a/sys/dev/drm/radeon_drv.h
+++ b/sys/dev/drm/radeon_drv.h
@@ -1,5 +1,5 @@
-/* radeon_drv.h -- Private header for radeon driver -*- linux-c -*- */
-/*-
+/* radeon_drv.h -- Private header for radeon driver -*- linux-c -*-
+ *
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
  * All rights reserved.
@@ -26,17 +26,14 @@
  * Authors:
  *    Kevin E. Martin <martin@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __RADEON_DRV_H__
 #define __RADEON_DRV_H__
 
-#ifdef __linux__
-#include "radeon_i2c.h"
-#endif /* __linux__ */
-
 /* General customization:
  */
 
@@ -44,7 +41,7 @@
 
 #define DRIVER_NAME		"radeon"
 #define DRIVER_DESC		"ATI Radeon"
-#define DRIVER_DATE		"20050311"
+#define DRIVER_DATE		"20050911"
 
 /* Interface history:
  *
@@ -88,18 +85,23 @@
  *     - Add support for r100 cube maps
  * 1.16- Add R200_EMIT_PP_TRI_PERF_CNTL packet to support brilinear
  *       texture filtering on r200
+ * 1.17- Add initial support for R300 (3D).
+ * 1.18- Add support for GL_ATI_fragment_shader, new packets R200_EMIT_PP_AFS_0/1,
+         R200_EMIT_PP_TXCTLALL_0-5 (replaces R200_EMIT_PP_TXFILTER_0-5, 2 more regs)
+         and R200_EMIT_ATF_TFACTOR (replaces R200_EMIT_TFACTOR_0 (8 consts instead of 6)
+ * 1.19- Add support for gart table in FB memory and PCIE r300
  */
 
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		16
+#define DRIVER_MINOR		19
 #define DRIVER_PATCHLEVEL	0
 
 enum radeon_family {
 	CHIP_R100,
 	CHIP_RS100,
 	CHIP_RV100,
-	CHIP_R200,
 	CHIP_RV200,
+	CHIP_R200,
 	CHIP_RS200,
 	CHIP_R250,
 	CHIP_RS250,
@@ -213,9 +215,6 @@ typedef struct drm_radeon_private {
 
 	int microcode_version;
 
-	unsigned long phys_pci_gart;
-	dma_addr_t bus_pci_gart;
-
 	struct {
 		u32 boxes;
 		int freelist_timeouts;
@@ -247,8 +246,6 @@ typedef struct drm_radeon_private {
 
 	drm_radeon_depth_clear_t depth_clear;
 
-	unsigned long fb_offset;
-	unsigned long mmio_offset;
 	unsigned long ring_offset;
 	unsigned long ring_rptr_offset;
 	unsigned long buffers_offset;
@@ -269,19 +266,22 @@ typedef struct drm_radeon_private {
 
 	struct radeon_surface surfaces[RADEON_MAX_SURFACES];
 	struct radeon_virt_surface virt_surfaces[2*RADEON_MAX_SURFACES];
-	
+
+	unsigned long pcigart_offset;
+	drm_ati_pcigart_info gart_info;
 	/* starting from here on, data is preserved accross an open */
 	uint32_t flags;		/* see radeon_chip_flags */
 
-#ifdef __linux__
-	struct radeon_i2c_chan i2c[4];
-#endif /* __linux__ */
 } drm_radeon_private_t;
 
 typedef struct drm_radeon_buf_priv {
 	u32 age;
 } drm_radeon_buf_priv_t;
 
+extern int radeon_no_wb;
+extern drm_ioctl_desc_t radeon_ioctls[];
+extern int radeon_max_ioctl;
+
 				/* radeon_cp.c */
 extern int radeon_cp_init(DRM_IOCTL_ARGS);
 extern int radeon_cp_start(DRM_IOCTL_ARGS);
@@ -317,12 +317,16 @@ extern irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS);
 extern void radeon_driver_irq_preinstall(drm_device_t * dev);
 extern void radeon_driver_irq_postinstall(drm_device_t * dev);
 extern void radeon_driver_irq_uninstall(drm_device_t * dev);
-extern void radeon_driver_prerelease(drm_device_t * dev, DRMFILE filp);
-extern void radeon_driver_pretakedown(drm_device_t * dev);
-extern int radeon_driver_open_helper(drm_device_t * dev,
-				     drm_file_t * filp_priv);
-extern void radeon_driver_free_filp_priv(drm_device_t * dev,
-					 drm_file_t * filp_priv);
+
+extern int radeon_driver_load(struct drm_device *dev, unsigned long flags);
+extern int radeon_driver_unload(struct drm_device *dev);
+extern int radeon_driver_firstopen(struct drm_device *dev);
+extern void radeon_driver_preclose(drm_device_t * dev, DRMFILE filp);
+extern void radeon_driver_postclose(drm_device_t * dev, drm_file_t * filp);
+extern void radeon_driver_lastclose(drm_device_t * dev);
+extern int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv);
+extern long radeon_compat_ioctl(struct file *filp, unsigned int cmd,
+					 unsigned long arg);
 
 /* r300_cmdbuf.c */
 extern void r300_init_reg_flags(void);
@@ -876,6 +880,39 @@ extern int r300_do_cp_cmdbuf( drm_device_t* dev,
 
 #define R200_PP_TRI_PERF                  0x2cf8
 
+#define R200_PP_AFS_0                     0x2f80
+#define R200_PP_AFS_1                     0x2f00 /* same as txcblend_0 */
+
+/* MPEG settings from VHA code */
+#define RADEON_VHA_SETTO16_1                       0x2694
+#define RADEON_VHA_SETTO16_2                       0x2680
+#define RADEON_VHA_SETTO0_1                        0x1840
+#define RADEON_VHA_FB_OFFSET                       0x19e4
+#define RADEON_VHA_SETTO1AND70S                    0x19d8
+#define RADEON_VHA_DST_PITCH                       0x1408
+
+// set as reference header
+#define RADEON_VHA_BACKFRAME0_OFF_Y              0x1840
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_Y        0x1844
+#define RADEON_VHA_BACKFRAME0_OFF_U              0x1848
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_U        0x184c
+#define RADOEN_VHA_BACKFRAME0_OFF_V              0x1850
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_V        0x1854
+#define RADEON_VHA_FORWFRAME0_OFF_Y              0x1858
+#define RADEON_VHA_FORWFRAME1_OFF_PITCH_Y        0x185c
+#define RADEON_VHA_FORWFRAME0_OFF_U              0x1860
+#define RADEON_VHA_FORWFRAME1_OFF_PITCH_U        0x1864
+#define RADEON_VHA_FORWFRAME0_OFF_V              0x1868
+#define RADEON_VHA_FORWFRAME0_OFF_PITCH_V        0x1880
+#define RADEON_VHA_BACKFRAME0_OFF_Y_2            0x1884
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_Y_2      0x1888
+#define RADEON_VHA_BACKFRAME0_OFF_U_2            0x188c
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_U_2      0x1890
+#define RADEON_VHA_BACKFRAME0_OFF_V_2            0x1894
+#define RADEON_VHA_BACKFRAME1_OFF_PITCH_V_2      0x1898
+
+
+
 /* Constants */
 #define RADEON_MAX_USEC_TIMEOUT		100000	/* 100 ms */
 
@@ -890,6 +927,8 @@ extern int r300_do_cp_cmdbuf( drm_device_t* dev,
 
 #define RADEON_RING_HIGH_MARK		128
 
+#define RADEON_PCIGART_TABLE_SIZE      (32*1024)
+
 #define RADEON_READ(reg)	DRM_READ32(  dev_priv->mmio, (reg) )
 #define RADEON_WRITE(reg,val)	DRM_WRITE32( dev_priv->mmio, (reg), (val) )
 #define RADEON_READ8(reg)	DRM_READ8(  dev_priv->mmio, (reg) )
@@ -909,9 +948,6 @@ do {									\
 	RADEON_WRITE( RADEON_PCIE_DATA, (val) );			\
 } while (0)
 
-extern int radeon_preinit(struct drm_device *dev, unsigned long flags);
-extern int radeon_postcleanup(struct drm_device *dev);
-
 #define CP_PACKET0( reg, n )						\
 	(RADEON_CP_PACKET0 | ((n) << 16) | ((reg) >> 2))
 #define CP_PACKET0_TABLE( reg, n )					\
diff --git a/sys/dev/drm/radeon_irq.c b/sys/dev/drm/radeon_irq.c
index 56be38a1139b..420fc84c1d38 100644
--- a/sys/dev/drm/radeon_irq.c
+++ b/sys/dev/drm/radeon_irq.c
@@ -28,15 +28,25 @@
  * Authors:
  *    Keith Whitwell <keith@tungstengraphics.com>
  *    Michel D�zer <michel@daenzer.net>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/radeon_drm.h"
 #include "dev/drm/radeon_drv.h"
 
+static __inline__ u32 radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv,
+					      u32 mask)
+{
+	u32 irqs = RADEON_READ(RADEON_GEN_INT_STATUS) & mask;
+	if (irqs)
+		RADEON_WRITE(RADEON_GEN_INT_STATUS, irqs);
+	return irqs;
+}
+
 /* Interrupts - Used for device synchronization and flushing in the
  * following circumstances:
  *
@@ -65,8 +75,8 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS)
 	/* Only consider the bits we're interested in - others could be used
 	 * outside the DRM
 	 */
-	stat = RADEON_READ(RADEON_GEN_INT_STATUS)
-	    & (RADEON_SW_INT_TEST | RADEON_CRTC_VBLANK_STAT);
+	stat = radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
+						  RADEON_CRTC_VBLANK_STAT));
 	if (!stat)
 		return IRQ_NONE;
 
@@ -82,19 +92,9 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS)
 		drm_vbl_send_signals(dev);
 	}
 
-	/* Acknowledge interrupts we handle */
-	RADEON_WRITE(RADEON_GEN_INT_STATUS, stat);
 	return IRQ_HANDLED;
 }
 
-static __inline__ void radeon_acknowledge_irqs(drm_radeon_private_t * dev_priv)
-{
-	u32 tmp = RADEON_READ(RADEON_GEN_INT_STATUS)
-	    & (RADEON_SW_INT_TEST_ACK | RADEON_CRTC_VBLANK_STAT);
-	if (tmp)
-		RADEON_WRITE(RADEON_GEN_INT_STATUS, tmp);
-}
-
 static int radeon_emit_irq(drm_device_t * dev)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -124,11 +124,6 @@ static int radeon_wait_irq(drm_device_t * dev, int swi_nr)
 
 	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 
-	/* This is a hack to work around mysterious freezes on certain
-	 * systems:
-	 */
-	radeon_acknowledge_irqs(dev_priv);
-
 	DRM_WAIT_ON(ret, dev_priv->swi_queue, 3 * DRM_HZ,
 		    RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr);
 
@@ -147,7 +142,7 @@ int radeon_driver_vblank_wait(drm_device_t * dev, unsigned int *sequence)
 		return DRM_ERR(EINVAL);
 	}
 
-	radeon_acknowledge_irqs(dev_priv);
+	radeon_acknowledge_irqs(dev_priv, RADEON_CRTC_VBLANK_STAT);
 
 	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 
@@ -223,7 +218,8 @@ void radeon_driver_irq_preinstall(drm_device_t * dev)
 	RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 
 	/* Clear bits if they're already high */
-	radeon_acknowledge_irqs(dev_priv);
+	radeon_acknowledge_irqs(dev_priv, (RADEON_SW_INT_TEST_ACK |
+					   RADEON_CRTC_VBLANK_STAT));
 }
 
 void radeon_driver_irq_postinstall(drm_device_t * dev)
diff --git a/sys/dev/drm/radeon_mem.c b/sys/dev/drm/radeon_mem.c
index 89c07afe3551..bb9ff302bb57 100644
--- a/sys/dev/drm/radeon_mem.c
+++ b/sys/dev/drm/radeon_mem.c
@@ -27,10 +27,11 @@
  *
  * Authors:
  *    Keith Whitwell <keith@tungstengraphics.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/radeon_drm.h"
diff --git a/sys/dev/drm/radeon_state.c b/sys/dev/drm/radeon_state.c
index 7d42df669751..23d55003ef4f 100644
--- a/sys/dev/drm/radeon_state.c
+++ b/sys/dev/drm/radeon_state.c
@@ -25,10 +25,11 @@
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
  *    Kevin E. Martin <martin@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/drm_sarea.h"
@@ -212,6 +213,15 @@ static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
 	case RADEON_EMIT_PP_CUBIC_FACES_1:
 	case RADEON_EMIT_PP_CUBIC_FACES_2:
 	case R200_EMIT_PP_TRI_PERF_CNTL:
+	case R200_EMIT_PP_AFS_0:
+	case R200_EMIT_PP_AFS_1:
+	case R200_EMIT_ATF_TFACTOR:
+	case R200_EMIT_PP_TXCTLALL_0:
+	case R200_EMIT_PP_TXCTLALL_1:
+	case R200_EMIT_PP_TXCTLALL_2:
+	case R200_EMIT_PP_TXCTLALL_3:
+	case R200_EMIT_PP_TXCTLALL_4:
+	case R200_EMIT_PP_TXCTLALL_5:
 		/* These packets don't contain memory offsets */
 		break;
 
@@ -492,100 +502,103 @@ static struct {
 	int len;
 	const char *name;
 } packet[RADEON_MAX_STATE_PACKETS] = {
-	{
-	RADEON_PP_MISC, 7, "RADEON_PP_MISC"}, {
-	RADEON_PP_CNTL, 3, "RADEON_PP_CNTL"}, {
-	RADEON_RB3D_COLORPITCH, 1, "RADEON_RB3D_COLORPITCH"}, {
-	RADEON_RE_LINE_PATTERN, 2, "RADEON_RE_LINE_PATTERN"}, {
-	RADEON_SE_LINE_WIDTH, 1, "RADEON_SE_LINE_WIDTH"}, {
-	RADEON_PP_LUM_MATRIX, 1, "RADEON_PP_LUM_MATRIX"}, {
-	RADEON_PP_ROT_MATRIX_0, 2, "RADEON_PP_ROT_MATRIX_0"}, {
-	RADEON_RB3D_STENCILREFMASK, 3, "RADEON_RB3D_STENCILREFMASK"}, {
-	RADEON_SE_VPORT_XSCALE, 6, "RADEON_SE_VPORT_XSCALE"}, {
-	RADEON_SE_CNTL, 2, "RADEON_SE_CNTL"}, {
-	RADEON_SE_CNTL_STATUS, 1, "RADEON_SE_CNTL_STATUS"}, {
-	RADEON_RE_MISC, 1, "RADEON_RE_MISC"}, {
-	RADEON_PP_TXFILTER_0, 6, "RADEON_PP_TXFILTER_0"}, {
-	RADEON_PP_BORDER_COLOR_0, 1, "RADEON_PP_BORDER_COLOR_0"}, {
-	RADEON_PP_TXFILTER_1, 6, "RADEON_PP_TXFILTER_1"}, {
-	RADEON_PP_BORDER_COLOR_1, 1, "RADEON_PP_BORDER_COLOR_1"}, {
-	RADEON_PP_TXFILTER_2, 6, "RADEON_PP_TXFILTER_2"}, {
-	RADEON_PP_BORDER_COLOR_2, 1, "RADEON_PP_BORDER_COLOR_2"}, {
-	RADEON_SE_ZBIAS_FACTOR, 2, "RADEON_SE_ZBIAS_FACTOR"}, {
-	RADEON_SE_TCL_OUTPUT_VTX_FMT, 11, "RADEON_SE_TCL_OUTPUT_VTX_FMT"}, {
-	RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 17,
-		    "RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED"}, {
-	R200_PP_TXCBLEND_0, 4, "R200_PP_TXCBLEND_0"}, {
-	R200_PP_TXCBLEND_1, 4, "R200_PP_TXCBLEND_1"}, {
-	R200_PP_TXCBLEND_2, 4, "R200_PP_TXCBLEND_2"}, {
-	R200_PP_TXCBLEND_3, 4, "R200_PP_TXCBLEND_3"}, {
-	R200_PP_TXCBLEND_4, 4, "R200_PP_TXCBLEND_4"}, {
-	R200_PP_TXCBLEND_5, 4, "R200_PP_TXCBLEND_5"}, {
-	R200_PP_TXCBLEND_6, 4, "R200_PP_TXCBLEND_6"}, {
-	R200_PP_TXCBLEND_7, 4, "R200_PP_TXCBLEND_7"}, {
-	R200_SE_TCL_LIGHT_MODEL_CTL_0, 6, "R200_SE_TCL_LIGHT_MODEL_CTL_0"},
-	{
-	R200_PP_TFACTOR_0, 6, "R200_PP_TFACTOR_0"}, {
-	R200_SE_VTX_FMT_0, 4, "R200_SE_VTX_FMT_0"}, {
-	R200_SE_VAP_CNTL, 1, "R200_SE_VAP_CNTL"}, {
-	R200_SE_TCL_MATRIX_SEL_0, 5, "R200_SE_TCL_MATRIX_SEL_0"}, {
-	R200_SE_TCL_TEX_PROC_CTL_2, 5, "R200_SE_TCL_TEX_PROC_CTL_2"}, {
-	R200_SE_TCL_UCP_VERT_BLEND_CTL, 1, "R200_SE_TCL_UCP_VERT_BLEND_CTL"},
-	{
-	R200_PP_TXFILTER_0, 6, "R200_PP_TXFILTER_0"}, {
-	R200_PP_TXFILTER_1, 6, "R200_PP_TXFILTER_1"}, {
-	R200_PP_TXFILTER_2, 6, "R200_PP_TXFILTER_2"}, {
-	R200_PP_TXFILTER_3, 6, "R200_PP_TXFILTER_3"}, {
-	R200_PP_TXFILTER_4, 6, "R200_PP_TXFILTER_4"}, {
-	R200_PP_TXFILTER_5, 6, "R200_PP_TXFILTER_5"}, {
-	R200_PP_TXOFFSET_0, 1, "R200_PP_TXOFFSET_0"}, {
-	R200_PP_TXOFFSET_1, 1, "R200_PP_TXOFFSET_1"}, {
-	R200_PP_TXOFFSET_2, 1, "R200_PP_TXOFFSET_2"}, {
-	R200_PP_TXOFFSET_3, 1, "R200_PP_TXOFFSET_3"}, {
-	R200_PP_TXOFFSET_4, 1, "R200_PP_TXOFFSET_4"}, {
-	R200_PP_TXOFFSET_5, 1, "R200_PP_TXOFFSET_5"}, {
-	R200_SE_VTE_CNTL, 1, "R200_SE_VTE_CNTL"}, {
-	R200_SE_TCL_OUTPUT_VTX_COMP_SEL, 1, "R200_SE_TCL_OUTPUT_VTX_COMP_SEL"},
-	{
-	R200_PP_TAM_DEBUG3, 1, "R200_PP_TAM_DEBUG3"}, {
-	R200_PP_CNTL_X, 1, "R200_PP_CNTL_X"}, {
-	R200_RB3D_DEPTHXY_OFFSET, 1, "R200_RB3D_DEPTHXY_OFFSET"}, {
-	R200_RE_AUX_SCISSOR_CNTL, 1, "R200_RE_AUX_SCISSOR_CNTL"}, {
-	R200_RE_SCISSOR_TL_0, 2, "R200_RE_SCISSOR_TL_0"}, {
-	R200_RE_SCISSOR_TL_1, 2, "R200_RE_SCISSOR_TL_1"}, {
-	R200_RE_SCISSOR_TL_2, 2, "R200_RE_SCISSOR_TL_2"}, {
-	R200_SE_VAP_CNTL_STATUS, 1, "R200_SE_VAP_CNTL_STATUS"}, {
-	R200_SE_VTX_STATE_CNTL, 1, "R200_SE_VTX_STATE_CNTL"}, {
-	R200_RE_POINTSIZE, 1, "R200_RE_POINTSIZE"}, {
-	R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0, 4,
-		    "R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0"}, {
-	R200_PP_CUBIC_FACES_0, 1, "R200_PP_CUBIC_FACES_0"},	/* 61 */
-	{
-	R200_PP_CUBIC_OFFSET_F1_0, 5, "R200_PP_CUBIC_OFFSET_F1_0"},	/* 62 */
-	{
-	R200_PP_CUBIC_FACES_1, 1, "R200_PP_CUBIC_FACES_1"}, {
-	R200_PP_CUBIC_OFFSET_F1_1, 5, "R200_PP_CUBIC_OFFSET_F1_1"}, {
-	R200_PP_CUBIC_FACES_2, 1, "R200_PP_CUBIC_FACES_2"}, {
-	R200_PP_CUBIC_OFFSET_F1_2, 5, "R200_PP_CUBIC_OFFSET_F1_2"}, {
-	R200_PP_CUBIC_FACES_3, 1, "R200_PP_CUBIC_FACES_3"}, {
-	R200_PP_CUBIC_OFFSET_F1_3, 5, "R200_PP_CUBIC_OFFSET_F1_3"}, {
-	R200_PP_CUBIC_FACES_4, 1, "R200_PP_CUBIC_FACES_4"}, {
-	R200_PP_CUBIC_OFFSET_F1_4, 5, "R200_PP_CUBIC_OFFSET_F1_4"}, {
-	R200_PP_CUBIC_FACES_5, 1, "R200_PP_CUBIC_FACES_5"}, {
-	R200_PP_CUBIC_OFFSET_F1_5, 5, "R200_PP_CUBIC_OFFSET_F1_5"}, {
-	RADEON_PP_TEX_SIZE_0, 2, "RADEON_PP_TEX_SIZE_0"}, {
-	RADEON_PP_TEX_SIZE_1, 2, "RADEON_PP_TEX_SIZE_1"}, {
-	RADEON_PP_TEX_SIZE_2, 2, "RADEON_PP_TEX_SIZE_2"}, {
-	R200_RB3D_BLENDCOLOR, 3, "R200_RB3D_BLENDCOLOR"}, {
-	R200_SE_TCL_POINT_SPRITE_CNTL, 1, "R200_SE_TCL_POINT_SPRITE_CNTL"},
-	{
-	RADEON_PP_CUBIC_FACES_0, 1, "RADEON_PP_CUBIC_FACES_0"}, {
-	RADEON_PP_CUBIC_OFFSET_T0_0, 5, "RADEON_PP_CUBIC_OFFSET_T0_0"}, {
-	RADEON_PP_CUBIC_FACES_1, 1, "RADEON_PP_CUBIC_FACES_1"}, {
-	RADEON_PP_CUBIC_OFFSET_T1_0, 5, "RADEON_PP_CUBIC_OFFSET_T1_0"}, {
-	RADEON_PP_CUBIC_FACES_2, 1, "RADEON_PP_CUBIC_FACES_2"}, {
-	RADEON_PP_CUBIC_OFFSET_T2_0, 5, "RADEON_PP_CUBIC_OFFSET_T2_0"}, {
-	R200_PP_TRI_PERF, 2, "R200_PP_TRI_PERF"},
+	{RADEON_PP_MISC, 7, "RADEON_PP_MISC"},
+	{RADEON_PP_CNTL, 3, "RADEON_PP_CNTL"},
+	{RADEON_RB3D_COLORPITCH, 1, "RADEON_RB3D_COLORPITCH"},
+	{RADEON_RE_LINE_PATTERN, 2, "RADEON_RE_LINE_PATTERN"},
+	{RADEON_SE_LINE_WIDTH, 1, "RADEON_SE_LINE_WIDTH"},
+	{RADEON_PP_LUM_MATRIX, 1, "RADEON_PP_LUM_MATRIX"},
+	{RADEON_PP_ROT_MATRIX_0, 2, "RADEON_PP_ROT_MATRIX_0"},
+	{RADEON_RB3D_STENCILREFMASK, 3, "RADEON_RB3D_STENCILREFMASK"},
+	{RADEON_SE_VPORT_XSCALE, 6, "RADEON_SE_VPORT_XSCALE"},
+	{RADEON_SE_CNTL, 2, "RADEON_SE_CNTL"},
+	{RADEON_SE_CNTL_STATUS, 1, "RADEON_SE_CNTL_STATUS"},
+	{RADEON_RE_MISC, 1, "RADEON_RE_MISC"},
+	{RADEON_PP_TXFILTER_0, 6, "RADEON_PP_TXFILTER_0"},
+	{RADEON_PP_BORDER_COLOR_0, 1, "RADEON_PP_BORDER_COLOR_0"},
+	{RADEON_PP_TXFILTER_1, 6, "RADEON_PP_TXFILTER_1"},
+	{RADEON_PP_BORDER_COLOR_1, 1, "RADEON_PP_BORDER_COLOR_1"},
+	{RADEON_PP_TXFILTER_2, 6, "RADEON_PP_TXFILTER_2"},
+	{RADEON_PP_BORDER_COLOR_2, 1, "RADEON_PP_BORDER_COLOR_2"},
+	{RADEON_SE_ZBIAS_FACTOR, 2, "RADEON_SE_ZBIAS_FACTOR"},
+	{RADEON_SE_TCL_OUTPUT_VTX_FMT, 11, "RADEON_SE_TCL_OUTPUT_VTX_FMT"},
+	{RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 17,
+	 "RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED"},
+	{R200_PP_TXCBLEND_0, 4, "R200_PP_TXCBLEND_0"},
+	{R200_PP_TXCBLEND_1, 4, "R200_PP_TXCBLEND_1"},
+	{R200_PP_TXCBLEND_2, 4, "R200_PP_TXCBLEND_2"},
+	{R200_PP_TXCBLEND_3, 4, "R200_PP_TXCBLEND_3"},
+	{R200_PP_TXCBLEND_4, 4, "R200_PP_TXCBLEND_4"},
+	{R200_PP_TXCBLEND_5, 4, "R200_PP_TXCBLEND_5"},
+	{R200_PP_TXCBLEND_6, 4, "R200_PP_TXCBLEND_6"},
+	{R200_PP_TXCBLEND_7, 4, "R200_PP_TXCBLEND_7"},
+	{R200_SE_TCL_LIGHT_MODEL_CTL_0, 6, "R200_SE_TCL_LIGHT_MODEL_CTL_0"},
+	{R200_PP_TFACTOR_0, 6, "R200_PP_TFACTOR_0"},
+	{R200_SE_VTX_FMT_0, 4, "R200_SE_VTX_FMT_0"},
+	{R200_SE_VAP_CNTL, 1, "R200_SE_VAP_CNTL"},
+	{R200_SE_TCL_MATRIX_SEL_0, 5, "R200_SE_TCL_MATRIX_SEL_0"},
+	{R200_SE_TCL_TEX_PROC_CTL_2, 5, "R200_SE_TCL_TEX_PROC_CTL_2"},
+	{R200_SE_TCL_UCP_VERT_BLEND_CTL, 1, "R200_SE_TCL_UCP_VERT_BLEND_CTL"},
+	{R200_PP_TXFILTER_0, 6, "R200_PP_TXFILTER_0"},
+	{R200_PP_TXFILTER_1, 6, "R200_PP_TXFILTER_1"},
+	{R200_PP_TXFILTER_2, 6, "R200_PP_TXFILTER_2"},
+	{R200_PP_TXFILTER_3, 6, "R200_PP_TXFILTER_3"},
+	{R200_PP_TXFILTER_4, 6, "R200_PP_TXFILTER_4"},
+	{R200_PP_TXFILTER_5, 6, "R200_PP_TXFILTER_5"},
+	{R200_PP_TXOFFSET_0, 1, "R200_PP_TXOFFSET_0"},
+	{R200_PP_TXOFFSET_1, 1, "R200_PP_TXOFFSET_1"},
+	{R200_PP_TXOFFSET_2, 1, "R200_PP_TXOFFSET_2"},
+	{R200_PP_TXOFFSET_3, 1, "R200_PP_TXOFFSET_3"},
+	{R200_PP_TXOFFSET_4, 1, "R200_PP_TXOFFSET_4"},
+	{R200_PP_TXOFFSET_5, 1, "R200_PP_TXOFFSET_5"},
+	{R200_SE_VTE_CNTL, 1, "R200_SE_VTE_CNTL"},
+	{R200_SE_TCL_OUTPUT_VTX_COMP_SEL, 1,
+	 "R200_SE_TCL_OUTPUT_VTX_COMP_SEL"},
+	{R200_PP_TAM_DEBUG3, 1, "R200_PP_TAM_DEBUG3"},
+	{R200_PP_CNTL_X, 1, "R200_PP_CNTL_X"},
+	{R200_RB3D_DEPTHXY_OFFSET, 1, "R200_RB3D_DEPTHXY_OFFSET"},
+	{R200_RE_AUX_SCISSOR_CNTL, 1, "R200_RE_AUX_SCISSOR_CNTL"},
+	{R200_RE_SCISSOR_TL_0, 2, "R200_RE_SCISSOR_TL_0"},
+	{R200_RE_SCISSOR_TL_1, 2, "R200_RE_SCISSOR_TL_1"},
+	{R200_RE_SCISSOR_TL_2, 2, "R200_RE_SCISSOR_TL_2"},
+	{R200_SE_VAP_CNTL_STATUS, 1, "R200_SE_VAP_CNTL_STATUS"},
+	{R200_SE_VTX_STATE_CNTL, 1, "R200_SE_VTX_STATE_CNTL"},
+	{R200_RE_POINTSIZE, 1, "R200_RE_POINTSIZE"},
+	{R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0, 4,
+	 "R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0"},
+	{R200_PP_CUBIC_FACES_0, 1, "R200_PP_CUBIC_FACES_0"},	/* 61 */
+	{R200_PP_CUBIC_OFFSET_F1_0, 5, "R200_PP_CUBIC_OFFSET_F1_0"}, /* 62 */
+	{R200_PP_CUBIC_FACES_1, 1, "R200_PP_CUBIC_FACES_1"},
+	{R200_PP_CUBIC_OFFSET_F1_1, 5, "R200_PP_CUBIC_OFFSET_F1_1"},
+	{R200_PP_CUBIC_FACES_2, 1, "R200_PP_CUBIC_FACES_2"},
+	{R200_PP_CUBIC_OFFSET_F1_2, 5, "R200_PP_CUBIC_OFFSET_F1_2"},
+	{R200_PP_CUBIC_FACES_3, 1, "R200_PP_CUBIC_FACES_3"},
+	{R200_PP_CUBIC_OFFSET_F1_3, 5, "R200_PP_CUBIC_OFFSET_F1_3"},
+	{R200_PP_CUBIC_FACES_4, 1, "R200_PP_CUBIC_FACES_4"},
+	{R200_PP_CUBIC_OFFSET_F1_4, 5, "R200_PP_CUBIC_OFFSET_F1_4"},
+	{R200_PP_CUBIC_FACES_5, 1, "R200_PP_CUBIC_FACES_5"},
+	{R200_PP_CUBIC_OFFSET_F1_5, 5, "R200_PP_CUBIC_OFFSET_F1_5"},
+	{RADEON_PP_TEX_SIZE_0, 2, "RADEON_PP_TEX_SIZE_0"},
+	{RADEON_PP_TEX_SIZE_1, 2, "RADEON_PP_TEX_SIZE_1"},
+	{RADEON_PP_TEX_SIZE_2, 2, "RADEON_PP_TEX_SIZE_2"},
+	{R200_RB3D_BLENDCOLOR, 3, "R200_RB3D_BLENDCOLOR"},
+	{R200_SE_TCL_POINT_SPRITE_CNTL, 1, "R200_SE_TCL_POINT_SPRITE_CNTL"},
+	{RADEON_PP_CUBIC_FACES_0, 1, "RADEON_PP_CUBIC_FACES_0"},
+	{RADEON_PP_CUBIC_OFFSET_T0_0, 5, "RADEON_PP_CUBIC_OFFSET_T0_0"},
+	{RADEON_PP_CUBIC_FACES_1, 1, "RADEON_PP_CUBIC_FACES_1"},
+	{RADEON_PP_CUBIC_OFFSET_T1_0, 5, "RADEON_PP_CUBIC_OFFSET_T1_0"},
+	{RADEON_PP_CUBIC_FACES_2, 1, "RADEON_PP_CUBIC_FACES_2"},
+	{RADEON_PP_CUBIC_OFFSET_T2_0, 5, "RADEON_PP_CUBIC_OFFSET_T2_0"},
+	{R200_PP_TRI_PERF, 2, "R200_PP_TRI_PERF"},
+	{R200_PP_AFS_0, 32, "R200_PP_AFS_0"},     /* 85 */
+	{R200_PP_AFS_1, 32, "R200_PP_AFS_1"},
+	{R200_PP_TFACTOR_0, 8, "R200_ATF_TFACTOR"},
+	{R200_PP_TXFILTER_0, 8, "R200_PP_TXCTLALL_0"},
+	{R200_PP_TXFILTER_1, 8, "R200_PP_TXCTLALL_1"},
+	{R200_PP_TXFILTER_2, 8, "R200_PP_TXCTLALL_2"},
+	{R200_PP_TXFILTER_3, 8, "R200_PP_TXCTLALL_3"},
+	{R200_PP_TXFILTER_4, 8, "R200_PP_TXCTLALL_4"},
+	{R200_PP_TXFILTER_5, 8, "R200_PP_TXCTLALL_5"},
 };
 
 /* ================================================================
@@ -2084,7 +2097,7 @@ static int radeon_cp_vertex(DRM_IOCTL_ARGS)
 	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_file_t *filp_priv;
-	drm_radeon_sarea_t *sarea_priv;
+	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_t *buf;
 	drm_radeon_vertex_t vertex;
@@ -2096,7 +2109,6 @@ static int radeon_cp_vertex(DRM_IOCTL_ARGS)
 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
 		return DRM_ERR(EINVAL);
 	}
-	sarea_priv = dev_priv->sarea_priv;
 
 	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
 
@@ -2173,7 +2185,7 @@ static int radeon_cp_indices(DRM_IOCTL_ARGS)
 	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_file_t *filp_priv;
-	drm_radeon_sarea_t *sarea_priv;
+	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_t *buf;
 	drm_radeon_indices_t elts;
@@ -2186,7 +2198,6 @@ static int radeon_cp_indices(DRM_IOCTL_ARGS)
 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
 		return DRM_ERR(EINVAL);
 	}
-	sarea_priv = dev_priv->sarea_priv;
 
 	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
 
@@ -2402,7 +2413,7 @@ static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
 	DRM_DEVICE;
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	drm_file_t *filp_priv;
-	drm_radeon_sarea_t *sarea_priv;
+	drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_t *buf;
 	drm_radeon_vertex2_t vertex;
@@ -2415,7 +2426,6 @@ static int radeon_cp_vertex2(DRM_IOCTL_ARGS)
 		DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
 		return DRM_ERR(EINVAL);
 	}
-	sarea_priv = dev_priv->sarea_priv;
 
 	DRM_GET_PRIV_WITH_RETURN(filp_priv, filp);
 
@@ -2922,7 +2932,7 @@ static int radeon_cp_getparam(DRM_IOCTL_ARGS)
 		value = dev_priv->gart_vm_start;
 		break;
 	case RADEON_PARAM_REGISTER_HANDLE:
-		value = dev_priv->mmio_offset;
+		value = dev_priv->mmio->offset;
 		break;
 	case RADEON_PARAM_STATUS_HANDLE:
 		value = dev_priv->ring_rptr_offset;
@@ -2994,6 +3004,9 @@ static int radeon_cp_setparam(DRM_IOCTL_ARGS)
 			dev_priv->sarea_priv->tiling_enabled = 1;
 		}
 		break;
+	case RADEON_SETPARAM_PCIGART_LOCATION:
+		dev_priv->pcigart_offset = sp.value;
+		break;
 	default:
 		DRM_DEBUG("Invalid parameter %d\n", sp.param);
 		return DRM_ERR(EINVAL);
@@ -3009,7 +3022,7 @@ static int radeon_cp_setparam(DRM_IOCTL_ARGS)
  *
  * DRM infrastructure takes care of reclaiming dma buffers.
  */
-void radeon_driver_prerelease(drm_device_t * dev, DRMFILE filp)
+void radeon_driver_preclose(drm_device_t * dev, DRMFILE filp)
 {
 	if (dev->dev_private) {
 		drm_radeon_private_t *dev_priv = dev->dev_private;
@@ -3022,12 +3035,12 @@ void radeon_driver_prerelease(drm_device_t * dev, DRMFILE filp)
 	}
 }
 
-void radeon_driver_pretakedown(drm_device_t * dev)
+void radeon_driver_lastclose(drm_device_t * dev)
 {
 	radeon_do_release(dev);
 }
 
-int radeon_driver_open_helper(drm_device_t * dev, drm_file_t * filp_priv)
+int radeon_driver_open(drm_device_t * dev, drm_file_t * filp_priv)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
 	struct drm_radeon_driver_file_fields *radeon_priv;
@@ -3049,7 +3062,7 @@ int radeon_driver_open_helper(drm_device_t * dev, drm_file_t * filp_priv)
 	return 0;
 }
 
-void radeon_driver_free_filp_priv(drm_device_t * dev, drm_file_t * filp_priv)
+void radeon_driver_postclose(drm_device_t * dev, drm_file_t * filp_priv)
 {
 	struct drm_radeon_driver_file_fields *radeon_priv =
 	    filp_priv->driver_priv;
@@ -3058,33 +3071,33 @@ void radeon_driver_free_filp_priv(drm_device_t * dev, drm_file_t * filp_priv)
 }
 
 drm_ioctl_desc_t radeon_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_RADEON_CP_INIT)] = {radeon_cp_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_START)] = {radeon_cp_start, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_STOP)] = {radeon_cp_stop, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_RESET)] = {radeon_cp_reset, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_IDLE)] = {radeon_cp_idle, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_CP_RESUME)] = {radeon_cp_resume, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_RESET)] = {radeon_engine_reset, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_FULLSCREEN)] = {radeon_fullscreen, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_SWAP)] = {radeon_cp_swap, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_CLEAR)] = {radeon_cp_clear, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_VERTEX)] = {radeon_cp_vertex, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_INDICES)] = {radeon_cp_indices, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_TEXTURE)] = {radeon_cp_texture, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_STIPPLE)] = {radeon_cp_stipple, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_INDIRECT)] = {radeon_cp_indirect, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_VERTEX2)] = {radeon_cp_vertex2, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_CMDBUF)] = {radeon_cp_cmdbuf, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_GETPARAM)] = {radeon_cp_getparam, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_FLIP)] = {radeon_cp_flip, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_ALLOC)] = {radeon_mem_alloc, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_FREE)] = {radeon_mem_free, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_INIT_HEAP)] = {radeon_mem_init_heap, 1, 1},
-	[DRM_IOCTL_NR(DRM_RADEON_IRQ_EMIT)] = {radeon_irq_emit, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_IRQ_WAIT)] = {radeon_irq_wait, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_SETPARAM)] = {radeon_cp_setparam, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_SURF_ALLOC)] = {radeon_surface_alloc, 1, 0},
-	[DRM_IOCTL_NR(DRM_RADEON_SURF_FREE)] = {radeon_surface_free, 1, 0}
+	[DRM_IOCTL_NR(DRM_RADEON_CP_INIT)] = {radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_CP_START)] = {radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_CP_STOP)] = {radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_CP_RESET)] = {radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_CP_IDLE)] = {radeon_cp_idle, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_CP_RESUME)] = {radeon_cp_resume, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_RESET)] = {radeon_engine_reset, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_FULLSCREEN)] = {radeon_fullscreen, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_SWAP)] = {radeon_cp_swap, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_CLEAR)] = {radeon_cp_clear, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_VERTEX)] = {radeon_cp_vertex, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_INDICES)] = {radeon_cp_indices, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_TEXTURE)] = {radeon_cp_texture, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_STIPPLE)] = {radeon_cp_stipple, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_INDIRECT)] = {radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_VERTEX2)] = {radeon_cp_vertex2, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_CMDBUF)] = {radeon_cp_cmdbuf, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_GETPARAM)] = {radeon_cp_getparam, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_FLIP)] = {radeon_cp_flip, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_ALLOC)] = {radeon_mem_alloc, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_FREE)] = {radeon_mem_free, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_INIT_HEAP)] = {radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_RADEON_IRQ_EMIT)] = {radeon_irq_emit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_IRQ_WAIT)] = {radeon_irq_wait, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_SETPARAM)] = {radeon_cp_setparam, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_SURF_ALLOC)] = {radeon_surface_alloc, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_RADEON_SURF_FREE)] = {radeon_surface_free, DRM_AUTH}
 };
 
 int radeon_max_ioctl = DRM_ARRAY_SIZE(radeon_ioctls);
diff --git a/sys/dev/drm/savage_bci.c b/sys/dev/drm/savage_bci.c
index f1a844a4d92c..34380856c8a7 100644
--- a/sys/dev/drm/savage_bci.c
+++ b/sys/dev/drm/savage_bci.c
@@ -21,13 +21,13 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
-#include "drmP.h"
-#include "savage_drm.h"
-#include "savage_drv.h"
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#include "dev/drm/drmP.h"
+#include "dev/drm/savage_drm.h"
+#include "dev/drm/savage_drv.h"
 
 /* Need a long timeout for shadow status updates can take a while
  * and so can waiting for events when the queue is full. */
@@ -165,8 +165,8 @@ savage_bci_wait_event_reg(drm_savage_private_t *dev_priv, uint16_t e)
 uint16_t savage_bci_emit_event(drm_savage_private_t *dev_priv,
 			       unsigned int flags)
 {
-	BCI_LOCALS;
 	uint16_t count;
+	BCI_LOCALS;
 
 	if (dev_priv->status_ptr) {
 		/* coordinate with Xserver */
@@ -419,12 +419,12 @@ uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv, unsigned int n)
 
 static void savage_dma_flush(drm_savage_private_t *dev_priv)
 {
-	BCI_LOCALS;
 	unsigned int first = dev_priv->first_dma_page;
 	unsigned int cur = dev_priv->current_dma_page;
 	uint16_t event;
 	unsigned int wrap, pad, align, len, i;
 	unsigned long phys_addr;
+	BCI_LOCALS;
 
 	if (first == cur &&
 	    dev_priv->dma_pages[cur].used == dev_priv->dma_pages[cur].flushed)
@@ -504,8 +504,9 @@ static void savage_dma_flush(drm_savage_private_t *dev_priv)
 
 static void savage_fake_dma_flush(drm_savage_private_t *dev_priv)
 {
-	BCI_LOCALS;
 	unsigned int i, j;
+	BCI_LOCALS;
+
 	if (dev_priv->first_dma_page == dev_priv->current_dma_page &&
 	    dev_priv->dma_pages[dev_priv->current_dma_page].used == 0)
 		return;
@@ -538,21 +539,9 @@ static void savage_fake_dma_flush(drm_savage_private_t *dev_priv)
 	dev_priv->first_dma_page = dev_priv->current_dma_page = 0;
 }
 
-/*
- * Initalize permanent mappings. On Savage4 and SavageIX the alignment
- * and size of the aperture is not suitable for automatic MTRR setup
- * in drm_initmap. Therefore we do it manually before the maps are
- * initialized. We also need to take care of deleting the MTRRs in
- * postcleanup.
- *
- * FIXME: this is linux-specific
- */
-int savage_preinit(drm_device_t *dev, unsigned long chipset)
+int savage_driver_load(drm_device_t *dev, unsigned long chipset)
 {
 	drm_savage_private_t *dev_priv;
-	unsigned long mmio_base, fb_base, fb_size, aperture_base;
-	unsigned int fb_rsrc, aper_rsrc;
-	int ret = 0;
 
 	dev_priv = drm_alloc(sizeof(drm_savage_private_t), DRM_MEM_DRIVER);
 	if (dev_priv == NULL)
@@ -560,8 +549,29 @@ int savage_preinit(drm_device_t *dev, unsigned long chipset)
 
 	memset(dev_priv, 0, sizeof(drm_savage_private_t));
 	dev->dev_private = (void *)dev_priv;
+
 	dev_priv->chipset = (enum savage_family)chipset;
 
+	return 0;
+}
+
+/*
+ * Initalize mappings. On Savage4 and SavageIX the alignment
+ * and size of the aperture is not suitable for automatic MTRR setup
+ * in drm_addmap. Therefore we add them manually before the maps are
+ * initialized, and tear them down on last close.
+ */
+int savage_driver_firstopen(drm_device_t *dev)
+{
+	drm_savage_private_t *dev_priv = dev->dev_private;
+	unsigned long mmio_base, fb_base, fb_size, aperture_base;
+	/* fb_rsrc and aper_rsrc aren't really used currently, but still exist
+	 * in case we decide we need information on the BAR for BSD in the
+	 * future.
+	 */
+	unsigned int fb_rsrc, aper_rsrc;
+	int ret = 0;
+
 	dev_priv->mtrr[0].handle = -1;
 	dev_priv->mtrr[1].handle = -1;
 	dev_priv->mtrr[2].handle = -1;
@@ -578,24 +588,25 @@ int savage_preinit(drm_device_t *dev, unsigned long chipset)
 			 * MTRRs. */
 			dev_priv->mtrr[0].base = fb_base;
 			dev_priv->mtrr[0].size = 0x01000000;
-			dev_priv->mtrr[0].handle = mtrr_add(
+			dev_priv->mtrr[0].handle = drm_mtrr_add(
 				dev_priv->mtrr[0].base, dev_priv->mtrr[0].size,
-				MTRR_TYPE_WRCOMB, 1);
+				DRM_MTRR_WC);
 			dev_priv->mtrr[1].base = fb_base+0x02000000;
 			dev_priv->mtrr[1].size = 0x02000000;
-			dev_priv->mtrr[1].handle = mtrr_add(
+			dev_priv->mtrr[1].handle = drm_mtrr_add(
 				dev_priv->mtrr[1].base, dev_priv->mtrr[1].size,
-				MTRR_TYPE_WRCOMB, 1);
+				DRM_MTRR_WC);
 			dev_priv->mtrr[2].base = fb_base+0x04000000;
 			dev_priv->mtrr[2].size = 0x04000000;
-			dev_priv->mtrr[2].handle = mtrr_add(
+			dev_priv->mtrr[2].handle = drm_mtrr_add(
 				dev_priv->mtrr[2].base, dev_priv->mtrr[2].size,
-				MTRR_TYPE_WRCOMB, 1);
+				DRM_MTRR_WC);
 		} else {
 			DRM_ERROR("strange pci_resource_len %08lx\n",
 				  drm_get_resource_len(dev, 0));
 		}
-	} else if (chipset != S3_SUPERSAVAGE && chipset != S3_SAVAGE2000) {
+	} else if (dev_priv->chipset != S3_SUPERSAVAGE &&
+		   dev_priv->chipset != S3_SAVAGE2000) {
 		mmio_base = drm_get_resource_start(dev, 0);
 		fb_rsrc = 1;
 		fb_base = drm_get_resource_start(dev, 1);
@@ -608,9 +619,9 @@ int savage_preinit(drm_device_t *dev, unsigned long chipset)
 			 * aperture. */
 			dev_priv->mtrr[0].base = fb_base;
 			dev_priv->mtrr[0].size = 0x08000000;
-			dev_priv->mtrr[0].handle = mtrr_add(
+			dev_priv->mtrr[0].handle = drm_mtrr_add(
 				dev_priv->mtrr[0].base, dev_priv->mtrr[0].size,
-				MTRR_TYPE_WRCOMB, 1);
+				DRM_MTRR_WC);
 		} else {
 			DRM_ERROR("strange pci_resource_len %08lx\n",
 				  drm_get_resource_len(dev, 1));
@@ -625,24 +636,21 @@ int savage_preinit(drm_device_t *dev, unsigned long chipset)
 		/* Automatic MTRR setup will do the right thing. */
 	}
 
-	if ((ret = drm_initmap(dev, mmio_base, SAVAGE_MMIO_SIZE, 0,
-			       _DRM_REGISTERS, 0)))
+	ret = drm_addmap(dev, mmio_base, SAVAGE_MMIO_SIZE, _DRM_REGISTERS,
+			 _DRM_READ_ONLY, &dev_priv->mmio);
+	if (ret)
 		return ret;
-	if (!(dev_priv->mmio = drm_core_findmap (dev, mmio_base)))
-		return DRM_ERR(ENOMEM);
 
-	if ((ret = drm_initmap(dev, fb_base, fb_size, fb_rsrc,
-			       _DRM_FRAME_BUFFER, _DRM_WRITE_COMBINING)))
+	ret = drm_addmap(dev, fb_base, fb_size, _DRM_FRAME_BUFFER,
+			 _DRM_WRITE_COMBINING, &dev_priv->fb);
+	if (ret)
 		return ret;
-	if (!(dev_priv->fb = drm_core_findmap (dev, fb_base)))
-		return DRM_ERR(ENOMEM);
 
-	if ((ret = drm_initmap(dev, aperture_base, SAVAGE_APERTURE_SIZE,
-			       aper_rsrc,
-			       _DRM_FRAME_BUFFER, _DRM_WRITE_COMBINING)))
+	ret = drm_addmap(dev, aperture_base, SAVAGE_APERTURE_SIZE,
+			 _DRM_FRAME_BUFFER, _DRM_WRITE_COMBINING,
+			 &dev_priv->aperture);
+	if (ret)
 		return ret;
-	if (!(dev_priv->aperture = drm_core_findmap (dev, aperture_base)))
-		return DRM_ERR(ENOMEM);
 
 	return ret;
 }
@@ -650,16 +658,22 @@ int savage_preinit(drm_device_t *dev, unsigned long chipset)
 /*
  * Delete MTRRs and free device-private data.
  */
-int savage_postcleanup(drm_device_t *dev)
+void savage_driver_lastclose(drm_device_t *dev)
 {
 	drm_savage_private_t *dev_priv = dev->dev_private;
 	int i;
 
 	for (i = 0; i < 3; ++i)
 		if (dev_priv->mtrr[i].handle >= 0)
-			mtrr_del(dev_priv->mtrr[i].handle,
-				 dev_priv->mtrr[i].base,
-				 dev_priv->mtrr[i].size);
+			drm_mtrr_del(dev_priv->mtrr[i].handle,
+				     dev_priv->mtrr[i].base,
+				     dev_priv->mtrr[i].size,
+				     DRM_MTRR_WC);
+}
+
+int savage_driver_unload(drm_device_t *dev)
+{
+	drm_savage_private_t *dev_priv = dev->dev_private;
 
 	drm_free(dev_priv, sizeof(drm_savage_private_t), DRM_MEM_DRIVER);
 
@@ -916,7 +930,7 @@ int savage_do_cleanup_bci(drm_device_t *dev)
 	return 0;
 }
 
-int savage_bci_init(DRM_IOCTL_ARGS)
+static int savage_bci_init(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_savage_init_t init;
@@ -936,7 +950,7 @@ int savage_bci_init(DRM_IOCTL_ARGS)
 	return DRM_ERR(EINVAL);
 }
 
-int savage_bci_event_emit(DRM_IOCTL_ARGS)
+static int savage_bci_event_emit(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_savage_private_t *dev_priv = dev->dev_private;
@@ -951,12 +965,12 @@ int savage_bci_event_emit(DRM_IOCTL_ARGS)
 
 	event.count = savage_bci_emit_event(dev_priv, event.flags);
 	event.count |= dev_priv->event_wrap << 16;
-	DRM_COPY_TO_USER_IOCTL(&((drm_savage_event_emit_t __user *)data)->count,
-			       event.count, sizeof(event.count));
+	DRM_COPY_TO_USER_IOCTL((drm_savage_event_emit_t __user *)data,
+			       event, sizeof(event));
 	return 0;
 }
 
-int savage_bci_event_wait(DRM_IOCTL_ARGS)
+static int savage_bci_event_wait(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_savage_private_t *dev_priv = dev->dev_private;
@@ -1087,3 +1101,13 @@ void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp) {
 
 	drm_core_reclaim_buffers(dev, filp);
 }
+
+drm_ioctl_desc_t savage_ioctls[] = {
+	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_INIT)] = {savage_bci_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_CMDBUF)] = {savage_bci_cmdbuf, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_EVENT_EMIT)] = {savage_bci_event_emit, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SAVAGE_BCI_EVENT_WAIT)] = {savage_bci_event_wait, DRM_AUTH},
+};
+
+int savage_max_ioctl = DRM_ARRAY_SIZE(savage_ioctls);
+
diff --git a/sys/dev/drm/savage_drm.h b/sys/dev/drm/savage_drm.h
index b6a9fde1c083..b0e0a04a200a 100644
--- a/sys/dev/drm/savage_drm.h
+++ b/sys/dev/drm/savage_drm.h
@@ -21,10 +21,11 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __SAVAGE_DRM_H__
 #define __SAVAGE_DRM_H__
 
diff --git a/sys/dev/drm/savage_drv.c b/sys/dev/drm/savage_drv.c
index 4d02e8d8e55a..98a8e91dfcf4 100644
--- a/sys/dev/drm/savage_drv.c
+++ b/sys/dev/drm/savage_drv.c
@@ -24,46 +24,46 @@
  *
  * Authors:
  *    Eric Anholt <anholt@FreeBSD.org>
- *
- * $FreeBSD$
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "savage_drm.h"
-#include "savage_drv.h"
-#include "drm_pciids.h"
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "dev/drm/drmP.h"
+#include "dev/drm/drm.h"
+#include "dev/drm/savage_drm.h"
+#include "dev/drm/savage_drv.h"
+#include "dev/drm/drm_pciids.h"
 
 /* drv_PCI_IDs comes from drm_pciids.h, generated from drm_pciids.txt. */
 static drm_pci_id_list_t savage_pciidlist[] = {
 	savage_PCI_IDS
 };
 
-extern drm_ioctl_desc_t savage_ioctls[];
-extern int savage_max_ioctl;
-
 static void savage_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = sizeof(drm_savage_buf_priv_t);
-	dev->preinit = savage_preinit;
-	dev->postcleanup = savage_postcleanup;
-	dev->reclaim_buffers = savage_reclaim_buffers;
-	dev->dma_ioctl = savage_bci_buffers;
+	dev->driver.buf_priv_size	= sizeof(drm_savage_buf_priv_t);
+	dev->driver.load		= savage_driver_load;
+	dev->driver.firstopen		= savage_driver_firstopen;
+	dev->driver.lastclose		= savage_driver_lastclose;
+	dev->driver.unload		= savage_driver_unload;
+	dev->driver.reclaim_buffers_locked = savage_reclaim_buffers;
+	dev->driver.dma_ioctl		= savage_bci_buffers;
 
-	dev->driver_ioctls = savage_ioctls;
-	dev->max_driver_ioctl = savage_max_ioctl;
+	dev->driver.ioctls		= savage_ioctls;
+	dev->driver.max_ioctl		= savage_max_ioctl;
 
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
 
-	dev->use_agp = 1;
-	dev->use_mtrr = 1;
-	dev->use_pci_dma = 1;
-	dev->use_dma = 1;
+	dev->driver.use_agp		= 1;
+	dev->driver.use_mtrr		= 1;
+	dev->driver.use_pci_dma		= 1;
+	dev->driver.use_dma		= 1;
 }
 
 #ifdef __FreeBSD__
diff --git a/sys/dev/drm/savage_drv.h b/sys/dev/drm/savage_drv.h
index d3d43a91592a..af3ccc86a1bd 100644
--- a/sys/dev/drm/savage_drv.h
+++ b/sys/dev/drm/savage_drv.h
@@ -1,5 +1,5 @@
-/* savage_drv.h -- Private header for the savage driver
- *
+/* savage_drv.h -- Private header for the savage driver */
+/*-
  * Copyright 2004  Felix Kuehling
  * All Rights Reserved.
  *
@@ -21,10 +21,11 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __SAVAGE_DRV_H__
 #define __SAVAGE_DRV_H__
 
@@ -106,6 +107,9 @@ enum savage_family {
 	S3_LAST
 };
 
+extern drm_ioctl_desc_t savage_ioctls[];
+extern int savage_max_ioctl;
+
 #define S3_SAVAGE3D_SERIES(chip)  ((chip>=S3_SAVAGE3D) && (chip<=S3_SAVAGE_MX))
 
 #define S3_SAVAGE4_SERIES(chip)  ((chip==S3_SAVAGE4)            \
@@ -191,15 +195,12 @@ typedef struct drm_savage_private {
 	/* Err, there is a macro wait_event in include/linux/wait.h.
 	 * Avoid unwanted macro expansion. */
 	void (*emit_clip_rect)(struct drm_savage_private *dev_priv,
-			       drm_clip_rect_t *pbox);
+			       const drm_clip_rect_t *pbox);
 	void (*dma_flush)(struct drm_savage_private *dev_priv);
 } drm_savage_private_t;
 
 /* ioctls */
-extern int savage_bci_init(DRM_IOCTL_ARGS);
 extern int savage_bci_cmdbuf(DRM_IOCTL_ARGS);
-extern int savage_bci_event_emit(DRM_IOCTL_ARGS);
-extern int savage_bci_event_wait(DRM_IOCTL_ARGS);
 extern int savage_bci_buffers(DRM_IOCTL_ARGS);
 
 /* BCI functions */
@@ -210,16 +211,18 @@ extern void savage_dma_reset(drm_savage_private_t *dev_priv);
 extern void savage_dma_wait(drm_savage_private_t *dev_priv, unsigned int page);
 extern uint32_t *savage_dma_alloc(drm_savage_private_t *dev_priv,
 				  unsigned int n);
-extern int savage_preinit(drm_device_t *dev, unsigned long chipset);
-extern int savage_postcleanup(drm_device_t *dev);
+extern int savage_driver_load(drm_device_t *dev, unsigned long chipset);
+extern int savage_driver_firstopen(drm_device_t *dev);
+extern void savage_driver_lastclose(drm_device_t *dev);
+extern int savage_driver_unload(drm_device_t *dev);
 extern int savage_do_cleanup_bci(drm_device_t *dev);
 extern void savage_reclaim_buffers(drm_device_t *dev, DRMFILE filp);
 
 /* state functions */
 extern void savage_emit_clip_rect_s3d(drm_savage_private_t *dev_priv,
-				      drm_clip_rect_t *pbox);
+				      const drm_clip_rect_t *pbox);
 extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
-				     drm_clip_rect_t *pbox);
+				     const drm_clip_rect_t *pbox);
 
 #define SAVAGE_FB_SIZE_S3	0x01000000	/*  16MB */
 #define SAVAGE_FB_SIZE_S4	0x02000000	/*  32MB */
@@ -502,15 +505,6 @@ extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
 
 #define BCI_WRITE( val ) *bci_ptr++ = (uint32_t)(val)
 
-#define BCI_COPY_FROM_USER(src,n) do {				\
-    unsigned int i;						\
-    for (i = 0; i < n; ++i) {					\
-	uint32_t val;						\
-	DRM_GET_USER_UNCHECKED(val, &((uint32_t*)(src))[i]);	\
-	BCI_WRITE(val);						\
-    }								\
-} while(0)
-
 /*
  * command DMA support
  */
@@ -536,8 +530,8 @@ extern void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
 
 #define DMA_WRITE( val ) *dma_ptr++ = (uint32_t)(val)
 
-#define DMA_COPY_FROM_USER(src,n) do {				\
-	DRM_COPY_FROM_USER_UNCHECKED(dma_ptr, (src), (n)*4);	\
+#define DMA_COPY(src, n) do {					\
+	memcpy(dma_ptr, (src), (n)*4);				\
 	dma_ptr += n;						\
 } while(0)
 
diff --git a/sys/dev/drm/savage_state.c b/sys/dev/drm/savage_state.c
index 5181cbdb0057..edff2a9a38f6 100644
--- a/sys/dev/drm/savage_state.c
+++ b/sys/dev/drm/savage_state.c
@@ -21,16 +21,16 @@
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD$
  */
 
-#include "drmP.h"
-#include "savage_drm.h"
-#include "savage_drv.h"
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#include "dev/drm/drmP.h"
+#include "dev/drm/savage_drm.h"
+#include "dev/drm/savage_drv.h"
 
 void savage_emit_clip_rect_s3d(drm_savage_private_t *dev_priv,
-			       drm_clip_rect_t *pbox)
+			       const drm_clip_rect_t *pbox)
 {
 	uint32_t scstart = dev_priv->state.s3d.new_scstart;
 	uint32_t scend   = dev_priv->state.s3d.new_scend;
@@ -56,7 +56,7 @@ void savage_emit_clip_rect_s3d(drm_savage_private_t *dev_priv,
 }
 
 void savage_emit_clip_rect_s4(drm_savage_private_t *dev_priv,
-			      drm_clip_rect_t *pbox)
+			      const drm_clip_rect_t *pbox)
 {
 	uint32_t drawctrl0 = dev_priv->state.s4.new_drawctrl0;
 	uint32_t drawctrl1 = dev_priv->state.s4.new_drawctrl1;
@@ -116,18 +116,18 @@ static int savage_verify_texaddr(drm_savage_private_t *dev_priv, int unit,
 
 #define SAVE_STATE(reg,where)			\
 	if(start <= reg && start+count > reg)	\
-		DRM_GET_USER_UNCHECKED(dev_priv->state.where, &regs[reg-start])
+		dev_priv->state.where = regs[reg - start]
 #define SAVE_STATE_MASK(reg,where,mask) do {			\
 	if(start <= reg && start+count > reg) {			\
 		uint32_t tmp;					\
-		DRM_GET_USER_UNCHECKED(tmp, &regs[reg-start]);	\
+		tmp = regs[reg - start];			\
 		dev_priv->state.where = (tmp & (mask)) |	\
 			(dev_priv->state.where & ~(mask));	\
 	}							\
 } while (0)
 static int savage_verify_state_s3d(drm_savage_private_t *dev_priv,
 				   unsigned int start, unsigned int count,
-				   const uint32_t __user *regs)
+				   const uint32_t *regs)
 {
 	if (start < SAVAGE_TEXPALADDR_S3D ||
 	    start+count-1 > SAVAGE_DESTTEXRWWATERMARK_S3D) {
@@ -157,7 +157,7 @@ static int savage_verify_state_s3d(drm_savage_private_t *dev_priv,
 
 static int savage_verify_state_s4(drm_savage_private_t *dev_priv,
 				  unsigned int start, unsigned int count,
-				  const uint32_t __user *regs)
+				  const uint32_t *regs)
 {
 	int ret = 0;
 
@@ -195,21 +195,18 @@ static int savage_verify_state_s4(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_state(drm_savage_private_t *dev_priv,
 				 const drm_savage_cmd_header_t *cmd_header,
-				 const uint32_t __user *regs)
+				 const uint32_t *regs)
 {
-	DMA_LOCALS;
 	unsigned int count = cmd_header->state.count;
 	unsigned int start = cmd_header->state.start;
 	unsigned int count2 = 0;
 	unsigned int bci_size;
 	int ret;
+	DMA_LOCALS;
 
 	if (!count)
 		return 0;
 
-	if (DRM_VERIFYAREA_READ(regs, count*4))
-		return DRM_ERR(EFAULT);
-
 	if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
 		ret = savage_verify_state_s3d(dev_priv, start, count, regs);
 		if (ret != 0)
@@ -260,7 +257,7 @@ static int savage_dispatch_state(drm_savage_private_t *dev_priv,
 		while (count > 0) {
 			unsigned int n = count < 255 ? count : 255;
 			DMA_SET_REGISTERS(start, n);
-			DMA_COPY_FROM_USER(regs, n);
+			DMA_COPY(regs, n);
 			count -= n;
 			start += n;
 			regs += n;
@@ -280,13 +277,13 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 				    const drm_savage_cmd_header_t *cmd_header,
 				    const drm_buf_t *dmabuf)
 {
-	BCI_LOCALS;
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->prim.prim;
 	unsigned int skip = cmd_header->prim.skip;
 	unsigned int n = cmd_header->prim.count;
 	unsigned int start = cmd_header->prim.start;
 	unsigned int i;
+	BCI_LOCALS;
 
 	if (!dmabuf) {
 	    DRM_ERROR("called without dma buffers!\n");
@@ -418,11 +415,9 @@ static int savage_dispatch_dma_prim(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 				   const drm_savage_cmd_header_t *cmd_header,
-				   const uint32_t __user *vtxbuf,
-				   unsigned int vb_size,
+				   const uint32_t *vtxbuf, unsigned int vb_size,
 				   unsigned int vb_stride)
 {
-	DMA_LOCALS;
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->prim.prim;
 	unsigned int skip = cmd_header->prim.skip;
@@ -430,6 +425,7 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 	unsigned int start = cmd_header->prim.start;
 	unsigned int vtx_size;
 	unsigned int i;
+	DMA_LOCALS;
 
 	if (!n)
 		return 0;
@@ -504,8 +500,7 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 
 			for (i = start; i < start+count; ++i) {
 				unsigned int j = i + reorder[i % 3];
-				DMA_COPY_FROM_USER(&vtxbuf[vb_stride*j],
-						   vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
 			}
 
 			DMA_COMMIT();
@@ -514,13 +509,12 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 			DMA_DRAW_PRIMITIVE(count, prim, skip);
 
 			if (vb_stride == vtx_size) {
-				DMA_COPY_FROM_USER(&vtxbuf[vb_stride*start],
-						   vtx_size*count);
+				DMA_COPY(&vtxbuf[vb_stride*start],
+					 vtx_size*count);
 			} else {
 				for (i = start; i < start+count; ++i) {
-					DMA_COPY_FROM_USER(
-						&vtxbuf[vb_stride*i],
-						vtx_size);
+					DMA_COPY(&vtxbuf[vb_stride*i],
+						 vtx_size);
 				}
 			}
 
@@ -538,15 +532,15 @@ static int savage_dispatch_vb_prim(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 				   const drm_savage_cmd_header_t *cmd_header,
-				   const uint16_t __user *usr_idx,
+				   const uint16_t *idx,
 				   const drm_buf_t *dmabuf)
 {
-	BCI_LOCALS;
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->idx.prim;
 	unsigned int skip = cmd_header->idx.skip;
 	unsigned int n = cmd_header->idx.count;
 	unsigned int i;
+	BCI_LOCALS;
 
 	if (!dmabuf) {
 	    DRM_ERROR("called without dma buffers!\n");
@@ -628,11 +622,8 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 	while (n != 0) {
 		/* Can emit up to 255 indices (85 triangles) at once. */
 		unsigned int count = n > 255 ? 255 : n;
-		/* Is it ok to allocate 510 bytes on the stack in an ioctl? */
-		uint16_t idx[255];
 
-		/* Copy and check indices */
-		DRM_COPY_FROM_USER_UNCHECKED(idx, usr_idx, count*2);
+		/* check indices */
 		for (i = 0; i < count; ++i) {
 			if (idx[i] > dmabuf->total/32) {
 				DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
@@ -673,7 +664,7 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 				BCI_WRITE(idx[i]);
 		}
 
-		usr_idx += count;
+		idx += count;
 		n -= count;
 
 		prim |= BCI_CMD_DRAW_CONT;
@@ -684,18 +675,18 @@ static int savage_dispatch_dma_idx(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 				  const drm_savage_cmd_header_t *cmd_header,
-				  const uint16_t __user *usr_idx,
-				  const uint32_t __user *vtxbuf,
+				  const uint16_t *idx,
+				  const uint32_t *vtxbuf,
 				  unsigned int vb_size,
 				  unsigned int vb_stride)
 {
-	DMA_LOCALS;
 	unsigned char reorder = 0;
 	unsigned int prim = cmd_header->idx.prim;
 	unsigned int skip = cmd_header->idx.skip;
 	unsigned int n = cmd_header->idx.count;
 	unsigned int vtx_size;
 	unsigned int i;
+	DMA_LOCALS;
 
 	if (!n)
 		return 0;
@@ -752,11 +743,8 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 	while (n != 0) {
 		/* Can emit up to 255 vertices (85 triangles) at once. */
 		unsigned int count = n > 255 ? 255 : n;
-		/* Is it ok to allocate 510 bytes on the stack in an ioctl? */
-		uint16_t idx[255];
 
-		/* Copy and check indices */
-		DRM_COPY_FROM_USER_UNCHECKED(idx, usr_idx, count*2);
+		/* Check indices */
 		for (i = 0; i < count; ++i) {
 			if (idx[i] > vb_size / (vb_stride*4)) {
 				DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
@@ -776,8 +764,7 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 
 			for (i = 0; i < count; ++i) {
 				unsigned int j = idx[i + reorder[i % 3]];
-				DMA_COPY_FROM_USER(&vtxbuf[vb_stride*j],
-						   vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
 			}
 
 			DMA_COMMIT();
@@ -787,14 +774,13 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 
 			for (i = 0; i < count; ++i) {
 				unsigned int j = idx[i];
-				DMA_COPY_FROM_USER(&vtxbuf[vb_stride*j],
-						   vtx_size);
+				DMA_COPY(&vtxbuf[vb_stride*j], vtx_size);
 			}
 
 			DMA_COMMIT();
 		}
 
-		usr_idx += count;
+		idx += count;
 		n -= count;
 
 		prim |= BCI_CMD_DRAW_CONT;
@@ -805,23 +791,18 @@ static int savage_dispatch_vb_idx(drm_savage_private_t *dev_priv,
 
 static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 				 const drm_savage_cmd_header_t *cmd_header,
-				 const drm_savage_cmd_header_t __user *data,
+				 const drm_savage_cmd_header_t *data,
 				 unsigned int nbox,
-				 const drm_clip_rect_t __user *usr_boxes)
+				 const drm_clip_rect_t *boxes)
 {
-	DMA_LOCALS;
-	unsigned int flags = cmd_header->clear0.flags, mask, value;
+	unsigned int flags = cmd_header->clear0.flags;
 	unsigned int clear_cmd;
 	unsigned int i, nbufs;
+	DMA_LOCALS;
 
 	if (nbox == 0)
 		return 0;
 
-	DRM_GET_USER_UNCHECKED(mask, &((drm_savage_cmd_header_t*)data)
-			       ->clear1.mask);
-	DRM_GET_USER_UNCHECKED(value, &((drm_savage_cmd_header_t*)data)
-			       ->clear1.value);
-
 	clear_cmd = BCI_CMD_RECT | BCI_CMD_RECT_XP | BCI_CMD_RECT_YP |
 		BCI_CMD_SEND_COLOR | BCI_CMD_DEST_PBD_NEW;
 	BCI_CMD_SET_ROP(clear_cmd,0xCC);
@@ -832,21 +813,20 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 	if (nbufs == 0)
 		return 0;
 
-	if (mask != 0xffffffff) {
+	if (data->clear1.mask != 0xffffffff) {
 		/* set mask */
 		BEGIN_DMA(2);
 		DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
-		DMA_WRITE(mask);
+		DMA_WRITE(data->clear1.mask);
 		DMA_COMMIT();
 	}
 	for (i = 0; i < nbox; ++i) {
-		drm_clip_rect_t box;
 		unsigned int x, y, w, h;
 		unsigned int buf;
-		DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
-		x = box.x1, y = box.y1;
-		w = box.x2 - box.x1;
-		h = box.y2 - box.y1;
+
+		x = boxes[i].x1, y = boxes[i].y1;
+		w = boxes[i].x2 - boxes[i].x1;
+		h = boxes[i].y2 - boxes[i].y1;
 		BEGIN_DMA(nbufs*6);
 		for (buf = SAVAGE_FRONT; buf <= SAVAGE_DEPTH; buf <<= 1) {
 			if (!(flags & buf))
@@ -866,13 +846,13 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 				DMA_WRITE(dev_priv->depth_bd);
 				break;
 			}
-			DMA_WRITE(value);
+			DMA_WRITE(data->clear1.value);
 			DMA_WRITE(BCI_X_Y(x, y));
 			DMA_WRITE(BCI_W_H(w, h));
 		}
 		DMA_COMMIT();
 	}
-	if (mask != 0xffffffff) {
+	if (data->clear1.mask != 0xffffffff) {
 		/* reset mask */
 		BEGIN_DMA(2);
 		DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
@@ -884,12 +864,11 @@ static int savage_dispatch_clear(drm_savage_private_t *dev_priv,
 }
 
 static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
-				unsigned int nbox,
-				const drm_clip_rect_t __user *usr_boxes)
+				unsigned int nbox, const drm_clip_rect_t *boxes)
 {
-	DMA_LOCALS;
 	unsigned int swap_cmd;
 	unsigned int i;
+	DMA_LOCALS;
 
 	if (nbox == 0)
 		return 0;
@@ -899,16 +878,14 @@ static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
 	BCI_CMD_SET_ROP(swap_cmd,0xCC);
 
 	for (i = 0; i < nbox; ++i) {
-		drm_clip_rect_t box;
-		DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
-
 		BEGIN_DMA(6);
 		DMA_WRITE(swap_cmd);
 		DMA_WRITE(dev_priv->back_offset);
 		DMA_WRITE(dev_priv->back_bd);
-		DMA_WRITE(BCI_X_Y(box.x1, box.y1));
-		DMA_WRITE(BCI_X_Y(box.x1, box.y1));
-		DMA_WRITE(BCI_W_H(box.x2-box.x1, box.y2-box.y1));
+		DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
+		DMA_WRITE(BCI_X_Y(boxes[i].x1, boxes[i].y1));
+		DMA_WRITE(BCI_W_H(boxes[i].x2-boxes[i].x1,
+				  boxes[i].y2-boxes[i].y1));
 		DMA_COMMIT();
 	}
 
@@ -916,29 +893,26 @@ static int savage_dispatch_swap(drm_savage_private_t *dev_priv,
 }
 
 static int savage_dispatch_draw(drm_savage_private_t *dev_priv,
-				const drm_savage_cmd_header_t __user *start,
-				const drm_savage_cmd_header_t __user *end,
+				const drm_savage_cmd_header_t *start,
+				const drm_savage_cmd_header_t *end,
 				const drm_buf_t *dmabuf,
-				const unsigned int __user *usr_vtxbuf,
+				const unsigned int *vtxbuf,
 				unsigned int vb_size, unsigned int vb_stride,
 				unsigned int nbox,
-				const drm_clip_rect_t __user *usr_boxes)
+				const drm_clip_rect_t *boxes)
 {
 	unsigned int i, j;
 	int ret;
 
 	for (i = 0; i < nbox; ++i) {
-		drm_clip_rect_t box;
-		const drm_savage_cmd_header_t __user *usr_cmdbuf;
-		DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
-		dev_priv->emit_clip_rect(dev_priv, &box);
+		const drm_savage_cmd_header_t *cmdbuf;
+		dev_priv->emit_clip_rect(dev_priv, &boxes[i]);
 
-		usr_cmdbuf = start;
-		while (usr_cmdbuf < end) {
+		cmdbuf = start;
+		while (cmdbuf < end) {
 			drm_savage_cmd_header_t cmd_header;
-			DRM_COPY_FROM_USER_UNCHECKED(&cmd_header, usr_cmdbuf,
-						     sizeof(cmd_header));
-			usr_cmdbuf++;
+			cmd_header = *cmdbuf;
+			cmdbuf++;
 			switch (cmd_header.cmd.cmd) {
 			case SAVAGE_CMD_DMA_PRIM:
 				ret = savage_dispatch_dma_prim(
@@ -947,27 +921,24 @@ static int savage_dispatch_draw(drm_savage_private_t *dev_priv,
 			case SAVAGE_CMD_VB_PRIM:
 				ret = savage_dispatch_vb_prim(
 					dev_priv, &cmd_header,
-					(uint32_t __user *)usr_vtxbuf,
-					vb_size, vb_stride);
+					vtxbuf, vb_size, vb_stride);
 				break;
 			case SAVAGE_CMD_DMA_IDX:
 				j = (cmd_header.idx.count + 3) / 4;
 				/* j was check in savage_bci_cmdbuf */
-				ret = savage_dispatch_dma_idx(
-					dev_priv, &cmd_header,
-					(uint16_t __user *)usr_cmdbuf,
+				ret = savage_dispatch_dma_idx(dev_priv,
+					&cmd_header, (const uint16_t *)cmdbuf,
 					dmabuf);
-				usr_cmdbuf += j;
+				cmdbuf += j;
 				break;
 			case SAVAGE_CMD_VB_IDX:
 				j = (cmd_header.idx.count + 3) / 4;
 				/* j was check in savage_bci_cmdbuf */
-				ret = savage_dispatch_vb_idx(
-					dev_priv, &cmd_header,
-					(uint16_t __user *)usr_cmdbuf,
-					(uint32_t __user *)usr_vtxbuf,
-					vb_size, vb_stride);
-				usr_cmdbuf += j;
+				ret = savage_dispatch_vb_idx(dev_priv,
+					&cmd_header, (const uint16_t *)cmdbuf,
+					(const uint32_t *)vtxbuf, vb_size,
+					vb_stride);
+				cmdbuf += j;
 				break;
 			default:
 				/* What's the best return code? EFAULT? */
@@ -992,10 +963,10 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 	drm_device_dma_t *dma = dev->dma;
 	drm_buf_t *dmabuf;
 	drm_savage_cmdbuf_t cmdbuf;
-	drm_savage_cmd_header_t __user *usr_cmdbuf;
-	drm_savage_cmd_header_t __user *first_draw_cmd;
-	unsigned int __user *usr_vtxbuf;
-	drm_clip_rect_t __user *usr_boxes;
+	drm_savage_cmd_header_t *kcmd_addr = NULL;
+	drm_savage_cmd_header_t *first_draw_cmd;
+	unsigned int *kvb_addr = NULL;
+	drm_clip_rect_t *kbox_addr = NULL;
 	unsigned int i, j;
 	int ret = 0;
 
@@ -1017,15 +988,53 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 		dmabuf = NULL;
 	}
 
-	usr_cmdbuf = (drm_savage_cmd_header_t __user *)cmdbuf.cmd_addr;
-	usr_vtxbuf = (unsigned int __user *)cmdbuf.vb_addr;
-	usr_boxes = (drm_clip_rect_t __user *)cmdbuf.box_addr;
-	if ((cmdbuf.size && DRM_VERIFYAREA_READ(usr_cmdbuf, cmdbuf.size*8)) ||
-	    (cmdbuf.vb_size && DRM_VERIFYAREA_READ(
-		    usr_vtxbuf, cmdbuf.vb_size)) ||
-	    (cmdbuf.nbox && DRM_VERIFYAREA_READ(
-		    usr_boxes, cmdbuf.nbox*sizeof(drm_clip_rect_t))))
-		return DRM_ERR(EFAULT);
+	/* Copy the user buffers into kernel temporary areas.  This hasn't been
+	 * a performance loss compared to VERIFYAREA_READ/
+	 * COPY_FROM_USER_UNCHECKED when done in other drivers, and is correct
+	 * for locking on FreeBSD.
+	 */
+	if (cmdbuf.size) {
+		kcmd_addr = drm_alloc(cmdbuf.size * 8, DRM_MEM_DRIVER);
+		if (kcmd_addr == NULL)
+			return ENOMEM;
+
+		if (DRM_COPY_FROM_USER(kcmd_addr, cmdbuf.cmd_addr,
+				       cmdbuf.size * 8))
+		{
+			drm_free(kcmd_addr, cmdbuf.size * 8, DRM_MEM_DRIVER);
+			return DRM_ERR(EFAULT);
+		}
+		cmdbuf.cmd_addr = kcmd_addr;
+	}
+	if (cmdbuf.vb_size) {
+		kvb_addr = drm_alloc(cmdbuf.vb_size, DRM_MEM_DRIVER);
+		if (kvb_addr == NULL) {
+			ret = DRM_ERR(ENOMEM);
+			goto done;
+		}
+
+		if (DRM_COPY_FROM_USER(kvb_addr, cmdbuf.vb_addr,
+				       cmdbuf.vb_size)) {
+			ret = DRM_ERR(EFAULT);
+			goto done;
+		}
+		cmdbuf.vb_addr = kvb_addr;
+	}
+	if (cmdbuf.nbox) {
+		kbox_addr = drm_alloc(cmdbuf.nbox * sizeof(drm_clip_rect_t),
+				       DRM_MEM_DRIVER);
+		if (kbox_addr == NULL) {
+			ret = DRM_ERR(ENOMEM);
+			goto done;
+		}
+
+		if (DRM_COPY_FROM_USER(kbox_addr, cmdbuf.box_addr,
+				       cmdbuf.nbox * sizeof(drm_clip_rect_t))) {
+			ret = DRM_ERR(EFAULT);
+			goto done;
+		}
+		cmdbuf.box_addr = kbox_addr;
+	}
 
 	/* Make sure writes to DMA buffers are finished before sending
 	 * DMA commands to the graphics hardware. */
@@ -1039,9 +1048,8 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 	first_draw_cmd = NULL;
 	while (i < cmdbuf.size) {
 		drm_savage_cmd_header_t cmd_header;
-		DRM_COPY_FROM_USER_UNCHECKED(&cmd_header, usr_cmdbuf,
-					     sizeof(cmd_header));
-		usr_cmdbuf++;
+		cmd_header = *(drm_savage_cmd_header_t *)cmdbuf.cmd_addr;
+		cmdbuf.cmd_addr++;
 		i++;
 
 		/* Group drawing commands with same state to minimize
@@ -1061,17 +1069,18 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 		case SAVAGE_CMD_DMA_PRIM:
 		case SAVAGE_CMD_VB_PRIM:
 			if (!first_draw_cmd)
-				first_draw_cmd = usr_cmdbuf-1;
-			usr_cmdbuf += j;
+				first_draw_cmd = cmdbuf.cmd_addr-1;
+			cmdbuf.cmd_addr += j;
 			i += j;
 			break;
 		default:
 			if (first_draw_cmd) {
 				ret = savage_dispatch_draw (
-					dev_priv, first_draw_cmd, usr_cmdbuf-1,
-					dmabuf, usr_vtxbuf, cmdbuf.vb_size,
+					dev_priv, first_draw_cmd,
+					cmdbuf.cmd_addr-1,
+					dmabuf, cmdbuf.vb_addr, cmdbuf.vb_size,
 					cmdbuf.vb_stride,
-					cmdbuf.nbox, usr_boxes);
+					cmdbuf.nbox, cmdbuf.box_addr);
 				if (ret != 0)
 					return ret;
 				first_draw_cmd = NULL;
@@ -1087,12 +1096,12 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 				DRM_ERROR("command SAVAGE_CMD_STATE extends "
 					  "beyond end of command buffer\n");
 				DMA_FLUSH();
-				return DRM_ERR(EINVAL);
+				ret = DRM_ERR(EINVAL);
+				goto done;
 			}
-			ret = savage_dispatch_state(
-				dev_priv, &cmd_header,
-				(uint32_t __user *)usr_cmdbuf);
-			usr_cmdbuf += j;
+			ret = savage_dispatch_state(dev_priv, &cmd_header,
+				(const uint32_t *)cmdbuf.cmd_addr);
+			cmdbuf.cmd_addr += j;
 			i += j;
 			break;
 		case SAVAGE_CMD_CLEAR:
@@ -1100,38 +1109,40 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 				DRM_ERROR("command SAVAGE_CMD_CLEAR extends "
 					  "beyond end of command buffer\n");
 				DMA_FLUSH();
-				return DRM_ERR(EINVAL);
+				ret = DRM_ERR(EINVAL);
+				goto done;
 			}
 			ret = savage_dispatch_clear(dev_priv, &cmd_header,
-						    usr_cmdbuf,
-						    cmdbuf.nbox, usr_boxes);
-			usr_cmdbuf++;
+						    cmdbuf.cmd_addr,
+						    cmdbuf.nbox, cmdbuf.box_addr);
+			cmdbuf.cmd_addr++;
 			i++;
 			break;
 		case SAVAGE_CMD_SWAP:
-			ret = savage_dispatch_swap(dev_priv,
-						   cmdbuf.nbox, usr_boxes);
+			ret = savage_dispatch_swap(dev_priv, cmdbuf.nbox,
+						   cmdbuf.box_addr);
 			break;
 		default:
 			DRM_ERROR("invalid command 0x%x\n", cmd_header.cmd.cmd);
 			DMA_FLUSH();
-			return DRM_ERR(EINVAL);
+			ret = DRM_ERR(EINVAL);
+			goto done;
 		}
 
 		if (ret != 0) {
 			DMA_FLUSH();
-			return ret;
+			goto done;
 		}
 	}
 
 	if (first_draw_cmd) {
 		ret = savage_dispatch_draw (
-			dev_priv, first_draw_cmd, usr_cmdbuf, dmabuf,
-			usr_vtxbuf, cmdbuf.vb_size, cmdbuf.vb_stride,
-			cmdbuf.nbox, usr_boxes);
+			dev_priv, first_draw_cmd, cmdbuf.cmd_addr, dmabuf,
+			cmdbuf.vb_addr, cmdbuf.vb_size, cmdbuf.vb_stride,
+			cmdbuf.nbox, cmdbuf.box_addr);
 		if (ret != 0) {
 			DMA_FLUSH();
-			return ret;
+			goto done;
 		}
 	}
 
@@ -1145,5 +1156,12 @@ int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
 		savage_freelist_put(dev, dmabuf);
 	}
 
-	return 0;
+done:
+	/* If we didn't need to allocate them, these'll be NULL */
+	drm_free(kcmd_addr, cmdbuf.size * 8, DRM_MEM_DRIVER);
+	drm_free(kvb_addr, cmdbuf.vb_size, DRM_MEM_DRIVER);
+	drm_free(kbox_addr, cmdbuf.nbox * sizeof(drm_clip_rect_t),
+		 DRM_MEM_DRIVER);
+
+	return ret;
 }
diff --git a/sys/dev/drm/sis_drm.h b/sys/dev/drm/sis_drm.h
index 3c6fb02efbd3..6b260a37386e 100644
--- a/sys/dev/drm/sis_drm.h
+++ b/sys/dev/drm/sis_drm.h
@@ -1,7 +1,32 @@
-/*
- * $FreeBSD$
+/* sis_drv.h -- Private header for sis driver -*- linux-c -*- */
+/*-
+ * Copyright 2005 Eric Anholt
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __SIS_DRM_H__
 #define __SIS_DRM_H__
 
diff --git a/sys/dev/drm/sis_drv.c b/sys/dev/drm/sis_drv.c
index 954d34b64dd6..5c7a1614e204 100644
--- a/sys/dev/drm/sis_drv.c
+++ b/sys/dev/drm/sis_drv.c
@@ -24,9 +24,11 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/sis_drm.h"
 #include "dev/drm/sis_drv.h"
@@ -37,27 +39,24 @@ static drm_pci_id_list_t sis_pciidlist[] = {
 	sis_PCI_IDS
 };
 
-extern drm_ioctl_desc_t sis_ioctls[];
-extern int sis_max_ioctl;
-
 static void sis_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = 1; /* No dev_priv */
-	dev->context_ctor = sis_init_context;
-	dev->context_dtor = sis_final_context;
+	dev->driver.buf_priv_size	= 1; /* No dev_priv */
+	dev->driver.context_ctor	= sis_init_context;
+	dev->driver.context_dtor	= sis_final_context;
 
-	dev->driver_ioctls = sis_ioctls;
-	dev->max_driver_ioctl = sis_max_ioctl;
+	dev->driver.ioctls		= sis_ioctls;
+	dev->driver.max_ioctl		= sis_max_ioctl;
 
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
 
-	dev->use_agp = 1;
-	dev->use_mtrr = 1;
+	dev->driver.use_agp		= 1;
+	dev->driver.use_mtrr		= 1;
 }
 
 #ifdef __FreeBSD__
@@ -97,5 +96,10 @@ DRIVER_MODULE(sisdrm, pci, sis_driver, drm_devclass, 0, 0);
 MODULE_DEPEND(sisdrm, drm, 1, 1, 1);
 
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
+#ifdef _LKM
 CFDRIVER_DECL(sis, DV_TTY, NULL);
+#else
+CFATTACH_DECL(sis, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+    drm_activate);
+#endif
 #endif
diff --git a/sys/dev/drm/sis_drv.h b/sys/dev/drm/sis_drv.h
index 4e06e005c4f5..7fc37716fa23 100644
--- a/sys/dev/drm/sis_drv.h
+++ b/sys/dev/drm/sis_drv.h
@@ -23,9 +23,11 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef _SIS_DRV_H_
 #define _SIS_DRV_H_
 
@@ -50,4 +52,7 @@ typedef struct drm_sis_private {
 extern int sis_init_context(drm_device_t * dev, int context);
 extern int sis_final_context(drm_device_t * dev, int context);
 
+extern drm_ioctl_desc_t sis_ioctls[];
+extern int sis_max_ioctl;
+
 #endif
diff --git a/sys/dev/drm/sis_ds.c b/sys/dev/drm/sis_ds.c
index 3f3a4b67fedc..57f55a30757d 100644
--- a/sys/dev/drm/sis_ds.c
+++ b/sys/dev/drm/sis_ds.c
@@ -1,6 +1,6 @@
 /* sis_ds.c -- Private header for Direct Rendering Manager -*- linux-c -*-
- * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw */
-/*-
+ * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw
+ *
  * Copyright 2000 Silicon Integrated Systems Corp, Inc., HsinChu, Taiwan.
  * All rights reserved.
  *
@@ -26,9 +26,11 @@
  * Authors:
  *    Sung-Ching Lin <sclin@sis.com.tw>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm.h"
 #include "dev/drm/sis_ds.h"
diff --git a/sys/dev/drm/sis_ds.h b/sys/dev/drm/sis_ds.h
index d5b0faa3b95b..b3ffadfb5ff9 100644
--- a/sys/dev/drm/sis_ds.h
+++ b/sys/dev/drm/sis_ds.h
@@ -1,5 +1,6 @@
 /* sis_ds.h -- Private header for Direct Rendering Manager -*- linux-c -*-
- * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw */
+ * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw
+ */
 /*-
  * Copyright 2000 Silicon Integrated Systems Corp, Inc., HsinChu, Taiwan.
  * All rights reserved.
@@ -26,9 +27,11 @@
  * Authors:
  *    Sung-Ching Lin <sclin@sis.com.tw>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __SIS_DS_H__
 #define __SIS_DS_H__
 
diff --git a/sys/dev/drm/sis_mm.c b/sys/dev/drm/sis_mm.c
index 8f4bc7c792b2..9c4a0cc8f22e 100644
--- a/sys/dev/drm/sis_mm.c
+++ b/sys/dev/drm/sis_mm.c
@@ -1,6 +1,6 @@
 /* sis_mm.c -- Private header for Direct Rendering Manager -*- linux-c -*-
- * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw */
-/*-
+ * Created: Mon Jan  4 10:05:05 1999 by sclin@sis.com.tw
+ *
  * Copyright 2000 Silicon Integrated Systems Corp, Inc., HsinChu, Taiwan.
  * All rights reserved.
  *
@@ -26,9 +26,11 @@
  * Authors:
  *    Sung-Ching Lin <sclin@sis.com.tw>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #if defined(__linux__) && defined(CONFIG_FB_SIS)
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
 #include <video/sisfb.h>
@@ -408,12 +410,12 @@ int sis_final_context(struct drm_device *dev, int context)
 }
 
 drm_ioctl_desc_t sis_ioctls[] = {
-	[DRM_IOCTL_NR(DRM_SIS_FB_ALLOC)] = {sis_fb_alloc, 1, 0},
-	[DRM_IOCTL_NR(DRM_SIS_FB_FREE)] = {sis_fb_free, 1, 0},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_INIT)] = {sis_ioctl_agp_init, 1, 1},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_ALLOC)] = {sis_ioctl_agp_alloc, 1, 0},
-	[DRM_IOCTL_NR(DRM_SIS_AGP_FREE)] = {sis_ioctl_agp_free, 1, 0},
-	[DRM_IOCTL_NR(DRM_SIS_FB_INIT)] = {sis_fb_init, 1, 1}
+	[DRM_IOCTL_NR(DRM_SIS_FB_ALLOC)] = {sis_fb_alloc, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SIS_FB_FREE)] = {sis_fb_free, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SIS_AGP_INIT)] = {sis_ioctl_agp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_SIS_AGP_ALLOC)] = {sis_ioctl_agp_alloc, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SIS_AGP_FREE)] = {sis_ioctl_agp_free, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_SIS_FB_INIT)] = {sis_fb_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY}
 };
 
 int sis_max_ioctl = DRM_ARRAY_SIZE(sis_ioctls);
diff --git a/sys/dev/drm/tdfx_drv.c b/sys/dev/drm/tdfx_drv.c
index e99bb3281a05..c193ecc02f31 100644
--- a/sys/dev/drm/tdfx_drv.c
+++ b/sys/dev/drm/tdfx_drv.c
@@ -30,9 +30,11 @@
  *    Daryll Strauss <daryll@valinux.com>
  *    Gareth Hughes <gareth@valinux.com>
  *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "dev/drm/tdfx_drv.h"
 #include "dev/drm/drmP.h"
 #include "dev/drm/drm_pciids.h"
@@ -42,23 +44,20 @@ static drm_pci_id_list_t tdfx_pciidlist[] = {
 	tdfx_PCI_IDS
 };
 
-extern drm_ioctl_desc_t tdfx_ioctls[];
-extern int tdfx_max_ioctl;
-
 static void tdfx_configure(drm_device_t *dev)
 {
-	dev->dev_priv_size = 1; /* No dev_priv */
+	dev->driver.buf_priv_size	= 1; /* No dev_priv */
 
-	dev->max_driver_ioctl = 0;
+	dev->driver.max_ioctl		= 0;
 
-	dev->driver_name = DRIVER_NAME;
-	dev->driver_desc = DRIVER_DESC;
-	dev->driver_date = DRIVER_DATE;
-	dev->driver_major = DRIVER_MAJOR;
-	dev->driver_minor = DRIVER_MINOR;
-	dev->driver_patchlevel = DRIVER_PATCHLEVEL;
+	dev->driver.name		= DRIVER_NAME;
+	dev->driver.desc		= DRIVER_DESC;
+	dev->driver.date		= DRIVER_DATE;
+	dev->driver.major		= DRIVER_MAJOR;
+	dev->driver.minor		= DRIVER_MINOR;
+	dev->driver.patchlevel		= DRIVER_PATCHLEVEL;
 
-	dev->use_mtrr = 1;
+	dev->driver.use_mtrr		= 1;
 }
 
 #ifdef __FreeBSD__
@@ -98,5 +97,10 @@ DRIVER_MODULE(tdfx, pci, tdfx_driver, drm_devclass, 0, 0);
 MODULE_DEPEND(tdfx, drm, 1, 1, 1);
 
 #elif defined(__NetBSD__) || defined(__OpenBSD__)
+#ifdef _LKM
 CFDRIVER_DECL(tdfx, DV_TTY, NULL);
+#else
+CFATTACH_DECL(tdfx, sizeof(drm_device_t), drm_probe, drm_attach, drm_detach,
+    drm_activate);
+#endif
 #endif
diff --git a/sys/dev/drm/tdfx_drv.h b/sys/dev/drm/tdfx_drv.h
index 05c1f7f9904c..d798569cb3fe 100644
--- a/sys/dev/drm/tdfx_drv.h
+++ b/sys/dev/drm/tdfx_drv.h
@@ -1,6 +1,7 @@
 /* tdfx.h -- 3dfx DRM template customization -*- linux-c -*-
  * Created: Wed Feb 14 12:32:32 2001 by gareth@valinux.com
- *
+ */
+/*-
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All Rights Reserved.
  *
@@ -25,10 +26,11 @@
  *
  * Authors:
  *    Gareth Hughes <gareth@valinux.com>
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #ifndef __TDFX_H__
 #define __TDFX_H__
 
diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES
index b34e4b63d9e9..347b43c1267c 100644
--- a/sys/i386/conf/NOTES
+++ b/sys/i386/conf/NOTES
@@ -471,6 +471,7 @@ device		mach64drm	# ATI Rage Pro, Rage Mobility P/M, Rage XL
 device		mgadrm		# AGP Matrox G200, G400, G450, G550
 device		r128drm		# ATI Rage 128
 device		radeondrm	# ATI Radeon
+device		savagedrm	# S3 Savage3D, Savage4
 device		sisdrm		# SiS 300/305, 540, 630
 device		tdfxdrm		# 3dfx Voodoo 3/4/5 and Banshee
 options 	DRM_DEBUG	# Include debug printfs (slow)
diff --git a/sys/modules/drm/Makefile b/sys/modules/drm/Makefile
index 6a8bd8e7f429..b2cfa040a1fa 100644
--- a/sys/modules/drm/Makefile
+++ b/sys/modules/drm/Makefile
@@ -6,6 +6,7 @@ SUBDIR = \
 	mga \
 	r128 \
 	radeon \
+	savage \
 	sis \
 	tdfx
 
diff --git a/sys/modules/drm/i915/Makefile b/sys/modules/drm/i915/Makefile
new file mode 100644
index 000000000000..a34695ad7870
--- /dev/null
+++ b/sys/modules/drm/i915/Makefile
@@ -0,0 +1,8 @@
+# $FreeBSD$
+
+.PATH:	${.CURDIR}/../../../dev/drm
+KMOD	= i915
+SRCS	= i915_dma.c i915_drv.c i915_irq.c i915_mem.c
+SRCS	+=device_if.h bus_if.h pci_if.h opt_drm.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/drm/savage/Makefile b/sys/modules/drm/savage/Makefile
new file mode 100644
index 000000000000..f1c9b42a5f33
--- /dev/null
+++ b/sys/modules/drm/savage/Makefile
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+.PATH:	${.CURDIR}/../../../dev/drm
+KMOD= 	savage
+CFLAGS += -I${.CURDIR}/../../../dev/drm
+SRCS=	savage_bci.c savage_drv.c savage_state.c
+SRCS+=	device_if.h bus_if.h pci_if.h opt_drm.h
+
+.include <bsd.kmod.mk>