38 files changed, 2390 insertions, 73 deletions

diff --git a/Makefile b/Makefile
index a37b087a042b..2894d2556728 100644
--- a/Makefile
+++ b/Makefile
@@ -209,7 +209,7 @@ WANT_MAKE_VERSION= 20160604
 # 20160220 - support .dinclude for FAST_DEPEND.
 WANT_MAKE_VERSION= 20160220
 .endif
-MYMAKE=		${MAKEOBJDIRPREFIX}${.CURDIR}/make.${MACHINE}/${WANT_MAKE}
+MYMAKE=		${OBJROOT}make.${MACHINE}/${WANT_MAKE}
 .if defined(.PARSEDIR)
 HAVE_MAKE=	bmake
 .else
@@ -407,10 +407,13 @@ upgrade_checks: .PHONY
 # headers, libraries and tools.  Also, allow the location of
 # the system bsdmake-like utility to be overridden.
 #
-MMAKEENV=	MAKEOBJDIRPREFIX=${MYMAKE:H} \
+MMAKEENV=	\
 		DESTDIR= \
 		INSTALL="sh ${.CURDIR}/tools/install.sh"
 MMAKE=		${MMAKEENV} ${MAKE} \
+		OBJTOP=${MYMAKE:H}/obj \
+		OBJROOT='$${OBJTOP}/' \
+		MAKEOBJDIRPREFIX= \
 		MAN= -DNO_SHARED \
 		-DNO_CPU_CFLAGS -DNO_WERROR \
 		-DNO_SUBDIR \
diff --git a/Makefile.inc1 b/Makefile.inc1
index ccb468610010..dc040e9a02a4 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -523,6 +523,7 @@ BMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
 BSARGS= 	DESTDIR= \
 		OBJTOP='${WORLDTMP}/obj-tools' \
 		OBJROOT='$${OBJTOP}/' \
+		MAKEOBJDIRPREFIX= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		BWPHASE=${.TARGET:C,^_,,} \
 		SSP_CFLAGS= \
@@ -566,6 +567,7 @@ KTMAKE=		TOOLS_PREFIX=${WORLDTMP} \
 		DESTDIR= \
 		OBJTOP='${WORLDTMP}/obj-kernel-tools' \
 		OBJROOT='$${OBJTOP}/' \
+		MAKEOBJDIRPREFIX= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		SSP_CFLAGS= \
 		MK_COVERAGE=no MK_HTML=no -DNO_LINT MK_MAN=no \
diff --git a/Makefile.libcompat b/Makefile.libcompat
index d3b617ea14ee..cc74a85ef971 100644
--- a/Makefile.libcompat
+++ b/Makefile.libcompat
@@ -108,8 +108,6 @@ LIBCOMPATCXXFLAGS+=	-isystem ${LIBCOMPATTMP}/usr/include/c++/v1 -std=c++11 \
 
 # Yes, the flags are redundant.
 LIBCOMPATWMAKEENV+= \
-		OBJTOP=${LIBCOMPAT_OBJTOP} \
-		OBJROOT='$${OBJTOP}/' \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH} \
 		SYSROOT=${LIBCOMPATTMP} \
@@ -130,6 +128,9 @@ LIBCOMPATWMAKEFLAGS+= CC="${XCC} ${LIBCOMPATCFLAGS}" \
 		-DNO_LINT \
 		MK_TESTS=no
 LIBCOMPATWMAKE+=	${LIBCOMPATWMAKEENV} ${MAKE} ${LIBCOMPATWMAKEFLAGS} \
+			OBJTOP=${LIBCOMPAT_OBJTOP} \
+			OBJROOT='$${OBJTOP}/' \
+			MAKEOBJDIRPREFIX= \
 			MK_MAN=no MK_HTML=no
 LIBCOMPATIMAKE+=	${LIBCOMPATWMAKE:NINSTALL=*:NDESTDIR=*} \
 			MK_TOOLCHAIN=no ${IMAKE_INSTALL} \
@@ -186,9 +187,10 @@ build${libcompat}: .PHONY
 	${_+_}cd ${.CURDIR}/${_dir}; \
 	    WORLDTMP=${WORLDTMP} \
 	    MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}" \
+	    ${MAKE} SSP_CFLAGS= DESTDIR= \
 	    OBJTOP=${LIBCOMPAT_OBJTOP} \
 	    OBJROOT='$${OBJTOP}/' \
-	    ${MAKE} SSP_CFLAGS= DESTDIR= \
+	    MAKEOBJDIRPREFIX= \
 	    DIRPRFX=${_dir}/ -DNO_LINT -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \
 	    build-tools
 .endfor
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index 26a1ad7c9345..979483ba44e7 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,21 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20171104: libcap_random should be in /lib not in /usr/lib
+OLD_LIBS+=usr/lib/libcap_random.so.0
+# 20171104: Casper can work only as shared library
+OLD_FILES+=usr/lib/libcasper.a
+OLD_FILES+=usr/lib/libcasper_p.a
+OLD_FILES+=usr/lib/libcap_dns.a
+OLD_FILES+=usr/lib/libcap_dns_p.a
+OLD_FILES+=usr/lib/libcap_grp.a
+OLD_FILES+=usr/lib/libcap_grp_p.a
+OLD_FILES+=usr/lib/libcap_pwd.a
+OLD_FILES+=usr/lib/libcap_pwd_p.a
+OLD_FILES+=usr/lib/libcap_random.a
+OLD_FILES+=usr/lib/libcap_random_p.a
+OLD_FILES+=usr/lib/libcap_sysctl.a
+OLD_FILES+=usr/lib/libcap_sysctl_p.a
 # 20171031: Removal of obsolete man files
 OLD_FILES+=usr/share/man/man7/adding_user.7.gz
 # 20171031: Disconnected libpathconv tests
diff --git a/contrib/llvm/lib/Support/Unix/Path.inc b/contrib/llvm/lib/Support/Unix/Path.inc
index 45097eb918b7..67edb46f0025 100644
--- a/contrib/llvm/lib/Support/Unix/Path.inc
+++ b/contrib/llvm/lib/Support/Unix/Path.inc
@@ -427,7 +427,7 @@ std::error_code resize_file(int FD, uint64_t Size) {
   // If we have posix_fallocate use it. Unlike ftruncate it always allocates
   // space, so we get an error if the disk is full.
   if (int Err = ::posix_fallocate(FD, 0, Size)) {
-    if (Err != EOPNOTSUPP)
+    if (Err != EINVAL && Err != EOPNOTSUPP)
       return std::error_code(Err, std::generic_category());
   }
 #endif
diff --git a/etc/autofs/Makefile b/etc/autofs/Makefile
index 6d1c9e85d30b..f2f8c4f3dd6c 100644
--- a/etc/autofs/Makefile
+++ b/etc/autofs/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-FILES=		include_ldap special_hosts special_media special_noauto special_null
+FILES=		include_ldap include_nis special_hosts special_media special_noauto special_null
 
 NO_OBJ=
 FILESDIR=	/etc/autofs
diff --git a/etc/autofs/include_nis b/etc/autofs/include_nis
new file mode 100644
index 000000000000..e671c42288dd
--- /dev/null
+++ b/etc/autofs/include_nis
@@ -0,0 +1,180 @@
+#!/usr/bin/awk -f
+#-
+# Copyright (c) 2017 G. Paul Ziemba
+# All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+#  OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+#  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+#  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+#  OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#  SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+#
+# /etc/autofs/include_nis
+#
+# automountd Directory Services script for NIS
+#
+# SYNOPSIS
+#       include_nis <mapname>
+#
+#       include_nis <mapname> <key>
+#
+# DESCRIPTION
+#
+#       This script provides a Directory Services map for automountd
+#       based on NIS. Please see auto_master(5) for general information.
+#
+#       The first form, with one argument, emits the entire named NIS map.
+#       The second form, with two arguments, emits the map entry for the
+#       key given in the second argument. 
+#
+#       This script attempts to determine the names and IP addresses
+#       of the local host. Map entries matching the local host are
+#       rewritten to specify nullfs mounts (instead of the default
+#       NFS) to reduce access overhead in the kernel.
+#
+#	If a map entry contains multiple location fields, it is not changed.
+#
+
+
+# Populate list of names and IP addrs thet mean "this host"
+# into myhostnames array
+BEGIN {
+    #
+    # Set self hostnames
+    #
+
+    "hostname -s" | getline;
+    myhostnames[$0] = 1;
+
+    "hostname -f" | getline;
+    myhostnames[$0] = 1;
+
+    myhostnames["localhost"] = 1
+
+    "hostname -f" | getline;
+    localdomain=$0
+    myhostnames["localhost."localdomain] = 1
+
+    while ("ifconfig" | getline) {
+	if ($1 == "inet") {
+	    myhostnames[$2] = 1;
+	}
+    }
+
+    # debug
+#            print "--- hostname list start ----"
+#            for (i in myhostnames) {
+#                print i
+#            }
+#            print "--- hostname list end ----"
+
+    if (ARGC == 2) {
+	# mapname only
+	while ("ypcat -k " ARGV[1] | getline) {
+	    proc_mapline(1)
+	}
+    }
+    if (ARGC == 3) {
+	# mapname and keyname
+	while ("ypmatch " ARGV[2] " " ARGV[1] | getline) {
+	    proc_mapline(0)
+	}
+    }
+    exit 0
+}
+
+function is_self(hostname)
+{
+    if (myhostnames[hostname]) {
+	return 1
+    }
+    return 0
+}
+
+#
+# Lines are of the form [key] [-opts] location1 [... locationN]
+#
+# indicate index of key field with first positional parameter
+# 1 means keyfield is the first field
+# 0 means keyfield is not present
+#
+function proc_mapline(keyfield)
+{
+    optionsfield = 0
+    locationfield = 0
+    locationcount = 0
+
+    for (i=keyfield+1; i <= NF; ++i) {
+	if (!optionsfield) {
+	    if ($i ~ /^-/) {
+		# the first options field found on the line
+		optionsfield = i;
+		continue
+	    }
+	}
+	# Assumption: location contains colon (":")
+	if (optionsfield && ($i ~ /:/) && ($i !~ /^-/)) {
+	    ++locationcount
+	    if (!locationfield) {
+		# the first location field found on the line
+		locationfield = i
+	    }
+	}
+    }
+
+    #
+    # If location not found, do not modify.
+    #
+    # If there is more than one location, do not modify. Rationale:
+    # Options are applied to all locations. We ca not have "nullfs"
+    # for only some locations and "nfs" for others for a given
+    # map key (i.e., a line). The usual reason for multiple
+    # locations is for redundancy using replicated volumes on
+    # multiple hosts, so multiple hosts imply fstype=nfs (the
+    # FreeBSD default for automounter maps).
+    #
+    # Hypothetically there could be a map entry with multiple
+    # locations all with host parts matching "me". In that case,
+    # it would be safe to rewrite the locations and specify
+    # nullfs, but the code does not handle this case.
+    #
+    if (locationcount == 1) {
+	#
+	# We have a line with exactly one location field
+	#
+	# Assumption: location has no more than one colon (":")
+	#
+	n=split($locationfield,location,":")
+	if (is_self(location[1])) {
+	    $locationfield = ":" location[2]
+	    if (optionsfield) {
+		# append to existing options
+		$optionsfield = $optionsfield ",fstype=nullfs"
+	    } else {
+		# sneak in ahead of location
+		$locationfield = "-fstype=nullfs " $locationfield
+	    }
+	}
+    }
+
+    print
+}
diff --git a/etc/rc.d/automount b/etc/rc.d/automount
index 7f43b4531125..e0789cb8d050 100644
--- a/etc/rc.d/automount
+++ b/etc/rc.d/automount
@@ -5,6 +5,7 @@
 
 # PROVIDE: automount
 # REQUIRE: nfsclient automountd
+# BEFORE: DAEMON
 # KEYWORD: nojail shutdown
 
 . /etc/rc.subr
diff --git a/etc/rc.d/automountd b/etc/rc.d/automountd
index c772494633f1..f9d8db8b2f37 100644
--- a/etc/rc.d/automountd
+++ b/etc/rc.d/automountd
@@ -4,7 +4,8 @@
 #
 
 # PROVIDE: automountd
-# REQUIRE: DAEMON
+# REQUIRE: rpcbind ypset nfsclient FILESYSTEMS ldconfig
+# BEFORE: DAEMON
 # KEYWORD: nojail
 
 . /etc/rc.subr
diff --git a/etc/rc.d/autounmountd b/etc/rc.d/autounmountd
index aa24d516ffff..2acbe09139ca 100644
--- a/etc/rc.d/autounmountd
+++ b/etc/rc.d/autounmountd
@@ -4,7 +4,8 @@
 #
 
 # PROVIDE: autounmountd
-# REQUIRE: DAEMON
+# REQUIRE: FILESYSTEMS
+# BEFORE: DAEMON
 # KEYWORD: nojail
 
 . /etc/rc.subr
diff --git a/lib/libc/stdlib/quick_exit.3 b/lib/libc/stdlib/quick_exit.3
index fb1e346bb106..9d316c7e46ea 100644
--- a/lib/libc/stdlib/quick_exit.3
+++ b/lib/libc/stdlib/quick_exit.3
@@ -24,7 +24,7 @@
 .\"
 .\"  $FreeBSD$
 .\"
-.Dd December 13, 2014
+.Dd November 4, 2017
 .Dt QUICK_EXIT 3
 .Os
 .Sh NAME
@@ -44,6 +44,17 @@ with
 .Xr at_quick_exit 3
 but not any C++ destructors or cleanup code registered with
 .Xr atexit 3 .
+The
+.Xr stdio 3
+file buffers are not flushed.
+.Pp
+The function
+.Fn quick_exit
+is
+.Em async-signal safe
+when the functions registered with
+.Xr at_quick_exit 3
+are.
 .Sh RETURN VALUES
 The
 .Fn quick_exit
diff --git a/lib/libc/stdlib/quick_exit.c b/lib/libc/stdlib/quick_exit.c
index ef8cdb1b401a..4d81a35f5375 100644
--- a/lib/libc/stdlib/quick_exit.c
+++ b/lib/libc/stdlib/quick_exit.c
@@ -26,6 +26,8 @@
  * $FreeBSD$
  */
 
+#include <sys/types.h>
+#include <machine/atomic.h>
 #include <stdlib.h>
 #include <pthread.h>
 
@@ -60,6 +62,7 @@ at_quick_exit(void (*func)(void))
 	h->cleanup = func;
 	pthread_mutex_lock(&atexit_mutex);
 	h->next = handlers;
+	__compiler_membar();
 	handlers = h;
 	pthread_mutex_unlock(&atexit_mutex);
 	return (0);
@@ -74,7 +77,9 @@ quick_exit(int status)
 	 * XXX: The C++ spec requires us to call std::terminate if there is an
 	 * exception here.
 	 */
-	for (h = handlers; NULL != h; h = h->next)
+	for (h = handlers; NULL != h; h = h->next) {
+		__compiler_membar();
 		h->cleanup();
+	}
 	_Exit(status);
 }
diff --git a/lib/libc/sys/posix_fallocate.2 b/lib/libc/sys/posix_fallocate.2
index 1bc59caf7239..4fba3dd87611 100644
--- a/lib/libc/sys/posix_fallocate.2
+++ b/lib/libc/sys/posix_fallocate.2
@@ -28,7 +28,7 @@
 .\"     @(#)open.2	8.2 (Berkeley) 11/16/93
 .\" $FreeBSD$
 .\"
-.Dd October 12, 2017
+.Dd November 4, 2017
 .Dt POSIX_FALLOCATE 2
 .Os
 .Sh NAME
@@ -106,9 +106,10 @@ A signal was caught during execution.
 .It Bq Er EINVAL
 The
 .Fa len
-argument was less than or equal to zero or the
+argument was less than or equal to zero, the
 .Fa offset
-argument was less than zero.
+argument was less than zero,
+or the operation is not supported by the file system.
 .It Bq Er EIO
 An I/O error occurred while reading from or writing to a file system.
 .It Bq Er ENODEV
diff --git a/lib/libcasper/libcasper/Makefile b/lib/libcasper/libcasper/Makefile
index 75ac69bf55e8..729bf071846a 100644
--- a/lib/libcasper/libcasper/Makefile
+++ b/lib/libcasper/libcasper/Makefile
@@ -1,14 +1,14 @@
 # $FreeBSD$
 
+SHLIBDIR?=	/lib
+
 .include <src.opts.mk>
 
 PACKAGE=${LIB}
 
-SHLIB_MAJOR=	0
-SHLIBDIR?=	/lib
-
 .if ${MK_CASPER} != "no"
-LIB=	casper
+SHLIB=	casper
+SHLIB_MAJOR=	0
 
 SRCS=	libcasper.c
 SRCS+=	libcasper_impl.c
diff --git a/lib/libcasper/services/cap_dns/Makefile b/lib/libcasper/services/cap_dns/Makefile
index 3e2daa2cfaaa..e89be7f5396d 100644
--- a/lib/libcasper/services/cap_dns/Makefile
+++ b/lib/libcasper/services/cap_dns/Makefile
@@ -10,7 +10,7 @@ SHLIB_MAJOR=	0
 INCSDIR?=	${INCLUDEDIR}/casper
 
 .if ${MK_CASPER} != "no"
-LIB=	cap_dns
+SHLIB=	cap_dns
 
 SRCS=	cap_dns.c
 .endif
diff --git a/lib/libcasper/services/cap_grp/Makefile b/lib/libcasper/services/cap_grp/Makefile
index d15cd72c20ce..5b464774f9f5 100644
--- a/lib/libcasper/services/cap_grp/Makefile
+++ b/lib/libcasper/services/cap_grp/Makefile
@@ -10,7 +10,7 @@ SHLIB_MAJOR=	0
 INCSDIR?=	${INCLUDEDIR}/casper
 
 .if ${MK_CASPER} != "no"
-LIB=	cap_grp
+SHLIB=	cap_grp
 
 SRCS=	cap_grp.c
 .endif
diff --git a/lib/libcasper/services/cap_pwd/Makefile b/lib/libcasper/services/cap_pwd/Makefile
index fc2a32850f79..be4ae89181c7 100644
--- a/lib/libcasper/services/cap_pwd/Makefile
+++ b/lib/libcasper/services/cap_pwd/Makefile
@@ -10,7 +10,7 @@ SHLIB_MAJOR=	0
 INCSDIR?=	${INCLUDEDIR}/casper
 
 .if ${MK_CASPER} != "no"
-LIB=	cap_pwd
+SHLIB=	cap_pwd
 
 SRCS=	cap_pwd.c
 .endif
diff --git a/lib/libcasper/services/cap_random/Makefile b/lib/libcasper/services/cap_random/Makefile
index d29cd147aeb0..fad8e3f9cdb7 100644
--- a/lib/libcasper/services/cap_random/Makefile
+++ b/lib/libcasper/services/cap_random/Makefile
@@ -1,15 +1,16 @@
 # $FreeBSD$
 
+SHLIBDIR?=	/lib/casper
+
 .include <src.opts.mk>
 
 PACKAGE=libcasper
 
 SHLIB_MAJOR=	0
-SHLIBDIR?=	/lib/casper
 INCSDIR?=	${INCLUDEDIR}/casper
 
 .if ${MK_CASPER} != "no"
-LIB=	cap_random
+SHLIB=	cap_random
 
 SRCS=	cap_random.c
 .endif
diff --git a/lib/libcasper/services/cap_sysctl/Makefile b/lib/libcasper/services/cap_sysctl/Makefile
index cd5c4b238724..14f2bd60cf97 100644
--- a/lib/libcasper/services/cap_sysctl/Makefile
+++ b/lib/libcasper/services/cap_sysctl/Makefile
@@ -10,7 +10,7 @@ SHLIB_MAJOR=	0
 INCSDIR?=	${INCLUDEDIR}/casper
 
 .if ${MK_CASPER} != "no"
-LIB=	cap_sysctl
+SHLIB=	cap_sysctl
 
 SRCS=	cap_sysctl.c
 .endif
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index f3c00af9f753..400995fb0d6f 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -760,7 +760,8 @@ MLINKS+=devstat.9 devicestat.9 \
 	devstat.9 devstat_end_transaction.9 \
 	devstat.9 devstat_remove_entry.9 \
 	devstat.9 devstat_start_transaction.9
-MLINKS+=disk.9 disk_alloc.9 \
+MLINKS+=disk.9 disk_add_alias.9 \
+	disk.9 disk_alloc.9 \
 	disk.9 disk_create.9 \
 	disk.9 disk_destroy.9 \
 	disk.9 disk_gone.9 \
diff --git a/share/mk/auto.obj.mk b/share/mk/auto.obj.mk
index b633937674be..7f903ee7efb8 100644
--- a/share/mk/auto.obj.mk
+++ b/share/mk/auto.obj.mk
@@ -61,11 +61,17 @@ __objdir_made != echo ${__objdir}/; umask ${OBJDIR_UMASK:U002}; \
 .endif
 # This causes make to use the specified directory as .OBJDIR
 .OBJDIR: ${__objdir}
-.if ${.OBJDIR:tA} != ${__objdir:tA} && ${__objdir_made:Uno:M${__objdir}/*} != ""
+.if ${.OBJDIR:tA} != ${__objdir:tA}
+# we did not get what we want - do we care?
+.if ${__objdir_made:Uno:M${__objdir}/*} != ""
 # watch out for __objdir being relative path
 .if !(${__objdir:M/*} == "" && ${.OBJDIR:tA} == ${${.CURDIR}/${__objdir}:L:tA})
 .error could not use ${__objdir}: .OBJDIR=${.OBJDIR}
 .endif
 .endif
+# apparently we can live with it
+# make sure we know what we have
+.OBJDIR: ${.CURDIR}
+.endif
 .endif
 .endif
diff --git a/share/mk/bsd.dep.mk b/share/mk/bsd.dep.mk
index 002f41e2d164..c6bf68b067f4 100644
--- a/share/mk/bsd.dep.mk
+++ b/share/mk/bsd.dep.mk
@@ -327,6 +327,10 @@ cleandepend:
 .endif
 .ORDER: cleandepend all
 .ORDER: cleandepend depend
+.if ${MK_AUTO_OBJ} == "yes"
+.ORDER: cleanobj depend
+.ORDER: cleandir depend
+.endif
 
 .if !target(checkdpadd) && (defined(DPADD) || defined(LDADD))
 _LDADD_FROM_DPADD=	${DPADD:R:T:C;^lib(.*)$;-l\1;g}
diff --git a/share/mk/bsd.init.mk b/share/mk/bsd.init.mk
index 407d71c0161e..f3851745a44d 100644
--- a/share/mk/bsd.init.mk
+++ b/share/mk/bsd.init.mk
@@ -10,6 +10,14 @@
 __<bsd.init.mk>__:
 .include <bsd.opts.mk>
 .-include "local.init.mk"
+
+.if ${MK_AUTO_OBJ} == "yes"
+# This is also done in bsd.obj.mk
+.if defined(NO_OBJ)
+.OBJDIR: ${.CURDIR}
+.endif
+.endif
+
 .if exists(${.CURDIR}/../Makefile.inc)
 .include "${.CURDIR}/../Makefile.inc"
 .endif
@@ -56,7 +64,8 @@ _SKIP_BUILD=	not building at level 0
     ${.TARGETS:M*install*} == ${.TARGETS} || \
     ${.TARGETS:Mclean*} == ${.TARGETS} || \
     ${.TARGETS:Mdestroy*} == ${.TARGETS} || \
-    make(obj) || make(analyze) || make(print-dir)
+    ${.TARGETS:Mobj} == ${.TARGETS} || \
+    make(analyze) || make(print-dir)
 # Skip building, but don't show a warning.
 _SKIP_BUILD=
 .endif
diff --git a/share/mk/bsd.obj.mk b/share/mk/bsd.obj.mk
index efbf59b85407..34ac227cd203 100644
--- a/share/mk/bsd.obj.mk
+++ b/share/mk/bsd.obj.mk
@@ -47,6 +47,7 @@ __<bsd.obj.mk>__:
 objwarn:
 obj:
 CANONICALOBJDIR= ${.OBJDIR}
+# This is also done in bsd.init.mk
 .if defined(NO_OBJ)
 # but this makefile does not want it!
 .OBJDIR: ${.CURDIR}
@@ -84,6 +85,11 @@ OBJTOP?= ${MAKEOBJDIR}
 CANONICALOBJDIR:=/usr/obj${.CURDIR}
 .endif
 
+.if defined(SRCTOP) && \
+    (${CANONICALOBJDIR} == /${RELDIR} || ${.OBJDIR} == /${RELDIR})
+.error .OBJDIR incorrectly set to /${RELDIR}
+.endif
+
 OBJTOP?= ${.OBJDIR:S,${.CURDIR},,}${SRCTOP}
 
 #
@@ -158,7 +164,8 @@ whereobj:
 .endif
 
 # Same check in bsd.progs.mk
-.if ${CANONICALOBJDIR} != ${.CURDIR} && exists(${CANONICALOBJDIR}/)
+.if ${CANONICALOBJDIR} != ${.CURDIR} && exists(${CANONICALOBJDIR}/) && \
+    (${MK_AUTO_OBJ} == "no" || ${.TARGETS:Nclean*:N*clean:Ndestroy*} == "")
 cleanobj:
 	-rm -rf ${CANONICALOBJDIR}
 .else
@@ -182,6 +189,10 @@ clean:
 .endif
 .endif
 .ORDER: clean all
+.if ${MK_AUTO_OBJ} == "yes"
+.ORDER: cleanobj all
+.ORDER: cleandir all
+.endif
 
 .include <bsd.subdir.mk>
 
diff --git a/share/mk/src.sys.obj.mk b/share/mk/src.sys.obj.mk
index 75541a5ba922..811a0ae20d40 100644
--- a/share/mk/src.sys.obj.mk
+++ b/share/mk/src.sys.obj.mk
@@ -77,13 +77,6 @@ OBJTOP:=	${OBJROOT}${TARGET:D${TARGET}.${TARGET_ARCH}:U${MACHINE}.${MACHINE_ARCH
 OBJTOP:=	${OBJROOT:H}
 .endif	# ${MK_UNIFIED_OBJDIR} == "yes"
 
-# Wait to validate MAKEOBJDIR until OBJTOP is set.
-.if defined(MAKEOBJDIR)
-.if ${MAKEOBJDIR:M/*} == ""
-.error Cannot use MAKEOBJDIR=${MAKEOBJDIR}${.newline}Unset MAKEOBJDIR to get default:  MAKEOBJDIR='${_default_makeobjdir}'
-.endif
-.endif
-
 # Fixup OBJROOT/OBJTOP if using MAKEOBJDIRPREFIX but leave it alone
 # for DIRDEPS_BUILD which really wants to know the absolute top at
 # all times.  This intenionally comes after adding TARGET.TARGET_ARCH
@@ -94,9 +87,24 @@ OBJTOP:=	${MAKEOBJDIRPREFIX}${SRCTOP}
 OBJROOT:=	${OBJTOP}/
 .endif
 
-# Try to enable MK_AUTO_OBJ by default if we can write to the OBJROOT.  Only
-# do this if AUTO_OBJ is not disabled by the user, not cleaning, and this
-# is the first make ran.
+# Wait to validate MAKEOBJDIR until OBJTOP is set.
+.if defined(MAKEOBJDIR)
+.if ${MAKEOBJDIR:M/*} == ""
+.error Cannot use MAKEOBJDIR=${MAKEOBJDIR}${.newline}Unset MAKEOBJDIR to get default:  MAKEOBJDIR='${_default_makeobjdir}'
+.endif
+.endif
+
+# __objdir is the expected .OBJDIR we want to use and that auto.obj.mk will
+# try to create.
+.if !empty(MAKEOBJDIRPREFIX)
+__objdir:=	${MAKEOBJDIRPREFIX}${.CURDIR}
+.elif !empty(MAKEOBJDIR)
+__objdir:=	${MAKEOBJDIR}
+.endif
+
+# Try to enable MK_AUTO_OBJ by default if we can write to the __objdir.  Only
+# do this if AUTO_OBJ is not disabled by the user, not cleaning, and this is
+# the first make ran.
 .if 0 && ${.MAKE.LEVEL} == 0 && \
     ${MK_AUTO_OBJ} == "no" && empty(.MAKEOVERRIDES:MMK_AUTO_OBJ) && \
     !defined(WITHOUT_AUTO_OBJ) && !make(showconfig) && !make(print-dir) && \
@@ -138,15 +146,13 @@ CheckAutoObj() { \
 		echo no; \
 	fi; \
 }
-.if !empty(MAKEOBJDIRPREFIX)
-WANTED_OBJDIR=	${MAKEOBJDIRPREFIX}${.CURDIR}
-.else
-WANTED_OBJDIR=	${MAKEOBJDIR}
+.if !empty(__objdir)
+__objdir_writable!= \
+	${CheckAutoObj}; CheckAutoObj "${__objdir}" || echo no
 .endif
-OBJDIR_WRITABLE!= \
-	${CheckAutoObj}; CheckAutoObj "${WANTED_OBJDIR}" || echo no
+__objdir_writable?= no
 # Export the decision to sub-makes.
-MK_AUTO_OBJ:=	${OBJDIR_WRITABLE}
+MK_AUTO_OBJ:=	${__objdir_writable}
 .export MK_AUTO_OBJ
 .elif make(showconfig)
 # Need to export for showconfig internally running make -dg1.  It is enabled
@@ -154,14 +160,11 @@ MK_AUTO_OBJ:=	${OBJDIR_WRITABLE}
 .export MK_AUTO_OBJ
 .endif	# ${MK_AUTO_OBJ} == "no" && ...
 
-# Assign this directory as .OBJDIR if possible after determining if AUTO_OBJ
-# can be enabled by default.
-.if ${MK_AUTO_OBJ} == "no"
+# Assign this directory as .OBJDIR if possible.
+#
 # The expected OBJDIR already exists, set it as .OBJDIR.
-.if !empty(MAKEOBJDIRPREFIX) && exists(${MAKEOBJDIRPREFIX}${.CURDIR})
-.OBJDIR: ${MAKEOBJDIRPREFIX}${.CURDIR}
-.elif exists(${MAKEOBJDIR})
-.OBJDIR: ${MAKEOBJDIR}
+.if !empty(__objdir) && exists(${__objdir})
+.OBJDIR: ${__objdir}
 # Special case to work around bmake bug.  If the top-level .OBJDIR does not yet
 # exist and MAKEOBJDIR is passed into environment and yield a blank value,
 # bmake will incorrectly set .OBJDIR=${SRCTOP}/ rather than the expected
@@ -169,5 +172,10 @@ MK_AUTO_OBJ:=	${OBJDIR_WRITABLE}
 .elif ${MAKE_VERSION} <= 20170720 && \
     ${.CURDIR} == ${SRCTOP} && ${.OBJDIR} == ${SRCTOP}/
 .OBJDIR: ${.CURDIR}
+.else
+# The OBJDIR we wanted does not yet exist, ensure we default to safe .CURDIR
+# in case make started with a bogus MAKEOBJDIR, that expanded before OBJTOP
+# was set, that happened to match some unexpected directory.  Either
+# auto.obj.mk or bsd.obj.mk will create the directory and fix .OBJDIR later.
+.OBJDIR: ${.CURDIR}
 .endif
-.endif	# ${MK_AUTO_OBJ} == "no"
diff --git a/share/mk/suite.test.mk b/share/mk/suite.test.mk
index 0b83002a5c27..50de9cc49cda 100644
--- a/share/mk/suite.test.mk
+++ b/share/mk/suite.test.mk
@@ -79,11 +79,11 @@ Kyuafile: Makefile
 
 KYUA= ${LOCALBASE}/bin/kyua
 
-MAKE_CHECK_SANDBOX_DIR=	${.OBJDIR}/checkdir
+MAKE_CHECK_SANDBOX_DIR=	checkdir
 CLEANDIRS+=	${MAKE_CHECK_SANDBOX_DIR}
 
 .if ${MK_MAKE_CHECK_USE_SANDBOX} != "no" && make(check)
-DESTDIR:=	${MAKE_CHECK_SANDBOX_DIR}
+DESTDIR:=	${.OBJDIR}/${MAKE_CHECK_SANDBOX_DIR}
 
 .if ${MK_MAKE_CHECK_TEST_WITH_COVERAGE} != "no"
 GCOV?=		gcov
diff --git a/share/mk/sys.mk b/share/mk/sys.mk
index c6bc8748fcc5..e8a6a98d9515 100644
--- a/share/mk/sys.mk
+++ b/share/mk/sys.mk
@@ -125,12 +125,6 @@ NO_META_IGNORE_HOST_HEADERS=	1
 # is not expected.
 .if !make(showconfig) && !make(print-dir)
 .sinclude <auto.obj.mk>
-# The .OBJDIR was not set, disable MK_AUTO_OBJ so downstream checks won't
-# assume .OBJDIR is proper.
-.if defined(__objdir) && ${.OBJDIR} != ${__objdir}
-.MAKEOVERRIDES+= MK_AUTO_OBJ
-MK_AUTO_OBJ=	no
-.endif
 .endif
 .endif	# ${MK_AUTO_OBJ} == "yes"
 .else # bmake
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 2ac09802fb5e..78777d814d7e 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -81,7 +81,6 @@ ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
 ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
 ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags));
 ASSYM(TD_PROC, offsetof(struct thread, td_proc));
-ASSYM(TD_TID, offsetof(struct thread, td_tid));
 ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
 
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
diff --git a/sys/arm/allwinner/if_awg.c b/sys/arm/allwinner/if_awg.c
index 2d3599b64f2b..d254b37e3cdd 100644
--- a/sys/arm/allwinner/if_awg.c
+++ b/sys/arm/allwinner/if_awg.c
@@ -92,7 +92,7 @@ __FBSDID("$FreeBSD$");
 #define	TX_SKIP(n, o)		(((n) + (o)) & (TX_DESC_COUNT - 1))
 #define	RX_NEXT(n)		(((n) + 1) & (RX_DESC_COUNT - 1))
 
-#define	TX_MAX_SEGS		10
+#define	TX_MAX_SEGS		20
 
 #define	SOFT_RST_RETRY		1000
 #define	MII_BUSY_RETRY		1000
@@ -192,6 +192,7 @@ struct awg_softc {
 	struct resource		*res[_RES_NITEMS];
 	struct mtx		mtx;
 	if_t			ifp;
+	device_t		dev;
 	device_t		miibus;
 	struct callout		stat_ch;
 	struct task		link_task;
@@ -421,14 +422,18 @@ awg_setup_txbuf(struct awg_softc *sc, int index, struct mbuf **mp)
 	    sc->tx.buf_map[index].map, m, segs, &nsegs, BUS_DMA_NOWAIT);
 	if (error == EFBIG) {
 		m = m_collapse(m, M_NOWAIT, TX_MAX_SEGS);
-		if (m == NULL)
+		if (m == NULL) {
+			device_printf(sc->dev, "awg_setup_txbuf: m_collapse failed\n");
 			return (0);
+		}
 		*mp = m;
 		error = bus_dmamap_load_mbuf_sg(sc->tx.buf_tag,
 		    sc->tx.buf_map[index].map, m, segs, &nsegs, BUS_DMA_NOWAIT);
 	}
-	if (error != 0)
+	if (error != 0) {
+		device_printf(sc->dev, "awg_setup_txbuf: bus_dmamap_load_mbuf_sg failed\n");
 		return (0);
+	}
 
 	bus_dmamap_sync(sc->tx.buf_tag, sc->tx.buf_map[index].map,
 	    BUS_DMASYNC_PREWRITE);
@@ -1613,6 +1618,7 @@ awg_attach(device_t dev)
 	int error;
 
 	sc = device_get_softc(dev);
+	sc->dev = dev;
 	sc->type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
 	node = ofw_bus_get_node(dev);
 
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index e6a3d367fd2e..932a2ae80c32 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -141,6 +141,7 @@ powerpc/mpc85xx/ds1553_bus_fdt.c	optional	ds1553 fdt
 powerpc/mpc85xx/ds1553_core.c	optional	ds1553
 powerpc/mpc85xx/fsl_diu.c	optional	mpc85xx diu
 powerpc/mpc85xx/fsl_espi.c	optional	mpc85xx spibus
+powerpc/mpc85xx/fsl_sata.c	optional	mpc85xx ata
 powerpc/mpc85xx/i2c.c		optional	iicbus fdt
 powerpc/mpc85xx/isa.c		optional	mpc85xx isa
 powerpc/mpc85xx/lbc.c		optional	mpc85xx
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c
index 7dbb22c610cf..c9c5357abebd 100644
--- a/sys/i386/i386/genassym.c
+++ b/sys/i386/i386/genassym.c
@@ -88,7 +88,6 @@ ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
 ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags));
 ASSYM(TD_PROC, offsetof(struct thread, td_proc));
 ASSYM(TD_MD, offsetof(struct thread, td_md));
-ASSYM(TD_TID, offsetof(struct thread, td_tid));
 
 ASSYM(TDP_CALLCHAIN, TDP_CALLCHAIN);
 
diff --git a/sys/i386/i386/swtch.s b/sys/i386/i386/swtch.s
index f38f52359a58..4562c7ed87f4 100644
--- a/sys/i386/i386/swtch.s
+++ b/sys/i386/i386/swtch.s
@@ -258,7 +258,6 @@ sw1:
 	movl	%eax,(%esp)
 
 	movl	%edx, PCPU(CURPCB)
-	movl	TD_TID(%ecx),%eax
 	movl	%ecx, PCPU(CURTHREAD)		/* into next thread */
 
 	/*
diff --git a/sys/kern/imgact_shell.c b/sys/kern/imgact_shell.c
index aaf521cf251e..bc21714aab29 100644
--- a/sys/kern/imgact_shell.c
+++ b/sys/kern/imgact_shell.c
@@ -97,8 +97,7 @@ CTASSERT(MAXSHELLCMDLEN >= MAXINTERP + 3);
  * 6.x branch on May 28, 2005 (matching __FreeBSD_version 600029).
  */
 int
-exec_shell_imgact(imgp)
-	struct image_params *imgp;
+exec_shell_imgact(struct image_params *imgp)
 {
 	const char *image_header = imgp->image_header;
 	const char *ihp, *interpb, *interpe, *maxp, *optb, *opte, *fname;
@@ -125,7 +124,7 @@ exec_shell_imgact(imgp)
 	 * However, we don't know how far into the page the contents are
 	 * valid -- the actual file might be much shorter than the page.
 	 * So find out the file size.
- 	 */
+	 */
 	error = VOP_GETATTR(imgp->vp, &vattr, imgp->proc->p_ucred);
 	if (error)
 		return (error);
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index d72c57d5f59b..58f37171718e 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -1579,8 +1579,7 @@ umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
 
 	uq_owner = owner->td_umtxq;
 	mtx_assert(&umtx_lock, MA_OWNED);
-	if (pi->pi_owner != NULL)
-		panic("pi_owner != NULL");
+	MPASS(pi->pi_owner == NULL);
 	pi->pi_owner = owner;
 	TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
 }
diff --git a/sys/powerpc/mpc85xx/fsl_sata.c b/sys/powerpc/mpc85xx/fsl_sata.c
new file mode 100644
index 000000000000..588825b400e0
--- /dev/null
+++ b/sys/powerpc/mpc85xx/fsl_sata.c
@@ -0,0 +1,1917 @@
+/*-
+ * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2017 Justin Hibbits <jhibbits@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/endian.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_debug.h>
+
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include "fsl_sata.h"
+
+struct fsl_sata_channel;
+struct fsl_sata_slot;
+enum fsl_sata_err_type;
+struct fsl_sata_cmd_tab;
+
+
+/* local prototypes */
+static int fsl_sata_init(device_t dev);
+static int fsl_sata_deinit(device_t dev);
+static int fsl_sata_suspend(device_t dev);
+static int fsl_sata_resume(device_t dev);
+static void fsl_sata_pm(void *arg);
+static void fsl_sata_intr(void *arg);
+static void fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus);
+static void fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb);
+static void fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
+static void fsl_sata_execute_transaction(struct fsl_sata_slot *slot);
+static void fsl_sata_timeout(struct fsl_sata_slot *slot);
+static void fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et);
+static int fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag);
+static void fsl_sata_dmainit(device_t dev);
+static void fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
+static void fsl_sata_dmafini(device_t dev);
+static void fsl_sata_slotsalloc(device_t dev);
+static void fsl_sata_slotsfree(device_t dev);
+static void fsl_sata_reset(struct fsl_sata_channel *ch);
+static void fsl_sata_start(struct fsl_sata_channel *ch);
+static void fsl_sata_stop(struct fsl_sata_channel *ch);
+
+static void fsl_sata_issue_recovery(struct fsl_sata_channel *ch);
+static void fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb);
+static void fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb);
+
+static void fsl_sataaction(struct cam_sim *sim, union ccb *ccb);
+static void fsl_satapoll(struct cam_sim *sim);
+
+static MALLOC_DEFINE(M_FSL_SATA, "FSL SATA driver", "FSL SATA driver data buffers");
+
+#define	recovery_type		spriv_field0
+#define	RECOVERY_NONE		0
+#define	RECOVERY_READ_LOG	1
+#define	RECOVERY_REQUEST_SENSE	2
+#define	recovery_slot		spriv_field1
+
+#define	FSL_SATA_P_CQR		0x0
+#define	FSL_SATA_P_CAR		0x4
+#define	FSL_SATA_P_CCR		0x10
+#define	FSL_SATA_P_CER			0x18
+#define	FSL_SATA_P_DER		0x20
+#define	FSL_SATA_P_CHBA		0x24
+#define	FSL_SATA_P_HSTS		0x28
+#define	  FSL_SATA_P_HSTS_HS_ON	  0x80000000
+#define	  FSL_SATA_P_HSTS_ME	  0x00040000
+#define	  FSL_SATA_P_HSTS_DLM	  0x00001000
+#define	  FSL_SATA_P_HSTS_FOT	  0x00000200
+#define	  FSL_SATA_P_HSTS_FOR	  0x00000100
+#define	  FSL_SATA_P_HSTS_FE	  0x00000020
+#define	  FSL_SATA_P_HSTS_PR	  0x00000010
+#define	  FSL_SATA_P_HSTS_SNTFU	  0x00000004
+#define	  FSL_SATA_P_HSTS_DE	  0x00000002
+#define	FSL_SATA_P_HCTRL	0x2c
+#define	  FSL_SATA_P_HCTRL_HC_ON  0x80000000
+#define	  FSL_SATA_P_HCTRL_HC_FORCE_OFF  0x40000000
+#define	  FSL_SATA_P_HCTRL_ENT	  0x10000000
+#define	  FSL_SATA_P_HCTRL_SNOOP  0x00000400
+#define	  FSL_SATA_P_HCTRL_PM	  0x00000200
+#define	  FSL_SATA_P_HCTRL_FATAL  0x00000020
+#define	  FSL_SATA_P_HCTRL_PHYRDY 0x00000010
+#define	  FSL_SATA_P_HCTRL_SIG	  0x00000008
+#define	  FSL_SATA_P_HCTRL_SNTFY  0x00000004
+#define	  FSL_SATA_P_HCTRL_DE	  0x00000002
+#define	  FSL_SATA_P_HCTRL_CC	  0x00000001
+#define	  FSL_SATA_P_HCTRL_INT_MASK	0x0000003f
+#define	FSL_SATA_P_CQPMP	0x30
+#define	FSL_SATA_P_SIG		0x34
+#define	FSL_SATA_P_ICC		0x38
+#define	  FSL_SATA_P_ICC_ITC_M	  0x1f000000
+#define	  FSL_SATA_P_ICC_ITC_S	  24
+#define	  FSL_SATA_P_ICC_ITTCV_M	  0x0007ffff
+#define	FSL_SATA_P_PCC		0x15c
+#define	  FSL_SATA_P_PCC_SLUMBER	  0x0000000c
+#define	  FSL_SATA_P_PCC_PARTIAL	  0x0000000a
+#define	  FSL_SATA_PCC_LPB_EN		  0x0000000e
+
+#define	FSL_SATA_MAX_SLOTS		16
+/* FSL_SATA register defines */
+
+#define	FSL_SATA_P_SSTS		0x100
+#define	FSL_SATA_P_SERR		0x104
+#define	FSL_SATA_P_SCTL		0x108
+#define	FSL_SATA_P_SNTF		0x10c
+
+/* Pessimistic prognosis on number of required S/G entries */
+#define	FSL_SATA_SG_ENTRIES	63
+/* Command list. 16 commands. First, 1Kbyte aligned. */
+#define	FSL_SATA_CL_OFFSET	0
+#define	FSL_SATA_CL_SIZE	16
+/* Command tables. Up to 32 commands, Each, 4-byte aligned. */
+#define	FSL_SATA_CT_OFFSET	(FSL_SATA_CL_OFFSET + FSL_SATA_CL_SIZE * FSL_SATA_MAX_SLOTS)
+#define	FSL_SATA_CT_SIZE	(96 + FSL_SATA_SG_ENTRIES * 16)
+/* Total main work area. */
+#define	FSL_SATA_WORK_SIZE	(FSL_SATA_CT_OFFSET + FSL_SATA_CT_SIZE * FSL_SATA_MAX_SLOTS)
+#define	FSL_SATA_MAX_XFER	(64 * 1024 * 1024)
+
+/* Some convenience macros for getting the CTP and CLP */
+#define	FSL_SATA_CTP_BUS(ch, slot)	\
+    ((ch->dma.work_bus + FSL_SATA_CT_OFFSET + (FSL_SATA_CT_SIZE * slot->slot)))
+#define FSL_SATA_PRD_OFFSET(prd) (96 + (prd) * 16)
+#define	FSL_SATA_CTP(ch, slot)		\
+    ((struct fsl_sata_cmd_tab *)(ch->dma.work + FSL_SATA_CT_OFFSET + \
+     (FSL_SATA_CT_SIZE * slot->slot)))
+#define	FSL_SATA_CLP(ch, slot)		\
+	((struct fsl_sata_cmd_list *) (ch->dma.work + FSL_SATA_CL_OFFSET + \
+	 (FSL_SATA_CL_SIZE * slot->slot)))
+
+struct fsl_sata_dma_prd {
+	uint32_t		dba;
+	uint32_t		reserved;
+	uint32_t		reserved2;
+	uint32_t		dwc_flg;		/* 0 based */
+#define	FSL_SATA_PRD_MASK		0x01fffffc	/* max 32MB */
+#define	FSL_SATA_PRD_MAX		(FSL_SATA_PRD_MASK + 4)
+#define	FSL_SATA_PRD_SNOOP		0x10000000
+#define	FSL_SATA_PRD_EXT		0x80000000
+} __packed;
+
+struct fsl_sata_cmd_tab {
+	uint8_t			cfis[32];
+	uint8_t			sfis[32];
+	uint8_t			acmd[16];
+	uint8_t			reserved[16];
+	struct fsl_sata_dma_prd	prd_tab[FSL_SATA_SG_ENTRIES];
+#define	FSL_SATA_PRD_EXT_INDEX	15
+#define FSL_SATA_PRD_MAX_DIRECT	16
+} __packed;
+
+struct fsl_sata_cmd_list {
+	uint32_t			cda;		/* word aligned */
+	uint16_t			fis_length;	/* length in bytes (aligned to words) */
+	uint16_t			prd_length;	/* PRD entries */
+	uint32_t			ttl;
+	uint32_t			cmd_flags;
+#define	FSL_SATA_CMD_TAG_MASK		0x001f
+#define	FSL_SATA_CMD_ATAPI		0x0020
+#define	FSL_SATA_CMD_BIST		0x0040
+#define	FSL_SATA_CMD_RESET		0x0080
+#define	FSL_SATA_CMD_QUEUED		0x0100
+#define	FSL_SATA_CMD_SNOOP		0x0200
+#define	FSL_SATA_CMD_VBIST		0x0400
+#define	FSL_SATA_CMD_WRITE		0x0800
+
+} __packed;
+
+/* misc defines */
+#define	ATA_IRQ_RID		0
+#define	ATA_INTR_FLAGS		(INTR_MPSAFE|INTR_TYPE_BIO|INTR_ENTROPY)
+
+struct ata_dmaslot {
+	bus_dmamap_t		data_map;	/* data DMA map */
+	int			nsegs;		/* Number of segs loaded */
+};
+
+/* structure holding DMA related information */
+struct ata_dma {
+	bus_dma_tag_t		 work_tag;	/* workspace DMA tag */
+	bus_dmamap_t		 work_map;	/* workspace DMA map */
+	uint8_t			*work;		/* workspace */
+	bus_addr_t		 work_bus;	/* bus address of work */
+	bus_dma_tag_t		data_tag;	/* data DMA tag */
+};
+
+enum fsl_sata_slot_states {
+	FSL_SATA_SLOT_EMPTY,
+	FSL_SATA_SLOT_LOADING,
+	FSL_SATA_SLOT_RUNNING,
+	FSL_SATA_SLOT_EXECUTING
+};
+
+struct fsl_sata_slot {
+	struct fsl_sata_channel		*ch;		/* Channel */
+	uint8_t				 slot;		/* Number of this slot */
+	enum fsl_sata_slot_states	 state;	/* Slot state */
+	union ccb			*ccb;		/* CCB occupying slot */
+	struct ata_dmaslot		 dma;	/* DMA data of this slot */
+	struct callout			 timeout;	/* Execution timeout */
+	uint32_t			 ttl;
+};
+
+struct fsl_sata_device {
+	int			revision;
+	int			mode;
+	u_int			bytecount;
+	u_int			atapi;
+	u_int			tags;
+	u_int			caps;
+};
+
+/* structure describing an ATA channel */
+struct fsl_sata_channel {
+	device_t		dev;		/* Device handle */
+	int			unit;		/* Physical channel */
+	struct resource		*r_mem;		/* Memory of this channel */
+	struct resource		*r_irq;		/* Interrupt of this channel */
+	void			*ih;		/* Interrupt handle */
+	struct ata_dma		dma;		/* DMA data */
+	struct cam_sim		*sim;
+	struct cam_path		*path;
+	uint32_t		caps;		/* Controller capabilities */
+	int			pm_level;	/* power management level */
+	int			devices;	/* What is present */
+	int			pm_present;	/* PM presence reported */
+
+	union ccb		*hold[FSL_SATA_MAX_SLOTS];
+	struct fsl_sata_slot	slot[FSL_SATA_MAX_SLOTS];
+	uint32_t		oslots;		/* Occupied slots */
+	uint32_t		rslots;		/* Running slots */
+	uint32_t		aslots;		/* Slots with atomic commands  */
+	uint32_t		eslots;		/* Slots in error */
+	uint32_t		toslots;	/* Slots in timeout */
+	int			lastslot;	/* Last used slot */
+	int			taggedtarget;	/* Last tagged target */
+	int			numrslots;	/* Number of running slots */
+	int			numrslotspd[16];/* Number of running slots per dev */
+	int			numtslots;	/* Number of tagged slots */
+	int			numtslotspd[16];/* Number of tagged slots per dev */
+	int			numhslots;	/* Number of held slots */
+	int			recoverycmd;	/* Our READ LOG active */
+	int			fatalerr;	/* Fatal error happend */
+	int			resetting;	/* Hard-reset in progress. */
+	int			resetpolldiv;	/* Hard-reset poll divider. */
+	union ccb		*frozen;	/* Frozen command */
+	struct callout		pm_timer;	/* Power management events */
+	struct callout		reset_timer;	/* Hard-reset timeout */
+
+	struct fsl_sata_device	user[16];	/* User-specified settings */
+	struct fsl_sata_device	curr[16];	/* Current settings */
+
+	struct mtx_padalign	mtx;		/* state lock */
+	STAILQ_HEAD(, ccb_hdr)	doneq;		/* queue of completed CCBs */
+	int			batch;		/* doneq is in use */
+};
+
+enum fsl_sata_err_type {
+	FSL_SATA_ERR_NONE,		/* No error */
+	FSL_SATA_ERR_INVALID,	/* Error detected by us before submitting. */
+	FSL_SATA_ERR_INNOCENT,	/* Innocent victim. */
+	FSL_SATA_ERR_TFE,		/* Task File Error. */
+	FSL_SATA_ERR_SATA,		/* SATA error. */
+	FSL_SATA_ERR_TIMEOUT,	/* Command execution timeout. */
+	FSL_SATA_ERR_NCQ,		/* NCQ command error. CCB should be put on hold
+				 * until READ LOG executed to reveal error. */
+};
+
+/* macros to hide busspace uglyness */
+#define	ATA_INB(res, offset) \
+	bus_read_1((res), (offset))
+#define	ATA_INW(res, offset) \
+	bus_read_2((res), (offset))
+#define	ATA_INL(res, offset) \
+	bus_read_4((res), (offset))
+#define	ATA_INSW(res, offset, addr, count) \
+	bus_read_multi_2((res), (offset), (addr), (count))
+#define	ATA_INSW_STRM(res, offset, addr, count) \
+	bus_read_multi_stream_2((res), (offset), (addr), (count))
+#define	ATA_INSL(res, offset, addr, count) \
+	bus_read_multi_4((res), (offset), (addr), (count))
+#define	ATA_INSL_STRM(res, offset, addr, count) \
+	bus_read_multi_stream_4((res), (offset), (addr), (count))
+#define	ATA_OUTB(res, offset, value) \
+	bus_write_1((res), (offset), (value))
+#define	ATA_OUTW(res, offset, value) \
+	bus_write_2((res), (offset), (value))
+#define	ATA_OUTL(res, offset, value) \
+	bus_write_4((res), (offset), (value))
+#define	ATA_OUTSW(res, offset, addr, count) \
+	bus_write_multi_2((res), (offset), (addr), (count))
+#define	ATA_OUTSW_STRM(res, offset, addr, count) \
+	bus_write_multi_stream_2((res), (offset), (addr), (count))
+#define	ATA_OUTSL(res, offset, addr, count) \
+	bus_write_multi_4((res), (offset), (addr), (count))
+#define	ATA_OUTSL_STRM(res, offset, addr, count) \
+	bus_write_multi_stream_4((res), (offset), (addr), (count))
+
+static int
+fsl_sata_probe(device_t dev)
+{
+
+	if (!ofw_bus_is_compatible(dev, "fsl,pq-sata-v2") &&
+	    !ofw_bus_is_compatible(dev, "fsl,pq-sata"))
+		return (ENXIO);
+
+	device_set_desc_copy(dev, "Freescale Integrated SATA Controller");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+fsl_sata_attach(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	struct cam_devq *devq;
+	int rid, error, i, sata_rev = 0;
+
+	ch->dev = dev;
+	ch->unit = (intptr_t)device_get_ivars(dev);
+	mtx_init(&ch->mtx, "FSL SATA channel lock", NULL, MTX_DEF);
+	ch->pm_level = 0;
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "pm_level", &ch->pm_level);
+	STAILQ_INIT(&ch->doneq);
+	if (ch->pm_level > 3)
+		callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "sata_rev", &sata_rev);
+	for (i = 0; i < 16; i++) {
+		ch->user[i].revision = sata_rev;
+		ch->user[i].mode = 0;
+		ch->user[i].bytecount = 8192;
+		ch->user[i].tags = FSL_SATA_MAX_SLOTS;
+		ch->user[i].caps = 0;
+		ch->curr[i] = ch->user[i];
+		if (ch->pm_level) {
+			ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
+			    CTS_SATA_CAPS_D_PMREQ;
+		}
+		ch->user[i].caps |= CTS_SATA_CAPS_H_AN;
+	}
+	rid = 0;
+	if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &rid, RF_ACTIVE)))
+		return (ENXIO);
+	rman_set_bustag(ch->r_mem, &bs_le_tag);
+	fsl_sata_dmainit(dev);
+	fsl_sata_slotsalloc(dev);
+	fsl_sata_init(dev);
+	rid = ATA_IRQ_RID;
+	if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &rid, RF_SHAREABLE | RF_ACTIVE))) {
+		device_printf(dev, "Unable to map interrupt\n");
+		error = ENXIO;
+		goto err0;
+	}
+	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
+	    fsl_sata_intr, ch, &ch->ih))) {
+		device_printf(dev, "Unable to setup interrupt\n");
+		error = ENXIO;
+		goto err1;
+	}
+	mtx_lock(&ch->mtx);
+	/* Create the device queue for our SIM. */
+	devq = cam_simq_alloc(FSL_SATA_MAX_SLOTS);
+	if (devq == NULL) {
+		device_printf(dev, "Unable to allocate simq\n");
+		error = ENOMEM;
+		goto err1;
+	}
+	/* Construct SIM entry */
+	ch->sim = cam_sim_alloc(fsl_sataaction, fsl_satapoll, "fslsata", ch,
+	    device_get_unit(dev), (struct mtx *)&ch->mtx, 2, FSL_SATA_MAX_SLOTS,
+	    devq);
+	if (ch->sim == NULL) {
+		cam_simq_free(devq);
+		device_printf(dev, "unable to allocate sim\n");
+		error = ENOMEM;
+		goto err1;
+	}
+	if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
+		device_printf(dev, "unable to register xpt bus\n");
+		error = ENXIO;
+		goto err2;
+	}
+	if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
+	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+		device_printf(dev, "unable to create path\n");
+		error = ENXIO;
+		goto err3;
+	}
+	if (ch->pm_level > 3) {
+		callout_reset(&ch->pm_timer,
+		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
+		    fsl_sata_pm, ch);
+	}
+	mtx_unlock(&ch->mtx);
+	return (0);
+
+err3:
+	xpt_bus_deregister(cam_sim_path(ch->sim));
+err2:
+	cam_sim_free(ch->sim, /*free_devq*/TRUE);
+err1:
+	mtx_unlock(&ch->mtx);
+	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
+err0:
+	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
+	mtx_destroy(&ch->mtx);
+	return (error);
+}
+
+static int
+fsl_sata_detach(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+
+	mtx_lock(&ch->mtx);
+	xpt_async(AC_LOST_DEVICE, ch->path, NULL);
+
+	xpt_free_path(ch->path);
+	xpt_bus_deregister(cam_sim_path(ch->sim));
+	cam_sim_free(ch->sim, /*free_devq*/TRUE);
+	mtx_unlock(&ch->mtx);
+
+	if (ch->pm_level > 3)
+		callout_drain(&ch->pm_timer);
+	bus_teardown_intr(dev, ch->r_irq, ch->ih);
+	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
+
+	fsl_sata_deinit(dev);
+	fsl_sata_slotsfree(dev);
+	fsl_sata_dmafini(dev);
+
+	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
+	mtx_destroy(&ch->mtx);
+	return (0);
+}
+
+static int
+fsl_sata_wait_register(struct fsl_sata_channel *ch, bus_size_t off,
+    unsigned int mask, unsigned int val, int t)
+{
+	int timeout = 0;
+	uint32_t rval;
+
+	while (((rval = ATA_INL(ch->r_mem, off)) & mask) != val) {
+		if (timeout > t) {
+			return (EBUSY);
+		}
+		DELAY(1000);
+		timeout++;
+	}
+	return (0);
+}
+
+static int
+fsl_sata_init(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	uint64_t work;
+	uint32_t r;
+
+	/* Disable port interrupts */
+	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
+	r &= ~FSL_SATA_P_HCTRL_HC_ON;
+	r |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
+	fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
+	    FSL_SATA_P_HSTS_HS_ON, 0, 1000);
+	/* Setup work areas */
+	work = ch->dma.work_bus + FSL_SATA_CL_OFFSET;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CHBA, work);
+	r &= ~FSL_SATA_P_HCTRL_ENT;
+	r &= ~FSL_SATA_P_HCTRL_PM;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r);
+	r = ATA_INL(ch->r_mem, FSL_SATA_P_PCC);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, r & ~FSL_SATA_PCC_LPB_EN);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_ICC, (1 << FSL_SATA_P_ICC_ITC_S));
+	fsl_sata_start(ch);
+	return (0);
+}
+
+static int
+fsl_sata_deinit(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	uint32_t r;
+
+	/* Disable port interrupts. */
+	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
+	/* Reset command register. */
+	fsl_sata_stop(ch);
+	/* Allow everything, including partial and slumber modes. */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, 0);
+	DELAY(100);
+	/* Disable PHY. */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, ATA_SC_DET_DISABLE);
+	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
+	/* Turn off the controller. */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_HC_ON);
+	return (0);
+}
+
+static int
+fsl_sata_suspend(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+
+	mtx_lock(&ch->mtx);
+	xpt_freeze_simq(ch->sim, 1);
+	while (ch->oslots)
+		msleep(ch, &ch->mtx, PRIBIO, "fsl_satasusp", hz/100);
+	fsl_sata_deinit(dev);
+	mtx_unlock(&ch->mtx);
+	return (0);
+}
+
+static int
+fsl_sata_resume(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+
+	mtx_lock(&ch->mtx);
+	fsl_sata_init(dev);
+	fsl_sata_reset(ch);
+	xpt_release_simq(ch->sim, TRUE);
+	mtx_unlock(&ch->mtx);
+	return (0);
+}
+
+devclass_t fsl_satach_devclass;
+static device_method_t fsl_satach_methods[] = {
+	DEVMETHOD(device_probe,     fsl_sata_probe),
+	DEVMETHOD(device_attach,    fsl_sata_attach),
+	DEVMETHOD(device_detach,    fsl_sata_detach),
+	DEVMETHOD(device_suspend,   fsl_sata_suspend),
+	DEVMETHOD(device_resume,    fsl_sata_resume),
+	DEVMETHOD_END
+};
+static driver_t fsl_satach_driver = {
+	"fslsata",
+	fsl_satach_methods,
+	sizeof(struct fsl_sata_channel)
+};
+DRIVER_MODULE(fsl_satach, simplebus, fsl_satach_driver, fsl_satach_devclass, NULL, NULL);
+
+struct fsl_sata_dc_cb_args {
+	bus_addr_t maddr;
+	int error;
+};
+
+static void
+fsl_sata_dmainit(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	struct fsl_sata_dc_cb_args dcba;
+
+	/* Command area. */
+	if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
+	    NULL, NULL, FSL_SATA_WORK_SIZE, 1, FSL_SATA_WORK_SIZE,
+	    0, NULL, NULL, &ch->dma.work_tag))
+		goto error;
+	if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work,
+	    BUS_DMA_ZERO, &ch->dma.work_map))
+		goto error;
+	if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
+	    FSL_SATA_WORK_SIZE, fsl_sata_dmasetupc_cb, &dcba, 0) || dcba.error) {
+		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
+		goto error;
+	}
+	ch->dma.work_bus = dcba.maddr;
+	/* Data area. */
+	if (bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
+	    NULL, NULL, FSL_SATA_MAX_XFER,
+	    FSL_SATA_SG_ENTRIES - 1, FSL_SATA_PRD_MAX,
+	    0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
+		goto error;
+	}
+	if (bootverbose)
+		device_printf(dev, "work area: %p\n", ch->dma.work);
+	return;
+
+error:
+	device_printf(dev, "WARNING - DMA initialization failed\n");
+	fsl_sata_dmafini(dev);
+}
+
+static void
+fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
+{
+	struct fsl_sata_dc_cb_args *dcba = (struct fsl_sata_dc_cb_args *)xsc;
+
+	if (!(dcba->error = error))
+		dcba->maddr = segs[0].ds_addr;
+}
+
+static void
+fsl_sata_dmafini(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+
+	if (ch->dma.data_tag) {
+		bus_dma_tag_destroy(ch->dma.data_tag);
+		ch->dma.data_tag = NULL;
+	}
+	if (ch->dma.work_bus) {
+		bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
+		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
+		ch->dma.work_bus = 0;
+		ch->dma.work = NULL;
+	}
+	if (ch->dma.work_tag) {
+		bus_dma_tag_destroy(ch->dma.work_tag);
+		ch->dma.work_tag = NULL;
+	}
+}
+
+static void
+fsl_sata_slotsalloc(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	int i;
+
+	/* Alloc and setup command/dma slots */
+	bzero(ch->slot, sizeof(ch->slot));
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		struct fsl_sata_slot *slot = &ch->slot[i];
+
+		slot->ch = ch;
+		slot->slot = i;
+		slot->state = FSL_SATA_SLOT_EMPTY;
+		slot->ccb = NULL;
+		callout_init_mtx(&slot->timeout, &ch->mtx, 0);
+
+		if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
+			device_printf(ch->dev, "FAILURE - create data_map\n");
+	}
+}
+
+static void
+fsl_sata_slotsfree(device_t dev)
+{
+	struct fsl_sata_channel *ch = device_get_softc(dev);
+	int i;
+
+	/* Free all dma slots */
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		struct fsl_sata_slot *slot = &ch->slot[i];
+
+		callout_drain(&slot->timeout);
+		if (slot->dma.data_map) {
+			bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
+			slot->dma.data_map = NULL;
+		}
+	}
+}
+
+static int
+fsl_sata_phy_check_events(struct fsl_sata_channel *ch, u_int32_t serr)
+{
+
+	if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
+	    ((ch->pm_level != 0) && (serr & ATA_SE_EXCHANGED))) {
+		u_int32_t status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS);
+		union ccb *ccb;
+
+		if (bootverbose) {
+			if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE)
+				device_printf(ch->dev, "CONNECT requested\n");
+			else
+				device_printf(ch->dev, "DISCONNECT requested\n");
+		}
+		/* Issue soft reset */
+		xpt_async(AC_BUS_RESET, ch->path, NULL);
+		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
+			return (0);
+		if (xpt_create_path(&ccb->ccb_h.path, NULL,
+		    cam_sim_path(ch->sim),
+		    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+			xpt_free_ccb(ccb);
+			return (0);
+		}
+		xpt_rescan(ccb);
+		return (1);
+	}
+	return (0);
+}
+
+static void
+fsl_sata_notify_events(struct fsl_sata_channel *ch, u_int32_t status)
+{
+	struct cam_path *dpath;
+	int i;
+
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_SNTF, status);
+	if (bootverbose)
+		device_printf(ch->dev, "SNTF 0x%04x\n", status);
+	for (i = 0; i < 16; i++) {
+		if ((status & (1 << i)) == 0)
+			continue;
+		if (xpt_create_path(&dpath, NULL,
+		    xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) {
+			xpt_async(AC_SCSI_AEN, dpath, NULL);
+			xpt_free_path(dpath);
+		}
+	}
+}
+
+static void
+fsl_sata_done(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+
+	mtx_assert(&ch->mtx, MA_OWNED);
+	if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 ||
+	    ch->batch == 0) {
+		xpt_done(ccb);
+		return;
+	}
+
+	STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe);
+}
+
+static void
+fsl_sata_intr(void *arg)
+{
+	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
+	struct ccb_hdr *ccb_h;
+	uint32_t istatus;
+	STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq);
+
+	/* Read interrupt statuses. */
+	istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & 0x7ffff;
+	if ((istatus & 0x3f) == 0)
+		return;
+
+	mtx_lock(&ch->mtx);
+	ch->batch = 1;
+	fsl_sata_intr_main(ch, istatus);
+	ch->batch = 0;
+	/*
+	 * Prevent the possibility of issues caused by processing the queue
+	 * while unlocked below by moving the contents to a local queue.
+	 */
+	STAILQ_CONCAT(&tmp_doneq, &ch->doneq);
+	mtx_unlock(&ch->mtx);
+	while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) {
+		STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe);
+		xpt_done_direct((union ccb *)ccb_h);
+	}
+	/* Clear interrupt statuses. */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, istatus & 0x3f);
+
+}
+
+static void
+fsl_sata_pm(void *arg)
+{
+	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
+	uint32_t work;
+
+	if (ch->numrslots != 0)
+		return;
+	work = ATA_INL(ch->r_mem, FSL_SATA_P_PCC) & ~FSL_SATA_PCC_LPB_EN;
+	if (ch->pm_level == 4)
+		work |= FSL_SATA_P_PCC_PARTIAL;
+	else
+		work |= FSL_SATA_P_PCC_SLUMBER;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, work);
+}
+
+/* XXX: interrupt todo */
+static void
+fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus)
+{
+	uint32_t cer, der, serr = 0, sntf = 0, ok, err;
+	enum fsl_sata_err_type et;
+	int i;
+
+	/* Complete all successful commands. */
+	ok = ATA_INL(ch->r_mem, FSL_SATA_P_CCR);
+	if (ch->aslots == 0)
+		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+			if (((ok >> i) & 1) && ch->slot[i].ccb != NULL)
+				fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_NONE);
+		}
+	/* Read command statuses. */
+	if (istatus & FSL_SATA_P_HSTS_SNTFU)
+		sntf = ATA_INL(ch->r_mem, FSL_SATA_P_SNTF);
+	/* XXX: Process PHY events */
+	serr = ATA_INL(ch->r_mem, FSL_SATA_P_SERR);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, serr);
+	if (istatus & (FSL_SATA_P_HSTS_PR)) {
+		if (serr) {
+			fsl_sata_phy_check_events(ch, serr);
+		}
+	}
+	/* Process command errors */
+	err = (istatus & (FSL_SATA_P_HSTS_FE | FSL_SATA_P_HSTS_DE));
+	cer = ATA_INL(ch->r_mem, FSL_SATA_P_CER);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, cer);
+	der = ATA_INL(ch->r_mem, FSL_SATA_P_DER);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, der);
+	/* On error, complete the rest of commands with error statuses. */
+	if (err) {
+		if (ch->frozen) {
+			union ccb *fccb = ch->frozen;
+			ch->frozen = NULL;
+			fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
+			if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
+				xpt_freeze_devq(fccb->ccb_h.path, 1);
+				fccb->ccb_h.status |= CAM_DEV_QFRZN;
+			}
+			fsl_sata_done(ch, fccb);
+		}
+		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+			if (ch->slot[i].ccb == NULL)
+				continue;
+			if ((cer & (1 << i)) != 0)
+				et = FSL_SATA_ERR_TFE;
+			else if ((der & (1 << ch->slot[i].ccb->ccb_h.target_id)) != 0)
+				et = FSL_SATA_ERR_SATA;
+			else
+				et = FSL_SATA_ERR_INVALID;
+			fsl_sata_end_transaction(&ch->slot[i], et);
+		}
+	}
+	/* Process NOTIFY events */
+	if (sntf)
+		fsl_sata_notify_events(ch, sntf);
+}
+
+/* Must be called with channel locked. */
+static int
+fsl_sata_check_collision(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+	int t = ccb->ccb_h.target_id;
+
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
+		/* Tagged command while we have no supported tag free. */
+		if (((~ch->oslots) & (0xffffffff >> (32 -
+		    ch->curr[t].tags))) == 0)
+			return (1);
+		/* Tagged command while untagged are active. */
+		if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0)
+			return (1);
+	} else {
+		/* Untagged command while tagged are active. */
+		if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0)
+			return (1);
+	}
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) {
+		/* Atomic command while anything active. */
+		if (ch->numrslots != 0)
+			return (1);
+	}
+	/* We have some atomic command running. */
+	if (ch->aslots != 0)
+		return (1);
+	return (0);
+}
+
+/* Must be called with channel locked. */
+static void
+fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+	struct fsl_sata_slot *slot;
+	int tag, tags;
+
+	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
+	    ("fsl_sata_begin_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
+	/* Choose empty slot. */
+	tags = FSL_SATA_MAX_SLOTS;
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
+		tags = ch->curr[ccb->ccb_h.target_id].tags;
+	if (ch->lastslot + 1 < tags)
+		tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
+	else
+		tag = 0;
+	if (tag == 0 || tag + ch->lastslot >= tags)
+		tag = ffs(~ch->oslots) - 1;
+	else
+		tag += ch->lastslot;
+	ch->lastslot = tag;
+	/* Occupy chosen slot. */
+	slot = &ch->slot[tag];
+	slot->ccb = ccb;
+	slot->ttl = 0;
+	/* Stop PM timer. */
+	if (ch->numrslots == 0 && ch->pm_level > 3)
+		callout_stop(&ch->pm_timer);
+	/* Update channel stats. */
+	ch->oslots |= (1 << tag);
+	ch->numrslots++;
+	ch->numrslotspd[ccb->ccb_h.target_id]++;
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
+		ch->numtslots++;
+		ch->numtslotspd[ccb->ccb_h.target_id]++;
+		ch->taggedtarget = ccb->ccb_h.target_id;
+	}
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
+		ch->aslots |= (1 << tag);
+	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+		slot->state = FSL_SATA_SLOT_LOADING;
+		bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
+		    fsl_sata_dmasetprd, slot, 0);
+	} else {
+		slot->dma.nsegs = 0;
+		fsl_sata_execute_transaction(slot);
+	}
+
+	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
+	    ("fsl_sata_begin_transaction exit\n"));
+}
+
+/* Locked by busdma engine. */
+static void
+fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{    
+	struct fsl_sata_slot *slot = arg;
+	struct fsl_sata_channel *ch = slot->ch;
+	struct fsl_sata_cmd_tab *ctp;
+	struct fsl_sata_dma_prd *prd;
+	int i, j, len, extlen;
+
+	if (error) {
+		device_printf(ch->dev, "DMA load error %d\n", error);
+		fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
+		return;
+	}
+	KASSERT(nsegs <= FSL_SATA_SG_ENTRIES - 1,
+	    ("too many DMA segment entries\n"));
+	/* Get a piece of the workspace for this request */
+	ctp = FSL_SATA_CTP(ch, slot);
+	/* Fill S/G table */
+	prd = &ctp->prd_tab[0];
+	for (i = 0, j = 0; i < nsegs; i++, j++) {
+		if (j == FSL_SATA_PRD_EXT_INDEX &&
+		    FSL_SATA_PRD_MAX_DIRECT < nsegs) {
+			prd[j].dba = FSL_SATA_CTP_BUS(ch, slot) +
+				     FSL_SATA_PRD_OFFSET(j+1);
+			j++;
+		}
+		len = segs[i].ds_len;
+		len = roundup2(len, sizeof(uint32_t));
+		prd[i].dba = htole32((uint32_t)segs[i].ds_addr);
+		prd[i].dwc_flg = htole32(FSL_SATA_PRD_SNOOP | len);
+		slot->ttl += len;
+		if (j > FSL_SATA_PRD_MAX_DIRECT)
+			extlen += len;
+	}
+	slot->dma.nsegs = j;
+	if (j > FSL_SATA_PRD_MAX_DIRECT)
+		prd[FSL_SATA_PRD_EXT_INDEX].dwc_flg = 
+		    htole32(FSL_SATA_PRD_SNOOP | FSL_SATA_PRD_EXT | extlen);
+	bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
+	    ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
+	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
+	fsl_sata_execute_transaction(slot);
+}
+
+/* Must be called with channel locked. */
+static void
+fsl_sata_execute_transaction(struct fsl_sata_slot *slot)
+{
+	struct fsl_sata_channel *ch = slot->ch;
+	struct fsl_sata_cmd_tab *ctp;
+	struct fsl_sata_cmd_list *clp;
+	union ccb *ccb = slot->ccb;
+	int port = ccb->ccb_h.target_id & 0x0f;
+	int fis_size, i, softreset;
+	uint32_t tmp;
+	uint32_t cmd_flags = FSL_SATA_CMD_WRITE | FSL_SATA_CMD_SNOOP;
+
+	softreset = 0;
+	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
+	    ("fsl_sata_execute_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
+	/* Get a piece of the workspace for this request */
+	ctp = FSL_SATA_CTP(ch, slot);
+	/* Setup the FIS for this request */
+	if (!(fis_size = fsl_sata_setup_fis(ch, ctp, ccb, slot->slot))) {
+		device_printf(ch->dev, "Setting up SATA FIS failed\n");
+		fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
+		return;
+	}
+	/* Setup the command list entry */
+	clp = FSL_SATA_CLP(ch, slot);
+	clp->fis_length = htole16(fis_size);
+	clp->prd_length = htole16(slot->dma.nsegs);
+	/* Special handling for Soft Reset command. */
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
+		if (ccb->ataio.cmd.control & ATA_A_RESET) {
+			softreset = 1;
+			cmd_flags |= FSL_SATA_CMD_RESET;
+		} else {
+			/* Prepare FIS receive area for check. */
+			for (i = 0; i < 32; i++)
+				ctp->sfis[i] = 0xff;
+			softreset = 2;
+		}
+	}
+	if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)
+		cmd_flags |= FSL_SATA_CMD_QUEUED;
+	clp->cmd_flags = htole32(cmd_flags |
+	    (ccb->ccb_h.func_code == XPT_SCSI_IO ?  FSL_SATA_CMD_ATAPI : 0) |
+	    slot->slot);
+	clp->ttl = htole32(slot->ttl);
+	clp->cda = htole32(FSL_SATA_CTP_BUS(ch, slot));
+	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
+	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	/* Issue command to the controller. */
+	slot->state = FSL_SATA_SLOT_RUNNING;
+	ch->rslots |= (1 << slot->slot);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CQPMP, port);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CQR, (1 << slot->slot));
+	/* Device reset commands don't interrupt. Poll them. */
+	if (ccb->ccb_h.func_code == XPT_ATA_IO &&
+	    (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) {
+		int count, timeout = ccb->ccb_h.timeout * 100;
+		enum fsl_sata_err_type et = FSL_SATA_ERR_NONE;
+
+		for (count = 0; count < timeout; count++) {
+			DELAY(10);
+			tmp = 0;
+			if (softreset == 2) {
+				tmp = ATA_INL(ch->r_mem, FSL_SATA_P_SIG);
+				if (tmp != 0 && tmp != 0xffffffff)
+					break;
+				continue;
+			}
+			if ((ATA_INL(ch->r_mem, FSL_SATA_P_CCR) & (1 << slot->slot)) != 0)
+				break;
+		}
+
+		if (timeout && (count >= timeout)) {
+			device_printf(ch->dev, "Poll timeout on slot %d port %d (round %d)\n",
+			    slot->slot, port, softreset);
+			device_printf(ch->dev, "hsts %08x cqr %08x ccr %08x ss %08x "
+			    "rs %08x cer %08x der %08x serr %08x car %08x sig %08x\n",
+			    ATA_INL(ch->r_mem, FSL_SATA_P_HSTS),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_CQR),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_CCR),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_SSTS), ch->rslots,
+			    ATA_INL(ch->r_mem, FSL_SATA_P_CER),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_DER),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_SERR),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_CAR),
+			    ATA_INL(ch->r_mem, FSL_SATA_P_SIG));
+			et = FSL_SATA_ERR_TIMEOUT;
+		}
+
+		fsl_sata_end_transaction(slot, et);
+		return;
+	}
+	/* Start command execution timeout */
+	callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2,
+	    0, (timeout_t*)fsl_sata_timeout, slot, 0);
+	return;
+}
+
+/* Must be called with channel locked. */
+static void
+fsl_sata_process_timeout(struct fsl_sata_channel *ch)
+{
+	int i;
+
+	mtx_assert(&ch->mtx, MA_OWNED);
+	/* Handle the rest of commands. */
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		/* Do we have a running request on slot? */
+		if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
+			continue;
+		fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_TIMEOUT);
+	}
+}
+
+/* Must be called with channel locked. */
+static void
+fsl_sata_rearm_timeout(struct fsl_sata_channel *ch)
+{
+	int i;
+
+	mtx_assert(&ch->mtx, MA_OWNED);
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		struct fsl_sata_slot *slot = &ch->slot[i];
+
+		/* Do we have a running request on slot? */
+		if (slot->state < FSL_SATA_SLOT_RUNNING)
+			continue;
+		if ((ch->toslots & (1 << i)) == 0)
+			continue;
+		callout_reset_sbt(&slot->timeout,
+ 	    	    SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
+		    (timeout_t*)fsl_sata_timeout, slot, 0);
+	}
+}
+
+/* Locked by callout mechanism. */
+static void
+fsl_sata_timeout(struct fsl_sata_slot *slot)
+{
+	struct fsl_sata_channel *ch = slot->ch;
+	device_t dev = ch->dev;
+	uint32_t sstatus;
+
+	/* Check for stale timeout. */
+	if (slot->state < FSL_SATA_SLOT_RUNNING)
+		return;
+
+	/* Check if slot was not being executed last time we checked. */
+	if (slot->state < FSL_SATA_SLOT_EXECUTING) {
+		/* Check if slot started executing. */
+		sstatus = ATA_INL(ch->r_mem, FSL_SATA_P_CAR);
+		if ((sstatus & (1 << slot->slot)) != 0)
+			slot->state = FSL_SATA_SLOT_EXECUTING;
+
+		callout_reset_sbt(&slot->timeout,
+	    	    SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
+		    (timeout_t*)fsl_sata_timeout, slot, 0);
+		return;
+	}
+
+	device_printf(dev, "Timeout on slot %d port %d\n",
+	    slot->slot, slot->ccb->ccb_h.target_id & 0x0f);
+
+	/* Handle frozen command. */
+	if (ch->frozen) {
+		union ccb *fccb = ch->frozen;
+		ch->frozen = NULL;
+		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
+		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
+			xpt_freeze_devq(fccb->ccb_h.path, 1);
+			fccb->ccb_h.status |= CAM_DEV_QFRZN;
+		}
+		fsl_sata_done(ch, fccb);
+	}
+	if (ch->toslots == 0)
+		xpt_freeze_simq(ch->sim, 1);
+	ch->toslots |= (1 << slot->slot);
+	if ((ch->rslots & ~ch->toslots) == 0)
+		fsl_sata_process_timeout(ch);
+	else
+		device_printf(dev, " ... waiting for slots %08x\n",
+		    ch->rslots & ~ch->toslots);
+}
+
+/* Must be called with channel locked. */
+static void
+fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et)
+{
+	struct fsl_sata_channel *ch = slot->ch;
+	union ccb *ccb = slot->ccb;
+	struct fsl_sata_cmd_list *clp;
+	int lastto;
+	uint32_t sig;
+
+	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+	clp = FSL_SATA_CLP(ch, slot);
+	/* Read result registers to the result struct */
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		struct ata_res *res = &ccb->ataio.res;
+
+		if ((et == FSL_SATA_ERR_TFE) ||
+		    (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
+			struct fsl_sata_cmd_tab *ctp = FSL_SATA_CTP(ch, slot);
+			uint8_t *fis = ctp->sfis;
+
+			res->status = fis[2];
+			res->error = fis[3];
+			res->lba_low = fis[4];
+			res->lba_mid = fis[5];
+			res->lba_high = fis[6];
+			res->device = fis[7];
+			res->lba_low_exp = fis[8];
+			res->lba_mid_exp = fis[9];
+			res->lba_high_exp = fis[10];
+			res->sector_count = fis[12];
+			res->sector_count_exp = fis[13];
+
+			if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
+			    (ccb->ataio.cmd.control & ATA_A_RESET) == 0) {
+				sig = ATA_INL(ch->r_mem,  FSL_SATA_P_SIG);
+				res->lba_high = sig >> 24;
+				res->lba_mid = sig >> 16;
+				res->lba_low = sig >> 8;
+				res->sector_count = sig;
+			}
+		} else
+			bzero(res, sizeof(*res));
+		if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 &&
+		    (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+			ccb->ataio.resid =
+			    ccb->ataio.dxfer_len - le32toh(clp->ttl);
+		}
+	} else {
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+			ccb->csio.resid =
+			    ccb->csio.dxfer_len - le32toh(clp->ttl);
+		}
+	}
+	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+		bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
+		    (ccb->ccb_h.flags & CAM_DIR_IN) ?
+		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+		bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
+	}
+	if (et != FSL_SATA_ERR_NONE)
+		ch->eslots |= (1 << slot->slot);
+	/* In case of error, freeze device for proper recovery. */
+	if ((et != FSL_SATA_ERR_NONE) && (!ch->recoverycmd) &&
+	    !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status |= CAM_DEV_QFRZN;
+	}
+	/* Set proper result status. */
+	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+	switch (et) {
+	case FSL_SATA_ERR_NONE:
+		ccb->ccb_h.status |= CAM_REQ_CMP;
+		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
+			ccb->csio.scsi_status = SCSI_STATUS_OK;
+		break;
+	case FSL_SATA_ERR_INVALID:
+		ch->fatalerr = 1;
+		ccb->ccb_h.status |= CAM_REQ_INVALID;
+		break;
+	case FSL_SATA_ERR_INNOCENT:
+		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
+		break;
+	case FSL_SATA_ERR_TFE:
+	case FSL_SATA_ERR_NCQ:
+		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
+			ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
+			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+		} else {
+			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
+		}
+		break;
+	case FSL_SATA_ERR_SATA:
+		ch->fatalerr = 1;
+		if (!ch->recoverycmd) {
+			xpt_freeze_simq(ch->sim, 1);
+			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
+		}
+		ccb->ccb_h.status |= CAM_UNCOR_PARITY;
+		break;
+	case FSL_SATA_ERR_TIMEOUT:
+		if (!ch->recoverycmd) {
+			xpt_freeze_simq(ch->sim, 1);
+			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
+		}
+		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
+		break;
+	default:
+		ch->fatalerr = 1;
+		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
+	}
+	/* Free slot. */
+	ch->oslots &= ~(1 << slot->slot);
+	ch->rslots &= ~(1 << slot->slot);
+	ch->aslots &= ~(1 << slot->slot);
+	slot->state = FSL_SATA_SLOT_EMPTY;
+	slot->ccb = NULL;
+	/* Update channel stats. */
+	ch->numrslots--;
+	ch->numrslotspd[ccb->ccb_h.target_id]--;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, 1 << slot->slot);
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
+		ch->numtslots--;
+		ch->numtslotspd[ccb->ccb_h.target_id]--;
+	}
+	/* Cancel timeout state if request completed normally. */
+	if (et != FSL_SATA_ERR_TIMEOUT) {
+		lastto = (ch->toslots == (1 << slot->slot));
+		ch->toslots &= ~(1 << slot->slot);
+		if (lastto)
+			xpt_release_simq(ch->sim, TRUE);
+	}
+	/* If it was first request of reset sequence and there is no error,
+	 * proceed to second request. */
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
+	    (ccb->ataio.cmd.control & ATA_A_RESET) &&
+	    et == FSL_SATA_ERR_NONE) {
+		ccb->ataio.cmd.control &= ~ATA_A_RESET;
+		fsl_sata_begin_transaction(ch, ccb);
+		return;
+	}
+	/* If it was our READ LOG command - process it. */
+	if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
+		fsl_sata_process_read_log(ch, ccb);
+	/* If it was our REQUEST SENSE command - process it. */
+	} else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
+		fsl_sata_process_request_sense(ch, ccb);
+	/* If it was NCQ or ATAPI command error, put result on hold. */
+	} else if (et == FSL_SATA_ERR_NCQ ||
+	    ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
+	     (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) {
+		ch->hold[slot->slot] = ccb;
+		ch->numhslots++;
+	} else
+		fsl_sata_done(ch, ccb);
+	/* If we have no other active commands, ... */
+	if (ch->rslots == 0) {
+		/* if there was fatal error - reset port. */
+		if (ch->toslots != 0 || ch->fatalerr) {
+			fsl_sata_reset(ch);
+		} else {
+			/* if we have slots in error, we can reinit port. */
+			if (ch->eslots != 0) {
+				fsl_sata_stop(ch);
+				fsl_sata_start(ch);
+			}
+			/* if there commands on hold, we can do READ LOG. */
+			if (!ch->recoverycmd && ch->numhslots)
+				fsl_sata_issue_recovery(ch);
+		}
+	/* If all the rest of commands are in timeout - give them chance. */
+	} else if ((ch->rslots & ~ch->toslots) == 0 &&
+	    et != FSL_SATA_ERR_TIMEOUT)
+		fsl_sata_rearm_timeout(ch);
+	/* Unfreeze frozen command. */
+	if (ch->frozen && !fsl_sata_check_collision(ch, ch->frozen)) {
+		union ccb *fccb = ch->frozen;
+		ch->frozen = NULL;
+		fsl_sata_begin_transaction(ch, fccb);
+		xpt_release_simq(ch->sim, TRUE);
+	}
+	/* Start PM timer. */
+	if (ch->numrslots == 0 && ch->pm_level > 3 &&
+	    (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
+		callout_schedule(&ch->pm_timer,
+		    (ch->pm_level == 4) ? hz / 1000 : hz / 8);
+	}
+}
+
+static void
+fsl_sata_issue_recovery(struct fsl_sata_channel *ch)
+{
+	union ccb *ccb;
+	struct ccb_ataio *ataio;
+	struct ccb_scsiio *csio;
+	int i;
+
+	/* Find some held command. */
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		if (ch->hold[i])
+			break;
+	}
+	ccb = xpt_alloc_ccb_nowait();
+	if (ccb == NULL) {
+		device_printf(ch->dev, "Unable to allocate recovery command\n");
+completeall:
+		/* We can't do anything -- complete held commands. */
+		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+			if (ch->hold[i] == NULL)
+				continue;
+			ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
+			ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
+			fsl_sata_done(ch, ch->hold[i]);
+			ch->hold[i] = NULL;
+			ch->numhslots--;
+		}
+		fsl_sata_reset(ch);
+		return;
+	}
+	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		/* READ LOG */
+		ccb->ccb_h.recovery_type = RECOVERY_READ_LOG;
+		ccb->ccb_h.func_code = XPT_ATA_IO;
+		ccb->ccb_h.flags = CAM_DIR_IN;
+		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
+		ataio = &ccb->ataio;
+		ataio->data_ptr = malloc(512, M_FSL_SATA, M_NOWAIT);
+		if (ataio->data_ptr == NULL) {
+			xpt_free_ccb(ccb);
+			device_printf(ch->dev,
+			    "Unable to allocate memory for READ LOG command\n");
+			goto completeall;
+		}
+		ataio->dxfer_len = 512;
+		bzero(&ataio->cmd, sizeof(ataio->cmd));
+		ataio->cmd.flags = CAM_ATAIO_48BIT;
+		ataio->cmd.command = 0x2F;	/* READ LOG EXT */
+		ataio->cmd.sector_count = 1;
+		ataio->cmd.sector_count_exp = 0;
+		ataio->cmd.lba_low = 0x10;
+		ataio->cmd.lba_mid = 0;
+		ataio->cmd.lba_mid_exp = 0;
+	} else {
+		/* REQUEST SENSE */
+		ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE;
+		ccb->ccb_h.recovery_slot = i;
+		ccb->ccb_h.func_code = XPT_SCSI_IO;
+		ccb->ccb_h.flags = CAM_DIR_IN;
+		ccb->ccb_h.status = 0;
+		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
+		csio = &ccb->csio;
+		csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
+		csio->dxfer_len = ch->hold[i]->csio.sense_len;
+		csio->cdb_len = 6;
+		bzero(&csio->cdb_io, sizeof(csio->cdb_io));
+		csio->cdb_io.cdb_bytes[0] = 0x03;
+		csio->cdb_io.cdb_bytes[4] = csio->dxfer_len;
+	}
+	/* Freeze SIM while doing recovery. */
+	ch->recoverycmd = 1;
+	xpt_freeze_simq(ch->sim, 1);
+	fsl_sata_begin_transaction(ch, ccb);
+}
+
+static void
+fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+	uint8_t *data;
+	struct ata_res *res;
+	int i;
+
+	ch->recoverycmd = 0;
+
+	data = ccb->ataio.data_ptr;
+	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
+	    (data[0] & 0x80) == 0) {
+		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+			if (!ch->hold[i])
+				continue;
+			if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
+				continue;
+			if ((data[0] & 0x1F) == i) {
+				res = &ch->hold[i]->ataio.res;
+				res->status = data[2];
+				res->error = data[3];
+				res->lba_low = data[4];
+				res->lba_mid = data[5];
+				res->lba_high = data[6];
+				res->device = data[7];
+				res->lba_low_exp = data[8];
+				res->lba_mid_exp = data[9];
+				res->lba_high_exp = data[10];
+				res->sector_count = data[12];
+				res->sector_count_exp = data[13];
+			} else {
+				ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
+				ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
+			}
+			fsl_sata_done(ch, ch->hold[i]);
+			ch->hold[i] = NULL;
+			ch->numhslots--;
+		}
+	} else {
+		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
+			device_printf(ch->dev, "Error while READ LOG EXT\n");
+		else if ((data[0] & 0x80) == 0) {
+			device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
+		}
+		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+			if (!ch->hold[i])
+				continue;
+			if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
+				continue;
+			fsl_sata_done(ch, ch->hold[i]);
+			ch->hold[i] = NULL;
+			ch->numhslots--;
+		}
+	}
+	free(ccb->ataio.data_ptr, M_FSL_SATA);
+	xpt_free_ccb(ccb);
+	xpt_release_simq(ch->sim, TRUE);
+}
+
+static void
+fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+	int i;
+
+	ch->recoverycmd = 0;
+
+	i = ccb->ccb_h.recovery_slot;
+	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
+		ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
+	} else {
+		ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
+		ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
+	}
+	fsl_sata_done(ch, ch->hold[i]);
+	ch->hold[i] = NULL;
+	ch->numhslots--;
+	xpt_free_ccb(ccb);
+	xpt_release_simq(ch->sim, TRUE);
+}
+
+static void
+fsl_sata_start(struct fsl_sata_channel *ch)
+{
+	u_int32_t cmd;
+
+	/* Clear SATA error register */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, 0xFFFFFFFF);
+	/* Clear any interrupts pending on this channel */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, 0x3F);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, 0xFFFF);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, 0xFFFF);
+	/* Start operations on this channel */
+	cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
+	cmd |= FSL_SATA_P_HCTRL_HC_ON | FSL_SATA_P_HCTRL_SNOOP;
+	cmd &= ~FSL_SATA_P_HCTRL_HC_FORCE_OFF;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd | 
+	    (ch->pm_present ? FSL_SATA_P_HCTRL_PM : 0));
+	fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
+	    FSL_SATA_P_HSTS_PR, FSL_SATA_P_HSTS_PR, 500);
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS,
+	    ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & FSL_SATA_P_HSTS_PR);
+}
+
+static void
+fsl_sata_stop(struct fsl_sata_channel *ch)
+{
+	uint32_t cmd;
+	int i;
+
+	/* Kill all activity on this channel */
+	cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
+	cmd &= ~FSL_SATA_P_HCTRL_HC_ON;
+
+	for (i = 0; i < 2; i++) {
+		ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd);
+		if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
+		    FSL_SATA_P_HSTS_HS_ON, 0, 500)) {
+			if (i != 0)
+				device_printf(ch->dev,
+				    "stopping FSL SATA engine failed\n");
+			cmd |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
+		} else
+			break;
+	}
+	ch->eslots = 0;
+}
+
+static void
+fsl_sata_reset(struct fsl_sata_channel *ch)
+{
+	uint32_t ctrl;
+	int i;
+
+	xpt_freeze_simq(ch->sim, 1);
+	if (bootverbose)
+		device_printf(ch->dev, "FSL SATA reset...\n");
+
+	/* Requeue freezed command. */
+	if (ch->frozen) {
+		union ccb *fccb = ch->frozen;
+		ch->frozen = NULL;
+		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
+		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
+			xpt_freeze_devq(fccb->ccb_h.path, 1);
+			fccb->ccb_h.status |= CAM_DEV_QFRZN;
+		}
+		fsl_sata_done(ch, fccb);
+	}
+	/* Kill the engine and requeue all running commands. */
+	fsl_sata_stop(ch);
+	DELAY(1000);	/* sleep for 1ms */
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		/* Do we have a running request on slot? */
+		if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
+			continue;
+		/* XXX; Commands in loading state. */
+		fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_INNOCENT);
+	}
+	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
+		if (!ch->hold[i])
+			continue;
+		fsl_sata_done(ch, ch->hold[i]);
+		ch->hold[i] = NULL;
+		ch->numhslots--;
+	}
+	if (ch->toslots != 0)
+		xpt_release_simq(ch->sim, TRUE);
+	ch->eslots = 0;
+	ch->toslots = 0;
+	ch->fatalerr = 0;
+	/* Tell the XPT about the event */
+	xpt_async(AC_BUS_RESET, ch->path, NULL);
+	/* Disable port interrupts */
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
+	    ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f);
+	/* Reset and reconnect PHY, */
+	fsl_sata_start(ch);
+	if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 0x08, 0x08, 500)) {
+		if (bootverbose)
+			device_printf(ch->dev,
+			    "FSL SATA reset: device not found\n");
+		ch->devices = 0;
+		/* Enable wanted port interrupts */
+		ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
+		    ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) | FSL_SATA_P_HCTRL_PHYRDY);
+		xpt_release_simq(ch->sim, TRUE);
+		return;
+	}
+	if (bootverbose)
+		device_printf(ch->dev, "FSL SATA reset: device found\n");
+	ch->devices = 1;
+	/* Enable wanted port interrupts */
+	ctrl = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f;
+	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
+	    ctrl | FSL_SATA_P_HCTRL_FATAL | FSL_SATA_P_HCTRL_PHYRDY |
+	    FSL_SATA_P_HCTRL_SIG | FSL_SATA_P_HCTRL_SNTFY |
+	    FSL_SATA_P_HCTRL_DE | FSL_SATA_P_HCTRL_CC);
+	xpt_release_simq(ch->sim, TRUE);
+}
+
+static int
+fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag)
+{
+	uint8_t *fis = &ctp->cfis[0];
+
+	bzero(fis, 32);
+	fis[0] = 0x27;  		/* host to device */
+	fis[1] = (ccb->ccb_h.target_id & 0x0f);
+	if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
+		fis[1] |= 0x80;
+		fis[2] = ATA_PACKET_CMD;
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
+		    ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA)
+			fis[3] = ATA_F_DMA;
+		else {
+			fis[5] = ccb->csio.dxfer_len;
+			fis[6] = ccb->csio.dxfer_len >> 8;
+		}
+		fis[7] = ATA_D_LBA;
+		fis[15] = ATA_A_4BIT;
+		bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
+		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes,
+		    ctp->acmd, ccb->csio.cdb_len);
+		bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len);
+	} else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) {
+		fis[1] |= 0x80;
+		fis[2] = ccb->ataio.cmd.command;
+		fis[3] = ccb->ataio.cmd.features;
+		fis[4] = ccb->ataio.cmd.lba_low;
+		fis[5] = ccb->ataio.cmd.lba_mid;
+		fis[6] = ccb->ataio.cmd.lba_high;
+		fis[7] = ccb->ataio.cmd.device;
+		fis[8] = ccb->ataio.cmd.lba_low_exp;
+		fis[9] = ccb->ataio.cmd.lba_mid_exp;
+		fis[10] = ccb->ataio.cmd.lba_high_exp;
+		fis[11] = ccb->ataio.cmd.features_exp;
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			fis[12] = tag << 3;
+			fis[13] = 0;
+		} else {
+			fis[12] = ccb->ataio.cmd.sector_count;
+			fis[13] = ccb->ataio.cmd.sector_count_exp;
+		}
+		fis[15] = ATA_A_4BIT;
+	} else {
+		fis[15] = ccb->ataio.cmd.control;
+	}
+	return (20);
+}
+
+static int
+fsl_sata_check_ids(struct fsl_sata_channel *ch, union ccb *ccb)
+{
+
+	if (ccb->ccb_h.target_id > 15) {
+		ccb->ccb_h.status = CAM_TID_INVALID;
+		fsl_sata_done(ch, ccb);
+		return (-1);
+	}
+	if (ccb->ccb_h.target_lun != 0) {
+		ccb->ccb_h.status = CAM_LUN_INVALID;
+		fsl_sata_done(ch, ccb);
+		return (-1);
+	}
+	return (0);
+}
+
+static void
+fsl_sataaction(struct cam_sim *sim, union ccb *ccb)
+{
+	struct fsl_sata_channel *ch;
+
+	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
+	    ("fsl_sataaction func_code=0x%x\n", ccb->ccb_h.func_code));
+
+	ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
+	switch (ccb->ccb_h.func_code) {
+	/* Common cases first */
+	case XPT_ATA_IO:	/* Execute the requested I/O operation */
+	case XPT_SCSI_IO:
+		if (fsl_sata_check_ids(ch, ccb))
+			return;
+		if (ch->devices == 0 ||
+		    (ch->pm_present == 0 &&
+		     ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
+			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
+			break;
+		}
+		ccb->ccb_h.recovery_type = RECOVERY_NONE;
+		/* Check for command collision. */
+		if (fsl_sata_check_collision(ch, ccb)) {
+			/* Freeze command. */
+			ch->frozen = ccb;
+			/* We have only one frozen slot, so freeze simq also. */
+			xpt_freeze_simq(ch->sim, 1);
+			return;
+		}
+		fsl_sata_begin_transaction(ch, ccb);
+		return;
+	case XPT_ABORT:			/* Abort the specified CCB */
+		/* XXX Implement */
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	case XPT_SET_TRAN_SETTINGS:
+	{
+		struct	ccb_trans_settings *cts = &ccb->cts;
+		struct	fsl_sata_device *d; 
+
+		if (fsl_sata_check_ids(ch, ccb))
+			return;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
+			d = &ch->curr[ccb->ccb_h.target_id];
+		else
+			d = &ch->user[ccb->ccb_h.target_id];
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
+			d->revision = cts->xport_specific.sata.revision;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
+			d->mode = cts->xport_specific.sata.mode;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
+			d->bytecount = min(8192, cts->xport_specific.sata.bytecount);
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
+			d->tags = min(FSL_SATA_MAX_SLOTS, cts->xport_specific.sata.tags);
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
+			ch->pm_present = cts->xport_specific.sata.pm_present;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
+			d->atapi = cts->xport_specific.sata.atapi;
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_GET_TRAN_SETTINGS:
+	/* Get default/user set transfer settings for the target */
+	{
+		struct	ccb_trans_settings *cts = &ccb->cts;
+		struct  fsl_sata_device *d;
+		uint32_t status;
+
+		if (fsl_sata_check_ids(ch, ccb))
+			return;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
+			d = &ch->curr[ccb->ccb_h.target_id];
+		else
+			d = &ch->user[ccb->ccb_h.target_id];
+		cts->protocol = PROTO_UNSPECIFIED;
+		cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
+		cts->transport = XPORT_SATA;
+		cts->transport_version = XPORT_VERSION_UNSPECIFIED;
+		cts->proto_specific.valid = 0;
+		cts->xport_specific.sata.valid = 0;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
+		    (ccb->ccb_h.target_id == 15 ||
+		    (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
+			status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS) & ATA_SS_SPD_MASK;
+			if (status & 0x0f0) {
+				cts->xport_specific.sata.revision =
+				    (status & 0x0f0) >> 4;
+				cts->xport_specific.sata.valid |=
+				    CTS_SATA_VALID_REVISION;
+			}
+			cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
+			if (ch->pm_level) {
+				cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
+			}
+			cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN;
+			cts->xport_specific.sata.caps &=
+			    ch->user[ccb->ccb_h.target_id].caps;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
+		} else {
+			cts->xport_specific.sata.revision = d->revision;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
+			cts->xport_specific.sata.caps = d->caps;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
+		}
+		cts->xport_specific.sata.mode = d->mode;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
+		cts->xport_specific.sata.bytecount = d->bytecount;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
+		cts->xport_specific.sata.pm_present = ch->pm_present;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
+		cts->xport_specific.sata.tags = d->tags;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
+		cts->xport_specific.sata.atapi = d->atapi;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
+	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
+		fsl_sata_reset(ch);
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	case XPT_TERM_IO:		/* Terminate the I/O process */
+		/* XXX Implement */
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	case XPT_PATH_INQ:		/* Path routing inquiry */
+	{
+		struct ccb_pathinq *cpi = &ccb->cpi;
+
+		cpi->version_num = 1; /* XXX??? */
+		cpi->hba_inquiry = PI_SDTR_ABLE;
+		cpi->hba_inquiry |= PI_TAG_ABLE;
+#if 0
+		/*
+		 * XXX: CAM tries to reset port 15 if it sees port multiplier
+		 * support.  Disable it for now.
+		 */
+		cpi->hba_inquiry |= PI_SATAPM;
+#endif
+		cpi->target_sprt = 0;
+		cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
+		cpi->hba_eng_cnt = 0;
+		/*
+		 * XXX: This should be 15, since hardware *does* support a port
+		 * multiplier.  See above.
+		 */
+		cpi->max_target = 0;
+		cpi->max_lun = 0;
+		cpi->initiator_id = 0;
+		cpi->bus_id = cam_sim_bus(sim);
+		cpi->base_transfer_speed = 150000;
+		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
+		strncpy(cpi->hba_vid, "FSL SATA", HBA_IDLEN);
+		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
+		cpi->unit_number = cam_sim_unit(sim);
+		cpi->transport = XPORT_SATA;
+		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
+		cpi->protocol = PROTO_ATA;
+		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
+		cpi->maxio = MAXPHYS;
+		cpi->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	default:
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	}
+	fsl_sata_done(ch, ccb);
+}
+
+static void
+fsl_satapoll(struct cam_sim *sim)
+{
+	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
+	uint32_t istatus;
+
+	/* Read interrupt statuses and process if any. */
+	istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS);
+	if (istatus != 0)
+		fsl_sata_intr_main(ch, istatus);
+}
+MODULE_VERSION(fsl_sata, 1);
+MODULE_DEPEND(fsl_sata, cam, 1, 1, 1);
diff --git a/sys/powerpc/mpc85xx/fsl_sata.h b/sys/powerpc/mpc85xx/fsl_sata.h
new file mode 100644
index 000000000000..3a9ea20484b0
--- /dev/null
+++ b/sys/powerpc/mpc85xx/fsl_sata.h
@@ -0,0 +1,141 @@
+/*-
+ * Copyright (c) 1998 - 2008 Søren Schmidt <sos@FreeBSD.org>
+ * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2015-2016 Justin Hibbits <jhibbits@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/* ATA register defines */
+
+#define ATA_FEATURE                     1       /* (W) feature */
+#define         ATA_F_DMA               0x01    /* enable DMA */
+#define         ATA_F_OVL               0x02    /* enable overlap */
+
+#define ATA_COUNT                       2       /* (W) sector count */
+
+#define ATA_SECTOR                      3       /* (RW) sector # */
+#define ATA_CYL_LSB                     4       /* (RW) cylinder# LSB */
+#define ATA_CYL_MSB                     5       /* (RW) cylinder# MSB */
+#define ATA_DRIVE                       6       /* (W) Sector/Drive/Head */
+#define         ATA_D_LBA               0x40    /* use LBA addressing */
+#define         ATA_D_IBM               0xa0    /* 512 byte sectors, ECC */
+
+#define ATA_COMMAND                     7       /* (W) command */
+
+#define ATA_ERROR                       8       /* (R) error */
+#define         ATA_E_ILI               0x01    /* illegal length */
+#define         ATA_E_NM                0x02    /* no media */
+#define         ATA_E_ABORT             0x04    /* command aborted */
+#define         ATA_E_MCR               0x08    /* media change request */
+#define         ATA_E_IDNF              0x10    /* ID not found */
+#define         ATA_E_MC                0x20    /* media changed */
+#define         ATA_E_UNC               0x40    /* uncorrectable data */
+#define         ATA_E_ICRC              0x80    /* UDMA crc error */
+#define		ATA_E_ATAPI_SENSE_MASK	0xf0	/* ATAPI sense key mask */
+
+#define ATA_IREASON                     9       /* (R) interrupt reason */
+#define         ATA_I_CMD               0x01    /* cmd (1) | data (0) */
+#define         ATA_I_IN                0x02    /* read (1) | write (0) */
+#define         ATA_I_RELEASE           0x04    /* released bus (1) */
+#define         ATA_I_TAGMASK           0xf8    /* tag mask */
+
+#define ATA_STATUS                      10      /* (R) status */
+#define ATA_ALTSTAT                     11      /* (R) alternate status */
+#define         ATA_S_ERROR             0x01    /* error */
+#define         ATA_S_INDEX             0x02    /* index */
+#define         ATA_S_CORR              0x04    /* data corrected */
+#define         ATA_S_DRQ               0x08    /* data request */
+#define         ATA_S_DSC               0x10    /* drive seek completed */
+#define         ATA_S_SERVICE           0x10    /* drive needs service */
+#define         ATA_S_DWF               0x20    /* drive write fault */
+#define         ATA_S_DMA               0x20    /* DMA ready */
+#define         ATA_S_READY             0x40    /* drive ready */
+#define         ATA_S_BUSY              0x80    /* busy */
+
+#define ATA_CONTROL                     12      /* (W) control */
+#define         ATA_A_IDS               0x02    /* disable interrupts */
+#define         ATA_A_RESET             0x04    /* RESET controller */
+#define         ATA_A_4BIT              0x08    /* 4 head bits */
+#define         ATA_A_HOB               0x80    /* High Order Byte enable */
+
+/* SATA register defines */
+#define ATA_SSTATUS                     13
+#define         ATA_SS_DET_MASK         0x0000000f
+#define         ATA_SS_DET_NO_DEVICE    0x00000000
+#define         ATA_SS_DET_DEV_PRESENT  0x00000001
+#define         ATA_SS_DET_PHY_ONLINE   0x00000003
+#define         ATA_SS_DET_PHY_OFFLINE  0x00000004
+
+#define         ATA_SS_SPD_MASK         0x000000f0
+#define         ATA_SS_SPD_NO_SPEED     0x00000000
+#define         ATA_SS_SPD_GEN1         0x00000010
+#define         ATA_SS_SPD_GEN2         0x00000020
+#define         ATA_SS_SPD_GEN3         0x00000030
+
+#define         ATA_SS_IPM_MASK         0x00000f00
+#define         ATA_SS_IPM_NO_DEVICE    0x00000000
+#define         ATA_SS_IPM_ACTIVE       0x00000100
+#define         ATA_SS_IPM_PARTIAL      0x00000200
+#define         ATA_SS_IPM_SLUMBER      0x00000600
+#define         ATA_SS_IPM_DEVSLEEP     0x00000800
+
+#define ATA_SERROR                      14
+#define         ATA_SE_DATA_CORRECTED   0x00000001
+#define         ATA_SE_COMM_CORRECTED   0x00000002
+#define         ATA_SE_DATA_ERR         0x00000100
+#define         ATA_SE_COMM_ERR         0x00000200
+#define         ATA_SE_PROT_ERR         0x00000400
+#define         ATA_SE_HOST_ERR         0x00000800
+#define         ATA_SE_PHY_CHANGED      0x00010000
+#define         ATA_SE_PHY_IERROR       0x00020000
+#define         ATA_SE_COMM_WAKE        0x00040000
+#define         ATA_SE_DECODE_ERR       0x00080000
+#define         ATA_SE_PARITY_ERR       0x00100000
+#define         ATA_SE_CRC_ERR          0x00200000
+#define         ATA_SE_HANDSHAKE_ERR    0x00400000
+#define         ATA_SE_LINKSEQ_ERR      0x00800000
+#define         ATA_SE_TRANSPORT_ERR    0x01000000
+#define         ATA_SE_UNKNOWN_FIS      0x02000000
+#define         ATA_SE_EXCHANGED        0x04000000
+
+#define ATA_SCONTROL                    15
+#define         ATA_SC_DET_MASK         0x0000000f
+#define         ATA_SC_DET_IDLE         0x00000000
+#define         ATA_SC_DET_RESET        0x00000001
+#define         ATA_SC_DET_DISABLE      0x00000004
+
+#define         ATA_SC_SPD_MASK         0x000000f0
+#define         ATA_SC_SPD_NO_SPEED     0x00000000
+#define         ATA_SC_SPD_SPEED_GEN1   0x00000010
+#define         ATA_SC_SPD_SPEED_GEN2   0x00000020
+#define         ATA_SC_SPD_SPEED_GEN3   0x00000030
+
+#define         ATA_SC_IPM_MASK         0x00000f00
+#define         ATA_SC_IPM_NONE         0x00000000
+#define         ATA_SC_IPM_DIS_PARTIAL  0x00000100
+#define         ATA_SC_IPM_DIS_SLUMBER  0x00000200
+#define         ATA_SC_IPM_DIS_DEVSLEEP 0x00000400
+
diff --git a/targets/pseudo/bootstrap-tools/Makefile b/targets/pseudo/bootstrap-tools/Makefile
index 49b168e37211..525320702a41 100644
--- a/targets/pseudo/bootstrap-tools/Makefile
+++ b/targets/pseudo/bootstrap-tools/Makefile
@@ -36,6 +36,7 @@ OSRELDATE?= 0
 BSARGS= 	DESTDIR= \
 		OBJTOP='${WORLDTMP}/obj-tools' \
 		OBJROOT='$${OBJTOP}/' \
+		MAKEOBJDIRPREFIX= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		BWPHASE=${.TARGET} \
 		SSP_CFLAGS= \
diff --git a/usr.sbin/ctld/login.c b/usr.sbin/ctld/login.c
index 221b6eb7edda..6df037f5dee6 100644
--- a/usr.sbin/ctld/login.c
+++ b/usr.sbin/ctld/login.c
@@ -435,7 +435,7 @@ login_chap(struct connection *conn, struct auth_group *ag)
 	 * Yay, authentication succeeded!
 	 */
 	log_debugx("authentication succeeded for user \"%s\"; "
-	    "transitioning to Negotiation Phase", auth->a_user);
+	    "transitioning to operational parameter negotiation", auth->a_user);
 	login_send_chap_success(request, auth);
 	pdu_delete(request);