Add linux kernel disk support

Native Linux vdev disk interfaces

Signed-off-by: Brian Behlendorf <[email protected]>

27 files changed, 2571 insertions, 112 deletions

diff --git a/module/zcommon/include/sys/fs/zfs.h b/module/zcommon/include/sys/fs/zfs.h
index b2c946919..25f256251 100644
--- a/module/zcommon/include/sys/fs/zfs.h
+++ b/module/zcommon/include/sys/fs/zfs.h
@@ -701,12 +701,12 @@ typedef struct ddt_histogram {
 #define	ZFS_DEV		"/dev/zfs"
 
 /* general zvol path */
-#define	ZVOL_DIR		"/dev/zvol"
-/* expansion */
-#define	ZVOL_PSEUDO_DEV		"/devices/pseudo/zfs@0:"
-/* for dump and swap */
-#define	ZVOL_FULL_DEV_DIR	ZVOL_DIR "/dsk/"
-#define	ZVOL_FULL_RDEV_DIR	ZVOL_DIR "/rdsk/"
+#define	ZVOL_DIR	"/dev"
+
+#define	ZVOL_MAJOR		230
+#define	ZVOL_MINOR_BITS		4
+#define	ZVOL_MINOR_MASK		((1U << ZVOL_MINOR_BITS) - 1)
+#define	ZVOL_MINORS		(1 << 4)
 
 #define	ZVOL_PROP_NAME		"name"
 #define	ZVOL_DEFAULT_BLOCKSIZE	8192
@@ -740,6 +740,8 @@ typedef enum zfs_ioc {
 	ZFS_IOC_DATASET_LIST_NEXT,
 	ZFS_IOC_SNAPSHOT_LIST_NEXT,
 	ZFS_IOC_SET_PROP,
+	ZFS_IOC_CREATE_MINOR,
+	ZFS_IOC_REMOVE_MINOR,
 	ZFS_IOC_CREATE,
 	ZFS_IOC_DESTROY,
 	ZFS_IOC_ROLLBACK,
diff --git a/module/zcommon/zfs_namecheck.c b/module/zcommon/zfs_namecheck.c
index 5cfafea47..2398c2755 100644
--- a/module/zcommon/zfs_namecheck.c
+++ b/module/zcommon/zfs_namecheck.c
@@ -142,9 +142,22 @@ dataset_namecheck(const char *path, namecheck_err_t *why, char *what)
 	 * which is the same as MAXNAMELEN used in the kernel.
 	 * If ZFS_MAXNAMELEN value is changed, make sure to cleanup all
 	 * places using MAXNAMELEN.
+	 *
+	 * When HAVE_KOBJ_NAME_LEN is defined the maximum safe kobject name
+	 * length is 20 bytes.  This 20 bytes is broken down as follows to
+	 * provide a maximum safe <pool>/<dataset>[@snapshot] length of only
+	 * 18 bytes.  To ensure bytes are left for <dataset>[@snapshot] the
+	 * <pool> portition is futher limited to 9 bytes.  For 2.6.27 and
+	 * newer kernels this limit is set to MAXNAMELEN.
+	 *
+	 *   <pool>/<dataset> + <partition> + <newline>
+	 *   (18)             + (1)         + (1)
 	 */
-
+#ifdef HAVE_KOBJ_NAME_LEN
+	if (strlen(path) > 18) {
+#else
 	if (strlen(path) >= MAXNAMELEN) {
+#endif /* HAVE_KOBJ_NAME_LEN */
 		if (why)
 			*why = NAME_ERR_TOOLONG;
 		return (-1);
@@ -303,8 +316,22 @@ pool_namecheck(const char *pool, namecheck_err_t *why, char *what)
 	 * which is the same as MAXNAMELEN used in the kernel.
 	 * If ZPOOL_MAXNAMELEN value is changed, make sure to cleanup all
 	 * places using MAXNAMELEN.
+	 *
+	 * When HAVE_KOBJ_NAME_LEN is defined the maximum safe kobject name
+	 * length is 20 bytes.  This 20 bytes is broken down as follows to
+	 * provide a maximum safe <pool>/<dataset>[@snapshot] length of only
+	 * 18 bytes.  To ensure bytes are left for <dataset>[@snapshot] the
+	 * <pool> portition is futher limited to 8 bytes.  For 2.6.27 and
+	 * newer kernels this limit is set to MAXNAMELEN.
+	 *
+	 *   <pool>/<dataset> + <partition> + <newline>
+	 *   (18)             + (1)         + (1)
 	 */
+#ifdef HAVE_KOBJ_NAME_LEN
+	if (strlen(pool) > 8) {
+#else
 	if (strlen(pool) >= MAXNAMELEN) {
+#endif /* HAVE_KOBJ_NAME_LEN */
 		if (why)
 			*why = NAME_ERR_TOOLONG;
 		return (-1);
diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in
index b9effcbf1..ef2e56208 100644
--- a/module/zfs/Makefile.in
+++ b/module/zfs/Makefile.in
@@ -53,6 +53,7 @@ ${MODULE}-objs += uberblock.o
 ${MODULE}-objs += unique.o
 ${MODULE}-objs += vdev.o
 ${MODULE}-objs += vdev_cache.o
+${MODULE}-objs += vdev_disk.o
 ${MODULE}-objs += vdev_file.o
 ${MODULE}-objs += vdev_label.o
 ${MODULE}-objs += vdev_mirror.o
diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c
index 7f17c736f..fea9cd502 100644
--- a/module/zfs/dmu.c
+++ b/module/zfs/dmu.c
@@ -793,7 +793,7 @@ dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 		else
 			dmu_buf_will_dirty(db, tx);
 
-		bcopy(buf, (char *)db->db_data + bufoff, tocpy);
+		(void) memcpy((char *)db->db_data + bufoff, buf, tocpy);
 
 		if (tocpy == db->db_size)
 			dmu_buf_fill_done(db, tx);
@@ -975,85 +975,126 @@ xuio_stat_wbuf_nocopy()
 }
 
 #ifdef _KERNEL
+
+/*
+ * Copy up to size bytes between arg_buf and req based on the data direction
+ * described by the req.  If an entire req's data cannot be transfered the
+ * req's is updated such that it's current index and bv offsets correctly
+ * reference any residual data which could not be copied.  The return value
+ * is the number of bytes successfully copied to arg_buf.
+ */
+static int
+dmu_req_copy(void *arg_buf, int size, int *offset, struct request *req)
+{
+	struct bio_vec *bv;
+	struct req_iterator iter;
+	char *bv_buf;
+	int tocpy;
+
+	*offset = 0;
+	rq_for_each_segment(bv, req, iter) {
+
+		/* Fully consumed the passed arg_buf */
+		ASSERT3S(*offset, <=, size);
+		if (size == *offset)
+			break;
+
+		/* Skip fully consumed bv's */
+		if (bv->bv_len == 0)
+			continue;
+
+		tocpy = MIN(bv->bv_len, size - *offset);
+		ASSERT3S(tocpy, >=, 0);
+
+		bv_buf = page_address(bv->bv_page) + bv->bv_offset;
+		ASSERT3P(bv_buf, !=, NULL);
+
+		if (rq_data_dir(req) == WRITE)
+			memcpy(arg_buf + *offset, bv_buf, tocpy);
+		else
+			memcpy(bv_buf, arg_buf + *offset, tocpy);
+
+		*offset += tocpy;
+		bv->bv_offset += tocpy;
+		bv->bv_len -= tocpy;
+	}
+
+	return 0;
+}
+
 int
-dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
+dmu_read_req(objset_t *os, uint64_t object, struct request *req)
 {
+	uint64_t size = blk_rq_bytes(req);
+	uint64_t offset = blk_rq_pos(req) << 9;
 	dmu_buf_t **dbp;
 	int numbufs, i, err;
-	xuio_t *xuio = NULL;
 
 	/*
 	 * NB: we could do this block-at-a-time, but it's nice
 	 * to be reading in parallel.
 	 */
-	err = dmu_buf_hold_array(os, object, uio->uio_loffset, size, TRUE, FTAG,
-	    &numbufs, &dbp);
+	err = dmu_buf_hold_array(os, object, offset, size, TRUE, FTAG,
+				 &numbufs, &dbp);
 	if (err)
 		return (err);
 
-	if (uio->uio_extflg == UIO_XUIO)
-		xuio = (xuio_t *)uio;
-
 	for (i = 0; i < numbufs; i++) {
-		int tocpy;
-		int bufoff;
+		int tocpy, didcpy, bufoff;
 		dmu_buf_t *db = dbp[i];
 
-		ASSERT(size > 0);
+		bufoff = offset - db->db_offset;
+		ASSERT3S(bufoff, >=, 0);
 
-		bufoff = uio->uio_loffset - db->db_offset;
 		tocpy = (int)MIN(db->db_size - bufoff, size);
+		if (tocpy == 0)
+			break;
 
-		if (xuio) {
-			dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
-			arc_buf_t *dbuf_abuf = dbi->db_buf;
-			arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
-			err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
-			if (!err) {
-				uio->uio_resid -= tocpy;
-				uio->uio_loffset += tocpy;
-			}
+		err = dmu_req_copy(db->db_data + bufoff, tocpy, &didcpy, req);
+
+		if (didcpy < tocpy)
+			err = EIO;
 
-			if (abuf == dbuf_abuf)
-				XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
-			else
-				XUIOSTAT_BUMP(xuiostat_rbuf_copied);
-		} else {
-			err = uiomove((char *)db->db_data + bufoff, tocpy,
-			    UIO_READ, uio);
-		}
 		if (err)
 			break;
 
 		size -= tocpy;
+		offset += didcpy;
+		err = 0;
 	}
 	dmu_buf_rele_array(dbp, numbufs, FTAG);
 
 	return (err);
 }
 
-static int
-dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
+int
+dmu_write_req(objset_t *os, uint64_t object, struct request *req, dmu_tx_t *tx)
 {
+	uint64_t size = blk_rq_bytes(req);
+	uint64_t offset = blk_rq_pos(req) << 9;
 	dmu_buf_t **dbp;
 	int numbufs;
 	int err = 0;
 	int i;
 
-	err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
-	    FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
+	if (size == 0)
+		return (0);
+
+	err = dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG,
+				 &numbufs, &dbp);
 	if (err)
 		return (err);
 
 	for (i = 0; i < numbufs; i++) {
-		int tocpy;
-		int bufoff;
+		int tocpy, didcpy, bufoff;
 		dmu_buf_t *db = dbp[i];
 
-		ASSERT(size > 0);
+		bufoff = offset - db->db_offset;
+		ASSERT3S(bufoff, >=, 0);
 
-		bufoff = uio->uio_loffset - db->db_offset;
 		tocpy = (int)MIN(db->db_size - bufoff, size);
+		if (tocpy == 0)
+			break;
 
 		ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
 
@@ -1062,28 +1103,28 @@ dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
 		else
 			dmu_buf_will_dirty(db, tx);
 
-		/*
-		 * XXX uiomove could block forever (eg. nfs-backed
-		 * pages).  There needs to be a uiolockdown() function
-		 * to lock the pages in memory, so that uiomove won't
-		 * block.
-		 */
-		err = uiomove((char *)db->db_data + bufoff, tocpy,
-		    UIO_WRITE, uio);
+		err = dmu_req_copy(db->db_data + bufoff, tocpy, &didcpy, req);
 
 		if (tocpy == db->db_size)
 			dmu_buf_fill_done(db, tx);
 
+		if (didcpy < tocpy)
+			err = EIO;
+
 		if (err)
 			break;
 
 		size -= tocpy;
+		offset += didcpy;
+		err = 0;
 	}
 
 	dmu_buf_rele_array(dbp, numbufs, FTAG);
 	return (err);
 }
+#endif
 
+#ifdef HAVE_ZPL
 int
 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
     dmu_tx_t *tx)
diff --git a/module/zfs/include/sys/blkdev.h b/module/zfs/include/sys/blkdev.h
new file mode 100644
index 000000000..b84f66aed
--- /dev/null
+++ b/module/zfs/include/sys/blkdev.h
@@ -0,0 +1,211 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Written by Brian Behlendorf <[email protected]>.
+ * LLNL-CODE-403049.
+ */
+
+#ifndef	_SYS_BLKDEV_H
+#define	_SYS_BLKDEV_H
+
+#ifdef _KERNEL
+
+#include <linux/blkdev.h>
+#include <linux/elevator.h>
+
+#ifndef HAVE_FMODE_T
+typedef unsigned __bitwise__ fmode_t;
+#endif /* HAVE_FMODE_T */
+
+#ifndef HAVE_BLK_FETCH_REQUEST
+static inline struct request *
+blk_fetch_request(struct request_queue *q)
+{
+	struct request *req;
+
+	req = elv_next_request(q);
+	if (req)
+		blkdev_dequeue_request(req);
+
+	return req;
+}
+#endif /* HAVE_BLK_FETCH_REQUEST */
+
+#ifndef HAVE_BLK_REQUEUE_REQUEST
+static inline void
+blk_requeue_request(request_queue_t *q, struct request *req)
+{
+	elv_requeue_request(q, req);
+}
+#endif /* HAVE_BLK_REQUEUE_REQUEST */
+
+#ifndef HAVE_BLK_END_REQUEST
+static inline bool
+__blk_end_request(struct request *req, int error, unsigned int nr_bytes)
+{
+	LIST_HEAD(list);
+
+	/*
+	 * Request has already been dequeued but 2.6.18 version of
+	 * end_request() unconditionally dequeues the request so we
+	 * add it to a local list to prevent hitting the BUG_ON.
+	 */
+	list_add(&req->queuelist, &list);
+
+	/*
+	 * The old API required the driver to end each segment and not
+	 * the entire request.  In our case we always need to end the
+	 * entire request partial requests are not supported.
+	 */
+	req->hard_cur_sectors = nr_bytes >> 9;
+	end_request(req, ((error == 0) ? 1 : error));
+
+	return 0;
+}
+
+static inline bool
+blk_end_request(struct request *req, int error, unsigned int nr_bytes)
+{
+	struct request_queue *q = req->q;
+	bool rc;
+
+	spin_lock_irq(q->queue_lock);
+	rc = __blk_end_request(req, error, nr_bytes);
+	spin_unlock_irq(q->queue_lock);
+
+	return rc;
+}
+#else
+# ifdef HAVE_BLK_END_REQUEST_GPL_ONLY
+/*
+ * Define required to avoid conflicting 2.6.29 non-static prototype for a
+ * GPL-only version of the helper.  As of 2.6.31 the helper is available
+ * to non-GPL modules and is not explicitly exported GPL-only.
+ */
+# define __blk_end_request __blk_end_request_x
+# define blk_end_request blk_end_request_x
+
+static inline bool
+__blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
+{
+	/*
+	 * The old API required the driver to end each segment and not
+	 * the entire request.  In our case we always need to end the
+	 * entire request partial requests are not supported.
+	 */
+	req->hard_cur_sectors = nr_bytes >> 9;
+	end_request(req, ((error == 0) ? 1 : error));
+
+	return 0;
+}
+static inline bool
+blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
+{
+	struct request_queue *q = req->q;
+	bool rc;
+
+	spin_lock_irq(q->queue_lock);
+	rc = __blk_end_request_x(req, error, nr_bytes);
+	spin_unlock_irq(q->queue_lock);
+
+	return rc;
+}
+# endif /* HAVE_BLK_END_REQUEST_GPL_ONLY */
+#endif /* HAVE_BLK_END_REQUEST */
+
+#ifndef HAVE_BLK_RQ_POS
+static inline sector_t
+blk_rq_pos(struct request *req)
+{
+	return req->sector;
+}
+#endif /* HAVE_BLK_RQ_POS */
+
+#ifndef HAVE_BLK_RQ_SECTORS
+static inline unsigned int
+blk_rq_sectors(struct request *req)
+{
+	return req->nr_sectors;
+}
+#endif /* HAVE_BLK_RQ_SECTORS */
+
+#if !defined(HAVE_BLK_RQ_BYTES) || defined(HAVE_BLK_RQ_BYTES_GPL_ONLY)
+/*
+ * Define required to avoid conflicting 2.6.29 non-static prototype for a
+ * GPL-only version of the helper.  As of 2.6.31 the helper is available
+ * to non-GPL modules in the form of a static inline in the header.
+ */
+#define blk_rq_bytes __blk_rq_bytes
+static inline unsigned int
+__blk_rq_bytes(struct request *req)
+{
+	return blk_rq_sectors(req) << 9;
+}
+#endif /* !HAVE_BLK_RQ_BYTES || HAVE_BLK_RQ_BYTES_GPL_ONLY */
+
+#ifndef HAVE_GET_DISK_RO
+static inline int
+get_disk_ro(struct gendisk *disk)
+{
+	int policy = 0;
+
+	if (disk->part[0])
+		policy = disk->part[0]->policy;
+
+	return policy;
+}
+#endif /* HAVE_GET_DISK_RO */
+
+#ifndef HAVE_RQ_IS_SYNC
+static inline bool
+rq_is_sync(struct request *req)
+{
+	return (req->flags & REQ_RW_SYNC);
+}
+#endif /* HAVE_RQ_IS_SYNC */
+
+#ifndef HAVE_RQ_FOR_EACH_SEGMENT
+struct req_iterator {
+	int i;
+	struct bio *bio;
+};
+
+# define for_each_bio(_bio)              \
+	for (; _bio; _bio = _bio->bi_next)
+
+# define __rq_for_each_bio(_bio, rq)    \
+	if ((rq->bio))                  \
+		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
+
+# define rq_for_each_segment(bvl, _rq, _iter)                   \
+	__rq_for_each_bio(_iter.bio, _rq)                       \
+		bio_for_each_segment(bvl, _iter.bio, _iter.i)
+#endif /* HAVE_RQ_FOR_EACH_SEGMENT */
+
+#ifndef DISK_NAME_LEN
+#define DISK_NAME_LEN	32
+#endif /* DISK_NAME_LEN */
+
+#endif /* KERNEL */
+
+#endif	/* _SYS_BLKDEV_H */
diff --git a/module/zfs/include/sys/dmu.h b/module/zfs/include/sys/dmu.h
index ad4127602..575cb2d10 100644
--- a/module/zfs/include/sys/dmu.h
+++ b/module/zfs/include/sys/dmu.h
@@ -41,13 +41,14 @@
 #include <sys/cred.h>
 #include <sys/time.h>
 #include <sys/uio.h>
+#ifdef _KERNEL
+#include <sys/blkdev.h>
+#endif
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
-struct uio;
-struct xuio;
 struct page;
 struct vnode;
 struct spa;
@@ -512,13 +513,14 @@ void dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 	const void *buf, dmu_tx_t *tx);
 void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
 	dmu_tx_t *tx);
-int dmu_read_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size);
-int dmu_write_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size,
-    dmu_tx_t *tx);
-int dmu_write_uio_dbuf(dmu_buf_t *zdb, struct uio *uio, uint64_t size,
-    dmu_tx_t *tx);
+#ifdef _KERNEL
+int dmu_read_req(objset_t *os, uint64_t object, struct request *req);
+int dmu_write_req(objset_t *os, uint64_t object, struct request *req, dmu_tx_t *tx);
+#endif
+#ifdef HAVE_ZPL
 int dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset,
     uint64_t size, struct page *pp, dmu_tx_t *tx);
+#endif
 struct arc_buf *dmu_request_arcbuf(dmu_buf_t *handle, int size);
 void dmu_return_arcbuf(struct arc_buf *buf);
 void dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, struct arc_buf *buf,
diff --git a/module/zfs/include/sys/spa_impl.h b/module/zfs/include/sys/spa_impl.h
index c965ffbbe..8cab66992 100644
--- a/module/zfs/include/sys/spa_impl.h
+++ b/module/zfs/include/sys/spa_impl.h
@@ -210,7 +210,7 @@ struct spa {
 	kmutex_t	spa_proc_lock;		/* protects spa_proc* */
 	kcondvar_t	spa_proc_cv;		/* spa_proc_state transitions */
 	spa_proc_state_t spa_proc_state;	/* see definition */
-	struct proc	*spa_proc;		/* "zpool-poolname" process */
+	proc_t		*spa_proc;		/* "zpool-poolname" process */
 	uint64_t	spa_did;		/* if procp != p0, did of t1 */
 	boolean_t	spa_autoreplace;	/* autoreplace set in open */
 	int		spa_vdev_locks;		/* locks grabbed */
diff --git a/module/zfs/include/sys/vdev_disk.h b/module/zfs/include/sys/vdev_disk.h
new file mode 100644
index 000000000..986c33624
--- /dev/null
+++ b/module/zfs/include/sys/vdev_disk.h
@@ -0,0 +1,97 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Written by Brian Behlendorf <[email protected]>.
+ * LLNL-CODE-403049.
+ */
+
+#ifndef _SYS_VDEV_DISK_H
+#define _SYS_VDEV_DISK_H
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+#include <sys/vdev.h>
+#include <sys/ddi.h>
+#include <sys/sunldi.h>
+#include <sys/sunddi.h>
+
+typedef struct vdev_disk {
+	ddi_devid_t		vd_devid;
+	char			*vd_minor;
+	struct block_device	*vd_bdev;
+} vdev_disk_t;
+
+extern int vdev_disk_physio(struct block_device *, caddr_t,
+			    size_t, uint64_t, int);
+extern int vdev_disk_read_rootlabel(char *, char *, nvlist_t **);
+
+/* 2.6.24 API change */
+#ifdef HAVE_2ARGS_BIO_END_IO_T
+# define BIO_END_IO_PROTO(fn, x, y, z)	static void fn(struct bio *x, int z)
+# define BIO_END_IO_RETURN(rc)		return
+#else
+# define BIO_END_IO_PROTO(fn, x, y, z)	static int fn(struct bio *x, \
+					              unsigned int y, int z)
+# define BIO_END_IO_RETURN(rc)		return rc
+#endif /* HAVE_2ARGS_BIO_END_IO_T */
+
+/* 2.6.29 API change */
+#ifdef HAVE_BIO_RW_SYNCIO
+# define DIO_RW_SYNCIO			BIO_RW_SYNCIO
+#else
+# define DIO_RW_SYNCIO			BIO_RW_SYNC
+#endif /* HAVE_BIO_RW_SYNCIO */
+
+/* 2.6.28 API change */
+#ifdef HAVE_OPEN_BDEV_EXCLUSIVE
+# define vdev_bdev_open(path, md, hld)	open_bdev_exclusive(path, md, hld)
+# define vdev_bdev_close(bdev, md)	close_bdev_exclusive(bdev, md)
+#else
+# define vdev_bdev_open(path, md, hld)	open_bdev_excl(path, md, hld)
+# define vdev_bdev_close(bdev, md)	close_bdev_excl(bdev)
+#endif /* HAVE_OPEN_BDEV_EXCLUSIVE */
+
+/* 2.6.22 API change */
+#ifdef HAVE_1ARG_INVALIDATE_BDEV
+# define vdev_bdev_invalidate(bdev)	invalidate_bdev(bdev)
+#else
+# define vdev_bdev_invalidate(bdev)	invalidate_bdev(bdev, 1)
+#endif /* HAVE_1ARG_INVALIDATE_BDEV */
+
+/* 2.6.30 API change */
+#ifdef HAVE_BDEV_LOGICAL_BLOCK_SIZE
+# define vdev_bdev_block_size(bdev)	bdev_logical_block_size(bdev)
+#else
+# define vdev_bdev_block_size(bdev)	bdev_hardsect_size(bdev)
+#endif
+
+#endif /* _KERNEL */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_VDEV_DISK_H */
diff --git a/module/zfs/include/sys/zfs_fuid.h b/module/zfs/include/sys/zfs_fuid.h
index 0feb3ce4b..d2989c852 100644
--- a/module/zfs/include/sys/zfs_fuid.h
+++ b/module/zfs/include/sys/zfs_fuid.h
@@ -33,6 +33,7 @@
 #include <sys/zfs_vfsops.h>
 #endif
 #include <sys/avl.h>
+#include <sys/list.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -98,6 +99,7 @@ typedef struct zfs_fuid_info {
 } zfs_fuid_info_t;
 
 #ifdef _KERNEL
+#ifdef HAVE_ZPL
 struct znode;
 extern uid_t zfs_fuid_map_id(zfsvfs_t *, uint64_t, cred_t *, zfs_fuid_type_t);
 extern void zfs_fuid_node_add(zfs_fuid_info_t **, const char *, uint32_t,
@@ -117,6 +119,7 @@ extern int zfs_fuid_find_by_domain(zfsvfs_t *, const char *domain,
     char **retdomain, boolean_t addok);
 extern const char *zfs_fuid_find_by_idx(zfsvfs_t *zfsvfs, uint32_t idx);
 extern void zfs_fuid_txhold(zfsvfs_t *zfsvfs, dmu_tx_t *tx);
+#endif /* HAVE_ZPL */
 #endif
 
 char *zfs_fuid_idx_domain(avl_tree_t *, uint32_t);
diff --git a/module/zfs/include/sys/zfs_ioctl.h b/module/zfs/include/sys/zfs_ioctl.h
index 4a21cc8e0..e3fd2c3fc 100644
--- a/module/zfs/include/sys/zfs_ioctl.h
+++ b/module/zfs/include/sys/zfs_ioctl.h
@@ -316,7 +316,6 @@ extern int zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr);
 extern int zfs_secpolicy_rename_perms(const char *from,
     const char *to, cred_t *cr);
 extern int zfs_secpolicy_destroy_perms(const char *name, cred_t *cr);
-extern int zfs_busy(void);
 extern int zfs_unmount_snap(const char *, void *);
 
 enum zfsdev_state_type {
diff --git a/module/zfs/include/sys/zfs_znode.h b/module/zfs/include/sys/zfs_znode.h
index 9d4b913ed..b2477eddb 100644
--- a/module/zfs/include/sys/zfs_znode.h
+++ b/module/zfs/include/sys/zfs_znode.h
@@ -343,8 +343,10 @@ extern void zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx);
 extern void zfs_upgrade(zfsvfs_t *zfsvfs, dmu_tx_t *tx);
 extern int zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx);
 
+#if defined(HAVE_UIO_RW)
 extern caddr_t zfs_map_page(page_t *, enum seg_rw);
 extern void zfs_unmap_page(page_t *, caddr_t);
+#endif /* HAVE_UIO_RW */
 
 extern zil_get_data_t zfs_get_data;
 extern zil_replay_func_t *zfs_replay_vector[TX_MAX_TYPE];
diff --git a/module/zfs/include/sys/zvol.h b/module/zfs/include/sys/zvol.h
index 0059bf510..c8b9d6507 100644
--- a/module/zfs/include/sys/zvol.h
+++ b/module/zfs/include/sys/zvol.h
@@ -28,49 +28,26 @@
 
 #include <sys/zfs_context.h>
 
-#ifdef	__cplusplus
-extern "C" {
-#endif
-
 #define	ZVOL_OBJ		1ULL
 #define	ZVOL_ZAP_OBJ		2ULL
 
 #ifdef _KERNEL
+
+#include <sys/blkdev.h>
+
 extern int zvol_check_volsize(uint64_t volsize, uint64_t blocksize);
 extern int zvol_check_volblocksize(uint64_t volblocksize);
 extern int zvol_get_stats(objset_t *os, nvlist_t *nv);
 extern void zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx);
 extern int zvol_create_minor(const char *);
+extern int zvol_create_minors(const char *);
 extern int zvol_remove_minor(const char *);
 extern void zvol_remove_minors(const char *);
-extern int zvol_set_volsize(const char *, major_t, uint64_t);
+extern int zvol_set_volsize(const char *, uint64_t);
+extern int zvol_set_volblocksize(const char *, uint64_t);
 
-extern int zvol_open(dev_t *devp, int flag, int otyp, cred_t *cr);
-extern int zvol_dump(dev_t dev, caddr_t addr, daddr_t offset, int nblocks);
-extern int zvol_close(dev_t dev, int flag, int otyp, cred_t *cr);
-extern int zvol_strategy(buf_t *bp);
-extern int zvol_read(dev_t dev, uio_t *uiop, cred_t *cr);
-extern int zvol_write(dev_t dev, uio_t *uiop, cred_t *cr);
-extern int zvol_aread(dev_t dev, struct aio_req *aio, cred_t *cr);
-extern int zvol_awrite(dev_t dev, struct aio_req *aio, cred_t *cr);
-extern int zvol_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *cr,
-    int *rvalp);
-extern int zvol_busy(void);
-extern void zvol_init(void);
+extern int zvol_init(void);
 extern void zvol_fini(void);
 
-extern int zvol_get_volume_params(minor_t minor, uint64_t *blksize,
-    uint64_t *max_xfer_len, void **minor_hdl, void **objset_hdl, void **zil_hdl,
-    void **rl_hdl, void **bonus_hdl);
-extern uint64_t zvol_get_volume_size(void *minor_hdl);
-extern int zvol_get_volume_wce(void *minor_hdl);
-extern void zvol_log_write_minor(void *minor_hdl, dmu_tx_t *tx, offset_t off,
-    ssize_t resid, boolean_t sync);
-
-#endif
-
-#ifdef	__cplusplus
-}
-#endif
-
-#endif	/* _SYS_ZVOL_H */
+#endif /* _KERNEL */
+#endif /* _SYS_ZVOL_H */
diff --git a/module/zfs/rrwlock.c b/module/zfs/rrwlock.c
index 4cef53f95..8d05d9387 100644
--- a/module/zfs/rrwlock.c
+++ b/module/zfs/rrwlock.c
@@ -23,6 +23,8 @@
  * Use is subject to license terms.
  */
 
+#ifdef HAVE_ZPL
+
 #include <sys/refcount.h>
 #include <sys/rrwlock.h>
 
@@ -262,3 +264,4 @@ rrw_held(rrwlock_t *rrl, krw_t rw)
 
 	return (held);
 }
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c
index 4027d0f4f..6fb3f90e3 100644
--- a/module/zfs/spa_misc.c
+++ b/module/zfs/spa_misc.c
@@ -1510,12 +1510,6 @@ spa_name_compare(const void *a1, const void *a2)
 	return (0);
 }
 
-int
-spa_busy(void)
-{
-	return (spa_active_count);
-}
-
 void
 spa_boot_init(void)
 {
diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c
index 4613e951a..1c14c7750 100644
--- a/module/zfs/vdev.c
+++ b/module/zfs/vdev.c
@@ -1069,6 +1069,15 @@ vdev_open_child(void *arg)
 boolean_t
 vdev_uses_zvols(vdev_t *vd)
 {
+/*
+ * Stacking zpools on top of zvols is unsupported until we implement a method
+ * for determining if an arbitrary block device is a zvol without using the
+ * path.  Solaris would check the 'zvol' path component but this does not
+ * exist in the Linux port, so we really should do something like stat the
+ * file and check the major number.  This is complicated by the fact that
+ * we need to do this portably in user or kernel space.
+ */
+#if 0
 	int c;
 
 	if (vd->vdev_path && strncmp(vd->vdev_path, ZVOL_DIR,
@@ -1077,6 +1086,7 @@ vdev_uses_zvols(vdev_t *vd)
 	for (c = 0; c < vd->vdev_children; c++)
 		if (vdev_uses_zvols(vd->vdev_child[c]))
 			return (B_TRUE);
+#endif
 	return (B_FALSE);
 }
 
diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c
new file mode 100644
index 000000000..dbf9b08e7
--- /dev/null
+++ b/module/zfs/vdev_disk.c
@@ -0,0 +1,654 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Rewritten for Linux by Brian Behlendorf <[email protected]>.
+ * LLNL-CODE-403049.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/spa.h>
+#include <sys/vdev_disk.h>
+#include <sys/vdev_impl.h>
+#include <sys/fs/zfs.h>
+#include <sys/zio.h>
+#include <sys/sunldi.h>
+
+/*
+ * Virtual device vector for disks.
+ */
+typedef struct dio_request {
+	struct completion	dr_comp;	/* Completion for sync IO */
+	atomic_t		dr_ref;		/* References */
+	zio_t			*dr_zio;	/* Parent ZIO */
+	int			dr_rw;		/* Read/Write */
+	int			dr_error;	/* Bio error */
+	int			dr_bio_count;	/* Count of bio's */
+        struct bio		*dr_bio[0];	/* Attached bio's */
+} dio_request_t;
+
+
+#ifdef HAVE_OPEN_BDEV_EXCLUSIVE
+static fmode_t
+vdev_bdev_mode(int smode)
+{
+	fmode_t mode = 0;
+
+	ASSERT3S(smode & (FREAD | FWRITE), !=, 0);
+
+	if (smode & FREAD)
+		mode |= FMODE_READ;
+
+	if (smode & FWRITE)
+		mode |= FMODE_WRITE;
+
+	return mode;
+}
+#else
+static int
+vdev_bdev_mode(int smode)
+{
+	int mode = 0;
+
+	ASSERT3S(smode & (FREAD | FWRITE), !=, 0);
+
+	if ((smode & FREAD) && !(smode & FWRITE))
+		mode = MS_RDONLY;
+
+	return mode;
+}
+#endif /* HAVE_OPEN_BDEV_EXCLUSIVE */
+
+static uint64_t
+bdev_capacity(struct block_device *bdev)
+{
+	struct hd_struct *part = bdev->bd_part;
+
+	/* The partition capacity referenced by the block device */
+	if (part)
+	       return part->nr_sects;
+
+	/* Otherwise assume the full device capacity */
+	return get_capacity(bdev->bd_disk);
+}
+
+static int
+vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *ashift)
+{
+	struct block_device *bdev;
+	vdev_disk_t *vd;
+	int mode, block_size;
+
+	/* Must have a pathname and it must be absolute. */
+	if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
+		v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
+		return EINVAL;
+	}
+
+	vd = kmem_zalloc(sizeof(vdev_disk_t), KM_SLEEP);
+	if (vd == NULL)
+		return ENOMEM;
+
+	/*
+	 * Devices are always opened by the path provided at configuration
+	 * time.  This means that if the provided path is a udev by-id path
+	 * then drives may be recabled without an issue.  If the provided
+	 * path is a udev by-path path then the physical location information
+	 * will be preserved.  This can be critical for more complicated
+	 * configurations where drives are located in specific physical
+	 * locations to maximize the systems tolerence to component failure.
+	 * Alternately you can provide your own udev rule to flexibly map
+	 * the drives as you see fit.  It is not advised that you use the
+	 * /dev/[hd]d devices which may be reorder due to probing order.
+	 * Devices in the wrong locations will be detected by the higher
+	 * level vdev validation.
+	 */
+	mode = spa_mode(v->vdev_spa);
+	bdev = vdev_bdev_open(v->vdev_path, vdev_bdev_mode(mode), vd);
+	if (IS_ERR(bdev)) {
+		kmem_free(vd, sizeof(vdev_disk_t));
+		return -PTR_ERR(bdev);
+	}
+
+	v->vdev_tsd = vd;
+	vd->vd_bdev = bdev;
+	block_size =  vdev_bdev_block_size(bdev);
+
+	/* Check if this is a whole device.  When bdev->bd_contains ==
+	 * bdev we have a whole device and not simply a partition. */
+	v->vdev_wholedisk = !!(bdev->bd_contains == bdev);
+
+	/* Clear the nowritecache bit, causes vdev_reopen() to try again. */
+	v->vdev_nowritecache = B_FALSE;
+
+	/* Physical volume size in bytes */
+	*psize = bdev_capacity(bdev) * block_size;
+
+	/* Based on the minimum sector size set the block size */
+	*ashift = highbit(MAX(block_size, SPA_MINBLOCKSIZE)) - 1;
+
+	return 0;
+}
+
+static void
+vdev_disk_close(vdev_t *v)
+{
+	vdev_disk_t *vd = v->vdev_tsd;
+
+	if (vd == NULL)
+		return;
+
+	if (vd->vd_bdev != NULL)
+		vdev_bdev_close(vd->vd_bdev,
+		                vdev_bdev_mode(spa_mode(v->vdev_spa)));
+
+	kmem_free(vd, sizeof(vdev_disk_t));
+	v->vdev_tsd = NULL;
+}
+
+static dio_request_t *
+vdev_disk_dio_alloc(int bio_count)
+{
+	dio_request_t *dr;
+	int i;
+
+	dr = kmem_zalloc(sizeof(dio_request_t) +
+	                 sizeof(struct bio *) * bio_count, KM_SLEEP);
+	if (dr) {
+		init_completion(&dr->dr_comp);
+		atomic_set(&dr->dr_ref, 0);
+		dr->dr_bio_count = bio_count;
+		dr->dr_error = 0;
+
+		for (i = 0; i < dr->dr_bio_count; i++)
+			dr->dr_bio[i] = NULL;
+	}
+
+	return dr;
+}
+
+static void
+vdev_disk_dio_free(dio_request_t *dr)
+{
+	int i;
+
+	for (i = 0; i < dr->dr_bio_count; i++)
+		if (dr->dr_bio[i])
+			bio_put(dr->dr_bio[i]);
+
+	kmem_free(dr, sizeof(dio_request_t) +
+	          sizeof(struct bio *) * dr->dr_bio_count);
+}
+
+static void
+vdev_disk_dio_get(dio_request_t *dr)
+{
+	atomic_inc(&dr->dr_ref);
+}
+
+static int
+vdev_disk_dio_put(dio_request_t *dr)
+{
+	int rc = atomic_dec_return(&dr->dr_ref);
+
+	/*
+	 * Free the dio_request when the last reference is dropped and
+	 * ensure zio_interpret is called only once with the correct zio
+	 */
+	if (rc == 0) {
+		zio_t *zio = dr->dr_zio;
+		int error = dr->dr_error;
+
+		vdev_disk_dio_free(dr);
+
+		if (zio) {
+			zio->io_error = error;
+			zio_interrupt(zio);
+		}
+	}
+
+	return rc;
+}
+
+BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, size, error)
+{
+	dio_request_t *dr = bio->bi_private;
+	int rc;
+
+	/* Fatal error but print some useful debugging before asserting */
+	if (dr == NULL)
+		PANIC("dr == NULL, bio->bi_private == NULL\n"
+		    "bi_next: %p, bi_flags: %lx, bi_rw: %lu, bi_vcnt: %d\n"
+		    "bi_idx: %d, bi_size: %d, bi_end_io: %p, bi_cnt: %d\n",
+		    bio->bi_next, bio->bi_flags, bio->bi_rw, bio->bi_vcnt,
+		    bio->bi_idx, bio->bi_size, bio->bi_end_io,
+		    atomic_read(&bio->bi_cnt));
+
+#ifndef HAVE_2ARGS_BIO_END_IO_T
+	if (bio->bi_size)
+		return 1;
+#endif /* HAVE_2ARGS_BIO_END_IO_T */
+
+	if (error == 0 && !test_bit(BIO_UPTODATE, &bio->bi_flags))
+		error = EIO;
+
+	if (dr->dr_error == 0)
+		dr->dr_error = error;
+
+	/* Drop reference aquired by __vdev_disk_physio */
+	rc = vdev_disk_dio_put(dr);
+
+	/* Wake up synchronous waiter this is the last outstanding bio */
+	if ((rc == 1) && (dr->dr_rw & (1 << DIO_RW_SYNCIO)))
+		complete(&dr->dr_comp);
+
+	BIO_END_IO_RETURN(0);
+}
+
+static inline unsigned long
+bio_nr_pages(void *bio_ptr, unsigned int bio_size)
+{
+	return ((((unsigned long)bio_ptr + bio_size + PAGE_SIZE - 1) >>
+	        PAGE_SHIFT) - ((unsigned long)bio_ptr >> PAGE_SHIFT));
+}
+
+static unsigned int
+bio_map(struct bio *bio, void *bio_ptr, unsigned int bio_size)
+{
+	unsigned int offset, size, i;
+	struct page *page;
+
+	offset = offset_in_page(bio_ptr);
+	for (i = 0; i < bio->bi_max_vecs; i++) {
+		size = PAGE_SIZE - offset;
+
+		if (bio_size <= 0)
+			break;
+
+		if (size > bio_size)
+			size = bio_size;
+
+		if (kmem_virt(bio_ptr))
+			page = vmalloc_to_page(bio_ptr);
+		else
+			page = virt_to_page(bio_ptr);
+
+		if (bio_add_page(bio, page, size, offset) != size)
+			break;
+
+		bio_ptr  += size;
+		bio_size -= size;
+		offset = 0;
+	}
+
+        return bio_size;
+}
+
+static int
+__vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr,
+                   size_t kbuf_size, uint64_t kbuf_offset, int flags)
+{
+        dio_request_t *dr;
+	caddr_t bio_ptr;
+	uint64_t bio_offset;
+	int bio_size, bio_count = 16;
+	int i = 0, error = 0, block_size;
+
+retry:
+	dr = vdev_disk_dio_alloc(bio_count);
+	if (dr == NULL)
+		return ENOMEM;
+
+	dr->dr_zio = zio;
+	dr->dr_rw = flags;
+	block_size = vdev_bdev_block_size(bdev);
+
+#ifdef BIO_RW_FAILFAST
+	if (flags & (1 << BIO_RW_FAILFAST))
+		dr->dr_rw |= 1 << BIO_RW_FAILFAST;
+#endif /* BIO_RW_FAILFAST */
+
+	/*
+	 * When the IO size exceeds the maximum bio size for the request
+	 * queue we are forced to break the IO in multiple bio's and wait
+	 * for them all to complete.  Ideally, all pool users will set
+	 * their volume block size to match the maximum request size and
+	 * the common case will be one bio per vdev IO request.
+	 */
+	bio_ptr    = kbuf_ptr;
+	bio_offset = kbuf_offset;
+	bio_size   = kbuf_size;
+	for (i = 0; i <= dr->dr_bio_count; i++) {
+
+		/* Finished constructing bio's for given buffer */
+		if (bio_size <= 0)
+			break;
+
+		/*
+		 * By default only 'bio_count' bio's per dio are allowed.
+		 * However, if we find ourselves in a situation where more
+		 * are needed we allocate a larger dio and warn the user.
+		 */
+		if (dr->dr_bio_count == i) {
+			vdev_disk_dio_free(dr);
+			bio_count *= 2;
+			printk("WARNING: Resized bio's/dio to %d\n",bio_count);
+			goto retry;
+		}
+
+		dr->dr_bio[i] = bio_alloc(GFP_NOIO,
+		                          bio_nr_pages(bio_ptr, bio_size));
+		if (dr->dr_bio[i] == NULL) {
+			vdev_disk_dio_free(dr);
+			return ENOMEM;
+		}
+
+		/* Matching put called by vdev_disk_physio_completion */
+		vdev_disk_dio_get(dr);
+
+		dr->dr_bio[i]->bi_bdev = bdev;
+		dr->dr_bio[i]->bi_sector = bio_offset / block_size;
+		dr->dr_bio[i]->bi_rw = dr->dr_rw;
+		dr->dr_bio[i]->bi_end_io = vdev_disk_physio_completion;
+		dr->dr_bio[i]->bi_private = dr;
+
+		/* Remaining size is returned to become the new size */
+		bio_size = bio_map(dr->dr_bio[i], bio_ptr, bio_size);
+
+		/* Advance in buffer and construct another bio if needed */
+		bio_ptr    += dr->dr_bio[i]->bi_size;
+		bio_offset += dr->dr_bio[i]->bi_size;
+	}
+
+	/* Extra reference to protect dio_request during submit_bio */
+	vdev_disk_dio_get(dr);
+
+	/* Submit all bio's associated with this dio */
+	for (i = 0; i < dr->dr_bio_count; i++)
+		if (dr->dr_bio[i])
+			submit_bio(dr->dr_rw, dr->dr_bio[i]);
+
+	/*
+	 * On synchronous blocking requests we wait for all bio the completion
+	 * callbacks to run.  We will be woken when the last callback runs
+	 * for this dio.  We are responsible for putting the last dio_request
+	 * reference will in turn put back the last bio references.  The
+	 * only synchronous consumer is vdev_disk_read_rootlabel() all other
+	 * IO originating from vdev_disk_io_start() is asynchronous.
+	 */
+	if (dr->dr_rw & (1 << DIO_RW_SYNCIO)) {
+		wait_for_completion(&dr->dr_comp);
+		error = dr->dr_error;
+		ASSERT3S(atomic_read(&dr->dr_ref), ==, 1);
+	}
+
+	(void)vdev_disk_dio_put(dr);
+
+	return error;
+}
+
+int
+vdev_disk_physio(struct block_device *bdev, caddr_t kbuf,
+		 size_t size, uint64_t offset, int flags)
+{
+	return __vdev_disk_physio(bdev, NULL, kbuf, size, offset, flags);
+}
+
+/* 2.6.24 API change */
+#ifdef HAVE_BIO_EMPTY_BARRIER
+BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, size, rc)
+{
+	zio_t *zio = bio->bi_private;
+
+	zio->io_error = -rc;
+	if (rc && (rc == -EOPNOTSUPP))
+		zio->io_vd->vdev_nowritecache = B_TRUE;
+
+	bio_put(bio);
+	zio_interrupt(zio);
+
+	BIO_END_IO_RETURN(0);
+}
+
+static int
+vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
+{
+	struct request_queue *q;
+	struct bio *bio;
+
+	q = bdev_get_queue(bdev);
+	if (!q)
+		return ENXIO;
+
+	bio = bio_alloc(GFP_KERNEL, 0);
+	if (!bio)
+		return ENOMEM;
+
+	bio->bi_end_io = vdev_disk_io_flush_completion;
+	bio->bi_private = zio;
+	bio->bi_bdev = bdev;
+	submit_bio(WRITE_BARRIER, bio);
+
+	return 0;
+}
+#else
+static int
+vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
+{
+	return ENOTSUP;
+}
+#endif /* HAVE_BIO_EMPTY_BARRIER */
+
+static int
+vdev_disk_io_start(zio_t *zio)
+{
+	vdev_t *v = zio->io_vd;
+	vdev_disk_t *vd = v->vdev_tsd;
+	int flags, error;
+
+	switch (zio->io_type) {
+	case ZIO_TYPE_IOCTL:
+
+		if (!vdev_readable(v)) {
+			zio->io_error = ENXIO;
+			return ZIO_PIPELINE_CONTINUE;
+		}
+
+		switch (zio->io_cmd) {
+		case DKIOCFLUSHWRITECACHE:
+
+			if (zfs_nocacheflush)
+				break;
+
+			if (v->vdev_nowritecache) {
+				zio->io_error = ENOTSUP;
+				break;
+			}
+
+			error = vdev_disk_io_flush(vd->vd_bdev, zio);
+			if (error == 0)
+				return ZIO_PIPELINE_STOP;
+
+			zio->io_error = error;
+			if (error == ENOTSUP)
+				v->vdev_nowritecache = B_TRUE;
+
+			break;
+
+		default:
+			zio->io_error = ENOTSUP;
+		}
+
+		return ZIO_PIPELINE_CONTINUE;
+
+	case ZIO_TYPE_WRITE:
+		flags = WRITE;
+		break;
+
+	case ZIO_TYPE_READ:
+		flags = READ;
+		break;
+
+	default:
+		zio->io_error = ENOTSUP;
+		return ZIO_PIPELINE_CONTINUE;
+	}
+
+#ifdef BIO_RW_FAILFAST
+	if (zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD))
+		flags |= (1 << BIO_RW_FAILFAST);
+#endif /* BIO_RW_FAILFAST */
+
+	error = __vdev_disk_physio(vd->vd_bdev, zio, zio->io_data,
+		                   zio->io_size, zio->io_offset, flags);
+	if (error) {
+		zio->io_error = error;
+		return ZIO_PIPELINE_CONTINUE;
+	}
+
+	return ZIO_PIPELINE_STOP;
+}
+
+static void
+vdev_disk_io_done(zio_t *zio)
+{
+	/*
+	 * If the device returned EIO, we revalidate the media.  If it is
+	 * determined the media has changed this triggers the asynchronous
+	 * removal of the device from the configuration.
+	 */
+	if (zio->io_error == EIO) {
+	        vdev_t *v = zio->io_vd;
+		vdev_disk_t *vd = v->vdev_tsd;
+
+		if (check_disk_change(vd->vd_bdev)) {
+			vdev_bdev_invalidate(vd->vd_bdev);
+			v->vdev_remove_wanted = B_TRUE;
+			spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
+		}
+	}
+}
+
+static void
+vdev_disk_hold(vdev_t *vd)
+{
+	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
+
+	/* We must have a pathname, and it must be absolute. */
+	if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
+		return;
+
+	/*
+	 * Only prefetch path and devid info if the device has
+	 * never been opened.
+	 */
+	if (vd->vdev_tsd != NULL)
+		return;
+
+	/* XXX: Implement me as a vnode lookup for the device */
+	vd->vdev_name_vp = NULL;
+	vd->vdev_devid_vp = NULL;
+}
+
+static void
+vdev_disk_rele(vdev_t *vd)
+{
+	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
+
+	/* XXX: Implement me as a vnode rele for the device */
+}
+
+vdev_ops_t vdev_disk_ops = {
+	vdev_disk_open,
+	vdev_disk_close,
+	vdev_default_asize,
+	vdev_disk_io_start,
+	vdev_disk_io_done,
+	NULL,
+	vdev_disk_hold,
+	vdev_disk_rele,
+	VDEV_TYPE_DISK,		/* name of this vdev type */
+	B_TRUE			/* leaf vdev */
+};
+
+/*
+ * Given the root disk device devid or pathname, read the label from
+ * the device, and construct a configuration nvlist.
+ */
+int
+vdev_disk_read_rootlabel(char *devpath, char *devid, nvlist_t **config)
+{
+	struct block_device *bdev;
+	vdev_label_t *label;
+	uint64_t s, size;
+	int i;
+
+	bdev = vdev_bdev_open(devpath, vdev_bdev_mode(FREAD), NULL);
+	if (IS_ERR(bdev))
+		return -PTR_ERR(bdev);
+
+	s = bdev_capacity(bdev) * vdev_bdev_block_size(bdev);
+	if (s == 0) {
+		vdev_bdev_close(bdev, vdev_bdev_mode(FREAD));
+		return EIO;
+	}
+
+	size = P2ALIGN_TYPED(s, sizeof(vdev_label_t), uint64_t);
+	label = vmem_alloc(sizeof(vdev_label_t), KM_SLEEP);
+
+	for (i = 0; i < VDEV_LABELS; i++) {
+	        uint64_t offset, state, txg = 0;
+
+		/* read vdev label */
+		offset = vdev_label_offset(size, i, 0);
+		if (vdev_disk_physio(bdev, (caddr_t)label,
+		    VDEV_SKIP_SIZE + VDEV_PHYS_SIZE, offset, READ_SYNC) != 0)
+			continue;
+
+		if (nvlist_unpack(label->vl_vdev_phys.vp_nvlist,
+		    sizeof (label->vl_vdev_phys.vp_nvlist), config, 0) != 0) {
+			*config = NULL;
+			continue;
+		}
+
+		if (nvlist_lookup_uint64(*config, ZPOOL_CONFIG_POOL_STATE,
+		    &state) != 0 || state >= POOL_STATE_DESTROYED) {
+			nvlist_free(*config);
+			*config = NULL;
+			continue;
+		}
+
+		if (nvlist_lookup_uint64(*config, ZPOOL_CONFIG_POOL_TXG,
+		    &txg) != 0 || txg == 0) {
+			nvlist_free(*config);
+			*config = NULL;
+			continue;
+		}
+
+		break;
+	}
+
+	vmem_free(label, sizeof(vdev_label_t));
+	vdev_bdev_close(bdev, vdev_bdev_mode(FREAD));
+
+	return 0;
+}
diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c
index 843b5ff06..f1ba9af57 100644
--- a/module/zfs/zfs_acl.c
+++ b/module/zfs/zfs_acl.c
@@ -22,6 +22,8 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
+#ifdef HAVE_ZPL
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/time.h>
@@ -2746,3 +2748,5 @@ zfs_zaccess_rename(znode_t *sdzp, znode_t *szp, znode_t *tdzp,
 
 	return (error);
 }
+
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_ctldir.c b/module/zfs/zfs_ctldir.c
index 815f8895e..51b12a1d5 100644
--- a/module/zfs/zfs_ctldir.c
+++ b/module/zfs/zfs_ctldir.c
@@ -63,6 +63,8 @@
  * so that it cannot be freed until all snapshots have been unmounted.
  */
 
+#ifdef HAVE_ZPL
+
 #include <fs/fs_subr.h>
 #include <sys/zfs_ctldir.h>
 #include <sys/zfs_ioctl.h>
@@ -1347,3 +1349,4 @@ zfsctl_umount_snapshots(vfs_t *vfsp, int fflags, cred_t *cr)
 
 	return (error);
 }
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_dir.c b/module/zfs/zfs_dir.c
index b06d29ab3..f0084cd4e 100644
--- a/module/zfs/zfs_dir.c
+++ b/module/zfs/zfs_dir.c
@@ -22,6 +22,8 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
+#ifdef HAVE_ZPL
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/time.h>
@@ -1087,3 +1089,4 @@ zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr)
 	else
 		return (secpolicy_vnode_remove(cr));
 }
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_fuid.c b/module/zfs/zfs_fuid.c
index 1773a14a1..97fceea00 100644
--- a/module/zfs/zfs_fuid.c
+++ b/module/zfs/zfs_fuid.c
@@ -192,6 +192,7 @@ zfs_fuid_idx_domain(avl_tree_t *idx_tree, uint32_t idx)
 }
 
 #ifdef _KERNEL
+#ifdef HAVE_ZPL
 /*
  * Load the fuid table(s) into memory.
  */
@@ -753,4 +754,5 @@ zfs_fuid_txhold(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
 		    FUID_SIZE_ESTIMATE(zfsvfs));
 	}
 }
+#endif /* HAVE_ZPL */
 #endif
diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c
index 99287e443..902c2342a 100644
--- a/module/zfs/zfs_ioctl.c
+++ b/module/zfs/zfs_ioctl.c
@@ -1292,6 +1292,9 @@ zfs_ioc_pool_import(zfs_cmd_t *zc)
 			error = err;
 	}
 
+	if (error == 0)
+		zvol_create_minors(zc->zc_name);
+
 	nvlist_free(config);
 
 	if (props)
@@ -2179,8 +2182,7 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 		err = dsl_dataset_set_reservation(dsname, source, intval);
 		break;
 	case ZFS_PROP_VOLSIZE:
-		err = zvol_set_volsize(dsname, ddi_driver_major(zfs_dip),
-		    intval);
+		err = zvol_set_volsize(dsname, intval);
 		break;
 	case ZFS_PROP_VERSION:
 	{
@@ -2654,6 +2656,30 @@ zfs_ioc_pool_get_props(zfs_cmd_t *zc)
 
 /*
  * inputs:
+ * zc_name              name of volume
+ *
+ * outputs:             none
+ */
+static int
+zfs_ioc_create_minor(zfs_cmd_t *zc)
+{
+	return (zvol_create_minor(zc->zc_name));
+}
+
+/*
+ * inputs:
+ * zc_name              name of volume
+ *
+ * outputs:             none
+ */
+static int
+zfs_ioc_remove_minor(zfs_cmd_t *zc)
+{
+	return (zvol_remove_minor(zc->zc_name));
+}
+
+/*
+ * inputs:
  * zc_name		name of filesystem
  * zc_nvlist_src{_size}	nvlist of delegated permissions
  * zc_perm_action	allow/unallow flag
@@ -4805,6 +4831,10 @@ static zfs_ioc_vec_t zfs_ioc_vec[] = {
 	    POOL_CHECK_SUSPENDED },
 	{ zfs_ioc_set_prop, zfs_secpolicy_none, DATASET_NAME, B_TRUE,
 	    POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY },
+	{ zfs_ioc_create_minor, zfs_secpolicy_config, DATASET_NAME, B_FALSE,
+	    POOL_CHECK_NONE },
+	{ zfs_ioc_remove_minor, zfs_secpolicy_config, DATASET_NAME, B_FALSE,
+	    POOL_CHECK_NONE },
 	{ zfs_ioc_create, zfs_secpolicy_create, DATASET_NAME, B_TRUE,
 	    POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY },
 	{ zfs_ioc_destroy, zfs_secpolicy_destroy, DATASET_NAME, B_TRUE,
diff --git a/module/zfs/zfs_log.c b/module/zfs/zfs_log.c
index 26ab78279..59a6451c7 100644
--- a/module/zfs/zfs_log.c
+++ b/module/zfs/zfs_log.c
@@ -22,6 +22,8 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
+#ifdef HAVE_ZPL
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -674,3 +676,5 @@ zfs_log_acl(zilog_t *zilog, dmu_tx_t *tx, znode_t *zp,
 	itx->itx_sync = (zp->z_sync_cnt != 0);
 	zil_itx_assign(zilog, itx, tx);
 }
+
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_replay.c b/module/zfs/zfs_replay.c
index 9fb336856..bd29ab9a1 100644
--- a/module/zfs/zfs_replay.c
+++ b/module/zfs/zfs_replay.c
@@ -22,6 +22,8 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
+#ifdef HAVE_ZPL
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -929,3 +931,4 @@ zil_replay_func_t *zfs_replay_vector[TX_MAX_TYPE] = {
 	zfs_replay_create_acl,	/* TX_MKDIR_ACL_ATTR */
 	zfs_replay_write2,	/* TX_WRITE2 */
 };
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_vfsops.c b/module/zfs/zfs_vfsops.c
index cb8c1d086..7e4e82f5e 100644
--- a/module/zfs/zfs_vfsops.c
+++ b/module/zfs/zfs_vfsops.c
@@ -65,6 +65,7 @@
 #include <sys/sa.h>
 #include "zfs_comutil.h"
 
+#ifdef HAVE_ZPL
 int zfsfstype;
 vfsops_t *zfs_vfsops = NULL;
 static major_t zfs_major;
@@ -2127,10 +2128,12 @@ zfs_vfsinit(int fstype, char *name)
 
 	return (0);
 }
+#endif /* HAVE_ZPL */
 
 void
 zfs_init(void)
 {
+#ifdef HAVE_ZPL
 	/*
 	 * Initialize .zfs directory structures
 	 */
@@ -2142,21 +2145,19 @@ zfs_init(void)
 	zfs_znode_init();
 
 	dmu_objset_register_type(DMU_OST_ZFS, zfs_space_delta_cb);
+#endif /* HAVE_ZPL */
 }
 
 void
 zfs_fini(void)
 {
+#ifdef HAVE_ZPL
 	zfsctl_fini();
 	zfs_znode_fini();
+#endif /* HAVE_ZPL */
 }
 
-int
-zfs_busy(void)
-{
-	return (zfs_active_fs_count != 0);
-}
-
+#ifdef HAVE_ZPL
 int
 zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers)
 {
@@ -2224,6 +2225,7 @@ zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers)
 
 	return (0);
 }
+#endif /* HAVE_ZPL */
 
 /*
  * Read a property stored within the master node.
@@ -2267,6 +2269,7 @@ zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value)
 	return (error);
 }
 
+#ifdef HAVE_ZPL
 static vfsdef_t vfw = {
 	VFSDEF_VERSION,
 	MNTTYPE_ZFS,
@@ -2279,3 +2282,4 @@ static vfsdef_t vfw = {
 struct modlfs zfs_modlfs = {
 	&mod_fsops, "ZFS filesystem version " SPA_VERSION_STRING, &vfw
 };
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c
index a0720079c..5899c7f7d 100644
--- a/module/zfs/zfs_vnops.c
+++ b/module/zfs/zfs_vnops.c
@@ -25,6 +25,8 @@
 /* Portions Copyright 2007 Jeremy Teo */
 /* Portions Copyright 2010 Robert Milkowski */
 
+#ifdef HAVE_ZPL
+
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/time.h>
@@ -319,6 +321,7 @@ zfs_ioctl(vnode_t *vp, int com, intptr_t data, int flag, cred_t *cred,
 	return (ENOTTY);
 }
 
+#if defined(_KERNEL) && defined(HAVE_UIO_RW)
 /*
  * Utility functions to map and unmap a single physical page.  These
  * are used to manage the mappable copies of ZFS file data, and therefore
@@ -343,6 +346,7 @@ zfs_unmap_page(page_t *pp, caddr_t addr)
 		ppmapout(addr);
 	}
 }
+#endif /* _KERNEL && HAVE_UIO_RW */
 
 /*
  * When a file is memory mapped, we must keep the IO data synchronized
@@ -5241,3 +5245,4 @@ const fs_operation_def_t zfs_evnodeops_template[] = {
 	VOPNAME_PATHCONF,	{ .vop_pathconf = zfs_pathconf },
 	NULL,			NULL
 };
+#endif /* HAVE_ZPL */
diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c
index 79d8d2e7e..5e6bab81a 100644
--- a/module/zfs/zfs_znode.c
+++ b/module/zfs/zfs_znode.c
@@ -88,6 +88,7 @@
  * (such as VFS logic) that will not compile easily in userland.
  */
 #ifdef _KERNEL
+#ifdef HAVE_ZPL
 /*
  * Needed to close a small window in zfs_znode_move() that allows the zfsvfs to
  * be freed before it can be safely accessed.
@@ -1737,22 +1738,29 @@ log:
 	dmu_tx_commit(tx);
 	return (0);
 }
+#endif /* HAVE_ZPL */
 
 void
 zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 {
-	zfsvfs_t	zfsvfs;
 	uint64_t	moid, obj, sa_obj, version;
 	uint64_t	sense = ZFS_CASE_SENSITIVE;
 	uint64_t	norm = 0;
 	nvpair_t	*elem;
 	int		error;
+#ifdef HAVE_ZPL
+	zfsvfs_t	zfsvfs;
 	int		i;
 	znode_t		*rootzp = NULL;
 	vnode_t		*vp;
 	vattr_t		vattr;
 	znode_t		*zp;
 	zfs_acl_ids_t	acl_ids;
+#else
+	timestruc_t	now;
+	dmu_buf_t	*db;
+	znode_phys_t	*pzp;
+#endif /* HAVE_ZPL */
 
 	/*
 	 * First attempt to create master node.
@@ -1814,6 +1822,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	error = zap_add(os, moid, ZFS_UNLINKED_SET, 8, 1, &obj, tx);
 	ASSERT(error == 0);
 
+#ifdef HAVE_ZPL
 	/*
 	 * Create root znode.  Create minimal znode/vnode/zfsvfs
 	 * to allow zfs_mknode to work.
@@ -1879,17 +1888,49 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
 	ZTOV(rootzp)->v_count = 0;
 	sa_handle_destroy(rootzp->z_sa_hdl);
 	kmem_cache_free(znode_cache, rootzp);
+	error = zfs_create_share_dir(&zfsvfs, tx);
 
+	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
+		mutex_destroy(&zfsvfs.z_hold_mtx[i]);
+#else
 	/*
-	 * Create shares directory
+	 * Create root znode with code free of VFS dependencies
 	 */
+	obj = zap_create_norm(os, norm, DMU_OT_DIRECTORY_CONTENTS,
+	                      DMU_OT_ZNODE, sizeof (znode_phys_t), tx);
 
-	error = zfs_create_share_dir(&zfsvfs, tx);
+	VERIFY(0 == dmu_bonus_hold(os, obj, FTAG, &db));
+	dmu_buf_will_dirty(db, tx);
 
+	/*
+	 * Initialize the znode physical data to zero.
+	 */
+	ASSERT(db->db_size >= sizeof (znode_phys_t));
+	bzero(db->db_data, db->db_size);
+	pzp = db->db_data;
+
+	if (USE_FUIDS(version, os))
+		pzp->zp_flags = ZFS_ARCHIVE | ZFS_AV_MODIFIED;
+
+	pzp->zp_size = 2; /* "." and ".." */
+	pzp->zp_links = 2;
+	pzp->zp_parent = obj;
+	pzp->zp_gen = dmu_tx_get_txg(tx);
+	pzp->zp_mode = S_IFDIR | 0755;
+	pzp->zp_flags = ZFS_ACL_TRIVIAL;
+
+	gethrestime(&now);
+
+	ZFS_TIME_ENCODE(&now, pzp->zp_crtime);
+	ZFS_TIME_ENCODE(&now, pzp->zp_ctime);
+	ZFS_TIME_ENCODE(&now, pzp->zp_atime);
+	ZFS_TIME_ENCODE(&now, pzp->zp_mtime);
+
+	error = zap_add(os, moid, ZFS_ROOT_OBJ, 8, 1, &obj, tx);
 	ASSERT(error == 0);
 
-	for (i = 0; i != ZFS_OBJ_MTX_SZ; i++)
-		mutex_destroy(&zfsvfs.z_hold_mtx[i]);
+	dmu_buf_rele(db, FTAG);
+#endif /* HAVE_ZPL */
 }
 
 #endif /* _KERNEL */
diff --git a/module/zfs/zvol.c b/module/zfs/zvol.c
new file mode 100644
index 000000000..6e9294292
--- /dev/null
+++ b/module/zfs/zvol.c
@@ -0,0 +1,1337 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Rewritten for Linux by Brian Behlendorf <[email protected]>.
+ * LLNL-CODE-403049.
+ *
+ * ZFS volume emulation driver.
+ *
+ * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes.
+ * Volumes are accessed through the symbolic links named:
+ *
+ * /dev/<pool_name>/<dataset_name>
+ *
+ * Volumes are persistent through reboot and module load.  No user command
+ * needs to be run before opening and using a device.
+ */
+
+#include <sys/dmu_traverse.h>
+#include <sys/dsl_dataset.h>
+#include <sys/dsl_prop.h>
+#include <sys/zap.h>
+#include <sys/zil_impl.h>
+#include <sys/zio.h>
+#include <sys/zfs_rlock.h>
+#include <sys/zfs_znode.h>
+#include <sys/zvol.h>
+
+unsigned int zvol_major = ZVOL_MAJOR;
+unsigned int zvol_threads = 0;
+
+static taskq_t *zvol_taskq;
+static kmutex_t zvol_state_lock;
+static list_t zvol_state_list;
+static char *zvol_tag = "zvol_tag";
+
+/*
+ * The in-core state of each volume.
+ */
+typedef struct zvol_state {
+	char			zv_name[DISK_NAME_LEN];	/* name */
+	uint64_t		zv_volsize;	/* advertised space */
+	uint64_t		zv_volblocksize;/* volume block size */
+	objset_t		*zv_objset;	/* objset handle */
+	uint32_t		zv_flags;	/* ZVOL_* flags */
+	uint32_t		zv_open_count;	/* open counts */
+	uint32_t		zv_changed;	/* disk changed */
+	zilog_t			*zv_zilog;	/* ZIL handle */
+	znode_t			zv_znode;	/* for range locking */
+	dmu_buf_t		*zv_dbuf;	/* bonus handle */
+	dev_t			zv_dev;		/* device id */
+	struct gendisk		*zv_disk;	/* generic disk */
+	struct request_queue	*zv_queue;	/* request queue */
+	spinlock_t		zv_lock;	/* request queue lock */
+	list_node_t		zv_next;	/* next zvol_state_t linkage */
+} zvol_state_t;
+
+#define	ZVOL_RDONLY	0x1
+
+/*
+ * Find the next available range of ZVOL_MINORS minor numbers.  The
+ * zvol_state_list is kept in ascending minor order so we simply need
+ * to scan the list for the first gap in the sequence.  This allows us
+ * to recycle minor number as devices are created and removed.
+ */
+static int
+zvol_find_minor(unsigned *minor)
+{
+	zvol_state_t *zv;
+
+	*minor = 0;
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	     zv = list_next(&zvol_state_list, zv), *minor += ZVOL_MINORS) {
+		if (MINOR(zv->zv_dev) != MINOR(*minor))
+			break;
+	}
+
+	/* All minors are in use */
+	if (*minor >= (1 << MINORBITS))
+		return ENXIO;
+
+	return 0;
+}
+
+/*
+ * Find a zvol_state_t given the full major+minor dev_t.
+ */
+static zvol_state_t *
+zvol_find_by_dev(dev_t dev)
+{
+	zvol_state_t *zv;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	     zv = list_next(&zvol_state_list, zv)) {
+		if (zv->zv_dev == dev)
+			return zv;
+	}
+
+	return NULL;
+}
+
+/*
+ * Find a zvol_state_t given the name provided at zvol_alloc() time.
+ */
+static zvol_state_t *
+zvol_find_by_name(const char *name)
+{
+	zvol_state_t *zv;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	     zv = list_next(&zvol_state_list, zv)) {
+		if (!strncmp(zv->zv_name, name, DISK_NAME_LEN))
+			return zv;
+	}
+
+	return NULL;
+}
+
+/*
+ * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation.
+ */
+void
+zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
+{
+	zfs_creat_t *zct = arg;
+	nvlist_t *nvprops = zct->zct_props;
+	int error;
+	uint64_t volblocksize, volsize;
+
+	VERIFY(nvlist_lookup_uint64(nvprops,
+	    zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) == 0);
+	if (nvlist_lookup_uint64(nvprops,
+	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0)
+		volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE);
+
+	/*
+	 * These properties must be removed from the list so the generic
+	 * property setting step won't apply to them.
+	 */
+	VERIFY(nvlist_remove_all(nvprops,
+	    zfs_prop_to_name(ZFS_PROP_VOLSIZE)) == 0);
+	(void) nvlist_remove_all(nvprops,
+	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE));
+
+	error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize,
+	    DMU_OT_NONE, 0, tx);
+	ASSERT(error == 0);
+
+	error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP,
+	    DMU_OT_NONE, 0, tx);
+	ASSERT(error == 0);
+
+	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx);
+	ASSERT(error == 0);
+}
+
+/*
+ * ZFS_IOC_OBJSET_STATS entry point.
+ */
+int
+zvol_get_stats(objset_t *os, nvlist_t *nv)
+{
+	int error;
+	dmu_object_info_t *doi;
+	uint64_t val;
+
+	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val);
+	if (error)
+		return (error);
+
+	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val);
+	doi = kmem_alloc(sizeof(dmu_object_info_t), KM_SLEEP);
+	error = dmu_object_info(os, ZVOL_OBJ, doi);
+
+	if (error == 0) {
+		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE,
+		    doi->doi_data_block_size);
+	}
+
+	kmem_free(doi, sizeof(dmu_object_info_t));
+
+	return (error);
+}
+
+/*
+ * Sanity check volume size.
+ */
+int
+zvol_check_volsize(uint64_t volsize, uint64_t blocksize)
+{
+	if (volsize == 0)
+		return (EINVAL);
+
+	if (volsize % blocksize != 0)
+		return (EINVAL);
+
+#ifdef _ILP32
+	if (volsize - 1 > MAXOFFSET_T)
+		return (EOVERFLOW);
+#endif
+	return (0);
+}
+
+/*
+ * Ensure the zap is flushed then inform the VFS of the capacity change.
+ */
+static int
+zvol_update_volsize(zvol_state_t *zv, uint64_t volsize)
+{
+	struct block_device *bdev;
+	dmu_tx_t *tx;
+	int error;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+
+	tx = dmu_tx_create(zv->zv_objset);
+	dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
+	error = dmu_tx_assign(tx, TXG_WAIT);
+	if (error) {
+		dmu_tx_abort(tx);
+		return (error);
+	}
+
+	error = zap_update(zv->zv_objset, ZVOL_ZAP_OBJ, "size", 8, 1,
+	    &volsize, tx);
+	dmu_tx_commit(tx);
+
+	if (error)
+		return (error);
+
+	error = dmu_free_long_range(zv->zv_objset,
+	    ZVOL_OBJ, volsize, DMU_OBJECT_END);
+	if (error)
+		return (error);
+
+	zv->zv_volsize = volsize;
+	zv->zv_changed = 1;
+
+	bdev = bdget_disk(zv->zv_disk, 0);
+	if (!bdev)
+		return EIO;
+
+	error = check_disk_change(bdev);
+	ASSERT3U(error, !=, 0);
+	bdput(bdev);
+
+	return (0);
+}
+
+/*
+ * Set ZFS_PROP_VOLSIZE set entry point.
+ */
+int
+zvol_set_volsize(const char *name, uint64_t volsize)
+{
+	zvol_state_t *zv;
+	dmu_object_info_t *doi;
+	objset_t *os = NULL;
+	uint64_t readonly;
+	int error;
+
+	mutex_enter(&zvol_state_lock);
+
+	zv = zvol_find_by_name(name);
+	if (zv == NULL) {
+		error = ENXIO;
+		goto out;
+	}
+
+	doi = kmem_alloc(sizeof(dmu_object_info_t), KM_SLEEP);
+
+	error = dmu_objset_hold(name, FTAG, &os);
+	if (error)
+		goto out_doi;
+
+	if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) != 0 ||
+	    (error = zvol_check_volsize(volsize,doi->doi_data_block_size)) != 0)
+		goto out_doi;
+
+	VERIFY(dsl_prop_get_integer(name, "readonly", &readonly, NULL) == 0);
+	if (readonly) {
+		error = EROFS;
+		goto out_doi;
+	}
+
+	if (get_disk_ro(zv->zv_disk) || (zv->zv_flags & ZVOL_RDONLY)) {
+		error = EROFS;
+		goto out_doi;
+	}
+
+	error = zvol_update_volsize(zv, volsize);
+out_doi:
+	kmem_free(doi, sizeof(dmu_object_info_t));
+out:
+	if (os)
+		dmu_objset_rele(os, FTAG);
+
+	mutex_exit(&zvol_state_lock);
+
+	return (error);
+}
+
+/*
+ * Sanity check volume block size.
+ */
+int
+zvol_check_volblocksize(uint64_t volblocksize)
+{
+	if (volblocksize < SPA_MINBLOCKSIZE ||
+	    volblocksize > SPA_MAXBLOCKSIZE ||
+	    !ISP2(volblocksize))
+		return (EDOM);
+
+	return (0);
+}
+
+/*
+ * Set ZFS_PROP_VOLBLOCKSIZE set entry point.
+ */
+int
+zvol_set_volblocksize(const char *name, uint64_t volblocksize)
+{
+	zvol_state_t *zv;
+	dmu_tx_t *tx;
+	int error;
+
+	mutex_enter(&zvol_state_lock);
+
+	zv = zvol_find_by_name(name);
+	if (zv == NULL) {
+		error = ENXIO;
+		goto out;
+	}
+
+	if (get_disk_ro(zv->zv_disk) || (zv->zv_flags & ZVOL_RDONLY)) {
+		error = EROFS;
+		goto out;
+	}
+
+	tx = dmu_tx_create(zv->zv_objset);
+	dmu_tx_hold_bonus(tx, ZVOL_OBJ);
+	error = dmu_tx_assign(tx, TXG_WAIT);
+	if (error) {
+		dmu_tx_abort(tx);
+	} else {
+		error = dmu_object_set_blocksize(zv->zv_objset, ZVOL_OBJ,
+		    volblocksize, 0, tx);
+		if (error == ENOTSUP)
+			error = EBUSY;
+		dmu_tx_commit(tx);
+		if (error == 0)
+			zv->zv_volblocksize = volblocksize;
+	}
+out:
+	mutex_exit(&zvol_state_lock);
+
+	return (error);
+}
+
+/*
+ * Replay a TX_WRITE ZIL transaction that didn't get committed
+ * after a system failure
+ */
+static int
+zvol_replay_write(zvol_state_t *zv, lr_write_t *lr, boolean_t byteswap)
+{
+	objset_t *os = zv->zv_objset;
+	char *data = (char *)(lr + 1);	/* data follows lr_write_t */
+	uint64_t off = lr->lr_offset;
+	uint64_t len = lr->lr_length;
+	dmu_tx_t *tx;
+	int error;
+
+	if (byteswap)
+		byteswap_uint64_array(lr, sizeof (*lr));
+
+	tx = dmu_tx_create(os);
+	dmu_tx_hold_write(tx, ZVOL_OBJ, off, len);
+	error = dmu_tx_assign(tx, TXG_WAIT);
+	if (error) {
+		dmu_tx_abort(tx);
+	} else {
+		dmu_write(os, ZVOL_OBJ, off, len, data, tx);
+		dmu_tx_commit(tx);
+	}
+
+	return (error);
+}
+
+static int
+zvol_replay_err(zvol_state_t *zv, lr_t *lr, boolean_t byteswap)
+{
+	return (ENOTSUP);
+}
+
+/*
+ * Callback vectors for replaying records.
+ * Only TX_WRITE is needed for zvol.
+ */
+zil_replay_func_t *zvol_replay_vector[TX_MAX_TYPE] = {
+	(zil_replay_func_t *)zvol_replay_err,	/* no such transaction type */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_CREATE */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_MKDIR */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_MKXATTR */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_SYMLINK */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_REMOVE */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_RMDIR */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_LINK */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_RENAME */
+	(zil_replay_func_t *)zvol_replay_write,	/* TX_WRITE */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_TRUNCATE */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_SETATTR */
+	(zil_replay_func_t *)zvol_replay_err,	/* TX_ACL */
+};
+
+/*
+ * zvol_log_write() handles synchronous writes using TX_WRITE ZIL transactions.
+ *
+ * We store data in the log buffers if it's small enough.
+ * Otherwise we will later flush the data out via dmu_sync().
+ */
+ssize_t zvol_immediate_write_sz = 32768;
+
+static void
+zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx,
+	       uint64_t offset, uint64_t size, int sync)
+{
+	uint32_t blocksize = zv->zv_volblocksize;
+	zilog_t *zilog = zv->zv_zilog;
+	boolean_t slogging;
+
+	if (zil_replaying(zilog, tx))
+		return;
+
+	slogging = spa_has_slogs(zilog->zl_spa);
+
+	while (size) {
+		itx_t *itx;
+		lr_write_t *lr;
+		ssize_t len;
+		itx_wr_state_t write_state;
+
+		/*
+		 * Unlike zfs_log_write() we can be called with
+		 * up to DMU_MAX_ACCESS/2 (5MB) writes.
+		 */
+		if (blocksize > zvol_immediate_write_sz && !slogging &&
+		    size >= blocksize && offset % blocksize == 0) {
+			write_state = WR_INDIRECT; /* uses dmu_sync */
+			len = blocksize;
+		} else if (sync) {
+			write_state = WR_COPIED;
+			len = MIN(ZIL_MAX_LOG_DATA, size);
+		} else {
+			write_state = WR_NEED_COPY;
+			len = MIN(ZIL_MAX_LOG_DATA, size);
+		}
+
+		itx = zil_itx_create(TX_WRITE, sizeof (*lr) +
+		    (write_state == WR_COPIED ? len : 0));
+		lr = (lr_write_t *)&itx->itx_lr;
+		if (write_state == WR_COPIED && dmu_read(zv->zv_objset,
+		    ZVOL_OBJ, offset, len, lr+1, DMU_READ_NO_PREFETCH) != 0) {
+			zil_itx_destroy(itx);
+			itx = zil_itx_create(TX_WRITE, sizeof (*lr));
+			lr = (lr_write_t *)&itx->itx_lr;
+			write_state = WR_NEED_COPY;
+		}
+
+		itx->itx_wr_state = write_state;
+		if (write_state == WR_NEED_COPY)
+			itx->itx_sod += len;
+		lr->lr_foid = ZVOL_OBJ;
+		lr->lr_offset = offset;
+		lr->lr_length = len;
+		lr->lr_blkoff = 0;
+		BP_ZERO(&lr->lr_blkptr);
+
+		itx->itx_private = zv;
+		itx->itx_sync = sync;
+
+		(void) zil_itx_assign(zilog, itx, tx);
+
+		offset += len;
+		size -= len;
+	}
+}
+
+/*
+ * Common write path running under the zvol taskq context.  This function
+ * is responsible for copying the request structure data in to the DMU and
+ * signaling the request queue with the result of the copy.
+ */
+static void
+zvol_write(void *arg)
+{
+	struct request *req = (struct request *)arg;
+	struct request_queue *q = req->q;
+	zvol_state_t *zv = q->queuedata;
+	uint64_t offset = blk_rq_pos(req) << 9;
+	uint64_t size = blk_rq_bytes(req);
+	int error = 0;
+	dmu_tx_t *tx;
+	rl_t *rl;
+
+	rl = zfs_range_lock(&zv->zv_znode, offset, size, RL_WRITER);
+
+	tx = dmu_tx_create(zv->zv_objset);
+	dmu_tx_hold_write(tx, ZVOL_OBJ, offset, size);
+
+	/* This will only fail for ENOSPC */
+	error = dmu_tx_assign(tx, TXG_WAIT);
+	if (error) {
+		dmu_tx_abort(tx);
+		zfs_range_unlock(rl);
+		blk_end_request(req, -error, size);
+		return;
+	}
+
+	error = dmu_write_req(zv->zv_objset, ZVOL_OBJ, req, tx);
+	if (error == 0)
+		zvol_log_write(zv, tx, offset, size, rq_is_sync(req));
+
+	dmu_tx_commit(tx);
+	zfs_range_unlock(rl);
+
+	if (rq_is_sync(req))
+		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+
+	blk_end_request(req, -error, size);
+}
+
+/*
+ * Common read path running under the zvol taskq context.  This function
+ * is responsible for copying the requested data out of the DMU and in to
+ * a linux request structure.  It then must signal the request queue with
+ * an error code describing the result of the copy.
+ */
+static void
+zvol_read(void *arg)
+{
+	struct request *req = (struct request *)arg;
+	struct request_queue *q = req->q;
+	zvol_state_t *zv = q->queuedata;
+	uint64_t offset = blk_rq_pos(req) << 9;
+	uint64_t size = blk_rq_bytes(req);
+	int error;
+	rl_t *rl;
+
+	rl = zfs_range_lock(&zv->zv_znode, offset, size, RL_READER);
+
+	error = dmu_read_req(zv->zv_objset, ZVOL_OBJ, req);
+
+	zfs_range_unlock(rl);
+
+	/* convert checksum errors into IO errors */
+	if (error == ECKSUM)
+		error = EIO;
+
+	blk_end_request(req, -error, size);
+}
+
+/*
+ * Request will be added back to the request queue and retried if
+ * it cannot be immediately dispatched to the taskq for handling
+ */
+static inline void
+zvol_dispatch(task_func_t func, struct request *req)
+{
+	if (!taskq_dispatch(zvol_taskq, func, (void *)req, TQ_NOSLEEP))
+		blk_requeue_request(req->q, req);
+}
+
+/*
+ * Common request path.  Rather than registering a custom make_request()
+ * function we use the generic Linux version.  This is done because it allows
+ * us to easily merge read requests which would otherwise we performed
+ * synchronously by the DMU.  This is less critical in write case where the
+ * DMU will perform the correct merging within a transaction group.  Using
+ * the generic make_request() also let's use leverage the fact that the
+ * elevator with ensure correct ordering in regards to barrior IOs.  On
+ * the downside it means that in the write case we end up doing request
+ * merging twice once in the elevator and once in the DMU.
+ *
+ * The request handler is called under a spin lock so all the real work
+ * is handed off to be done in the context of the zvol taskq.  This function
+ * simply performs basic request sanity checking and hands off the request.
+ */
+static void
+zvol_request(struct request_queue *q)
+{
+	zvol_state_t *zv = q->queuedata;
+	struct request *req;
+	unsigned int size;
+
+	while ((req = blk_fetch_request(q)) != NULL) {
+		size = blk_rq_bytes(req);
+
+		if (blk_rq_pos(req) + blk_rq_sectors(req) >
+		    get_capacity(zv->zv_disk)) {
+			printk(KERN_INFO
+			       "%s: bad access: block=%llu, count=%lu\n",
+			       req->rq_disk->disk_name,
+			       (long long unsigned)blk_rq_pos(req),
+			       (long unsigned)blk_rq_sectors(req));
+			__blk_end_request(req, -EIO, size);
+			continue;
+		}
+
+		if (!blk_fs_request(req)) {
+			printk(KERN_INFO "%s: non-fs cmd\n",
+			       req->rq_disk->disk_name);
+			__blk_end_request(req, -EIO, size);
+			continue;
+		}
+
+		switch (rq_data_dir(req)) {
+		case READ:
+			zvol_dispatch(zvol_read, req);
+			break;
+		case WRITE:
+			if (unlikely(get_disk_ro(zv->zv_disk)) ||
+			    unlikely(zv->zv_flags & ZVOL_RDONLY)) {
+				__blk_end_request(req, -EROFS, size);
+				break;
+			}
+
+			zvol_dispatch(zvol_write, req);
+			break;
+		default:
+			printk(KERN_INFO "%s: unknown cmd: %d\n",
+			       req->rq_disk->disk_name, (int)rq_data_dir(req));
+			__blk_end_request(req, -EIO, size);
+			break;
+		}
+	}
+}
+
+static void
+zvol_get_done(zgd_t *zgd, int error)
+{
+	if (zgd->zgd_db)
+		dmu_buf_rele(zgd->zgd_db, zgd);
+
+	zfs_range_unlock(zgd->zgd_rl);
+
+	if (error == 0 && zgd->zgd_bp)
+		zil_add_block(zgd->zgd_zilog, zgd->zgd_bp);
+
+	kmem_free(zgd, sizeof (zgd_t));
+}
+
+/*
+ * Get data to generate a TX_WRITE intent log record.
+ */
+static int
+zvol_get_data(void *arg, lr_write_t *lr, char *buf, zio_t *zio)
+{
+	zvol_state_t *zv = arg;
+	objset_t *os = zv->zv_objset;
+	uint64_t offset = lr->lr_offset;
+	uint64_t size = lr->lr_length;
+	dmu_buf_t *db;
+	zgd_t *zgd;
+	int error;
+
+	ASSERT(zio != NULL);
+	ASSERT(size != 0);
+
+	zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
+	zgd->zgd_zilog = zv->zv_zilog;
+	zgd->zgd_rl = zfs_range_lock(&zv->zv_znode, offset, size, RL_READER);
+
+	/*
+	 * Write records come in two flavors: immediate and indirect.
+	 * For small writes it's cheaper to store the data with the
+	 * log record (immediate); for large writes it's cheaper to
+	 * sync the data and get a pointer to it (indirect) so that
+	 * we don't have to write the data twice.
+	 */
+	if (buf != NULL) { /* immediate write */
+		error = dmu_read(os, ZVOL_OBJ, offset, size, buf,
+		    DMU_READ_NO_PREFETCH);
+	} else {
+		size = zv->zv_volblocksize;
+		offset = P2ALIGN_TYPED(offset, size, uint64_t);
+		error = dmu_buf_hold(os, ZVOL_OBJ, offset, zgd, &db,
+		    DMU_READ_NO_PREFETCH);
+		if (error == 0) {
+			zgd->zgd_db = db;
+			zgd->zgd_bp = &lr->lr_blkptr;
+
+			ASSERT(db != NULL);
+			ASSERT(db->db_offset == offset);
+			ASSERT(db->db_size == size);
+
+			error = dmu_sync(zio, lr->lr_common.lrc_txg,
+			    zvol_get_done, zgd);
+
+			if (error == 0)
+				return (0);
+		}
+	}
+
+	zvol_get_done(zgd, error);
+
+	return (error);
+}
+
+/*
+ * The zvol_state_t's are inserted in increasing MINOR(dev_t) order.
+ */
+static void
+zvol_insert(zvol_state_t *zv_insert)
+{
+	zvol_state_t *zv = NULL;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+	ASSERT3U(MINOR(zv_insert->zv_dev) & ZVOL_MINOR_MASK, ==, 0);
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	     zv = list_next(&zvol_state_list, zv)) {
+		if (MINOR(zv->zv_dev) > MINOR(zv_insert->zv_dev))
+			break;
+	}
+
+	list_insert_before(&zvol_state_list, zv, zv_insert);
+}
+
+/*
+ * Simply remove the zvol from to list of zvols.
+ */
+static void
+zvol_remove(zvol_state_t *zv_remove)
+{
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+	list_remove(&zvol_state_list, zv_remove);
+}
+
+static int
+zvol_first_open(zvol_state_t *zv)
+{
+	objset_t *os;
+	uint64_t volsize;
+	int error;
+	uint64_t ro;
+
+	/* lie and say we're read-only */
+	error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, 1, zvol_tag, &os);
+	if (error)
+		return (-error);
+
+	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
+	if (error) {
+	        dmu_objset_disown(os, zvol_tag);
+	        return (-error);
+	}
+
+	zv->zv_objset = os;
+	error = dmu_bonus_hold(os, ZVOL_OBJ, zvol_tag, &zv->zv_dbuf);
+	if (error) {
+	        dmu_objset_disown(os, zvol_tag);
+	        return (-error);
+	}
+
+	set_capacity(zv->zv_disk, volsize >> 9);
+	zv->zv_volsize = volsize;
+	zv->zv_zilog = zil_open(os, zvol_get_data);
+
+	VERIFY(dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL) == 0);
+	if (ro || dmu_objset_is_snapshot(os)) {
+                set_disk_ro(zv->zv_disk, 1);
+	        zv->zv_flags |= ZVOL_RDONLY;
+	} else {
+                set_disk_ro(zv->zv_disk, 0);
+	        zv->zv_flags &= ~ZVOL_RDONLY;
+	}
+
+	return (-error);
+}
+
+static void
+zvol_last_close(zvol_state_t *zv)
+{
+	zil_close(zv->zv_zilog);
+	zv->zv_zilog = NULL;
+	dmu_buf_rele(zv->zv_dbuf, zvol_tag);
+	zv->zv_dbuf = NULL;
+	dmu_objset_disown(zv->zv_objset, zvol_tag);
+	zv->zv_objset = NULL;
+}
+
+static int
+zvol_open(struct block_device *bdev, fmode_t flag)
+{
+	zvol_state_t *zv = bdev->bd_disk->private_data;
+	int error = 0, drop_mutex = 0;
+
+	/*
+	 * If the caller is already holding the mutex do not take it
+	 * again, this will happen as part of zvol_create_minor().
+	 * Once add_disk() is called the device is live and the kernel
+	 * will attempt to open it to read the partition information.
+	 */
+	if (!mutex_owned(&zvol_state_lock)) {
+		mutex_enter(&zvol_state_lock);
+		drop_mutex = 1;
+	}
+
+	ASSERT3P(zv, !=, NULL);
+
+	if (zv->zv_open_count == 0) {
+		error = zvol_first_open(zv);
+		if (error)
+			goto out_mutex;
+	}
+
+	if ((flag & FMODE_WRITE) &&
+	    (get_disk_ro(zv->zv_disk) || (zv->zv_flags & ZVOL_RDONLY))) {
+		error = -EROFS;
+		goto out_open_count;
+	}
+
+	zv->zv_open_count++;
+
+out_open_count:
+	if (zv->zv_open_count == 0)
+		zvol_last_close(zv);
+
+out_mutex:
+	if (drop_mutex)
+		mutex_exit(&zvol_state_lock);
+
+	check_disk_change(bdev);
+
+	return (error);
+}
+
+static int
+zvol_release(struct gendisk *disk, fmode_t mode)
+{
+	zvol_state_t *zv = disk->private_data;
+	int drop_mutex = 0;
+
+	if (!mutex_owned(&zvol_state_lock)) {
+		mutex_enter(&zvol_state_lock);
+		drop_mutex = 1;
+	}
+
+	ASSERT3P(zv, !=, NULL);
+	ASSERT3U(zv->zv_open_count, >, 0);
+	zv->zv_open_count--;
+	if (zv->zv_open_count == 0)
+		zvol_last_close(zv);
+
+	if (drop_mutex)
+		mutex_exit(&zvol_state_lock);
+
+	return (0);
+}
+
+static int
+zvol_ioctl(struct block_device *bdev, fmode_t mode,
+           unsigned int cmd, unsigned long arg)
+{
+	zvol_state_t *zv = bdev->bd_disk->private_data;
+	int error = 0;
+
+	if (zv == NULL)
+		return (-ENXIO);
+
+	switch (cmd) {
+	case BLKFLSBUF:
+		zil_commit(zv->zv_zilog, ZVOL_OBJ);
+		break;
+
+	default:
+		error = -ENOTTY;
+		break;
+
+	}
+
+	return (error);
+}
+
+#ifdef CONFIG_COMPAT
+static int
+zvol_compat_ioctl(struct block_device *bdev, fmode_t mode,
+                  unsigned cmd, unsigned long arg)
+{
+	return zvol_ioctl(bdev, mode, cmd, arg);
+}
+#else
+#define zvol_compat_ioctl   NULL
+#endif
+
+static int zvol_media_changed(struct gendisk *disk)
+{
+	zvol_state_t *zv = disk->private_data;
+
+	return zv->zv_changed;
+}
+
+static int zvol_revalidate_disk(struct gendisk *disk)
+{
+	zvol_state_t *zv = disk->private_data;
+
+	zv->zv_changed = 0;
+	set_capacity(zv->zv_disk, zv->zv_volsize >> 9);
+
+	return 0;
+}
+
+/*
+ * Provide a simple virtual geometry for legacy compatibility.  For devices
+ * smaller than 1 MiB a small head and sector count is used to allow very
+ * tiny devices.  For devices over 1 Mib a standard head and sector count
+ * is used to keep the cylinders count reasonable.
+ */
+static int
+zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
+{
+	zvol_state_t *zv = bdev->bd_disk->private_data;
+	sector_t sectors = get_capacity(zv->zv_disk);
+
+	if (sectors > 2048) {
+		geo->heads = 16;
+		geo->sectors = 63;
+	} else {
+		geo->heads = 2;
+		geo->sectors = 4;
+	}
+
+	geo->start = 0;
+	geo->cylinders = sectors / (geo->heads * geo->sectors);
+
+	return 0;
+}
+
+static struct kobject *
+zvol_probe(dev_t dev, int *part, void *arg)
+{
+	zvol_state_t *zv;
+	struct kobject *kobj;
+
+	mutex_enter(&zvol_state_lock);
+	zv = zvol_find_by_dev(dev);
+	kobj = zv ? get_disk(zv->zv_disk) : ERR_PTR(-ENOENT);
+	mutex_exit(&zvol_state_lock);
+
+	return kobj;
+}
+
+#ifdef HAVE_BDEV_BLOCK_DEVICE_OPERATIONS
+static struct block_device_operations zvol_ops = {
+	.open            = zvol_open,
+	.release         = zvol_release,
+	.ioctl           = zvol_ioctl,
+	.compat_ioctl    = zvol_compat_ioctl,
+	.media_changed   = zvol_media_changed,
+	.revalidate_disk = zvol_revalidate_disk,
+	.getgeo          = zvol_getgeo,
+        .owner           = THIS_MODULE,
+};
+
+#else /* HAVE_BDEV_BLOCK_DEVICE_OPERATIONS */
+
+static int
+zvol_open_by_inode(struct inode *inode, struct file *file)
+{
+	return zvol_open(inode->i_bdev, file->f_mode);
+}
+
+static int
+zvol_release_by_inode(struct inode *inode, struct file *file)
+{
+	return zvol_release(inode->i_bdev->bd_disk, file->f_mode);
+}
+
+static int
+zvol_ioctl_by_inode(struct inode *inode, struct file *file,
+                    unsigned int cmd, unsigned long arg)
+{
+	return zvol_ioctl(inode->i_bdev, file->f_mode, cmd, arg);
+}
+
+# ifdef CONFIG_COMPAT
+static long
+zvol_compat_ioctl_by_inode(struct file *file,
+                           unsigned int cmd, unsigned long arg)
+{
+	return zvol_compat_ioctl(file->f_dentry->d_inode->i_bdev,
+	                         file->f_mode, cmd, arg);
+}
+# else
+# define zvol_compat_ioctl_by_inode   NULL
+# endif
+
+static struct block_device_operations zvol_ops = {
+	.open            = zvol_open_by_inode,
+	.release         = zvol_release_by_inode,
+	.ioctl           = zvol_ioctl_by_inode,
+	.compat_ioctl    = zvol_compat_ioctl_by_inode,
+	.media_changed   = zvol_media_changed,
+	.revalidate_disk = zvol_revalidate_disk,
+	.getgeo          = zvol_getgeo,
+        .owner           = THIS_MODULE,
+};
+#endif /* HAVE_BDEV_BLOCK_DEVICE_OPERATIONS */
+
+/*
+ * Allocate memory for a new zvol_state_t and setup the required
+ * request queue and generic disk structures for the block device.
+ */
+static zvol_state_t *
+zvol_alloc(dev_t dev, const char *name)
+{
+	zvol_state_t *zv;
+
+	zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
+	if (zv == NULL)
+		goto out;
+
+	zv->zv_queue = blk_init_queue(zvol_request, &zv->zv_lock);
+	if (zv->zv_queue == NULL)
+		goto out_kmem;
+
+	zv->zv_disk = alloc_disk(ZVOL_MINORS);
+	if (zv->zv_disk == NULL)
+		goto out_queue;
+
+	zv->zv_queue->queuedata = zv;
+	zv->zv_dev = dev;
+	zv->zv_open_count = 0;
+	strlcpy(zv->zv_name, name, DISK_NAME_LEN);
+
+	mutex_init(&zv->zv_znode.z_range_lock, NULL, MUTEX_DEFAULT, NULL);
+	avl_create(&zv->zv_znode.z_range_avl, zfs_range_compare,
+	    sizeof (rl_t), offsetof(rl_t, r_node));
+	spin_lock_init(&zv->zv_lock);
+	list_link_init(&zv->zv_next);
+
+	zv->zv_disk->major = zvol_major;
+	zv->zv_disk->first_minor = (dev & MINORMASK);
+	zv->zv_disk->fops = &zvol_ops;
+	zv->zv_disk->private_data = zv;
+	zv->zv_disk->queue = zv->zv_queue;
+	snprintf(zv->zv_disk->disk_name, DISK_NAME_LEN, "%s", name);
+
+	return zv;
+
+out_queue:
+	blk_cleanup_queue(zv->zv_queue);
+out_kmem:
+	kmem_free(zv, sizeof (zvol_state_t));
+out:
+	return NULL;
+}
+
+/*
+ * Cleanup then free a zvol_state_t which was created by zvol_alloc().
+ */
+static void
+zvol_free(zvol_state_t *zv)
+{
+	avl_destroy(&zv->zv_znode.z_range_avl);
+	mutex_destroy(&zv->zv_znode.z_range_lock);
+
+	del_gendisk(zv->zv_disk);
+	blk_cleanup_queue(zv->zv_queue);
+	put_disk(zv->zv_disk);
+
+	kmem_free(zv, sizeof (zvol_state_t));
+}
+
+static int
+__zvol_create_minor(const char *name)
+{
+	zvol_state_t *zv;
+	objset_t *os;
+	dmu_object_info_t *doi;
+	uint64_t volsize;
+	unsigned minor = 0;
+	int error = 0;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+
+	zv = zvol_find_by_name(name);
+	if (zv) {
+		error = EEXIST;
+		goto out;
+	}
+
+	doi = kmem_alloc(sizeof(dmu_object_info_t), KM_SLEEP);
+
+	error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, zvol_tag, &os);
+	if (error)
+		goto out_doi;
+
+	error = dmu_object_info(os, ZVOL_OBJ, doi);
+	if (error)
+		goto out_dmu_objset_disown;
+
+	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
+	if (error)
+		goto out_dmu_objset_disown;
+
+	error = zvol_find_minor(&minor);
+	if (error)
+		goto out_dmu_objset_disown;
+
+	zv = zvol_alloc(MKDEV(zvol_major, minor), name);
+	if (zv == NULL) {
+		error = EAGAIN;
+		goto out_dmu_objset_disown;
+	}
+
+	if (dmu_objset_is_snapshot(os))
+		zv->zv_flags |= ZVOL_RDONLY;
+
+	zv->zv_volblocksize = doi->doi_data_block_size;
+	zv->zv_volsize = volsize;
+	zv->zv_objset = os;
+
+	set_capacity(zv->zv_disk, zv->zv_volsize >> 9);
+
+	if (zil_replay_disable)
+		zil_destroy(dmu_objset_zil(os), B_FALSE);
+	else
+		zil_replay(os, zv, zvol_replay_vector);
+
+out_dmu_objset_disown:
+	dmu_objset_disown(os, zvol_tag);
+	zv->zv_objset = NULL;
+out_doi:
+	kmem_free(doi, sizeof(dmu_object_info_t));
+out:
+
+	if (error == 0) {
+		zvol_insert(zv);
+		add_disk(zv->zv_disk);
+	}
+
+	return (error);
+}
+
+/*
+ * Create a block device minor node and setup the linkage between it
+ * and the specified volume.  Once this function returns the block
+ * device is live and ready for use.
+ */
+int
+zvol_create_minor(const char *name)
+{
+	int error;
+
+	mutex_enter(&zvol_state_lock);
+	error = __zvol_create_minor(name);
+	mutex_exit(&zvol_state_lock);
+
+	return (error);
+}
+
+static int
+__zvol_remove_minor(const char *name)
+{
+	zvol_state_t *zv;
+
+	ASSERT(MUTEX_HELD(&zvol_state_lock));
+
+	zv = zvol_find_by_name(name);
+	if (zv == NULL)
+		return (ENXIO);
+
+	if (zv->zv_open_count > 0)
+		return (EBUSY);
+
+	zvol_remove(zv);
+	zvol_free(zv);
+
+	return (0);
+}
+
+/*
+ * Remove a block device minor node for the specified volume.
+ */
+int
+zvol_remove_minor(const char *name)
+{
+	int error;
+
+	mutex_enter(&zvol_state_lock);
+	error = __zvol_remove_minor(name);
+	mutex_exit(&zvol_state_lock);
+
+	return (error);
+}
+
+static int
+zvol_create_minors_cb(spa_t *spa, uint64_t dsobj,
+		      const char *dsname, void *arg)
+{
+	if (strchr(dsname, '/') == NULL)
+		return 0;
+
+	return __zvol_create_minor(dsname);
+}
+
+/*
+ * Create minors for specified pool, if pool is NULL create minors
+ * for all available pools.
+ */
+int
+zvol_create_minors(const char *pool)
+{
+	spa_t *spa = NULL;
+	int error = 0;
+
+	mutex_enter(&zvol_state_lock);
+	if (pool) {
+		error = dmu_objset_find_spa(NULL, pool, zvol_create_minors_cb,
+		    NULL, DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
+	} else {
+		mutex_enter(&spa_namespace_lock);
+		while ((spa = spa_next(spa)) != NULL) {
+			error = dmu_objset_find_spa(NULL,
+			    spa_name(spa), zvol_create_minors_cb, NULL,
+			    DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
+			if (error)
+				break;
+		}
+		mutex_exit(&spa_namespace_lock);
+	}
+	mutex_exit(&zvol_state_lock);
+
+	return error;
+}
+
+/*
+ * Remove minors for specified pool, if pool is NULL remove all minors.
+ */
+void
+zvol_remove_minors(const char *pool)
+{
+	zvol_state_t *zv, *zv_next;
+	char *str;
+
+	str = kmem_zalloc(DISK_NAME_LEN, KM_SLEEP);
+	if (pool) {
+		(void) strncpy(str, pool, strlen(pool));
+		(void) strcat(str, "/");
+	}
+
+	mutex_enter(&zvol_state_lock);
+	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
+		zv_next = list_next(&zvol_state_list, zv);
+
+		if (pool == NULL || !strncmp(str, zv->zv_name, strlen(str))) {
+			zvol_remove(zv);
+			zvol_free(zv);
+		}
+	}
+	mutex_exit(&zvol_state_lock);
+	kmem_free(str, DISK_NAME_LEN);
+}
+
+int
+zvol_init(void)
+{
+	int error;
+
+	if (!zvol_threads)
+		zvol_threads = num_online_cpus();
+
+	zvol_taskq = taskq_create(ZVOL_DRIVER, zvol_threads, maxclsyspri,
+		                  zvol_threads, INT_MAX, TASKQ_PREPOPULATE);
+	if (zvol_taskq == NULL) {
+		printk(KERN_INFO "ZFS: taskq_create() failed\n");
+		return (-ENOMEM);
+	}
+
+	error = register_blkdev(zvol_major, ZVOL_DRIVER);
+	if (error) {
+		printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
+		taskq_destroy(zvol_taskq);
+		return (error);
+	}
+
+	blk_register_region(MKDEV(zvol_major, 0), 1UL << MINORBITS,
+	                    THIS_MODULE, zvol_probe, NULL, NULL);
+
+	mutex_init(&zvol_state_lock, NULL, MUTEX_DEFAULT, NULL);
+	list_create(&zvol_state_list, sizeof (zvol_state_t),
+	            offsetof(zvol_state_t, zv_next));
+
+	(void) zvol_create_minors(NULL);
+
+	return (0);
+}
+
+void
+zvol_fini(void)
+{
+	zvol_remove_minors(NULL);
+	blk_unregister_region(MKDEV(zvol_major, 0), 1UL << MINORBITS);
+	unregister_blkdev(zvol_major, ZVOL_DRIVER);
+	taskq_destroy(zvol_taskq);
+	mutex_destroy(&zvol_state_lock);
+	list_destroy(&zvol_state_list);
+}
+
+module_param(zvol_major, uint, 0);
+MODULE_PARM_DESC(zvol_major, "Major number for zvol device");
+
+module_param(zvol_threads, uint, 0);
+MODULE_PARM_DESC(zvol_threads, "Number of threads for zvol device");