diff options
author | Isaac Huang <he.huang@intel.com> | 2016-08-31 00:26:43 -0600 |
---|---|---|
committer | Brian Behlendorf <behlendorf1@llnl.gov> | 2016-11-29 14:34:32 -0800 |
commit | b0be93e81a4ace96a799d3705da6832633eb4325 (patch) | |
tree | 7f8782cb7c0a62a3e079b03f96a5fa157e461bcb /module | |
parent | a6255b7fce400d485a0e87cbe369aa0ed7dc5dc4 (diff) |
ABD page support to vdev_disk.c
Signed-off-by: Isaac Huang <he.huang@intel.com>
Diffstat (limited to 'module')
-rw-r--r-- | module/zfs/abd.c | 60 | ||||
-rw-r--r-- | module/zfs/vdev_disk.c | 79 |
2 files changed, 83 insertions, 56 deletions
diff --git a/module/zfs/abd.c b/module/zfs/abd.c index 9fa4a5d43..306c47536 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -999,8 +999,66 @@ abd_cmp(abd_t *dabd, abd_t *sabd) abd_cmp_cb, NULL)); } - #if defined(_KERNEL) && defined(HAVE_SPL) +/* + * bio_nr_pages for ABD. + * @off is the offset in @abd + */ +unsigned long +abd_nr_pages_off(abd_t *abd, unsigned int size, size_t off) +{ + unsigned long pos; + + if (abd_is_linear(abd)) + pos = (unsigned long)abd_to_buf(abd) + off; + else + pos = abd->abd_u.abd_scatter.abd_offset + off; + + return ((pos + size + PAGESIZE - 1) >> PAGE_SHIFT) + - (pos >> PAGE_SHIFT); +} + +/* + * bio_map for scatter ABD. + * @off is the offset in @abd + * Remaining IO size is returned + */ +unsigned int +abd_scatter_bio_map_off(struct bio *bio, abd_t *abd, + unsigned int io_size, size_t off) +{ + int i; + struct abd_iter aiter; + + ASSERT(!abd_is_linear(abd)); + ASSERT3U(io_size, <=, abd->abd_size - off); + + abd_iter_init(&aiter, abd); + abd_iter_advance(&aiter, off); + + for (i = 0; i < bio->bi_max_vecs; i++) { + struct page *pg; + size_t len, pgoff, index; + + if (io_size <= 0) + break; + + pgoff = abd_iter_scatter_chunk_offset(&aiter); + len = MIN(io_size, PAGESIZE - pgoff); + ASSERT(len > 0); + + index = abd_iter_scatter_chunk_index(&aiter); + pg = abd->abd_u.abd_scatter.abd_chunks[index]; + if (bio_add_page(bio, pg, len, pgoff) != len) + break; + + io_size -= len; + abd_iter_advance(&aiter, len); + } + + return (io_size); +} + /* Tunable Parameters */ module_param(zfs_abd_scatter_enabled, int, 0644); MODULE_PARM_DESC(zfs_abd_scatter_enabled, diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c index 67759d021..ae6ed4de9 100644 --- a/module/zfs/vdev_disk.c +++ b/module/zfs/vdev_disk.c @@ -43,7 +43,6 @@ static void *zfs_vdev_holder = VDEV_HOLDER; */ typedef struct dio_request { zio_t *dr_zio; /* Parent ZIO */ - void *dr_loanbuf; /* borrowed abd buffer */ atomic_t dr_ref; /* References */ int dr_error; /* Bio error */ int dr_bio_count; /* Count of bio's */ @@ -404,7 +403,6 @@ vdev_disk_dio_put(dio_request_t *dr) */ if (rc == 0) { zio_t *zio = dr->dr_zio; - void *loanbuf = dr->dr_loanbuf; int error = dr->dr_error; vdev_disk_dio_free(dr); @@ -414,14 +412,6 @@ vdev_disk_dio_put(dio_request_t *dr) ASSERT3S(zio->io_error, >=, 0); if (zio->io_error) vdev_disk_error(zio); - /* ABD placeholder */ - if (loanbuf != NULL) { - if (zio->io_type == ZIO_TYPE_READ) { - abd_copy_from_buf(zio->io_abd, loanbuf, - zio->io_size); - } - zio_buf_free(loanbuf, zio->io_size); - } zio_delay_interrupt(zio); } @@ -446,17 +436,10 @@ BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, error) #endif } - /* Drop reference aquired by __vdev_disk_physio */ + /* Drop reference acquired by __vdev_disk_physio */ rc = vdev_disk_dio_put(dr); } -static inline unsigned long -bio_nr_pages(void *bio_ptr, unsigned int bio_size) -{ - return ((((unsigned long)bio_ptr + bio_size + PAGE_SIZE - 1) >> - PAGE_SHIFT) - ((unsigned long)bio_ptr >> PAGE_SHIFT)); -} - static unsigned int bio_map(struct bio *bio, void *bio_ptr, unsigned int bio_size) { @@ -496,6 +479,15 @@ bio_map(struct bio *bio, void *bio_ptr, unsigned int bio_size) return (bio_size); } +static unsigned int +bio_map_abd_off(struct bio *bio, abd_t *abd, unsigned int size, size_t off) +{ + if (abd_is_linear(abd)) + return (bio_map(bio, ((char *)abd_to_buf(abd)) + off, size)); + + return (abd_scatter_bio_map_off(bio, abd, size, off)); +} + #ifndef bio_set_op_attrs #define bio_set_op_attrs(bio, rw, flags) \ do { (bio)->bi_rw |= (rw)|(flags); } while (0) @@ -528,11 +520,11 @@ vdev_submit_bio(struct bio *bio) } static int -__vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, - size_t kbuf_size, uint64_t kbuf_offset, int rw, int flags) +__vdev_disk_physio(struct block_device *bdev, zio_t *zio, + size_t io_size, uint64_t io_offset, int rw, int flags) { dio_request_t *dr; - caddr_t bio_ptr; + uint64_t abd_offset; uint64_t bio_offset; int bio_size, bio_count = 16; int i = 0, error = 0; @@ -540,7 +532,8 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, struct blk_plug plug; #endif - ASSERT3U(kbuf_offset + kbuf_size, <=, bdev->bd_inode->i_size); + ASSERT(zio != NULL); + ASSERT3U(io_offset + io_size, <=, bdev->bd_inode->i_size); retry: dr = vdev_disk_dio_alloc(bio_count); @@ -559,32 +552,10 @@ retry: * their volume block size to match the maximum request size and * the common case will be one bio per vdev IO request. */ - if (zio != NULL) { - abd_t *abd = zio->io_abd; - - /* - * ABD placeholder - * We can't use abd_borrow_buf routines here since our - * completion context is interrupt and abd refcounts - * take a mutex (in debug mode). - */ - if (abd_is_linear(abd)) { - bio_ptr = abd_to_buf(abd); - dr->dr_loanbuf = NULL; - } else { - bio_ptr = zio_buf_alloc(zio->io_size); - dr->dr_loanbuf = bio_ptr; - if (zio->io_type != ZIO_TYPE_READ) - abd_copy_to_buf(bio_ptr, abd, zio->io_size); - - } - } else { - bio_ptr = kbuf_ptr; - dr->dr_loanbuf = NULL; - } - bio_offset = kbuf_offset; - bio_size = kbuf_size; + abd_offset = 0; + bio_offset = io_offset; + bio_size = io_size; for (i = 0; i <= dr->dr_bio_count; i++) { /* Finished constructing bio's for given buffer */ @@ -597,8 +568,6 @@ retry: * are needed we allocate a larger dio and warn the user. */ if (dr->dr_bio_count == i) { - if (dr->dr_loanbuf) - zio_buf_free(dr->dr_loanbuf, zio->io_size); vdev_disk_dio_free(dr); bio_count *= 2; goto retry; @@ -606,10 +575,9 @@ retry: /* bio_alloc() with __GFP_WAIT never returns NULL */ dr->dr_bio[i] = bio_alloc(GFP_NOIO, - MIN(bio_nr_pages(bio_ptr, bio_size), BIO_MAX_PAGES)); + MIN(abd_nr_pages_off(zio->io_abd, bio_size, abd_offset), + BIO_MAX_PAGES)); if (unlikely(dr->dr_bio[i] == NULL)) { - if (dr->dr_loanbuf) - zio_buf_free(dr->dr_loanbuf, zio->io_size); vdev_disk_dio_free(dr); return (ENOMEM); } @@ -624,10 +592,11 @@ retry: bio_set_op_attrs(dr->dr_bio[i], rw, flags); /* Remaining size is returned to become the new size */ - bio_size = bio_map(dr->dr_bio[i], bio_ptr, bio_size); + bio_size = bio_map_abd_off(dr->dr_bio[i], zio->io_abd, + bio_size, abd_offset); /* Advance in buffer and construct another bio if needed */ - bio_ptr += BIO_BI_SIZE(dr->dr_bio[i]); + abd_offset += BIO_BI_SIZE(dr->dr_bio[i]); bio_offset += BIO_BI_SIZE(dr->dr_bio[i]); } @@ -769,7 +738,7 @@ vdev_disk_io_start(zio_t *zio) } zio->io_target_timestamp = zio_handle_io_delay(zio); - error = __vdev_disk_physio(vd->vd_bdev, zio, NULL, + error = __vdev_disk_physio(vd->vd_bdev, zio, zio->io_size, zio->io_offset, rw, flags); if (error) { zio->io_error = error; |