ABD kmap to kmap_atomic

Convert usage of kmap to kmap_atomic while correctly saving off
irq state.

1 files changed, 49 insertions, 35 deletions

diff --git a/module/zfs/abd.c b/module/zfs/abd.c
index 08bb0c52e..6436d3b1f 100644
--- a/module/zfs/abd.c
+++ b/module/zfs/abd.c
@@ -119,6 +119,9 @@
 #include <sys/zio.h>
 #include <sys/zfs_context.h>
 #include <sys/zfs_znode.h>
+#ifdef _KERNEL
+#include <linux/kmap_compat.h>
+#endif
 
 #ifndef KMC_NOTOUCH
 #define	KMC_NOTOUCH	0
@@ -187,22 +190,6 @@ abd_free_chunk(struct page *c)
 	__free_pages(c, 0);
 }
 
-static void *
-abd_map_chunk(struct page *c)
-{
-	/*
-	 * Use of segkpm means we don't care if this is mapped S_READ or S_WRITE
-	 * but S_WRITE is conceptually more accurate.
-	 */
-	return (kmap(c));
-}
-
-static void
-abd_unmap_chunk(struct page *c)
-{
-	kunmap(c);
-}
-
 void
 abd_init(void)
 {
@@ -230,11 +217,10 @@ struct page;
 #define	abd_alloc_chunk() \
 	((struct page *) umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP))
 #define	abd_free_chunk(chunk) 		umem_free(chunk, PAGESIZE)
-#define	abd_map_chunk(chunk)		((void *)chunk)
-static void
-abd_unmap_chunk(struct page *c)
-{
-}
+#define	zfs_kmap_atomic(chunk, km)	((void *)chunk)
+#define	zfs_kunmap_atomic(addr, km)	do { (void)(addr); } while (0)
+#define	local_irq_save(flags)		do { (void)(flags); } while (0)
+#define	local_irq_restore(flags)	do { (void)(flags); } while (0)
 
 void
 abd_init(void)
@@ -697,11 +683,28 @@ abd_release_ownership_of_buf(abd_t *abd)
 	ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
 }
 
+#ifndef HAVE_1ARG_KMAP_ATOMIC
+#define	NR_KM_TYPE (6)
+#ifdef _KERNEL
+int km_table[NR_KM_TYPE] = {
+	KM_USER0,
+	KM_USER1,
+	KM_BIO_SRC_IRQ,
+	KM_BIO_DST_IRQ,
+	KM_PTE0,
+	KM_PTE1,
+};
+#endif
+#endif
+
 struct abd_iter {
 	abd_t		*iter_abd;	/* ABD being iterated through */
 	size_t		iter_pos;	/* position (relative to abd_offset) */
 	void		*iter_mapaddr;	/* addr corresponding to iter_pos */
 	size_t		iter_mapsize;	/* length of data valid at mapaddr */
+#ifndef HAVE_1ARG_KMAP_ATOMIC
+	int		iter_km;	/* KM_* for kmap_atomic */
+#endif
 };
 
 static inline size_t
@@ -724,13 +727,17 @@ abd_iter_scatter_chunk_index(struct abd_iter *aiter)
  * Initialize the abd_iter.
  */
 static void
-abd_iter_init(struct abd_iter *aiter, abd_t *abd)
+abd_iter_init(struct abd_iter *aiter, abd_t *abd, int km_type)
 {
 	abd_verify(abd);
 	aiter->iter_abd = abd;
 	aiter->iter_pos = 0;
 	aiter->iter_mapaddr = NULL;
 	aiter->iter_mapsize = 0;
+#ifndef HAVE_1ARG_KMAP_ATOMIC
+	ASSERT3U(km_type, <, NR_KM_TYPE);
+	aiter->iter_km = km_type;
+#endif
 }
 
 /*
@@ -779,8 +786,9 @@ abd_iter_map(struct abd_iter *aiter)
 		aiter->iter_mapsize = MIN(PAGESIZE - offset,
 		    aiter->iter_abd->abd_size - aiter->iter_pos);
 
-		paddr = abd_map_chunk(
-			aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]);
+		paddr = zfs_kmap_atomic(
+			aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index],
+			km_table[aiter->iter_km]);
 	}
 
 	aiter->iter_mapaddr = (char *)paddr + offset;
@@ -799,9 +807,9 @@ abd_iter_unmap(struct abd_iter *aiter)
 
 	if (!abd_is_linear(aiter->iter_abd)) {
 		/* LINTED E_FUNC_SET_NOT_USED */
-		size_t index = abd_iter_scatter_chunk_index(aiter);
-		abd_unmap_chunk(
-		    aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]);
+		zfs_kunmap_atomic(aiter->iter_mapaddr -
+		    abd_iter_scatter_chunk_offset(aiter),
+		    km_table[aiter->iter_km]);
 	}
 
 	ASSERT3P(aiter->iter_mapaddr, !=, NULL);
@@ -821,7 +829,7 @@ abd_iterate_func(abd_t *abd, size_t off, size_t size,
 	abd_verify(abd);
 	ASSERT3U(off + size, <=, abd->abd_size);
 
-	abd_iter_init(&aiter, abd);
+	abd_iter_init(&aiter, abd, 0);
 	abd_iter_advance(&aiter, off);
 
 	while (size > 0) {
@@ -953,8 +961,8 @@ abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff,
 	ASSERT3U(doff + size, <=, dabd->abd_size);
 	ASSERT3U(soff + size, <=, sabd->abd_size);
 
-	abd_iter_init(&daiter, dabd);
-	abd_iter_init(&saiter, sabd);
+	abd_iter_init(&daiter, dabd, 0);
+	abd_iter_init(&saiter, sabd, 1);
 	abd_iter_advance(&daiter, doff);
 	abd_iter_advance(&saiter, soff);
 
@@ -1039,17 +1047,19 @@ abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 	struct abd_iter caiters[3];
 	struct abd_iter daiter;
 	void *caddrs[3];
+	unsigned long flags;
 
 	ASSERT3U(parity, <=, 3);
 
 	for (i = 0; i < parity; i++)
-		abd_iter_init(&caiters[i], cabds[i]);
+		abd_iter_init(&caiters[i], cabds[i], i);
 
 	if (dabd)
-		abd_iter_init(&daiter, dabd);
+		abd_iter_init(&daiter, dabd, i);
 
 	ASSERT3S(dsize, >=, 0);
 
+	local_irq_save(flags);
 	while (csize > 0) {
 		len = csize;
 
@@ -1106,6 +1116,7 @@ abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 		ASSERT3S(dsize, >=, 0);
 		ASSERT3S(csize, >=, 0);
 	}
+	local_irq_restore(flags);
 }
 
 /*
@@ -1130,14 +1141,16 @@ abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
 	struct abd_iter citers[3];
 	struct abd_iter xiters[3];
 	void *caddrs[3], *xaddrs[3];
+	unsigned long flags;
 
 	ASSERT3U(parity, <=, 3);
 
 	for (i = 0; i < parity; i++) {
-		abd_iter_init(&citers[i], cabds[i]);
-		abd_iter_init(&xiters[i], tabds[i]);
+		abd_iter_init(&citers[i], cabds[i], 2*i);
+		abd_iter_init(&xiters[i], tabds[i], 2*i+1);
 	}
 
+	local_irq_save(flags);
 	while (tsize > 0) {
 
 		for (i = 0; i < parity; i++) {
@@ -1179,6 +1192,7 @@ abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
 		tsize -= len;
 		ASSERT3S(tsize, >=, 0);
 	}
+	local_irq_restore(flags);
 }
 
 #if defined(_KERNEL) && defined(HAVE_SPL)
@@ -1215,7 +1229,7 @@ abd_scatter_bio_map_off(struct bio *bio, abd_t *abd,
 	ASSERT(!abd_is_linear(abd));
 	ASSERT3U(io_size, <=, abd->abd_size - off);
 
-	abd_iter_init(&aiter, abd);
+	abd_iter_init(&aiter, abd, 0);
 	abd_iter_advance(&aiter, off);
 
 	for (i = 0; i < bio->bi_max_vecs; i++) {