zfetch: Don't issue new streams when old have not completed

The current dmu_zfetch code implicitly assumes that I/Os complete
within min_sec_reap seconds. With async dmu and a readonly workload
(and thus no exponential backoff in operations from the "write
throttle") such as L2ARC rebuild it is possible to saturate the drives
with I/O requests. These are then effectively compounded with prefetch
requests.

This change reference counts streams and prevents them from being
recycled after their min_sec_reap timeout if they still have
outstanding I/Os.

Reviewed-by: Alexander Motin <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Matt Macy <[email protected]>
Closes #10900 

1 files changed, 10 insertions, 6 deletions

diff --git a/include/sys/dmu_zfetch.h b/include/sys/dmu_zfetch.h
index 4303ab314..34b711fc0 100644
--- a/include/sys/dmu_zfetch.h
+++ b/include/sys/dmu_zfetch.h
@@ -40,6 +40,13 @@ extern unsigned long	zfetch_array_rd_sz;
 
 struct dnode;				/* so we can reference dnode */
 
+typedef struct zfetch {
+	kmutex_t	zf_lock;	/* protects zfetch structure */
+	list_t		zf_stream;	/* list of zstream_t's */
+	struct dnode	*zf_dnode;	/* dnode that owns this zfetch */
+	int		zf_numstreams;	/* number of zstream_t's */
+} zfetch_t;
+
 typedef struct zstream {
 	uint64_t	zs_blkid;	/* expect next access at this blkid */
 	uint64_t	zs_pf_blkid;	/* next block to prefetch */
@@ -52,15 +59,12 @@ typedef struct zstream {
 
 	kmutex_t	zs_lock;	/* protects stream */
 	hrtime_t	zs_atime;	/* time last prefetch issued */
+	hrtime_t	zs_start_time;	/* start of last prefetch */
 	list_node_t	zs_node;	/* link for zf_stream */
+	zfetch_t	*zs_fetch;	/* parent fetch */
+	zfs_refcount_t	zs_blocks; /* number of pending blocks in the stream */
 } zstream_t;
 
-typedef struct zfetch {
-	kmutex_t	zf_lock;	/* protects zfetch structure */
-	list_t		zf_stream;	/* list of zstream_t's */
-	struct dnode	*zf_dnode;	/* dnode that owns this zfetch */
-} zfetch_t;
-
 void		zfetch_init(void);
 void		zfetch_fini(void);