Persistent L2ARC

This commit makes the L2ARC persistent across reboots. We implement
a light-weight persistent L2ARC metadata structure that allows L2ARC
contents to be recovered after a reboot. This significantly eases the
impact a reboot has on read performance on systems with large caches.

Reviewed-by: Matthew Ahrens <[email protected]>
Reviewed-by: George Wilson <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Co-authored-by: Saso Kiselkov <[email protected]>
Co-authored-by: Jorgen Lundman <[email protected]>
Co-authored-by: George Amanakis <[email protected]>
Ported-by: Yuxuan Shui <[email protected]>
Signed-off-by: George Amanakis <[email protected]>
Closes #925 
Closes #1823 
Closes #2672 
Closes #3744 
Closes #9582 

15 files changed, 971 insertions, 0 deletions

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 897a6a955..61df6d420 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -164,3 +164,9 @@ tags = ['functional', 'user_namespace']
 tests = ['groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos',
     'userquota_013_pos', 'userspace_003_pos']
 tags = ['functional', 'userquota']
+
+[tests/functional/persist_l2arc:Linux]
+tests = ['persist_l2arc_001_pos', 'persist_l2arc_002_pos',
+    'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos',
+    'persist_l2arc_006_pos', 'persist_l2arc_007_pos', 'persist_l2arc_008_pos']
+tags = ['functional', 'persist_l2arc']
diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg
index 62d335abe..680fcf42c 100644
--- a/tests/zfs-tests/include/tunables.cfg
+++ b/tests/zfs-tests/include/tunables.cfg
@@ -36,6 +36,8 @@ INITIALIZE_CHUNK_SIZE		initialize_chunk_size		zfs_initialize_chunk_size
 INITIALIZE_VALUE		initialize_value		zfs_initialize_value
 KEEP_LOG_SPACEMAPS_AT_EXPORT	keep_log_spacemaps_at_export	zfs_keep_log_spacemaps_at_export
 L2ARC_NOPREFETCH		l2arc.noprefetch		l2arc_noprefetch
+L2ARC_REBUILD_BLOCKS_MIN_L2SIZE	UNSUPPORTED			l2arc_rebuild_blocks_min_l2size
+L2ARC_REBUILD_ENABLED		UNSUPPORTED			l2arc_rebuild_enabled
 L2ARC_WRITE_BOOST		l2arc.write_boost		l2arc_write_boost
 L2ARC_WRITE_MAX			l2arc.write_max			l2arc_write_max
 LIVELIST_CONDENSE_NEW_ALLOC	livelist.condense.new_alloc	zfs_livelist_condense_new_alloc
diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am
index bd484b2da..776222f08 100644
--- a/tests/zfs-tests/tests/functional/Makefile.am
+++ b/tests/zfs-tests/tests/functional/Makefile.am
@@ -45,6 +45,7 @@ SUBDIRS = \
 	no_space \
 	nopwrite \
 	online_offline \
+	persist_l2arc \
 	pool_checkpoint \
 	pool_names \
 	poolversion \
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am b/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am
new file mode 100644
index 000000000..14a43de9c
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am
@@ -0,0 +1,15 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/persist_l2arc
+dist_pkgdata_SCRIPTS = \
+	cleanup.ksh \
+	setup.ksh \
+	persist_l2arc_001_pos.ksh \
+	persist_l2arc_002_pos.ksh \
+	persist_l2arc_003_neg.ksh \
+	persist_l2arc_004_pos.ksh \
+	persist_l2arc_005_pos.ksh \
+	persist_l2arc_006_pos.ksh \
+	persist_l2arc_007_pos.ksh \
+	persist_l2arc_008_pos.ksh
+
+dist_pkgdata_DATA = \
+	persist_l2arc.cfg
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh
new file mode 100755
index 000000000..828de3862
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh
@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+verify_runnable "global"
+
+if datasetexists $TESTPOOL ; then
+	log_must zpool destroy -f $TESTPOOL
+fi
+
+log_must rm -rf $VDIR
+
+log_pass
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg
new file mode 100644
index 000000000..60bb24637
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg
@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+export SIZE=1G
+export VDIR=$TESTDIR/disk.persist_l2arc
+export VDEV="$VDIR/a"
+export VDEV_CACHE="$VDIR/b"
+
+# fio options
+export DIRECTORY=/$TESTPOOL
+export NUMJOBS=4
+export RUNTIME=30
+export PERF_RANDSEED=1234
+export PERF_COMPPERCENT=66
+export PERF_COMPCHUNK=0
+export BLOCKSIZE=128K
+export SYNC_TYPE=0
+export DIRECT=1
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh
new file mode 100755
index 000000000..b202fac40
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh
@@ -0,0 +1,106 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Persistent L2ARC with an unencrypted ZFS file system succeeds
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Export and re-import pool without writing any data.
+#	3. Create a random file in that pool and random read for 30 sec.
+#	4. Export pool.
+#	5. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	6. Import pool.
+#	7. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(4).
+#	8. Check if the labels of the L2ARC device are intact.
+#
+#	* We can predict the minimum bytes of L2ARC restored if we subtract
+#	from the effective size of the cache device the bytes l2arc_evict()
+#	evicts:
+#	l2: L2ARC device size - VDEV_LABEL_START_SIZE - l2ad_dev_hdr_asize
+#	wr_sz: l2arc_write_max + l2arc_write_boost (worst case)
+#	blk_overhead: wr_sz / SPA_MINBLOCKSIZE / (l2 / SPA_MAXBLOCKSIZE) *
+#		sizeof (l2arc_log_blk_phys_t)
+#	min restored size: l2 - (wr_sz + blk_overhead)
+#
+
+verify_runnable "global"
+
+log_assert "Persistent L2ARC with an unencrypted ZFS file system succeeds."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must zpool export $TESTPOOL
+log_must zpool import -d $VDIR $TESTPOOL
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool export $TESTPOOL
+
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool import -d $VDIR $TESTPOOL
+
+sleep 2
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk -gt 0
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Persistent L2ARC with an unencrypted ZFS file system succeeds."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh
new file mode 100755
index 000000000..ae219e01a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh
@@ -0,0 +1,112 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+
+#
+# DESCRIPTION:
+#	Persistent L2ARC with an encrypted ZFS file system succeeds
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a an encrypted ZFS file system.
+#	3. Create a random file in the encyrpted file system and random
+#		read for 30 sec.
+#	4. Export pool.
+#	5. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	5. Import pool.
+#	6. Mount the encypted ZFS file system.
+#	7. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(5).
+#	8. Check if the labels of the L2ARC device are intact.
+#
+#	* We can predict the minimum bytes of L2ARC restored if we subtract
+#	from the effective size of the cache device the bytes l2arc_evict()
+#	evicts:
+#	l2: L2ARC device size - VDEV_LABEL_START_SIZE - l2ad_dev_hdr_asize
+#	wr_sz: l2arc_write_max + l2arc_write_boost (worst case)
+#	blk_overhead: wr_sz / SPA_MINBLOCKSIZE / (l2 / SPA_MAXBLOCKSIZE) *
+#		sizeof (l2arc_log_blk_phys_t)
+#	min restored size: l2 - (wr_sz + blk_overhead)
+#
+
+verify_runnable "global"
+
+log_assert "Persistent L2ARC with an encrypted ZFS file system succeeds."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \
+	"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool export $TESTPOOL
+
+sleep 2
+
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool import -d $VDIR $TESTPOOL
+log_must eval "echo $PASSPHRASE | zfs mount -l $TESTPOOL/$TESTFS1"
+
+sleep 2
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk -gt 0
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Persistent L2ARC with an encrypted ZFS file system succeeds."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh
new file mode 100755
index 000000000..7824dfe8f
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh
@@ -0,0 +1,87 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0
+#
+# STRATEGY:
+#	1. Set L2ARC_REBUILD_ENABLED = 0
+#	2. Create pool with a cache device.
+#	3. Create a random file in that pool and random read for 30 sec.
+#	4. Export pool.
+#	5. Import pool.
+#	6. Check in zpool iostat if the cache device has space allocated.
+#	7. Read the file written in (2) and check if l2_hits in
+#		/proc/spl/kstat/zfs/arcstats increased.
+#
+
+verify_runnable "global"
+
+log_assert "Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_REBUILD_ENABLED $rebuild_enabled
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+
+# disable L2ARC rebuild
+typeset rebuild_enabled=$(get_tunable L2ARC_REBUILD_ENABLED)
+log_must set_tunable32 L2ARC_REBUILD_ENABLED 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( 2 * $fill_mb ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool export $TESTPOOL
+
+typeset l2_success_start=$(get_arcstat l2_rebuild_success)
+
+log_must zpool import -d $VDIR $TESTPOOL
+log_mustnot test "$(zpool iostat -Hpv $TESTPOOL $VDEV_CACHE | awk '{print $2}')" -gt 80000000
+
+typeset l2_success_end=$(get_arcstat l2_rebuild_success)
+
+log_mustnot test $l2_success_end -gt $l2_success_start
+
+log_must zpool destroy -f $TESTPOOL
+log_must set_tunable32 L2ARC_REBUILD_ENABLED $rebuild_enabled
+
+log_pass "Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh
new file mode 100755
index 000000000..6620131d1
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh
@@ -0,0 +1,101 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Persistent L2ARC restores all written log blocks
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a random file in that pool, smaller than the cache device
+#		and random read for 30 sec.
+#	3. Export pool.
+#	4. Read amount of log blocks written.
+#	5. Import pool.
+#	6. Read amount of log blocks built.
+#	7. Compare the two amounts
+#	8. Read the file written in (2) and check if l2_hits in
+#		/proc/spl/kstat/zfs/arcstats increased.
+#	9. Check if the labels of the L2ARC device are intact.
+#
+
+verify_runnable "global"
+
+log_assert "Persistent L2ARC restores all written log blocks."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( 2 * $fill_mb ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+typeset log_blk_start=$(get_arcstat l2_log_blk_writes)
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool export $TESTPOOL
+
+sleep 2
+
+typeset log_blk_end=$(get_arcstat l2_log_blk_writes)
+
+typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool import -d $VDIR $TESTPOOL
+
+typeset l2_hits_start=$(get_arcstat l2_hits)
+
+export RUNTIME=10
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+typeset l2_hits_end=$(get_arcstat l2_hits)
+
+typeset log_blk_rebuild_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \
+	$(( $log_blk_end - $log_blk_start ))
+
+log_must test $l2_hits_end -gt $l2_hits_start
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Persistent L2ARC restores all written log blocks."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh
new file mode 100755
index 000000000..b2cad9d1f
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh
@@ -0,0 +1,108 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+
+#
+# DESCRIPTION:
+#	Persistent L2ARC restores all written log blocks with encryption
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a an encrypted ZFS file system.
+#	3. Create a random file in the entrypted file system,
+#		smaller than the cache device, and random read for 30 sec.
+#	4. Export pool.
+#	5. Read amount of log blocks written.
+#	6. Import pool.
+#	7. Mount the encypted ZFS file system.
+#	8. Read amount of log blocks built.
+#	9. Compare the two amounts
+#	10. Read the file written in (3) and check if l2_hits in
+#		/proc/spl/kstat/zfs/arcstats increased.
+#	11. Check if the labels of the L2ARC device are intact.
+#
+
+verify_runnable "global"
+
+log_assert "Persistent L2ARC restores all written log blocks with encryption."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( 2 * $fill_mb ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+typeset log_blk_start=$(get_arcstat l2_log_blk_writes)
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \
+	"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool export $TESTPOOL
+
+sleep 2
+
+typeset log_blk_end=$(get_arcstat l2_log_blk_writes)
+
+typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool import -d $VDIR $TESTPOOL
+log_must eval "echo $PASSPHRASE | zfs mount -l $TESTPOOL/$TESTFS1"
+
+typeset l2_hits_start=$(get_arcstat l2_hits)
+
+export RUNTIME=10
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+typeset l2_hits_end=$(get_arcstat l2_hits)
+
+typeset log_blk_rebuild_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \
+	$(( $log_blk_end - $log_blk_start ))
+
+log_must test $l2_hits_end -gt $l2_hits_start
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Persistent L2ARC restores all written log blocks with encryption."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh
new file mode 100755
index 000000000..55e9f9585
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh
@@ -0,0 +1,98 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not
+#	present.
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a random file in that pool and random read for 30 sec.
+#	3. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	4. Offline the L2ARC device and export pool.
+#	5. Import pool and online the L2ARC device.
+#	6. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(3).
+#	7. Check if the labels of the L2ARC device are intact.
+#
+
+verify_runnable "global"
+
+log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+log_must zpool export $TESTPOOL
+
+sleep 5
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+log_must zpool import -d $VDIR $TESTPOOL
+log_must zpool online $TESTPOOL $VDEV_CACHE
+
+sleep 5
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk -gt 0
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh
new file mode 100755
index 000000000..e3c983be8
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh
@@ -0,0 +1,95 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present.
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a random file in that pool and random read for 30 sec.
+#	3. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	4. Offline the L2ARC device.
+#	5. Online the L2ARC device.
+#	6. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(3).
+#	7. Check if the labels of the L2ARC device are intact.
+#
+
+verify_runnable "global"
+
+log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+
+sleep 5
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+log_must zpool online $TESTPOOL $VDEV_CACHE
+
+sleep 5
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk -gt 0
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh
new file mode 100755
index 000000000..a64bd94d3
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh
@@ -0,0 +1,143 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+#
+# DESCRIPTION:
+#	Off/onlining an L2ARC device restores all written blocks, vdev present.
+#
+# STRATEGY:
+#	1. Create pool with a cache device.
+#	2. Create a random file in that pool and random read for 30 sec.
+#	3. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	4. Offline the L2ARC device.
+#	5. Online the L2ARC device.
+#	6. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(3).
+#	7. Create another random file in that pool and random read for 30 sec.
+#	8. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	9. Offline the L2ARC device.
+#	10. Online the L2ARC device.
+#	11. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(7).
+#	12. Check if the amount of log blocks on the cache device has
+#		increased.
+#	13. Export the pool.
+#	14. Read the amount of log blocks on the cache device.
+#	15. Import the pool.
+#	16. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(14).
+#	17. Check if the labels of the L2ARC device are intact.
+#
+
+verify_runnable "global"
+
+log_assert "Off/onlining an L2ARC device restores all written blocks , vdev present."
+
+function cleanup
+{
+	if poolexists $TESTPOOL ; then
+		destroy_pool $TESTPOOL
+	fi
+
+	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+}
+log_onexit cleanup
+
+# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
+typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+log_must set_tunable32 L2ARC_NOPREFETCH 0
+
+typeset fill_mb=400
+typeset cache_sz=$(( 3 * $fill_mb ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+
+sleep 2
+
+typeset l2_dh_log_blk1=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool online $TESTPOOL $VDEV_CACHE
+
+sleep 5
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk1 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk1 -gt 0
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+
+sleep 2
+
+typeset l2_dh_log_blk2=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool online $TESTPOOL $VDEV_CACHE
+
+sleep 5
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk2 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+
+log_must test $l2_dh_log_blk2 -gt $l2_dh_log_blk1
+
+log_must zpool export $TESTPOOL
+
+typeset l2_dh_log_blk3=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
+
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+
+log_must zpool import -d $VDIR $TESTPOOL
+
+sleep 5
+
+typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks)
+
+log_must test $l2_dh_log_blk3 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk3 -gt 0
+
+log_must zdb -lq $VDEV_CACHE
+
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "Off/onlining an L2ARC device restores all written blocks, vdev present."
diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh
new file mode 100755
index 000000000..ef95c84cd
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh
@@ -0,0 +1,29 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020, George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg
+
+verify_runnable "global"
+
+log_must rm -rf $VDIR
+log_must mkdir -p $VDIR
+log_must mkfile $SIZE $VDEV
+
+log_pass