From 2e0e443ac40c7e825a02519a497328226bd866ff Mon Sep 17 00:00:00 2001 From: George Melikov Date: Sat, 4 Feb 2017 20:10:24 +0300 Subject: OpenZFS 7247 - zfs receive of deduplicated stream fails Authored by: Chris Williamson Reviewed by: Matthew Ahrens Reviewed by: Dan Kimmel Approved by: Robert Mustacchi Reviewed-by: loli10K Reviewed-by: Brian Behlendorf Ported-by: George Melikov OpenZFS-issue: https://www.illumos.org/issues/7247 OpenZFS-commit: https://github.com/openzfs/openzfs/commit/2ad25b4 Closes #5689 Porting notes: - tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh renamed as zfs_receive_015_pos.ksh, zfs_receive_013_pos.ksh is now used for OpenZFS test. - libzfs_sendrecv.c: SMALLEST_POSSIBLE_MAX_DDT_MB is always used for all 32-bit builds. --- lib/libzfs/libzfs_sendrecv.c | 6 +- module/zfs/dmu_send.c | 44 ++++++------ tests/runfiles/linux.run | 2 +- .../functional/cli_root/zfs_receive/Makefile.am | 3 +- .../cli_root/zfs_receive/zfs_receive_013_pos.ksh | 80 +++++++++------------ .../cli_root/zfs_receive/zfs_receive_015_pos.ksh | 83 ++++++++++++++++++++++ 6 files changed, 147 insertions(+), 71 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index b08cd2b63..c1a09e3a9 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -245,12 +245,16 @@ cksummer(void *arg) int outfd; dedup_table_t ddt; zio_cksum_t stream_cksum; - uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); uint64_t numbuckets; +#ifdef _ILP32 + ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20; +#else + uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); ddt.max_ddt_size = MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100, SMALLEST_POSSIBLE_MAX_DDT_MB << 20); +#endif numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t)); diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c index 6e117651b..6f17671ee 100644 --- a/module/zfs/dmu_send.c +++ b/module/zfs/dmu_send.c @@ -3230,6 +3230,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dsl_dataset_phys(origin_head)->ds_flags &= ~DS_FLAG_INCONSISTENT; + drc->drc_newsnapobj = + dsl_dataset_phys(origin_head)->ds_prev_snap_obj; + dsl_dataset_rele(origin_head, FTAG); dsl_destroy_head_sync_impl(drc->drc_ds, tx); @@ -3265,8 +3268,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) (void) zap_remove(dp->dp_meta_objset, ds->ds_object, DS_FIELD_RESUME_TONAME, tx); } + drc->drc_newsnapobj = + dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; } - drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; zvol_create_minors(dp->dp_spa, drc->drc_tofs, B_TRUE); /* * Release the hold from dmu_recv_begin. This must be done before @@ -3310,8 +3314,6 @@ static int dmu_recv_end_modified_blocks = 3; static int dmu_recv_existing_end(dmu_recv_cookie_t *drc) { - int error; - #ifdef _KERNEL /* * We will be destroying the ds; make sure its origin is unmounted if @@ -3322,23 +3324,30 @@ dmu_recv_existing_end(dmu_recv_cookie_t *drc) zfs_destroy_unmount_origin(name); #endif - error = dsl_sync_task(drc->drc_tofs, + return (dsl_sync_task(drc->drc_tofs, dmu_recv_end_check, dmu_recv_end_sync, drc, - dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL); - - if (error != 0) - dmu_recv_cleanup_ds(drc); - return (error); + dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); } static int dmu_recv_new_end(dmu_recv_cookie_t *drc) +{ + return (dsl_sync_task(drc->drc_tofs, + dmu_recv_end_check, dmu_recv_end_sync, drc, + dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); +} + +int +dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) { int error; - error = dsl_sync_task(drc->drc_tofs, - dmu_recv_end_check, dmu_recv_end_sync, drc, - dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL); + drc->drc_owner = owner; + + if (drc->drc_newfs) + error = dmu_recv_new_end(drc); + else + error = dmu_recv_existing_end(drc); if (error != 0) { dmu_recv_cleanup_ds(drc); @@ -3350,17 +3359,6 @@ dmu_recv_new_end(dmu_recv_cookie_t *drc) return (error); } -int -dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) -{ - drc->drc_owner = owner; - - if (drc->drc_newfs) - return (dmu_recv_new_end(drc)); - else - return (dmu_recv_existing_end(drc)); -} - /* * Return TRUE if this objset is currently being received into. */ diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 0a8198236..117117661 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -150,7 +150,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', 'zfs_receive_005_neg', 'zfs_receive_006_pos', 'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg', 'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos', - 'zfs_receive_013_pos', 'zfs_receive_014_pos'] + 'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos'] # DISABLED: # zfs_rename_006_pos - https://github.com/zfsonlinux/zfs/issues/5647 diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am index 564aab756..3c87a9927 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am @@ -15,4 +15,5 @@ dist_pkgdata_SCRIPTS = \ zfs_receive_011_pos.ksh \ zfs_receive_012_pos.ksh \ zfs_receive_013_pos.ksh \ - zfs_receive_014_pos.ksh + zfs_receive_014_pos.ksh \ + zfs_receive_015_pos.ksh diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh index 11081bb7f..f70d0f4bf 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh @@ -15,69 +15,59 @@ # # -# Copyright 2016, loli10K. All rights reserved. +# Copyright (c) 2015 by Delphix. All rights reserved. # -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib +. $STF_SUITE/tests/functional/cli_root/cli_common.kshlib # # DESCRIPTION: -# Verify ZFS can receive custom properties on both filesystems and -# snapshots from full and incremental streams. +# Verifying 'zfs receive' works correctly on deduplicated streams # # STRATEGY: -# 1. Create a filesystem. -# 2. Snapshot the filesystem. -# 3. Set custom properties on both the fs and snapshots. -# 4. Create different send streams with the properties. -# 5. Receive the send streams and verify the properties. +# 1. Create some snapshots with duplicated data +# 2. Send a deduplicated stream of the last snapshot +# 3. Attempt to receive the deduplicated stream # -verify_runnable "both" - -typeset streamfile_full=/var/tmp/streamfile_full.$$ -typeset streamfile_incr=/var/tmp/streamfile_incr.$$ -orig=$TESTPOOL/$TESTFS1 -dest=$TESTPOOL/$TESTFS2 -typeset user_prop=$(valid_user_property 8) -typeset value=$(user_property_value 8) +src_fs=$TESTPOOL/drecvsrc +temppool=recvtank +dst_fs=$temppool/drecvdest +streamfile=/var/tmp/drecvstream.$$ +tpoolfile=/temptank.$$ function cleanup { - log_must $RM $streamfile_full - log_must $RM $streamfile_incr - log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1 - log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2 + for fs in $src_fs $dst_fs; do + datasetexists $fs && log_must $ZFS destroy -rf $fs + done + $ZPOOL destroy $temppool + [[ -f $streamfile ]] && log_must $RM -f $streamfile + [[ -f $tpoolfile ]] && log_must $RM -f $tpoolfile } -log_assert "ZFS can receive custom properties." +log_assert "Verifying 'zfs receive' works correctly on deduplicated streams" log_onexit cleanup -# 1. Create a filesystem. -log_must $ZFS create $orig +truncate -s 100M $tpoolfile +log_must $ZPOOL create $temppool $tpoolfile +log_must $ZFS create $src_fs +src_mnt=$(get_prop mountpoint $src_fs) || log_fail "get_prop mountpoint $src_fs" + +echo blah > $src_mnt/blah +$ZFS snapshot $src_fs@base -# 2. Snapshot the filesystem. -log_must $ZFS snapshot $orig@snap1 -log_must $ZFS snapshot $orig@snap2 -log_must $ZFS snapshot $orig@snap3 +echo grumble > $src_mnt/grumble +echo blah > $src_mnt/blah2 +$ZFS snapshot $src_fs@snap2 -# 3. Set custom properties on both the fs and snapshots. -log_must eval "$ZFS set '$user_prop'='$value' $orig" -log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1" -log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2" -log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3" +echo grumble > $src_mnt/mumble +echo blah > $src_mnt/blah3 +$ZFS snapshot $src_fs@snap3 -# 4. Create different send streams with the properties. -log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full" -log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr" +log_must eval "$ZFS send -D -R $src_fs@snap3 > $streamfile" +log_must eval "$ZFS receive -v $dst_fs < $streamfile" -# 5. Receive the send streams and verify the properties. -log_must eval "$ZFS recv $dest < $streamfile_full" -log_must eval "check_user_prop $dest $user_prop '$value'" -log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'" -log_must eval "$ZFS recv $dest < $streamfile_incr" -log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'" -log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'" +cleanup -log_pass "ZFS can receive custom properties passed." +log_pass "Verifying 'zfs receive' works correctly on deduplicated streams" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh new file mode 100755 index 000000000..11081bb7f --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh @@ -0,0 +1,83 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright 2016, loli10K. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib + +# +# DESCRIPTION: +# Verify ZFS can receive custom properties on both filesystems and +# snapshots from full and incremental streams. +# +# STRATEGY: +# 1. Create a filesystem. +# 2. Snapshot the filesystem. +# 3. Set custom properties on both the fs and snapshots. +# 4. Create different send streams with the properties. +# 5. Receive the send streams and verify the properties. +# + +verify_runnable "both" + +typeset streamfile_full=/var/tmp/streamfile_full.$$ +typeset streamfile_incr=/var/tmp/streamfile_incr.$$ +orig=$TESTPOOL/$TESTFS1 +dest=$TESTPOOL/$TESTFS2 +typeset user_prop=$(valid_user_property 8) +typeset value=$(user_property_value 8) + +function cleanup +{ + log_must $RM $streamfile_full + log_must $RM $streamfile_incr + log_must $ZFS destroy -rf $TESTPOOL/$TESTFS1 + log_must $ZFS destroy -rf $TESTPOOL/$TESTFS2 +} + +log_assert "ZFS can receive custom properties." +log_onexit cleanup + +# 1. Create a filesystem. +log_must $ZFS create $orig + +# 2. Snapshot the filesystem. +log_must $ZFS snapshot $orig@snap1 +log_must $ZFS snapshot $orig@snap2 +log_must $ZFS snapshot $orig@snap3 + +# 3. Set custom properties on both the fs and snapshots. +log_must eval "$ZFS set '$user_prop'='$value' $orig" +log_must eval "$ZFS set '$user_prop:snap1'='$value:snap1' $orig@snap1" +log_must eval "$ZFS set '$user_prop:snap2'='$value:snap2' $orig@snap2" +log_must eval "$ZFS set '$user_prop:snap3'='$value:snap3' $orig@snap3" + +# 4. Create different send streams with the properties. +log_must eval "$ZFS send -p $orig@snap1 > $streamfile_full" +log_must eval "$ZFS send -p -I $orig@snap1 $orig@snap3 > $streamfile_incr" + +# 5. Receive the send streams and verify the properties. +log_must eval "$ZFS recv $dest < $streamfile_full" +log_must eval "check_user_prop $dest $user_prop '$value'" +log_must eval "check_user_prop $dest@snap1 '$user_prop:snap1' '$value:snap1'" +log_must eval "$ZFS recv $dest < $streamfile_incr" +log_must eval "check_user_prop $dest@snap2 '$user_prop:snap2' '$value:snap2'" +log_must eval "check_user_prop $dest@snap3 '$user_prop:snap3' '$value:snap3'" + +log_pass "ZFS can receive custom properties passed." -- cgit v1.2.3