diff options
26 files changed, 219 insertions, 1081 deletions
diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index ae71cdc88..15e350313 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2019 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright 2012 Milan Jurik. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. @@ -4266,7 +4266,10 @@ zfs_do_send(int argc, char **argv) flags.progress = B_TRUE; break; case 'D': - flags.dedup = B_TRUE; + (void) fprintf(stderr, + gettext("WARNING: deduplicated send is no " + "longer supported. A regular,\n" + "non-deduplicated stream will be generated.\n\n")); break; case 'n': flags.dryrun = B_TRUE; @@ -4333,16 +4336,6 @@ zfs_do_send(int argc, char **argv) } } - if (flags.dedup) { - (void) fprintf(stderr, - gettext("WARNING: deduplicated send is " - "deprecated, and will be removed in a\n" - "future release. (In the future, the flag will be " - "accepted, but a\n" - "regular, non-deduplicated stream will be " - "generated.)\n\n")); - } - if (flags.parsable && flags.verbosity == 0) flags.verbosity = 1; @@ -4351,7 +4344,7 @@ zfs_do_send(int argc, char **argv) if (resume_token != NULL) { if (fromname != NULL || flags.replicate || flags.props || - flags.backup || flags.dedup || flags.holds || + flags.backup || flags.holds || flags.saved || redactbook != NULL) { (void) fprintf(stderr, gettext("invalid flags combined with -t\n")); @@ -4375,7 +4368,7 @@ zfs_do_send(int argc, char **argv) if (flags.saved) { if (fromname != NULL || flags.replicate || flags.props || - flags.doall || flags.backup || flags.dedup || + flags.doall || flags.backup || flags.holds || flags.largeblock || flags.embed_data || flags.compress || flags.raw || redactbook != NULL) { (void) fprintf(stderr, gettext("incompatible flags " diff --git a/include/libzfs.h b/include/libzfs.h index 7633579d4..c4f08882e 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright Joyent, Inc. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2016, Intel Corporation. @@ -651,8 +651,8 @@ typedef struct sendflags { /* if dataset is a clone, do incremental from its origin */ boolean_t fromorigin; - /* do deduplication */ - boolean_t dedup; + /* field no longer used, maintained for backwards compatibility */ + boolean_t pad; /* send properties (ie, -p) */ boolean_t props; diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index 45c344c83..5cec62913 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2017 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2018 Datto Inc. */ @@ -71,7 +71,6 @@ struct libzfs_handle { int libzfs_pool_iter; char libzfs_chassis_id[256]; boolean_t libzfs_prop_debug; - boolean_t libzfs_dedup_warning_printed; }; #define ZFSSHARE_MISS 0x01 /* Didn't find entry in cache */ diff --git a/include/sys/dmu.h b/include/sys/dmu.h index 928ee763d..139f3cbdf 100644 --- a/include/sys/dmu.h +++ b/include/sys/dmu.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright 2014 HybridCluster. All rights reserved. @@ -864,8 +864,6 @@ int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, struct arc_buf *buf, dmu_tx_t *tx); #define dmu_assign_arcbuf dmu_assign_arcbuf_by_dbuf -void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset, - dmu_buf_t *handle, dmu_tx_t *tx); #ifdef HAVE_UIO_ZEROCOPY int dmu_xuio_init(struct xuio *uio, int niov); void dmu_xuio_fini(struct xuio *uio); diff --git a/include/sys/dmu_recv.h b/include/sys/dmu_recv.h index e3b54e3d1..c0a562115 100644 --- a/include/sys/dmu_recv.h +++ b/include/sys/dmu_recv.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. */ @@ -53,10 +53,8 @@ typedef struct dmu_recv_cookie { boolean_t drc_raw; boolean_t drc_clone; boolean_t drc_spill; - struct avl_tree *drc_guid_to_ds_map; nvlist_t *drc_keynvl; uint64_t drc_fromsnapobj; - uint64_t drc_newsnapobj; uint64_t drc_ivset_guid; void *drc_owner; cred_t *drc_cred; @@ -80,13 +78,11 @@ typedef struct dmu_recv_cookie { objlist_t *drc_ignore_objlist; } dmu_recv_cookie_t; -int dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin, - boolean_t force, boolean_t resumable, nvlist_t *localprops, - nvlist_t *hidden_args, char *origin, dmu_recv_cookie_t *drc, - zfs_file_t *fp, offset_t *voffp); -int dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, - uint64_t *action_handlep, offset_t *voffp); -int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner); -boolean_t dmu_objset_is_receiving(objset_t *os); +int dmu_recv_begin(char *, char *, dmu_replay_record_t *, + boolean_t, boolean_t, nvlist_t *, nvlist_t *, char *, + dmu_recv_cookie_t *, zfs_file_t *, offset_t *); +int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *); +int dmu_recv_end(dmu_recv_cookie_t *, void *); +boolean_t dmu_objset_is_receiving(objset_t *); #endif /* _DMU_RECV_H */ diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index da5a3ee60..d4ffe70bb 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright 2016 RackTop Systems. * Copyright (c) 2017, Intel Corporation. */ @@ -111,8 +111,7 @@ typedef enum drr_headertype { /* * Mask of all supported backup features */ -#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_DEDUP | \ - DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \ +#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_SA_SPILL | \ DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_LZ4 | \ DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \ DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \ diff --git a/include/sys/zfs_onexit.h b/include/sys/zfs_onexit.h index 4982bd4d0..0fab23ff8 100644 --- a/include/sys/zfs_onexit.h +++ b/include/sys/zfs_onexit.h @@ -21,6 +21,7 @@ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 by Delphix. All rights reserved. */ #ifndef _SYS_ZFS_ONEXIT_H @@ -54,10 +55,6 @@ extern int zfs_onexit_fd_hold(int fd, minor_t *minorp); extern void zfs_onexit_fd_rele(int fd); extern int zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, uint64_t *action_handle); -extern int zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, - boolean_t fire); -extern int zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, - void **data); #ifdef __cplusplus } diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 43a39e789..c65673dd3 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright (c) 2012 Pawel Jakub Dawidek <[email protected]>. * All rights reserved @@ -73,22 +73,14 @@ extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *); static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *, - recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int, - uint64_t *, const char *, nvlist_t *); + recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, + const char *, nvlist_t *); static int guid_to_name_redact_snaps(libzfs_handle_t *hdl, const char *parent, uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids, uint64_t num_redact_snaps, char *name); static int guid_to_name(libzfs_handle_t *, const char *, uint64_t, boolean_t, char *); -static const zio_cksum_t zero_cksum = { { 0 } }; - -typedef struct dedup_arg { - int inputfd; - int outputfd; - libzfs_handle_t *dedup_hdl; -} dedup_arg_t; - typedef struct progress_arg { zfs_handle_t *pa_zhp; int pa_fd; @@ -97,112 +89,6 @@ typedef struct progress_arg { int pa_verbosity; } progress_arg_t; -typedef struct dataref { - uint64_t ref_guid; - uint64_t ref_object; - uint64_t ref_offset; -} dataref_t; - -typedef struct dedup_entry { - struct dedup_entry *dde_next; - zio_cksum_t dde_chksum; - uint64_t dde_prop; - dataref_t dde_ref; -} dedup_entry_t; - -#define MAX_DDT_PHYSMEM_PERCENT 20 -#define SMALLEST_POSSIBLE_MAX_DDT_MB 128 - -typedef struct dedup_table { - dedup_entry_t **dedup_hash_array; - umem_cache_t *ddecache; - uint64_t max_ddt_size; /* max dedup table size in bytes */ - uint64_t cur_ddt_size; /* current dedup table size in bytes */ - uint64_t ddt_count; - int numhashbits; - boolean_t ddt_full; -} dedup_table_t; - -static int -high_order_bit(uint64_t n) -{ - int count; - - for (count = 0; n != 0; count++) - n >>= 1; - return (count); -} - -static size_t -ssread(void *buf, size_t len, FILE *stream) -{ - size_t outlen; - - if ((outlen = fread(buf, len, 1, stream)) == 0) - return (0); - - return (outlen); -} - -static void -ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp, - zio_cksum_t *cs, uint64_t prop, dataref_t *dr) -{ - dedup_entry_t *dde; - - if (ddt->cur_ddt_size >= ddt->max_ddt_size) { - if (ddt->ddt_full == B_FALSE) { - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "Dedup table full. Deduplication will continue " - "with existing table entries")); - ddt->ddt_full = B_TRUE; - } - return; - } - - if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT)) - != NULL) { - assert(*ddepp == NULL); - dde->dde_next = NULL; - dde->dde_chksum = *cs; - dde->dde_prop = prop; - dde->dde_ref = *dr; - *ddepp = dde; - ddt->cur_ddt_size += sizeof (dedup_entry_t); - ddt->ddt_count++; - } -} - -/* - * Using the specified dedup table, do a lookup for an entry with - * the checksum cs. If found, return the block's reference info - * in *dr. Otherwise, insert a new entry in the dedup table, using - * the reference information specified by *dr. - * - * return value: true - entry was found - * false - entry was not found - */ -static boolean_t -ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs, - uint64_t prop, dataref_t *dr) -{ - uint32_t hashcode; - dedup_entry_t **ddepp; - - hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits); - - for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL; - ddepp = &((*ddepp)->dde_next)) { - if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) && - (*ddepp)->dde_prop == prop) { - *dr = (*ddepp)->dde_ref; - return (B_TRUE); - } - } - ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr); - return (B_FALSE); -} - static int dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, zio_cksum_t *zc, int outfd) @@ -229,274 +115,6 @@ dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, } /* - * This function is started in a separate thread when the dedup option - * has been requested. The main send thread determines the list of - * snapshots to be included in the send stream and makes the ioctl calls - * for each one. But instead of having the ioctl send the output to the - * the output fd specified by the caller of zfs_send()), the - * ioctl is told to direct the output to a pipe, which is read by the - * alternate thread running THIS function. This function does the - * dedup'ing by: - * 1. building a dedup table (the DDT) - * 2. doing checksums on each data block and inserting a record in the DDT - * 3. looking for matching checksums, and - * 4. sending a DRR_WRITE_BYREF record instead of a write record whenever - * a duplicate block is found. - * The output of this function then goes to the output fd requested - * by the caller of zfs_send(). - */ -static void * -cksummer(void *arg) -{ - dedup_arg_t *dda = arg; - char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE); - dmu_replay_record_t thedrr = { 0 }; - dmu_replay_record_t *drr = &thedrr; - FILE *ofp; - int outfd; - dedup_table_t ddt; - zio_cksum_t stream_cksum; - uint64_t numbuckets; - -#ifdef _ILP32 - ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20; -#else - uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); - ddt.max_ddt_size = - MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100, - SMALLEST_POSSIBLE_MAX_DDT_MB << 20); -#endif - - numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t)); - - /* - * numbuckets must be a power of 2. Increase number to - * a power of 2 if necessary. - */ - if (!ISP2(numbuckets)) - numbuckets = 1ULL << high_order_bit(numbuckets); - - ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *)); - ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0, - NULL, NULL, NULL, NULL, NULL, 0); - ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *); - ddt.numhashbits = high_order_bit(numbuckets) - 1; - ddt.ddt_full = B_FALSE; - - outfd = dda->outputfd; - ofp = fdopen(dda->inputfd, "r"); - while (ssread(drr, sizeof (*drr), ofp) != 0) { - - /* - * kernel filled in checksum, we are going to write same - * record, but need to regenerate checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - bzero(&drr->drr_u.drr_checksum.drr_checksum, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - - switch (drr->drr_type) { - case DRR_BEGIN: - { - struct drr_begin *drrb = &drr->drr_u.drr_begin; - int fflags; - int sz = 0; - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - - ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); - - /* set the DEDUP feature flag for this stream */ - fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); - fflags |= (DMU_BACKUP_FEATURE_DEDUP | - DMU_BACKUP_FEATURE_DEDUPPROPS); - DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags); - - if (drr->drr_payloadlen != 0) { - sz = drr->drr_payloadlen; - - if (sz > SPA_MAXBLOCKSIZE) { - buf = zfs_realloc(dda->dedup_hdl, buf, - SPA_MAXBLOCKSIZE, sz); - } - (void) ssread(buf, sz, ofp); - if (ferror(stdin)) - perror("fread"); - } - if (dump_record(drr, buf, sz, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* use the recalculated checksum */ - drre->drr_checksum = stream_cksum; - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - if (drro->drr_bonuslen > 0) { - (void) ssread(buf, - DRR_OBJECT_PAYLOAD_SIZE(drro), ofp); - } - if (dump_record(drr, buf, DRR_OBJECT_PAYLOAD_SIZE(drro), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - (void) ssread(buf, DRR_SPILL_PAYLOAD_SIZE(drrs), ofp); - if (dump_record(drr, buf, DRR_SPILL_PAYLOAD_SIZE(drrs), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_FREEOBJECTS: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_WRITE: - { - struct drr_write *drrw = &drr->drr_u.drr_write; - dataref_t dataref; - uint64_t payload_size; - - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - (void) ssread(buf, payload_size, ofp); - - /* - * Use the existing checksum if it's dedup-capable, - * else calculate a SHA256 checksum for it. - */ - - if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum, - zero_cksum) || - !DRR_IS_DEDUP_CAPABLE(drrw->drr_flags)) { - SHA2_CTX ctx; - zio_cksum_t tmpsha256; - - SHA2Init(SHA256, &ctx); - SHA2Update(&ctx, buf, payload_size); - SHA2Final(&tmpsha256, &ctx); - - drrw->drr_key.ddk_cksum.zc_word[0] = - BE_64(tmpsha256.zc_word[0]); - drrw->drr_key.ddk_cksum.zc_word[1] = - BE_64(tmpsha256.zc_word[1]); - drrw->drr_key.ddk_cksum.zc_word[2] = - BE_64(tmpsha256.zc_word[2]); - drrw->drr_key.ddk_cksum.zc_word[3] = - BE_64(tmpsha256.zc_word[3]); - drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256; - drrw->drr_flags |= DRR_CHECKSUM_DEDUP; - } - - dataref.ref_guid = drrw->drr_toguid; - dataref.ref_object = drrw->drr_object; - dataref.ref_offset = drrw->drr_offset; - - if (ddt_update(dda->dedup_hdl, &ddt, - &drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop, - &dataref)) { - dmu_replay_record_t wbr_drr = {0}; - struct drr_write_byref *wbr_drrr = - &wbr_drr.drr_u.drr_write_byref; - - /* block already present in stream */ - wbr_drr.drr_type = DRR_WRITE_BYREF; - - wbr_drrr->drr_object = drrw->drr_object; - wbr_drrr->drr_offset = drrw->drr_offset; - wbr_drrr->drr_length = drrw->drr_logical_size; - wbr_drrr->drr_toguid = drrw->drr_toguid; - wbr_drrr->drr_refguid = dataref.ref_guid; - wbr_drrr->drr_refobject = - dataref.ref_object; - wbr_drrr->drr_refoffset = - dataref.ref_offset; - - wbr_drrr->drr_checksumtype = - drrw->drr_checksumtype; - wbr_drrr->drr_flags = drrw->drr_flags; - wbr_drrr->drr_key.ddk_cksum = - drrw->drr_key.ddk_cksum; - wbr_drrr->drr_key.ddk_prop = - drrw->drr_key.ddk_prop; - - if (dump_record(&wbr_drr, NULL, 0, - &stream_cksum, outfd) != 0) - goto out; - } else { - /* block not previously seen */ - if (dump_record(drr, buf, payload_size, - &stream_cksum, outfd) != 0) - goto out; - } - break; - } - - case DRR_WRITE_EMBEDDED: - { - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - (void) ssread(buf, - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), ofp); - if (dump_record(drr, buf, - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_FREE: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_OBJECT_RANGE: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } - } -out: - umem_cache_destroy(ddt.ddecache); - free(ddt.dedup_hash_array); - free(buf); - (void) fclose(ofp); - - return (NULL); -} - -/* * Routines for dealing with the AVL tree of fs-nvlists */ typedef struct fsavl_node { @@ -2478,7 +2096,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, int spa_version; pthread_t tid = 0; int pipefd[2]; - dedup_arg_t dda = { 0 }; int featureflags = 0; FILE *fout; @@ -2502,33 +2119,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, if (flags->holds) featureflags |= DMU_BACKUP_FEATURE_HOLDS; - /* - * Start the dedup thread if this is a dedup stream. We do not bother - * doing this if this a raw send of an encrypted dataset with dedup off - * because normal encrypted blocks won't dedup. - */ - if (flags->dedup && !flags->dryrun && !(flags->raw && - zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF && - zfs_prop_get_int(zhp, ZFS_PROP_DEDUP) == ZIO_CHECKSUM_OFF)) { - featureflags |= (DMU_BACKUP_FEATURE_DEDUP | - DMU_BACKUP_FEATURE_DEDUPPROPS); - if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) { - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED, - errbuf)); - } - dda.outputfd = outfd; - dda.inputfd = pipefd[1]; - dda.dedup_hdl = zhp->zfs_hdl; - if ((err = pthread_create(&tid, NULL, cksummer, &dda)) != 0) { - (void) close(pipefd[0]); - (void) close(pipefd[1]); - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, - EZFS_THREADCREATEFAILED, errbuf)); - } - } - if (flags->replicate || flags->doall || flags->props || flags->holds || flags->backup) { char full_tosnap_name[ZFS_MAX_DATASET_NAME_LEN]; @@ -2706,34 +2296,6 @@ err_out: return (err); } -static int -get_dedup_fd(zfs_handle_t *zhp, dedup_arg_t *dda, int fd, pthread_t *tid, - int *outfd) -{ - int pipefd[2]; - char errbuf[1024]; - int err; - (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, - "warning: cannot send '%s'"), zhp->zfs_name); - if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) { - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED, - errbuf)); - } - dda->outputfd = fd; - dda->inputfd = pipefd[1]; - dda->dedup_hdl = zhp->zfs_hdl; - if ((err = pthread_create(tid, NULL, cksummer, dda)) != 0) { - (void) close(pipefd[0]); - (void) close(pipefd[1]); - zfs_error_aux(zhp->zfs_hdl, strerror(err)); - return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED, - errbuf)); - } - *outfd = pipefd[0]; - return (0); -} - zfs_handle_t * name_to_dir_handle(libzfs_handle_t *hdl, const char *snapname) { @@ -2819,9 +2381,8 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, libzfs_handle_t *hdl = zhp->zfs_hdl; char *name = zhp->zfs_name; int orig_fd = fd; - pthread_t ddtid, ptid; + pthread_t ptid; progress_arg_t pa = { 0 }; - dedup_arg_t dda = { 0 }; char errbuf[1024]; (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, @@ -2915,16 +2476,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, return (0); /* - * If deduplication is requested, spawn a thread that will deduplicate - * the data coming out of the kernel. - */ - if (flags->dedup) { - err = get_dedup_fd(zhp, &dda, fd, &ddtid, &fd); - if (err != 0) - return (err); - } - - /* * If progress reporting is requested, spawn a new thread to poll * ZFS_IOC_SEND_PROGRESS at a regular interval. */ @@ -2939,11 +2490,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, send_progress_thread, &pa); if (err != 0) { zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - if (flags->dedup) { - (void) pthread_cancel(ddtid); - (void) close(fd); - (void) pthread_join(ddtid, NULL); - } return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED, errbuf)); } @@ -2966,12 +2512,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, return (zfs_standard_error(hdl, error, errbuf)); } } - if (flags->dedup) { - if (err != 0) - (void) pthread_cancel(ddtid); - (void) close(fd); - (void) pthread_join(ddtid, NULL); - } if (flags->props || flags->holds || flags->backup) { /* Write the final end record. */ @@ -3965,8 +3505,7 @@ doagain: static int zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc, - char **top_zfs, int cleanup_fd, uint64_t *action_handlep, - nvlist_t *cmdprops) + char **top_zfs, nvlist_t *cmdprops) { nvlist_t *stream_nv = NULL; avl_tree_t *stream_avl = NULL; @@ -4143,8 +3682,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, * recv_skip() and return 0). */ error = zfs_receive_impl(hdl, destname, NULL, flags, fd, - sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd, - action_handlep, sendsnap, cmdprops); + sendfs, stream_nv, stream_avl, top_zfs, sendsnap, cmdprops); if (error == ENODATA) { error = 0; break; @@ -4530,8 +4068,8 @@ static int zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr, dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv, - avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops) + avl_tree_t *stream_avl, char **top_zfs, + const char *finalsnap, nvlist_t *cmdprops) { time_t begin_time; int ioctl_err, ioctl_errno, err; @@ -4741,24 +4279,16 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, (void) printf("found clone origin %s\n", origin); } - if (!hdl->libzfs_dedup_warning_printed && - (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & + if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & DMU_BACKUP_FEATURE_DEDUP)) { (void) fprintf(stderr, - gettext("WARNING: This is a deduplicated send stream. " - "The ability to send and\n" - "receive deduplicated send streams is deprecated. " - "In the future, the\n" - "ability to receive a deduplicated send stream with " - "\"zfs receive\" will be\n" - "removed. However, in the future, a utility will be " - "provided to convert a\n" - "deduplicated send stream to a regular " - "(non-deduplicated) stream. This\n" - "future utility will require that the send stream be " - "located in a\n" - "seek-able file, rather than provided by a pipe.\n\n")); - hdl->libzfs_dedup_warning_printed = B_TRUE; + gettext("ERROR: \"zfs receive\" no longer supports " + "deduplicated send streams. Use\n" + "the \"zstream redup\" command to convert this stream " + "to a regular,\n" + "non-deduplicated stream.\n")); + err = zfs_error(hdl, EZFS_NOTSUP, errbuf); + goto out; } boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & @@ -5103,8 +4633,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops, oxprops, wkeydata, wkeylen, origin, flags->force, flags->resumable, - raw, infd, drr_noswap, cleanup_fd, &read_bytes, &errflags, - action_handlep, &prop_errors); + raw, infd, drr_noswap, -1, &read_bytes, &errflags, + NULL, &prop_errors); ioctl_errno = ioctl_err; prop_errflags = errflags; @@ -5435,8 +4965,8 @@ zfs_receive_checkprops(libzfs_handle_t *hdl, nvlist_t *props, static int zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, const char *originsnap, recvflags_t *flags, int infd, const char *sendfs, - nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops) + nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, + const char *finalsnap, nvlist_t *cmdprops) { int err; dmu_replay_record_t drr, drr_noswap; @@ -5546,12 +5076,12 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, } return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags, &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs, - cleanup_fd, action_handlep, finalsnap, cmdprops)); + finalsnap, cmdprops)); } else { assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_COMPOUNDSTREAM); return (zfs_receive_package(hdl, infd, tosnap, flags, &drr, - &zcksum, top_zfs, cleanup_fd, action_handlep, cmdprops)); + &zcksum, top_zfs, cmdprops)); } } @@ -5568,8 +5098,6 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, { char *top_zfs = NULL; int err; - int cleanup_fd; - uint64_t action_handle = 0; struct stat sb; char *originsnap = NULL; @@ -5595,13 +5123,8 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, return (err); } - cleanup_fd = open(ZFS_DEV, O_RDWR); - VERIFY(cleanup_fd >= 0); - err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL, - stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL, props); - - VERIFY(0 == close(cleanup_fd)); + stream_avl, &top_zfs, NULL, props); if (err == 0 && !flags->nomount && flags->domount && top_zfs) { zfs_handle_t *zhp = NULL; diff --git a/lib/libzfs_core/libzfs_core.c b/lib/libzfs_core/libzfs_core.c index 18143d364..4e83b624b 100644 --- a/lib/libzfs_core/libzfs_core.c +++ b/lib/libzfs_core/libzfs_core.c @@ -20,7 +20,7 @@ */ /* - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2017 Datto Inc. * Copyright 2017 RackTop Systems. @@ -783,9 +783,8 @@ static int recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, uint8_t *wkeydata, uint_t wkeylen, const char *origin, boolean_t force, boolean_t resumable, boolean_t raw, int input_fd, - const dmu_replay_record_t *begin_record, int cleanup_fd, - uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle, - nvlist_t **errors) + const dmu_replay_record_t *begin_record, uint64_t *read_bytes, + uint64_t *errflags, nvlist_t **errors) { dmu_replay_record_t drr; char fsname[MAXPATHLEN]; @@ -868,12 +867,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, if (resumable) fnvlist_add_boolean(innvl, "resumable"); - if (cleanup_fd >= 0) - fnvlist_add_int32(innvl, "cleanup_fd", cleanup_fd); - - if (action_handle != NULL) - fnvlist_add_uint64(innvl, "action_handle", - *action_handle); error = lzc_ioctl(ZFS_IOC_RECV_NEW, fsname, innvl, &outnvl); @@ -885,10 +878,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, error = nvlist_lookup_uint64(outnvl, "error_flags", errflags); - if (error == 0 && action_handle != NULL) - error = nvlist_lookup_uint64(outnvl, "action_handle", - action_handle); - if (error == 0 && errors != NULL) { nvlist_t *nvl; error = nvlist_lookup_nvlist(outnvl, "errors", &nvl); @@ -931,12 +920,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, zc.zc_cleanup_fd = -1; zc.zc_action_handle = 0; - if (cleanup_fd >= 0) - zc.zc_cleanup_fd = cleanup_fd; - - if (action_handle != NULL) - zc.zc_action_handle = *action_handle; - zc.zc_nvlist_dst_size = 128 * 1024; zc.zc_nvlist_dst = (uint64_t)(uintptr_t) malloc(zc.zc_nvlist_dst_size); @@ -951,9 +934,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, if (errflags != NULL) *errflags = zc.zc_obj; - if (action_handle != NULL) - *action_handle = zc.zc_action_handle; - if (errors != NULL) VERIFY0(nvlist_unpack( (void *)(uintptr_t)zc.zc_nvlist_dst, @@ -986,7 +966,7 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin, boolean_t force, boolean_t raw, int fd) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - B_FALSE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL)); + B_FALSE, raw, fd, NULL, NULL, NULL, NULL)); } /* @@ -1000,7 +980,7 @@ lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin, boolean_t force, boolean_t raw, int fd) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - B_TRUE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL)); + B_TRUE, raw, fd, NULL, NULL, NULL, NULL)); } /* @@ -1023,7 +1003,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props, return (EINVAL); return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - resumable, raw, fd, begin_record, -1, NULL, NULL, NULL, NULL)); + resumable, raw, fd, begin_record, NULL, NULL, NULL)); } /* @@ -1039,9 +1019,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props, * The 'errflags' value will contain zprop_errflags_t flags which are * used to describe any failures. * - * The 'action_handle' is used to pass the handle for this guid/ds mapping. - * It should be set to zero on first call and will contain an updated handle - * on success, it should be passed in subsequent calls. + * The 'action_handle' and 'cleanup_fd' are no longer used, and are ignored. * * The 'errors' nvlist contains an entry for each unapplied received * property. Callers are responsible for freeing this nvlist. @@ -1053,8 +1031,8 @@ int lzc_receive_one(const char *snapname, nvlist_t *props, nvlist_t **errors) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - resumable, raw, input_fd, begin_record, cleanup_fd, read_bytes, - errflags, action_handle, errors)); + resumable, raw, input_fd, begin_record, + read_bytes, errflags, errors)); } /* @@ -1073,8 +1051,8 @@ int lzc_receive_with_cmdprops(const char *snapname, nvlist_t *props, nvlist_t **errors) { return (recv_impl(snapname, props, cmdprops, wkeydata, wkeylen, origin, - force, resumable, raw, input_fd, begin_record, cleanup_fd, - read_bytes, errflags, action_handle, errors)); + force, resumable, raw, input_fd, begin_record, + read_bytes, errflags, errors)); } /* diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index a60a5c7bb..2d799a2da 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -105,17 +105,11 @@ destroyed by using the .Nm zfs Cm destroy Fl d command. .Pp -Deduplicated send streams can be generated by using the -.Nm zfs Cm send Fl D +The ability to send and receive deduplicated send streams has been removed. +However, a deduplicated send stream created with older software can be converted +to a regular (non-deduplicated) stream by using the +.Nm zstream Cm redup command. -\fBThe ability to send and receive deduplicated send streams is deprecated.\fR -In the future, the ability to receive a deduplicated send stream with -.Nm zfs Cm receive -will be removed. -However, in the future, a utility will be provided to convert a -deduplicated send stream to a regular (non-deduplicated) stream. -This future utility will require that the send stream be located in a -seek-able file, rather than provided by a pipe. .Pp If .Fl o Em property Ns = Ns Ar value @@ -378,3 +372,4 @@ deleting its saved partially received state. .El .Sh SEE ALSO .Xr zfs-send 8 +.Xr zstream 8 diff --git a/man/man8/zfs-send.8 b/man/man8/zfs-send.8 index 1deedc214..c1b2134d1 100644 --- a/man/man8/zfs-send.8 +++ b/man/man8/zfs-send.8 @@ -86,21 +86,9 @@ The output can be redirected to a file or to a different system By default, a full stream is generated. .Bl -tag -width "-D" .It Fl D, -dedup -Generate a deduplicated stream. -\fBDeduplicated send is deprecated and will be removed in a future release.\fR -(In the future, the flag will be accepted but a regular, non-deduplicated -stream will be generated.) -Blocks which would have been sent multiple times in the send stream will only be -sent once. -The receiving system must also support this feature to receive a deduplicated -stream. -This flag can be used regardless of the dataset's -.Sy dedup -property, but performance will be much better if the filesystem uses a -dedup-capable checksum -.Po for example, -.Sy sha256 -.Pc . +Deduplicated send is no longer supported. +This flag is accepted for backwards compatibility, but a regular, +non-deduplicated stream will be generated. .It Fl I Ar snapshot Generate a stream package that sends all intermediary snapshots from the first snapshot to the second snapshot. diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index 6eb935720..a21ac8d74 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved. @@ -1560,56 +1560,6 @@ dmu_return_arcbuf(arc_buf_t *buf) arc_buf_destroy(buf, FTAG); } -void -dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset, - dmu_buf_t *handle, dmu_tx_t *tx) -{ - dmu_buf_t *dst_handle; - dmu_buf_impl_t *dstdb; - dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle; - dmu_object_type_t type; - arc_buf_t *abuf; - uint64_t datalen; - boolean_t byteorder; - uint8_t salt[ZIO_DATA_SALT_LEN]; - uint8_t iv[ZIO_DATA_IV_LEN]; - uint8_t mac[ZIO_DATA_MAC_LEN]; - - ASSERT3P(srcdb->db_buf, !=, NULL); - - /* hold the db that we want to write to */ - VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle, - DMU_READ_NO_DECRYPT)); - dstdb = (dmu_buf_impl_t *)dst_handle; - datalen = arc_buf_size(srcdb->db_buf); - - DB_DNODE_ENTER(dstdb); - type = DB_DNODE(dstdb)->dn_type; - DB_DNODE_EXIT(dstdb); - - /* allocated an arc buffer that matches the type of srcdb->db_buf */ - if (arc_is_encrypted(srcdb->db_buf)) { - arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac); - abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os), - byteorder, salt, iv, mac, type, - datalen, arc_buf_lsize(srcdb->db_buf), - arc_get_compression(srcdb->db_buf)); - } else { - /* we won't get a compressed db back from dmu_buf_hold() */ - ASSERT3U(arc_get_compression(srcdb->db_buf), - ==, ZIO_COMPRESS_OFF); - abuf = arc_loan_buf(os->os_spa, - DMU_OT_IS_METADATA(type), datalen); - } - - ASSERT3U(datalen, ==, arc_buf_size(abuf)); - - /* copy the data to the new buffer and assign it to the dstdb */ - bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen); - dbuf_assign_arcbuf(dstdb, abuf, tx); - dmu_buf_rele(dst_handle, FTAG); -} - /* * When possible directly assign passed loaned arc buffer to a dbuf. * If this is not possible copy the contents of passed arc buf via diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index 9c82cecc6..817c85b93 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. * Copyright 2014 HybridCluster. All rights reserved. * Copyright (c) 2018, loli10K <[email protected]>. All rights reserved. @@ -101,8 +101,6 @@ struct receive_writer_arg { boolean_t done; int err; - /* A map from guid to dataset to help handle dedup'd streams. */ - avl_tree_t *guid_to_ds_map; boolean_t resumable; boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */ boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */ @@ -123,13 +121,6 @@ struct receive_writer_arg { boolean_t or_byteorder; }; -typedef struct guid_map_entry { - uint64_t guid; - boolean_t raw; - objset_t *gme_os; - avl_node_t avlnode; -} guid_map_entry_t; - typedef struct dmu_recv_begin_arg { const char *drba_origin; dmu_recv_cookie_t *drba_cookie; @@ -1213,38 +1204,6 @@ dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin, } static int -guid_compare(const void *arg1, const void *arg2) -{ - const guid_map_entry_t *gmep1 = (const guid_map_entry_t *)arg1; - const guid_map_entry_t *gmep2 = (const guid_map_entry_t *)arg2; - - return (TREE_CMP(gmep1->guid, gmep2->guid)); -} - -static void -free_guid_map_onexit(void *arg) -{ - avl_tree_t *ca = arg; - void *cookie = NULL; - guid_map_entry_t *gmep; - - while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) { - ds_hold_flags_t dsflags = DS_HOLD_FLAG_DECRYPT; - - if (gmep->raw) { - gmep->gme_os->os_raw_receive = B_FALSE; - dsflags &= ~DS_HOLD_FLAG_DECRYPT; - } - - dsl_dataset_disown(gmep->gme_os->os_dsl_dataset, - dsflags, gmep); - kmem_free(gmep, sizeof (guid_map_entry_t)); - } - avl_destroy(ca); - kmem_free(ca, sizeof (avl_tree_t)); -} - -static int receive_read(dmu_recv_cookie_t *drc, int len, void *buf) { int done = 0; @@ -1845,81 +1804,6 @@ receive_process_write_record(struct receive_writer_arg *rwa, return (EAGAIN); } -/* - * Handle a DRR_WRITE_BYREF record. This record is used in dedup'ed - * streams to refer to a copy of the data that is already on the - * system because it came in earlier in the stream. This function - * finds the earlier copy of the data, and uses that copy instead of - * data from the stream to fulfill this write. - */ -noinline static int -receive_write_byref(struct receive_writer_arg *rwa, - struct drr_write_byref *drrwbr) -{ - dmu_tx_t *tx; - int err; - guid_map_entry_t gmesrch; - guid_map_entry_t *gmep; - avl_index_t where; - objset_t *ref_os = NULL; - int flags = DMU_READ_PREFETCH; - dmu_buf_t *dbp; - - if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset) - return (SET_ERROR(EINVAL)); - - /* - * If the GUID of the referenced dataset is different from the - * GUID of the target dataset, find the referenced dataset. - */ - if (drrwbr->drr_toguid != drrwbr->drr_refguid) { - gmesrch.guid = drrwbr->drr_refguid; - if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch, - &where)) == NULL) { - return (SET_ERROR(EINVAL)); - } - ref_os = gmep->gme_os; - } else { - ref_os = rwa->os; - } - - if (drrwbr->drr_object > rwa->max_object) - rwa->max_object = drrwbr->drr_object; - - if (rwa->raw) - flags |= DMU_READ_NO_DECRYPT; - - /* may return either a regular db or an encrypted one */ - err = dmu_buf_hold(ref_os, drrwbr->drr_refobject, - drrwbr->drr_refoffset, FTAG, &dbp, flags); - if (err != 0) - return (err); - - tx = dmu_tx_create(rwa->os); - - dmu_tx_hold_write(tx, drrwbr->drr_object, - drrwbr->drr_offset, drrwbr->drr_length); - err = dmu_tx_assign(tx, TXG_WAIT); - if (err != 0) { - dmu_tx_abort(tx); - return (err); - } - - if (rwa->raw) { - dmu_copy_from_buf(rwa->os, drrwbr->drr_object, - drrwbr->drr_offset, dbp, tx); - } else { - dmu_write(rwa->os, drrwbr->drr_object, - drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx); - } - dmu_buf_rele(dbp, FTAG); - - /* See comment in restore_write. */ - save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx); - dmu_tx_commit(tx); - return (0); -} - static int receive_write_embedded(struct receive_writer_arg *rwa, struct drr_write_embedded *drrwe, void *data) @@ -2607,13 +2491,6 @@ receive_process_record(struct receive_writer_arg *rwa, } break; } - case DRR_WRITE_BYREF: - { - struct drr_write_byref *drrwbr = - &rrd->header.drr_u.drr_write_byref; - err = receive_write_byref(rwa, drrwbr); - break; - } case DRR_WRITE_EMBEDDED: { struct drr_write_embedded *drrwe = @@ -2754,8 +2631,7 @@ resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl) * NB: callers *must* call dmu_recv_end() if this succeeds. */ int -dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, - uint64_t *action_handlep, offset_t *voffp) +dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp) { int err = 0; struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP); @@ -2779,41 +2655,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, ASSERT0(drc->drc_os->os_encrypted && (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)); - /* if this stream is dedup'ed, set up the avl tree for guid mapping */ - if (drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) { - minor_t minor; - - if (cleanup_fd == -1) { - err = SET_ERROR(EBADF); - goto out; - } - err = zfs_onexit_fd_hold(cleanup_fd, &minor); - if (err != 0) { - cleanup_fd = -1; - goto out; - } - - if (*action_handlep == 0) { - rwa->guid_to_ds_map = - kmem_alloc(sizeof (avl_tree_t), KM_SLEEP); - avl_create(rwa->guid_to_ds_map, guid_compare, - sizeof (guid_map_entry_t), - offsetof(guid_map_entry_t, avlnode)); - err = zfs_onexit_add_cb(minor, - free_guid_map_onexit, rwa->guid_to_ds_map, - action_handlep); - if (err != 0) - goto out; - } else { - err = zfs_onexit_cb_data(minor, *action_handlep, - (void **)&rwa->guid_to_ds_map); - if (err != 0) - goto out; - } - - drc->drc_guid_to_ds_map = rwa->guid_to_ds_map; - } - /* handle DSL encryption key payload */ if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { nvlist_t *keynvl = NULL; @@ -2980,9 +2821,6 @@ out: kmem_free(rwa, sizeof (*rwa)); nvlist_free(drc->drc_begin_nvl); - if ((drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) && - (cleanup_fd != -1)) - zfs_onexit_fd_rele(cleanup_fd); if (err != 0) { /* @@ -3083,6 +2921,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dmu_recv_cookie_t *drc = arg; dsl_pool_t *dp = dmu_tx_pool(tx); boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0; + uint64_t newsnapobj; spa_history_log_internal_ds(drc->drc_ds, "finish receiving", tx, "snap=%s", drc->drc_tosnap); @@ -3148,7 +2987,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dsl_dataset_phys(origin_head)->ds_flags &= ~DS_FLAG_INCONSISTENT; - drc->drc_newsnapobj = + newsnapobj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; dsl_dataset_rele(origin_head, FTAG); @@ -3188,7 +3027,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) (void) zap_remove(dp->dp_meta_objset, ds->ds_object, DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx); } - drc->drc_newsnapobj = + newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; } @@ -3203,9 +3042,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) * value. */ if (drc->drc_raw && drc->drc_ivset_guid != 0) { - dmu_object_zapify(dp->dp_meta_objset, drc->drc_newsnapobj, + dmu_object_zapify(dp->dp_meta_objset, newsnapobj, DMU_OT_DSL_DATASET, tx); - VERIFY0(zap_update(dp->dp_meta_objset, drc->drc_newsnapobj, + VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj, DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1, &drc->drc_ivset_guid, tx)); } @@ -3226,54 +3065,6 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) drc->drc_ds = NULL; } -static int -add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj, - boolean_t raw) -{ - dsl_pool_t *dp; - dsl_dataset_t *snapds; - guid_map_entry_t *gmep; - objset_t *os; - ds_hold_flags_t dsflags = (raw) ? 0 : DS_HOLD_FLAG_DECRYPT; - int err; - - ASSERT(guid_map != NULL); - - err = dsl_pool_hold(name, FTAG, &dp); - if (err != 0) - return (err); - gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP); - err = dsl_dataset_own_obj(dp, snapobj, dsflags, gmep, &snapds); - - if (err == 0) { - err = dmu_objset_from_ds(snapds, &os); - if (err != 0) { - dsl_dataset_disown(snapds, dsflags, FTAG); - dsl_pool_rele(dp, FTAG); - kmem_free(gmep, sizeof (*gmep)); - return (err); - } - /* - * If this is a deduplicated raw send stream, we need - * to make sure that we can still read raw blocks from - * earlier datasets in the stream, so we set the - * os_raw_receive flag now. - */ - if (raw) - os->os_raw_receive = B_TRUE; - - gmep->raw = raw; - gmep->guid = dsl_dataset_phys(snapds)->ds_guid; - gmep->gme_os = os; - avl_add(guid_map, gmep); - } else { - kmem_free(gmep, sizeof (*gmep)); - } - - dsl_pool_rele(dp, FTAG); - return (err); -} - static int dmu_recv_end_modified_blocks = 3; static int @@ -3325,12 +3116,6 @@ dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) drc->drc_tofs, drc->drc_tosnap); zvol_create_minor(snapname); kmem_strfree(snapname); - - if (drc->drc_guid_to_ds_map != NULL) { - (void) add_ds_to_guidmap(drc->drc_tofs, - drc->drc_guid_to_ds_map, - drc->drc_newsnapobj, drc->drc_raw); - } } return (error); } diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index 5c53eb637..2104bef71 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -27,7 +27,7 @@ * Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved. * Copyright 2016 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2014 Integros [integros.com] @@ -4765,9 +4765,9 @@ static boolean_t zfs_ioc_recv_inject_err; static int zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops, nvlist_t *localprops, nvlist_t *hidden_args, boolean_t force, - boolean_t resumable, int input_fd, dmu_replay_record_t *begin_record, - int cleanup_fd, uint64_t *read_bytes, uint64_t *errflags, - uint64_t *action_handle, nvlist_t **errors) + boolean_t resumable, int input_fd, + dmu_replay_record_t *begin_record, uint64_t *read_bytes, + uint64_t *errflags, nvlist_t **errors) { dmu_recv_cookie_t drc; int error = 0; @@ -4896,7 +4896,7 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops, nvlist_free(xprops); } - error = dmu_recv_stream(&drc, cleanup_fd, action_handle, &off); + error = dmu_recv_stream(&drc, &off); if (error == 0) { zfsvfs_t *zfsvfs = NULL; @@ -5088,13 +5088,10 @@ out: * zc_cookie file descriptor to recv from * zc_begin_record the BEGIN record of the stream (not byteswapped) * zc_guid force flag - * zc_cleanup_fd cleanup-on-exit file descriptor - * zc_action_handle handle for this guid/ds mapping (or zero on first call) * * outputs: * zc_cookie number of bytes read * zc_obj zprop_errflags_t - * zc_action_handle handle for this guid/ds mapping * zc_nvlist_dst{_size} error for each unapplied received property */ static int @@ -5137,8 +5134,7 @@ zfs_ioc_recv(zfs_cmd_t *zc) error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvdprops, localprops, NULL, zc->zc_guid, B_FALSE, zc->zc_cookie, &begin_record, - zc->zc_cleanup_fd, &zc->zc_cookie, &zc->zc_obj, - &zc->zc_action_handle, &errors); + &zc->zc_cookie, &zc->zc_obj, &errors); nvlist_free(recvdprops); nvlist_free(localprops); @@ -5171,15 +5167,14 @@ zfs_ioc_recv(zfs_cmd_t *zc) * "input_fd" -> file descriptor to read stream from (int32) * (optional) "force" -> force flag (value ignored) * (optional) "resumable" -> resumable flag (value ignored) - * (optional) "cleanup_fd" -> cleanup-on-exit file descriptor - * (optional) "action_handle" -> handle for this guid/ds mapping + * (optional) "cleanup_fd" -> unused + * (optional) "action_handle" -> unused * (optional) "hidden_args" -> { "wkeydata" -> value } * } * * outnvl: { * "read_bytes" -> number of bytes read * "error_flags" -> zprop_errflags_t - * "action_handle" -> handle for this guid/ds mapping * "errors" -> error for each unapplied received property (nvlist) * } */ @@ -5212,11 +5207,9 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) char tofs[ZFS_MAX_DATASET_NAME_LEN]; boolean_t force; boolean_t resumable; - uint64_t action_handle = 0; uint64_t read_bytes = 0; uint64_t errflags = 0; int input_fd = -1; - int cleanup_fd = -1; int error; snapname = fnvlist_lookup_string(innvl, "snapname"); @@ -5244,14 +5237,6 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) force = nvlist_exists(innvl, "force"); resumable = nvlist_exists(innvl, "resumable"); - error = nvlist_lookup_int32(innvl, "cleanup_fd", &cleanup_fd); - if (error && error != ENOENT) - return (error); - - error = nvlist_lookup_uint64(innvl, "action_handle", &action_handle); - if (error && error != ENOENT) - return (error); - /* we still use "props" here for backwards compatibility */ error = nvlist_lookup_nvlist(innvl, "props", &recvprops); if (error && error != ENOENT) @@ -5266,12 +5251,11 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) return (error); error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvprops, localprops, - hidden_args, force, resumable, input_fd, begin_record, cleanup_fd, - &read_bytes, &errflags, &action_handle, &errors); + hidden_args, force, resumable, input_fd, begin_record, + &read_bytes, &errflags, &errors); fnvlist_add_uint64(outnvl, "read_bytes", read_bytes); fnvlist_add_uint64(outnvl, "error_flags", errflags); - fnvlist_add_uint64(outnvl, "action_handle", action_handle); fnvlist_add_nvlist(outnvl, "errors", errors); nvlist_free(errors); diff --git a/module/zfs/zfs_onexit.c b/module/zfs/zfs_onexit.c index bf86446d4..2a1332e71 100644 --- a/module/zfs/zfs_onexit.c +++ b/module/zfs/zfs_onexit.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2013, 2020 by Delphix. All rights reserved. */ #include <sys/types.h> @@ -171,80 +171,3 @@ zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, return (0); } - -static zfs_onexit_action_node_t * -zfs_onexit_find_cb(zfs_onexit_t *zo, uint64_t action_handle) -{ - zfs_onexit_action_node_t *match; - zfs_onexit_action_node_t *ap; - list_t *l; - - ASSERT(MUTEX_HELD(&zo->zo_lock)); - - match = (zfs_onexit_action_node_t *)(uintptr_t)action_handle; - l = &zo->zo_actions; - for (ap = list_head(l); ap != NULL; ap = list_next(l, ap)) { - if (match == ap) - break; - } - return (ap); -} - -/* - * Delete the callback, triggering it first if 'fire' is set. - */ -int -zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire) -{ - zfs_onexit_t *zo; - zfs_onexit_action_node_t *ap; - int error; - - error = zfs_onexit_minor_to_state(minor, &zo); - if (error) - return (error); - - mutex_enter(&zo->zo_lock); - ap = zfs_onexit_find_cb(zo, action_handle); - if (ap != NULL) { - list_remove(&zo->zo_actions, ap); - mutex_exit(&zo->zo_lock); - if (fire) - ap->za_func(ap->za_data); - kmem_free(ap, sizeof (zfs_onexit_action_node_t)); - } else { - mutex_exit(&zo->zo_lock); - error = SET_ERROR(ENOENT); - } - - return (error); -} - -/* - * Return the data associated with this callback. This allows consumers - * of the cleanup-on-exit interfaces to stash kernel data across system - * calls, knowing that it will be cleaned up if the calling process exits. - */ -int -zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data) -{ - zfs_onexit_t *zo; - zfs_onexit_action_node_t *ap; - int error; - - *data = NULL; - - error = zfs_onexit_minor_to_state(minor, &zo); - if (error) - return (error); - - mutex_enter(&zo->zo_lock); - ap = zfs_onexit_find_cb(zo, action_handle); - if (ap != NULL) - *data = ap->za_data; - else - error = SET_ERROR(ENOENT); - mutex_exit(&zo->zo_lock); - - return (error); -} diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index b75be778b..6f83dcd8b 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos'] tags = ['functional', 'rootpool'] [tests/functional/rsend] -tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', - 'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', - 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', - 'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg', - 'rsend_019_pos', 'rsend_020_pos', +tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', + 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos', + 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos', + 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos', + 'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos', 'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props', 'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump', 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', - 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD', + 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', 'send-c_recv_dedup', 'send_encrypted_hierarchy', 'send_encrypted_props', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', - 'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol', + 'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol', 'send_partial_dataset'] tags = ['functional', 'rsend'] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index 8e16f8847..7728a6481 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend dist_pkgdata_SCRIPTS = \ setup.ksh \ cleanup.ksh \ + recv_dedup.ksh \ + recv_dedup_encrypted_zvol.ksh \ rsend_001_pos.ksh \ rsend_002_pos.ksh \ rsend_003_pos.ksh \ @@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \ send_encrypted_hierarchy.ksh \ send_encrypted_props.ksh \ send_encrypted_truncated_files.ksh \ - send-cD.ksh \ send-c_embedded_blocks.ksh \ send-c_incremental.ksh \ send-c_lz4_disabled.ksh \ @@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \ send_holds.ksh \ send_hole_birth.ksh \ send_mixed_raw.ksh \ - send-wDR_encrypted_zvol.ksh + send-wR_encrypted_zvol.ksh dist_pkgdata_DATA = \ + dedup.zsend.bz2 \ + dedup_encrypted_zvol.bz2 \ + dedup_encrypted_zvol.zsend.bz2 \ + fs.tar.gz \ rsend.cfg \ rsend.kshlib diff --git a/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 Binary files differnew file mode 100644 index 000000000..585e14852 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 diff --git a/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 Binary files differnew file mode 100644 index 000000000..73a5742fc --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 diff --git a/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 Binary files differnew file mode 100644 index 000000000..04a6cb53f --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 diff --git a/tests/zfs-tests/tests/functional/rsend/fs.tar.gz b/tests/zfs-tests/tests/functional/rsend/fs.tar.gz Binary files differnew file mode 100644 index 000000000..cb6861c15 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/fs.tar.gz diff --git a/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh b/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh new file mode 100755 index 000000000..e6e282a1c --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh @@ -0,0 +1,53 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verifies that we can receive a dedup send stream by processing it with +# "zstream redup". +# + +verify_runnable "both" + +function cleanup +{ + destroy_dataset $TESTPOOL/recv "-r" + rm -r /$TESTPOOL/tar + rm $sendfile +} +log_onexit cleanup + +log_assert "Verify zfs can receive dedup send streams with 'zstream redup'" + +typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2 +typeset sendfile=/$TESTPOOL/dedup.zsend +typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz + +log_must eval "bzcat <$sendfile_compressed >$sendfile" +log_must zfs create $TESTPOOL/recv +log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv" + +log_must mkdir /$TESTPOOL/tar +log_must tar --directory /$TESTPOOL/tar -xzf $tarfile +log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv + +log_pass "zfs can receive dedup send streams with 'zstream redup'" diff --git a/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh b/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh new file mode 100755 index 000000000..569fcd893 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh @@ -0,0 +1,60 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verifies that we can receive a dedup send stream of a zvol by processing it +# with "zstream redup". +# + +verify_runnable "both" + +function cleanup +{ + destroy_dataset $TESTPOOL/recv "-r" + rm $sendfile + rm $volfile + rm $keyfile +} +log_onexit cleanup + +log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams" + +typeset keyfile=/$TESTPOOL/pkey +typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv +typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 +typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend +typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2 +typeset volfile=/$TESTPOOL/dedup_encrypted_zvol + +log_must eval "echo 'password' > $keyfile" + +log_must eval "bzcat <$sendfile_compressed >$sendfile" +log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv" + +log_must zfs load-key $TESTPOOL/recv +block_device_wait + +log_must eval "bzcat <$volfile_compressed >$volfile" +log_must diff $volfile $recvdev + +log_pass "zfs can receive raw, recursive, and deduplicated send streams" diff --git a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh deleted file mode 100755 index fcbec2d9e..000000000 --- a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/ksh -p - -# -# This file and its contents are supplied under the terms of the -# Common Development and Distribution License ("CDDL"), version 1.0. -# You may only use this file in accordance with the terms of version -# 1.0 of the CDDL. -# -# A full copy of the text of the CDDL should have accompanied this -# source. A copy of the CDDL is also available via the Internet at -# http://www.illumos.org/license/CDDL. -# - -# -# Copyright (c) 2015, 2018 by Delphix. All rights reserved. -# - -. $STF_SUITE/tests/functional/rsend/rsend.kshlib - -# -# Description: -# Verify that the -c and -D flags do not interfere with each other. -# -# Strategy: -# 1. Write unique data to a filesystem and create a compressed, deduplicated -# full stream. -# 2. Verify that the stream and send dataset show the same size -# 3. Make several copies of the original data, and create both full and -# incremental compressed, deduplicated send streams -# 4. Verify the full stream is no bigger than the stream from step 1 -# 5. Verify the streams can be received correctly. -# - -verify_runnable "both" - -log_assert "Verify that the -c and -D flags do not interfere with each other" -log_onexit cleanup_pool $POOL2 - -typeset sendfs=$POOL2/sendfs -typeset recvfs=$POOL2/recvfs -typeset stream0=$BACKDIR/stream.0 -typeset stream1=$BACKDIR/stream.1 -typeset inc=$BACKDIR/stream.inc - -log_must zfs create -o compress=lz4 $sendfs -log_must zfs create -o compress=lz4 $recvfs -typeset dir=$(get_prop mountpoint $sendfs) -# Don't use write_compressible: we want compressible but undeduplicable data. -log_must eval "dd if=/dev/urandom bs=1024k count=4 | base64 >$dir/file" -log_must zfs snapshot $sendfs@snap0 -log_must eval "zfs send -D -c $sendfs@snap0 >$stream0" - -# The stream size should match at this point because the data is all unique -verify_stream_size $stream0 $sendfs - -for i in {0..3}; do - log_must cp $dir/file $dir/file.$i -done -log_must zfs snapshot $sendfs@snap1 - -# The stream sizes should match, since the second stream contains no new blocks -log_must eval "zfs send -D -c $sendfs@snap1 >$stream1" -typeset size0=$(stat_size $stream0) -typeset size1=$(stat_size $stream1) -within_percent $size0 $size1 90 || log_fail "$size0 and $size1" - -# make sure the receive works correctly. -log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc" -log_must eval "zfs recv -d $recvfs <$stream0" -log_must eval "zfs recv -d $recvfs <$inc" -cmp_ds_cont $sendfs $recvfs - -# check receive with redup. -log_must zfs destroy -r $recvfs -log_must zfs create -o compress=lz4 $recvfs -log_must eval "zstream redup $stream0 | zfs recv -d $recvfs" -log_must eval "zstream redup $inc | zfs recv -d $recvfs" -cmp_ds_cont $sendfs $recvfs - -# The size of the incremental should be the same as the initial send. -typeset size2=$(stat_size $inc) -within_percent $size0 $size2 90 || log_fail "$size0 and $size1" - -# The redup'ed size should be 4x -typeset size3=$(zstream redup $inc | wc -c) -let size4=size0*4 -within_percent $size4 $size3 90 || log_fail "$size4 and $size3" - -log_pass "The -c and -D flags do not interfere with each other" diff --git a/tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh b/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh index f9cfeee21..b95fc3da3 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh @@ -16,20 +16,21 @@ # # Copyright (c) 2018 by Datto Inc. All rights reserved. +# Copyright (c) 2020 by Delphix. All rights reserved. # . $STF_SUITE/tests/functional/rsend/rsend.kshlib # # DESCRIPTION: -# Verify that zvols with dedup=on and encryption=on can be sent and received -# with a deduplicated raw send stream. +# Verify that zvols with encryption=on can be sent and received with a raw +# send stream. # # STRATEGY: -# 1. Create a zvol with dedup and encryption on and put a filesystem on it +# 1. Create a zvol with encryption on and put a filesystem on it # 2. Copy a file into the zvol a few times and take a snapshot # 3. Repeat step 2 a few times to create more snapshots -# 4. Send all snapshots in a recursive, raw, deduplicated send stream +# 4. Send all snapshots in a recursive, raw send stream # 5. Mount the received zvol and verify that all of the data there is correct # @@ -48,7 +49,7 @@ function cleanup } log_onexit cleanup -log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams" +log_assert "Verify zfs can receive raw, recursive send streams" typeset keyfile=/$TESTPOOL/pkey typeset snap_count=5 @@ -93,7 +94,7 @@ for ((i = 1; i <= $snap_count; i++)); do log_must mount $remount_rw $zdev $mntpnt done -log_must eval "zfs send -wDR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile" +log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile" log_must eval "zfs recv $TESTPOOL/recv < $sendfile" log_must zfs load-key $TESTPOOL/recv block_device_wait @@ -104,4 +105,4 @@ md5_1=$(cat $mntpnt/* | md5digest) md5_2=$(cat $recvmnt/* | md5digest) [[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2" -log_pass "zfs can receive raw, recursive, and deduplicated send streams" +log_pass "zfs can receive raw, recursive send streams" diff --git a/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh b/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh index 8a0971596..d5eb9a0ed 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh @@ -13,6 +13,7 @@ # # Copyright (c) 2019 Datto Inc. +# Copyright (c) 2020 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -98,7 +99,6 @@ set -A badargs \ "-R $POOL/recvfs" \ "-p $POOL/recvfs" \ "-I $POOL/recvfs" \ - "-D $POOL/recvfs" \ "-h $POOL/recvfs" while (( i < ${#badargs[*]} )) |