summaryrefslogtreecommitdiffstats
path: root/module/zfs
diff options
context:
space:
mode:
authorMatthew Macy <[email protected]>2019-09-06 11:26:26 -0700
committerBrian Behlendorf <[email protected]>2019-09-06 11:26:26 -0700
commitbced7e3aaa3cf54d5e8e4f94e067144b27cb744b (patch)
tree729dac6996f4f11b88bc3a831b2b8d6852e6fbb6 /module/zfs
parent870e7a52c105f26ef4254b90230d396f4ce39ea7 (diff)
OpenZFS restructuring - move platform specific sources
Move platform specific Linux source under module/os/linux/ and update the build system accordingly. Additional code restructuring will follow to make the common code fully portable. Reviewed-by: Jorgen Lundman <[email protected]> Reviewed-by: Igor Kozhukhov <[email protected]> Reviewed-by: Brian Behlendorf <[email protected]> Signed-off-by: Matthew Macy <[email protected]> Closes #9206
Diffstat (limited to 'module/zfs')
-rw-r--r--module/zfs/Makefile.in43
-rw-r--r--module/zfs/abd.c1638
-rw-r--r--module/zfs/gzip.c2
-rw-r--r--module/zfs/policy.c355
-rw-r--r--module/zfs/qat.c105
-rw-r--r--module/zfs/qat.h204
-rw-r--r--module/zfs/qat_compress.c574
-rw-r--r--module/zfs/qat_crypt.c631
-rw-r--r--module/zfs/sha256.c2
-rw-r--r--module/zfs/spa_misc.c2
-rw-r--r--module/zfs/spa_stats.c1034
-rw-r--r--module/zfs/vdev_disk.c954
-rw-r--r--module/zfs/vdev_file.c331
-rw-r--r--module/zfs/zfs_acl.c2816
-rw-r--r--module/zfs/zfs_ctldir.c1240
-rw-r--r--module/zfs/zfs_debug.c253
-rw-r--r--module/zfs/zfs_dir.c1205
-rw-r--r--module/zfs/zfs_sysfs.c661
-rw-r--r--module/zfs/zfs_vfsops.c2562
-rw-r--r--module/zfs/zfs_vnops.c5275
-rw-r--r--module/zfs/zfs_znode.c2234
-rw-r--r--module/zfs/zio_crypt.c2036
-rw-r--r--module/zfs/zpl_ctldir.c572
-rw-r--r--module/zfs/zpl_export.c177
-rw-r--r--module/zfs/zpl_file.c1075
-rw-r--r--module/zfs/zpl_inode.c826
-rw-r--r--module/zfs/zpl_super.c426
-rw-r--r--module/zfs/zpl_xattr.c1548
28 files changed, 14 insertions, 28767 deletions
diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in
index 5adea9fb5..7c560fad7 100644
--- a/module/zfs/Makefile.in
+++ b/module/zfs/Makefile.in
@@ -16,18 +16,17 @@ endif
# Suppress unused but set variable warnings often due to ASSERTs
ccflags-y += $(NO_UNUSED_BUT_SET_VARIABLE)
-$(MODULE)-objs += abd.o
$(MODULE)-objs += aggsum.o
$(MODULE)-objs += arc.o
$(MODULE)-objs += blkptr.o
$(MODULE)-objs += bplist.o
$(MODULE)-objs += bpobj.o
-$(MODULE)-objs += cityhash.o
-$(MODULE)-objs += dbuf.o
-$(MODULE)-objs += dbuf_stats.o
$(MODULE)-objs += bptree.o
$(MODULE)-objs += bqueue.o
+$(MODULE)-objs += cityhash.o
$(MODULE)-objs += dataset_kstats.o
+$(MODULE)-objs += dbuf.o
+$(MODULE)-objs += dbuf_stats.o
$(MODULE)-objs += ddt.o
$(MODULE)-objs += ddt_zap.o
$(MODULE)-objs += dmu.o
@@ -42,28 +41,29 @@ $(MODULE)-objs += dmu_tx.o
$(MODULE)-objs += dmu_zfetch.o
$(MODULE)-objs += dnode.o
$(MODULE)-objs += dnode_sync.o
+$(MODULE)-objs += dsl_bookmark.o
+$(MODULE)-objs += dsl_crypt.o
$(MODULE)-objs += dsl_dataset.o
$(MODULE)-objs += dsl_deadlist.o
$(MODULE)-objs += dsl_deleg.o
-$(MODULE)-objs += dsl_bookmark.o
+$(MODULE)-objs += dsl_destroy.o
$(MODULE)-objs += dsl_dir.o
-$(MODULE)-objs += dsl_crypt.o
$(MODULE)-objs += dsl_pool.o
$(MODULE)-objs += dsl_prop.o
$(MODULE)-objs += dsl_scan.o
$(MODULE)-objs += dsl_synctask.o
+$(MODULE)-objs += dsl_userhold.o
$(MODULE)-objs += edonr_zfs.o
$(MODULE)-objs += fm.o
$(MODULE)-objs += gzip.o
$(MODULE)-objs += hkdf.o
-$(MODULE)-objs += lzjb.o
$(MODULE)-objs += lz4.o
+$(MODULE)-objs += lzjb.o
$(MODULE)-objs += metaslab.o
$(MODULE)-objs += mmp.o
$(MODULE)-objs += multilist.o
$(MODULE)-objs += objlist.o
$(MODULE)-objs += pathname.o
-$(MODULE)-objs += policy.o
$(MODULE)-objs += range_tree.o
$(MODULE)-objs += refcount.o
$(MODULE)-objs += rrwlock.o
@@ -78,17 +78,14 @@ $(MODULE)-objs += spa_errlog.o
$(MODULE)-objs += spa_history.o
$(MODULE)-objs += spa_log_spacemap.o
$(MODULE)-objs += spa_misc.o
-$(MODULE)-objs += spa_stats.o
$(MODULE)-objs += space_map.o
$(MODULE)-objs += space_reftree.o
-$(MODULE)-objs += txg.o
$(MODULE)-objs += trace.o
+$(MODULE)-objs += txg.o
$(MODULE)-objs += uberblock.o
$(MODULE)-objs += unique.o
$(MODULE)-objs += vdev.o
$(MODULE)-objs += vdev_cache.o
-$(MODULE)-objs += vdev_disk.o
-$(MODULE)-objs += vdev_file.o
$(MODULE)-objs += vdev_indirect.o
$(MODULE)-objs += vdev_indirect_births.o
$(MODULE)-objs += vdev_indirect_mapping.o
@@ -112,11 +109,7 @@ $(MODULE)-objs += zcp_global.o
$(MODULE)-objs += zcp_iter.o
$(MODULE)-objs += zcp_synctask.o
$(MODULE)-objs += zfeature.o
-$(MODULE)-objs += zfs_acl.o
$(MODULE)-objs += zfs_byteswap.o
-$(MODULE)-objs += zfs_ctldir.o
-$(MODULE)-objs += zfs_debug.o
-$(MODULE)-objs += zfs_dir.o
$(MODULE)-objs += zfs_fm.o
$(MODULE)-objs += zfs_fuid.o
$(MODULE)-objs += zfs_ioctl.o
@@ -126,31 +119,15 @@ $(MODULE)-objs += zfs_ratelimit.o
$(MODULE)-objs += zfs_replay.o
$(MODULE)-objs += zfs_rlock.o
$(MODULE)-objs += zfs_sa.o
-$(MODULE)-objs += zfs_sysfs.o
-$(MODULE)-objs += zfs_vfsops.o
-$(MODULE)-objs += zfs_vnops.o
-$(MODULE)-objs += zfs_znode.o
$(MODULE)-objs += zil.o
$(MODULE)-objs += zio.o
$(MODULE)-objs += zio_checksum.o
$(MODULE)-objs += zio_compress.o
-$(MODULE)-objs += zio_crypt.o
$(MODULE)-objs += zio_inject.o
$(MODULE)-objs += zle.o
-$(MODULE)-objs += zpl_ctldir.o
-$(MODULE)-objs += zpl_export.o
-$(MODULE)-objs += zpl_file.o
-$(MODULE)-objs += zpl_inode.o
-$(MODULE)-objs += zpl_super.o
-$(MODULE)-objs += zpl_xattr.o
$(MODULE)-objs += zrlock.o
$(MODULE)-objs += zthr.o
$(MODULE)-objs += zvol.o
-$(MODULE)-objs += dsl_destroy.o
-$(MODULE)-objs += dsl_userhold.o
-$(MODULE)-objs += qat.o
-$(MODULE)-objs += qat_compress.o
-$(MODULE)-objs += qat_crypt.o
# Suppress incorrect warnings from versions of objtool which are not
# aware of x86 EVEX prefix instructions used for AVX512.
@@ -165,3 +142,5 @@ $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx512bw.o
$(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neon.o
$(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neonx2.o
+
+-include @abs_top_builddir@/module/os/linux/zfs/Makefile
diff --git a/module/zfs/abd.c b/module/zfs/abd.c
deleted file mode 100644
index ac6b0b742..000000000
--- a/module/zfs/abd.c
+++ /dev/null
@@ -1,1638 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
- * Copyright (c) 2019 by Delphix. All rights reserved.
- */
-
-/*
- * ARC buffer data (ABD).
- *
- * ABDs are an abstract data structure for the ARC which can use two
- * different ways of storing the underlying data:
- *
- * (a) Linear buffer. In this case, all the data in the ABD is stored in one
- * contiguous buffer in memory (from a zio_[data_]buf_* kmem cache).
- *
- * +-------------------+
- * | ABD (linear) |
- * | abd_flags = ... |
- * | abd_size = ... | +--------------------------------+
- * | abd_buf ------------->| raw buffer of size abd_size |
- * +-------------------+ +--------------------------------+
- * no abd_chunks
- *
- * (b) Scattered buffer. In this case, the data in the ABD is split into
- * equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers
- * to the chunks recorded in an array at the end of the ABD structure.
- *
- * +-------------------+
- * | ABD (scattered) |
- * | abd_flags = ... |
- * | abd_size = ... |
- * | abd_offset = 0 | +-----------+
- * | abd_chunks[0] ----------------------------->| chunk 0 |
- * | abd_chunks[1] ---------------------+ +-----------+
- * | ... | | +-----------+
- * | abd_chunks[N-1] ---------+ +------->| chunk 1 |
- * +-------------------+ | +-----------+
- * | ...
- * | +-----------+
- * +----------------->| chunk N-1 |
- * +-----------+
- *
- * Linear buffers act exactly like normal buffers and are always mapped into the
- * kernel's virtual memory space, while scattered ABD data chunks are allocated
- * as physical pages and then mapped in only while they are actually being
- * accessed through one of the abd_* library functions. Using scattered ABDs
- * provides several benefits:
- *
- * (1) They avoid use of kmem_*, preventing performance problems where running
- * kmem_reap on very large memory systems never finishes and causes
- * constant TLB shootdowns.
- *
- * (2) Fragmentation is less of an issue since when we are at the limit of
- * allocatable space, we won't have to search around for a long free
- * hole in the VA space for large ARC allocations. Each chunk is mapped in
- * individually, so even if we are using HIGHMEM (see next point) we
- * wouldn't need to worry about finding a contiguous address range.
- *
- * (3) If we are not using HIGHMEM, then all physical memory is always
- * mapped into the kernel's address space, so we also avoid the map /
- * unmap costs on each ABD access.
- *
- * If we are not using HIGHMEM, scattered buffers which have only one chunk
- * can be treated as linear buffers, because they are contiguous in the
- * kernel's virtual address space. See abd_alloc_pages() for details.
- *
- * It is possible to make all ABDs linear by setting zfs_abd_scatter_enabled to
- * B_FALSE.
- *
- * In addition to directly allocating a linear or scattered ABD, it is also
- * possible to create an ABD by requesting the "sub-ABD" starting at an offset
- * within an existing ABD. In linear buffers this is simple (set abd_buf of
- * the new ABD to the starting point within the original raw buffer), but
- * scattered ABDs are a little more complex. The new ABD makes a copy of the
- * relevant abd_chunks pointers (but not the underlying data). However, to
- * provide arbitrary rather than only chunk-aligned starting offsets, it also
- * tracks an abd_offset field which represents the starting point of the data
- * within the first chunk in abd_chunks. For both linear and scattered ABDs,
- * creating an offset ABD marks the original ABD as the offset's parent, and the
- * original ABD's abd_children refcount is incremented. This data allows us to
- * ensure the root ABD isn't deleted before its children.
- *
- * Most consumers should never need to know what type of ABD they're using --
- * the ABD public API ensures that it's possible to transparently switch from
- * using a linear ABD to a scattered one when doing so would be beneficial.
- *
- * If you need to use the data within an ABD directly, if you know it's linear
- * (because you allocated it) you can use abd_to_buf() to access the underlying
- * raw buffer. Otherwise, you should use one of the abd_borrow_buf* functions
- * which will allocate a raw buffer if necessary. Use the abd_return_buf*
- * functions to return any raw buffers that are no longer necessary when you're
- * done using them.
- *
- * There are a variety of ABD APIs that implement basic buffer operations:
- * compare, copy, read, write, and fill with zeroes. If you need a custom
- * function which progressively accesses the whole ABD, use the abd_iterate_*
- * functions.
- */
-
-#include <sys/abd.h>
-#include <sys/param.h>
-#include <sys/zio.h>
-#include <sys/zfs_context.h>
-#include <sys/zfs_znode.h>
-#ifdef _KERNEL
-#include <linux/scatterlist.h>
-#include <linux/kmap_compat.h>
-#else
-#define MAX_ORDER 1
-#endif
-
-typedef struct abd_stats {
- kstat_named_t abdstat_struct_size;
- kstat_named_t abdstat_linear_cnt;
- kstat_named_t abdstat_linear_data_size;
- kstat_named_t abdstat_scatter_cnt;
- kstat_named_t abdstat_scatter_data_size;
- kstat_named_t abdstat_scatter_chunk_waste;
- kstat_named_t abdstat_scatter_orders[MAX_ORDER];
- kstat_named_t abdstat_scatter_page_multi_chunk;
- kstat_named_t abdstat_scatter_page_multi_zone;
- kstat_named_t abdstat_scatter_page_alloc_retry;
- kstat_named_t abdstat_scatter_sg_table_retry;
-} abd_stats_t;
-
-static abd_stats_t abd_stats = {
- /* Amount of memory occupied by all of the abd_t struct allocations */
- { "struct_size", KSTAT_DATA_UINT64 },
- /*
- * The number of linear ABDs which are currently allocated, excluding
- * ABDs which don't own their data (for instance the ones which were
- * allocated through abd_get_offset() and abd_get_from_buf()). If an
- * ABD takes ownership of its buf then it will become tracked.
- */
- { "linear_cnt", KSTAT_DATA_UINT64 },
- /* Amount of data stored in all linear ABDs tracked by linear_cnt */
- { "linear_data_size", KSTAT_DATA_UINT64 },
- /*
- * The number of scatter ABDs which are currently allocated, excluding
- * ABDs which don't own their data (for instance the ones which were
- * allocated through abd_get_offset()).
- */
- { "scatter_cnt", KSTAT_DATA_UINT64 },
- /* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
- { "scatter_data_size", KSTAT_DATA_UINT64 },
- /*
- * The amount of space wasted at the end of the last chunk across all
- * scatter ABDs tracked by scatter_cnt.
- */
- { "scatter_chunk_waste", KSTAT_DATA_UINT64 },
- /*
- * The number of compound allocations of a given order. These
- * allocations are spread over all currently allocated ABDs, and
- * act as a measure of memory fragmentation.
- */
- { { "scatter_order_N", KSTAT_DATA_UINT64 } },
- /*
- * The number of scatter ABDs which contain multiple chunks.
- * ABDs are preferentially allocated from the minimum number of
- * contiguous multi-page chunks, a single chunk is optimal.
- */
- { "scatter_page_multi_chunk", KSTAT_DATA_UINT64 },
- /*
- * The number of scatter ABDs which are split across memory zones.
- * ABDs are preferentially allocated using pages from a single zone.
- */
- { "scatter_page_multi_zone", KSTAT_DATA_UINT64 },
- /*
- * The total number of retries encountered when attempting to
- * allocate the pages to populate the scatter ABD.
- */
- { "scatter_page_alloc_retry", KSTAT_DATA_UINT64 },
- /*
- * The total number of retries encountered when attempting to
- * allocate the sg table for an ABD.
- */
- { "scatter_sg_table_retry", KSTAT_DATA_UINT64 },
-};
-
-#define ABDSTAT(stat) (abd_stats.stat.value.ui64)
-#define ABDSTAT_INCR(stat, val) \
- atomic_add_64(&abd_stats.stat.value.ui64, (val))
-#define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1)
-#define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1)
-
-#define ABD_SCATTER(abd) (abd->abd_u.abd_scatter)
-#define ABD_BUF(abd) (abd->abd_u.abd_linear.abd_buf)
-#define abd_for_each_sg(abd, sg, n, i) \
- for_each_sg(ABD_SCATTER(abd).abd_sgl, sg, n, i)
-
-/* see block comment above for description */
-int zfs_abd_scatter_enabled = B_TRUE;
-unsigned zfs_abd_scatter_max_order = MAX_ORDER - 1;
-
-/*
- * zfs_abd_scatter_min_size is the minimum allocation size to use scatter
- * ABD's. Smaller allocations will use linear ABD's which uses
- * zio_[data_]buf_alloc().
- *
- * Scatter ABD's use at least one page each, so sub-page allocations waste
- * some space when allocated as scatter (e.g. 2KB scatter allocation wastes
- * half of each page). Using linear ABD's for small allocations means that
- * they will be put on slabs which contain many allocations. This can
- * improve memory efficiency, but it also makes it much harder for ARC
- * evictions to actually free pages, because all the buffers on one slab need
- * to be freed in order for the slab (and underlying pages) to be freed.
- * Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's
- * possible for them to actually waste more memory than scatter (one page per
- * buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th).
- *
- * Spill blocks are typically 512B and are heavily used on systems running
- * selinux with the default dnode size and the `xattr=sa` property set.
- *
- * By default we use linear allocations for 512B and 1KB, and scatter
- * allocations for larger (1.5KB and up).
- */
-int zfs_abd_scatter_min_size = 512 * 3;
-
-static kmem_cache_t *abd_cache = NULL;
-static kstat_t *abd_ksp;
-
-static inline size_t
-abd_chunkcnt_for_bytes(size_t size)
-{
- return (P2ROUNDUP(size, PAGESIZE) / PAGESIZE);
-}
-
-#ifdef _KERNEL
-/*
- * Mark zfs data pages so they can be excluded from kernel crash dumps
- */
-#ifdef _LP64
-#define ABD_FILE_CACHE_PAGE 0x2F5ABDF11ECAC4E
-
-static inline void
-abd_mark_zfs_page(struct page *page)
-{
- get_page(page);
- SetPagePrivate(page);
- set_page_private(page, ABD_FILE_CACHE_PAGE);
-}
-
-static inline void
-abd_unmark_zfs_page(struct page *page)
-{
- set_page_private(page, 0UL);
- ClearPagePrivate(page);
- put_page(page);
-}
-#else
-#define abd_mark_zfs_page(page)
-#define abd_unmark_zfs_page(page)
-#endif /* _LP64 */
-
-#ifndef CONFIG_HIGHMEM
-
-#ifndef __GFP_RECLAIM
-#define __GFP_RECLAIM __GFP_WAIT
-#endif
-
-/*
- * The goal is to minimize fragmentation by preferentially populating ABDs
- * with higher order compound pages from a single zone. Allocation size is
- * progressively decreased until it can be satisfied without performing
- * reclaim or compaction. When necessary this function will degenerate to
- * allocating individual pages and allowing reclaim to satisfy allocations.
- */
-static void
-abd_alloc_pages(abd_t *abd, size_t size)
-{
- struct list_head pages;
- struct sg_table table;
- struct scatterlist *sg;
- struct page *page, *tmp_page = NULL;
- gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
- gfp_t gfp_comp = (gfp | __GFP_NORETRY | __GFP_COMP) & ~__GFP_RECLAIM;
- int max_order = MIN(zfs_abd_scatter_max_order, MAX_ORDER - 1);
- int nr_pages = abd_chunkcnt_for_bytes(size);
- int chunks = 0, zones = 0;
- size_t remaining_size;
- int nid = NUMA_NO_NODE;
- int alloc_pages = 0;
-
- INIT_LIST_HEAD(&pages);
-
- while (alloc_pages < nr_pages) {
- unsigned chunk_pages;
- int order;
-
- order = MIN(highbit64(nr_pages - alloc_pages) - 1, max_order);
- chunk_pages = (1U << order);
-
- page = alloc_pages_node(nid, order ? gfp_comp : gfp, order);
- if (page == NULL) {
- if (order == 0) {
- ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
- schedule_timeout_interruptible(1);
- } else {
- max_order = MAX(0, order - 1);
- }
- continue;
- }
-
- list_add_tail(&page->lru, &pages);
-
- if ((nid != NUMA_NO_NODE) && (page_to_nid(page) != nid))
- zones++;
-
- nid = page_to_nid(page);
- ABDSTAT_BUMP(abdstat_scatter_orders[order]);
- chunks++;
- alloc_pages += chunk_pages;
- }
-
- ASSERT3S(alloc_pages, ==, nr_pages);
-
- while (sg_alloc_table(&table, chunks, gfp)) {
- ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
- schedule_timeout_interruptible(1);
- }
-
- sg = table.sgl;
- remaining_size = size;
- list_for_each_entry_safe(page, tmp_page, &pages, lru) {
- size_t sg_size = MIN(PAGESIZE << compound_order(page),
- remaining_size);
- sg_set_page(sg, page, sg_size, 0);
- abd_mark_zfs_page(page);
- remaining_size -= sg_size;
-
- sg = sg_next(sg);
- list_del(&page->lru);
- }
-
- /*
- * These conditions ensure that a possible transformation to a linear
- * ABD would be valid.
- */
- ASSERT(!PageHighMem(sg_page(table.sgl)));
- ASSERT0(ABD_SCATTER(abd).abd_offset);
-
- if (table.nents == 1) {
- /*
- * Since there is only one entry, this ABD can be represented
- * as a linear buffer. All single-page (4K) ABD's can be
- * represented this way. Some multi-page ABD's can also be
- * represented this way, if we were able to allocate a single
- * "chunk" (higher-order "page" which represents a power-of-2
- * series of physically-contiguous pages). This is often the
- * case for 2-page (8K) ABD's.
- *
- * Representing a single-entry scatter ABD as a linear ABD
- * has the performance advantage of avoiding the copy (and
- * allocation) in abd_borrow_buf_copy / abd_return_buf_copy.
- * A performance increase of around 5% has been observed for
- * ARC-cached reads (of small blocks which can take advantage
- * of this).
- *
- * Note that this optimization is only possible because the
- * pages are always mapped into the kernel's address space.
- * This is not the case for highmem pages, so the
- * optimization can not be made there.
- */
- abd->abd_flags |= ABD_FLAG_LINEAR;
- abd->abd_flags |= ABD_FLAG_LINEAR_PAGE;
- abd->abd_u.abd_linear.abd_sgl = table.sgl;
- abd->abd_u.abd_linear.abd_buf =
- page_address(sg_page(table.sgl));
- } else if (table.nents > 1) {
- ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
- abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
-
- if (zones) {
- ABDSTAT_BUMP(abdstat_scatter_page_multi_zone);
- abd->abd_flags |= ABD_FLAG_MULTI_ZONE;
- }
-
- ABD_SCATTER(abd).abd_sgl = table.sgl;
- ABD_SCATTER(abd).abd_nents = table.nents;
- }
-}
-#else
-/*
- * Allocate N individual pages to construct a scatter ABD. This function
- * makes no attempt to request contiguous pages and requires the minimal
- * number of kernel interfaces. It's designed for maximum compatibility.
- */
-static void
-abd_alloc_pages(abd_t *abd, size_t size)
-{
- struct scatterlist *sg = NULL;
- struct sg_table table;
- struct page *page;
- gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
- int nr_pages = abd_chunkcnt_for_bytes(size);
- int i = 0;
-
- while (sg_alloc_table(&table, nr_pages, gfp)) {
- ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
- schedule_timeout_interruptible(1);
- }
-
- ASSERT3U(table.nents, ==, nr_pages);
- ABD_SCATTER(abd).abd_sgl = table.sgl;
- ABD_SCATTER(abd).abd_nents = nr_pages;
-
- abd_for_each_sg(abd, sg, nr_pages, i) {
- while ((page = __page_cache_alloc(gfp)) == NULL) {
- ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
- schedule_timeout_interruptible(1);
- }
-
- ABDSTAT_BUMP(abdstat_scatter_orders[0]);
- sg_set_page(sg, page, PAGESIZE, 0);
- abd_mark_zfs_page(page);
- }
-
- if (nr_pages > 1) {
- ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
- abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
- }
-}
-#endif /* !CONFIG_HIGHMEM */
-
-static void
-abd_free_pages(abd_t *abd)
-{
- struct scatterlist *sg = NULL;
- struct sg_table table;
- struct page *page;
- int nr_pages = ABD_SCATTER(abd).abd_nents;
- int order, i = 0;
-
- if (abd->abd_flags & ABD_FLAG_MULTI_ZONE)
- ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_zone);
-
- if (abd->abd_flags & ABD_FLAG_MULTI_CHUNK)
- ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);
-
- abd_for_each_sg(abd, sg, nr_pages, i) {
- page = sg_page(sg);
- abd_unmark_zfs_page(page);
- order = compound_order(page);
- __free_pages(page, order);
- ASSERT3U(sg->length, <=, PAGE_SIZE << order);
- ABDSTAT_BUMPDOWN(abdstat_scatter_orders[order]);
- }
-
- table.sgl = ABD_SCATTER(abd).abd_sgl;
- table.nents = table.orig_nents = nr_pages;
- sg_free_table(&table);
-}
-
-#else /* _KERNEL */
-
-#ifndef PAGE_SHIFT
-#define PAGE_SHIFT (highbit64(PAGESIZE)-1)
-#endif
-
-struct page;
-
-#define zfs_kmap_atomic(chunk, km) ((void *)chunk)
-#define zfs_kunmap_atomic(addr, km) do { (void)(addr); } while (0)
-#define local_irq_save(flags) do { (void)(flags); } while (0)
-#define local_irq_restore(flags) do { (void)(flags); } while (0)
-#define nth_page(pg, i) \
- ((struct page *)((void *)(pg) + (i) * PAGESIZE))
-
-struct scatterlist {
- struct page *page;
- int length;
- int end;
-};
-
-static void
-sg_init_table(struct scatterlist *sg, int nr)
-{
- memset(sg, 0, nr * sizeof (struct scatterlist));
- sg[nr - 1].end = 1;
-}
-
-#define for_each_sg(sgl, sg, nr, i) \
- for ((i) = 0, (sg) = (sgl); (i) < (nr); (i)++, (sg) = sg_next(sg))
-
-static inline void
-sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
- unsigned int offset)
-{
- /* currently we don't use offset */
- ASSERT(offset == 0);
- sg->page = page;
- sg->length = len;
-}
-
-static inline struct page *
-sg_page(struct scatterlist *sg)
-{
- return (sg->page);
-}
-
-static inline struct scatterlist *
-sg_next(struct scatterlist *sg)
-{
- if (sg->end)
- return (NULL);
-
- return (sg + 1);
-}
-
-static void
-abd_alloc_pages(abd_t *abd, size_t size)
-{
- unsigned nr_pages = abd_chunkcnt_for_bytes(size);
- struct scatterlist *sg;
- int i;
-
- ABD_SCATTER(abd).abd_sgl = vmem_alloc(nr_pages *
- sizeof (struct scatterlist), KM_SLEEP);
- sg_init_table(ABD_SCATTER(abd).abd_sgl, nr_pages);
-
- abd_for_each_sg(abd, sg, nr_pages, i) {
- struct page *p = umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP);
- sg_set_page(sg, p, PAGESIZE, 0);
- }
- ABD_SCATTER(abd).abd_nents = nr_pages;
-}
-
-static void
-abd_free_pages(abd_t *abd)
-{
- int i, n = ABD_SCATTER(abd).abd_nents;
- struct scatterlist *sg;
-
- abd_for_each_sg(abd, sg, n, i) {
- for (int j = 0; j < sg->length; j += PAGESIZE) {
- struct page *p = nth_page(sg_page(sg), j >> PAGE_SHIFT);
- umem_free(p, PAGESIZE);
- }
- }
-
- vmem_free(ABD_SCATTER(abd).abd_sgl, n * sizeof (struct scatterlist));
-}
-
-#endif /* _KERNEL */
-
-void
-abd_init(void)
-{
- int i;
-
- abd_cache = kmem_cache_create("abd_t", sizeof (abd_t),
- 0, NULL, NULL, NULL, NULL, NULL, 0);
-
- abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
- sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
- if (abd_ksp != NULL) {
- abd_ksp->ks_data = &abd_stats;
- kstat_install(abd_ksp);
-
- for (i = 0; i < MAX_ORDER; i++) {
- snprintf(abd_stats.abdstat_scatter_orders[i].name,
- KSTAT_STRLEN, "scatter_order_%d", i);
- abd_stats.abdstat_scatter_orders[i].data_type =
- KSTAT_DATA_UINT64;
- }
- }
-}
-
-void
-abd_fini(void)
-{
- if (abd_ksp != NULL) {
- kstat_delete(abd_ksp);
- abd_ksp = NULL;
- }
-
- if (abd_cache) {
- kmem_cache_destroy(abd_cache);
- abd_cache = NULL;
- }
-}
-
-static inline void
-abd_verify(abd_t *abd)
-{
- ASSERT3U(abd->abd_size, >, 0);
- ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
- ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
- ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE |
- ABD_FLAG_MULTI_CHUNK | ABD_FLAG_LINEAR_PAGE));
- IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
- IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
- if (abd_is_linear(abd)) {
- ASSERT3P(abd->abd_u.abd_linear.abd_buf, !=, NULL);
- } else {
- size_t n;
- int i = 0;
- struct scatterlist *sg = NULL;
-
- ASSERT3U(ABD_SCATTER(abd).abd_nents, >, 0);
- ASSERT3U(ABD_SCATTER(abd).abd_offset, <,
- ABD_SCATTER(abd).abd_sgl->length);
- n = ABD_SCATTER(abd).abd_nents;
- abd_for_each_sg(abd, sg, n, i) {
- ASSERT3P(sg_page(sg), !=, NULL);
- }
- }
-}
-
-static inline abd_t *
-abd_alloc_struct(void)
-{
- abd_t *abd = kmem_cache_alloc(abd_cache, KM_PUSHPAGE);
-
- ASSERT3P(abd, !=, NULL);
- ABDSTAT_INCR(abdstat_struct_size, sizeof (abd_t));
-
- return (abd);
-}
-
-static inline void
-abd_free_struct(abd_t *abd)
-{
- kmem_cache_free(abd_cache, abd);
- ABDSTAT_INCR(abdstat_struct_size, -(int)sizeof (abd_t));
-}
-
-/*
- * Allocate an ABD, along with its own underlying data buffers. Use this if you
- * don't care whether the ABD is linear or not.
- */
-abd_t *
-abd_alloc(size_t size, boolean_t is_metadata)
-{
- /* see the comment above zfs_abd_scatter_min_size */
- if (!zfs_abd_scatter_enabled || size < zfs_abd_scatter_min_size)
- return (abd_alloc_linear(size, is_metadata));
-
- VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);
-
- abd_t *abd = abd_alloc_struct();
- abd->abd_flags = ABD_FLAG_OWNER;
- abd->abd_u.abd_scatter.abd_offset = 0;
- abd_alloc_pages(abd, size);
-
- if (is_metadata) {
- abd->abd_flags |= ABD_FLAG_META;
- }
- abd->abd_size = size;
- abd->abd_parent = NULL;
- zfs_refcount_create(&abd->abd_children);
-
- ABDSTAT_BUMP(abdstat_scatter_cnt);
- ABDSTAT_INCR(abdstat_scatter_data_size, size);
- ABDSTAT_INCR(abdstat_scatter_chunk_waste,
- P2ROUNDUP(size, PAGESIZE) - size);
-
- return (abd);
-}
-
-static void
-abd_free_scatter(abd_t *abd)
-{
- abd_free_pages(abd);
-
- zfs_refcount_destroy(&abd->abd_children);
- ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
- ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
- ABDSTAT_INCR(abdstat_scatter_chunk_waste,
- (int)abd->abd_size - (int)P2ROUNDUP(abd->abd_size, PAGESIZE));
-
- abd_free_struct(abd);
-}
-
-/*
- * Allocate an ABD that must be linear, along with its own underlying data
- * buffer. Only use this when it would be very annoying to write your ABD
- * consumer with a scattered ABD.
- */
-abd_t *
-abd_alloc_linear(size_t size, boolean_t is_metadata)
-{
- abd_t *abd = abd_alloc_struct();
-
- VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);
-
- abd->abd_flags = ABD_FLAG_LINEAR | ABD_FLAG_OWNER;
- if (is_metadata) {
- abd->abd_flags |= ABD_FLAG_META;
- }
- abd->abd_size = size;
- abd->abd_parent = NULL;
- zfs_refcount_create(&abd->abd_children);
-
- if (is_metadata) {
- abd->abd_u.abd_linear.abd_buf = zio_buf_alloc(size);
- } else {
- abd->abd_u.abd_linear.abd_buf = zio_data_buf_alloc(size);
- }
-
- ABDSTAT_BUMP(abdstat_linear_cnt);
- ABDSTAT_INCR(abdstat_linear_data_size, size);
-
- return (abd);
-}
-
-static void
-abd_free_linear(abd_t *abd)
-{
- if (abd_is_linear_page(abd)) {
- /* Transform it back into a scatter ABD for freeing */
- struct scatterlist *sg = abd->abd_u.abd_linear.abd_sgl;
- abd->abd_flags &= ~ABD_FLAG_LINEAR;
- abd->abd_flags &= ~ABD_FLAG_LINEAR_PAGE;
- ABD_SCATTER(abd).abd_nents = 1;
- ABD_SCATTER(abd).abd_offset = 0;
- ABD_SCATTER(abd).abd_sgl = sg;
- abd_free_scatter(abd);
- return;
- }
- if (abd->abd_flags & ABD_FLAG_META) {
- zio_buf_free(abd->abd_u.abd_linear.abd_buf, abd->abd_size);
- } else {
- zio_data_buf_free(abd->abd_u.abd_linear.abd_buf, abd->abd_size);
- }
-
- zfs_refcount_destroy(&abd->abd_children);
- ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
- ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
-
- abd_free_struct(abd);
-}
-
-/*
- * Free an ABD. Only use this on ABDs allocated with abd_alloc() or
- * abd_alloc_linear().
- */
-void
-abd_free(abd_t *abd)
-{
- abd_verify(abd);
- ASSERT3P(abd->abd_parent, ==, NULL);
- ASSERT(abd->abd_flags & ABD_FLAG_OWNER);
- if (abd_is_linear(abd))
- abd_free_linear(abd);
- else
- abd_free_scatter(abd);
-}
-
-/*
- * Allocate an ABD of the same format (same metadata flag, same scatterize
- * setting) as another ABD.
- */
-abd_t *
-abd_alloc_sametype(abd_t *sabd, size_t size)
-{
- boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0;
- if (abd_is_linear(sabd) &&
- !abd_is_linear_page(sabd)) {
- return (abd_alloc_linear(size, is_metadata));
- } else {
- return (abd_alloc(size, is_metadata));
- }
-}
-
-/*
- * If we're going to use this ABD for doing I/O using the block layer, the
- * consumer of the ABD data doesn't care if it's scattered or not, and we don't
- * plan to store this ABD in memory for a long period of time, we should
- * allocate the ABD type that requires the least data copying to do the I/O.
- *
- * On Illumos this is linear ABDs, however if ldi_strategy() can ever issue I/Os
- * using a scatter/gather list we should switch to that and replace this call
- * with vanilla abd_alloc().
- *
- * On Linux the optimal thing to do would be to use abd_get_offset() and
- * construct a new ABD which shares the original pages thereby eliminating
- * the copy. But for the moment a new linear ABD is allocated until this
- * performance optimization can be implemented.
- */
-abd_t *
-abd_alloc_for_io(size_t size, boolean_t is_metadata)
-{
- return (abd_alloc(size, is_metadata));
-}
-
-/*
- * Allocate a new ABD to point to offset off of sabd. It shares the underlying
- * buffer data with sabd. Use abd_put() to free. sabd must not be freed while
- * any derived ABDs exist.
- */
-static inline abd_t *
-abd_get_offset_impl(abd_t *sabd, size_t off, size_t size)
-{
- abd_t *abd;
-
- abd_verify(sabd);
- ASSERT3U(off, <=, sabd->abd_size);
-
- if (abd_is_linear(sabd)) {
- abd = abd_alloc_struct();
-
- /*
- * Even if this buf is filesystem metadata, we only track that
- * if we own the underlying data buffer, which is not true in
- * this case. Therefore, we don't ever use ABD_FLAG_META here.
- */
- abd->abd_flags = ABD_FLAG_LINEAR;
-
- abd->abd_u.abd_linear.abd_buf =
- (char *)sabd->abd_u.abd_linear.abd_buf + off;
- } else {
- int i = 0;
- struct scatterlist *sg = NULL;
- size_t new_offset = sabd->abd_u.abd_scatter.abd_offset + off;
-
- abd = abd_alloc_struct();
-
- /*
- * Even if this buf is filesystem metadata, we only track that
- * if we own the underlying data buffer, which is not true in
- * this case. Therefore, we don't ever use ABD_FLAG_META here.
- */
- abd->abd_flags = 0;
-
- abd_for_each_sg(sabd, sg, ABD_SCATTER(sabd).abd_nents, i) {
- if (new_offset < sg->length)
- break;
- new_offset -= sg->length;
- }
-
- ABD_SCATTER(abd).abd_sgl = sg;
- ABD_SCATTER(abd).abd_offset = new_offset;
- ABD_SCATTER(abd).abd_nents = ABD_SCATTER(sabd).abd_nents - i;
- }
-
- abd->abd_size = size;
- abd->abd_parent = sabd;
- zfs_refcount_create(&abd->abd_children);
- (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
-
- return (abd);
-}
-
-abd_t *
-abd_get_offset(abd_t *sabd, size_t off)
-{
- size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;
-
- VERIFY3U(size, >, 0);
-
- return (abd_get_offset_impl(sabd, off, size));
-}
-
-abd_t *
-abd_get_offset_size(abd_t *sabd, size_t off, size_t size)
-{
- ASSERT3U(off + size, <=, sabd->abd_size);
-
- return (abd_get_offset_impl(sabd, off, size));
-}
-
-/*
- * Allocate a linear ABD structure for buf. You must free this with abd_put()
- * since the resulting ABD doesn't own its own buffer.
- */
-abd_t *
-abd_get_from_buf(void *buf, size_t size)
-{
- abd_t *abd = abd_alloc_struct();
-
- VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);
-
- /*
- * Even if this buf is filesystem metadata, we only track that if we
- * own the underlying data buffer, which is not true in this case.
- * Therefore, we don't ever use ABD_FLAG_META here.
- */
- abd->abd_flags = ABD_FLAG_LINEAR;
- abd->abd_size = size;
- abd->abd_parent = NULL;
- zfs_refcount_create(&abd->abd_children);
-
- abd->abd_u.abd_linear.abd_buf = buf;
-
- return (abd);
-}
-
-/*
- * Free an ABD allocated from abd_get_offset() or abd_get_from_buf(). Will not
- * free the underlying scatterlist or buffer.
- */
-void
-abd_put(abd_t *abd)
-{
- abd_verify(abd);
- ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
-
- if (abd->abd_parent != NULL) {
- (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
- abd->abd_size, abd);
- }
-
- zfs_refcount_destroy(&abd->abd_children);
- abd_free_struct(abd);
-}
-
-/*
- * Get the raw buffer associated with a linear ABD.
- */
-void *
-abd_to_buf(abd_t *abd)
-{
- ASSERT(abd_is_linear(abd));
- abd_verify(abd);
- return (abd->abd_u.abd_linear.abd_buf);
-}
-
-/*
- * Borrow a raw buffer from an ABD without copying the contents of the ABD
- * into the buffer. If the ABD is scattered, this will allocate a raw buffer
- * whose contents are undefined. To copy over the existing data in the ABD, use
- * abd_borrow_buf_copy() instead.
- */
-void *
-abd_borrow_buf(abd_t *abd, size_t n)
-{
- void *buf;
- abd_verify(abd);
- ASSERT3U(abd->abd_size, >=, n);
- if (abd_is_linear(abd)) {
- buf = abd_to_buf(abd);
- } else {
- buf = zio_buf_alloc(n);
- }
- (void) zfs_refcount_add_many(&abd->abd_children, n, buf);
-
- return (buf);
-}
-
-void *
-abd_borrow_buf_copy(abd_t *abd, size_t n)
-{
- void *buf = abd_borrow_buf(abd, n);
- if (!abd_is_linear(abd)) {
- abd_copy_to_buf(buf, abd, n);
- }
- return (buf);
-}
-
-/*
- * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will
- * not change the contents of the ABD and will ASSERT that you didn't modify
- * the buffer since it was borrowed. If you want any changes you made to buf to
- * be copied back to abd, use abd_return_buf_copy() instead.
- */
-void
-abd_return_buf(abd_t *abd, void *buf, size_t n)
-{
- abd_verify(abd);
- ASSERT3U(abd->abd_size, >=, n);
- if (abd_is_linear(abd)) {
- ASSERT3P(buf, ==, abd_to_buf(abd));
- } else {
- ASSERT0(abd_cmp_buf(abd, buf, n));
- zio_buf_free(buf, n);
- }
- (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
-}
-
-void
-abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
-{
- if (!abd_is_linear(abd)) {
- abd_copy_from_buf(abd, buf, n);
- }
- abd_return_buf(abd, buf, n);
-}
-
-/*
- * Give this ABD ownership of the buffer that it's storing. Can only be used on
- * linear ABDs which were allocated via abd_get_from_buf(), or ones allocated
- * with abd_alloc_linear() which subsequently released ownership of their buf
- * with abd_release_ownership_of_buf().
- */
-void
-abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata)
-{
- ASSERT(abd_is_linear(abd));
- ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
- abd_verify(abd);
-
- abd->abd_flags |= ABD_FLAG_OWNER;
- if (is_metadata) {
- abd->abd_flags |= ABD_FLAG_META;
- }
-
- ABDSTAT_BUMP(abdstat_linear_cnt);
- ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
-}
-
-void
-abd_release_ownership_of_buf(abd_t *abd)
-{
- ASSERT(abd_is_linear(abd));
- ASSERT(abd->abd_flags & ABD_FLAG_OWNER);
-
- /*
- * abd_free() needs to handle LINEAR_PAGE ABD's specially.
- * Since that flag does not survive the
- * abd_release_ownership_of_buf() -> abd_get_from_buf() ->
- * abd_take_ownership_of_buf() sequence, we don't allow releasing
- * these "linear but not zio_[data_]buf_alloc()'ed" ABD's.
- */
- ASSERT(!abd_is_linear_page(abd));
-
- abd_verify(abd);
-
- abd->abd_flags &= ~ABD_FLAG_OWNER;
- /* Disable this flag since we no longer own the data buffer */
- abd->abd_flags &= ~ABD_FLAG_META;
-
- ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
- ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
-}
-
-#ifndef HAVE_1ARG_KMAP_ATOMIC
-#define NR_KM_TYPE (6)
-#ifdef _KERNEL
-int km_table[NR_KM_TYPE] = {
- KM_USER0,
- KM_USER1,
- KM_BIO_SRC_IRQ,
- KM_BIO_DST_IRQ,
- KM_PTE0,
- KM_PTE1,
-};
-#endif
-#endif
-
-struct abd_iter {
- /* public interface */
- void *iter_mapaddr; /* addr corresponding to iter_pos */
- size_t iter_mapsize; /* length of data valid at mapaddr */
-
- /* private */
- abd_t *iter_abd; /* ABD being iterated through */
- size_t iter_pos;
- size_t iter_offset; /* offset in current sg/abd_buf, */
- /* abd_offset included */
- struct scatterlist *iter_sg; /* current sg */
-#ifndef HAVE_1ARG_KMAP_ATOMIC
- int iter_km; /* KM_* for kmap_atomic */
-#endif
-};
-
-/*
- * Initialize the abd_iter.
- */
-static void
-abd_iter_init(struct abd_iter *aiter, abd_t *abd, int km_type)
-{
- abd_verify(abd);
- aiter->iter_abd = abd;
- aiter->iter_mapaddr = NULL;
- aiter->iter_mapsize = 0;
- aiter->iter_pos = 0;
- if (abd_is_linear(abd)) {
- aiter->iter_offset = 0;
- aiter->iter_sg = NULL;
- } else {
- aiter->iter_offset = ABD_SCATTER(abd).abd_offset;
- aiter->iter_sg = ABD_SCATTER(abd).abd_sgl;
- }
-#ifndef HAVE_1ARG_KMAP_ATOMIC
- ASSERT3U(km_type, <, NR_KM_TYPE);
- aiter->iter_km = km_type;
-#endif
-}
-
-/*
- * Advance the iterator by a certain amount. Cannot be called when a chunk is
- * in use. This can be safely called when the aiter has already exhausted, in
- * which case this does nothing.
- */
-static void
-abd_iter_advance(struct abd_iter *aiter, size_t amount)
-{
- ASSERT3P(aiter->iter_mapaddr, ==, NULL);
- ASSERT0(aiter->iter_mapsize);
-
- /* There's nothing left to advance to, so do nothing */
- if (aiter->iter_pos == aiter->iter_abd->abd_size)
- return;
-
- aiter->iter_pos += amount;
- aiter->iter_offset += amount;
- if (!abd_is_linear(aiter->iter_abd)) {
- while (aiter->iter_offset >= aiter->iter_sg->length) {
- aiter->iter_offset -= aiter->iter_sg->length;
- aiter->iter_sg = sg_next(aiter->iter_sg);
- if (aiter->iter_sg == NULL) {
- ASSERT0(aiter->iter_offset);
- break;
- }
- }
- }
-}
-
-/*
- * Map the current chunk into aiter. This can be safely called when the aiter
- * has already exhausted, in which case this does nothing.
- */
-static void
-abd_iter_map(struct abd_iter *aiter)
-{
- void *paddr;
- size_t offset = 0;
-
- ASSERT3P(aiter->iter_mapaddr, ==, NULL);
- ASSERT0(aiter->iter_mapsize);
-
- /* There's nothing left to iterate over, so do nothing */
- if (aiter->iter_pos == aiter->iter_abd->abd_size)
- return;
-
- if (abd_is_linear(aiter->iter_abd)) {
- ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
- offset = aiter->iter_offset;
- aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
- paddr = aiter->iter_abd->abd_u.abd_linear.abd_buf;
- } else {
- offset = aiter->iter_offset;
- aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset,
- aiter->iter_abd->abd_size - aiter->iter_pos);
-
- paddr = zfs_kmap_atomic(sg_page(aiter->iter_sg),
- km_table[aiter->iter_km]);
- }
-
- aiter->iter_mapaddr = (char *)paddr + offset;
-}
-
-/*
- * Unmap the current chunk from aiter. This can be safely called when the aiter
- * has already exhausted, in which case this does nothing.
- */
-static void
-abd_iter_unmap(struct abd_iter *aiter)
-{
- /* There's nothing left to unmap, so do nothing */
- if (aiter->iter_pos == aiter->iter_abd->abd_size)
- return;
-
- if (!abd_is_linear(aiter->iter_abd)) {
- /* LINTED E_FUNC_SET_NOT_USED */
- zfs_kunmap_atomic(aiter->iter_mapaddr - aiter->iter_offset,
- km_table[aiter->iter_km]);
- }
-
- ASSERT3P(aiter->iter_mapaddr, !=, NULL);
- ASSERT3U(aiter->iter_mapsize, >, 0);
-
- aiter->iter_mapaddr = NULL;
- aiter->iter_mapsize = 0;
-}
-
-int
-abd_iterate_func(abd_t *abd, size_t off, size_t size,
- abd_iter_func_t *func, void *private)
-{
- int ret = 0;
- struct abd_iter aiter;
-
- abd_verify(abd);
- ASSERT3U(off + size, <=, abd->abd_size);
-
- abd_iter_init(&aiter, abd, 0);
- abd_iter_advance(&aiter, off);
-
- while (size > 0) {
- abd_iter_map(&aiter);
-
- size_t len = MIN(aiter.iter_mapsize, size);
- ASSERT3U(len, >, 0);
-
- ret = func(aiter.iter_mapaddr, len, private);
-
- abd_iter_unmap(&aiter);
-
- if (ret != 0)
- break;
-
- size -= len;
- abd_iter_advance(&aiter, len);
- }
-
- return (ret);
-}
-
-struct buf_arg {
- void *arg_buf;
-};
-
-static int
-abd_copy_to_buf_off_cb(void *buf, size_t size, void *private)
-{
- struct buf_arg *ba_ptr = private;
-
- (void) memcpy(ba_ptr->arg_buf, buf, size);
- ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
-
- return (0);
-}
-
-/*
- * Copy abd to buf. (off is the offset in abd.)
- */
-void
-abd_copy_to_buf_off(void *buf, abd_t *abd, size_t off, size_t size)
-{
- struct buf_arg ba_ptr = { buf };
-
- (void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb,
- &ba_ptr);
-}
-
-static int
-abd_cmp_buf_off_cb(void *buf, size_t size, void *private)
-{
- int ret;
- struct buf_arg *ba_ptr = private;
-
- ret = memcmp(buf, ba_ptr->arg_buf, size);
- ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
-
- return (ret);
-}
-
-/*
- * Compare the contents of abd to buf. (off is the offset in abd.)
- */
-int
-abd_cmp_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
-{
- struct buf_arg ba_ptr = { (void *) buf };
-
- return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr));
-}
-
-static int
-abd_copy_from_buf_off_cb(void *buf, size_t size, void *private)
-{
- struct buf_arg *ba_ptr = private;
-
- (void) memcpy(buf, ba_ptr->arg_buf, size);
- ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
-
- return (0);
-}
-
-/*
- * Copy from buf to abd. (off is the offset in abd.)
- */
-void
-abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
-{
- struct buf_arg ba_ptr = { (void *) buf };
-
- (void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb,
- &ba_ptr);
-}
-
-/*ARGSUSED*/
-static int
-abd_zero_off_cb(void *buf, size_t size, void *private)
-{
- (void) memset(buf, 0, size);
- return (0);
-}
-
-/*
- * Zero out the abd from a particular offset to the end.
- */
-void
-abd_zero_off(abd_t *abd, size_t off, size_t size)
-{
- (void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL);
-}
-
-/*
- * Iterate over two ABDs and call func incrementally on the two ABDs' data in
- * equal-sized chunks (passed to func as raw buffers). func could be called many
- * times during this iteration.
- */
-int
-abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff,
- size_t size, abd_iter_func2_t *func, void *private)
-{
- int ret = 0;
- struct abd_iter daiter, saiter;
-
- abd_verify(dabd);
- abd_verify(sabd);
-
- ASSERT3U(doff + size, <=, dabd->abd_size);
- ASSERT3U(soff + size, <=, sabd->abd_size);
-
- abd_iter_init(&daiter, dabd, 0);
- abd_iter_init(&saiter, sabd, 1);
- abd_iter_advance(&daiter, doff);
- abd_iter_advance(&saiter, soff);
-
- while (size > 0) {
- abd_iter_map(&daiter);
- abd_iter_map(&saiter);
-
- size_t dlen = MIN(daiter.iter_mapsize, size);
- size_t slen = MIN(saiter.iter_mapsize, size);
- size_t len = MIN(dlen, slen);
- ASSERT(dlen > 0 || slen > 0);
-
- ret = func(daiter.iter_mapaddr, saiter.iter_mapaddr, len,
- private);
-
- abd_iter_unmap(&saiter);
- abd_iter_unmap(&daiter);
-
- if (ret != 0)
- break;
-
- size -= len;
- abd_iter_advance(&daiter, len);
- abd_iter_advance(&saiter, len);
- }
-
- return (ret);
-}
-
-/*ARGSUSED*/
-static int
-abd_copy_off_cb(void *dbuf, void *sbuf, size_t size, void *private)
-{
- (void) memcpy(dbuf, sbuf, size);
- return (0);
-}
-
-/*
- * Copy from sabd to dabd starting from soff and doff.
- */
-void
-abd_copy_off(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, size_t size)
-{
- (void) abd_iterate_func2(dabd, sabd, doff, soff, size,
- abd_copy_off_cb, NULL);
-}
-
-/*ARGSUSED*/
-static int
-abd_cmp_cb(void *bufa, void *bufb, size_t size, void *private)
-{
- return (memcmp(bufa, bufb, size));
-}
-
-/*
- * Compares the contents of two ABDs.
- */
-int
-abd_cmp(abd_t *dabd, abd_t *sabd)
-{
- ASSERT3U(dabd->abd_size, ==, sabd->abd_size);
- return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size,
- abd_cmp_cb, NULL));
-}
-
-/*
- * Iterate over code ABDs and a data ABD and call @func_raidz_gen.
- *
- * @cabds parity ABDs, must have equal size
- * @dabd data ABD. Can be NULL (in this case @dsize = 0)
- * @func_raidz_gen should be implemented so that its behaviour
- * is the same when taking linear and when taking scatter
- */
-void
-abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
- ssize_t csize, ssize_t dsize, const unsigned parity,
- void (*func_raidz_gen)(void **, const void *, size_t, size_t))
-{
- int i;
- ssize_t len, dlen;
- struct abd_iter caiters[3];
- struct abd_iter daiter = {0};
- void *caddrs[3];
- unsigned long flags;
-
- ASSERT3U(parity, <=, 3);
-
- for (i = 0; i < parity; i++)
- abd_iter_init(&caiters[i], cabds[i], i);
-
- if (dabd)
- abd_iter_init(&daiter, dabd, i);
-
- ASSERT3S(dsize, >=, 0);
-
- local_irq_save(flags);
- while (csize > 0) {
- len = csize;
-
- if (dabd && dsize > 0)
- abd_iter_map(&daiter);
-
- for (i = 0; i < parity; i++) {
- abd_iter_map(&caiters[i]);
- caddrs[i] = caiters[i].iter_mapaddr;
- }
-
- switch (parity) {
- case 3:
- len = MIN(caiters[2].iter_mapsize, len);
- /* falls through */
- case 2:
- len = MIN(caiters[1].iter_mapsize, len);
- /* falls through */
- case 1:
- len = MIN(caiters[0].iter_mapsize, len);
- }
-
- /* must be progressive */
- ASSERT3S(len, >, 0);
-
- if (dabd && dsize > 0) {
- /* this needs precise iter.length */
- len = MIN(daiter.iter_mapsize, len);
- dlen = len;
- } else
- dlen = 0;
-
- /* must be progressive */
- ASSERT3S(len, >, 0);
- /*
- * The iterated function likely will not do well if each
- * segment except the last one is not multiple of 512 (raidz).
- */
- ASSERT3U(((uint64_t)len & 511ULL), ==, 0);
-
- func_raidz_gen(caddrs, daiter.iter_mapaddr, len, dlen);
-
- for (i = parity-1; i >= 0; i--) {
- abd_iter_unmap(&caiters[i]);
- abd_iter_advance(&caiters[i], len);
- }
-
- if (dabd && dsize > 0) {
- abd_iter_unmap(&daiter);
- abd_iter_advance(&daiter, dlen);
- dsize -= dlen;
- }
-
- csize -= len;
-
- ASSERT3S(dsize, >=, 0);
- ASSERT3S(csize, >=, 0);
- }
- local_irq_restore(flags);
-}
-
-/*
- * Iterate over code ABDs and data reconstruction target ABDs and call
- * @func_raidz_rec. Function maps at most 6 pages atomically.
- *
- * @cabds parity ABDs, must have equal size
- * @tabds rec target ABDs, at most 3
- * @tsize size of data target columns
- * @func_raidz_rec expects syndrome data in target columns. Function
- * reconstructs data and overwrites target columns.
- */
-void
-abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
- ssize_t tsize, const unsigned parity,
- void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
- const unsigned *mul),
- const unsigned *mul)
-{
- int i;
- ssize_t len;
- struct abd_iter citers[3];
- struct abd_iter xiters[3];
- void *caddrs[3], *xaddrs[3];
- unsigned long flags;
-
- ASSERT3U(parity, <=, 3);
-
- for (i = 0; i < parity; i++) {
- abd_iter_init(&citers[i], cabds[i], 2*i);
- abd_iter_init(&xiters[i], tabds[i], 2*i+1);
- }
-
- local_irq_save(flags);
- while (tsize > 0) {
-
- for (i = 0; i < parity; i++) {
- abd_iter_map(&citers[i]);
- abd_iter_map(&xiters[i]);
- caddrs[i] = citers[i].iter_mapaddr;
- xaddrs[i] = xiters[i].iter_mapaddr;
- }
-
- len = tsize;
- switch (parity) {
- case 3:
- len = MIN(xiters[2].iter_mapsize, len);
- len = MIN(citers[2].iter_mapsize, len);
- /* falls through */
- case 2:
- len = MIN(xiters[1].iter_mapsize, len);
- len = MIN(citers[1].iter_mapsize, len);
- /* falls through */
- case 1:
- len = MIN(xiters[0].iter_mapsize, len);
- len = MIN(citers[0].iter_mapsize, len);
- }
- /* must be progressive */
- ASSERT3S(len, >, 0);
- /*
- * The iterated function likely will not do well if each
- * segment except the last one is not multiple of 512 (raidz).
- */
- ASSERT3U(((uint64_t)len & 511ULL), ==, 0);
-
- func_raidz_rec(xaddrs, len, caddrs, mul);
-
- for (i = parity-1; i >= 0; i--) {
- abd_iter_unmap(&xiters[i]);
- abd_iter_unmap(&citers[i]);
- abd_iter_advance(&xiters[i], len);
- abd_iter_advance(&citers[i], len);
- }
-
- tsize -= len;
- ASSERT3S(tsize, >=, 0);
- }
- local_irq_restore(flags);
-}
-
-#if defined(_KERNEL)
-/*
- * bio_nr_pages for ABD.
- * @off is the offset in @abd
- */
-unsigned long
-abd_nr_pages_off(abd_t *abd, unsigned int size, size_t off)
-{
- unsigned long pos;
-
- if (abd_is_linear(abd))
- pos = (unsigned long)abd_to_buf(abd) + off;
- else
- pos = abd->abd_u.abd_scatter.abd_offset + off;
-
- return ((pos + size + PAGESIZE - 1) >> PAGE_SHIFT) -
- (pos >> PAGE_SHIFT);
-}
-
-/*
- * bio_map for scatter ABD.
- * @off is the offset in @abd
- * Remaining IO size is returned
- */
-unsigned int
-abd_scatter_bio_map_off(struct bio *bio, abd_t *abd,
- unsigned int io_size, size_t off)
-{
- int i;
- struct abd_iter aiter;
-
- ASSERT(!abd_is_linear(abd));
- ASSERT3U(io_size, <=, abd->abd_size - off);
-
- abd_iter_init(&aiter, abd, 0);
- abd_iter_advance(&aiter, off);
-
- for (i = 0; i < bio->bi_max_vecs; i++) {
- struct page *pg;
- size_t len, sgoff, pgoff;
- struct scatterlist *sg;
-
- if (io_size <= 0)
- break;
-
- sg = aiter.iter_sg;
- sgoff = aiter.iter_offset;
- pgoff = sgoff & (PAGESIZE - 1);
- len = MIN(io_size, PAGESIZE - pgoff);
- ASSERT(len > 0);
-
- pg = nth_page(sg_page(sg), sgoff >> PAGE_SHIFT);
- if (bio_add_page(bio, pg, len, pgoff) != len)
- break;
-
- io_size -= len;
- abd_iter_advance(&aiter, len);
- }
-
- return (io_size);
-}
-
-/* Tunable Parameters */
-module_param(zfs_abd_scatter_enabled, int, 0644);
-MODULE_PARM_DESC(zfs_abd_scatter_enabled,
- "Toggle whether ABD allocations must be linear.");
-module_param(zfs_abd_scatter_min_size, int, 0644);
-MODULE_PARM_DESC(zfs_abd_scatter_min_size,
- "Minimum size of scatter allocations.");
-/* CSTYLED */
-module_param(zfs_abd_scatter_max_order, uint, 0644);
-MODULE_PARM_DESC(zfs_abd_scatter_max_order,
- "Maximum order allocation used for a scatter ABD.");
-#endif
diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c
index 5cac2a7de..9d8af3228 100644
--- a/module/zfs/gzip.c
+++ b/module/zfs/gzip.c
@@ -29,7 +29,7 @@
#include <sys/debug.h>
#include <sys/types.h>
#include <sys/strings.h>
-#include "qat.h"
+#include <sys/qat.h>
#ifdef _KERNEL
diff --git a/module/zfs/policy.c b/module/zfs/policy.c
deleted file mode 100644
index 7f9456a67..000000000
--- a/module/zfs/policy.c
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2013, Joyent, Inc. All rights reserved.
- * Copyright (C) 2016 Lawrence Livermore National Security, LLC.
- *
- * For Linux the vast majority of this enforcement is already handled via
- * the standard Linux VFS permission checks. However certain administrative
- * commands which bypass the standard mechanisms may need to make use of
- * this functionality.
- */
-
-#include <sys/policy.h>
-#include <linux/security.h>
-#include <linux/vfs_compat.h>
-
-/*
- * The passed credentials cannot be directly verified because Linux only
- * provides and interface to check the *current* process credentials. In
- * order to handle this the capable() test is only run when the passed
- * credentials match the current process credentials or the kcred. In
- * all other cases this function must fail and return the passed err.
- */
-static int
-priv_policy_ns(const cred_t *cr, int capability, boolean_t all, int err,
- struct user_namespace *ns)
-{
- ASSERT3S(all, ==, B_FALSE);
-
- if (cr != CRED() && (cr != kcred))
- return (err);
-
-#if defined(CONFIG_USER_NS) && defined(HAVE_NS_CAPABLE)
- if (!(ns ? ns_capable(ns, capability) : capable(capability)))
-#else
- if (!capable(capability))
-#endif
- return (err);
-
- return (0);
-}
-
-static int
-priv_policy(const cred_t *cr, int capability, boolean_t all, int err)
-{
- return (priv_policy_ns(cr, capability, all, err, NULL));
-}
-
-static int
-priv_policy_user(const cred_t *cr, int capability, boolean_t all, int err)
-{
- /*
- * All priv_policy_user checks are preceded by kuid/kgid_has_mapping()
- * checks. If we cannot do them, we shouldn't be using ns_capable()
- * since we don't know whether the affected files are valid in our
- * namespace. Note that kuid_has_mapping() came after cred->user_ns, so
- * we shouldn't need to re-check for HAVE_CRED_USER_NS
- */
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- return (priv_policy_ns(cr, capability, all, err, cr->user_ns));
-#else
- return (priv_policy_ns(cr, capability, all, err, NULL));
-#endif
-}
-
-/*
- * Checks for operations that are either client-only or are used by
- * both clients and servers.
- */
-int
-secpolicy_nfs(const cred_t *cr)
-{
- return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EPERM));
-}
-
-/*
- * Catch all system configuration.
- */
-int
-secpolicy_sys_config(const cred_t *cr, boolean_t checkonly)
-{
- return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EPERM));
-}
-
-/*
- * Like secpolicy_vnode_access() but we get the actual wanted mode and the
- * current mode of the file, not the missing bits.
- *
- * Enforced in the Linux VFS.
- */
-int
-secpolicy_vnode_access2(const cred_t *cr, struct inode *ip, uid_t owner,
- mode_t curmode, mode_t wantmode)
-{
- return (0);
-}
-
-/*
- * This is a special routine for ZFS; it is used to determine whether
- * any of the privileges in effect allow any form of access to the
- * file. There's no reason to audit this or any reason to record
- * this. More work is needed to do the "KPLD" stuff.
- */
-int
-secpolicy_vnode_any_access(const cred_t *cr, struct inode *ip, uid_t owner)
-{
- if (crgetfsuid(cr) == owner)
- return (0);
-
- if (zpl_inode_owner_or_capable(ip))
- return (0);
-
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
- return (EPERM);
-#endif
-
- if (priv_policy_user(cr, CAP_DAC_OVERRIDE, B_FALSE, EPERM) == 0)
- return (0);
-
- if (priv_policy_user(cr, CAP_DAC_READ_SEARCH, B_FALSE, EPERM) == 0)
- return (0);
-
- return (EPERM);
-}
-
-/*
- * Determine if subject can chown owner of a file.
- */
-int
-secpolicy_vnode_chown(const cred_t *cr, uid_t owner)
-{
- if (crgetfsuid(cr) == owner)
- return (0);
-
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
- return (EPERM);
-#endif
-
- return (priv_policy_user(cr, CAP_FOWNER, B_FALSE, EPERM));
-}
-
-/*
- * Determine if subject can change group ownership of a file.
- */
-int
-secpolicy_vnode_create_gid(const cred_t *cr)
-{
- return (priv_policy(cr, CAP_SETGID, B_FALSE, EPERM));
-}
-
-/*
- * Policy determines whether we can remove an entry from a directory,
- * regardless of permission bits.
- */
-int
-secpolicy_vnode_remove(const cred_t *cr)
-{
- return (priv_policy(cr, CAP_FOWNER, B_FALSE, EPERM));
-}
-
-/*
- * Determine that subject can modify the mode of a file. allzone privilege
- * needed when modifying root owned object.
- */
-int
-secpolicy_vnode_setdac(const cred_t *cr, uid_t owner)
-{
- if (crgetfsuid(cr) == owner)
- return (0);
-
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
- return (EPERM);
-#endif
-
- return (priv_policy_user(cr, CAP_FOWNER, B_FALSE, EPERM));
-}
-
-/*
- * Are we allowed to retain the set-uid/set-gid bits when
- * changing ownership or when writing to a file?
- * "issuid" should be true when set-uid; only in that case
- * root ownership is checked (setgid is assumed).
- *
- * Enforced in the Linux VFS.
- */
-int
-secpolicy_vnode_setid_retain(const cred_t *cr, boolean_t issuidroot)
-{
- return (priv_policy_user(cr, CAP_FSETID, B_FALSE, EPERM));
-}
-
-/*
- * Determine that subject can set the file setgid flag.
- */
-int
-secpolicy_vnode_setids_setgids(const cred_t *cr, gid_t gid)
-{
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- if (!kgid_has_mapping(cr->user_ns, SGID_TO_KGID(gid)))
- return (EPERM);
-#endif
- if (crgetfsgid(cr) != gid && !groupmember(gid, cr))
- return (priv_policy_user(cr, CAP_FSETID, B_FALSE, EPERM));
-
- return (0);
-}
-
-/*
- * Determine if the subject can inject faults in the ZFS fault injection
- * framework. Requires all privileges.
- */
-int
-secpolicy_zinject(const cred_t *cr)
-{
- return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EACCES));
-}
-
-/*
- * Determine if the subject has permission to manipulate ZFS datasets
- * (not pools). Equivalent to the SYS_MOUNT privilege.
- */
-int
-secpolicy_zfs(const cred_t *cr)
-{
- return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EACCES));
-}
-
-void
-secpolicy_setid_clear(vattr_t *vap, cred_t *cr)
-{
- if ((vap->va_mode & (S_ISUID | S_ISGID)) != 0 &&
- secpolicy_vnode_setid_retain(cr,
- (vap->va_mode & S_ISUID) != 0 &&
- (vap->va_mask & AT_UID) != 0 && vap->va_uid == 0) != 0) {
- vap->va_mask |= AT_MODE;
- vap->va_mode &= ~(S_ISUID|S_ISGID);
- }
-}
-
-/*
- * Determine that subject can set the file setid flags.
- */
-static int
-secpolicy_vnode_setid_modify(const cred_t *cr, uid_t owner)
-{
- if (crgetfsuid(cr) == owner)
- return (0);
-
-#if defined(CONFIG_USER_NS) && defined(HAVE_KUID_HAS_MAPPING)
- if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
- return (EPERM);
-#endif
-
- return (priv_policy_user(cr, CAP_FSETID, B_FALSE, EPERM));
-}
-
-/*
- * Determine that subject can make a file a "sticky".
- *
- * Enforced in the Linux VFS.
- */
-static int
-secpolicy_vnode_stky_modify(const cred_t *cr)
-{
- return (0);
-}
-
-int
-secpolicy_setid_setsticky_clear(struct inode *ip, vattr_t *vap,
- const vattr_t *ovap, cred_t *cr)
-{
- int error;
-
- if ((vap->va_mode & S_ISUID) != 0 &&
- (error = secpolicy_vnode_setid_modify(cr,
- ovap->va_uid)) != 0) {
- return (error);
- }
-
- /*
- * Check privilege if attempting to set the
- * sticky bit on a non-directory.
- */
- if (!S_ISDIR(ip->i_mode) && (vap->va_mode & S_ISVTX) != 0 &&
- secpolicy_vnode_stky_modify(cr) != 0) {
- vap->va_mode &= ~S_ISVTX;
- }
-
- /*
- * Check for privilege if attempting to set the
- * group-id bit.
- */
- if ((vap->va_mode & S_ISGID) != 0 &&
- secpolicy_vnode_setids_setgids(cr, ovap->va_gid) != 0) {
- vap->va_mode &= ~S_ISGID;
- }
-
- return (0);
-}
-
-/*
- * Check privileges for setting xvattr attributes
- */
-int
-secpolicy_xvattr(xvattr_t *xvap, uid_t owner, cred_t *cr, vtype_t vtype)
-{
- return (secpolicy_vnode_chown(cr, owner));
-}
-
-/*
- * Check privileges for setattr attributes.
- *
- * Enforced in the Linux VFS.
- */
-int
-secpolicy_vnode_setattr(cred_t *cr, struct inode *ip, struct vattr *vap,
- const struct vattr *ovap, int flags,
- int unlocked_access(void *, int, cred_t *), void *node)
-{
- return (0);
-}
-
-/*
- * Check privileges for links.
- *
- * Enforced in the Linux VFS.
- */
-int
-secpolicy_basic_link(const cred_t *cr)
-{
- return (0);
-}
diff --git a/module/zfs/qat.c b/module/zfs/qat.c
deleted file mode 100644
index a6f024cb4..000000000
--- a/module/zfs/qat.c
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-#if defined(_KERNEL) && defined(HAVE_QAT)
-#include <sys/zfs_context.h>
-#include "qat.h"
-
-qat_stats_t qat_stats = {
- { "comp_requests", KSTAT_DATA_UINT64 },
- { "comp_total_in_bytes", KSTAT_DATA_UINT64 },
- { "comp_total_out_bytes", KSTAT_DATA_UINT64 },
- { "decomp_requests", KSTAT_DATA_UINT64 },
- { "decomp_total_in_bytes", KSTAT_DATA_UINT64 },
- { "decomp_total_out_bytes", KSTAT_DATA_UINT64 },
- { "dc_fails", KSTAT_DATA_UINT64 },
- { "encrypt_requests", KSTAT_DATA_UINT64 },
- { "encrypt_total_in_bytes", KSTAT_DATA_UINT64 },
- { "encrypt_total_out_bytes", KSTAT_DATA_UINT64 },
- { "decrypt_requests", KSTAT_DATA_UINT64 },
- { "decrypt_total_in_bytes", KSTAT_DATA_UINT64 },
- { "decrypt_total_out_bytes", KSTAT_DATA_UINT64 },
- { "crypt_fails", KSTAT_DATA_UINT64 },
- { "cksum_requests", KSTAT_DATA_UINT64 },
- { "cksum_total_in_bytes", KSTAT_DATA_UINT64 },
- { "cksum_fails", KSTAT_DATA_UINT64 },
-};
-
-static kstat_t *qat_ksp = NULL;
-
-CpaStatus
-qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
-{
- *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
- if (*pp_mem_addr == NULL)
- return (CPA_STATUS_RESOURCE);
- return (CPA_STATUS_SUCCESS);
-}
-
-void
-qat_mem_free_contig(void **pp_mem_addr)
-{
- if (*pp_mem_addr != NULL) {
- kfree(*pp_mem_addr);
- *pp_mem_addr = NULL;
- }
-}
-
-int
-qat_init(void)
-{
- qat_ksp = kstat_create("zfs", 0, "qat", "misc",
- KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
- KSTAT_FLAG_VIRTUAL);
- if (qat_ksp != NULL) {
- qat_ksp->ks_data = &qat_stats;
- kstat_install(qat_ksp);
- }
-
- /*
- * Just set the disable flag when qat init failed, qat can be
- * turned on again in post-process after zfs module is loaded, e.g.:
- * echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
- */
- if (qat_dc_init() != 0)
- zfs_qat_compress_disable = 1;
-
- if (qat_cy_init() != 0) {
- zfs_qat_checksum_disable = 1;
- zfs_qat_encrypt_disable = 1;
- }
-
- return (0);
-}
-
-void
-qat_fini(void)
-{
- if (qat_ksp != NULL) {
- kstat_delete(qat_ksp);
- qat_ksp = NULL;
- }
-
- qat_cy_fini();
- qat_dc_fini();
-}
-
-#endif
diff --git a/module/zfs/qat.h b/module/zfs/qat.h
deleted file mode 100644
index fdd608139..000000000
--- a/module/zfs/qat.h
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-#ifndef _SYS_QAT_H
-#define _SYS_QAT_H
-
-typedef enum qat_compress_dir {
- QAT_DECOMPRESS = 0,
- QAT_COMPRESS = 1,
-} qat_compress_dir_t;
-
-typedef enum qat_encrypt_dir {
- QAT_DECRYPT = 0,
- QAT_ENCRYPT = 1,
-} qat_encrypt_dir_t;
-
-
-#if defined(_KERNEL) && defined(HAVE_QAT)
-#include <sys/zio.h>
-#include <sys/crypto/api.h>
-#include "cpa.h"
-#include "dc/cpa_dc.h"
-#include "lac/cpa_cy_sym.h"
-
-/*
- * Timeout - no response from hardware after 0.5 seconds
- */
-#define QAT_TIMEOUT_MS 500
-
-/*
- * The minimal and maximal buffer size which are not restricted
- * in the QAT hardware, but with the input buffer size between 4KB
- * and 128KB the hardware can provide the optimal performance.
- */
-#define QAT_MIN_BUF_SIZE (4*1024)
-#define QAT_MAX_BUF_SIZE (128*1024)
-
-/*
- * Used for QAT kstat.
- */
-typedef struct qat_stats {
- /*
- * Number of jobs submitted to QAT compression engine.
- */
- kstat_named_t comp_requests;
- /*
- * Total bytes sent to QAT compression engine.
- */
- kstat_named_t comp_total_in_bytes;
- /*
- * Total bytes output from QAT compression engine.
- */
- kstat_named_t comp_total_out_bytes;
- /*
- * Number of jobs submitted to QAT de-compression engine.
- */
- kstat_named_t decomp_requests;
- /*
- * Total bytes sent to QAT de-compression engine.
- */
- kstat_named_t decomp_total_in_bytes;
- /*
- * Total bytes output from QAT de-compression engine.
- */
- kstat_named_t decomp_total_out_bytes;
- /*
- * Number of fails in the QAT compression / decompression engine.
- * Note: when a QAT error happens, it doesn't necessarily indicate a
- * critical hardware issue. Sometimes it is because the output buffer
- * is not big enough. The compression job will be transferred to the
- * gzip software implementation so the functionality of ZFS is not
- * impacted.
- */
- kstat_named_t dc_fails;
-
- /*
- * Number of jobs submitted to QAT encryption engine.
- */
- kstat_named_t encrypt_requests;
- /*
- * Total bytes sent to QAT encryption engine.
- */
- kstat_named_t encrypt_total_in_bytes;
- /*
- * Total bytes output from QAT encryption engine.
- */
- kstat_named_t encrypt_total_out_bytes;
- /*
- * Number of jobs submitted to QAT decryption engine.
- */
- kstat_named_t decrypt_requests;
- /*
- * Total bytes sent to QAT decryption engine.
- */
- kstat_named_t decrypt_total_in_bytes;
- /*
- * Total bytes output from QAT decryption engine.
- */
- kstat_named_t decrypt_total_out_bytes;
- /*
- * Number of fails in the QAT encryption / decryption engine.
- * Note: when a QAT error happens, it doesn't necessarily indicate a
- * critical hardware issue. The encryption job will be transferred
- * to the software implementation so the functionality of ZFS is
- * not impacted.
- */
- kstat_named_t crypt_fails;
-
- /*
- * Number of jobs submitted to QAT checksum engine.
- */
- kstat_named_t cksum_requests;
- /*
- * Total bytes sent to QAT checksum engine.
- */
- kstat_named_t cksum_total_in_bytes;
- /*
- * Number of fails in the QAT checksum engine.
- * Note: when a QAT error happens, it doesn't necessarily indicate a
- * critical hardware issue. The checksum job will be transferred to the
- * software implementation so the functionality of ZFS is not impacted.
- */
- kstat_named_t cksum_fails;
-} qat_stats_t;
-
-#define QAT_STAT_INCR(stat, val) \
- atomic_add_64(&qat_stats.stat.value.ui64, (val))
-#define QAT_STAT_BUMP(stat) \
- QAT_STAT_INCR(stat, 1)
-
-extern qat_stats_t qat_stats;
-extern int zfs_qat_compress_disable;
-extern int zfs_qat_checksum_disable;
-extern int zfs_qat_encrypt_disable;
-
-/* inlined for performance */
-static inline struct page *
-qat_mem_to_page(void *addr)
-{
- if (!is_vmalloc_addr(addr))
- return (virt_to_page(addr));
-
- return (vmalloc_to_page(addr));
-}
-
-CpaStatus qat_mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes);
-void qat_mem_free_contig(void **pp_mem_addr);
-#define QAT_PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \
- qat_mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
-#define QAT_PHYS_CONTIG_FREE(p_mem_addr) \
- qat_mem_free_contig((void *)&(p_mem_addr))
-
-extern int qat_dc_init(void);
-extern void qat_dc_fini(void);
-extern int qat_cy_init(void);
-extern void qat_cy_fini(void);
-extern int qat_init(void);
-extern void qat_fini(void);
-
-/* fake CpaStatus used to indicate data was not compressible */
-#define CPA_STATUS_INCOMPRESSIBLE (-127)
-
-extern boolean_t qat_dc_use_accel(size_t s_len);
-extern boolean_t qat_crypt_use_accel(size_t s_len);
-extern boolean_t qat_checksum_use_accel(size_t s_len);
-extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len,
- char *dst, int dst_len, size_t *c_len);
-extern int qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
- uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
- crypto_key_t *key, uint64_t crypt, uint32_t enc_len);
-extern int qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size,
- zio_cksum_t *zcp);
-#else
-#define CPA_STATUS_SUCCESS 0
-#define CPA_STATUS_INCOMPRESSIBLE (-127)
-#define qat_init()
-#define qat_fini()
-#define qat_dc_use_accel(s_len) 0
-#define qat_crypt_use_accel(s_len) 0
-#define qat_checksum_use_accel(s_len) 0
-#define qat_compress(dir, s, sl, d, dl, cl) 0
-#define qat_crypt(dir, s, d, a, al, i, db, k, c, el) 0
-#define qat_checksum(c, buf, s, z) 0
-#endif
-
-#endif /* _SYS_QAT_H */
diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c
deleted file mode 100644
index 4136b6555..000000000
--- a/module/zfs/qat_compress.c
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-#if defined(_KERNEL) && defined(HAVE_QAT)
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-#include <linux/pagemap.h>
-#include <linux/completion.h>
-#include <sys/zfs_context.h>
-#include <sys/byteorder.h>
-#include <sys/zio.h>
-#include "qat.h"
-
-/*
- * Max instances in a QAT device, each instance is a channel to submit
- * jobs to QAT hardware, this is only for pre-allocating instance and
- * session arrays; the actual number of instances are defined in the
- * QAT driver's configuration file.
- */
-#define QAT_DC_MAX_INSTANCES 48
-
-/*
- * ZLIB head and foot size
- */
-#define ZLIB_HEAD_SZ 2
-#define ZLIB_FOOT_SZ 4
-
-static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
-static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
-static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
-static Cpa16U num_inst = 0;
-static Cpa32U inst_num = 0;
-static boolean_t qat_dc_init_done = B_FALSE;
-int zfs_qat_compress_disable = 0;
-
-boolean_t
-qat_dc_use_accel(size_t s_len)
-{
- return (!zfs_qat_compress_disable &&
- qat_dc_init_done &&
- s_len >= QAT_MIN_BUF_SIZE &&
- s_len <= QAT_MAX_BUF_SIZE);
-}
-
-static void
-qat_dc_callback(void *p_callback, CpaStatus status)
-{
- if (p_callback != NULL)
- complete((struct completion *)p_callback);
-}
-
-static void
-qat_dc_clean(void)
-{
- Cpa16U buff_num = 0;
- Cpa16U num_inter_buff_lists = 0;
-
- for (Cpa16U i = 0; i < num_inst; i++) {
- cpaDcStopInstance(dc_inst_handles[i]);
- QAT_PHYS_CONTIG_FREE(session_handles[i]);
- /* free intermediate buffers */
- if (buffer_array[i] != NULL) {
- cpaDcGetNumIntermediateBuffers(
- dc_inst_handles[i], &num_inter_buff_lists);
- for (buff_num = 0; buff_num < num_inter_buff_lists;
- buff_num++) {
- CpaBufferList *buffer_inter =
- buffer_array[i][buff_num];
- if (buffer_inter->pBuffers) {
- QAT_PHYS_CONTIG_FREE(
- buffer_inter->pBuffers->pData);
- QAT_PHYS_CONTIG_FREE(
- buffer_inter->pBuffers);
- }
- QAT_PHYS_CONTIG_FREE(
- buffer_inter->pPrivateMetaData);
- QAT_PHYS_CONTIG_FREE(buffer_inter);
- }
- }
- }
-
- num_inst = 0;
- qat_dc_init_done = B_FALSE;
-}
-
-int
-qat_dc_init(void)
-{
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa32U sess_size = 0;
- Cpa32U ctx_size = 0;
- Cpa16U num_inter_buff_lists = 0;
- Cpa16U buff_num = 0;
- Cpa32U buff_meta_size = 0;
- CpaDcSessionSetupData sd = {0};
-
- if (qat_dc_init_done)
- return (0);
-
- status = cpaDcGetNumInstances(&num_inst);
- if (status != CPA_STATUS_SUCCESS)
- return (-1);
-
- /* if the user has configured no QAT compression units just return */
- if (num_inst == 0)
- return (0);
-
- if (num_inst > QAT_DC_MAX_INSTANCES)
- num_inst = QAT_DC_MAX_INSTANCES;
-
- status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
- if (status != CPA_STATUS_SUCCESS)
- return (-1);
-
- for (Cpa16U i = 0; i < num_inst; i++) {
- cpaDcSetAddressTranslation(dc_inst_handles[i],
- (void*)virt_to_phys);
-
- status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
- 1, &buff_meta_size);
-
- if (status == CPA_STATUS_SUCCESS)
- status = cpaDcGetNumIntermediateBuffers(
- dc_inst_handles[i], &num_inter_buff_lists);
-
- if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
- status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
- num_inter_buff_lists *
- sizeof (CpaBufferList *));
-
- for (buff_num = 0; buff_num < num_inter_buff_lists;
- buff_num++) {
- if (status == CPA_STATUS_SUCCESS)
- status = QAT_PHYS_CONTIG_ALLOC(
- &buffer_array[i][buff_num],
- sizeof (CpaBufferList));
-
- if (status == CPA_STATUS_SUCCESS)
- status = QAT_PHYS_CONTIG_ALLOC(
- &buffer_array[i][buff_num]->
- pPrivateMetaData,
- buff_meta_size);
-
- if (status == CPA_STATUS_SUCCESS)
- status = QAT_PHYS_CONTIG_ALLOC(
- &buffer_array[i][buff_num]->pBuffers,
- sizeof (CpaFlatBuffer));
-
- if (status == CPA_STATUS_SUCCESS) {
- /*
- * implementation requires an intermediate
- * buffer approximately twice the size of
- * output buffer, which is 2x max buffer
- * size here.
- */
- status = QAT_PHYS_CONTIG_ALLOC(
- &buffer_array[i][buff_num]->pBuffers->
- pData, 2 * QAT_MAX_BUF_SIZE);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- buffer_array[i][buff_num]->numBuffers = 1;
- buffer_array[i][buff_num]->pBuffers->
- dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
- }
- }
-
- status = cpaDcStartInstance(dc_inst_handles[i],
- num_inter_buff_lists, buffer_array[i]);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- sd.compLevel = CPA_DC_L1;
- sd.compType = CPA_DC_DEFLATE;
- sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
- sd.sessDirection = CPA_DC_DIR_COMBINED;
- sd.sessState = CPA_DC_STATELESS;
- sd.deflateWindowSize = 7;
- sd.checksum = CPA_DC_ADLER32;
- status = cpaDcGetSessionSize(dc_inst_handles[i],
- &sd, &sess_size, &ctx_size);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
- if (session_handles[i] == NULL)
- goto fail;
-
- status = cpaDcInitSession(dc_inst_handles[i],
- session_handles[i],
- &sd, NULL, qat_dc_callback);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- }
-
- qat_dc_init_done = B_TRUE;
- return (0);
-fail:
- qat_dc_clean();
- return (-1);
-}
-
-void
-qat_dc_fini(void)
-{
- if (!qat_dc_init_done)
- return;
-
- qat_dc_clean();
-}
-
-/*
- * The "add" parameter is an additional buffer which is passed
- * to QAT as a scratch buffer alongside the destination buffer
- * in case the "compressed" data ends up being larger than the
- * original source data. This is necessary to prevent QAT from
- * generating buffer overflow warnings for incompressible data.
- */
-static int
-qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
- char *dst, int dst_len, char *add, int add_len, size_t *c_len)
-{
- CpaInstanceHandle dc_inst_handle;
- CpaDcSessionHandle session_handle;
- CpaBufferList *buf_list_src = NULL;
- CpaBufferList *buf_list_dst = NULL;
- CpaFlatBuffer *flat_buf_src = NULL;
- CpaFlatBuffer *flat_buf_dst = NULL;
- Cpa8U *buffer_meta_src = NULL;
- Cpa8U *buffer_meta_dst = NULL;
- Cpa32U buffer_meta_size = 0;
- CpaDcRqResults dc_results;
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa32U hdr_sz = 0;
- Cpa32U compressed_sz;
- Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
- Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
- Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
- Cpa32U bytes_left;
- Cpa32U dst_pages = 0;
- Cpa32U adler32 = 0;
- char *data;
- struct page *page;
- struct page **in_pages = NULL;
- struct page **out_pages = NULL;
- struct page **add_pages = NULL;
- Cpa32U page_off = 0;
- struct completion complete;
- Cpa32U page_num = 0;
- Cpa16U i;
-
- /*
- * We increment num_src_buf and num_dst_buf by 2 to allow
- * us to handle non page-aligned buffer addresses and buffers
- * whose sizes are not divisible by PAGE_SIZE.
- */
- Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
- (num_src_buf * sizeof (CpaFlatBuffer));
- Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
- ((num_dst_buf + num_add_buf) * sizeof (CpaFlatBuffer));
-
- if (QAT_PHYS_CONTIG_ALLOC(&in_pages,
- num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
- goto fail;
-
- if (QAT_PHYS_CONTIG_ALLOC(&out_pages,
- num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
- goto fail;
-
- if (QAT_PHYS_CONTIG_ALLOC(&add_pages,
- num_add_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
- goto fail;
-
- i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
- dc_inst_handle = dc_inst_handles[i];
- session_handle = session_handles[i];
-
- cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
- &buffer_meta_size);
- if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
- CPA_STATUS_SUCCESS)
- goto fail;
-
- cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf + num_add_buf,
- &buffer_meta_size);
- if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
- CPA_STATUS_SUCCESS)
- goto fail;
-
- /* build source buffer list */
- if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
- CPA_STATUS_SUCCESS)
- goto fail;
-
- flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
-
- buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
-
- /* build destination buffer list */
- if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
- CPA_STATUS_SUCCESS)
- goto fail;
-
- flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
-
- buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
-
- buf_list_src->numBuffers = 0;
- buf_list_src->pPrivateMetaData = buffer_meta_src;
- bytes_left = src_len;
- data = src;
- page_num = 0;
- while (bytes_left > 0) {
- page_off = ((long)data & ~PAGE_MASK);
- page = qat_mem_to_page(data);
- in_pages[page_num] = page;
- flat_buf_src->pData = kmap(page) + page_off;
- flat_buf_src->dataLenInBytes =
- min((long)PAGE_SIZE - page_off, (long)bytes_left);
-
- bytes_left -= flat_buf_src->dataLenInBytes;
- data += flat_buf_src->dataLenInBytes;
- flat_buf_src++;
- buf_list_src->numBuffers++;
- page_num++;
- }
-
- buf_list_dst->numBuffers = 0;
- buf_list_dst->pPrivateMetaData = buffer_meta_dst;
- bytes_left = dst_len;
- data = dst;
- page_num = 0;
- while (bytes_left > 0) {
- page_off = ((long)data & ~PAGE_MASK);
- page = qat_mem_to_page(data);
- flat_buf_dst->pData = kmap(page) + page_off;
- out_pages[page_num] = page;
- flat_buf_dst->dataLenInBytes =
- min((long)PAGE_SIZE - page_off, (long)bytes_left);
-
- bytes_left -= flat_buf_dst->dataLenInBytes;
- data += flat_buf_dst->dataLenInBytes;
- flat_buf_dst++;
- buf_list_dst->numBuffers++;
- page_num++;
- dst_pages++;
- }
-
- /* map additional scratch pages into the destination buffer list */
- bytes_left = add_len;
- data = add;
- page_num = 0;
- while (bytes_left > 0) {
- page_off = ((long)data & ~PAGE_MASK);
- page = qat_mem_to_page(data);
- flat_buf_dst->pData = kmap(page) + page_off;
- add_pages[page_num] = page;
- flat_buf_dst->dataLenInBytes =
- min((long)PAGE_SIZE - page_off, (long)bytes_left);
-
- bytes_left -= flat_buf_dst->dataLenInBytes;
- data += flat_buf_dst->dataLenInBytes;
- flat_buf_dst++;
- buf_list_dst->numBuffers++;
- page_num++;
- }
-
- init_completion(&complete);
-
- if (dir == QAT_COMPRESS) {
- QAT_STAT_BUMP(comp_requests);
- QAT_STAT_INCR(comp_total_in_bytes, src_len);
-
- cpaDcGenerateHeader(session_handle,
- buf_list_dst->pBuffers, &hdr_sz);
- buf_list_dst->pBuffers->pData += hdr_sz;
- buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
- status = cpaDcCompressData(
- dc_inst_handle, session_handle,
- buf_list_src, buf_list_dst,
- &dc_results, CPA_DC_FLUSH_FINAL,
- &complete);
- if (status != CPA_STATUS_SUCCESS) {
- goto fail;
- }
-
- /* we now wait until the completion of the operation. */
- if (!wait_for_completion_interruptible_timeout(&complete,
- QAT_TIMEOUT_MS)) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- if (dc_results.status != CPA_STATUS_SUCCESS) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- compressed_sz = dc_results.produced;
- if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
- status = CPA_STATUS_INCOMPRESSIBLE;
- goto fail;
- }
-
- flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
- /* move to the last page */
- flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
-
- /* no space for gzip footer in the last page */
- if (((compressed_sz + hdr_sz) % PAGE_SIZE)
- + ZLIB_FOOT_SZ > PAGE_SIZE) {
- status = CPA_STATUS_INCOMPRESSIBLE;
- goto fail;
- }
-
- /* jump to the end of the buffer and append footer */
- flat_buf_dst->pData =
- (char *)((unsigned long)flat_buf_dst->pData & PAGE_MASK)
- + ((compressed_sz + hdr_sz) % PAGE_SIZE);
- flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
-
- dc_results.produced = 0;
- status = cpaDcGenerateFooter(session_handle,
- flat_buf_dst, &dc_results);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- *c_len = compressed_sz + dc_results.produced + hdr_sz;
- QAT_STAT_INCR(comp_total_out_bytes, *c_len);
- } else {
- ASSERT3U(dir, ==, QAT_DECOMPRESS);
- QAT_STAT_BUMP(decomp_requests);
- QAT_STAT_INCR(decomp_total_in_bytes, src_len);
-
- buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
- buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
- status = cpaDcDecompressData(dc_inst_handle, session_handle,
- buf_list_src, buf_list_dst, &dc_results, CPA_DC_FLUSH_FINAL,
- &complete);
-
- if (CPA_STATUS_SUCCESS != status) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- /* we now wait until the completion of the operation. */
- if (!wait_for_completion_interruptible_timeout(&complete,
- QAT_TIMEOUT_MS)) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- if (dc_results.status != CPA_STATUS_SUCCESS) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- /* verify adler checksum */
- adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
- if (adler32 != BSWAP_32(dc_results.checksum)) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
- *c_len = dc_results.produced;
- QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
- }
-
-fail:
- if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_INCOMPRESSIBLE)
- QAT_STAT_BUMP(dc_fails);
-
- if (in_pages) {
- for (page_num = 0;
- page_num < buf_list_src->numBuffers;
- page_num++) {
- kunmap(in_pages[page_num]);
- }
- QAT_PHYS_CONTIG_FREE(in_pages);
- }
-
- if (out_pages) {
- for (page_num = 0; page_num < dst_pages; page_num++) {
- kunmap(out_pages[page_num]);
- }
- QAT_PHYS_CONTIG_FREE(out_pages);
- }
-
- if (add_pages) {
- for (page_num = 0;
- page_num < buf_list_dst->numBuffers - dst_pages;
- page_num++) {
- kunmap(add_pages[page_num]);
- }
- QAT_PHYS_CONTIG_FREE(add_pages);
- }
-
- QAT_PHYS_CONTIG_FREE(buffer_meta_src);
- QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
- QAT_PHYS_CONTIG_FREE(buf_list_src);
- QAT_PHYS_CONTIG_FREE(buf_list_dst);
-
- return (status);
-}
-
-/*
- * Entry point for QAT accelerated compression / decompression.
- */
-int
-qat_compress(qat_compress_dir_t dir, char *src, int src_len,
- char *dst, int dst_len, size_t *c_len)
-{
- int ret;
- size_t add_len = 0;
- void *add = NULL;
-
- if (dir == QAT_COMPRESS) {
- add_len = dst_len;
- add = zio_data_buf_alloc(add_len);
- }
-
- ret = qat_compress_impl(dir, src, src_len, dst,
- dst_len, add, add_len, c_len);
-
- if (dir == QAT_COMPRESS)
- zio_data_buf_free(add, add_len);
-
- return (ret);
-}
-
-static int
-param_set_qat_compress(const char *val, zfs_kernel_param_t *kp)
-{
- int ret;
- int *pvalue = kp->arg;
- ret = param_set_int(val, kp);
- if (ret)
- return (ret);
- /*
- * zfs_qat_compress_disable = 0: enable qat compress
- * try to initialize qat instance if it has not been done
- */
- if (*pvalue == 0 && !qat_dc_init_done) {
- ret = qat_dc_init();
- if (ret != 0) {
- zfs_qat_compress_disable = 1;
- return (ret);
- }
- }
- return (ret);
-}
-
-module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
- param_get_int, &zfs_qat_compress_disable, 0644);
-MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");
-
-#endif
diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c
deleted file mode 100644
index 02e19d21d..000000000
--- a/module/zfs/qat_crypt.c
+++ /dev/null
@@ -1,631 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * This file represents the QAT implementation of checksums and encryption.
- * Internally, QAT shares the same cryptographic instances for both of these
- * operations, so the code has been combined here. QAT data compression uses
- * compression instances, so that code is separated into qat_compress.c
- */
-
-#if defined(_KERNEL) && defined(HAVE_QAT)
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-#include <linux/pagemap.h>
-#include <linux/completion.h>
-#include <sys/zfs_context.h>
-#include <sys/zio_crypt.h>
-#include "lac/cpa_cy_im.h"
-#include "lac/cpa_cy_common.h"
-#include "qat.h"
-
-/*
- * Max instances in a QAT device, each instance is a channel to submit
- * jobs to QAT hardware, this is only for pre-allocating instances
- * and session arrays; the actual number of instances are defined in
- * the QAT driver's configure file.
- */
-#define QAT_CRYPT_MAX_INSTANCES 48
-
-#define MAX_PAGE_NUM 1024
-
-static Cpa32U inst_num = 0;
-static Cpa16U num_inst = 0;
-static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
-static boolean_t qat_cy_init_done = B_FALSE;
-int zfs_qat_encrypt_disable = 0;
-int zfs_qat_checksum_disable = 0;
-
-typedef struct cy_callback {
- CpaBoolean verify_result;
- struct completion complete;
-} cy_callback_t;
-
-static void
-symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
- void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
-{
- cy_callback_t *cb = p_callback;
-
- if (cb != NULL) {
- /* indicate that the function has been called */
- cb->verify_result = verify;
- complete(&cb->complete);
- }
-}
-
-boolean_t
-qat_crypt_use_accel(size_t s_len)
-{
- return (!zfs_qat_encrypt_disable &&
- qat_cy_init_done &&
- s_len >= QAT_MIN_BUF_SIZE &&
- s_len <= QAT_MAX_BUF_SIZE);
-}
-
-boolean_t
-qat_checksum_use_accel(size_t s_len)
-{
- return (!zfs_qat_checksum_disable &&
- qat_cy_init_done &&
- s_len >= QAT_MIN_BUF_SIZE &&
- s_len <= QAT_MAX_BUF_SIZE);
-}
-
-void
-qat_cy_clean(void)
-{
- for (Cpa16U i = 0; i < num_inst; i++)
- cpaCyStopInstance(cy_inst_handles[i]);
-
- num_inst = 0;
- qat_cy_init_done = B_FALSE;
-}
-
-int
-qat_cy_init(void)
-{
- CpaStatus status = CPA_STATUS_FAIL;
-
- if (qat_cy_init_done)
- return (0);
-
- status = cpaCyGetNumInstances(&num_inst);
- if (status != CPA_STATUS_SUCCESS)
- return (-1);
-
- /* if the user has configured no QAT encryption units just return */
- if (num_inst == 0)
- return (0);
-
- if (num_inst > QAT_CRYPT_MAX_INSTANCES)
- num_inst = QAT_CRYPT_MAX_INSTANCES;
-
- status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
- if (status != CPA_STATUS_SUCCESS)
- return (-1);
-
- for (Cpa16U i = 0; i < num_inst; i++) {
- status = cpaCySetAddressTranslation(cy_inst_handles[i],
- (void *)virt_to_phys);
- if (status != CPA_STATUS_SUCCESS)
- goto error;
-
- status = cpaCyStartInstance(cy_inst_handles[i]);
- if (status != CPA_STATUS_SUCCESS)
- goto error;
- }
-
- qat_cy_init_done = B_TRUE;
- return (0);
-
-error:
- qat_cy_clean();
- return (-1);
-}
-
-void
-qat_cy_fini(void)
-{
- if (!qat_cy_init_done)
- return;
-
- qat_cy_clean();
-}
-
-static CpaStatus
-qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
- CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
- Cpa64U crypt, Cpa32U aad_len)
-{
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa32U ctx_size;
- Cpa32U ciper_algorithm;
- Cpa32U hash_algorithm;
- CpaCySymSessionSetupData sd = { 0 };
-
- if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
- return (CPA_STATUS_FAIL);
- } else {
- ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
- hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
- }
-
- sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
- sd.cipherSetupData.pCipherKey = key->ck_data;
- sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
- sd.hashSetupData.hashAlgorithm = hash_algorithm;
- sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
- sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
- sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
- sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
- sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
- sd.digestIsAppended = CPA_FALSE;
- sd.verifyDigest = CPA_FALSE;
-
- if (dir == QAT_ENCRYPT) {
- sd.cipherSetupData.cipherDirection =
- CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
- sd.algChainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
- } else {
- ASSERT3U(dir, ==, QAT_DECRYPT);
- sd.cipherSetupData.cipherDirection =
- CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
- sd.algChainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
- }
-
- status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
- if (status != CPA_STATUS_SUCCESS)
- return (status);
-
- status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
- if (status != CPA_STATUS_SUCCESS)
- return (status);
-
- status = cpaCySymInitSession(inst_handle, symcallback, &sd,
- *cy_session_ctx);
- if (status != CPA_STATUS_SUCCESS) {
- QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
- return (status);
- }
-
- return (CPA_STATUS_SUCCESS);
-}
-
-static CpaStatus
-qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
- CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
-{
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa32U ctx_size;
- Cpa32U hash_algorithm;
- CpaCySymSessionSetupData sd = { 0 };
-
- /*
- * ZFS's SHA512 checksum is actually SHA512/256, which uses
- * a different IV from standard SHA512. QAT does not support
- * SHA512/256, so we can only support SHA256.
- */
- if (cksum == ZIO_CHECKSUM_SHA256)
- hash_algorithm = CPA_CY_SYM_HASH_SHA256;
- else
- return (CPA_STATUS_FAIL);
-
- sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
- sd.symOperation = CPA_CY_SYM_OP_HASH;
- sd.hashSetupData.hashAlgorithm = hash_algorithm;
- sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
- sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
- sd.digestIsAppended = CPA_FALSE;
- sd.verifyDigest = CPA_FALSE;
-
- status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
- if (status != CPA_STATUS_SUCCESS)
- return (status);
-
- status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
- if (status != CPA_STATUS_SUCCESS)
- return (status);
-
- status = cpaCySymInitSession(inst_handle, symcallback, &sd,
- *cy_session_ctx);
- if (status != CPA_STATUS_SUCCESS) {
- QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
- return (status);
- }
-
- return (CPA_STATUS_SUCCESS);
-}
-
-static CpaStatus
-qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
- CpaBufferList *src, CpaBufferList *dst)
-{
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa32U meta_size = 0;
-
- status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
- if (status != CPA_STATUS_SUCCESS)
- return (status);
-
- status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
- if (status != CPA_STATUS_SUCCESS)
- goto error;
-
- if (src != dst) {
- status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
- meta_size);
- if (status != CPA_STATUS_SUCCESS)
- goto error;
- }
-
- return (CPA_STATUS_SUCCESS);
-
-error:
- QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
- if (src != dst)
- QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
-
- return (status);
-}
-
-int
-qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
- uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
- crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
-{
- CpaStatus status = CPA_STATUS_SUCCESS;
- Cpa16U i;
- CpaInstanceHandle cy_inst_handle;
- Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
- Cpa32U bytes_left = 0;
- Cpa8S *data = NULL;
- CpaCySymSessionCtx *cy_session_ctx = NULL;
- cy_callback_t cb;
- CpaCySymOpData op_data = { 0 };
- CpaBufferList src_buffer_list = { 0 };
- CpaBufferList dst_buffer_list = { 0 };
- CpaFlatBuffer *flat_src_buf_array = NULL;
- CpaFlatBuffer *flat_src_buf = NULL;
- CpaFlatBuffer *flat_dst_buf_array = NULL;
- CpaFlatBuffer *flat_dst_buf = NULL;
- struct page *in_pages[MAX_PAGE_NUM];
- struct page *out_pages[MAX_PAGE_NUM];
- Cpa32U in_page_num = 0;
- Cpa32U out_page_num = 0;
- Cpa32U in_page_off = 0;
- Cpa32U out_page_off = 0;
-
- if (dir == QAT_ENCRYPT) {
- QAT_STAT_BUMP(encrypt_requests);
- QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
- } else {
- QAT_STAT_BUMP(decrypt_requests);
- QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
- }
-
- i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
- cy_inst_handle = cy_inst_handles[i];
-
- status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
- &cy_session_ctx, key, crypt, aad_len);
- if (status != CPA_STATUS_SUCCESS) {
- /* don't count CCM as a failure since it's not supported */
- if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
- QAT_STAT_BUMP(crypt_fails);
- return (status);
- }
-
- /*
- * We increment nr_bufs by 2 to allow us to handle non
- * page-aligned buffer addresses and buffers whose sizes
- * are not divisible by PAGE_SIZE.
- */
- status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
- &src_buffer_list, &dst_buffer_list);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
- nr_bufs * sizeof (CpaFlatBuffer));
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
- nr_bufs * sizeof (CpaFlatBuffer));
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
- ZIO_DATA_MAC_LEN);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
- ZIO_DATA_IV_LEN);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- if (aad_len > 0) {
- status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
- aad_len);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
- }
-
- bytes_left = enc_len;
- data = src_buf;
- flat_src_buf = flat_src_buf_array;
- while (bytes_left > 0) {
- in_page_off = ((long)data & ~PAGE_MASK);
- in_pages[in_page_num] = qat_mem_to_page(data);
- flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
- flat_src_buf->dataLenInBytes =
- min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
- data += flat_src_buf->dataLenInBytes;
- bytes_left -= flat_src_buf->dataLenInBytes;
- flat_src_buf++;
- in_page_num++;
- }
- src_buffer_list.pBuffers = flat_src_buf_array;
- src_buffer_list.numBuffers = in_page_num;
-
- bytes_left = enc_len;
- data = dst_buf;
- flat_dst_buf = flat_dst_buf_array;
- while (bytes_left > 0) {
- out_page_off = ((long)data & ~PAGE_MASK);
- out_pages[out_page_num] = qat_mem_to_page(data);
- flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
- out_page_off;
- flat_dst_buf->dataLenInBytes =
- min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
- data += flat_dst_buf->dataLenInBytes;
- bytes_left -= flat_dst_buf->dataLenInBytes;
- flat_dst_buf++;
- out_page_num++;
- }
- dst_buffer_list.pBuffers = flat_dst_buf_array;
- dst_buffer_list.numBuffers = out_page_num;
-
- op_data.sessionCtx = cy_session_ctx;
- op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
- op_data.cryptoStartSrcOffsetInBytes = 0;
- op_data.messageLenToCipherInBytes = 0;
- op_data.hashStartSrcOffsetInBytes = 0;
- op_data.messageLenToHashInBytes = 0;
- op_data.messageLenToCipherInBytes = enc_len;
- op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
- bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
-
- cb.verify_result = CPA_FALSE;
- init_completion(&cb.complete);
- status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
- &src_buffer_list, &dst_buffer_list, NULL);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- if (!wait_for_completion_interruptible_timeout(&cb.complete,
- QAT_TIMEOUT_MS)) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- if (cb.verify_result == CPA_FALSE) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- /* save digest result to digest_buf */
- bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
- if (dir == QAT_ENCRYPT)
- QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
- else
- QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
-
-fail:
- if (status != CPA_STATUS_SUCCESS)
- QAT_STAT_BUMP(crypt_fails);
-
- for (i = 0; i < in_page_num; i++)
- kunmap(in_pages[i]);
- for (i = 0; i < out_page_num; i++)
- kunmap(out_pages[i]);
-
- cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
- if (aad_len > 0)
- QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
- QAT_PHYS_CONTIG_FREE(op_data.pIv);
- QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
- QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
- QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
- QAT_PHYS_CONTIG_FREE(cy_session_ctx);
- QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
- QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
-
- return (status);
-}
-
-int
-qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
-{
- CpaStatus status;
- Cpa16U i;
- CpaInstanceHandle cy_inst_handle;
- Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
- Cpa32U bytes_left = 0;
- Cpa8S *data = NULL;
- CpaCySymSessionCtx *cy_session_ctx = NULL;
- cy_callback_t cb;
- Cpa8U *digest_buffer = NULL;
- CpaCySymOpData op_data = { 0 };
- CpaBufferList src_buffer_list = { 0 };
- CpaFlatBuffer *flat_src_buf_array = NULL;
- CpaFlatBuffer *flat_src_buf = NULL;
- struct page *in_pages[MAX_PAGE_NUM];
- Cpa32U page_num = 0;
- Cpa32U page_off = 0;
-
- QAT_STAT_BUMP(cksum_requests);
- QAT_STAT_INCR(cksum_total_in_bytes, size);
-
- i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
- cy_inst_handle = cy_inst_handles[i];
-
- status = qat_init_checksum_session_ctx(cy_inst_handle,
- &cy_session_ctx, cksum);
- if (status != CPA_STATUS_SUCCESS) {
- /* don't count unsupported checksums as a failure */
- if (cksum == ZIO_CHECKSUM_SHA256 ||
- cksum == ZIO_CHECKSUM_SHA512)
- QAT_STAT_BUMP(cksum_fails);
- return (status);
- }
-
- /*
- * We increment nr_bufs by 2 to allow us to handle non
- * page-aligned buffer addresses and buffers whose sizes
- * are not divisible by PAGE_SIZE.
- */
- status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
- &src_buffer_list, &src_buffer_list);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
- nr_bufs * sizeof (CpaFlatBuffer));
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
- status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
- sizeof (zio_cksum_t));
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- bytes_left = size;
- data = buf;
- flat_src_buf = flat_src_buf_array;
- while (bytes_left > 0) {
- page_off = ((long)data & ~PAGE_MASK);
- in_pages[page_num] = qat_mem_to_page(data);
- flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
- flat_src_buf->dataLenInBytes =
- min((long)PAGE_SIZE - page_off, (long)bytes_left);
- data += flat_src_buf->dataLenInBytes;
- bytes_left -= flat_src_buf->dataLenInBytes;
- flat_src_buf++;
- page_num++;
- }
- src_buffer_list.pBuffers = flat_src_buf_array;
- src_buffer_list.numBuffers = page_num;
-
- op_data.sessionCtx = cy_session_ctx;
- op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
- op_data.hashStartSrcOffsetInBytes = 0;
- op_data.messageLenToHashInBytes = size;
- op_data.pDigestResult = digest_buffer;
-
- cb.verify_result = CPA_FALSE;
- init_completion(&cb.complete);
- status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
- &src_buffer_list, &src_buffer_list, NULL);
- if (status != CPA_STATUS_SUCCESS)
- goto fail;
-
- if (!wait_for_completion_interruptible_timeout(&cb.complete,
- QAT_TIMEOUT_MS)) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
- if (cb.verify_result == CPA_FALSE) {
- status = CPA_STATUS_FAIL;
- goto fail;
- }
-
- bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
-
-fail:
- if (status != CPA_STATUS_SUCCESS)
- QAT_STAT_BUMP(cksum_fails);
-
- for (i = 0; i < page_num; i++)
- kunmap(in_pages[i]);
-
- cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
- QAT_PHYS_CONTIG_FREE(digest_buffer);
- QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
- QAT_PHYS_CONTIG_FREE(cy_session_ctx);
- QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
-
- return (status);
-}
-
-static int
-param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp)
-{
- int ret;
- int *pvalue = kp->arg;
- ret = param_set_int(val, kp);
- if (ret)
- return (ret);
- /*
- * zfs_qat_encrypt_disable = 0: enable qat encrypt
- * try to initialize qat instance if it has not been done
- */
- if (*pvalue == 0 && !qat_cy_init_done) {
- ret = qat_cy_init();
- if (ret != 0) {
- zfs_qat_encrypt_disable = 1;
- return (ret);
- }
- }
- return (ret);
-}
-
-static int
-param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp)
-{
- int ret;
- int *pvalue = kp->arg;
- ret = param_set_int(val, kp);
- if (ret)
- return (ret);
- /*
- * set_checksum_param_ops = 0: enable qat checksum
- * try to initialize qat instance if it has not been done
- */
- if (*pvalue == 0 && !qat_cy_init_done) {
- ret = qat_cy_init();
- if (ret != 0) {
- zfs_qat_checksum_disable = 1;
- return (ret);
- }
- }
- return (ret);
-}
-
-module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
- param_get_int, &zfs_qat_encrypt_disable, 0644);
-MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
-
-module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
- param_get_int, &zfs_qat_checksum_disable, 0644);
-MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
-
-#endif
diff --git a/module/zfs/sha256.c b/module/zfs/sha256.c
index 2adadf56f..406c926a0 100644
--- a/module/zfs/sha256.c
+++ b/module/zfs/sha256.c
@@ -30,7 +30,7 @@
#include <sys/zio.h>
#include <sys/sha2.h>
#include <sys/abd.h>
-#include "qat.h"
+#include <sys/qat.h>
static int
sha_incremental(void *buf, size_t size, void *arg)
diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c
index b0c1ae1e6..a18f9604a 100644
--- a/module/zfs/spa_misc.c
+++ b/module/zfs/spa_misc.c
@@ -59,7 +59,7 @@
#include <sys/kstat.h>
#include "zfs_prop.h"
#include <sys/zfeature.h>
-#include "qat.h"
+#include <sys/qat.h>
/*
* SPA locking
diff --git a/module/zfs/spa_stats.c b/module/zfs/spa_stats.c
deleted file mode 100644
index 6895428f4..000000000
--- a/module/zfs/spa_stats.c
+++ /dev/null
@@ -1,1034 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-#include <sys/zfs_context.h>
-#include <sys/spa_impl.h>
-#include <sys/vdev_impl.h>
-#include <sys/spa.h>
-#include <zfs_comutil.h>
-
-/*
- * Keeps stats on last N reads per spa_t, disabled by default.
- */
-int zfs_read_history = 0;
-
-/*
- * Include cache hits in history, disabled by default.
- */
-int zfs_read_history_hits = 0;
-
-/*
- * Keeps stats on the last 100 txgs by default.
- */
-int zfs_txg_history = 100;
-
-/*
- * Keeps stats on the last N MMP updates, disabled by default.
- */
-int zfs_multihost_history = 0;
-
-/*
- * ==========================================================================
- * SPA Read History Routines
- * ==========================================================================
- */
-
-/*
- * Read statistics - Information exported regarding each arc_read call
- */
-typedef struct spa_read_history {
- hrtime_t start; /* time read completed */
- uint64_t objset; /* read from this objset */
- uint64_t object; /* read of this object number */
- uint64_t level; /* block's indirection level */
- uint64_t blkid; /* read of this block id */
- char origin[24]; /* read originated from here */
- uint32_t aflags; /* ARC flags (cached, prefetch, etc.) */
- pid_t pid; /* PID of task doing read */
- char comm[16]; /* process name of task doing read */
- procfs_list_node_t srh_node;
-} spa_read_history_t;
-
-static int
-spa_read_history_show_header(struct seq_file *f)
-{
- seq_printf(f, "%-8s %-16s %-8s %-8s %-8s %-8s %-8s "
- "%-24s %-8s %-16s\n", "UID", "start", "objset", "object",
- "level", "blkid", "aflags", "origin", "pid", "process");
-
- return (0);
-}
-
-static int
-spa_read_history_show(struct seq_file *f, void *data)
-{
- spa_read_history_t *srh = (spa_read_history_t *)data;
-
- seq_printf(f, "%-8llu %-16llu 0x%-6llx "
- "%-8lli %-8lli %-8lli 0x%-6x %-24s %-8i %-16s\n",
- (u_longlong_t)srh->srh_node.pln_id, srh->start,
- (longlong_t)srh->objset, (longlong_t)srh->object,
- (longlong_t)srh->level, (longlong_t)srh->blkid,
- srh->aflags, srh->origin, srh->pid, srh->comm);
-
- return (0);
-}
-
-/* Remove oldest elements from list until there are no more than 'size' left */
-static void
-spa_read_history_truncate(spa_history_list_t *shl, unsigned int size)
-{
- spa_read_history_t *srh;
- while (shl->size > size) {
- srh = list_remove_head(&shl->procfs_list.pl_list);
- ASSERT3P(srh, !=, NULL);
- kmem_free(srh, sizeof (spa_read_history_t));
- shl->size--;
- }
-
- if (size == 0)
- ASSERT(list_is_empty(&shl->procfs_list.pl_list));
-}
-
-static int
-spa_read_history_clear(procfs_list_t *procfs_list)
-{
- spa_history_list_t *shl = procfs_list->pl_private;
- mutex_enter(&procfs_list->pl_lock);
- spa_read_history_truncate(shl, 0);
- mutex_exit(&procfs_list->pl_lock);
- return (0);
-}
-
-static void
-spa_read_history_init(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.read_history;
- char *module;
-
- shl->size = 0;
-
- module = kmem_asprintf("zfs/%s", spa_name(spa));
-
- shl->procfs_list.pl_private = shl;
- procfs_list_install(module,
- "reads",
- 0600,
- &shl->procfs_list,
- spa_read_history_show,
- spa_read_history_show_header,
- spa_read_history_clear,
- offsetof(spa_read_history_t, srh_node));
-
- strfree(module);
-}
-
-static void
-spa_read_history_destroy(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.read_history;
- procfs_list_uninstall(&shl->procfs_list);
- spa_read_history_truncate(shl, 0);
- procfs_list_destroy(&shl->procfs_list);
-}
-
-void
-spa_read_history_add(spa_t *spa, const zbookmark_phys_t *zb, uint32_t aflags)
-{
- spa_history_list_t *shl = &spa->spa_stats.read_history;
- spa_read_history_t *srh;
-
- ASSERT3P(spa, !=, NULL);
- ASSERT3P(zb, !=, NULL);
-
- if (zfs_read_history == 0 && shl->size == 0)
- return;
-
- if (zfs_read_history_hits == 0 && (aflags & ARC_FLAG_CACHED))
- return;
-
- srh = kmem_zalloc(sizeof (spa_read_history_t), KM_SLEEP);
- strlcpy(srh->comm, getcomm(), sizeof (srh->comm));
- srh->start = gethrtime();
- srh->objset = zb->zb_objset;
- srh->object = zb->zb_object;
- srh->level = zb->zb_level;
- srh->blkid = zb->zb_blkid;
- srh->aflags = aflags;
- srh->pid = getpid();
-
- mutex_enter(&shl->procfs_list.pl_lock);
-
- procfs_list_add(&shl->procfs_list, srh);
- shl->size++;
-
- spa_read_history_truncate(shl, zfs_read_history);
-
- mutex_exit(&shl->procfs_list.pl_lock);
-}
-
-/*
- * ==========================================================================
- * SPA TXG History Routines
- * ==========================================================================
- */
-
-/*
- * Txg statistics - Information exported regarding each txg sync
- */
-
-typedef struct spa_txg_history {
- uint64_t txg; /* txg id */
- txg_state_t state; /* active txg state */
- uint64_t nread; /* number of bytes read */
- uint64_t nwritten; /* number of bytes written */
- uint64_t reads; /* number of read operations */
- uint64_t writes; /* number of write operations */
- uint64_t ndirty; /* number of dirty bytes */
- hrtime_t times[TXG_STATE_COMMITTED]; /* completion times */
- procfs_list_node_t sth_node;
-} spa_txg_history_t;
-
-static int
-spa_txg_history_show_header(struct seq_file *f)
-{
- seq_printf(f, "%-8s %-16s %-5s %-12s %-12s %-12s "
- "%-8s %-8s %-12s %-12s %-12s %-12s\n", "txg", "birth", "state",
- "ndirty", "nread", "nwritten", "reads", "writes",
- "otime", "qtime", "wtime", "stime");
- return (0);
-}
-
-static int
-spa_txg_history_show(struct seq_file *f, void *data)
-{
- spa_txg_history_t *sth = (spa_txg_history_t *)data;
- uint64_t open = 0, quiesce = 0, wait = 0, sync = 0;
- char state;
-
- switch (sth->state) {
- case TXG_STATE_BIRTH: state = 'B'; break;
- case TXG_STATE_OPEN: state = 'O'; break;
- case TXG_STATE_QUIESCED: state = 'Q'; break;
- case TXG_STATE_WAIT_FOR_SYNC: state = 'W'; break;
- case TXG_STATE_SYNCED: state = 'S'; break;
- case TXG_STATE_COMMITTED: state = 'C'; break;
- default: state = '?'; break;
- }
-
- if (sth->times[TXG_STATE_OPEN])
- open = sth->times[TXG_STATE_OPEN] -
- sth->times[TXG_STATE_BIRTH];
-
- if (sth->times[TXG_STATE_QUIESCED])
- quiesce = sth->times[TXG_STATE_QUIESCED] -
- sth->times[TXG_STATE_OPEN];
-
- if (sth->times[TXG_STATE_WAIT_FOR_SYNC])
- wait = sth->times[TXG_STATE_WAIT_FOR_SYNC] -
- sth->times[TXG_STATE_QUIESCED];
-
- if (sth->times[TXG_STATE_SYNCED])
- sync = sth->times[TXG_STATE_SYNCED] -
- sth->times[TXG_STATE_WAIT_FOR_SYNC];
-
- seq_printf(f, "%-8llu %-16llu %-5c %-12llu "
- "%-12llu %-12llu %-8llu %-8llu %-12llu %-12llu %-12llu %-12llu\n",
- (longlong_t)sth->txg, sth->times[TXG_STATE_BIRTH], state,
- (u_longlong_t)sth->ndirty,
- (u_longlong_t)sth->nread, (u_longlong_t)sth->nwritten,
- (u_longlong_t)sth->reads, (u_longlong_t)sth->writes,
- (u_longlong_t)open, (u_longlong_t)quiesce, (u_longlong_t)wait,
- (u_longlong_t)sync);
-
- return (0);
-}
-
-/* Remove oldest elements from list until there are no more than 'size' left */
-static void
-spa_txg_history_truncate(spa_history_list_t *shl, unsigned int size)
-{
- spa_txg_history_t *sth;
- while (shl->size > size) {
- sth = list_remove_head(&shl->procfs_list.pl_list);
- ASSERT3P(sth, !=, NULL);
- kmem_free(sth, sizeof (spa_txg_history_t));
- shl->size--;
- }
-
- if (size == 0)
- ASSERT(list_is_empty(&shl->procfs_list.pl_list));
-
-}
-
-static int
-spa_txg_history_clear(procfs_list_t *procfs_list)
-{
- spa_history_list_t *shl = procfs_list->pl_private;
- mutex_enter(&procfs_list->pl_lock);
- spa_txg_history_truncate(shl, 0);
- mutex_exit(&procfs_list->pl_lock);
- return (0);
-}
-
-static void
-spa_txg_history_init(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.txg_history;
- char *module;
-
- shl->size = 0;
-
- module = kmem_asprintf("zfs/%s", spa_name(spa));
-
- shl->procfs_list.pl_private = shl;
- procfs_list_install(module,
- "txgs",
- 0644,
- &shl->procfs_list,
- spa_txg_history_show,
- spa_txg_history_show_header,
- spa_txg_history_clear,
- offsetof(spa_txg_history_t, sth_node));
-
- strfree(module);
-}
-
-static void
-spa_txg_history_destroy(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.txg_history;
- procfs_list_uninstall(&shl->procfs_list);
- spa_txg_history_truncate(shl, 0);
- procfs_list_destroy(&shl->procfs_list);
-}
-
-/*
- * Add a new txg to historical record.
- */
-void
-spa_txg_history_add(spa_t *spa, uint64_t txg, hrtime_t birth_time)
-{
- spa_history_list_t *shl = &spa->spa_stats.txg_history;
- spa_txg_history_t *sth;
-
- if (zfs_txg_history == 0 && shl->size == 0)
- return;
-
- sth = kmem_zalloc(sizeof (spa_txg_history_t), KM_SLEEP);
- sth->txg = txg;
- sth->state = TXG_STATE_OPEN;
- sth->times[TXG_STATE_BIRTH] = birth_time;
-
- mutex_enter(&shl->procfs_list.pl_lock);
- procfs_list_add(&shl->procfs_list, sth);
- shl->size++;
- spa_txg_history_truncate(shl, zfs_txg_history);
- mutex_exit(&shl->procfs_list.pl_lock);
-}
-
-/*
- * Set txg state completion time and increment current state.
- */
-int
-spa_txg_history_set(spa_t *spa, uint64_t txg, txg_state_t completed_state,
- hrtime_t completed_time)
-{
- spa_history_list_t *shl = &spa->spa_stats.txg_history;
- spa_txg_history_t *sth;
- int error = ENOENT;
-
- if (zfs_txg_history == 0)
- return (0);
-
- mutex_enter(&shl->procfs_list.pl_lock);
- for (sth = list_tail(&shl->procfs_list.pl_list); sth != NULL;
- sth = list_prev(&shl->procfs_list.pl_list, sth)) {
- if (sth->txg == txg) {
- sth->times[completed_state] = completed_time;
- sth->state++;
- error = 0;
- break;
- }
- }
- mutex_exit(&shl->procfs_list.pl_lock);
-
- return (error);
-}
-
-/*
- * Set txg IO stats.
- */
-static int
-spa_txg_history_set_io(spa_t *spa, uint64_t txg, uint64_t nread,
- uint64_t nwritten, uint64_t reads, uint64_t writes, uint64_t ndirty)
-{
- spa_history_list_t *shl = &spa->spa_stats.txg_history;
- spa_txg_history_t *sth;
- int error = ENOENT;
-
- if (zfs_txg_history == 0)
- return (0);
-
- mutex_enter(&shl->procfs_list.pl_lock);
- for (sth = list_tail(&shl->procfs_list.pl_list); sth != NULL;
- sth = list_prev(&shl->procfs_list.pl_list, sth)) {
- if (sth->txg == txg) {
- sth->nread = nread;
- sth->nwritten = nwritten;
- sth->reads = reads;
- sth->writes = writes;
- sth->ndirty = ndirty;
- error = 0;
- break;
- }
- }
- mutex_exit(&shl->procfs_list.pl_lock);
-
- return (error);
-}
-
-txg_stat_t *
-spa_txg_history_init_io(spa_t *spa, uint64_t txg, dsl_pool_t *dp)
-{
- txg_stat_t *ts;
-
- if (zfs_txg_history == 0)
- return (NULL);
-
- ts = kmem_alloc(sizeof (txg_stat_t), KM_SLEEP);
-
- spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
- vdev_get_stats(spa->spa_root_vdev, &ts->vs1);
- spa_config_exit(spa, SCL_CONFIG, FTAG);
-
- ts->txg = txg;
- ts->ndirty = dp->dp_dirty_pertxg[txg & TXG_MASK];
-
- spa_txg_history_set(spa, txg, TXG_STATE_WAIT_FOR_SYNC, gethrtime());
-
- return (ts);
-}
-
-void
-spa_txg_history_fini_io(spa_t *spa, txg_stat_t *ts)
-{
- if (ts == NULL)
- return;
-
- if (zfs_txg_history == 0) {
- kmem_free(ts, sizeof (txg_stat_t));
- return;
- }
-
- spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
- vdev_get_stats(spa->spa_root_vdev, &ts->vs2);
- spa_config_exit(spa, SCL_CONFIG, FTAG);
-
- spa_txg_history_set(spa, ts->txg, TXG_STATE_SYNCED, gethrtime());
- spa_txg_history_set_io(spa, ts->txg,
- ts->vs2.vs_bytes[ZIO_TYPE_READ] - ts->vs1.vs_bytes[ZIO_TYPE_READ],
- ts->vs2.vs_bytes[ZIO_TYPE_WRITE] - ts->vs1.vs_bytes[ZIO_TYPE_WRITE],
- ts->vs2.vs_ops[ZIO_TYPE_READ] - ts->vs1.vs_ops[ZIO_TYPE_READ],
- ts->vs2.vs_ops[ZIO_TYPE_WRITE] - ts->vs1.vs_ops[ZIO_TYPE_WRITE],
- ts->ndirty);
-
- kmem_free(ts, sizeof (txg_stat_t));
-}
-
-/*
- * ==========================================================================
- * SPA TX Assign Histogram Routines
- * ==========================================================================
- */
-
-/*
- * Tx statistics - Information exported regarding dmu_tx_assign time.
- */
-
-/*
- * When the kstat is written zero all buckets. When the kstat is read
- * count the number of trailing buckets set to zero and update ks_ndata
- * such that they are not output.
- */
-static int
-spa_tx_assign_update(kstat_t *ksp, int rw)
-{
- spa_t *spa = ksp->ks_private;
- spa_history_kstat_t *shk = &spa->spa_stats.tx_assign_histogram;
- int i;
-
- if (rw == KSTAT_WRITE) {
- for (i = 0; i < shk->count; i++)
- ((kstat_named_t *)shk->private)[i].value.ui64 = 0;
- }
-
- for (i = shk->count; i > 0; i--)
- if (((kstat_named_t *)shk->private)[i-1].value.ui64 != 0)
- break;
-
- ksp->ks_ndata = i;
- ksp->ks_data_size = i * sizeof (kstat_named_t);
-
- return (0);
-}
-
-static void
-spa_tx_assign_init(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.tx_assign_histogram;
- char *name;
- kstat_named_t *ks;
- kstat_t *ksp;
- int i;
-
- mutex_init(&shk->lock, NULL, MUTEX_DEFAULT, NULL);
-
- shk->count = 42; /* power of two buckets for 1ns to 2,199s */
- shk->size = shk->count * sizeof (kstat_named_t);
- shk->private = kmem_alloc(shk->size, KM_SLEEP);
-
- name = kmem_asprintf("zfs/%s", spa_name(spa));
-
- for (i = 0; i < shk->count; i++) {
- ks = &((kstat_named_t *)shk->private)[i];
- ks->data_type = KSTAT_DATA_UINT64;
- ks->value.ui64 = 0;
- (void) snprintf(ks->name, KSTAT_STRLEN, "%llu ns",
- (u_longlong_t)1 << i);
- }
-
- ksp = kstat_create(name, 0, "dmu_tx_assign", "misc",
- KSTAT_TYPE_NAMED, 0, KSTAT_FLAG_VIRTUAL);
- shk->kstat = ksp;
-
- if (ksp) {
- ksp->ks_lock = &shk->lock;
- ksp->ks_data = shk->private;
- ksp->ks_ndata = shk->count;
- ksp->ks_data_size = shk->size;
- ksp->ks_private = spa;
- ksp->ks_update = spa_tx_assign_update;
- kstat_install(ksp);
- }
- strfree(name);
-}
-
-static void
-spa_tx_assign_destroy(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.tx_assign_histogram;
- kstat_t *ksp;
-
- ksp = shk->kstat;
- if (ksp)
- kstat_delete(ksp);
-
- kmem_free(shk->private, shk->size);
- mutex_destroy(&shk->lock);
-}
-
-void
-spa_tx_assign_add_nsecs(spa_t *spa, uint64_t nsecs)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.tx_assign_histogram;
- uint64_t idx = 0;
-
- while (((1ULL << idx) < nsecs) && (idx < shk->size - 1))
- idx++;
-
- atomic_inc_64(&((kstat_named_t *)shk->private)[idx].value.ui64);
-}
-
-/*
- * ==========================================================================
- * SPA IO History Routines
- * ==========================================================================
- */
-static int
-spa_io_history_update(kstat_t *ksp, int rw)
-{
- if (rw == KSTAT_WRITE)
- memset(ksp->ks_data, 0, ksp->ks_data_size);
-
- return (0);
-}
-
-static void
-spa_io_history_init(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
- char *name;
- kstat_t *ksp;
-
- mutex_init(&shk->lock, NULL, MUTEX_DEFAULT, NULL);
-
- name = kmem_asprintf("zfs/%s", spa_name(spa));
-
- ksp = kstat_create(name, 0, "io", "disk", KSTAT_TYPE_IO, 1, 0);
- shk->kstat = ksp;
-
- if (ksp) {
- ksp->ks_lock = &shk->lock;
- ksp->ks_private = spa;
- ksp->ks_update = spa_io_history_update;
- kstat_install(ksp);
- }
- strfree(name);
-}
-
-static void
-spa_io_history_destroy(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
- if (shk->kstat)
- kstat_delete(shk->kstat);
-
- mutex_destroy(&shk->lock);
-}
-
-/*
- * ==========================================================================
- * SPA MMP History Routines
- * ==========================================================================
- */
-
-/*
- * MMP statistics - Information exported regarding attempted MMP writes
- * For MMP writes issued, fields used as per comments below.
- * For MMP writes skipped, an entry represents a span of time when
- * writes were skipped for same reason (error from mmp_random_leaf).
- * Differences are:
- * timestamp time first write skipped, if >1 skipped in a row
- * mmp_delay delay value at timestamp
- * vdev_guid number of writes skipped
- * io_error one of enum mmp_error
- * duration time span (ns) of skipped writes
- */
-
-typedef struct spa_mmp_history {
- uint64_t mmp_node_id; /* unique # for updates */
- uint64_t txg; /* txg of last sync */
- uint64_t timestamp; /* UTC time MMP write issued */
- uint64_t mmp_delay; /* mmp_thread.mmp_delay at timestamp */
- uint64_t vdev_guid; /* unique ID of leaf vdev */
- char *vdev_path;
- int vdev_label; /* vdev label */
- int io_error; /* error status of MMP write */
- hrtime_t error_start; /* hrtime of start of error period */
- hrtime_t duration; /* time from submission to completion */
- procfs_list_node_t smh_node;
-} spa_mmp_history_t;
-
-static int
-spa_mmp_history_show_header(struct seq_file *f)
-{
- seq_printf(f, "%-10s %-10s %-10s %-6s %-10s %-12s %-24s "
- "%-10s %s\n", "id", "txg", "timestamp", "error", "duration",
- "mmp_delay", "vdev_guid", "vdev_label", "vdev_path");
- return (0);
-}
-
-static int
-spa_mmp_history_show(struct seq_file *f, void *data)
-{
- spa_mmp_history_t *smh = (spa_mmp_history_t *)data;
- char skip_fmt[] = "%-10llu %-10llu %10llu %#6llx %10lld %12llu %-24llu "
- "%-10lld %s\n";
- char write_fmt[] = "%-10llu %-10llu %10llu %6lld %10lld %12llu %-24llu "
- "%-10lld %s\n";
-
- seq_printf(f, (smh->error_start ? skip_fmt : write_fmt),
- (u_longlong_t)smh->mmp_node_id, (u_longlong_t)smh->txg,
- (u_longlong_t)smh->timestamp, (longlong_t)smh->io_error,
- (longlong_t)smh->duration, (u_longlong_t)smh->mmp_delay,
- (u_longlong_t)smh->vdev_guid, (u_longlong_t)smh->vdev_label,
- (smh->vdev_path ? smh->vdev_path : "-"));
-
- return (0);
-}
-
-/* Remove oldest elements from list until there are no more than 'size' left */
-static void
-spa_mmp_history_truncate(spa_history_list_t *shl, unsigned int size)
-{
- spa_mmp_history_t *smh;
- while (shl->size > size) {
- smh = list_remove_head(&shl->procfs_list.pl_list);
- if (smh->vdev_path)
- strfree(smh->vdev_path);
- kmem_free(smh, sizeof (spa_mmp_history_t));
- shl->size--;
- }
-
- if (size == 0)
- ASSERT(list_is_empty(&shl->procfs_list.pl_list));
-
-}
-
-static int
-spa_mmp_history_clear(procfs_list_t *procfs_list)
-{
- spa_history_list_t *shl = procfs_list->pl_private;
- mutex_enter(&procfs_list->pl_lock);
- spa_mmp_history_truncate(shl, 0);
- mutex_exit(&procfs_list->pl_lock);
- return (0);
-}
-
-static void
-spa_mmp_history_init(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.mmp_history;
- char *module;
-
- shl->size = 0;
-
- module = kmem_asprintf("zfs/%s", spa_name(spa));
-
- shl->procfs_list.pl_private = shl;
- procfs_list_install(module,
- "multihost",
- 0644,
- &shl->procfs_list,
- spa_mmp_history_show,
- spa_mmp_history_show_header,
- spa_mmp_history_clear,
- offsetof(spa_mmp_history_t, smh_node));
-
- strfree(module);
-}
-
-static void
-spa_mmp_history_destroy(spa_t *spa)
-{
- spa_history_list_t *shl = &spa->spa_stats.mmp_history;
- procfs_list_uninstall(&shl->procfs_list);
- spa_mmp_history_truncate(shl, 0);
- procfs_list_destroy(&shl->procfs_list);
-}
-
-/*
- * Set duration in existing "skip" record to how long we have waited for a leaf
- * vdev to become available.
- *
- * Important that we start search at the tail of the list where new
- * records are inserted, so this is normally an O(1) operation.
- */
-int
-spa_mmp_history_set_skip(spa_t *spa, uint64_t mmp_node_id)
-{
- spa_history_list_t *shl = &spa->spa_stats.mmp_history;
- spa_mmp_history_t *smh;
- int error = ENOENT;
-
- if (zfs_multihost_history == 0 && shl->size == 0)
- return (0);
-
- mutex_enter(&shl->procfs_list.pl_lock);
- for (smh = list_tail(&shl->procfs_list.pl_list); smh != NULL;
- smh = list_prev(&shl->procfs_list.pl_list, smh)) {
- if (smh->mmp_node_id == mmp_node_id) {
- ASSERT3U(smh->io_error, !=, 0);
- smh->duration = gethrtime() - smh->error_start;
- smh->vdev_guid++;
- error = 0;
- break;
- }
- }
- mutex_exit(&shl->procfs_list.pl_lock);
-
- return (error);
-}
-
-/*
- * Set MMP write duration and error status in existing record.
- * See comment re: search order above spa_mmp_history_set_skip().
- */
-int
-spa_mmp_history_set(spa_t *spa, uint64_t mmp_node_id, int io_error,
- hrtime_t duration)
-{
- spa_history_list_t *shl = &spa->spa_stats.mmp_history;
- spa_mmp_history_t *smh;
- int error = ENOENT;
-
- if (zfs_multihost_history == 0 && shl->size == 0)
- return (0);
-
- mutex_enter(&shl->procfs_list.pl_lock);
- for (smh = list_tail(&shl->procfs_list.pl_list); smh != NULL;
- smh = list_prev(&shl->procfs_list.pl_list, smh)) {
- if (smh->mmp_node_id == mmp_node_id) {
- ASSERT(smh->io_error == 0);
- smh->io_error = io_error;
- smh->duration = duration;
- error = 0;
- break;
- }
- }
- mutex_exit(&shl->procfs_list.pl_lock);
-
- return (error);
-}
-
-/*
- * Add a new MMP historical record.
- * error == 0 : a write was issued.
- * error != 0 : a write was not issued because no leaves were found.
- */
-void
-spa_mmp_history_add(spa_t *spa, uint64_t txg, uint64_t timestamp,
- uint64_t mmp_delay, vdev_t *vd, int label, uint64_t mmp_node_id,
- int error)
-{
- spa_history_list_t *shl = &spa->spa_stats.mmp_history;
- spa_mmp_history_t *smh;
-
- if (zfs_multihost_history == 0 && shl->size == 0)
- return;
-
- smh = kmem_zalloc(sizeof (spa_mmp_history_t), KM_SLEEP);
- smh->txg = txg;
- smh->timestamp = timestamp;
- smh->mmp_delay = mmp_delay;
- if (vd) {
- smh->vdev_guid = vd->vdev_guid;
- if (vd->vdev_path)
- smh->vdev_path = strdup(vd->vdev_path);
- }
- smh->vdev_label = label;
- smh->mmp_node_id = mmp_node_id;
-
- if (error) {
- smh->io_error = error;
- smh->error_start = gethrtime();
- smh->vdev_guid = 1;
- }
-
- mutex_enter(&shl->procfs_list.pl_lock);
- procfs_list_add(&shl->procfs_list, smh);
- shl->size++;
- spa_mmp_history_truncate(shl, zfs_multihost_history);
- mutex_exit(&shl->procfs_list.pl_lock);
-}
-
-static void *
-spa_state_addr(kstat_t *ksp, loff_t n)
-{
- return (ksp->ks_private); /* return the spa_t */
-}
-
-static int
-spa_state_data(char *buf, size_t size, void *data)
-{
- spa_t *spa = (spa_t *)data;
- (void) snprintf(buf, size, "%s\n", spa_state_to_name(spa));
- return (0);
-}
-
-/*
- * Return the state of the pool in /proc/spl/kstat/zfs/<pool>/state.
- *
- * This is a lock-less read of the pool's state (unlike using 'zpool', which
- * can potentially block for seconds). Because it doesn't block, it can useful
- * as a pool heartbeat value.
- */
-static void
-spa_state_init(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.state;
- char *name;
- kstat_t *ksp;
-
- mutex_init(&shk->lock, NULL, MUTEX_DEFAULT, NULL);
-
- name = kmem_asprintf("zfs/%s", spa_name(spa));
- ksp = kstat_create(name, 0, "state", "misc",
- KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
-
- shk->kstat = ksp;
- if (ksp) {
- ksp->ks_lock = &shk->lock;
- ksp->ks_data = NULL;
- ksp->ks_private = spa;
- ksp->ks_flags |= KSTAT_FLAG_NO_HEADERS;
- kstat_set_raw_ops(ksp, NULL, spa_state_data, spa_state_addr);
- kstat_install(ksp);
- }
-
- strfree(name);
-}
-
-static void
-spa_health_destroy(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.state;
- kstat_t *ksp = shk->kstat;
- if (ksp)
- kstat_delete(ksp);
-
- mutex_destroy(&shk->lock);
-}
-
-static spa_iostats_t spa_iostats_template = {
- { "trim_extents_written", KSTAT_DATA_UINT64 },
- { "trim_bytes_written", KSTAT_DATA_UINT64 },
- { "trim_extents_skipped", KSTAT_DATA_UINT64 },
- { "trim_bytes_skipped", KSTAT_DATA_UINT64 },
- { "trim_extents_failed", KSTAT_DATA_UINT64 },
- { "trim_bytes_failed", KSTAT_DATA_UINT64 },
- { "autotrim_extents_written", KSTAT_DATA_UINT64 },
- { "autotrim_bytes_written", KSTAT_DATA_UINT64 },
- { "autotrim_extents_skipped", KSTAT_DATA_UINT64 },
- { "autotrim_bytes_skipped", KSTAT_DATA_UINT64 },
- { "autotrim_extents_failed", KSTAT_DATA_UINT64 },
- { "autotrim_bytes_failed", KSTAT_DATA_UINT64 },
-};
-
-#define SPA_IOSTATS_ADD(stat, val) \
- atomic_add_64(&iostats->stat.value.ui64, (val));
-
-void
-spa_iostats_trim_add(spa_t *spa, trim_type_t type,
- uint64_t extents_written, uint64_t bytes_written,
- uint64_t extents_skipped, uint64_t bytes_skipped,
- uint64_t extents_failed, uint64_t bytes_failed)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.iostats;
- kstat_t *ksp = shk->kstat;
- spa_iostats_t *iostats;
-
- if (ksp == NULL)
- return;
-
- iostats = ksp->ks_data;
- if (type == TRIM_TYPE_MANUAL) {
- SPA_IOSTATS_ADD(trim_extents_written, extents_written);
- SPA_IOSTATS_ADD(trim_bytes_written, bytes_written);
- SPA_IOSTATS_ADD(trim_extents_skipped, extents_skipped);
- SPA_IOSTATS_ADD(trim_bytes_skipped, bytes_skipped);
- SPA_IOSTATS_ADD(trim_extents_failed, extents_failed);
- SPA_IOSTATS_ADD(trim_bytes_failed, bytes_failed);
- } else {
- SPA_IOSTATS_ADD(autotrim_extents_written, extents_written);
- SPA_IOSTATS_ADD(autotrim_bytes_written, bytes_written);
- SPA_IOSTATS_ADD(autotrim_extents_skipped, extents_skipped);
- SPA_IOSTATS_ADD(autotrim_bytes_skipped, bytes_skipped);
- SPA_IOSTATS_ADD(autotrim_extents_failed, extents_failed);
- SPA_IOSTATS_ADD(autotrim_bytes_failed, bytes_failed);
- }
-}
-
-int
-spa_iostats_update(kstat_t *ksp, int rw)
-{
- if (rw == KSTAT_WRITE) {
- memcpy(ksp->ks_data, &spa_iostats_template,
- sizeof (spa_iostats_t));
- }
-
- return (0);
-}
-
-static void
-spa_iostats_init(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.iostats;
-
- mutex_init(&shk->lock, NULL, MUTEX_DEFAULT, NULL);
-
- char *name = kmem_asprintf("zfs/%s", spa_name(spa));
- kstat_t *ksp = kstat_create(name, 0, "iostats", "misc",
- KSTAT_TYPE_NAMED, sizeof (spa_iostats_t) / sizeof (kstat_named_t),
- KSTAT_FLAG_VIRTUAL);
-
- shk->kstat = ksp;
- if (ksp) {
- int size = sizeof (spa_iostats_t);
- ksp->ks_lock = &shk->lock;
- ksp->ks_private = spa;
- ksp->ks_update = spa_iostats_update;
- ksp->ks_data = kmem_alloc(size, KM_SLEEP);
- memcpy(ksp->ks_data, &spa_iostats_template, size);
- kstat_install(ksp);
- }
-
- strfree(name);
-}
-
-static void
-spa_iostats_destroy(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.iostats;
- kstat_t *ksp = shk->kstat;
- if (ksp) {
- kmem_free(ksp->ks_data, sizeof (spa_iostats_t));
- kstat_delete(ksp);
- }
-
- mutex_destroy(&shk->lock);
-}
-
-void
-spa_stats_init(spa_t *spa)
-{
- spa_read_history_init(spa);
- spa_txg_history_init(spa);
- spa_tx_assign_init(spa);
- spa_io_history_init(spa);
- spa_mmp_history_init(spa);
- spa_state_init(spa);
- spa_iostats_init(spa);
-}
-
-void
-spa_stats_destroy(spa_t *spa)
-{
- spa_iostats_destroy(spa);
- spa_health_destroy(spa);
- spa_tx_assign_destroy(spa);
- spa_txg_history_destroy(spa);
- spa_read_history_destroy(spa);
- spa_io_history_destroy(spa);
- spa_mmp_history_destroy(spa);
-}
-
-#if defined(_KERNEL)
-/* CSTYLED */
-module_param(zfs_read_history, int, 0644);
-MODULE_PARM_DESC(zfs_read_history,
- "Historical statistics for the last N reads");
-
-module_param(zfs_read_history_hits, int, 0644);
-MODULE_PARM_DESC(zfs_read_history_hits,
- "Include cache hits in read history");
-
-module_param(zfs_txg_history, int, 0644);
-MODULE_PARM_DESC(zfs_txg_history,
- "Historical statistics for the last N txgs");
-
-module_param(zfs_multihost_history, int, 0644);
-MODULE_PARM_DESC(zfs_multihost_history,
- "Historical statistics for last N multihost writes");
-/* END CSTYLED */
-#endif
diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c
deleted file mode 100644
index 21f9ae454..000000000
--- a/module/zfs/vdev_disk.c
+++ /dev/null
@@ -1,954 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
- * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- * Rewritten for Linux by Brian Behlendorf <[email protected]>.
- * LLNL-CODE-403049.
- * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
- */
-
-#include <sys/zfs_context.h>
-#include <sys/spa_impl.h>
-#include <sys/vdev_disk.h>
-#include <sys/vdev_impl.h>
-#include <sys/vdev_trim.h>
-#include <sys/abd.h>
-#include <sys/fs/zfs.h>
-#include <sys/zio.h>
-#include <linux/msdos_fs.h>
-#include <linux/vfs_compat.h>
-
-char *zfs_vdev_scheduler = VDEV_SCHEDULER;
-static void *zfs_vdev_holder = VDEV_HOLDER;
-
-/* size of the "reserved" partition, in blocks */
-#define EFI_MIN_RESV_SIZE (16 * 1024)
-
-/*
- * Virtual device vector for disks.
- */
-typedef struct dio_request {
- zio_t *dr_zio; /* Parent ZIO */
- atomic_t dr_ref; /* References */
- int dr_error; /* Bio error */
- int dr_bio_count; /* Count of bio's */
- struct bio *dr_bio[0]; /* Attached bio's */
-} dio_request_t;
-
-
-#if defined(HAVE_OPEN_BDEV_EXCLUSIVE) || defined(HAVE_BLKDEV_GET_BY_PATH)
-static fmode_t
-vdev_bdev_mode(int smode)
-{
- fmode_t mode = 0;
-
- ASSERT3S(smode & (FREAD | FWRITE), !=, 0);
-
- if (smode & FREAD)
- mode |= FMODE_READ;
-
- if (smode & FWRITE)
- mode |= FMODE_WRITE;
-
- return (mode);
-}
-#else
-static int
-vdev_bdev_mode(int smode)
-{
- int mode = 0;
-
- ASSERT3S(smode & (FREAD | FWRITE), !=, 0);
-
- if ((smode & FREAD) && !(smode & FWRITE))
- mode = SB_RDONLY;
-
- return (mode);
-}
-#endif /* HAVE_OPEN_BDEV_EXCLUSIVE */
-
-/*
- * Returns the usable capacity (in bytes) for the partition or disk.
- */
-static uint64_t
-bdev_capacity(struct block_device *bdev)
-{
- return (i_size_read(bdev->bd_inode));
-}
-
-/*
- * Returns the maximum expansion capacity of the block device (in bytes).
- *
- * It is possible to expand a vdev when it has been created as a wholedisk
- * and the containing block device has increased in capacity. Or when the
- * partition containing the pool has been manually increased in size.
- *
- * This function is only responsible for calculating the potential expansion
- * size so it can be reported by 'zpool list'. The efi_use_whole_disk() is
- * responsible for verifying the expected partition layout in the wholedisk
- * case, and updating the partition table if appropriate. Once the partition
- * size has been increased the additional capacity will be visible using
- * bdev_capacity().
- *
- * The returned maximum expansion capacity is always expected to be larger, or
- * at the very least equal, to its usable capacity to prevent overestimating
- * the pool expandsize.
- */
-static uint64_t
-bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
-{
- uint64_t psize;
- int64_t available;
-
- if (wholedisk && bdev->bd_part != NULL && bdev != bdev->bd_contains) {
- /*
- * When reporting maximum expansion capacity for a wholedisk
- * deduct any capacity which is expected to be lost due to
- * alignment restrictions. Over reporting this value isn't
- * harmful and would only result in slightly less capacity
- * than expected post expansion.
- * The estimated available space may be slightly smaller than
- * bdev_capacity() for devices where the number of sectors is
- * not a multiple of the alignment size and the partition layout
- * is keeping less than PARTITION_END_ALIGNMENT bytes after the
- * "reserved" EFI partition: in such cases return the device
- * usable capacity.
- */
- available = i_size_read(bdev->bd_contains->bd_inode) -
- ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
- PARTITION_END_ALIGNMENT) << SECTOR_BITS);
- psize = MAX(available, bdev_capacity(bdev));
- } else {
- psize = bdev_capacity(bdev);
- }
-
- return (psize);
-}
-
-static void
-vdev_disk_error(zio_t *zio)
-{
- /*
- * This function can be called in interrupt context, for instance while
- * handling IRQs coming from a misbehaving disk device; use printk()
- * which is safe from any context.
- */
- printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
- "offset=%llu size=%llu flags=%x\n", spa_name(zio->io_spa),
- zio->io_vd->vdev_path, zio->io_error, zio->io_type,
- (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
- zio->io_flags);
-}
-
-/*
- * Use the Linux 'noop' elevator for zfs managed block devices. This
- * strikes the ideal balance by allowing the zfs elevator to do all
- * request ordering and prioritization. While allowing the Linux
- * elevator to do the maximum front/back merging allowed by the
- * physical device. This yields the largest possible requests for
- * the device with the lowest total overhead.
- */
-static void
-vdev_elevator_switch(vdev_t *v, char *elevator)
-{
- vdev_disk_t *vd = v->vdev_tsd;
- struct request_queue *q;
- char *device;
- int error;
-
- for (int c = 0; c < v->vdev_children; c++)
- vdev_elevator_switch(v->vdev_child[c], elevator);
-
- if (!v->vdev_ops->vdev_op_leaf || vd->vd_bdev == NULL)
- return;
-
- q = bdev_get_queue(vd->vd_bdev);
- device = vd->vd_bdev->bd_disk->disk_name;
-
- /*
- * Skip devices which are not whole disks (partitions).
- * Device-mapper devices are excepted since they may be whole
- * disks despite the vdev_wholedisk flag, in which case we can
- * and should switch the elevator. If the device-mapper device
- * does not have an elevator (i.e. dm-raid, dm-crypt, etc.) the
- * "Skip devices without schedulers" check below will fail.
- */
- if (!v->vdev_wholedisk && strncmp(device, "dm-", 3) != 0)
- return;
-
- /* Leave existing scheduler when set to "none" */
- if ((strncmp(elevator, "none", 4) == 0) && (strlen(elevator) == 4))
- return;
-
- /*
- * The elevator_change() function was available in kernels from
- * 2.6.36 to 4.11. When not available fall back to using the user
- * mode helper functionality to set the elevator via sysfs. This
- * requires /bin/echo and sysfs to be mounted which may not be true
- * early in the boot process.
- */
-#ifdef HAVE_ELEVATOR_CHANGE
- error = elevator_change(q, elevator);
-#else
-#define SET_SCHEDULER_CMD \
- "exec 0</dev/null " \
- " 1>/sys/block/%s/queue/scheduler " \
- " 2>/dev/null; " \
- "echo %s"
-
- char *argv[] = { "/bin/sh", "-c", NULL, NULL };
- char *envp[] = { NULL };
-
- argv[2] = kmem_asprintf(SET_SCHEDULER_CMD, device, elevator);
- error = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
- strfree(argv[2]);
-#endif /* HAVE_ELEVATOR_CHANGE */
- if (error) {
- zfs_dbgmsg("Unable to set \"%s\" scheduler for %s (%s): %d",
- elevator, v->vdev_path, device, error);
- }
-}
-
-static int
-vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
- uint64_t *ashift)
-{
- struct block_device *bdev;
- fmode_t mode = vdev_bdev_mode(spa_mode(v->vdev_spa));
- int count = 0, block_size;
- int bdev_retry_count = 50;
- vdev_disk_t *vd;
-
- /* Must have a pathname and it must be absolute. */
- if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
- v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
- vdev_dbgmsg(v, "invalid vdev_path");
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Reopen the device if it is currently open. When expanding a
- * partition force re-scanning the partition table while closed
- * in order to get an accurate updated block device size. Then
- * since udev may need to recreate the device links increase the
- * open retry count before reporting the device as unavailable.
- */
- vd = v->vdev_tsd;
- if (vd) {
- char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
- boolean_t reread_part = B_FALSE;
-
- rw_enter(&vd->vd_lock, RW_WRITER);
- bdev = vd->vd_bdev;
- vd->vd_bdev = NULL;
-
- if (bdev) {
- if (v->vdev_expanding && bdev != bdev->bd_contains) {
- bdevname(bdev->bd_contains, disk_name + 5);
- reread_part = B_TRUE;
- }
-
- vdev_bdev_close(bdev, mode);
- }
-
- if (reread_part) {
- bdev = vdev_bdev_open(disk_name, mode, zfs_vdev_holder);
- if (!IS_ERR(bdev)) {
- int error = vdev_bdev_reread_part(bdev);
- vdev_bdev_close(bdev, mode);
- if (error == 0)
- bdev_retry_count = 100;
- }
- }
- } else {
- vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);
-
- rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
- rw_enter(&vd->vd_lock, RW_WRITER);
- }
-
- /*
- * Devices are always opened by the path provided at configuration
- * time. This means that if the provided path is a udev by-id path
- * then drives may be re-cabled without an issue. If the provided
- * path is a udev by-path path, then the physical location information
- * will be preserved. This can be critical for more complicated
- * configurations where drives are located in specific physical
- * locations to maximize the systems tolerance to component failure.
- *
- * Alternatively, you can provide your own udev rule to flexibly map
- * the drives as you see fit. It is not advised that you use the
- * /dev/[hd]d devices which may be reordered due to probing order.
- * Devices in the wrong locations will be detected by the higher
- * level vdev validation.
- *
- * The specified paths may be briefly removed and recreated in
- * response to udev events. This should be exceptionally unlikely
- * because the zpool command makes every effort to verify these paths
- * have already settled prior to reaching this point. Therefore,
- * a ENOENT failure at this point is highly likely to be transient
- * and it is reasonable to sleep and retry before giving up. In
- * practice delays have been observed to be on the order of 100ms.
- */
- bdev = ERR_PTR(-ENXIO);
- while (IS_ERR(bdev) && count < bdev_retry_count) {
- bdev = vdev_bdev_open(v->vdev_path, mode, zfs_vdev_holder);
- if (unlikely(PTR_ERR(bdev) == -ENOENT)) {
- schedule_timeout(MSEC_TO_TICK(10));
- count++;
- } else if (IS_ERR(bdev)) {
- break;
- }
- }
-
- if (IS_ERR(bdev)) {
- int error = -PTR_ERR(bdev);
- vdev_dbgmsg(v, "open error=%d count=%d", error, count);
- vd->vd_bdev = NULL;
- v->vdev_tsd = vd;
- rw_exit(&vd->vd_lock);
- return (SET_ERROR(error));
- } else {
- vd->vd_bdev = bdev;
- v->vdev_tsd = vd;
- rw_exit(&vd->vd_lock);
- }
-
- struct request_queue *q = bdev_get_queue(vd->vd_bdev);
-
- /* Determine the physical block size */
- block_size = vdev_bdev_block_size(vd->vd_bdev);
-
- /* Clear the nowritecache bit, causes vdev_reopen() to try again. */
- v->vdev_nowritecache = B_FALSE;
-
- /* Set when device reports it supports TRIM. */
- v->vdev_has_trim = !!blk_queue_discard(q);
-
- /* Set when device reports it supports secure TRIM. */
- v->vdev_has_securetrim = !!blk_queue_discard_secure(q);
-
- /* Inform the ZIO pipeline that we are non-rotational */
- v->vdev_nonrot = blk_queue_nonrot(q);
-
- /* Physical volume size in bytes for the partition */
- *psize = bdev_capacity(vd->vd_bdev);
-
- /* Physical volume size in bytes including possible expansion space */
- *max_psize = bdev_max_capacity(vd->vd_bdev, v->vdev_wholedisk);
-
- /* Based on the minimum sector size set the block size */
- *ashift = highbit64(MAX(block_size, SPA_MINBLOCKSIZE)) - 1;
-
- /* Try to set the io scheduler elevator algorithm */
- (void) vdev_elevator_switch(v, zfs_vdev_scheduler);
-
- return (0);
-}
-
-static void
-vdev_disk_close(vdev_t *v)
-{
- vdev_disk_t *vd = v->vdev_tsd;
-
- if (v->vdev_reopening || vd == NULL)
- return;
-
- if (vd->vd_bdev != NULL) {
- vdev_bdev_close(vd->vd_bdev,
- vdev_bdev_mode(spa_mode(v->vdev_spa)));
- }
-
- rw_destroy(&vd->vd_lock);
- kmem_free(vd, sizeof (vdev_disk_t));
- v->vdev_tsd = NULL;
-}
-
-static dio_request_t *
-vdev_disk_dio_alloc(int bio_count)
-{
- dio_request_t *dr;
- int i;
-
- dr = kmem_zalloc(sizeof (dio_request_t) +
- sizeof (struct bio *) * bio_count, KM_SLEEP);
- if (dr) {
- atomic_set(&dr->dr_ref, 0);
- dr->dr_bio_count = bio_count;
- dr->dr_error = 0;
-
- for (i = 0; i < dr->dr_bio_count; i++)
- dr->dr_bio[i] = NULL;
- }
-
- return (dr);
-}
-
-static void
-vdev_disk_dio_free(dio_request_t *dr)
-{
- int i;
-
- for (i = 0; i < dr->dr_bio_count; i++)
- if (dr->dr_bio[i])
- bio_put(dr->dr_bio[i]);
-
- kmem_free(dr, sizeof (dio_request_t) +
- sizeof (struct bio *) * dr->dr_bio_count);
-}
-
-static void
-vdev_disk_dio_get(dio_request_t *dr)
-{
- atomic_inc(&dr->dr_ref);
-}
-
-static int
-vdev_disk_dio_put(dio_request_t *dr)
-{
- int rc = atomic_dec_return(&dr->dr_ref);
-
- /*
- * Free the dio_request when the last reference is dropped and
- * ensure zio_interpret is called only once with the correct zio
- */
- if (rc == 0) {
- zio_t *zio = dr->dr_zio;
- int error = dr->dr_error;
-
- vdev_disk_dio_free(dr);
-
- if (zio) {
- zio->io_error = error;
- ASSERT3S(zio->io_error, >=, 0);
- if (zio->io_error)
- vdev_disk_error(zio);
-
- zio_delay_interrupt(zio);
- }
- }
-
- return (rc);
-}
-
-BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, error)
-{
- dio_request_t *dr = bio->bi_private;
- int rc;
-
- if (dr->dr_error == 0) {
-#ifdef HAVE_1ARG_BIO_END_IO_T
- dr->dr_error = BIO_END_IO_ERROR(bio);
-#else
- if (error)
- dr->dr_error = -(error);
- else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
- dr->dr_error = EIO;
-#endif
- }
-
- /* Drop reference acquired by __vdev_disk_physio */
- rc = vdev_disk_dio_put(dr);
-}
-
-static unsigned int
-bio_map(struct bio *bio, void *bio_ptr, unsigned int bio_size)
-{
- unsigned int offset, size, i;
- struct page *page;
-
- offset = offset_in_page(bio_ptr);
- for (i = 0; i < bio->bi_max_vecs; i++) {
- size = PAGE_SIZE - offset;
-
- if (bio_size <= 0)
- break;
-
- if (size > bio_size)
- size = bio_size;
-
- if (is_vmalloc_addr(bio_ptr))
- page = vmalloc_to_page(bio_ptr);
- else
- page = virt_to_page(bio_ptr);
-
- /*
- * Some network related block device uses tcp_sendpage, which
- * doesn't behave well when using 0-count page, this is a
- * safety net to catch them.
- */
- ASSERT3S(page_count(page), >, 0);
-
- if (bio_add_page(bio, page, size, offset) != size)
- break;
-
- bio_ptr += size;
- bio_size -= size;
- offset = 0;
- }
-
- return (bio_size);
-}
-
-static unsigned int
-bio_map_abd_off(struct bio *bio, abd_t *abd, unsigned int size, size_t off)
-{
- if (abd_is_linear(abd))
- return (bio_map(bio, ((char *)abd_to_buf(abd)) + off, size));
-
- return (abd_scatter_bio_map_off(bio, abd, size, off));
-}
-
-static inline void
-vdev_submit_bio_impl(struct bio *bio)
-{
-#ifdef HAVE_1ARG_SUBMIT_BIO
- submit_bio(bio);
-#else
- submit_bio(0, bio);
-#endif
-}
-
-#ifdef HAVE_BIO_SET_DEV
-#if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
-/*
- * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
- * GPL-only bio_associate_blkg() symbol thus inadvertently converting
- * the entire macro. Provide a minimal version which always assigns the
- * request queue's root_blkg to the bio.
- */
-static inline void
-vdev_bio_associate_blkg(struct bio *bio)
-{
- struct request_queue *q = bio->bi_disk->queue;
-
- ASSERT3P(q, !=, NULL);
- ASSERT3P(bio->bi_blkg, ==, NULL);
-
- if (blkg_tryget(q->root_blkg))
- bio->bi_blkg = q->root_blkg;
-}
-#define bio_associate_blkg vdev_bio_associate_blkg
-#endif
-#else
-/*
- * Provide a bio_set_dev() helper macro for pre-Linux 4.14 kernels.
- */
-static inline void
-bio_set_dev(struct bio *bio, struct block_device *bdev)
-{
- bio->bi_bdev = bdev;
-}
-#endif /* HAVE_BIO_SET_DEV */
-
-static inline void
-vdev_submit_bio(struct bio *bio)
-{
-#ifdef HAVE_CURRENT_BIO_TAIL
- struct bio **bio_tail = current->bio_tail;
- current->bio_tail = NULL;
- vdev_submit_bio_impl(bio);
- current->bio_tail = bio_tail;
-#else
- struct bio_list *bio_list = current->bio_list;
- current->bio_list = NULL;
- vdev_submit_bio_impl(bio);
- current->bio_list = bio_list;
-#endif
-}
-
-static int
-__vdev_disk_physio(struct block_device *bdev, zio_t *zio,
- size_t io_size, uint64_t io_offset, int rw, int flags)
-{
- dio_request_t *dr;
- uint64_t abd_offset;
- uint64_t bio_offset;
- int bio_size, bio_count = 16;
- int i = 0, error = 0;
-#if defined(HAVE_BLK_QUEUE_HAVE_BLK_PLUG)
- struct blk_plug plug;
-#endif
- /*
- * Accessing outside the block device is never allowed.
- */
- if (io_offset + io_size > bdev->bd_inode->i_size) {
- vdev_dbgmsg(zio->io_vd,
- "Illegal access %llu size %llu, device size %llu",
- io_offset, io_size, i_size_read(bdev->bd_inode));
- return (SET_ERROR(EIO));
- }
-
-retry:
- dr = vdev_disk_dio_alloc(bio_count);
- if (dr == NULL)
- return (SET_ERROR(ENOMEM));
-
- if (zio && !(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))
- bio_set_flags_failfast(bdev, &flags);
-
- dr->dr_zio = zio;
-
- /*
- * When the IO size exceeds the maximum bio size for the request
- * queue we are forced to break the IO in multiple bio's and wait
- * for them all to complete. Ideally, all pool users will set
- * their volume block size to match the maximum request size and
- * the common case will be one bio per vdev IO request.
- */
-
- abd_offset = 0;
- bio_offset = io_offset;
- bio_size = io_size;
- for (i = 0; i <= dr->dr_bio_count; i++) {
-
- /* Finished constructing bio's for given buffer */
- if (bio_size <= 0)
- break;
-
- /*
- * By default only 'bio_count' bio's per dio are allowed.
- * However, if we find ourselves in a situation where more
- * are needed we allocate a larger dio and warn the user.
- */
- if (dr->dr_bio_count == i) {
- vdev_disk_dio_free(dr);
- bio_count *= 2;
- goto retry;
- }
-
- /* bio_alloc() with __GFP_WAIT never returns NULL */
- dr->dr_bio[i] = bio_alloc(GFP_NOIO,
- MIN(abd_nr_pages_off(zio->io_abd, bio_size, abd_offset),
- BIO_MAX_PAGES));
- if (unlikely(dr->dr_bio[i] == NULL)) {
- vdev_disk_dio_free(dr);
- return (SET_ERROR(ENOMEM));
- }
-
- /* Matching put called by vdev_disk_physio_completion */
- vdev_disk_dio_get(dr);
-
- bio_set_dev(dr->dr_bio[i], bdev);
- BIO_BI_SECTOR(dr->dr_bio[i]) = bio_offset >> 9;
- dr->dr_bio[i]->bi_end_io = vdev_disk_physio_completion;
- dr->dr_bio[i]->bi_private = dr;
- bio_set_op_attrs(dr->dr_bio[i], rw, flags);
-
- /* Remaining size is returned to become the new size */
- bio_size = bio_map_abd_off(dr->dr_bio[i], zio->io_abd,
- bio_size, abd_offset);
-
- /* Advance in buffer and construct another bio if needed */
- abd_offset += BIO_BI_SIZE(dr->dr_bio[i]);
- bio_offset += BIO_BI_SIZE(dr->dr_bio[i]);
- }
-
- /* Extra reference to protect dio_request during vdev_submit_bio */
- vdev_disk_dio_get(dr);
-
-#if defined(HAVE_BLK_QUEUE_HAVE_BLK_PLUG)
- if (dr->dr_bio_count > 1)
- blk_start_plug(&plug);
-#endif
-
- /* Submit all bio's associated with this dio */
- for (i = 0; i < dr->dr_bio_count; i++)
- if (dr->dr_bio[i])
- vdev_submit_bio(dr->dr_bio[i]);
-
-#if defined(HAVE_BLK_QUEUE_HAVE_BLK_PLUG)
- if (dr->dr_bio_count > 1)
- blk_finish_plug(&plug);
-#endif
-
- (void) vdev_disk_dio_put(dr);
-
- return (error);
-}
-
-BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error)
-{
- zio_t *zio = bio->bi_private;
-#ifdef HAVE_1ARG_BIO_END_IO_T
- zio->io_error = BIO_END_IO_ERROR(bio);
-#else
- zio->io_error = -error;
-#endif
-
- if (zio->io_error && (zio->io_error == EOPNOTSUPP))
- zio->io_vd->vdev_nowritecache = B_TRUE;
-
- bio_put(bio);
- ASSERT3S(zio->io_error, >=, 0);
- if (zio->io_error)
- vdev_disk_error(zio);
- zio_interrupt(zio);
-}
-
-static int
-vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
-{
- struct request_queue *q;
- struct bio *bio;
-
- q = bdev_get_queue(bdev);
- if (!q)
- return (SET_ERROR(ENXIO));
-
- bio = bio_alloc(GFP_NOIO, 0);
- /* bio_alloc() with __GFP_WAIT never returns NULL */
- if (unlikely(bio == NULL))
- return (SET_ERROR(ENOMEM));
-
- bio->bi_end_io = vdev_disk_io_flush_completion;
- bio->bi_private = zio;
- bio_set_dev(bio, bdev);
- bio_set_flush(bio);
- vdev_submit_bio(bio);
- invalidate_bdev(bdev);
-
- return (0);
-}
-
-static void
-vdev_disk_io_start(zio_t *zio)
-{
- vdev_t *v = zio->io_vd;
- vdev_disk_t *vd = v->vdev_tsd;
- unsigned long trim_flags = 0;
- int rw, flags, error;
-
- /*
- * If the vdev is closed, it's likely in the REMOVED or FAULTED state.
- * Nothing to be done here but return failure.
- */
- if (vd == NULL) {
- zio->io_error = ENXIO;
- zio_interrupt(zio);
- return;
- }
-
- rw_enter(&vd->vd_lock, RW_READER);
-
- /*
- * If the vdev is closed, it's likely due to a failed reopen and is
- * in the UNAVAIL state. Nothing to be done here but return failure.
- */
- if (vd->vd_bdev == NULL) {
- rw_exit(&vd->vd_lock);
- zio->io_error = ENXIO;
- zio_interrupt(zio);
- return;
- }
-
- switch (zio->io_type) {
- case ZIO_TYPE_IOCTL:
-
- if (!vdev_readable(v)) {
- rw_exit(&vd->vd_lock);
- zio->io_error = SET_ERROR(ENXIO);
- zio_interrupt(zio);
- return;
- }
-
- switch (zio->io_cmd) {
- case DKIOCFLUSHWRITECACHE:
-
- if (zfs_nocacheflush)
- break;
-
- if (v->vdev_nowritecache) {
- zio->io_error = SET_ERROR(ENOTSUP);
- break;
- }
-
- error = vdev_disk_io_flush(vd->vd_bdev, zio);
- if (error == 0) {
- rw_exit(&vd->vd_lock);
- return;
- }
-
- zio->io_error = error;
-
- break;
-
- default:
- zio->io_error = SET_ERROR(ENOTSUP);
- }
-
- rw_exit(&vd->vd_lock);
- zio_execute(zio);
- return;
- case ZIO_TYPE_WRITE:
- rw = WRITE;
-#if defined(HAVE_BLK_QUEUE_HAVE_BIO_RW_UNPLUG)
- flags = (1 << BIO_RW_UNPLUG);
-#elif defined(REQ_UNPLUG)
- flags = REQ_UNPLUG;
-#else
- flags = 0;
-#endif
- break;
-
- case ZIO_TYPE_READ:
- rw = READ;
-#if defined(HAVE_BLK_QUEUE_HAVE_BIO_RW_UNPLUG)
- flags = (1 << BIO_RW_UNPLUG);
-#elif defined(REQ_UNPLUG)
- flags = REQ_UNPLUG;
-#else
- flags = 0;
-#endif
- break;
-
- case ZIO_TYPE_TRIM:
-#if defined(BLKDEV_DISCARD_SECURE)
- if (zio->io_trim_flags & ZIO_TRIM_SECURE)
- trim_flags |= BLKDEV_DISCARD_SECURE;
-#endif
- zio->io_error = -blkdev_issue_discard(vd->vd_bdev,
- zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS,
- trim_flags);
-
- rw_exit(&vd->vd_lock);
- zio_interrupt(zio);
- return;
-
- default:
- rw_exit(&vd->vd_lock);
- zio->io_error = SET_ERROR(ENOTSUP);
- zio_interrupt(zio);
- return;
- }
-
- zio->io_target_timestamp = zio_handle_io_delay(zio);
- error = __vdev_disk_physio(vd->vd_bdev, zio,
- zio->io_size, zio->io_offset, rw, flags);
- rw_exit(&vd->vd_lock);
-
- if (error) {
- zio->io_error = error;
- zio_interrupt(zio);
- return;
- }
-}
-
-static void
-vdev_disk_io_done(zio_t *zio)
-{
- /*
- * If the device returned EIO, we revalidate the media. If it is
- * determined the media has changed this triggers the asynchronous
- * removal of the device from the configuration.
- */
- if (zio->io_error == EIO) {
- vdev_t *v = zio->io_vd;
- vdev_disk_t *vd = v->vdev_tsd;
-
- if (check_disk_change(vd->vd_bdev)) {
- vdev_bdev_invalidate(vd->vd_bdev);
- v->vdev_remove_wanted = B_TRUE;
- spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
- }
- }
-}
-
-static void
-vdev_disk_hold(vdev_t *vd)
-{
- ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
-
- /* We must have a pathname, and it must be absolute. */
- if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
- return;
-
- /*
- * Only prefetch path and devid info if the device has
- * never been opened.
- */
- if (vd->vdev_tsd != NULL)
- return;
-
- /* XXX: Implement me as a vnode lookup for the device */
- vd->vdev_name_vp = NULL;
- vd->vdev_devid_vp = NULL;
-}
-
-static void
-vdev_disk_rele(vdev_t *vd)
-{
- ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
-
- /* XXX: Implement me as a vnode rele for the device */
-}
-
-static int
-param_set_vdev_scheduler(const char *val, zfs_kernel_param_t *kp)
-{
- spa_t *spa = NULL;
- char *p;
-
- if (val == NULL)
- return (SET_ERROR(-EINVAL));
-
- if ((p = strchr(val, '\n')) != NULL)
- *p = '\0';
-
- if (spa_mode_global != 0) {
- mutex_enter(&spa_namespace_lock);
- while ((spa = spa_next(spa)) != NULL) {
- if (spa_state(spa) != POOL_STATE_ACTIVE ||
- !spa_writeable(spa) || spa_suspended(spa))
- continue;
-
- spa_open_ref(spa, FTAG);
- mutex_exit(&spa_namespace_lock);
- vdev_elevator_switch(spa->spa_root_vdev, (char *)val);
- mutex_enter(&spa_namespace_lock);
- spa_close(spa, FTAG);
- }
- mutex_exit(&spa_namespace_lock);
- }
-
- return (param_set_charp(val, kp));
-}
-
-vdev_ops_t vdev_disk_ops = {
- .vdev_op_open = vdev_disk_open,
- .vdev_op_close = vdev_disk_close,
- .vdev_op_asize = vdev_default_asize,
- .vdev_op_io_start = vdev_disk_io_start,
- .vdev_op_io_done = vdev_disk_io_done,
- .vdev_op_state_change = NULL,
- .vdev_op_need_resilver = NULL,
- .vdev_op_hold = vdev_disk_hold,
- .vdev_op_rele = vdev_disk_rele,
- .vdev_op_remap = NULL,
- .vdev_op_xlate = vdev_default_xlate,
- .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
- .vdev_op_leaf = B_TRUE /* leaf vdev */
-};
-
-module_param_call(zfs_vdev_scheduler, param_set_vdev_scheduler,
- param_get_charp, &zfs_vdev_scheduler, 0644);
-MODULE_PARM_DESC(zfs_vdev_scheduler, "I/O scheduler");
diff --git a/module/zfs/vdev_file.c b/module/zfs/vdev_file.c
deleted file mode 100644
index b79017f3a..000000000
--- a/module/zfs/vdev_file.c
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
- */
-
-#include <sys/zfs_context.h>
-#include <sys/spa.h>
-#include <sys/spa_impl.h>
-#include <sys/vdev_file.h>
-#include <sys/vdev_impl.h>
-#include <sys/vdev_trim.h>
-#include <sys/zio.h>
-#include <sys/fs/zfs.h>
-#include <sys/fm/fs/zfs.h>
-#include <sys/abd.h>
-#include <sys/fcntl.h>
-#include <sys/vnode.h>
-
-/*
- * Virtual device vector for files.
- */
-
-static taskq_t *vdev_file_taskq;
-
-static void
-vdev_file_hold(vdev_t *vd)
-{
- ASSERT(vd->vdev_path != NULL);
-}
-
-static void
-vdev_file_rele(vdev_t *vd)
-{
- ASSERT(vd->vdev_path != NULL);
-}
-
-static int
-vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
- uint64_t *ashift)
-{
- vdev_file_t *vf;
- vnode_t *vp;
- vattr_t vattr;
- int error;
-
- /*
- * Rotational optimizations only make sense on block devices.
- */
- vd->vdev_nonrot = B_TRUE;
-
- /*
- * Allow TRIM on file based vdevs. This may not always be supported,
- * since it depends on your kernel version and underlying filesystem
- * type but it is always safe to attempt.
- */
- vd->vdev_has_trim = B_TRUE;
-
- /*
- * Disable secure TRIM on file based vdevs. There is no way to
- * request this behavior from the underlying filesystem.
- */
- vd->vdev_has_securetrim = B_FALSE;
-
- /*
- * We must have a pathname, and it must be absolute.
- */
- if (vd->vdev_path == NULL || vd->vdev_path[0] != '/') {
- vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Reopen the device if it's not currently open. Otherwise,
- * just update the physical size of the device.
- */
- if (vd->vdev_tsd != NULL) {
- ASSERT(vd->vdev_reopening);
- vf = vd->vdev_tsd;
- goto skip_open;
- }
-
- vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_SLEEP);
-
- /*
- * We always open the files from the root of the global zone, even if
- * we're in a local zone. If the user has gotten to this point, the
- * administrator has already decided that the pool should be available
- * to local zone users, so the underlying devices should be as well.
- */
- ASSERT(vd->vdev_path != NULL && vd->vdev_path[0] == '/');
- error = vn_openat(vd->vdev_path + 1, UIO_SYSSPACE,
- spa_mode(vd->vdev_spa) | FOFFMAX, 0, &vp, 0, 0, rootdir, -1);
-
- if (error) {
- vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
- return (error);
- }
-
- vf->vf_vnode = vp;
-
-#ifdef _KERNEL
- /*
- * Make sure it's a regular file.
- */
- if (vp->v_type != VREG) {
- vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
- return (SET_ERROR(ENODEV));
- }
-#endif
-
-skip_open:
- /*
- * Determine the physical size of the file.
- */
- vattr.va_mask = AT_SIZE;
- error = VOP_GETATTR(vf->vf_vnode, &vattr, 0, kcred, NULL);
- if (error) {
- vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
- return (error);
- }
-
- *max_psize = *psize = vattr.va_size;
- *ashift = SPA_MINBLOCKSHIFT;
-
- return (0);
-}
-
-static void
-vdev_file_close(vdev_t *vd)
-{
- vdev_file_t *vf = vd->vdev_tsd;
-
- if (vd->vdev_reopening || vf == NULL)
- return;
-
- if (vf->vf_vnode != NULL) {
- (void) VOP_PUTPAGE(vf->vf_vnode, 0, 0, B_INVAL, kcred, NULL);
- (void) VOP_CLOSE(vf->vf_vnode, spa_mode(vd->vdev_spa), 1, 0,
- kcred, NULL);
- }
-
- vd->vdev_delayed_close = B_FALSE;
- kmem_free(vf, sizeof (vdev_file_t));
- vd->vdev_tsd = NULL;
-}
-
-static void
-vdev_file_io_strategy(void *arg)
-{
- zio_t *zio = (zio_t *)arg;
- vdev_t *vd = zio->io_vd;
- vdev_file_t *vf = vd->vdev_tsd;
- ssize_t resid;
- void *buf;
-
- if (zio->io_type == ZIO_TYPE_READ)
- buf = abd_borrow_buf(zio->io_abd, zio->io_size);
- else
- buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size);
-
- zio->io_error = vn_rdwr(zio->io_type == ZIO_TYPE_READ ?
- UIO_READ : UIO_WRITE, vf->vf_vnode, buf, zio->io_size,
- zio->io_offset, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred, &resid);
-
- if (zio->io_type == ZIO_TYPE_READ)
- abd_return_buf_copy(zio->io_abd, buf, zio->io_size);
- else
- abd_return_buf(zio->io_abd, buf, zio->io_size);
-
- if (resid != 0 && zio->io_error == 0)
- zio->io_error = SET_ERROR(ENOSPC);
-
- zio_delay_interrupt(zio);
-}
-
-static void
-vdev_file_io_fsync(void *arg)
-{
- zio_t *zio = (zio_t *)arg;
- vdev_file_t *vf = zio->io_vd->vdev_tsd;
-
- zio->io_error = VOP_FSYNC(vf->vf_vnode, FSYNC | FDSYNC, kcred, NULL);
-
- zio_interrupt(zio);
-}
-
-static void
-vdev_file_io_start(zio_t *zio)
-{
- vdev_t *vd = zio->io_vd;
- vdev_file_t *vf = vd->vdev_tsd;
-
- if (zio->io_type == ZIO_TYPE_IOCTL) {
- /* XXPOLICY */
- if (!vdev_readable(vd)) {
- zio->io_error = SET_ERROR(ENXIO);
- zio_interrupt(zio);
- return;
- }
-
- switch (zio->io_cmd) {
- case DKIOCFLUSHWRITECACHE:
-
- if (zfs_nocacheflush)
- break;
-
- /*
- * We cannot safely call vfs_fsync() when PF_FSTRANS
- * is set in the current context. Filesystems like
- * XFS include sanity checks to verify it is not
- * already set, see xfs_vm_writepage(). Therefore
- * the sync must be dispatched to a different context.
- */
- if (__spl_pf_fstrans_check()) {
- VERIFY3U(taskq_dispatch(vdev_file_taskq,
- vdev_file_io_fsync, zio, TQ_SLEEP), !=,
- TASKQID_INVALID);
- return;
- }
-
- zio->io_error = VOP_FSYNC(vf->vf_vnode, FSYNC | FDSYNC,
- kcred, NULL);
- break;
- default:
- zio->io_error = SET_ERROR(ENOTSUP);
- }
-
- zio_execute(zio);
- return;
- } else if (zio->io_type == ZIO_TYPE_TRIM) {
- struct flock flck;
-
- ASSERT3U(zio->io_size, !=, 0);
- bzero(&flck, sizeof (flck));
- flck.l_type = F_FREESP;
- flck.l_start = zio->io_offset;
- flck.l_len = zio->io_size;
- flck.l_whence = SEEK_SET;
-
- zio->io_error = VOP_SPACE(vf->vf_vnode, F_FREESP, &flck,
- 0, 0, kcred, NULL);
-
- zio_execute(zio);
- return;
- }
-
- zio->io_target_timestamp = zio_handle_io_delay(zio);
-
- VERIFY3U(taskq_dispatch(vdev_file_taskq, vdev_file_io_strategy, zio,
- TQ_SLEEP), !=, TASKQID_INVALID);
-}
-
-/* ARGSUSED */
-static void
-vdev_file_io_done(zio_t *zio)
-{
-}
-
-vdev_ops_t vdev_file_ops = {
- .vdev_op_open = vdev_file_open,
- .vdev_op_close = vdev_file_close,
- .vdev_op_asize = vdev_default_asize,
- .vdev_op_io_start = vdev_file_io_start,
- .vdev_op_io_done = vdev_file_io_done,
- .vdev_op_state_change = NULL,
- .vdev_op_need_resilver = NULL,
- .vdev_op_hold = vdev_file_hold,
- .vdev_op_rele = vdev_file_rele,
- .vdev_op_remap = NULL,
- .vdev_op_xlate = vdev_default_xlate,
- .vdev_op_type = VDEV_TYPE_FILE, /* name of this vdev type */
- .vdev_op_leaf = B_TRUE /* leaf vdev */
-};
-
-void
-vdev_file_init(void)
-{
- vdev_file_taskq = taskq_create("z_vdev_file", MAX(boot_ncpus, 16),
- minclsyspri, boot_ncpus, INT_MAX, TASKQ_DYNAMIC);
-
- VERIFY(vdev_file_taskq);
-}
-
-void
-vdev_file_fini(void)
-{
- taskq_destroy(vdev_file_taskq);
-}
-
-/*
- * From userland we access disks just like files.
- */
-#ifndef _KERNEL
-
-vdev_ops_t vdev_disk_ops = {
- .vdev_op_open = vdev_file_open,
- .vdev_op_close = vdev_file_close,
- .vdev_op_asize = vdev_default_asize,
- .vdev_op_io_start = vdev_file_io_start,
- .vdev_op_io_done = vdev_file_io_done,
- .vdev_op_state_change = NULL,
- .vdev_op_need_resilver = NULL,
- .vdev_op_hold = vdev_file_hold,
- .vdev_op_rele = vdev_file_rele,
- .vdev_op_remap = NULL,
- .vdev_op_xlate = vdev_default_xlate,
- .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
- .vdev_op_leaf = B_TRUE /* leaf vdev */
-};
-
-#endif
diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c
deleted file mode 100644
index 26af91e27..000000000
--- a/module/zfs/zfs_acl.c
+++ /dev/null
@@ -1,2816 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
- */
-
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysmacros.h>
-#include <sys/vfs.h>
-#include <sys/vnode.h>
-#include <sys/sid.h>
-#include <sys/file.h>
-#include <sys/stat.h>
-#include <sys/kmem.h>
-#include <sys/cmn_err.h>
-#include <sys/errno.h>
-#include <sys/sdt.h>
-#include <sys/fs/zfs.h>
-#include <sys/mode.h>
-#include <sys/policy.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_fuid.h>
-#include <sys/zfs_acl.h>
-#include <sys/zfs_dir.h>
-#include <sys/zfs_vfsops.h>
-#include <sys/dmu.h>
-#include <sys/dnode.h>
-#include <sys/zap.h>
-#include <sys/sa.h>
-#include <sys/trace_acl.h>
-#include <sys/zpl.h>
-
-#define ALLOW ACE_ACCESS_ALLOWED_ACE_TYPE
-#define DENY ACE_ACCESS_DENIED_ACE_TYPE
-#define MAX_ACE_TYPE ACE_SYSTEM_ALARM_CALLBACK_OBJECT_ACE_TYPE
-#define MIN_ACE_TYPE ALLOW
-
-#define OWNING_GROUP (ACE_GROUP|ACE_IDENTIFIER_GROUP)
-#define EVERYONE_ALLOW_MASK (ACE_READ_ACL|ACE_READ_ATTRIBUTES | \
- ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE)
-#define EVERYONE_DENY_MASK (ACE_WRITE_ACL|ACE_WRITE_OWNER | \
- ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS)
-#define OWNER_ALLOW_MASK (ACE_WRITE_ACL | ACE_WRITE_OWNER | \
- ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS)
-
-#define ZFS_CHECKED_MASKS (ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_DATA| \
- ACE_READ_NAMED_ATTRS|ACE_WRITE_DATA|ACE_WRITE_ATTRIBUTES| \
- ACE_WRITE_NAMED_ATTRS|ACE_APPEND_DATA|ACE_EXECUTE|ACE_WRITE_OWNER| \
- ACE_WRITE_ACL|ACE_DELETE|ACE_DELETE_CHILD|ACE_SYNCHRONIZE)
-
-#define WRITE_MASK_DATA (ACE_WRITE_DATA|ACE_APPEND_DATA|ACE_WRITE_NAMED_ATTRS)
-#define WRITE_MASK_ATTRS (ACE_WRITE_ACL|ACE_WRITE_OWNER|ACE_WRITE_ATTRIBUTES| \
- ACE_DELETE|ACE_DELETE_CHILD)
-#define WRITE_MASK (WRITE_MASK_DATA|WRITE_MASK_ATTRS)
-
-#define OGE_CLEAR (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \
- ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_EXECUTE)
-
-#define OKAY_MASK_BITS (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \
- ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_EXECUTE)
-
-#define ALL_INHERIT (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE | \
- ACE_NO_PROPAGATE_INHERIT_ACE|ACE_INHERIT_ONLY_ACE|ACE_INHERITED_ACE)
-
-#define RESTRICTED_CLEAR (ACE_WRITE_ACL|ACE_WRITE_OWNER)
-
-#define V4_ACL_WIDE_FLAGS (ZFS_ACL_AUTO_INHERIT|ZFS_ACL_DEFAULTED|\
- ZFS_ACL_PROTECTED)
-
-#define ZFS_ACL_WIDE_FLAGS (V4_ACL_WIDE_FLAGS|ZFS_ACL_TRIVIAL|ZFS_INHERIT_ACE|\
- ZFS_ACL_OBJ_ACE)
-
-#define ALL_MODE_EXECS (S_IXUSR | S_IXGRP | S_IXOTH)
-
-#define IDMAP_WK_CREATOR_OWNER_UID 2147483648U
-
-static uint16_t
-zfs_ace_v0_get_type(void *acep)
-{
- return (((zfs_oldace_t *)acep)->z_type);
-}
-
-static uint16_t
-zfs_ace_v0_get_flags(void *acep)
-{
- return (((zfs_oldace_t *)acep)->z_flags);
-}
-
-static uint32_t
-zfs_ace_v0_get_mask(void *acep)
-{
- return (((zfs_oldace_t *)acep)->z_access_mask);
-}
-
-static uint64_t
-zfs_ace_v0_get_who(void *acep)
-{
- return (((zfs_oldace_t *)acep)->z_fuid);
-}
-
-static void
-zfs_ace_v0_set_type(void *acep, uint16_t type)
-{
- ((zfs_oldace_t *)acep)->z_type = type;
-}
-
-static void
-zfs_ace_v0_set_flags(void *acep, uint16_t flags)
-{
- ((zfs_oldace_t *)acep)->z_flags = flags;
-}
-
-static void
-zfs_ace_v0_set_mask(void *acep, uint32_t mask)
-{
- ((zfs_oldace_t *)acep)->z_access_mask = mask;
-}
-
-static void
-zfs_ace_v0_set_who(void *acep, uint64_t who)
-{
- ((zfs_oldace_t *)acep)->z_fuid = who;
-}
-
-/*ARGSUSED*/
-static size_t
-zfs_ace_v0_size(void *acep)
-{
- return (sizeof (zfs_oldace_t));
-}
-
-static size_t
-zfs_ace_v0_abstract_size(void)
-{
- return (sizeof (zfs_oldace_t));
-}
-
-static int
-zfs_ace_v0_mask_off(void)
-{
- return (offsetof(zfs_oldace_t, z_access_mask));
-}
-
-/*ARGSUSED*/
-static int
-zfs_ace_v0_data(void *acep, void **datap)
-{
- *datap = NULL;
- return (0);
-}
-
-static acl_ops_t zfs_acl_v0_ops = {
- .ace_mask_get = zfs_ace_v0_get_mask,
- .ace_mask_set = zfs_ace_v0_set_mask,
- .ace_flags_get = zfs_ace_v0_get_flags,
- .ace_flags_set = zfs_ace_v0_set_flags,
- .ace_type_get = zfs_ace_v0_get_type,
- .ace_type_set = zfs_ace_v0_set_type,
- .ace_who_get = zfs_ace_v0_get_who,
- .ace_who_set = zfs_ace_v0_set_who,
- .ace_size = zfs_ace_v0_size,
- .ace_abstract_size = zfs_ace_v0_abstract_size,
- .ace_mask_off = zfs_ace_v0_mask_off,
- .ace_data = zfs_ace_v0_data
-};
-
-static uint16_t
-zfs_ace_fuid_get_type(void *acep)
-{
- return (((zfs_ace_hdr_t *)acep)->z_type);
-}
-
-static uint16_t
-zfs_ace_fuid_get_flags(void *acep)
-{
- return (((zfs_ace_hdr_t *)acep)->z_flags);
-}
-
-static uint32_t
-zfs_ace_fuid_get_mask(void *acep)
-{
- return (((zfs_ace_hdr_t *)acep)->z_access_mask);
-}
-
-static uint64_t
-zfs_ace_fuid_get_who(void *args)
-{
- uint16_t entry_type;
- zfs_ace_t *acep = args;
-
- entry_type = acep->z_hdr.z_flags & ACE_TYPE_FLAGS;
-
- if (entry_type == ACE_OWNER || entry_type == OWNING_GROUP ||
- entry_type == ACE_EVERYONE)
- return (-1);
- return (((zfs_ace_t *)acep)->z_fuid);
-}
-
-static void
-zfs_ace_fuid_set_type(void *acep, uint16_t type)
-{
- ((zfs_ace_hdr_t *)acep)->z_type = type;
-}
-
-static void
-zfs_ace_fuid_set_flags(void *acep, uint16_t flags)
-{
- ((zfs_ace_hdr_t *)acep)->z_flags = flags;
-}
-
-static void
-zfs_ace_fuid_set_mask(void *acep, uint32_t mask)
-{
- ((zfs_ace_hdr_t *)acep)->z_access_mask = mask;
-}
-
-static void
-zfs_ace_fuid_set_who(void *arg, uint64_t who)
-{
- zfs_ace_t *acep = arg;
-
- uint16_t entry_type = acep->z_hdr.z_flags & ACE_TYPE_FLAGS;
-
- if (entry_type == ACE_OWNER || entry_type == OWNING_GROUP ||
- entry_type == ACE_EVERYONE)
- return;
- acep->z_fuid = who;
-}
-
-static size_t
-zfs_ace_fuid_size(void *acep)
-{
- zfs_ace_hdr_t *zacep = acep;
- uint16_t entry_type;
-
- switch (zacep->z_type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- return (sizeof (zfs_object_ace_t));
- case ALLOW:
- case DENY:
- entry_type =
- (((zfs_ace_hdr_t *)acep)->z_flags & ACE_TYPE_FLAGS);
- if (entry_type == ACE_OWNER ||
- entry_type == OWNING_GROUP ||
- entry_type == ACE_EVERYONE)
- return (sizeof (zfs_ace_hdr_t));
- /*FALLTHROUGH*/
- default:
- return (sizeof (zfs_ace_t));
- }
-}
-
-static size_t
-zfs_ace_fuid_abstract_size(void)
-{
- return (sizeof (zfs_ace_hdr_t));
-}
-
-static int
-zfs_ace_fuid_mask_off(void)
-{
- return (offsetof(zfs_ace_hdr_t, z_access_mask));
-}
-
-static int
-zfs_ace_fuid_data(void *acep, void **datap)
-{
- zfs_ace_t *zacep = acep;
- zfs_object_ace_t *zobjp;
-
- switch (zacep->z_hdr.z_type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- zobjp = acep;
- *datap = (caddr_t)zobjp + sizeof (zfs_ace_t);
- return (sizeof (zfs_object_ace_t) - sizeof (zfs_ace_t));
- default:
- *datap = NULL;
- return (0);
- }
-}
-
-static acl_ops_t zfs_acl_fuid_ops = {
- .ace_mask_get = zfs_ace_fuid_get_mask,
- .ace_mask_set = zfs_ace_fuid_set_mask,
- .ace_flags_get = zfs_ace_fuid_get_flags,
- .ace_flags_set = zfs_ace_fuid_set_flags,
- .ace_type_get = zfs_ace_fuid_get_type,
- .ace_type_set = zfs_ace_fuid_set_type,
- .ace_who_get = zfs_ace_fuid_get_who,
- .ace_who_set = zfs_ace_fuid_set_who,
- .ace_size = zfs_ace_fuid_size,
- .ace_abstract_size = zfs_ace_fuid_abstract_size,
- .ace_mask_off = zfs_ace_fuid_mask_off,
- .ace_data = zfs_ace_fuid_data
-};
-
-/*
- * The following three functions are provided for compatibility with
- * older ZPL version in order to determine if the file use to have
- * an external ACL and what version of ACL previously existed on the
- * file. Would really be nice to not need this, sigh.
- */
-uint64_t
-zfs_external_acl(znode_t *zp)
-{
- zfs_acl_phys_t acl_phys;
- int error;
-
- if (zp->z_is_sa)
- return (0);
-
- /*
- * Need to deal with a potential
- * race where zfs_sa_upgrade could cause
- * z_isa_sa to change.
- *
- * If the lookup fails then the state of z_is_sa should have
- * changed.
- */
-
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(ZTOZSB(zp)),
- &acl_phys, sizeof (acl_phys))) == 0)
- return (acl_phys.z_acl_extern_obj);
- else {
- /*
- * after upgrade the SA_ZPL_ZNODE_ACL should have been
- * removed
- */
- VERIFY(zp->z_is_sa && error == ENOENT);
- return (0);
- }
-}
-
-/*
- * Determine size of ACL in bytes
- *
- * This is more complicated than it should be since we have to deal
- * with old external ACLs.
- */
-static int
-zfs_acl_znode_info(znode_t *zp, int *aclsize, int *aclcount,
- zfs_acl_phys_t *aclphys)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- uint64_t acl_count;
- int size;
- int error;
-
- ASSERT(MUTEX_HELD(&zp->z_acl_lock));
- if (zp->z_is_sa) {
- if ((error = sa_size(zp->z_sa_hdl, SA_ZPL_DACL_ACES(zfsvfs),
- &size)) != 0)
- return (error);
- *aclsize = size;
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_DACL_COUNT(zfsvfs),
- &acl_count, sizeof (acl_count))) != 0)
- return (error);
- *aclcount = acl_count;
- } else {
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(zfsvfs),
- aclphys, sizeof (*aclphys))) != 0)
- return (error);
-
- if (aclphys->z_acl_version == ZFS_ACL_VERSION_INITIAL) {
- *aclsize = ZFS_ACL_SIZE(aclphys->z_acl_size);
- *aclcount = aclphys->z_acl_size;
- } else {
- *aclsize = aclphys->z_acl_size;
- *aclcount = aclphys->z_acl_count;
- }
- }
- return (0);
-}
-
-int
-zfs_znode_acl_version(znode_t *zp)
-{
- zfs_acl_phys_t acl_phys;
-
- if (zp->z_is_sa)
- return (ZFS_ACL_VERSION_FUID);
- else {
- int error;
-
- /*
- * Need to deal with a potential
- * race where zfs_sa_upgrade could cause
- * z_isa_sa to change.
- *
- * If the lookup fails then the state of z_is_sa should have
- * changed.
- */
- if ((error = sa_lookup(zp->z_sa_hdl,
- SA_ZPL_ZNODE_ACL(ZTOZSB(zp)),
- &acl_phys, sizeof (acl_phys))) == 0)
- return (acl_phys.z_acl_version);
- else {
- /*
- * After upgrade SA_ZPL_ZNODE_ACL should have
- * been removed.
- */
- VERIFY(zp->z_is_sa && error == ENOENT);
- return (ZFS_ACL_VERSION_FUID);
- }
- }
-}
-
-static int
-zfs_acl_version(int version)
-{
- if (version < ZPL_VERSION_FUID)
- return (ZFS_ACL_VERSION_INITIAL);
- else
- return (ZFS_ACL_VERSION_FUID);
-}
-
-static int
-zfs_acl_version_zp(znode_t *zp)
-{
- return (zfs_acl_version(ZTOZSB(zp)->z_version));
-}
-
-zfs_acl_t *
-zfs_acl_alloc(int vers)
-{
- zfs_acl_t *aclp;
-
- aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_SLEEP);
- list_create(&aclp->z_acl, sizeof (zfs_acl_node_t),
- offsetof(zfs_acl_node_t, z_next));
- aclp->z_version = vers;
- if (vers == ZFS_ACL_VERSION_FUID)
- aclp->z_ops = &zfs_acl_fuid_ops;
- else
- aclp->z_ops = &zfs_acl_v0_ops;
- return (aclp);
-}
-
-zfs_acl_node_t *
-zfs_acl_node_alloc(size_t bytes)
-{
- zfs_acl_node_t *aclnode;
-
- aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_SLEEP);
- if (bytes) {
- aclnode->z_acldata = kmem_alloc(bytes, KM_SLEEP);
- aclnode->z_allocdata = aclnode->z_acldata;
- aclnode->z_allocsize = bytes;
- aclnode->z_size = bytes;
- }
-
- return (aclnode);
-}
-
-static void
-zfs_acl_node_free(zfs_acl_node_t *aclnode)
-{
- if (aclnode->z_allocsize)
- kmem_free(aclnode->z_allocdata, aclnode->z_allocsize);
- kmem_free(aclnode, sizeof (zfs_acl_node_t));
-}
-
-static void
-zfs_acl_release_nodes(zfs_acl_t *aclp)
-{
- zfs_acl_node_t *aclnode;
-
- while ((aclnode = list_head(&aclp->z_acl))) {
- list_remove(&aclp->z_acl, aclnode);
- zfs_acl_node_free(aclnode);
- }
- aclp->z_acl_count = 0;
- aclp->z_acl_bytes = 0;
-}
-
-void
-zfs_acl_free(zfs_acl_t *aclp)
-{
- zfs_acl_release_nodes(aclp);
- list_destroy(&aclp->z_acl);
- kmem_free(aclp, sizeof (zfs_acl_t));
-}
-
-static boolean_t
-zfs_acl_valid_ace_type(uint_t type, uint_t flags)
-{
- uint16_t entry_type;
-
- switch (type) {
- case ALLOW:
- case DENY:
- case ACE_SYSTEM_AUDIT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_ACE_TYPE:
- entry_type = flags & ACE_TYPE_FLAGS;
- return (entry_type == ACE_OWNER ||
- entry_type == OWNING_GROUP ||
- entry_type == ACE_EVERYONE || entry_type == 0 ||
- entry_type == ACE_IDENTIFIER_GROUP);
- default:
- if (type >= MIN_ACE_TYPE && type <= MAX_ACE_TYPE)
- return (B_TRUE);
- }
- return (B_FALSE);
-}
-
-static boolean_t
-zfs_ace_valid(umode_t obj_mode, zfs_acl_t *aclp, uint16_t type, uint16_t iflags)
-{
- /*
- * first check type of entry
- */
-
- if (!zfs_acl_valid_ace_type(type, iflags))
- return (B_FALSE);
-
- switch (type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- if (aclp->z_version < ZFS_ACL_VERSION_FUID)
- return (B_FALSE);
- aclp->z_hints |= ZFS_ACL_OBJ_ACE;
- }
-
- /*
- * next check inheritance level flags
- */
-
- if (S_ISDIR(obj_mode) &&
- (iflags & (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE)))
- aclp->z_hints |= ZFS_INHERIT_ACE;
-
- if (iflags & (ACE_INHERIT_ONLY_ACE|ACE_NO_PROPAGATE_INHERIT_ACE)) {
- if ((iflags & (ACE_FILE_INHERIT_ACE|
- ACE_DIRECTORY_INHERIT_ACE)) == 0) {
- return (B_FALSE);
- }
- }
-
- return (B_TRUE);
-}
-
-static void *
-zfs_acl_next_ace(zfs_acl_t *aclp, void *start, uint64_t *who,
- uint32_t *access_mask, uint16_t *iflags, uint16_t *type)
-{
- zfs_acl_node_t *aclnode;
-
- ASSERT(aclp);
-
- if (start == NULL) {
- aclnode = list_head(&aclp->z_acl);
- if (aclnode == NULL)
- return (NULL);
-
- aclp->z_next_ace = aclnode->z_acldata;
- aclp->z_curr_node = aclnode;
- aclnode->z_ace_idx = 0;
- }
-
- aclnode = aclp->z_curr_node;
-
- if (aclnode == NULL)
- return (NULL);
-
- if (aclnode->z_ace_idx >= aclnode->z_ace_count) {
- aclnode = list_next(&aclp->z_acl, aclnode);
- if (aclnode == NULL)
- return (NULL);
- else {
- aclp->z_curr_node = aclnode;
- aclnode->z_ace_idx = 0;
- aclp->z_next_ace = aclnode->z_acldata;
- }
- }
-
- if (aclnode->z_ace_idx < aclnode->z_ace_count) {
- void *acep = aclp->z_next_ace;
- size_t ace_size;
-
- /*
- * Make sure we don't overstep our bounds
- */
- ace_size = aclp->z_ops->ace_size(acep);
-
- if (((caddr_t)acep + ace_size) >
- ((caddr_t)aclnode->z_acldata + aclnode->z_size)) {
- return (NULL);
- }
-
- *iflags = aclp->z_ops->ace_flags_get(acep);
- *type = aclp->z_ops->ace_type_get(acep);
- *access_mask = aclp->z_ops->ace_mask_get(acep);
- *who = aclp->z_ops->ace_who_get(acep);
- aclp->z_next_ace = (caddr_t)aclp->z_next_ace + ace_size;
- aclnode->z_ace_idx++;
-
- return ((void *)acep);
- }
- return (NULL);
-}
-
-/*ARGSUSED*/
-static uint64_t
-zfs_ace_walk(void *datap, uint64_t cookie, int aclcnt,
- uint16_t *flags, uint16_t *type, uint32_t *mask)
-{
- zfs_acl_t *aclp = datap;
- zfs_ace_hdr_t *acep = (zfs_ace_hdr_t *)(uintptr_t)cookie;
- uint64_t who;
-
- acep = zfs_acl_next_ace(aclp, acep, &who, mask,
- flags, type);
- return ((uint64_t)(uintptr_t)acep);
-}
-
-/*
- * Copy ACE to internal ZFS format.
- * While processing the ACL each ACE will be validated for correctness.
- * ACE FUIDs will be created later.
- */
-int
-zfs_copy_ace_2_fuid(zfsvfs_t *zfsvfs, umode_t obj_mode, zfs_acl_t *aclp,
- void *datap, zfs_ace_t *z_acl, uint64_t aclcnt, size_t *size,
- zfs_fuid_info_t **fuidp, cred_t *cr)
-{
- int i;
- uint16_t entry_type;
- zfs_ace_t *aceptr = z_acl;
- ace_t *acep = datap;
- zfs_object_ace_t *zobjacep;
- ace_object_t *aceobjp;
-
- for (i = 0; i != aclcnt; i++) {
- aceptr->z_hdr.z_access_mask = acep->a_access_mask;
- aceptr->z_hdr.z_flags = acep->a_flags;
- aceptr->z_hdr.z_type = acep->a_type;
- entry_type = aceptr->z_hdr.z_flags & ACE_TYPE_FLAGS;
- if (entry_type != ACE_OWNER && entry_type != OWNING_GROUP &&
- entry_type != ACE_EVERYONE) {
- aceptr->z_fuid = zfs_fuid_create(zfsvfs, acep->a_who,
- cr, (entry_type == 0) ?
- ZFS_ACE_USER : ZFS_ACE_GROUP, fuidp);
- }
-
- /*
- * Make sure ACE is valid
- */
- if (zfs_ace_valid(obj_mode, aclp, aceptr->z_hdr.z_type,
- aceptr->z_hdr.z_flags) != B_TRUE)
- return (SET_ERROR(EINVAL));
-
- switch (acep->a_type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- zobjacep = (zfs_object_ace_t *)aceptr;
- aceobjp = (ace_object_t *)acep;
-
- bcopy(aceobjp->a_obj_type, zobjacep->z_object_type,
- sizeof (aceobjp->a_obj_type));
- bcopy(aceobjp->a_inherit_obj_type,
- zobjacep->z_inherit_type,
- sizeof (aceobjp->a_inherit_obj_type));
- acep = (ace_t *)((caddr_t)acep + sizeof (ace_object_t));
- break;
- default:
- acep = (ace_t *)((caddr_t)acep + sizeof (ace_t));
- }
-
- aceptr = (zfs_ace_t *)((caddr_t)aceptr +
- aclp->z_ops->ace_size(aceptr));
- }
-
- *size = (caddr_t)aceptr - (caddr_t)z_acl;
-
- return (0);
-}
-
-/*
- * Copy ZFS ACEs to fixed size ace_t layout
- */
-static void
-zfs_copy_fuid_2_ace(zfsvfs_t *zfsvfs, zfs_acl_t *aclp, cred_t *cr,
- void *datap, int filter)
-{
- uint64_t who;
- uint32_t access_mask;
- uint16_t iflags, type;
- zfs_ace_hdr_t *zacep = NULL;
- ace_t *acep = datap;
- ace_object_t *objacep;
- zfs_object_ace_t *zobjacep;
- size_t ace_size;
- uint16_t entry_type;
-
- while ((zacep = zfs_acl_next_ace(aclp, zacep,
- &who, &access_mask, &iflags, &type))) {
-
- switch (type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- if (filter) {
- continue;
- }
- zobjacep = (zfs_object_ace_t *)zacep;
- objacep = (ace_object_t *)acep;
- bcopy(zobjacep->z_object_type,
- objacep->a_obj_type,
- sizeof (zobjacep->z_object_type));
- bcopy(zobjacep->z_inherit_type,
- objacep->a_inherit_obj_type,
- sizeof (zobjacep->z_inherit_type));
- ace_size = sizeof (ace_object_t);
- break;
- default:
- ace_size = sizeof (ace_t);
- break;
- }
-
- entry_type = (iflags & ACE_TYPE_FLAGS);
- if ((entry_type != ACE_OWNER &&
- entry_type != OWNING_GROUP &&
- entry_type != ACE_EVERYONE)) {
- acep->a_who = zfs_fuid_map_id(zfsvfs, who,
- cr, (entry_type & ACE_IDENTIFIER_GROUP) ?
- ZFS_ACE_GROUP : ZFS_ACE_USER);
- } else {
- acep->a_who = (uid_t)(int64_t)who;
- }
- acep->a_access_mask = access_mask;
- acep->a_flags = iflags;
- acep->a_type = type;
- acep = (ace_t *)((caddr_t)acep + ace_size);
- }
-}
-
-static int
-zfs_copy_ace_2_oldace(umode_t obj_mode, zfs_acl_t *aclp, ace_t *acep,
- zfs_oldace_t *z_acl, int aclcnt, size_t *size)
-{
- int i;
- zfs_oldace_t *aceptr = z_acl;
-
- for (i = 0; i != aclcnt; i++, aceptr++) {
- aceptr->z_access_mask = acep[i].a_access_mask;
- aceptr->z_type = acep[i].a_type;
- aceptr->z_flags = acep[i].a_flags;
- aceptr->z_fuid = acep[i].a_who;
- /*
- * Make sure ACE is valid
- */
- if (zfs_ace_valid(obj_mode, aclp, aceptr->z_type,
- aceptr->z_flags) != B_TRUE)
- return (SET_ERROR(EINVAL));
- }
- *size = (caddr_t)aceptr - (caddr_t)z_acl;
- return (0);
-}
-
-/*
- * convert old ACL format to new
- */
-void
-zfs_acl_xform(znode_t *zp, zfs_acl_t *aclp, cred_t *cr)
-{
- zfs_oldace_t *oldaclp;
- int i;
- uint16_t type, iflags;
- uint32_t access_mask;
- uint64_t who;
- void *cookie = NULL;
- zfs_acl_node_t *newaclnode;
-
- ASSERT(aclp->z_version == ZFS_ACL_VERSION_INITIAL);
- /*
- * First create the ACE in a contiguous piece of memory
- * for zfs_copy_ace_2_fuid().
- *
- * We only convert an ACL once, so this won't happen
- * every time.
- */
- oldaclp = kmem_alloc(sizeof (zfs_oldace_t) * aclp->z_acl_count,
- KM_SLEEP);
- i = 0;
- while ((cookie = zfs_acl_next_ace(aclp, cookie, &who,
- &access_mask, &iflags, &type))) {
- oldaclp[i].z_flags = iflags;
- oldaclp[i].z_type = type;
- oldaclp[i].z_fuid = who;
- oldaclp[i++].z_access_mask = access_mask;
- }
-
- newaclnode = zfs_acl_node_alloc(aclp->z_acl_count *
- sizeof (zfs_object_ace_t));
- aclp->z_ops = &zfs_acl_fuid_ops;
- VERIFY(zfs_copy_ace_2_fuid(ZTOZSB(zp), ZTOI(zp)->i_mode,
- aclp, oldaclp, newaclnode->z_acldata, aclp->z_acl_count,
- &newaclnode->z_size, NULL, cr) == 0);
- newaclnode->z_ace_count = aclp->z_acl_count;
- aclp->z_version = ZFS_ACL_VERSION;
- kmem_free(oldaclp, aclp->z_acl_count * sizeof (zfs_oldace_t));
-
- /*
- * Release all previous ACL nodes
- */
-
- zfs_acl_release_nodes(aclp);
-
- list_insert_head(&aclp->z_acl, newaclnode);
-
- aclp->z_acl_bytes = newaclnode->z_size;
- aclp->z_acl_count = newaclnode->z_ace_count;
-
-}
-
-/*
- * Convert unix access mask to v4 access mask
- */
-static uint32_t
-zfs_unix_to_v4(uint32_t access_mask)
-{
- uint32_t new_mask = 0;
-
- if (access_mask & S_IXOTH)
- new_mask |= ACE_EXECUTE;
- if (access_mask & S_IWOTH)
- new_mask |= ACE_WRITE_DATA;
- if (access_mask & S_IROTH)
- new_mask |= ACE_READ_DATA;
- return (new_mask);
-}
-
-static void
-zfs_set_ace(zfs_acl_t *aclp, void *acep, uint32_t access_mask,
- uint16_t access_type, uint64_t fuid, uint16_t entry_type)
-{
- uint16_t type = entry_type & ACE_TYPE_FLAGS;
-
- aclp->z_ops->ace_mask_set(acep, access_mask);
- aclp->z_ops->ace_type_set(acep, access_type);
- aclp->z_ops->ace_flags_set(acep, entry_type);
- if ((type != ACE_OWNER && type != OWNING_GROUP &&
- type != ACE_EVERYONE))
- aclp->z_ops->ace_who_set(acep, fuid);
-}
-
-/*
- * Determine mode of file based on ACL.
- * Also, create FUIDs for any User/Group ACEs
- */
-uint64_t
-zfs_mode_compute(uint64_t fmode, zfs_acl_t *aclp,
- uint64_t *pflags, uint64_t fuid, uint64_t fgid)
-{
- int entry_type;
- mode_t mode;
- mode_t seen = 0;
- zfs_ace_hdr_t *acep = NULL;
- uint64_t who;
- uint16_t iflags, type;
- uint32_t access_mask;
- boolean_t an_exec_denied = B_FALSE;
-
- mode = (fmode & (S_IFMT | S_ISUID | S_ISGID | S_ISVTX));
-
- while ((acep = zfs_acl_next_ace(aclp, acep, &who,
- &access_mask, &iflags, &type))) {
-
- if (!zfs_acl_valid_ace_type(type, iflags))
- continue;
-
- entry_type = (iflags & ACE_TYPE_FLAGS);
-
- /*
- * Skip over owner@, group@ or everyone@ inherit only ACEs
- */
- if ((iflags & ACE_INHERIT_ONLY_ACE) &&
- (entry_type == ACE_OWNER || entry_type == ACE_EVERYONE ||
- entry_type == OWNING_GROUP))
- continue;
-
- if (entry_type == ACE_OWNER || (entry_type == 0 &&
- who == fuid)) {
- if ((access_mask & ACE_READ_DATA) &&
- (!(seen & S_IRUSR))) {
- seen |= S_IRUSR;
- if (type == ALLOW) {
- mode |= S_IRUSR;
- }
- }
- if ((access_mask & ACE_WRITE_DATA) &&
- (!(seen & S_IWUSR))) {
- seen |= S_IWUSR;
- if (type == ALLOW) {
- mode |= S_IWUSR;
- }
- }
- if ((access_mask & ACE_EXECUTE) &&
- (!(seen & S_IXUSR))) {
- seen |= S_IXUSR;
- if (type == ALLOW) {
- mode |= S_IXUSR;
- }
- }
- } else if (entry_type == OWNING_GROUP ||
- (entry_type == ACE_IDENTIFIER_GROUP && who == fgid)) {
- if ((access_mask & ACE_READ_DATA) &&
- (!(seen & S_IRGRP))) {
- seen |= S_IRGRP;
- if (type == ALLOW) {
- mode |= S_IRGRP;
- }
- }
- if ((access_mask & ACE_WRITE_DATA) &&
- (!(seen & S_IWGRP))) {
- seen |= S_IWGRP;
- if (type == ALLOW) {
- mode |= S_IWGRP;
- }
- }
- if ((access_mask & ACE_EXECUTE) &&
- (!(seen & S_IXGRP))) {
- seen |= S_IXGRP;
- if (type == ALLOW) {
- mode |= S_IXGRP;
- }
- }
- } else if (entry_type == ACE_EVERYONE) {
- if ((access_mask & ACE_READ_DATA)) {
- if (!(seen & S_IRUSR)) {
- seen |= S_IRUSR;
- if (type == ALLOW) {
- mode |= S_IRUSR;
- }
- }
- if (!(seen & S_IRGRP)) {
- seen |= S_IRGRP;
- if (type == ALLOW) {
- mode |= S_IRGRP;
- }
- }
- if (!(seen & S_IROTH)) {
- seen |= S_IROTH;
- if (type == ALLOW) {
- mode |= S_IROTH;
- }
- }
- }
- if ((access_mask & ACE_WRITE_DATA)) {
- if (!(seen & S_IWUSR)) {
- seen |= S_IWUSR;
- if (type == ALLOW) {
- mode |= S_IWUSR;
- }
- }
- if (!(seen & S_IWGRP)) {
- seen |= S_IWGRP;
- if (type == ALLOW) {
- mode |= S_IWGRP;
- }
- }
- if (!(seen & S_IWOTH)) {
- seen |= S_IWOTH;
- if (type == ALLOW) {
- mode |= S_IWOTH;
- }
- }
- }
- if ((access_mask & ACE_EXECUTE)) {
- if (!(seen & S_IXUSR)) {
- seen |= S_IXUSR;
- if (type == ALLOW) {
- mode |= S_IXUSR;
- }
- }
- if (!(seen & S_IXGRP)) {
- seen |= S_IXGRP;
- if (type == ALLOW) {
- mode |= S_IXGRP;
- }
- }
- if (!(seen & S_IXOTH)) {
- seen |= S_IXOTH;
- if (type == ALLOW) {
- mode |= S_IXOTH;
- }
- }
- }
- } else {
- /*
- * Only care if this IDENTIFIER_GROUP or
- * USER ACE denies execute access to someone,
- * mode is not affected
- */
- if ((access_mask & ACE_EXECUTE) && type == DENY)
- an_exec_denied = B_TRUE;
- }
- }
-
- /*
- * Failure to allow is effectively a deny, so execute permission
- * is denied if it was never mentioned or if we explicitly
- * weren't allowed it.
- */
- if (!an_exec_denied &&
- ((seen & ALL_MODE_EXECS) != ALL_MODE_EXECS ||
- (mode & ALL_MODE_EXECS) != ALL_MODE_EXECS))
- an_exec_denied = B_TRUE;
-
- if (an_exec_denied)
- *pflags &= ~ZFS_NO_EXECS_DENIED;
- else
- *pflags |= ZFS_NO_EXECS_DENIED;
-
- return (mode);
-}
-
-/*
- * Read an external acl object. If the intent is to modify, always
- * create a new acl and leave any cached acl in place.
- */
-int
-zfs_acl_node_read(struct znode *zp, boolean_t have_lock, zfs_acl_t **aclpp,
- boolean_t will_modify)
-{
- zfs_acl_t *aclp;
- int aclsize = 0;
- int acl_count = 0;
- zfs_acl_node_t *aclnode;
- zfs_acl_phys_t znode_acl;
- int version;
- int error;
- boolean_t drop_lock = B_FALSE;
-
- ASSERT(MUTEX_HELD(&zp->z_acl_lock));
-
- if (zp->z_acl_cached && !will_modify) {
- *aclpp = zp->z_acl_cached;
- return (0);
- }
-
- /*
- * close race where znode could be upgrade while trying to
- * read the znode attributes.
- *
- * But this could only happen if the file isn't already an SA
- * znode
- */
- if (!zp->z_is_sa && !have_lock) {
- mutex_enter(&zp->z_lock);
- drop_lock = B_TRUE;
- }
- version = zfs_znode_acl_version(zp);
-
- if ((error = zfs_acl_znode_info(zp, &aclsize,
- &acl_count, &znode_acl)) != 0) {
- goto done;
- }
-
- aclp = zfs_acl_alloc(version);
-
- aclp->z_acl_count = acl_count;
- aclp->z_acl_bytes = aclsize;
-
- aclnode = zfs_acl_node_alloc(aclsize);
- aclnode->z_ace_count = aclp->z_acl_count;
- aclnode->z_size = aclsize;
-
- if (!zp->z_is_sa) {
- if (znode_acl.z_acl_extern_obj) {
- error = dmu_read(ZTOZSB(zp)->z_os,
- znode_acl.z_acl_extern_obj, 0, aclnode->z_size,
- aclnode->z_acldata, DMU_READ_PREFETCH);
- } else {
- bcopy(znode_acl.z_ace_data, aclnode->z_acldata,
- aclnode->z_size);
- }
- } else {
- error = sa_lookup(zp->z_sa_hdl, SA_ZPL_DACL_ACES(ZTOZSB(zp)),
- aclnode->z_acldata, aclnode->z_size);
- }
-
- if (error != 0) {
- zfs_acl_free(aclp);
- zfs_acl_node_free(aclnode);
- /* convert checksum errors into IO errors */
- if (error == ECKSUM)
- error = SET_ERROR(EIO);
- goto done;
- }
-
- list_insert_head(&aclp->z_acl, aclnode);
-
- *aclpp = aclp;
- if (!will_modify)
- zp->z_acl_cached = aclp;
-done:
- if (drop_lock)
- mutex_exit(&zp->z_lock);
- return (error);
-}
-
-/*ARGSUSED*/
-void
-zfs_acl_data_locator(void **dataptr, uint32_t *length, uint32_t buflen,
- boolean_t start, void *userdata)
-{
- zfs_acl_locator_cb_t *cb = (zfs_acl_locator_cb_t *)userdata;
-
- if (start) {
- cb->cb_acl_node = list_head(&cb->cb_aclp->z_acl);
- } else {
- cb->cb_acl_node = list_next(&cb->cb_aclp->z_acl,
- cb->cb_acl_node);
- }
- *dataptr = cb->cb_acl_node->z_acldata;
- *length = cb->cb_acl_node->z_size;
-}
-
-int
-zfs_acl_chown_setattr(znode_t *zp)
-{
- int error;
- zfs_acl_t *aclp;
-
- if (ZTOZSB(zp)->z_acl_type == ZFS_ACLTYPE_POSIXACL)
- return (0);
-
- ASSERT(MUTEX_HELD(&zp->z_lock));
- ASSERT(MUTEX_HELD(&zp->z_acl_lock));
-
- error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE);
- if (error == 0 && aclp->z_acl_count > 0)
- zp->z_mode = ZTOI(zp)->i_mode =
- zfs_mode_compute(zp->z_mode, aclp,
- &zp->z_pflags, KUID_TO_SUID(ZTOI(zp)->i_uid),
- KGID_TO_SGID(ZTOI(zp)->i_gid));
-
- /*
- * Some ZFS implementations (ZEVO) create neither a ZNODE_ACL
- * nor a DACL_ACES SA in which case ENOENT is returned from
- * zfs_acl_node_read() when the SA can't be located.
- * Allow chown/chgrp to succeed in these cases rather than
- * returning an error that makes no sense in the context of
- * the caller.
- */
- if (error == ENOENT)
- return (0);
-
- return (error);
-}
-
-static void
-acl_trivial_access_masks(mode_t mode, uint32_t *allow0, uint32_t *deny1,
- uint32_t *deny2, uint32_t *owner, uint32_t *group, uint32_t *everyone)
-{
- *deny1 = *deny2 = *allow0 = *group = 0;
-
- if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH)))
- *deny1 |= ACE_READ_DATA;
- if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH)))
- *deny1 |= ACE_WRITE_DATA;
- if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH)))
- *deny1 |= ACE_EXECUTE;
-
- if (!(mode & S_IRGRP) && (mode & S_IROTH))
- *deny2 = ACE_READ_DATA;
- if (!(mode & S_IWGRP) && (mode & S_IWOTH))
- *deny2 |= ACE_WRITE_DATA;
- if (!(mode & S_IXGRP) && (mode & S_IXOTH))
- *deny2 |= ACE_EXECUTE;
-
- if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH)))
- *allow0 |= ACE_READ_DATA;
- if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH)))
- *allow0 |= ACE_WRITE_DATA;
- if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH)))
- *allow0 |= ACE_EXECUTE;
-
- *owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL|
- ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES|
- ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE;
- if (mode & S_IRUSR)
- *owner |= ACE_READ_DATA;
- if (mode & S_IWUSR)
- *owner |= ACE_WRITE_DATA|ACE_APPEND_DATA;
- if (mode & S_IXUSR)
- *owner |= ACE_EXECUTE;
-
- *group = ACE_READ_ACL|ACE_READ_ATTRIBUTES| ACE_READ_NAMED_ATTRS|
- ACE_SYNCHRONIZE;
- if (mode & S_IRGRP)
- *group |= ACE_READ_DATA;
- if (mode & S_IWGRP)
- *group |= ACE_WRITE_DATA|ACE_APPEND_DATA;
- if (mode & S_IXGRP)
- *group |= ACE_EXECUTE;
-
- *everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES| ACE_READ_NAMED_ATTRS|
- ACE_SYNCHRONIZE;
- if (mode & S_IROTH)
- *everyone |= ACE_READ_DATA;
- if (mode & S_IWOTH)
- *everyone |= ACE_WRITE_DATA|ACE_APPEND_DATA;
- if (mode & S_IXOTH)
- *everyone |= ACE_EXECUTE;
-}
-
-/*
- * ace_trivial:
- * determine whether an ace_t acl is trivial
- *
- * Trivialness implies that the acl is composed of only
- * owner, group, everyone entries. ACL can't
- * have read_acl denied, and write_owner/write_acl/write_attributes
- * can only be owner@ entry.
- */
-static int
-ace_trivial_common(void *acep, int aclcnt,
- uint64_t (*walk)(void *, uint64_t, int aclcnt,
- uint16_t *, uint16_t *, uint32_t *))
-{
- uint16_t flags;
- uint32_t mask;
- uint16_t type;
- uint64_t cookie = 0;
-
- while ((cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask))) {
- switch (flags & ACE_TYPE_FLAGS) {
- case ACE_OWNER:
- case ACE_GROUP|ACE_IDENTIFIER_GROUP:
- case ACE_EVERYONE:
- break;
- default:
- return (1);
- }
-
- if (flags & (ACE_FILE_INHERIT_ACE|
- ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
- ACE_INHERIT_ONLY_ACE))
- return (1);
-
- /*
- * Special check for some special bits
- *
- * Don't allow anybody to deny reading basic
- * attributes or a files ACL.
- */
- if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
- (type == ACE_ACCESS_DENIED_ACE_TYPE))
- return (1);
-
- /*
- * Delete permissions are never set by default
- */
- if (mask & (ACE_DELETE|ACE_DELETE_CHILD))
- return (1);
- /*
- * only allow owner@ to have
- * write_acl/write_owner/write_attributes/write_xattr/
- */
- if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
- (!(flags & ACE_OWNER) && (mask &
- (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES|
- ACE_WRITE_NAMED_ATTRS))))
- return (1);
-
- }
-
- return (0);
-}
-
-/*
- * common code for setting ACLs.
- *
- * This function is called from zfs_mode_update, zfs_perm_init, and zfs_setacl.
- * zfs_setacl passes a non-NULL inherit pointer (ihp) to indicate that it's
- * already checked the acl and knows whether to inherit.
- */
-int
-zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx)
-{
- int error;
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- dmu_object_type_t otype;
- zfs_acl_locator_cb_t locate = { 0 };
- uint64_t mode;
- sa_bulk_attr_t bulk[5];
- uint64_t ctime[2];
- int count = 0;
- zfs_acl_phys_t acl_phys;
-
- mode = zp->z_mode;
-
- mode = zfs_mode_compute(mode, aclp, &zp->z_pflags,
- KUID_TO_SUID(ZTOI(zp)->i_uid), KGID_TO_SGID(ZTOI(zp)->i_gid));
-
- zp->z_mode = ZTOI(zp)->i_mode = mode;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
- &mode, sizeof (mode));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, sizeof (zp->z_pflags));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
- &ctime, sizeof (ctime));
-
- if (zp->z_acl_cached) {
- zfs_acl_free(zp->z_acl_cached);
- zp->z_acl_cached = NULL;
- }
-
- /*
- * Upgrade needed?
- */
- if (!zfsvfs->z_use_fuids) {
- otype = DMU_OT_OLDACL;
- } else {
- if ((aclp->z_version == ZFS_ACL_VERSION_INITIAL) &&
- (zfsvfs->z_version >= ZPL_VERSION_FUID))
- zfs_acl_xform(zp, aclp, cr);
- ASSERT(aclp->z_version >= ZFS_ACL_VERSION_FUID);
- otype = DMU_OT_ACL;
- }
-
- /*
- * Arrgh, we have to handle old on disk format
- * as well as newer (preferred) SA format.
- */
-
- if (zp->z_is_sa) { /* the easy case, just update the ACL attribute */
- locate.cb_aclp = aclp;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_DACL_ACES(zfsvfs),
- zfs_acl_data_locator, &locate, aclp->z_acl_bytes);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_DACL_COUNT(zfsvfs),
- NULL, &aclp->z_acl_count, sizeof (uint64_t));
- } else { /* Painful legacy way */
- zfs_acl_node_t *aclnode;
- uint64_t off = 0;
- uint64_t aoid;
-
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(zfsvfs),
- &acl_phys, sizeof (acl_phys))) != 0)
- return (error);
-
- aoid = acl_phys.z_acl_extern_obj;
-
- if (aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- /*
- * If ACL was previously external and we are now
- * converting to new ACL format then release old
- * ACL object and create a new one.
- */
- if (aoid &&
- aclp->z_version != acl_phys.z_acl_version) {
- error = dmu_object_free(zfsvfs->z_os, aoid, tx);
- if (error)
- return (error);
- aoid = 0;
- }
- if (aoid == 0) {
- aoid = dmu_object_alloc(zfsvfs->z_os,
- otype, aclp->z_acl_bytes,
- otype == DMU_OT_ACL ?
- DMU_OT_SYSACL : DMU_OT_NONE,
- otype == DMU_OT_ACL ?
- DN_OLD_MAX_BONUSLEN : 0, tx);
- } else {
- (void) dmu_object_set_blocksize(zfsvfs->z_os,
- aoid, aclp->z_acl_bytes, 0, tx);
- }
- acl_phys.z_acl_extern_obj = aoid;
- for (aclnode = list_head(&aclp->z_acl); aclnode;
- aclnode = list_next(&aclp->z_acl, aclnode)) {
- if (aclnode->z_ace_count == 0)
- continue;
- dmu_write(zfsvfs->z_os, aoid, off,
- aclnode->z_size, aclnode->z_acldata, tx);
- off += aclnode->z_size;
- }
- } else {
- void *start = acl_phys.z_ace_data;
- /*
- * Migrating back embedded?
- */
- if (acl_phys.z_acl_extern_obj) {
- error = dmu_object_free(zfsvfs->z_os,
- acl_phys.z_acl_extern_obj, tx);
- if (error)
- return (error);
- acl_phys.z_acl_extern_obj = 0;
- }
-
- for (aclnode = list_head(&aclp->z_acl); aclnode;
- aclnode = list_next(&aclp->z_acl, aclnode)) {
- if (aclnode->z_ace_count == 0)
- continue;
- bcopy(aclnode->z_acldata, start,
- aclnode->z_size);
- start = (caddr_t)start + aclnode->z_size;
- }
- }
- /*
- * If Old version then swap count/bytes to match old
- * layout of znode_acl_phys_t.
- */
- if (aclp->z_version == ZFS_ACL_VERSION_INITIAL) {
- acl_phys.z_acl_size = aclp->z_acl_count;
- acl_phys.z_acl_count = aclp->z_acl_bytes;
- } else {
- acl_phys.z_acl_size = aclp->z_acl_bytes;
- acl_phys.z_acl_count = aclp->z_acl_count;
- }
- acl_phys.z_acl_version = aclp->z_version;
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ZNODE_ACL(zfsvfs), NULL,
- &acl_phys, sizeof (acl_phys));
- }
-
- /*
- * Replace ACL wide bits, but first clear them.
- */
- zp->z_pflags &= ~ZFS_ACL_WIDE_FLAGS;
-
- zp->z_pflags |= aclp->z_hints;
-
- if (ace_trivial_common(aclp, 0, zfs_ace_walk) == 0)
- zp->z_pflags |= ZFS_ACL_TRIVIAL;
-
- zfs_tstamp_update_setup(zp, STATE_CHANGED, NULL, ctime);
- return (sa_bulk_update(zp->z_sa_hdl, bulk, count, tx));
-}
-
-static void
-zfs_acl_chmod(zfsvfs_t *zfsvfs, uint64_t mode, zfs_acl_t *aclp)
-{
- void *acep = NULL;
- uint64_t who;
- int new_count, new_bytes;
- int ace_size;
- int entry_type;
- uint16_t iflags, type;
- uint32_t access_mask;
- zfs_acl_node_t *newnode;
- size_t abstract_size = aclp->z_ops->ace_abstract_size();
- void *zacep;
- uint32_t owner, group, everyone;
- uint32_t deny1, deny2, allow0;
-
- new_count = new_bytes = 0;
-
- acl_trivial_access_masks((mode_t)mode, &allow0, &deny1, &deny2,
- &owner, &group, &everyone);
-
- newnode = zfs_acl_node_alloc((abstract_size * 6) + aclp->z_acl_bytes);
-
- zacep = newnode->z_acldata;
- if (allow0) {
- zfs_set_ace(aclp, zacep, allow0, ALLOW, -1, ACE_OWNER);
- zacep = (void *)((uintptr_t)zacep + abstract_size);
- new_count++;
- new_bytes += abstract_size;
- }
- if (deny1) {
- zfs_set_ace(aclp, zacep, deny1, DENY, -1, ACE_OWNER);
- zacep = (void *)((uintptr_t)zacep + abstract_size);
- new_count++;
- new_bytes += abstract_size;
- }
- if (deny2) {
- zfs_set_ace(aclp, zacep, deny2, DENY, -1, OWNING_GROUP);
- zacep = (void *)((uintptr_t)zacep + abstract_size);
- new_count++;
- new_bytes += abstract_size;
- }
-
- while ((acep = zfs_acl_next_ace(aclp, acep, &who, &access_mask,
- &iflags, &type))) {
- uint16_t inherit_flags;
-
- entry_type = (iflags & ACE_TYPE_FLAGS);
- inherit_flags = (iflags & ALL_INHERIT);
-
- if ((entry_type == ACE_OWNER || entry_type == ACE_EVERYONE ||
- (entry_type == OWNING_GROUP)) &&
- ((inherit_flags & ACE_INHERIT_ONLY_ACE) == 0)) {
- continue;
- }
-
- if ((type != ALLOW && type != DENY) ||
- (inherit_flags & ACE_INHERIT_ONLY_ACE)) {
- if (inherit_flags)
- aclp->z_hints |= ZFS_INHERIT_ACE;
- switch (type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- aclp->z_hints |= ZFS_ACL_OBJ_ACE;
- break;
- }
- } else {
-
- /*
- * Limit permissions to be no greater than
- * group permissions
- */
- if (zfsvfs->z_acl_inherit == ZFS_ACL_RESTRICTED) {
- if (!(mode & S_IRGRP))
- access_mask &= ~ACE_READ_DATA;
- if (!(mode & S_IWGRP))
- access_mask &=
- ~(ACE_WRITE_DATA|ACE_APPEND_DATA);
- if (!(mode & S_IXGRP))
- access_mask &= ~ACE_EXECUTE;
- access_mask &=
- ~(ACE_WRITE_OWNER|ACE_WRITE_ACL|
- ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS);
- }
- }
- zfs_set_ace(aclp, zacep, access_mask, type, who, iflags);
- ace_size = aclp->z_ops->ace_size(acep);
- zacep = (void *)((uintptr_t)zacep + ace_size);
- new_count++;
- new_bytes += ace_size;
- }
- zfs_set_ace(aclp, zacep, owner, 0, -1, ACE_OWNER);
- zacep = (void *)((uintptr_t)zacep + abstract_size);
- zfs_set_ace(aclp, zacep, group, 0, -1, OWNING_GROUP);
- zacep = (void *)((uintptr_t)zacep + abstract_size);
- zfs_set_ace(aclp, zacep, everyone, 0, -1, ACE_EVERYONE);
-
- new_count += 3;
- new_bytes += abstract_size * 3;
- zfs_acl_release_nodes(aclp);
- aclp->z_acl_count = new_count;
- aclp->z_acl_bytes = new_bytes;
- newnode->z_ace_count = new_count;
- newnode->z_size = new_bytes;
- list_insert_tail(&aclp->z_acl, newnode);
-}
-
-void
-zfs_acl_chmod_setattr(znode_t *zp, zfs_acl_t **aclp, uint64_t mode)
-{
- mutex_enter(&zp->z_acl_lock);
- mutex_enter(&zp->z_lock);
- *aclp = zfs_acl_alloc(zfs_acl_version_zp(zp));
- (*aclp)->z_hints = zp->z_pflags & V4_ACL_WIDE_FLAGS;
- zfs_acl_chmod(ZTOZSB(zp), mode, *aclp);
- mutex_exit(&zp->z_lock);
- mutex_exit(&zp->z_acl_lock);
- ASSERT(*aclp);
-}
-
-/*
- * strip off write_owner and write_acl
- */
-static void
-zfs_restricted_update(zfsvfs_t *zfsvfs, zfs_acl_t *aclp, void *acep)
-{
- uint32_t mask = aclp->z_ops->ace_mask_get(acep);
-
- if ((zfsvfs->z_acl_inherit == ZFS_ACL_RESTRICTED) &&
- (aclp->z_ops->ace_type_get(acep) == ALLOW)) {
- mask &= ~RESTRICTED_CLEAR;
- aclp->z_ops->ace_mask_set(acep, mask);
- }
-}
-
-/*
- * Should ACE be inherited?
- */
-static int
-zfs_ace_can_use(umode_t obj_mode, uint16_t acep_flags)
-{
- int iflags = (acep_flags & 0xf);
-
- if (S_ISDIR(obj_mode) && (iflags & ACE_DIRECTORY_INHERIT_ACE))
- return (1);
- else if (iflags & ACE_FILE_INHERIT_ACE)
- return (!(S_ISDIR(obj_mode) &&
- (iflags & ACE_NO_PROPAGATE_INHERIT_ACE)));
- return (0);
-}
-
-/*
- * inherit inheritable ACEs from parent
- */
-static zfs_acl_t *
-zfs_acl_inherit(zfsvfs_t *zfsvfs, umode_t obj_mode, zfs_acl_t *paclp,
- uint64_t mode, boolean_t *need_chmod)
-{
- void *pacep;
- void *acep;
- zfs_acl_node_t *aclnode;
- zfs_acl_t *aclp = NULL;
- uint64_t who;
- uint32_t access_mask;
- uint16_t iflags, newflags, type;
- size_t ace_size;
- void *data1, *data2;
- size_t data1sz, data2sz;
- boolean_t vdir = S_ISDIR(obj_mode);
- boolean_t vreg = S_ISREG(obj_mode);
- boolean_t passthrough, passthrough_x, noallow;
-
- passthrough_x =
- zfsvfs->z_acl_inherit == ZFS_ACL_PASSTHROUGH_X;
- passthrough = passthrough_x ||
- zfsvfs->z_acl_inherit == ZFS_ACL_PASSTHROUGH;
- noallow =
- zfsvfs->z_acl_inherit == ZFS_ACL_NOALLOW;
-
- *need_chmod = B_TRUE;
- pacep = NULL;
- aclp = zfs_acl_alloc(paclp->z_version);
- if (zfsvfs->z_acl_inherit == ZFS_ACL_DISCARD || S_ISLNK(obj_mode))
- return (aclp);
- while ((pacep = zfs_acl_next_ace(paclp, pacep, &who,
- &access_mask, &iflags, &type))) {
-
- /*
- * don't inherit bogus ACEs
- */
- if (!zfs_acl_valid_ace_type(type, iflags))
- continue;
-
- if (noallow && type == ALLOW)
- continue;
-
- ace_size = aclp->z_ops->ace_size(pacep);
-
- if (!zfs_ace_can_use(obj_mode, iflags))
- continue;
-
- /*
- * If owner@, group@, or everyone@ inheritable
- * then zfs_acl_chmod() isn't needed.
- */
- if (passthrough &&
- ((iflags & (ACE_OWNER|ACE_EVERYONE)) ||
- ((iflags & OWNING_GROUP) ==
- OWNING_GROUP)) && (vreg || (vdir && (iflags &
- ACE_DIRECTORY_INHERIT_ACE)))) {
- *need_chmod = B_FALSE;
- }
-
- if (!vdir && passthrough_x &&
- ((mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0)) {
- access_mask &= ~ACE_EXECUTE;
- }
-
- aclnode = zfs_acl_node_alloc(ace_size);
- list_insert_tail(&aclp->z_acl, aclnode);
- acep = aclnode->z_acldata;
-
- zfs_set_ace(aclp, acep, access_mask, type,
- who, iflags|ACE_INHERITED_ACE);
-
- /*
- * Copy special opaque data if any
- */
- if ((data1sz = paclp->z_ops->ace_data(pacep, &data1)) != 0) {
- VERIFY((data2sz = aclp->z_ops->ace_data(acep,
- &data2)) == data1sz);
- bcopy(data1, data2, data2sz);
- }
-
- aclp->z_acl_count++;
- aclnode->z_ace_count++;
- aclp->z_acl_bytes += aclnode->z_size;
- newflags = aclp->z_ops->ace_flags_get(acep);
-
- if (vdir)
- aclp->z_hints |= ZFS_INHERIT_ACE;
-
- if ((iflags & ACE_NO_PROPAGATE_INHERIT_ACE) || !vdir) {
- newflags &= ~ALL_INHERIT;
- aclp->z_ops->ace_flags_set(acep,
- newflags|ACE_INHERITED_ACE);
- zfs_restricted_update(zfsvfs, aclp, acep);
- continue;
- }
-
- ASSERT(vdir);
-
- /*
- * If only FILE_INHERIT is set then turn on
- * inherit_only
- */
- if ((iflags & (ACE_FILE_INHERIT_ACE |
- ACE_DIRECTORY_INHERIT_ACE)) == ACE_FILE_INHERIT_ACE) {
- newflags |= ACE_INHERIT_ONLY_ACE;
- aclp->z_ops->ace_flags_set(acep,
- newflags|ACE_INHERITED_ACE);
- } else {
- newflags &= ~ACE_INHERIT_ONLY_ACE;
- aclp->z_ops->ace_flags_set(acep,
- newflags|ACE_INHERITED_ACE);
- }
- }
- return (aclp);
-}
-
-/*
- * Create file system object initial permissions
- * including inheritable ACEs.
- */
-int
-zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr,
- vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids)
-{
- int error;
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- zfs_acl_t *paclp;
- gid_t gid = vap->va_gid;
- boolean_t need_chmod = B_TRUE;
- boolean_t inherited = B_FALSE;
-
- bzero(acl_ids, sizeof (zfs_acl_ids_t));
- acl_ids->z_mode = vap->va_mode;
-
- if (vsecp)
- if ((error = zfs_vsec_2_aclp(zfsvfs, vap->va_mode, vsecp,
- cr, &acl_ids->z_fuidp, &acl_ids->z_aclp)) != 0)
- return (error);
-
- acl_ids->z_fuid = vap->va_uid;
- acl_ids->z_fgid = vap->va_gid;
-#ifdef HAVE_KSID
- /*
- * Determine uid and gid.
- */
- if ((flag & IS_ROOT_NODE) || zfsvfs->z_replay ||
- ((flag & IS_XATTR) && (S_ISDIR(vap->va_mode)))) {
- acl_ids->z_fuid = zfs_fuid_create(zfsvfs, (uint64_t)vap->va_uid,
- cr, ZFS_OWNER, &acl_ids->z_fuidp);
- acl_ids->z_fgid = zfs_fuid_create(zfsvfs, (uint64_t)vap->va_gid,
- cr, ZFS_GROUP, &acl_ids->z_fuidp);
- gid = vap->va_gid;
- } else {
- acl_ids->z_fuid = zfs_fuid_create_cred(zfsvfs, ZFS_OWNER,
- cr, &acl_ids->z_fuidp);
- acl_ids->z_fgid = 0;
- if (vap->va_mask & AT_GID) {
- acl_ids->z_fgid = zfs_fuid_create(zfsvfs,
- (uint64_t)vap->va_gid,
- cr, ZFS_GROUP, &acl_ids->z_fuidp);
- gid = vap->va_gid;
- if (acl_ids->z_fgid != KGID_TO_SGID(ZTOI(dzp)->i_gid) &&
- !groupmember(vap->va_gid, cr) &&
- secpolicy_vnode_create_gid(cr) != 0)
- acl_ids->z_fgid = 0;
- }
- if (acl_ids->z_fgid == 0) {
- if (dzp->z_mode & S_ISGID) {
- char *domain;
- uint32_t rid;
-
- acl_ids->z_fgid = KGID_TO_SGID(
- ZTOI(dzp)->i_gid);
- gid = zfs_fuid_map_id(zfsvfs, acl_ids->z_fgid,
- cr, ZFS_GROUP);
-
- if (zfsvfs->z_use_fuids &&
- IS_EPHEMERAL(acl_ids->z_fgid)) {
- domain = zfs_fuid_idx_domain(
- &zfsvfs->z_fuid_idx,
- FUID_INDEX(acl_ids->z_fgid));
- rid = FUID_RID(acl_ids->z_fgid);
- zfs_fuid_node_add(&acl_ids->z_fuidp,
- domain, rid,
- FUID_INDEX(acl_ids->z_fgid),
- acl_ids->z_fgid, ZFS_GROUP);
- }
- } else {
- acl_ids->z_fgid = zfs_fuid_create_cred(zfsvfs,
- ZFS_GROUP, cr, &acl_ids->z_fuidp);
- gid = crgetgid(cr);
- }
- }
- }
-#endif /* HAVE_KSID */
-
- /*
- * If we're creating a directory, and the parent directory has the
- * set-GID bit set, set in on the new directory.
- * Otherwise, if the user is neither privileged nor a member of the
- * file's new group, clear the file's set-GID bit.
- */
-
- if (!(flag & IS_ROOT_NODE) && (dzp->z_mode & S_ISGID) &&
- (S_ISDIR(vap->va_mode))) {
- acl_ids->z_mode |= S_ISGID;
- } else {
- if ((acl_ids->z_mode & S_ISGID) &&
- secpolicy_vnode_setids_setgids(cr, gid) != 0)
- acl_ids->z_mode &= ~S_ISGID;
- }
-
- if (acl_ids->z_aclp == NULL) {
- mutex_enter(&dzp->z_acl_lock);
- mutex_enter(&dzp->z_lock);
- if (!(flag & IS_ROOT_NODE) && (S_ISDIR(ZTOI(dzp)->i_mode) &&
- (dzp->z_pflags & ZFS_INHERIT_ACE)) &&
- !(dzp->z_pflags & ZFS_XATTR)) {
- VERIFY(0 == zfs_acl_node_read(dzp, B_TRUE,
- &paclp, B_FALSE));
- acl_ids->z_aclp = zfs_acl_inherit(zfsvfs,
- vap->va_mode, paclp, acl_ids->z_mode, &need_chmod);
- inherited = B_TRUE;
- } else {
- acl_ids->z_aclp =
- zfs_acl_alloc(zfs_acl_version_zp(dzp));
- acl_ids->z_aclp->z_hints |= ZFS_ACL_TRIVIAL;
- }
- mutex_exit(&dzp->z_lock);
- mutex_exit(&dzp->z_acl_lock);
- if (need_chmod) {
- acl_ids->z_aclp->z_hints |= S_ISDIR(vap->va_mode) ?
- ZFS_ACL_AUTO_INHERIT : 0;
- zfs_acl_chmod(zfsvfs, acl_ids->z_mode, acl_ids->z_aclp);
- }
- }
-
- if (inherited || vsecp) {
- acl_ids->z_mode = zfs_mode_compute(acl_ids->z_mode,
- acl_ids->z_aclp, &acl_ids->z_aclp->z_hints,
- acl_ids->z_fuid, acl_ids->z_fgid);
- if (ace_trivial_common(acl_ids->z_aclp, 0, zfs_ace_walk) == 0)
- acl_ids->z_aclp->z_hints |= ZFS_ACL_TRIVIAL;
- }
-
- return (0);
-}
-
-/*
- * Free ACL and fuid_infop, but not the acl_ids structure
- */
-void
-zfs_acl_ids_free(zfs_acl_ids_t *acl_ids)
-{
- if (acl_ids->z_aclp)
- zfs_acl_free(acl_ids->z_aclp);
- if (acl_ids->z_fuidp)
- zfs_fuid_info_free(acl_ids->z_fuidp);
- acl_ids->z_aclp = NULL;
- acl_ids->z_fuidp = NULL;
-}
-
-boolean_t
-zfs_acl_ids_overquota(zfsvfs_t *zv, zfs_acl_ids_t *acl_ids, uint64_t projid)
-{
- return (zfs_id_overquota(zv, DMU_USERUSED_OBJECT, acl_ids->z_fuid) ||
- zfs_id_overquota(zv, DMU_GROUPUSED_OBJECT, acl_ids->z_fgid) ||
- (projid != ZFS_DEFAULT_PROJID && projid != ZFS_INVALID_PROJID &&
- zfs_id_overquota(zv, DMU_PROJECTUSED_OBJECT, projid)));
-}
-
-/*
- * Retrieve a file's ACL
- */
-int
-zfs_getacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr)
-{
- zfs_acl_t *aclp;
- ulong_t mask;
- int error;
- int count = 0;
- int largeace = 0;
-
- mask = vsecp->vsa_mask & (VSA_ACE | VSA_ACECNT |
- VSA_ACE_ACLFLAGS | VSA_ACE_ALLTYPES);
-
- if (mask == 0)
- return (SET_ERROR(ENOSYS));
-
- if ((error = zfs_zaccess(zp, ACE_READ_ACL, 0, skipaclchk, cr)))
- return (error);
-
- mutex_enter(&zp->z_acl_lock);
-
- error = zfs_acl_node_read(zp, B_FALSE, &aclp, B_FALSE);
- if (error != 0) {
- mutex_exit(&zp->z_acl_lock);
- return (error);
- }
-
- /*
- * Scan ACL to determine number of ACEs
- */
- if ((zp->z_pflags & ZFS_ACL_OBJ_ACE) && !(mask & VSA_ACE_ALLTYPES)) {
- void *zacep = NULL;
- uint64_t who;
- uint32_t access_mask;
- uint16_t type, iflags;
-
- while ((zacep = zfs_acl_next_ace(aclp, zacep,
- &who, &access_mask, &iflags, &type))) {
- switch (type) {
- case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE:
- case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE:
- case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE:
- largeace++;
- continue;
- default:
- count++;
- }
- }
- vsecp->vsa_aclcnt = count;
- } else
- count = (int)aclp->z_acl_count;
-
- if (mask & VSA_ACECNT) {
- vsecp->vsa_aclcnt = count;
- }
-
- if (mask & VSA_ACE) {
- size_t aclsz;
-
- aclsz = count * sizeof (ace_t) +
- sizeof (ace_object_t) * largeace;
-
- vsecp->vsa_aclentp = kmem_alloc(aclsz, KM_SLEEP);
- vsecp->vsa_aclentsz = aclsz;
-
- if (aclp->z_version == ZFS_ACL_VERSION_FUID)
- zfs_copy_fuid_2_ace(ZTOZSB(zp), aclp, cr,
- vsecp->vsa_aclentp, !(mask & VSA_ACE_ALLTYPES));
- else {
- zfs_acl_node_t *aclnode;
- void *start = vsecp->vsa_aclentp;
-
- for (aclnode = list_head(&aclp->z_acl); aclnode;
- aclnode = list_next(&aclp->z_acl, aclnode)) {
- bcopy(aclnode->z_acldata, start,
- aclnode->z_size);
- start = (caddr_t)start + aclnode->z_size;
- }
- ASSERT((caddr_t)start - (caddr_t)vsecp->vsa_aclentp ==
- aclp->z_acl_bytes);
- }
- }
- if (mask & VSA_ACE_ACLFLAGS) {
- vsecp->vsa_aclflags = 0;
- if (zp->z_pflags & ZFS_ACL_DEFAULTED)
- vsecp->vsa_aclflags |= ACL_DEFAULTED;
- if (zp->z_pflags & ZFS_ACL_PROTECTED)
- vsecp->vsa_aclflags |= ACL_PROTECTED;
- if (zp->z_pflags & ZFS_ACL_AUTO_INHERIT)
- vsecp->vsa_aclflags |= ACL_AUTO_INHERIT;
- }
-
- mutex_exit(&zp->z_acl_lock);
-
- return (0);
-}
-
-int
-zfs_vsec_2_aclp(zfsvfs_t *zfsvfs, umode_t obj_mode,
- vsecattr_t *vsecp, cred_t *cr, zfs_fuid_info_t **fuidp, zfs_acl_t **zaclp)
-{
- zfs_acl_t *aclp;
- zfs_acl_node_t *aclnode;
- int aclcnt = vsecp->vsa_aclcnt;
- int error;
-
- if (vsecp->vsa_aclcnt > MAX_ACL_ENTRIES || vsecp->vsa_aclcnt <= 0)
- return (SET_ERROR(EINVAL));
-
- aclp = zfs_acl_alloc(zfs_acl_version(zfsvfs->z_version));
-
- aclp->z_hints = 0;
- aclnode = zfs_acl_node_alloc(aclcnt * sizeof (zfs_object_ace_t));
- if (aclp->z_version == ZFS_ACL_VERSION_INITIAL) {
- if ((error = zfs_copy_ace_2_oldace(obj_mode, aclp,
- (ace_t *)vsecp->vsa_aclentp, aclnode->z_acldata,
- aclcnt, &aclnode->z_size)) != 0) {
- zfs_acl_free(aclp);
- zfs_acl_node_free(aclnode);
- return (error);
- }
- } else {
- if ((error = zfs_copy_ace_2_fuid(zfsvfs, obj_mode, aclp,
- vsecp->vsa_aclentp, aclnode->z_acldata, aclcnt,
- &aclnode->z_size, fuidp, cr)) != 0) {
- zfs_acl_free(aclp);
- zfs_acl_node_free(aclnode);
- return (error);
- }
- }
- aclp->z_acl_bytes = aclnode->z_size;
- aclnode->z_ace_count = aclcnt;
- aclp->z_acl_count = aclcnt;
- list_insert_head(&aclp->z_acl, aclnode);
-
- /*
- * If flags are being set then add them to z_hints
- */
- if (vsecp->vsa_mask & VSA_ACE_ACLFLAGS) {
- if (vsecp->vsa_aclflags & ACL_PROTECTED)
- aclp->z_hints |= ZFS_ACL_PROTECTED;
- if (vsecp->vsa_aclflags & ACL_DEFAULTED)
- aclp->z_hints |= ZFS_ACL_DEFAULTED;
- if (vsecp->vsa_aclflags & ACL_AUTO_INHERIT)
- aclp->z_hints |= ZFS_ACL_AUTO_INHERIT;
- }
-
- *zaclp = aclp;
-
- return (0);
-}
-
-/*
- * Set a file's ACL
- */
-int
-zfs_setacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- zilog_t *zilog = zfsvfs->z_log;
- ulong_t mask = vsecp->vsa_mask & (VSA_ACE | VSA_ACECNT);
- dmu_tx_t *tx;
- int error;
- zfs_acl_t *aclp;
- zfs_fuid_info_t *fuidp = NULL;
- boolean_t fuid_dirtied;
- uint64_t acl_obj;
-
- if (mask == 0)
- return (SET_ERROR(ENOSYS));
-
- if (zp->z_pflags & ZFS_IMMUTABLE)
- return (SET_ERROR(EPERM));
-
- if ((error = zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr)))
- return (error);
-
- error = zfs_vsec_2_aclp(zfsvfs, ZTOI(zp)->i_mode, vsecp, cr, &fuidp,
- &aclp);
- if (error)
- return (error);
-
- /*
- * If ACL wide flags aren't being set then preserve any
- * existing flags.
- */
- if (!(vsecp->vsa_mask & VSA_ACE_ACLFLAGS)) {
- aclp->z_hints |=
- (zp->z_pflags & V4_ACL_WIDE_FLAGS);
- }
-top:
- mutex_enter(&zp->z_acl_lock);
- mutex_enter(&zp->z_lock);
-
- tx = dmu_tx_create(zfsvfs->z_os);
-
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
-
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
-
- /*
- * If old version and ACL won't fit in bonus and we aren't
- * upgrading then take out necessary DMU holds
- */
-
- if ((acl_obj = zfs_external_acl(zp)) != 0) {
- if (zfsvfs->z_version >= ZPL_VERSION_FUID &&
- zfs_znode_acl_version(zp) <= ZFS_ACL_VERSION_INITIAL) {
- dmu_tx_hold_free(tx, acl_obj, 0,
- DMU_OBJECT_END);
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
- aclp->z_acl_bytes);
- } else {
- dmu_tx_hold_write(tx, acl_obj, 0, aclp->z_acl_bytes);
- }
- } else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, aclp->z_acl_bytes);
- }
-
- zfs_sa_upgrade_txholds(tx, zp);
- error = dmu_tx_assign(tx, TXG_NOWAIT);
- if (error) {
- mutex_exit(&zp->z_acl_lock);
- mutex_exit(&zp->z_lock);
-
- if (error == ERESTART) {
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- dmu_tx_abort(tx);
- zfs_acl_free(aclp);
- return (error);
- }
-
- error = zfs_aclset_common(zp, aclp, cr, tx);
- ASSERT(error == 0);
- ASSERT(zp->z_acl_cached == NULL);
- zp->z_acl_cached = aclp;
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- zfs_log_acl(zilog, tx, zp, vsecp, fuidp);
-
- if (fuidp)
- zfs_fuid_info_free(fuidp);
- dmu_tx_commit(tx);
-
- mutex_exit(&zp->z_lock);
- mutex_exit(&zp->z_acl_lock);
-
- return (error);
-}
-
-/*
- * Check accesses of interest (AoI) against attributes of the dataset
- * such as read-only. Returns zero if no AoI conflict with dataset
- * attributes, otherwise an appropriate errno is returned.
- */
-static int
-zfs_zaccess_dataset_check(znode_t *zp, uint32_t v4_mode)
-{
- if ((v4_mode & WRITE_MASK) && (zfs_is_readonly(ZTOZSB(zp))) &&
- (!S_ISDEV(ZTOI(zp)->i_mode) ||
- (S_ISDEV(ZTOI(zp)->i_mode) && (v4_mode & WRITE_MASK_ATTRS)))) {
- return (SET_ERROR(EROFS));
- }
-
- /*
- * Only check for READONLY on non-directories.
- */
- if ((v4_mode & WRITE_MASK_DATA) &&
- ((!S_ISDIR(ZTOI(zp)->i_mode) &&
- (zp->z_pflags & (ZFS_READONLY | ZFS_IMMUTABLE))) ||
- (S_ISDIR(ZTOI(zp)->i_mode) &&
- (zp->z_pflags & ZFS_IMMUTABLE)))) {
- return (SET_ERROR(EPERM));
- }
-
- if ((v4_mode & (ACE_DELETE | ACE_DELETE_CHILD)) &&
- (zp->z_pflags & ZFS_NOUNLINK)) {
- return (SET_ERROR(EPERM));
- }
-
- if (((v4_mode & (ACE_READ_DATA|ACE_EXECUTE)) &&
- (zp->z_pflags & ZFS_AV_QUARANTINED))) {
- return (SET_ERROR(EACCES));
- }
-
- return (0);
-}
-
-/*
- * The primary usage of this function is to loop through all of the
- * ACEs in the znode, determining what accesses of interest (AoI) to
- * the caller are allowed or denied. The AoI are expressed as bits in
- * the working_mode parameter. As each ACE is processed, bits covered
- * by that ACE are removed from the working_mode. This removal
- * facilitates two things. The first is that when the working mode is
- * empty (= 0), we know we've looked at all the AoI. The second is
- * that the ACE interpretation rules don't allow a later ACE to undo
- * something granted or denied by an earlier ACE. Removing the
- * discovered access or denial enforces this rule. At the end of
- * processing the ACEs, all AoI that were found to be denied are
- * placed into the working_mode, giving the caller a mask of denied
- * accesses. Returns:
- * 0 if all AoI granted
- * EACCES if the denied mask is non-zero
- * other error if abnormal failure (e.g., IO error)
- *
- * A secondary usage of the function is to determine if any of the
- * AoI are granted. If an ACE grants any access in
- * the working_mode, we immediately short circuit out of the function.
- * This mode is chosen by setting anyaccess to B_TRUE. The
- * working_mode is not a denied access mask upon exit if the function
- * is used in this manner.
- */
-static int
-zfs_zaccess_aces_check(znode_t *zp, uint32_t *working_mode,
- boolean_t anyaccess, cred_t *cr)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- zfs_acl_t *aclp;
- int error;
- uid_t uid = crgetuid(cr);
- uint64_t who;
- uint16_t type, iflags;
- uint16_t entry_type;
- uint32_t access_mask;
- uint32_t deny_mask = 0;
- zfs_ace_hdr_t *acep = NULL;
- boolean_t checkit;
- uid_t gowner;
- uid_t fowner;
-
- zfs_fuid_map_ids(zp, cr, &fowner, &gowner);
-
- mutex_enter(&zp->z_acl_lock);
-
- error = zfs_acl_node_read(zp, B_FALSE, &aclp, B_FALSE);
- if (error != 0) {
- mutex_exit(&zp->z_acl_lock);
- return (error);
- }
-
- ASSERT(zp->z_acl_cached);
-
- while ((acep = zfs_acl_next_ace(aclp, acep, &who, &access_mask,
- &iflags, &type))) {
- uint32_t mask_matched;
-
- if (!zfs_acl_valid_ace_type(type, iflags))
- continue;
-
- if (S_ISDIR(ZTOI(zp)->i_mode) &&
- (iflags & ACE_INHERIT_ONLY_ACE))
- continue;
-
- /* Skip ACE if it does not affect any AoI */
- mask_matched = (access_mask & *working_mode);
- if (!mask_matched)
- continue;
-
- entry_type = (iflags & ACE_TYPE_FLAGS);
-
- checkit = B_FALSE;
-
- switch (entry_type) {
- case ACE_OWNER:
- if (uid == fowner)
- checkit = B_TRUE;
- break;
- case OWNING_GROUP:
- who = gowner;
- /*FALLTHROUGH*/
- case ACE_IDENTIFIER_GROUP:
- checkit = zfs_groupmember(zfsvfs, who, cr);
- break;
- case ACE_EVERYONE:
- checkit = B_TRUE;
- break;
-
- /* USER Entry */
- default:
- if (entry_type == 0) {
- uid_t newid;
-
- newid = zfs_fuid_map_id(zfsvfs, who, cr,
- ZFS_ACE_USER);
- if (newid != IDMAP_WK_CREATOR_OWNER_UID &&
- uid == newid)
- checkit = B_TRUE;
- break;
- } else {
- mutex_exit(&zp->z_acl_lock);
- return (SET_ERROR(EIO));
- }
- }
-
- if (checkit) {
- if (type == DENY) {
- DTRACE_PROBE3(zfs__ace__denies,
- znode_t *, zp,
- zfs_ace_hdr_t *, acep,
- uint32_t, mask_matched);
- deny_mask |= mask_matched;
- } else {
- DTRACE_PROBE3(zfs__ace__allows,
- znode_t *, zp,
- zfs_ace_hdr_t *, acep,
- uint32_t, mask_matched);
- if (anyaccess) {
- mutex_exit(&zp->z_acl_lock);
- return (0);
- }
- }
- *working_mode &= ~mask_matched;
- }
-
- /* Are we done? */
- if (*working_mode == 0)
- break;
- }
-
- mutex_exit(&zp->z_acl_lock);
-
- /* Put the found 'denies' back on the working mode */
- if (deny_mask) {
- *working_mode |= deny_mask;
- return (SET_ERROR(EACCES));
- } else if (*working_mode) {
- return (-1);
- }
-
- return (0);
-}
-
-/*
- * Return true if any access whatsoever granted, we don't actually
- * care what access is granted.
- */
-boolean_t
-zfs_has_access(znode_t *zp, cred_t *cr)
-{
- uint32_t have = ACE_ALL_PERMS;
-
- if (zfs_zaccess_aces_check(zp, &have, B_TRUE, cr) != 0) {
- uid_t owner;
-
- owner = zfs_fuid_map_id(ZTOZSB(zp),
- KUID_TO_SUID(ZTOI(zp)->i_uid), cr, ZFS_OWNER);
- return (secpolicy_vnode_any_access(cr, ZTOI(zp), owner) == 0);
- }
- return (B_TRUE);
-}
-
-static int
-zfs_zaccess_common(znode_t *zp, uint32_t v4_mode, uint32_t *working_mode,
- boolean_t *check_privs, boolean_t skipaclchk, cred_t *cr)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- int err;
-
- *working_mode = v4_mode;
- *check_privs = B_TRUE;
-
- /*
- * Short circuit empty requests
- */
- if (v4_mode == 0 || zfsvfs->z_replay) {
- *working_mode = 0;
- return (0);
- }
-
- if ((err = zfs_zaccess_dataset_check(zp, v4_mode)) != 0) {
- *check_privs = B_FALSE;
- return (err);
- }
-
- /*
- * The caller requested that the ACL check be skipped. This
- * would only happen if the caller checked VOP_ACCESS() with a
- * 32 bit ACE mask and already had the appropriate permissions.
- */
- if (skipaclchk) {
- *working_mode = 0;
- return (0);
- }
-
- return (zfs_zaccess_aces_check(zp, working_mode, B_FALSE, cr));
-}
-
-static int
-zfs_zaccess_append(znode_t *zp, uint32_t *working_mode, boolean_t *check_privs,
- cred_t *cr)
-{
- if (*working_mode != ACE_WRITE_DATA)
- return (SET_ERROR(EACCES));
-
- return (zfs_zaccess_common(zp, ACE_APPEND_DATA, working_mode,
- check_privs, B_FALSE, cr));
-}
-
-int
-zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr)
-{
- boolean_t owner = B_FALSE;
- boolean_t groupmbr = B_FALSE;
- boolean_t is_attr;
- uid_t uid = crgetuid(cr);
- int error;
-
- if (zdp->z_pflags & ZFS_AV_QUARANTINED)
- return (SET_ERROR(EACCES));
-
- is_attr = ((zdp->z_pflags & ZFS_XATTR) &&
- (S_ISDIR(ZTOI(zdp)->i_mode)));
- if (is_attr)
- goto slow;
-
-
- mutex_enter(&zdp->z_acl_lock);
-
- if (zdp->z_pflags & ZFS_NO_EXECS_DENIED) {
- mutex_exit(&zdp->z_acl_lock);
- return (0);
- }
-
- if (KUID_TO_SUID(ZTOI(zdp)->i_uid) != 0 ||
- KGID_TO_SGID(ZTOI(zdp)->i_gid) != 0) {
- mutex_exit(&zdp->z_acl_lock);
- goto slow;
- }
-
- if (uid == KUID_TO_SUID(ZTOI(zdp)->i_uid)) {
- owner = B_TRUE;
- if (zdp->z_mode & S_IXUSR) {
- mutex_exit(&zdp->z_acl_lock);
- return (0);
- } else {
- mutex_exit(&zdp->z_acl_lock);
- goto slow;
- }
- }
- if (groupmember(KGID_TO_SGID(ZTOI(zdp)->i_gid), cr)) {
- groupmbr = B_TRUE;
- if (zdp->z_mode & S_IXGRP) {
- mutex_exit(&zdp->z_acl_lock);
- return (0);
- } else {
- mutex_exit(&zdp->z_acl_lock);
- goto slow;
- }
- }
- if (!owner && !groupmbr) {
- if (zdp->z_mode & S_IXOTH) {
- mutex_exit(&zdp->z_acl_lock);
- return (0);
- }
- }
-
- mutex_exit(&zdp->z_acl_lock);
-
-slow:
- DTRACE_PROBE(zfs__fastpath__execute__access__miss);
- ZFS_ENTER(ZTOZSB(zdp));
- error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr);
- ZFS_EXIT(ZTOZSB(zdp));
- return (error);
-}
-
-/*
- * Determine whether Access should be granted/denied.
- *
- * The least priv subsystem is always consulted as a basic privilege
- * can define any form of access.
- */
-int
-zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr)
-{
- uint32_t working_mode;
- int error;
- int is_attr;
- boolean_t check_privs;
- znode_t *xzp;
- znode_t *check_zp = zp;
- mode_t needed_bits;
- uid_t owner;
-
- is_attr = ((zp->z_pflags & ZFS_XATTR) && S_ISDIR(ZTOI(zp)->i_mode));
-
- /*
- * If attribute then validate against base file
- */
- if (is_attr) {
- if ((error = zfs_zget(ZTOZSB(zp),
- zp->z_xattr_parent, &xzp)) != 0) {
- return (error);
- }
-
- check_zp = xzp;
-
- /*
- * fixup mode to map to xattr perms
- */
-
- if (mode & (ACE_WRITE_DATA|ACE_APPEND_DATA)) {
- mode &= ~(ACE_WRITE_DATA|ACE_APPEND_DATA);
- mode |= ACE_WRITE_NAMED_ATTRS;
- }
-
- if (mode & (ACE_READ_DATA|ACE_EXECUTE)) {
- mode &= ~(ACE_READ_DATA|ACE_EXECUTE);
- mode |= ACE_READ_NAMED_ATTRS;
- }
- }
-
- owner = zfs_fuid_map_id(ZTOZSB(zp), KUID_TO_SUID(ZTOI(zp)->i_uid),
- cr, ZFS_OWNER);
- /*
- * Map the bits required to the standard inode flags
- * S_IRUSR|S_IWUSR|S_IXUSR in the needed_bits. Map the bits
- * mapped by working_mode (currently missing) in missing_bits.
- * Call secpolicy_vnode_access2() with (needed_bits & ~checkmode),
- * needed_bits.
- */
- needed_bits = 0;
-
- working_mode = mode;
- if ((working_mode & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
- owner == crgetuid(cr))
- working_mode &= ~(ACE_READ_ACL|ACE_READ_ATTRIBUTES);
-
- if (working_mode & (ACE_READ_DATA|ACE_READ_NAMED_ATTRS|
- ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_SYNCHRONIZE))
- needed_bits |= S_IRUSR;
- if (working_mode & (ACE_WRITE_DATA|ACE_WRITE_NAMED_ATTRS|
- ACE_APPEND_DATA|ACE_WRITE_ATTRIBUTES|ACE_SYNCHRONIZE))
- needed_bits |= S_IWUSR;
- if (working_mode & ACE_EXECUTE)
- needed_bits |= S_IXUSR;
-
- if ((error = zfs_zaccess_common(check_zp, mode, &working_mode,
- &check_privs, skipaclchk, cr)) == 0) {
- if (is_attr)
- iput(ZTOI(xzp));
- return (secpolicy_vnode_access2(cr, ZTOI(zp), owner,
- needed_bits, needed_bits));
- }
-
- if (error && !check_privs) {
- if (is_attr)
- iput(ZTOI(xzp));
- return (error);
- }
-
- if (error && (flags & V_APPEND)) {
- error = zfs_zaccess_append(zp, &working_mode, &check_privs, cr);
- }
-
- if (error && check_privs) {
- mode_t checkmode = 0;
-
- /*
- * First check for implicit owner permission on
- * read_acl/read_attributes
- */
-
- error = 0;
- ASSERT(working_mode != 0);
-
- if ((working_mode & (ACE_READ_ACL|ACE_READ_ATTRIBUTES) &&
- owner == crgetuid(cr)))
- working_mode &= ~(ACE_READ_ACL|ACE_READ_ATTRIBUTES);
-
- if (working_mode & (ACE_READ_DATA|ACE_READ_NAMED_ATTRS|
- ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_SYNCHRONIZE))
- checkmode |= S_IRUSR;
- if (working_mode & (ACE_WRITE_DATA|ACE_WRITE_NAMED_ATTRS|
- ACE_APPEND_DATA|ACE_WRITE_ATTRIBUTES|ACE_SYNCHRONIZE))
- checkmode |= S_IWUSR;
- if (working_mode & ACE_EXECUTE)
- checkmode |= S_IXUSR;
-
- error = secpolicy_vnode_access2(cr, ZTOI(check_zp), owner,
- needed_bits & ~checkmode, needed_bits);
-
- if (error == 0 && (working_mode & ACE_WRITE_OWNER))
- error = secpolicy_vnode_chown(cr, owner);
- if (error == 0 && (working_mode & ACE_WRITE_ACL))
- error = secpolicy_vnode_setdac(cr, owner);
-
- if (error == 0 && (working_mode &
- (ACE_DELETE|ACE_DELETE_CHILD)))
- error = secpolicy_vnode_remove(cr);
-
- if (error == 0 && (working_mode & ACE_SYNCHRONIZE)) {
- error = secpolicy_vnode_chown(cr, owner);
- }
- if (error == 0) {
- /*
- * See if any bits other than those already checked
- * for are still present. If so then return EACCES
- */
- if (working_mode & ~(ZFS_CHECKED_MASKS)) {
- error = SET_ERROR(EACCES);
- }
- }
- } else if (error == 0) {
- error = secpolicy_vnode_access2(cr, ZTOI(zp), owner,
- needed_bits, needed_bits);
- }
-
- if (is_attr)
- iput(ZTOI(xzp));
-
- return (error);
-}
-
-/*
- * Translate traditional unix S_IRUSR/S_IWUSR/S_IXUSR mode into
- * native ACL format and call zfs_zaccess()
- */
-int
-zfs_zaccess_rwx(znode_t *zp, mode_t mode, int flags, cred_t *cr)
-{
- return (zfs_zaccess(zp, zfs_unix_to_v4(mode >> 6), flags, B_FALSE, cr));
-}
-
-/*
- * Access function for secpolicy_vnode_setattr
- */
-int
-zfs_zaccess_unix(znode_t *zp, mode_t mode, cred_t *cr)
-{
- int v4_mode = zfs_unix_to_v4(mode >> 6);
-
- return (zfs_zaccess(zp, v4_mode, 0, B_FALSE, cr));
-}
-
-static int
-zfs_delete_final_check(znode_t *zp, znode_t *dzp,
- mode_t available_perms, cred_t *cr)
-{
- int error;
- uid_t downer;
-
- downer = zfs_fuid_map_id(ZTOZSB(dzp), KUID_TO_SUID(ZTOI(dzp)->i_uid),
- cr, ZFS_OWNER);
-
- error = secpolicy_vnode_access2(cr, ZTOI(dzp),
- downer, available_perms, S_IWUSR|S_IXUSR);
-
- if (error == 0)
- error = zfs_sticky_remove_access(dzp, zp, cr);
-
- return (error);
-}
-
-/*
- * Determine whether Access should be granted/deny, without
- * consulting least priv subsystem.
- *
- * The following chart is the recommended NFSv4 enforcement for
- * ability to delete an object.
- *
- * -------------------------------------------------------
- * | Parent Dir | Target Object Permissions |
- * | permissions | |
- * -------------------------------------------------------
- * | | ACL Allows | ACL Denies| Delete |
- * | | Delete | Delete | unspecified|
- * -------------------------------------------------------
- * | ACL Allows | Permit | Permit | Permit |
- * | DELETE_CHILD | |
- * -------------------------------------------------------
- * | ACL Denies | Permit | Deny | Deny |
- * | DELETE_CHILD | | | |
- * -------------------------------------------------------
- * | ACL specifies | | | |
- * | only allow | Permit | Permit | Permit |
- * | write and | | | |
- * | execute | | | |
- * -------------------------------------------------------
- * | ACL denies | | | |
- * | write and | Permit | Deny | Deny |
- * | execute | | | |
- * -------------------------------------------------------
- * ^
- * |
- * No search privilege, can't even look up file?
- *
- */
-int
-zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr)
-{
- uint32_t dzp_working_mode = 0;
- uint32_t zp_working_mode = 0;
- int dzp_error, zp_error;
- mode_t available_perms;
- boolean_t dzpcheck_privs = B_TRUE;
- boolean_t zpcheck_privs = B_TRUE;
-
- /*
- * We want specific DELETE permissions to
- * take precedence over WRITE/EXECUTE. We don't
- * want an ACL such as this to mess us up.
- * user:joe:write_data:deny,user:joe:delete:allow
- *
- * However, deny permissions may ultimately be overridden
- * by secpolicy_vnode_access().
- *
- * We will ask for all of the necessary permissions and then
- * look at the working modes from the directory and target object
- * to determine what was found.
- */
-
- if (zp->z_pflags & (ZFS_IMMUTABLE | ZFS_NOUNLINK))
- return (SET_ERROR(EPERM));
-
- /*
- * First row
- * If the directory permissions allow the delete, we are done.
- */
- if ((dzp_error = zfs_zaccess_common(dzp, ACE_DELETE_CHILD,
- &dzp_working_mode, &dzpcheck_privs, B_FALSE, cr)) == 0)
- return (0);
-
- /*
- * If target object has delete permission then we are done
- */
- if ((zp_error = zfs_zaccess_common(zp, ACE_DELETE, &zp_working_mode,
- &zpcheck_privs, B_FALSE, cr)) == 0)
- return (0);
-
- ASSERT(dzp_error && zp_error);
-
- if (!dzpcheck_privs)
- return (dzp_error);
- if (!zpcheck_privs)
- return (zp_error);
-
- /*
- * Second row
- *
- * If directory returns EACCES then delete_child was denied
- * due to deny delete_child. In this case send the request through
- * secpolicy_vnode_remove(). We don't use zfs_delete_final_check()
- * since that *could* allow the delete based on write/execute permission
- * and we want delete permissions to override write/execute.
- */
-
- if (dzp_error == EACCES)
- return (secpolicy_vnode_remove(cr));
-
- /*
- * Third Row
- * only need to see if we have write/execute on directory.
- */
-
- dzp_error = zfs_zaccess_common(dzp, ACE_EXECUTE|ACE_WRITE_DATA,
- &dzp_working_mode, &dzpcheck_privs, B_FALSE, cr);
-
- if (dzp_error != 0 && !dzpcheck_privs)
- return (dzp_error);
-
- /*
- * Fourth row
- */
-
- available_perms = (dzp_working_mode & ACE_WRITE_DATA) ? 0 : S_IWUSR;
- available_perms |= (dzp_working_mode & ACE_EXECUTE) ? 0 : S_IXUSR;
-
- return (zfs_delete_final_check(zp, dzp, available_perms, cr));
-
-}
-
-int
-zfs_zaccess_rename(znode_t *sdzp, znode_t *szp, znode_t *tdzp,
- znode_t *tzp, cred_t *cr)
-{
- int add_perm;
- int error;
-
- if (szp->z_pflags & ZFS_AV_QUARANTINED)
- return (SET_ERROR(EACCES));
-
- add_perm = S_ISDIR(ZTOI(szp)->i_mode) ?
- ACE_ADD_SUBDIRECTORY : ACE_ADD_FILE;
-
- /*
- * Rename permissions are combination of delete permission +
- * add file/subdir permission.
- */
-
- /*
- * first make sure we do the delete portion.
- *
- * If that succeeds then check for add_file/add_subdir permissions
- */
-
- if ((error = zfs_zaccess_delete(sdzp, szp, cr)))
- return (error);
-
- /*
- * If we have a tzp, see if we can delete it?
- */
- if (tzp) {
- if ((error = zfs_zaccess_delete(tdzp, tzp, cr)))
- return (error);
- }
-
- /*
- * Now check for add permissions
- */
- error = zfs_zaccess(tdzp, add_perm, 0, B_FALSE, cr);
-
- return (error);
-}
diff --git a/module/zfs/zfs_ctldir.c b/module/zfs/zfs_ctldir.c
deleted file mode 100644
index 1e61ef06d..000000000
--- a/module/zfs/zfs_ctldir.c
+++ /dev/null
@@ -1,1240 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- *
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
- * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- * LLNL-CODE-403049.
- * Rewritten for Linux by:
- * Rohan Puri <[email protected]>
- * Brian Behlendorf <[email protected]>
- * Copyright (c) 2013 by Delphix. All rights reserved.
- * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
- * Copyright (c) 2018 George Melikov. All Rights Reserved.
- * Copyright (c) 2019 Datto, Inc. All rights reserved.
- */
-
-/*
- * ZFS control directory (a.k.a. ".zfs")
- *
- * This directory provides a common location for all ZFS meta-objects.
- * Currently, this is only the 'snapshot' and 'shares' directory, but this may
- * expand in the future. The elements are built dynamically, as the hierarchy
- * does not actually exist on disk.
- *
- * For 'snapshot', we don't want to have all snapshots always mounted, because
- * this would take up a huge amount of space in /etc/mnttab. We have three
- * types of objects:
- *
- * ctldir ------> snapshotdir -------> snapshot
- * |
- * |
- * V
- * mounted fs
- *
- * The 'snapshot' node contains just enough information to lookup '..' and act
- * as a mountpoint for the snapshot. Whenever we lookup a specific snapshot, we
- * perform an automount of the underlying filesystem and return the
- * corresponding inode.
- *
- * All mounts are handled automatically by an user mode helper which invokes
- * the mount procedure. Unmounts are handled by allowing the mount
- * point to expire so the kernel may automatically unmount it.
- *
- * The '.zfs', '.zfs/snapshot', and all directories created under
- * '.zfs/snapshot' (ie: '.zfs/snapshot/<snapname>') all share the same
- * zfsvfs_t as the head filesystem (what '.zfs' lives under).
- *
- * File systems mounted on top of the '.zfs/snapshot/<snapname>' paths
- * (ie: snapshots) are complete ZFS filesystems and have their own unique
- * zfsvfs_t. However, the fsid reported by these mounts will be the same
- * as that used by the parent zfsvfs_t to make NFS happy.
- */
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysmacros.h>
-#include <sys/pathname.h>
-#include <sys/vfs.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zfs_ioctl.h>
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/stat.h>
-#include <sys/dmu.h>
-#include <sys/dmu_objset.h>
-#include <sys/dsl_destroy.h>
-#include <sys/dsl_deleg.h>
-#include <sys/zpl.h>
-#include <sys/mntent.h>
-#include "zfs_namecheck.h"
-
-/*
- * Two AVL trees are maintained which contain all currently automounted
- * snapshots. Every automounted snapshots maps to a single zfs_snapentry_t
- * entry which MUST:
- *
- * - be attached to both trees, and
- * - be unique, no duplicate entries are allowed.
- *
- * The zfs_snapshots_by_name tree is indexed by the full dataset name
- * while the zfs_snapshots_by_objsetid tree is indexed by the unique
- * objsetid. This allows for fast lookups either by name or objsetid.
- */
-static avl_tree_t zfs_snapshots_by_name;
-static avl_tree_t zfs_snapshots_by_objsetid;
-static krwlock_t zfs_snapshot_lock;
-
-/*
- * Control Directory Tunables (.zfs)
- */
-int zfs_expire_snapshot = ZFSCTL_EXPIRE_SNAPSHOT;
-int zfs_admin_snapshot = 0;
-
-typedef struct {
- char *se_name; /* full snapshot name */
- char *se_path; /* full mount path */
- spa_t *se_spa; /* pool spa */
- uint64_t se_objsetid; /* snapshot objset id */
- struct dentry *se_root_dentry; /* snapshot root dentry */
- taskqid_t se_taskqid; /* scheduled unmount taskqid */
- avl_node_t se_node_name; /* zfs_snapshots_by_name link */
- avl_node_t se_node_objsetid; /* zfs_snapshots_by_objsetid link */
- zfs_refcount_t se_refcount; /* reference count */
-} zfs_snapentry_t;
-
-static void zfsctl_snapshot_unmount_delay_impl(zfs_snapentry_t *se, int delay);
-
-/*
- * Allocate a new zfs_snapentry_t being careful to make a copy of the
- * the snapshot name and provided mount point. No reference is taken.
- */
-static zfs_snapentry_t *
-zfsctl_snapshot_alloc(char *full_name, char *full_path, spa_t *spa,
- uint64_t objsetid, struct dentry *root_dentry)
-{
- zfs_snapentry_t *se;
-
- se = kmem_zalloc(sizeof (zfs_snapentry_t), KM_SLEEP);
-
- se->se_name = strdup(full_name);
- se->se_path = strdup(full_path);
- se->se_spa = spa;
- se->se_objsetid = objsetid;
- se->se_root_dentry = root_dentry;
- se->se_taskqid = TASKQID_INVALID;
-
- zfs_refcount_create(&se->se_refcount);
-
- return (se);
-}
-
-/*
- * Free a zfs_snapentry_t the caller must ensure there are no active
- * references.
- */
-static void
-zfsctl_snapshot_free(zfs_snapentry_t *se)
-{
- zfs_refcount_destroy(&se->se_refcount);
- strfree(se->se_name);
- strfree(se->se_path);
-
- kmem_free(se, sizeof (zfs_snapentry_t));
-}
-
-/*
- * Hold a reference on the zfs_snapentry_t.
- */
-static void
-zfsctl_snapshot_hold(zfs_snapentry_t *se)
-{
- zfs_refcount_add(&se->se_refcount, NULL);
-}
-
-/*
- * Release a reference on the zfs_snapentry_t. When the number of
- * references drops to zero the structure will be freed.
- */
-static void
-zfsctl_snapshot_rele(zfs_snapentry_t *se)
-{
- if (zfs_refcount_remove(&se->se_refcount, NULL) == 0)
- zfsctl_snapshot_free(se);
-}
-
-/*
- * Add a zfs_snapentry_t to both the zfs_snapshots_by_name and
- * zfs_snapshots_by_objsetid trees. While the zfs_snapentry_t is part
- * of the trees a reference is held.
- */
-static void
-zfsctl_snapshot_add(zfs_snapentry_t *se)
-{
- ASSERT(RW_WRITE_HELD(&zfs_snapshot_lock));
- zfsctl_snapshot_hold(se);
- avl_add(&zfs_snapshots_by_name, se);
- avl_add(&zfs_snapshots_by_objsetid, se);
-}
-
-/*
- * Remove a zfs_snapentry_t from both the zfs_snapshots_by_name and
- * zfs_snapshots_by_objsetid trees. Upon removal a reference is dropped,
- * this can result in the structure being freed if that was the last
- * remaining reference.
- */
-static void
-zfsctl_snapshot_remove(zfs_snapentry_t *se)
-{
- ASSERT(RW_WRITE_HELD(&zfs_snapshot_lock));
- avl_remove(&zfs_snapshots_by_name, se);
- avl_remove(&zfs_snapshots_by_objsetid, se);
- zfsctl_snapshot_rele(se);
-}
-
-/*
- * Snapshot name comparison function for the zfs_snapshots_by_name.
- */
-static int
-snapentry_compare_by_name(const void *a, const void *b)
-{
- const zfs_snapentry_t *se_a = a;
- const zfs_snapentry_t *se_b = b;
- int ret;
-
- ret = strcmp(se_a->se_name, se_b->se_name);
-
- if (ret < 0)
- return (-1);
- else if (ret > 0)
- return (1);
- else
- return (0);
-}
-
-/*
- * Snapshot name comparison function for the zfs_snapshots_by_objsetid.
- */
-static int
-snapentry_compare_by_objsetid(const void *a, const void *b)
-{
- const zfs_snapentry_t *se_a = a;
- const zfs_snapentry_t *se_b = b;
-
- if (se_a->se_spa != se_b->se_spa)
- return ((ulong_t)se_a->se_spa < (ulong_t)se_b->se_spa ? -1 : 1);
-
- if (se_a->se_objsetid < se_b->se_objsetid)
- return (-1);
- else if (se_a->se_objsetid > se_b->se_objsetid)
- return (1);
- else
- return (0);
-}
-
-/*
- * Find a zfs_snapentry_t in zfs_snapshots_by_name. If the snapname
- * is found a pointer to the zfs_snapentry_t is returned and a reference
- * taken on the structure. The caller is responsible for dropping the
- * reference with zfsctl_snapshot_rele(). If the snapname is not found
- * NULL will be returned.
- */
-static zfs_snapentry_t *
-zfsctl_snapshot_find_by_name(char *snapname)
-{
- zfs_snapentry_t *se, search;
-
- ASSERT(RW_LOCK_HELD(&zfs_snapshot_lock));
-
- search.se_name = snapname;
- se = avl_find(&zfs_snapshots_by_name, &search, NULL);
- if (se)
- zfsctl_snapshot_hold(se);
-
- return (se);
-}
-
-/*
- * Find a zfs_snapentry_t in zfs_snapshots_by_objsetid given the objset id
- * rather than the snapname. In all other respects it behaves the same
- * as zfsctl_snapshot_find_by_name().
- */
-static zfs_snapentry_t *
-zfsctl_snapshot_find_by_objsetid(spa_t *spa, uint64_t objsetid)
-{
- zfs_snapentry_t *se, search;
-
- ASSERT(RW_LOCK_HELD(&zfs_snapshot_lock));
-
- search.se_spa = spa;
- search.se_objsetid = objsetid;
- se = avl_find(&zfs_snapshots_by_objsetid, &search, NULL);
- if (se)
- zfsctl_snapshot_hold(se);
-
- return (se);
-}
-
-/*
- * Rename a zfs_snapentry_t in the zfs_snapshots_by_name. The structure is
- * removed, renamed, and added back to the new correct location in the tree.
- */
-static int
-zfsctl_snapshot_rename(char *old_snapname, char *new_snapname)
-{
- zfs_snapentry_t *se;
-
- ASSERT(RW_WRITE_HELD(&zfs_snapshot_lock));
-
- se = zfsctl_snapshot_find_by_name(old_snapname);
- if (se == NULL)
- return (SET_ERROR(ENOENT));
-
- zfsctl_snapshot_remove(se);
- strfree(se->se_name);
- se->se_name = strdup(new_snapname);
- zfsctl_snapshot_add(se);
- zfsctl_snapshot_rele(se);
-
- return (0);
-}
-
-/*
- * Delayed task responsible for unmounting an expired automounted snapshot.
- */
-static void
-snapentry_expire(void *data)
-{
- zfs_snapentry_t *se = (zfs_snapentry_t *)data;
- spa_t *spa = se->se_spa;
- uint64_t objsetid = se->se_objsetid;
-
- if (zfs_expire_snapshot <= 0) {
- zfsctl_snapshot_rele(se);
- return;
- }
-
- se->se_taskqid = TASKQID_INVALID;
- (void) zfsctl_snapshot_unmount(se->se_name, MNT_EXPIRE);
- zfsctl_snapshot_rele(se);
-
- /*
- * Reschedule the unmount if the zfs_snapentry_t wasn't removed.
- * This can occur when the snapshot is busy.
- */
- rw_enter(&zfs_snapshot_lock, RW_READER);
- if ((se = zfsctl_snapshot_find_by_objsetid(spa, objsetid)) != NULL) {
- zfsctl_snapshot_unmount_delay_impl(se, zfs_expire_snapshot);
- zfsctl_snapshot_rele(se);
- }
- rw_exit(&zfs_snapshot_lock);
-}
-
-/*
- * Cancel an automatic unmount of a snapname. This callback is responsible
- * for dropping the reference on the zfs_snapentry_t which was taken when
- * during dispatch.
- */
-static void
-zfsctl_snapshot_unmount_cancel(zfs_snapentry_t *se)
-{
- if (taskq_cancel_id(system_delay_taskq, se->se_taskqid) == 0) {
- se->se_taskqid = TASKQID_INVALID;
- zfsctl_snapshot_rele(se);
- }
-}
-
-/*
- * Dispatch the unmount task for delayed handling with a hold protecting it.
- */
-static void
-zfsctl_snapshot_unmount_delay_impl(zfs_snapentry_t *se, int delay)
-{
- ASSERT3S(se->se_taskqid, ==, TASKQID_INVALID);
-
- if (delay <= 0)
- return;
-
- zfsctl_snapshot_hold(se);
- se->se_taskqid = taskq_dispatch_delay(system_delay_taskq,
- snapentry_expire, se, TQ_SLEEP, ddi_get_lbolt() + delay * HZ);
-}
-
-/*
- * Schedule an automatic unmount of objset id to occur in delay seconds from
- * now. Any previous delayed unmount will be cancelled in favor of the
- * updated deadline. A reference is taken by zfsctl_snapshot_find_by_name()
- * and held until the outstanding task is handled or cancelled.
- */
-int
-zfsctl_snapshot_unmount_delay(spa_t *spa, uint64_t objsetid, int delay)
-{
- zfs_snapentry_t *se;
- int error = ENOENT;
-
- rw_enter(&zfs_snapshot_lock, RW_READER);
- if ((se = zfsctl_snapshot_find_by_objsetid(spa, objsetid)) != NULL) {
- zfsctl_snapshot_unmount_cancel(se);
- zfsctl_snapshot_unmount_delay_impl(se, delay);
- zfsctl_snapshot_rele(se);
- error = 0;
- }
- rw_exit(&zfs_snapshot_lock);
-
- return (error);
-}
-
-/*
- * Check if snapname is currently mounted. Returned non-zero when mounted
- * and zero when unmounted.
- */
-static boolean_t
-zfsctl_snapshot_ismounted(char *snapname)
-{
- zfs_snapentry_t *se;
- boolean_t ismounted = B_FALSE;
-
- rw_enter(&zfs_snapshot_lock, RW_READER);
- if ((se = zfsctl_snapshot_find_by_name(snapname)) != NULL) {
- zfsctl_snapshot_rele(se);
- ismounted = B_TRUE;
- }
- rw_exit(&zfs_snapshot_lock);
-
- return (ismounted);
-}
-
-/*
- * Check if the given inode is a part of the virtual .zfs directory.
- */
-boolean_t
-zfsctl_is_node(struct inode *ip)
-{
- return (ITOZ(ip)->z_is_ctldir);
-}
-
-/*
- * Check if the given inode is a .zfs/snapshots/snapname directory.
- */
-boolean_t
-zfsctl_is_snapdir(struct inode *ip)
-{
- return (zfsctl_is_node(ip) && (ip->i_ino <= ZFSCTL_INO_SNAPDIRS));
-}
-
-/*
- * Allocate a new inode with the passed id and ops.
- */
-static struct inode *
-zfsctl_inode_alloc(zfsvfs_t *zfsvfs, uint64_t id,
- const struct file_operations *fops, const struct inode_operations *ops)
-{
- inode_timespec_t now;
- struct inode *ip;
- znode_t *zp;
-
- ip = new_inode(zfsvfs->z_sb);
- if (ip == NULL)
- return (NULL);
-
- now = current_time(ip);
- zp = ITOZ(ip);
- ASSERT3P(zp->z_dirlocks, ==, NULL);
- ASSERT3P(zp->z_acl_cached, ==, NULL);
- ASSERT3P(zp->z_xattr_cached, ==, NULL);
- zp->z_id = id;
- zp->z_unlinked = B_FALSE;
- zp->z_atime_dirty = B_FALSE;
- zp->z_zn_prefetch = B_FALSE;
- zp->z_moved = B_FALSE;
- zp->z_is_sa = B_FALSE;
- zp->z_is_mapped = B_FALSE;
- zp->z_is_ctldir = B_TRUE;
- zp->z_is_stale = B_FALSE;
- zp->z_sa_hdl = NULL;
- zp->z_blksz = 0;
- zp->z_seq = 0;
- zp->z_mapcnt = 0;
- zp->z_size = 0;
- zp->z_pflags = 0;
- zp->z_mode = 0;
- zp->z_sync_cnt = 0;
- ip->i_generation = 0;
- ip->i_ino = id;
- ip->i_mode = (S_IFDIR | S_IRWXUGO);
- ip->i_uid = SUID_TO_KUID(0);
- ip->i_gid = SGID_TO_KGID(0);
- ip->i_blkbits = SPA_MINBLOCKSHIFT;
- ip->i_atime = now;
- ip->i_mtime = now;
- ip->i_ctime = now;
- ip->i_fop = fops;
- ip->i_op = ops;
-#if defined(IOP_XATTR)
- ip->i_opflags &= ~IOP_XATTR;
-#endif
-
- if (insert_inode_locked(ip)) {
- unlock_new_inode(ip);
- iput(ip);
- return (NULL);
- }
-
- mutex_enter(&zfsvfs->z_znodes_lock);
- list_insert_tail(&zfsvfs->z_all_znodes, zp);
- zfsvfs->z_nr_znodes++;
- membar_producer();
- mutex_exit(&zfsvfs->z_znodes_lock);
-
- unlock_new_inode(ip);
-
- return (ip);
-}
-
-/*
- * Lookup the inode with given id, it will be allocated if needed.
- */
-static struct inode *
-zfsctl_inode_lookup(zfsvfs_t *zfsvfs, uint64_t id,
- const struct file_operations *fops, const struct inode_operations *ops)
-{
- struct inode *ip = NULL;
-
- while (ip == NULL) {
- ip = ilookup(zfsvfs->z_sb, (unsigned long)id);
- if (ip)
- break;
-
- /* May fail due to concurrent zfsctl_inode_alloc() */
- ip = zfsctl_inode_alloc(zfsvfs, id, fops, ops);
- }
-
- return (ip);
-}
-
-/*
- * Create the '.zfs' directory. This directory is cached as part of the VFS
- * structure. This results in a hold on the zfsvfs_t. The code in zfs_umount()
- * therefore checks against a vfs_count of 2 instead of 1. This reference
- * is removed when the ctldir is destroyed in the unmount. All other entities
- * under the '.zfs' directory are created dynamically as needed.
- *
- * Because the dynamically created '.zfs' directory entries assume the use
- * of 64-bit inode numbers this support must be disabled on 32-bit systems.
- */
-int
-zfsctl_create(zfsvfs_t *zfsvfs)
-{
- ASSERT(zfsvfs->z_ctldir == NULL);
-
- zfsvfs->z_ctldir = zfsctl_inode_alloc(zfsvfs, ZFSCTL_INO_ROOT,
- &zpl_fops_root, &zpl_ops_root);
- if (zfsvfs->z_ctldir == NULL)
- return (SET_ERROR(ENOENT));
-
- return (0);
-}
-
-/*
- * Destroy the '.zfs' directory or remove a snapshot from zfs_snapshots_by_name.
- * Only called when the filesystem is unmounted.
- */
-void
-zfsctl_destroy(zfsvfs_t *zfsvfs)
-{
- if (zfsvfs->z_issnap) {
- zfs_snapentry_t *se;
- spa_t *spa = zfsvfs->z_os->os_spa;
- uint64_t objsetid = dmu_objset_id(zfsvfs->z_os);
-
- rw_enter(&zfs_snapshot_lock, RW_WRITER);
- se = zfsctl_snapshot_find_by_objsetid(spa, objsetid);
- if (se != NULL)
- zfsctl_snapshot_remove(se);
- rw_exit(&zfs_snapshot_lock);
- if (se != NULL) {
- zfsctl_snapshot_unmount_cancel(se);
- zfsctl_snapshot_rele(se);
- }
- } else if (zfsvfs->z_ctldir) {
- iput(zfsvfs->z_ctldir);
- zfsvfs->z_ctldir = NULL;
- }
-}
-
-/*
- * Given a root znode, retrieve the associated .zfs directory.
- * Add a hold to the vnode and return it.
- */
-struct inode *
-zfsctl_root(znode_t *zp)
-{
- ASSERT(zfs_has_ctldir(zp));
- igrab(ZTOZSB(zp)->z_ctldir);
- return (ZTOZSB(zp)->z_ctldir);
-}
-
-/*
- * Generate a long fid to indicate a snapdir. We encode whether snapdir is
- * already mounted in gen field. We do this because nfsd lookup will not
- * trigger automount. Next time the nfsd does fh_to_dentry, we will notice
- * this and do automount and return ESTALE to force nfsd revalidate and follow
- * mount.
- */
-static int
-zfsctl_snapdir_fid(struct inode *ip, fid_t *fidp)
-{
- zfid_short_t *zfid = (zfid_short_t *)fidp;
- zfid_long_t *zlfid = (zfid_long_t *)fidp;
- uint32_t gen = 0;
- uint64_t object;
- uint64_t objsetid;
- int i;
- struct dentry *dentry;
-
- if (fidp->fid_len < LONG_FID_LEN) {
- fidp->fid_len = LONG_FID_LEN;
- return (SET_ERROR(ENOSPC));
- }
-
- object = ip->i_ino;
- objsetid = ZFSCTL_INO_SNAPDIRS - ip->i_ino;
- zfid->zf_len = LONG_FID_LEN;
-
- dentry = d_obtain_alias(igrab(ip));
- if (!IS_ERR(dentry)) {
- gen = !!d_mountpoint(dentry);
- dput(dentry);
- }
-
- for (i = 0; i < sizeof (zfid->zf_object); i++)
- zfid->zf_object[i] = (uint8_t)(object >> (8 * i));
-
- for (i = 0; i < sizeof (zfid->zf_gen); i++)
- zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));
-
- for (i = 0; i < sizeof (zlfid->zf_setid); i++)
- zlfid->zf_setid[i] = (uint8_t)(objsetid >> (8 * i));
-
- for (i = 0; i < sizeof (zlfid->zf_setgen); i++)
- zlfid->zf_setgen[i] = 0;
-
- return (0);
-}
-
-/*
- * Generate an appropriate fid for an entry in the .zfs directory.
- */
-int
-zfsctl_fid(struct inode *ip, fid_t *fidp)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- uint64_t object = zp->z_id;
- zfid_short_t *zfid;
- int i;
-
- ZFS_ENTER(zfsvfs);
-
- if (zfsctl_is_snapdir(ip)) {
- ZFS_EXIT(zfsvfs);
- return (zfsctl_snapdir_fid(ip, fidp));
- }
-
- if (fidp->fid_len < SHORT_FID_LEN) {
- fidp->fid_len = SHORT_FID_LEN;
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOSPC));
- }
-
- zfid = (zfid_short_t *)fidp;
-
- zfid->zf_len = SHORT_FID_LEN;
-
- for (i = 0; i < sizeof (zfid->zf_object); i++)
- zfid->zf_object[i] = (uint8_t)(object >> (8 * i));
-
- /* .zfs znodes always have a generation number of 0 */
- for (i = 0; i < sizeof (zfid->zf_gen); i++)
- zfid->zf_gen[i] = 0;
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*
- * Construct a full dataset name in full_name: "pool/dataset@snap_name"
- */
-static int
-zfsctl_snapshot_name(zfsvfs_t *zfsvfs, const char *snap_name, int len,
- char *full_name)
-{
- objset_t *os = zfsvfs->z_os;
-
- if (zfs_component_namecheck(snap_name, NULL, NULL) != 0)
- return (SET_ERROR(EILSEQ));
-
- dmu_objset_name(os, full_name);
- if ((strlen(full_name) + 1 + strlen(snap_name)) >= len)
- return (SET_ERROR(ENAMETOOLONG));
-
- (void) strcat(full_name, "@");
- (void) strcat(full_name, snap_name);
-
- return (0);
-}
-
-/*
- * Returns full path in full_path: "/pool/dataset/.zfs/snapshot/snap_name/"
- */
-static int
-zfsctl_snapshot_path_objset(zfsvfs_t *zfsvfs, uint64_t objsetid,
- int path_len, char *full_path)
-{
- objset_t *os = zfsvfs->z_os;
- fstrans_cookie_t cookie;
- char *snapname;
- boolean_t case_conflict;
- uint64_t id, pos = 0;
- int error = 0;
-
- if (zfsvfs->z_vfs->vfs_mntpoint == NULL)
- return (SET_ERROR(ENOENT));
-
- cookie = spl_fstrans_mark();
- snapname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
-
- while (error == 0) {
- dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
- error = dmu_snapshot_list_next(zfsvfs->z_os,
- ZFS_MAX_DATASET_NAME_LEN, snapname, &id, &pos,
- &case_conflict);
- dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
- if (error)
- goto out;
-
- if (id == objsetid)
- break;
- }
-
- snprintf(full_path, path_len, "%s/.zfs/snapshot/%s",
- zfsvfs->z_vfs->vfs_mntpoint, snapname);
-out:
- kmem_free(snapname, ZFS_MAX_DATASET_NAME_LEN);
- spl_fstrans_unmark(cookie);
-
- return (error);
-}
-
-/*
- * Special case the handling of "..".
- */
-int
-zfsctl_root_lookup(struct inode *dip, char *name, struct inode **ipp,
- int flags, cred_t *cr, int *direntflags, pathname_t *realpnp)
-{
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- int error = 0;
-
- ZFS_ENTER(zfsvfs);
-
- if (strcmp(name, "..") == 0) {
- *ipp = dip->i_sb->s_root->d_inode;
- } else if (strcmp(name, ZFS_SNAPDIR_NAME) == 0) {
- *ipp = zfsctl_inode_lookup(zfsvfs, ZFSCTL_INO_SNAPDIR,
- &zpl_fops_snapdir, &zpl_ops_snapdir);
- } else if (strcmp(name, ZFS_SHAREDIR_NAME) == 0) {
- *ipp = zfsctl_inode_lookup(zfsvfs, ZFSCTL_INO_SHARES,
- &zpl_fops_shares, &zpl_ops_shares);
- } else {
- *ipp = NULL;
- }
-
- if (*ipp == NULL)
- error = SET_ERROR(ENOENT);
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Lookup entry point for the 'snapshot' directory. Try to open the
- * snapshot if it exist, creating the pseudo filesystem inode as necessary.
- */
-int
-zfsctl_snapdir_lookup(struct inode *dip, char *name, struct inode **ipp,
- int flags, cred_t *cr, int *direntflags, pathname_t *realpnp)
-{
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- uint64_t id;
- int error;
-
- ZFS_ENTER(zfsvfs);
-
- error = dmu_snapshot_lookup(zfsvfs->z_os, name, &id);
- if (error) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- *ipp = zfsctl_inode_lookup(zfsvfs, ZFSCTL_INO_SNAPDIRS - id,
- &simple_dir_operations, &simple_dir_inode_operations);
- if (*ipp == NULL)
- error = SET_ERROR(ENOENT);
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Renaming a directory under '.zfs/snapshot' will automatically trigger
- * a rename of the snapshot to the new given name. The rename is confined
- * to the '.zfs/snapshot' directory snapshots cannot be moved elsewhere.
- */
-int
-zfsctl_snapdir_rename(struct inode *sdip, char *snm,
- struct inode *tdip, char *tnm, cred_t *cr, int flags)
-{
- zfsvfs_t *zfsvfs = ITOZSB(sdip);
- char *to, *from, *real, *fsname;
- int error;
-
- if (!zfs_admin_snapshot)
- return (SET_ERROR(EACCES));
-
- ZFS_ENTER(zfsvfs);
-
- to = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
- from = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
- real = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
- fsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
-
- if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE) {
- error = dmu_snapshot_realname(zfsvfs->z_os, snm, real,
- ZFS_MAX_DATASET_NAME_LEN, NULL);
- if (error == 0) {
- snm = real;
- } else if (error != ENOTSUP) {
- goto out;
- }
- }
-
- dmu_objset_name(zfsvfs->z_os, fsname);
-
- error = zfsctl_snapshot_name(ITOZSB(sdip), snm,
- ZFS_MAX_DATASET_NAME_LEN, from);
- if (error == 0)
- error = zfsctl_snapshot_name(ITOZSB(tdip), tnm,
- ZFS_MAX_DATASET_NAME_LEN, to);
- if (error == 0)
- error = zfs_secpolicy_rename_perms(from, to, cr);
- if (error != 0)
- goto out;
-
- /*
- * Cannot move snapshots out of the snapdir.
- */
- if (sdip != tdip) {
- error = SET_ERROR(EINVAL);
- goto out;
- }
-
- /*
- * No-op when names are identical.
- */
- if (strcmp(snm, tnm) == 0) {
- error = 0;
- goto out;
- }
-
- rw_enter(&zfs_snapshot_lock, RW_WRITER);
-
- error = dsl_dataset_rename_snapshot(fsname, snm, tnm, B_FALSE);
- if (error == 0)
- (void) zfsctl_snapshot_rename(snm, tnm);
-
- rw_exit(&zfs_snapshot_lock);
-out:
- kmem_free(from, ZFS_MAX_DATASET_NAME_LEN);
- kmem_free(to, ZFS_MAX_DATASET_NAME_LEN);
- kmem_free(real, ZFS_MAX_DATASET_NAME_LEN);
- kmem_free(fsname, ZFS_MAX_DATASET_NAME_LEN);
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Removing a directory under '.zfs/snapshot' will automatically trigger
- * the removal of the snapshot with the given name.
- */
-int
-zfsctl_snapdir_remove(struct inode *dip, char *name, cred_t *cr, int flags)
-{
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- char *snapname, *real;
- int error;
-
- if (!zfs_admin_snapshot)
- return (SET_ERROR(EACCES));
-
- ZFS_ENTER(zfsvfs);
-
- snapname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
- real = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
-
- if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE) {
- error = dmu_snapshot_realname(zfsvfs->z_os, name, real,
- ZFS_MAX_DATASET_NAME_LEN, NULL);
- if (error == 0) {
- name = real;
- } else if (error != ENOTSUP) {
- goto out;
- }
- }
-
- error = zfsctl_snapshot_name(ITOZSB(dip), name,
- ZFS_MAX_DATASET_NAME_LEN, snapname);
- if (error == 0)
- error = zfs_secpolicy_destroy_perms(snapname, cr);
- if (error != 0)
- goto out;
-
- error = zfsctl_snapshot_unmount(snapname, MNT_FORCE);
- if ((error == 0) || (error == ENOENT))
- error = dsl_destroy_snapshot(snapname, B_FALSE);
-out:
- kmem_free(snapname, ZFS_MAX_DATASET_NAME_LEN);
- kmem_free(real, ZFS_MAX_DATASET_NAME_LEN);
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Creating a directory under '.zfs/snapshot' will automatically trigger
- * the creation of a new snapshot with the given name.
- */
-int
-zfsctl_snapdir_mkdir(struct inode *dip, char *dirname, vattr_t *vap,
- struct inode **ipp, cred_t *cr, int flags)
-{
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- char *dsname;
- int error;
-
- if (!zfs_admin_snapshot)
- return (SET_ERROR(EACCES));
-
- dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
-
- if (zfs_component_namecheck(dirname, NULL, NULL) != 0) {
- error = SET_ERROR(EILSEQ);
- goto out;
- }
-
- dmu_objset_name(zfsvfs->z_os, dsname);
-
- error = zfs_secpolicy_snapshot_perms(dsname, cr);
- if (error != 0)
- goto out;
-
- if (error == 0) {
- error = dmu_objset_snapshot_one(dsname, dirname);
- if (error != 0)
- goto out;
-
- error = zfsctl_snapdir_lookup(dip, dirname, ipp,
- 0, cr, NULL, NULL);
- }
-out:
- kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN);
-
- return (error);
-}
-
-/*
- * Attempt to unmount a snapshot by making a call to user space.
- * There is no assurance that this can or will succeed, is just a
- * best effort. In the case where it does fail, perhaps because
- * it's in use, the unmount will fail harmlessly.
- */
-int
-zfsctl_snapshot_unmount(char *snapname, int flags)
-{
- char *argv[] = { "/usr/bin/env", "umount", "-t", "zfs", "-n", NULL,
- NULL };
- char *envp[] = { NULL };
- zfs_snapentry_t *se;
- int error;
-
- rw_enter(&zfs_snapshot_lock, RW_READER);
- if ((se = zfsctl_snapshot_find_by_name(snapname)) == NULL) {
- rw_exit(&zfs_snapshot_lock);
- return (SET_ERROR(ENOENT));
- }
- rw_exit(&zfs_snapshot_lock);
-
- if (flags & MNT_FORCE)
- argv[4] = "-fn";
- argv[5] = se->se_path;
- dprintf("unmount; path=%s\n", se->se_path);
- error = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
- zfsctl_snapshot_rele(se);
-
-
- /*
- * The umount system utility will return 256 on error. We must
- * assume this error is because the file system is busy so it is
- * converted to the more sensible EBUSY.
- */
- if (error)
- error = SET_ERROR(EBUSY);
-
- return (error);
-}
-
-int
-zfsctl_snapshot_mount(struct path *path, int flags)
-{
- struct dentry *dentry = path->dentry;
- struct inode *ip = dentry->d_inode;
- zfsvfs_t *zfsvfs;
- zfsvfs_t *snap_zfsvfs;
- zfs_snapentry_t *se;
- char *full_name, *full_path;
- char *argv[] = { "/usr/bin/env", "mount", "-t", "zfs", "-n", NULL, NULL,
- NULL };
- char *envp[] = { NULL };
- int error;
- struct path spath;
-
- if (ip == NULL)
- return (SET_ERROR(EISDIR));
-
- zfsvfs = ITOZSB(ip);
- ZFS_ENTER(zfsvfs);
-
- full_name = kmem_zalloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
- full_path = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
-
- error = zfsctl_snapshot_name(zfsvfs, dname(dentry),
- ZFS_MAX_DATASET_NAME_LEN, full_name);
- if (error)
- goto error;
-
- /*
- * Construct a mount point path from sb of the ctldir inode and dirent
- * name, instead of from d_path(), so that chroot'd process doesn't fail
- * on mount.zfs(8).
- */
- snprintf(full_path, MAXPATHLEN, "%s/.zfs/snapshot/%s",
- zfsvfs->z_vfs->vfs_mntpoint, dname(dentry));
-
- /*
- * Multiple concurrent automounts of a snapshot are never allowed.
- * The snapshot may be manually mounted as many times as desired.
- */
- if (zfsctl_snapshot_ismounted(full_name)) {
- error = 0;
- goto error;
- }
-
- /*
- * Attempt to mount the snapshot from user space. Normally this
- * would be done using the vfs_kern_mount() function, however that
- * function is marked GPL-only and cannot be used. On error we
- * careful to log the real error to the console and return EISDIR
- * to safely abort the automount. This should be very rare.
- *
- * If the user mode helper happens to return EBUSY, a concurrent
- * mount is already in progress in which case the error is ignored.
- * Take note that if the program was executed successfully the return
- * value from call_usermodehelper() will be (exitcode << 8 + signal).
- */
- dprintf("mount; name=%s path=%s\n", full_name, full_path);
- argv[5] = full_name;
- argv[6] = full_path;
- error = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
- if (error) {
- if (!(error & MOUNT_BUSY << 8)) {
- zfs_dbgmsg("Unable to automount %s error=%d",
- full_path, error);
- error = SET_ERROR(EISDIR);
- } else {
- /*
- * EBUSY, this could mean a concurrent mount, or the
- * snapshot has already been mounted at completely
- * different place. We return 0 so VFS will retry. For
- * the latter case the VFS will retry several times
- * and return ELOOP, which is probably not a very good
- * behavior.
- */
- error = 0;
- }
- goto error;
- }
-
- /*
- * Follow down in to the mounted snapshot and set MNT_SHRINKABLE
- * to identify this as an automounted filesystem.
- */
- spath = *path;
- path_get(&spath);
- if (zpl_follow_down_one(&spath)) {
- snap_zfsvfs = ITOZSB(spath.dentry->d_inode);
- snap_zfsvfs->z_parent = zfsvfs;
- dentry = spath.dentry;
- spath.mnt->mnt_flags |= MNT_SHRINKABLE;
-
- rw_enter(&zfs_snapshot_lock, RW_WRITER);
- se = zfsctl_snapshot_alloc(full_name, full_path,
- snap_zfsvfs->z_os->os_spa, dmu_objset_id(snap_zfsvfs->z_os),
- dentry);
- zfsctl_snapshot_add(se);
- zfsctl_snapshot_unmount_delay_impl(se, zfs_expire_snapshot);
- rw_exit(&zfs_snapshot_lock);
- }
- path_put(&spath);
-error:
- kmem_free(full_name, ZFS_MAX_DATASET_NAME_LEN);
- kmem_free(full_path, MAXPATHLEN);
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Get the snapdir inode from fid
- */
-int
-zfsctl_snapdir_vget(struct super_block *sb, uint64_t objsetid, int gen,
- struct inode **ipp)
-{
- int error;
- struct path path;
- char *mnt;
- struct dentry *dentry;
-
- mnt = kmem_alloc(MAXPATHLEN, KM_SLEEP);
-
- error = zfsctl_snapshot_path_objset(sb->s_fs_info, objsetid,
- MAXPATHLEN, mnt);
- if (error)
- goto out;
-
- /* Trigger automount */
- error = -kern_path(mnt, LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &path);
- if (error)
- goto out;
-
- path_put(&path);
- /*
- * Get the snapdir inode. Note, we don't want to use the above
- * path because it contains the root of the snapshot rather
- * than the snapdir.
- */
- *ipp = ilookup(sb, ZFSCTL_INO_SNAPDIRS - objsetid);
- if (*ipp == NULL) {
- error = SET_ERROR(ENOENT);
- goto out;
- }
-
- /* check gen, see zfsctl_snapdir_fid */
- dentry = d_obtain_alias(igrab(*ipp));
- if (gen != (!IS_ERR(dentry) && d_mountpoint(dentry))) {
- iput(*ipp);
- *ipp = NULL;
- error = SET_ERROR(ENOENT);
- }
- if (!IS_ERR(dentry))
- dput(dentry);
-out:
- kmem_free(mnt, MAXPATHLEN);
- return (error);
-}
-
-int
-zfsctl_shares_lookup(struct inode *dip, char *name, struct inode **ipp,
- int flags, cred_t *cr, int *direntflags, pathname_t *realpnp)
-{
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- struct inode *ip;
- znode_t *dzp;
- int error;
-
- ZFS_ENTER(zfsvfs);
-
- if (zfsvfs->z_shares_dir == 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOTSUP));
- }
-
- if ((error = zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp)) == 0) {
- error = zfs_lookup(ZTOI(dzp), name, &ip, 0, cr, NULL, NULL);
- iput(ZTOI(dzp));
- }
-
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*
- * Initialize the various pieces we'll need to create and manipulate .zfs
- * directories. Currently this is unused but available.
- */
-void
-zfsctl_init(void)
-{
- avl_create(&zfs_snapshots_by_name, snapentry_compare_by_name,
- sizeof (zfs_snapentry_t), offsetof(zfs_snapentry_t,
- se_node_name));
- avl_create(&zfs_snapshots_by_objsetid, snapentry_compare_by_objsetid,
- sizeof (zfs_snapentry_t), offsetof(zfs_snapentry_t,
- se_node_objsetid));
- rw_init(&zfs_snapshot_lock, NULL, RW_DEFAULT, NULL);
-}
-
-/*
- * Cleanup the various pieces we needed for .zfs directories. In particular
- * ensure the expiry timer is canceled safely.
- */
-void
-zfsctl_fini(void)
-{
- avl_destroy(&zfs_snapshots_by_name);
- avl_destroy(&zfs_snapshots_by_objsetid);
- rw_destroy(&zfs_snapshot_lock);
-}
-
-module_param(zfs_admin_snapshot, int, 0644);
-MODULE_PARM_DESC(zfs_admin_snapshot, "Enable mkdir/rmdir/mv in .zfs/snapshot");
-
-module_param(zfs_expire_snapshot, int, 0644);
-MODULE_PARM_DESC(zfs_expire_snapshot, "Seconds to expire .zfs/snapshot");
diff --git a/module/zfs/zfs_debug.c b/module/zfs/zfs_debug.c
deleted file mode 100644
index 538533d27..000000000
--- a/module/zfs/zfs_debug.c
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
- */
-
-#include <sys/zfs_context.h>
-
-typedef struct zfs_dbgmsg {
- procfs_list_node_t zdm_node;
- time_t zdm_timestamp;
- int zdm_size;
- char zdm_msg[1]; /* variable length allocation */
-} zfs_dbgmsg_t;
-
-procfs_list_t zfs_dbgmsgs;
-int zfs_dbgmsg_size = 0;
-int zfs_dbgmsg_maxsize = 4<<20; /* 4MB */
-
-/*
- * Internal ZFS debug messages are enabled by default.
- *
- * # Print debug messages
- * cat /proc/spl/kstat/zfs/dbgmsg
- *
- * # Disable the kernel debug message log.
- * echo 0 > /sys/module/zfs/parameters/zfs_dbgmsg_enable
- *
- * # Clear the kernel debug message log.
- * echo 0 >/proc/spl/kstat/zfs/dbgmsg
- */
-int zfs_dbgmsg_enable = 1;
-
-static int
-zfs_dbgmsg_show_header(struct seq_file *f)
-{
- seq_printf(f, "%-12s %-8s\n", "timestamp", "message");
- return (0);
-}
-
-static int
-zfs_dbgmsg_show(struct seq_file *f, void *p)
-{
- zfs_dbgmsg_t *zdm = (zfs_dbgmsg_t *)p;
- seq_printf(f, "%-12llu %-s\n",
- (u_longlong_t)zdm->zdm_timestamp, zdm->zdm_msg);
- return (0);
-}
-
-static void
-zfs_dbgmsg_purge(int max_size)
-{
- while (zfs_dbgmsg_size > max_size) {
- zfs_dbgmsg_t *zdm = list_remove_head(&zfs_dbgmsgs.pl_list);
- if (zdm == NULL)
- return;
-
- int size = zdm->zdm_size;
- kmem_free(zdm, size);
- zfs_dbgmsg_size -= size;
- }
-}
-
-static int
-zfs_dbgmsg_clear(procfs_list_t *procfs_list)
-{
- mutex_enter(&zfs_dbgmsgs.pl_lock);
- zfs_dbgmsg_purge(0);
- mutex_exit(&zfs_dbgmsgs.pl_lock);
- return (0);
-}
-
-void
-zfs_dbgmsg_init(void)
-{
- procfs_list_install("zfs",
- "dbgmsg",
- 0600,
- &zfs_dbgmsgs,
- zfs_dbgmsg_show,
- zfs_dbgmsg_show_header,
- zfs_dbgmsg_clear,
- offsetof(zfs_dbgmsg_t, zdm_node));
-}
-
-void
-zfs_dbgmsg_fini(void)
-{
- procfs_list_uninstall(&zfs_dbgmsgs);
- zfs_dbgmsg_purge(0);
-
- /*
- * TODO - decide how to make this permanent
- */
-#ifdef _KERNEL
- procfs_list_destroy(&zfs_dbgmsgs);
-#endif
-}
-
-void
-__set_error(const char *file, const char *func, int line, int err)
-{
- /*
- * To enable this:
- *
- * $ echo 512 >/sys/module/zfs/parameters/zfs_flags
- */
- if (zfs_flags & ZFS_DEBUG_SET_ERROR)
- __dprintf(B_FALSE, file, func, line, "error %lu", err);
-}
-
-void
-__zfs_dbgmsg(char *buf)
-{
- int size = sizeof (zfs_dbgmsg_t) + strlen(buf);
- zfs_dbgmsg_t *zdm = kmem_zalloc(size, KM_SLEEP);
- zdm->zdm_size = size;
- zdm->zdm_timestamp = gethrestime_sec();
- strcpy(zdm->zdm_msg, buf);
-
- mutex_enter(&zfs_dbgmsgs.pl_lock);
- procfs_list_add(&zfs_dbgmsgs, zdm);
- zfs_dbgmsg_size += size;
- zfs_dbgmsg_purge(MAX(zfs_dbgmsg_maxsize, 0));
- mutex_exit(&zfs_dbgmsgs.pl_lock);
-}
-
-#ifdef _KERNEL
-
-void
-__dprintf(boolean_t dprint, const char *file, const char *func,
- int line, const char *fmt, ...)
-{
- const char *newfile;
- va_list adx;
- size_t size;
- char *buf;
- char *nl;
- int i;
- char *prefix = (dprint) ? "dprintf: " : "";
-
- size = 1024;
- buf = kmem_alloc(size, KM_SLEEP);
-
- /*
- * Get rid of annoying prefix to filename.
- */
- newfile = strrchr(file, '/');
- if (newfile != NULL) {
- newfile = newfile + 1; /* Get rid of leading / */
- } else {
- newfile = file;
- }
-
- i = snprintf(buf, size, "%s%s:%d:%s(): ", prefix, newfile, line, func);
-
- if (i < size) {
- va_start(adx, fmt);
- (void) vsnprintf(buf + i, size - i, fmt, adx);
- va_end(adx);
- }
-
- /*
- * Get rid of trailing newline for dprintf logs.
- */
- if (dprint && buf[0] != '\0') {
- nl = &buf[strlen(buf) - 1];
- if (*nl == '\n')
- *nl = '\0';
- }
-
- /*
- * To get this data enable the zfs__dprintf trace point as shown:
- *
- * # Enable zfs__dprintf tracepoint, clear the tracepoint ring buffer
- * $ echo 1 > /sys/kernel/debug/tracing/events/zfs/enable
- * $ echo 0 > /sys/kernel/debug/tracing/trace
- *
- * # Dump the ring buffer.
- * $ cat /sys/kernel/debug/tracing/trace
- */
- DTRACE_PROBE1(zfs__dprintf, char *, buf);
-
- /*
- * To get this data:
- *
- * $ cat /proc/spl/kstat/zfs/dbgmsg
- *
- * To clear the buffer:
- * $ echo 0 > /proc/spl/kstat/zfs/dbgmsg
- */
- __zfs_dbgmsg(buf);
-
- kmem_free(buf, size);
-}
-
-#else
-
-void
-zfs_dbgmsg_print(const char *tag)
-{
- ssize_t ret __attribute__((unused));
-
- /*
- * We use write() in this function instead of printf()
- * so it is safe to call from a signal handler.
- */
- ret = write(STDOUT_FILENO, "ZFS_DBGMSG(", 11);
- ret = write(STDOUT_FILENO, tag, strlen(tag));
- ret = write(STDOUT_FILENO, ") START:\n", 9);
-
- mutex_enter(&zfs_dbgmsgs.pl_lock);
- for (zfs_dbgmsg_t *zdm = list_head(&zfs_dbgmsgs.pl_list); zdm != NULL;
- zdm = list_next(&zfs_dbgmsgs.pl_list, zdm)) {
- ret = write(STDOUT_FILENO, zdm->zdm_msg,
- strlen(zdm->zdm_msg));
- ret = write(STDOUT_FILENO, "\n", 1);
- }
-
- ret = write(STDOUT_FILENO, "ZFS_DBGMSG(", 11);
- ret = write(STDOUT_FILENO, tag, strlen(tag));
- ret = write(STDOUT_FILENO, ") END\n", 6);
-
- mutex_exit(&zfs_dbgmsgs.pl_lock);
-}
-#endif /* _KERNEL */
-
-#ifdef _KERNEL
-module_param(zfs_dbgmsg_enable, int, 0644);
-MODULE_PARM_DESC(zfs_dbgmsg_enable, "Enable ZFS debug message log");
-
-module_param(zfs_dbgmsg_maxsize, int, 0644);
-MODULE_PARM_DESC(zfs_dbgmsg_maxsize, "Maximum ZFS debug log size");
-#endif
diff --git a/module/zfs/zfs_dir.c b/module/zfs/zfs_dir.c
deleted file mode 100644
index 6bdad737c..000000000
--- a/module/zfs/zfs_dir.c
+++ /dev/null
@@ -1,1205 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, 2016 by Delphix. All rights reserved.
- * Copyright 2017 Nexenta Systems, Inc.
- */
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysmacros.h>
-#include <sys/vfs.h>
-#include <sys/vnode.h>
-#include <sys/file.h>
-#include <sys/mode.h>
-#include <sys/kmem.h>
-#include <sys/uio.h>
-#include <sys/pathname.h>
-#include <sys/cmn_err.h>
-#include <sys/errno.h>
-#include <sys/stat.h>
-#include <sys/sunddi.h>
-#include <sys/random.h>
-#include <sys/policy.h>
-#include <sys/zfs_dir.h>
-#include <sys/zfs_acl.h>
-#include <sys/zfs_vnops.h>
-#include <sys/fs/zfs.h>
-#include <sys/zap.h>
-#include <sys/dmu.h>
-#include <sys/atomic.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zfs_fuid.h>
-#include <sys/sa.h>
-#include <sys/zfs_sa.h>
-
-/*
- * zfs_match_find() is used by zfs_dirent_lock() to perform zap lookups
- * of names after deciding which is the appropriate lookup interface.
- */
-static int
-zfs_match_find(zfsvfs_t *zfsvfs, znode_t *dzp, char *name, matchtype_t mt,
- boolean_t update, int *deflags, pathname_t *rpnp, uint64_t *zoid)
-{
- boolean_t conflict = B_FALSE;
- int error;
-
- if (zfsvfs->z_norm) {
- size_t bufsz = 0;
- char *buf = NULL;
-
- if (rpnp) {
- buf = rpnp->pn_buf;
- bufsz = rpnp->pn_bufsize;
- }
-
- /*
- * In the non-mixed case we only expect there would ever
- * be one match, but we need to use the normalizing lookup.
- */
- error = zap_lookup_norm(zfsvfs->z_os, dzp->z_id, name, 8, 1,
- zoid, mt, buf, bufsz, &conflict);
- } else {
- error = zap_lookup(zfsvfs->z_os, dzp->z_id, name, 8, 1, zoid);
- }
-
- /*
- * Allow multiple entries provided the first entry is
- * the object id. Non-zpl consumers may safely make
- * use of the additional space.
- *
- * XXX: This should be a feature flag for compatibility
- */
- if (error == EOVERFLOW)
- error = 0;
-
- if (zfsvfs->z_norm && !error && deflags)
- *deflags = conflict ? ED_CASE_CONFLICT : 0;
-
- *zoid = ZFS_DIRENT_OBJ(*zoid);
-
- return (error);
-}
-
-/*
- * Lock a directory entry. A dirlock on <dzp, name> protects that name
- * in dzp's directory zap object. As long as you hold a dirlock, you can
- * assume two things: (1) dzp cannot be reaped, and (2) no other thread
- * can change the zap entry for (i.e. link or unlink) this name.
- *
- * Input arguments:
- * dzp - znode for directory
- * name - name of entry to lock
- * flag - ZNEW: if the entry already exists, fail with EEXIST.
- * ZEXISTS: if the entry does not exist, fail with ENOENT.
- * ZSHARED: allow concurrent access with other ZSHARED callers.
- * ZXATTR: we want dzp's xattr directory
- * ZCILOOK: On a mixed sensitivity file system,
- * this lookup should be case-insensitive.
- * ZCIEXACT: On a purely case-insensitive file system,
- * this lookup should be case-sensitive.
- * ZRENAMING: we are locking for renaming, force narrow locks
- * ZHAVELOCK: Don't grab the z_name_lock for this call. The
- * current thread already holds it.
- *
- * Output arguments:
- * zpp - pointer to the znode for the entry (NULL if there isn't one)
- * dlpp - pointer to the dirlock for this entry (NULL on error)
- * direntflags - (case-insensitive lookup only)
- * flags if multiple case-sensitive matches exist in directory
- * realpnp - (case-insensitive lookup only)
- * actual name matched within the directory
- *
- * Return value: 0 on success or errno on failure.
- *
- * NOTE: Always checks for, and rejects, '.' and '..'.
- * NOTE: For case-insensitive file systems we take wide locks (see below),
- * but return znode pointers to a single match.
- */
-int
-zfs_dirent_lock(zfs_dirlock_t **dlpp, znode_t *dzp, char *name, znode_t **zpp,
- int flag, int *direntflags, pathname_t *realpnp)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- zfs_dirlock_t *dl;
- boolean_t update;
- matchtype_t mt = 0;
- uint64_t zoid;
- int error = 0;
- int cmpflags;
-
- *zpp = NULL;
- *dlpp = NULL;
-
- /*
- * Verify that we are not trying to lock '.', '..', or '.zfs'
- */
- if ((name[0] == '.' &&
- (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) ||
- (zfs_has_ctldir(dzp) && strcmp(name, ZFS_CTLDIR_NAME) == 0))
- return (SET_ERROR(EEXIST));
-
- /*
- * Case sensitivity and normalization preferences are set when
- * the file system is created. These are stored in the
- * zfsvfs->z_case and zfsvfs->z_norm fields. These choices
- * affect what vnodes can be cached in the DNLC, how we
- * perform zap lookups, and the "width" of our dirlocks.
- *
- * A normal dirlock locks a single name. Note that with
- * normalization a name can be composed multiple ways, but
- * when normalized, these names all compare equal. A wide
- * dirlock locks multiple names. We need these when the file
- * system is supporting mixed-mode access. It is sometimes
- * necessary to lock all case permutations of file name at
- * once so that simultaneous case-insensitive/case-sensitive
- * behaves as rationally as possible.
- */
-
- /*
- * When matching we may need to normalize & change case according to
- * FS settings.
- *
- * Note that a normalized match is necessary for a case insensitive
- * filesystem when the lookup request is not exact because normalization
- * can fold case independent of normalizing code point sequences.
- *
- * See the table above zfs_dropname().
- */
- if (zfsvfs->z_norm != 0) {
- mt = MT_NORMALIZE;
-
- /*
- * Determine if the match needs to honor the case specified in
- * lookup, and if so keep track of that so that during
- * normalization we don't fold case.
- */
- if ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE &&
- (flag & ZCIEXACT)) ||
- (zfsvfs->z_case == ZFS_CASE_MIXED && !(flag & ZCILOOK))) {
- mt |= MT_MATCH_CASE;
- }
- }
-
- /*
- * Only look in or update the DNLC if we are looking for the
- * name on a file system that does not require normalization
- * or case folding. We can also look there if we happen to be
- * on a non-normalizing, mixed sensitivity file system IF we
- * are looking for the exact name.
- *
- * Maybe can add TO-UPPERed version of name to dnlc in ci-only
- * case for performance improvement?
- */
- update = !zfsvfs->z_norm ||
- (zfsvfs->z_case == ZFS_CASE_MIXED &&
- !(zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER) && !(flag & ZCILOOK));
-
- /*
- * ZRENAMING indicates we are in a situation where we should
- * take narrow locks regardless of the file system's
- * preferences for normalizing and case folding. This will
- * prevent us deadlocking trying to grab the same wide lock
- * twice if the two names happen to be case-insensitive
- * matches.
- */
- if (flag & ZRENAMING)
- cmpflags = 0;
- else
- cmpflags = zfsvfs->z_norm;
-
- /*
- * Wait until there are no locks on this name.
- *
- * Don't grab the lock if it is already held. However, cannot
- * have both ZSHARED and ZHAVELOCK together.
- */
- ASSERT(!(flag & ZSHARED) || !(flag & ZHAVELOCK));
- if (!(flag & ZHAVELOCK))
- rw_enter(&dzp->z_name_lock, RW_READER);
-
- mutex_enter(&dzp->z_lock);
- for (;;) {
- if (dzp->z_unlinked && !(flag & ZXATTR)) {
- mutex_exit(&dzp->z_lock);
- if (!(flag & ZHAVELOCK))
- rw_exit(&dzp->z_name_lock);
- return (SET_ERROR(ENOENT));
- }
- for (dl = dzp->z_dirlocks; dl != NULL; dl = dl->dl_next) {
- if ((u8_strcmp(name, dl->dl_name, 0, cmpflags,
- U8_UNICODE_LATEST, &error) == 0) || error != 0)
- break;
- }
- if (error != 0) {
- mutex_exit(&dzp->z_lock);
- if (!(flag & ZHAVELOCK))
- rw_exit(&dzp->z_name_lock);
- return (SET_ERROR(ENOENT));
- }
- if (dl == NULL) {
- /*
- * Allocate a new dirlock and add it to the list.
- */
- dl = kmem_alloc(sizeof (zfs_dirlock_t), KM_SLEEP);
- cv_init(&dl->dl_cv, NULL, CV_DEFAULT, NULL);
- dl->dl_name = name;
- dl->dl_sharecnt = 0;
- dl->dl_namelock = 0;
- dl->dl_namesize = 0;
- dl->dl_dzp = dzp;
- dl->dl_next = dzp->z_dirlocks;
- dzp->z_dirlocks = dl;
- break;
- }
- if ((flag & ZSHARED) && dl->dl_sharecnt != 0)
- break;
- cv_wait(&dl->dl_cv, &dzp->z_lock);
- }
-
- /*
- * If the z_name_lock was NOT held for this dirlock record it.
- */
- if (flag & ZHAVELOCK)
- dl->dl_namelock = 1;
-
- if ((flag & ZSHARED) && ++dl->dl_sharecnt > 1 && dl->dl_namesize == 0) {
- /*
- * We're the second shared reference to dl. Make a copy of
- * dl_name in case the first thread goes away before we do.
- * Note that we initialize the new name before storing its
- * pointer into dl_name, because the first thread may load
- * dl->dl_name at any time. It'll either see the old value,
- * which belongs to it, or the new shared copy; either is OK.
- */
- dl->dl_namesize = strlen(dl->dl_name) + 1;
- name = kmem_alloc(dl->dl_namesize, KM_SLEEP);
- bcopy(dl->dl_name, name, dl->dl_namesize);
- dl->dl_name = name;
- }
-
- mutex_exit(&dzp->z_lock);
-
- /*
- * We have a dirlock on the name. (Note that it is the dirlock,
- * not the dzp's z_lock, that protects the name in the zap object.)
- * See if there's an object by this name; if so, put a hold on it.
- */
- if (flag & ZXATTR) {
- error = sa_lookup(dzp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &zoid,
- sizeof (zoid));
- if (error == 0)
- error = (zoid == 0 ? SET_ERROR(ENOENT) : 0);
- } else {
- error = zfs_match_find(zfsvfs, dzp, name, mt,
- update, direntflags, realpnp, &zoid);
- }
- if (error) {
- if (error != ENOENT || (flag & ZEXISTS)) {
- zfs_dirent_unlock(dl);
- return (error);
- }
- } else {
- if (flag & ZNEW) {
- zfs_dirent_unlock(dl);
- return (SET_ERROR(EEXIST));
- }
- error = zfs_zget(zfsvfs, zoid, zpp);
- if (error) {
- zfs_dirent_unlock(dl);
- return (error);
- }
- }
-
- *dlpp = dl;
-
- return (0);
-}
-
-/*
- * Unlock this directory entry and wake anyone who was waiting for it.
- */
-void
-zfs_dirent_unlock(zfs_dirlock_t *dl)
-{
- znode_t *dzp = dl->dl_dzp;
- zfs_dirlock_t **prev_dl, *cur_dl;
-
- mutex_enter(&dzp->z_lock);
-
- if (!dl->dl_namelock)
- rw_exit(&dzp->z_name_lock);
-
- if (dl->dl_sharecnt > 1) {
- dl->dl_sharecnt--;
- mutex_exit(&dzp->z_lock);
- return;
- }
- prev_dl = &dzp->z_dirlocks;
- while ((cur_dl = *prev_dl) != dl)
- prev_dl = &cur_dl->dl_next;
- *prev_dl = dl->dl_next;
- cv_broadcast(&dl->dl_cv);
- mutex_exit(&dzp->z_lock);
-
- if (dl->dl_namesize != 0)
- kmem_free(dl->dl_name, dl->dl_namesize);
- cv_destroy(&dl->dl_cv);
- kmem_free(dl, sizeof (*dl));
-}
-
-/*
- * Look up an entry in a directory.
- *
- * NOTE: '.' and '..' are handled as special cases because
- * no directory entries are actually stored for them. If this is
- * the root of a filesystem, then '.zfs' is also treated as a
- * special pseudo-directory.
- */
-int
-zfs_dirlook(znode_t *dzp, char *name, struct inode **ipp, int flags,
- int *deflg, pathname_t *rpnp)
-{
- zfs_dirlock_t *dl;
- znode_t *zp;
- int error = 0;
- uint64_t parent;
-
- if (name[0] == 0 || (name[0] == '.' && name[1] == 0)) {
- *ipp = ZTOI(dzp);
- igrab(*ipp);
- } else if (name[0] == '.' && name[1] == '.' && name[2] == 0) {
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
-
- /*
- * If we are a snapshot mounted under .zfs, return
- * the inode pointer for the snapshot directory.
- */
- if ((error = sa_lookup(dzp->z_sa_hdl,
- SA_ZPL_PARENT(zfsvfs), &parent, sizeof (parent))) != 0)
- return (error);
-
- if (parent == dzp->z_id && zfsvfs->z_parent != zfsvfs) {
- error = zfsctl_root_lookup(zfsvfs->z_parent->z_ctldir,
- "snapshot", ipp, 0, kcred, NULL, NULL);
- return (error);
- }
- rw_enter(&dzp->z_parent_lock, RW_READER);
- error = zfs_zget(zfsvfs, parent, &zp);
- if (error == 0)
- *ipp = ZTOI(zp);
- rw_exit(&dzp->z_parent_lock);
- } else if (zfs_has_ctldir(dzp) && strcmp(name, ZFS_CTLDIR_NAME) == 0) {
- *ipp = zfsctl_root(dzp);
- } else {
- int zf;
-
- zf = ZEXISTS | ZSHARED;
- if (flags & FIGNORECASE)
- zf |= ZCILOOK;
-
- error = zfs_dirent_lock(&dl, dzp, name, &zp, zf, deflg, rpnp);
- if (error == 0) {
- *ipp = ZTOI(zp);
- zfs_dirent_unlock(dl);
- dzp->z_zn_prefetch = B_TRUE; /* enable prefetching */
- }
- rpnp = NULL;
- }
-
- if ((flags & FIGNORECASE) && rpnp && !error)
- (void) strlcpy(rpnp->pn_buf, name, rpnp->pn_bufsize);
-
- return (error);
-}
-
-/*
- * unlinked Set (formerly known as the "delete queue") Error Handling
- *
- * When dealing with the unlinked set, we dmu_tx_hold_zap(), but we
- * don't specify the name of the entry that we will be manipulating. We
- * also fib and say that we won't be adding any new entries to the
- * unlinked set, even though we might (this is to lower the minimum file
- * size that can be deleted in a full filesystem). So on the small
- * chance that the nlink list is using a fat zap (ie. has more than
- * 2000 entries), we *may* not pre-read a block that's needed.
- * Therefore it is remotely possible for some of the assertions
- * regarding the unlinked set below to fail due to i/o error. On a
- * nondebug system, this will result in the space being leaked.
- */
-void
-zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
-
- ASSERT(zp->z_unlinked);
- ASSERT(ZTOI(zp)->i_nlink == 0);
-
- VERIFY3U(0, ==,
- zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
-
- dataset_kstats_update_nunlinks_kstat(&zfsvfs->z_kstat, 1);
-}
-
-/*
- * Clean up any znodes that had no links when we either crashed or
- * (force) umounted the file system.
- */
-static void
-zfs_unlinked_drain_task(void *arg)
-{
- zfsvfs_t *zfsvfs = arg;
- zap_cursor_t zc;
- zap_attribute_t zap;
- dmu_object_info_t doi;
- znode_t *zp;
- int error;
-
- ASSERT3B(zfsvfs->z_draining, ==, B_TRUE);
-
- /*
- * Iterate over the contents of the unlinked set.
- */
- for (zap_cursor_init(&zc, zfsvfs->z_os, zfsvfs->z_unlinkedobj);
- zap_cursor_retrieve(&zc, &zap) == 0 && !zfsvfs->z_drain_cancel;
- zap_cursor_advance(&zc)) {
-
- /*
- * See what kind of object we have in list
- */
-
- error = dmu_object_info(zfsvfs->z_os,
- zap.za_first_integer, &doi);
- if (error != 0)
- continue;
-
- ASSERT((doi.doi_type == DMU_OT_PLAIN_FILE_CONTENTS) ||
- (doi.doi_type == DMU_OT_DIRECTORY_CONTENTS));
- /*
- * We need to re-mark these list entries for deletion,
- * so we pull them back into core and set zp->z_unlinked.
- */
- error = zfs_zget(zfsvfs, zap.za_first_integer, &zp);
-
- /*
- * We may pick up znodes that are already marked for deletion.
- * This could happen during the purge of an extended attribute
- * directory. All we need to do is skip over them, since they
- * are already in the system marked z_unlinked.
- */
- if (error != 0)
- continue;
-
- zp->z_unlinked = B_TRUE;
-
- /*
- * iput() is Linux's equivalent to illumos' VN_RELE(). It will
- * decrement the inode's ref count and may cause the inode to be
- * synchronously freed. We interrupt freeing of this inode, by
- * checking the return value of dmu_objset_zfs_unmounting() in
- * dmu_free_long_range(), when an unmount is requested.
- */
- iput(ZTOI(zp));
- ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
- }
- zap_cursor_fini(&zc);
-
- zfsvfs->z_draining = B_FALSE;
- zfsvfs->z_drain_task = TASKQID_INVALID;
-}
-
-/*
- * Sets z_draining then tries to dispatch async unlinked drain.
- * If that fails executes synchronous unlinked drain.
- */
-void
-zfs_unlinked_drain(zfsvfs_t *zfsvfs)
-{
- ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
- ASSERT3B(zfsvfs->z_draining, ==, B_FALSE);
-
- zfsvfs->z_draining = B_TRUE;
- zfsvfs->z_drain_cancel = B_FALSE;
-
- zfsvfs->z_drain_task = taskq_dispatch(
- dsl_pool_unlinked_drain_taskq(dmu_objset_pool(zfsvfs->z_os)),
- zfs_unlinked_drain_task, zfsvfs, TQ_SLEEP);
- if (zfsvfs->z_drain_task == TASKQID_INVALID) {
- zfs_dbgmsg("async zfs_unlinked_drain dispatch failed");
- zfs_unlinked_drain_task(zfsvfs);
- }
-}
-
-/*
- * Wait for the unlinked drain taskq task to stop. This will interrupt the
- * unlinked set processing if it is in progress.
- */
-void
-zfs_unlinked_drain_stop_wait(zfsvfs_t *zfsvfs)
-{
- ASSERT3B(zfsvfs->z_unmounted, ==, B_FALSE);
-
- if (zfsvfs->z_draining) {
- zfsvfs->z_drain_cancel = B_TRUE;
- taskq_cancel_id(dsl_pool_unlinked_drain_taskq(
- dmu_objset_pool(zfsvfs->z_os)), zfsvfs->z_drain_task);
- zfsvfs->z_drain_task = TASKQID_INVALID;
- zfsvfs->z_draining = B_FALSE;
- }
-}
-
-/*
- * Delete the entire contents of a directory. Return a count
- * of the number of entries that could not be deleted. If we encounter
- * an error, return a count of at least one so that the directory stays
- * in the unlinked set.
- *
- * NOTE: this function assumes that the directory is inactive,
- * so there is no need to lock its entries before deletion.
- * Also, it assumes the directory contents is *only* regular
- * files.
- */
-static int
-zfs_purgedir(znode_t *dzp)
-{
- zap_cursor_t zc;
- zap_attribute_t zap;
- znode_t *xzp;
- dmu_tx_t *tx;
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- zfs_dirlock_t dl;
- int skipped = 0;
- int error;
-
- for (zap_cursor_init(&zc, zfsvfs->z_os, dzp->z_id);
- (error = zap_cursor_retrieve(&zc, &zap)) == 0;
- zap_cursor_advance(&zc)) {
- error = zfs_zget(zfsvfs,
- ZFS_DIRENT_OBJ(zap.za_first_integer), &xzp);
- if (error) {
- skipped += 1;
- continue;
- }
-
- ASSERT(S_ISREG(ZTOI(xzp)->i_mode) ||
- S_ISLNK(ZTOI(xzp)->i_mode));
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_zap(tx, dzp->z_id, FALSE, zap.za_name);
- dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
- /* Is this really needed ? */
- zfs_sa_upgrade_txholds(tx, xzp);
- dmu_tx_mark_netfree(tx);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- zfs_iput_async(ZTOI(xzp));
- skipped += 1;
- continue;
- }
- bzero(&dl, sizeof (dl));
- dl.dl_dzp = dzp;
- dl.dl_name = zap.za_name;
-
- error = zfs_link_destroy(&dl, xzp, tx, 0, NULL);
- if (error)
- skipped += 1;
- dmu_tx_commit(tx);
-
- zfs_iput_async(ZTOI(xzp));
- }
- zap_cursor_fini(&zc);
- if (error != ENOENT)
- skipped += 1;
- return (skipped);
-}
-
-void
-zfs_rmnode(znode_t *zp)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- objset_t *os = zfsvfs->z_os;
- znode_t *xzp = NULL;
- dmu_tx_t *tx;
- uint64_t acl_obj;
- uint64_t xattr_obj;
- uint64_t links;
- int error;
-
- ASSERT(ZTOI(zp)->i_nlink == 0);
- ASSERT(atomic_read(&ZTOI(zp)->i_count) == 0);
-
- /*
- * If this is an attribute directory, purge its contents.
- */
- if (S_ISDIR(ZTOI(zp)->i_mode) && (zp->z_pflags & ZFS_XATTR)) {
- if (zfs_purgedir(zp) != 0) {
- /*
- * Not enough space to delete some xattrs.
- * Leave it in the unlinked set.
- */
- zfs_znode_dmu_fini(zp);
-
- return;
- }
- }
-
- /*
- * Free up all the data in the file. We don't do this for directories
- * because we need truncate and remove to be in the same tx, like in
- * zfs_znode_delete(). Otherwise, if we crash here we'll end up with
- * an inconsistent truncated zap object in the delete queue. Note a
- * truncated file is harmless since it only contains user data.
- */
- if (S_ISREG(ZTOI(zp)->i_mode)) {
- error = dmu_free_long_range(os, zp->z_id, 0, DMU_OBJECT_END);
- if (error) {
- /*
- * Not enough space or we were interrupted by unmount.
- * Leave the file in the unlinked set.
- */
- zfs_znode_dmu_fini(zp);
- return;
- }
- }
-
- /*
- * If the file has extended attributes, we're going to unlink
- * the xattr dir.
- */
- error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
- &xattr_obj, sizeof (xattr_obj));
- if (error == 0 && xattr_obj) {
- error = zfs_zget(zfsvfs, xattr_obj, &xzp);
- ASSERT(error == 0);
- }
-
- acl_obj = zfs_external_acl(zp);
-
- /*
- * Set up the final transaction.
- */
- tx = dmu_tx_create(os);
- dmu_tx_hold_free(tx, zp->z_id, 0, DMU_OBJECT_END);
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
- if (xzp) {
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, TRUE, NULL);
- dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
- }
- if (acl_obj)
- dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
-
- zfs_sa_upgrade_txholds(tx, zp);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- /*
- * Not enough space to delete the file. Leave it in the
- * unlinked set, leaking it until the fs is remounted (at
- * which point we'll call zfs_unlinked_drain() to process it).
- */
- dmu_tx_abort(tx);
- zfs_znode_dmu_fini(zp);
- goto out;
- }
-
- if (xzp) {
- ASSERT(error == 0);
- mutex_enter(&xzp->z_lock);
- xzp->z_unlinked = B_TRUE; /* mark xzp for deletion */
- clear_nlink(ZTOI(xzp)); /* no more links to it */
- links = 0;
- VERIFY(0 == sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
- &links, sizeof (links), tx));
- mutex_exit(&xzp->z_lock);
- zfs_unlinked_add(xzp, tx);
- }
-
- /* Remove this znode from the unlinked set */
- VERIFY3U(0, ==,
- zap_remove_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
-
- dataset_kstats_update_nunlinked_kstat(&zfsvfs->z_kstat, 1);
-
- zfs_znode_delete(zp, tx);
-
- dmu_tx_commit(tx);
-out:
- if (xzp)
- zfs_iput_async(ZTOI(xzp));
-}
-
-static uint64_t
-zfs_dirent(znode_t *zp, uint64_t mode)
-{
- uint64_t de = zp->z_id;
-
- if (ZTOZSB(zp)->z_version >= ZPL_VERSION_DIRENT_TYPE)
- de |= IFTODT(mode) << 60;
- return (de);
-}
-
-/*
- * Link zp into dl. Can fail in the following cases :
- * - if zp has been unlinked.
- * - if the number of entries with the same hash (aka. colliding entries)
- * exceed the capacity of a leaf-block of fatzap and splitting of the
- * leaf-block does not help.
- */
-int
-zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag)
-{
- znode_t *dzp = dl->dl_dzp;
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- uint64_t value;
- int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode);
- sa_bulk_attr_t bulk[5];
- uint64_t mtime[2], ctime[2];
- uint64_t links;
- int count = 0;
- int error;
-
- mutex_enter(&zp->z_lock);
-
- if (!(flag & ZRENAMING)) {
- if (zp->z_unlinked) { /* no new links to unlinked zp */
- ASSERT(!(flag & (ZNEW | ZEXISTS)));
- mutex_exit(&zp->z_lock);
- return (SET_ERROR(ENOENT));
- }
- if (!(flag & ZNEW)) {
- /*
- * ZNEW nodes come from zfs_mknode() where the link
- * count has already been initialised
- */
- inc_nlink(ZTOI(zp));
- links = ZTOI(zp)->i_nlink;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
- NULL, &links, sizeof (links));
- }
- }
-
- value = zfs_dirent(zp, zp->z_mode);
- error = zap_add(ZTOZSB(zp)->z_os, dzp->z_id, dl->dl_name, 8, 1,
- &value, tx);
-
- /*
- * zap_add could fail to add the entry if it exceeds the capacity of the
- * leaf-block and zap_leaf_split() failed to help.
- * The caller of this routine is responsible for failing the transaction
- * which will rollback the SA updates done above.
- */
- if (error != 0) {
- if (!(flag & ZRENAMING) && !(flag & ZNEW))
- drop_nlink(ZTOI(zp));
- mutex_exit(&zp->z_lock);
- return (error);
- }
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
- &dzp->z_id, sizeof (dzp->z_id));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, sizeof (zp->z_pflags));
-
- if (!(flag & ZNEW)) {
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
- ctime, sizeof (ctime));
- zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime,
- ctime);
- }
- error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
- ASSERT(error == 0);
-
- mutex_exit(&zp->z_lock);
-
- mutex_enter(&dzp->z_lock);
- dzp->z_size++;
- if (zp_is_dir)
- inc_nlink(ZTOI(dzp));
- links = ZTOI(dzp)->i_nlink;
- count = 0;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
- &dzp->z_size, sizeof (dzp->z_size));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL,
- &links, sizeof (links));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
- mtime, sizeof (mtime));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
- ctime, sizeof (ctime));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &dzp->z_pflags, sizeof (dzp->z_pflags));
- zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime);
- error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
- ASSERT(error == 0);
- mutex_exit(&dzp->z_lock);
-
- return (0);
-}
-
-/*
- * The match type in the code for this function should conform to:
- *
- * ------------------------------------------------------------------------
- * fs type | z_norm | lookup type | match type
- * ---------|-------------|-------------|----------------------------------
- * CS !norm | 0 | 0 | 0 (exact)
- * CS norm | formX | 0 | MT_NORMALIZE
- * CI !norm | upper | !ZCIEXACT | MT_NORMALIZE
- * CI !norm | upper | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE
- * CI norm | upper|formX | !ZCIEXACT | MT_NORMALIZE
- * CI norm | upper|formX | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE
- * CM !norm | upper | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE
- * CM !norm | upper | ZCILOOK | MT_NORMALIZE
- * CM norm | upper|formX | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE
- * CM norm | upper|formX | ZCILOOK | MT_NORMALIZE
- *
- * Abbreviations:
- * CS = Case Sensitive, CI = Case Insensitive, CM = Case Mixed
- * upper = case folding set by fs type on creation (U8_TEXTPREP_TOUPPER)
- * formX = unicode normalization form set on fs creation
- */
-static int
-zfs_dropname(zfs_dirlock_t *dl, znode_t *zp, znode_t *dzp, dmu_tx_t *tx,
- int flag)
-{
- int error;
-
- if (ZTOZSB(zp)->z_norm) {
- matchtype_t mt = MT_NORMALIZE;
-
- if ((ZTOZSB(zp)->z_case == ZFS_CASE_INSENSITIVE &&
- (flag & ZCIEXACT)) ||
- (ZTOZSB(zp)->z_case == ZFS_CASE_MIXED &&
- !(flag & ZCILOOK))) {
- mt |= MT_MATCH_CASE;
- }
-
- error = zap_remove_norm(ZTOZSB(zp)->z_os, dzp->z_id,
- dl->dl_name, mt, tx);
- } else {
- error = zap_remove(ZTOZSB(zp)->z_os, dzp->z_id, dl->dl_name,
- tx);
- }
-
- return (error);
-}
-
-/*
- * Unlink zp from dl, and mark zp for deletion if this was the last link. Can
- * fail if zp is a mount point (EBUSY) or a non-empty directory (ENOTEMPTY).
- * If 'unlinkedp' is NULL, we put unlinked znodes on the unlinked list.
- * If it's non-NULL, we use it to indicate whether the znode needs deletion,
- * and it's the caller's job to do it.
- */
-int
-zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag,
- boolean_t *unlinkedp)
-{
- znode_t *dzp = dl->dl_dzp;
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode);
- boolean_t unlinked = B_FALSE;
- sa_bulk_attr_t bulk[5];
- uint64_t mtime[2], ctime[2];
- uint64_t links;
- int count = 0;
- int error;
-
- if (!(flag & ZRENAMING)) {
- mutex_enter(&zp->z_lock);
-
- if (zp_is_dir && !zfs_dirempty(zp)) {
- mutex_exit(&zp->z_lock);
- return (SET_ERROR(ENOTEMPTY));
- }
-
- /*
- * If we get here, we are going to try to remove the object.
- * First try removing the name from the directory; if that
- * fails, return the error.
- */
- error = zfs_dropname(dl, zp, dzp, tx, flag);
- if (error != 0) {
- mutex_exit(&zp->z_lock);
- return (error);
- }
-
- if (ZTOI(zp)->i_nlink <= zp_is_dir) {
- zfs_panic_recover("zfs: link count on %lu is %u, "
- "should be at least %u", zp->z_id,
- (int)ZTOI(zp)->i_nlink, zp_is_dir + 1);
- set_nlink(ZTOI(zp), zp_is_dir + 1);
- }
- drop_nlink(ZTOI(zp));
- if (ZTOI(zp)->i_nlink == zp_is_dir) {
- zp->z_unlinked = B_TRUE;
- clear_nlink(ZTOI(zp));
- unlinked = B_TRUE;
- } else {
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs),
- NULL, &ctime, sizeof (ctime));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
- NULL, &zp->z_pflags, sizeof (zp->z_pflags));
- zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime,
- ctime);
- }
- links = ZTOI(zp)->i_nlink;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
- NULL, &links, sizeof (links));
- error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
- count = 0;
- ASSERT(error == 0);
- mutex_exit(&zp->z_lock);
- } else {
- error = zfs_dropname(dl, zp, dzp, tx, flag);
- if (error != 0)
- return (error);
- }
-
- mutex_enter(&dzp->z_lock);
- dzp->z_size--; /* one dirent removed */
- if (zp_is_dir)
- drop_nlink(ZTOI(dzp)); /* ".." link from zp */
- links = ZTOI(dzp)->i_nlink;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
- NULL, &links, sizeof (links));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs),
- NULL, &dzp->z_size, sizeof (dzp->z_size));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs),
- NULL, ctime, sizeof (ctime));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs),
- NULL, mtime, sizeof (mtime));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
- NULL, &dzp->z_pflags, sizeof (dzp->z_pflags));
- zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime);
- error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
- ASSERT(error == 0);
- mutex_exit(&dzp->z_lock);
-
- if (unlinkedp != NULL)
- *unlinkedp = unlinked;
- else if (unlinked)
- zfs_unlinked_add(zp, tx);
-
- return (0);
-}
-
-/*
- * Indicate whether the directory is empty. Works with or without z_lock
- * held, but can only be consider a hint in the latter case. Returns true
- * if only "." and ".." remain and there's no work in progress.
- *
- * The internal ZAP size, rather than zp->z_size, needs to be checked since
- * some consumers (Lustre) do not strictly maintain an accurate SA_ZPL_SIZE.
- */
-boolean_t
-zfs_dirempty(znode_t *dzp)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- uint64_t count;
- int error;
-
- if (dzp->z_dirlocks != NULL)
- return (B_FALSE);
-
- error = zap_count(zfsvfs->z_os, dzp->z_id, &count);
- if (error != 0 || count != 0)
- return (B_FALSE);
-
- return (B_TRUE);
-}
-
-int
-zfs_make_xattrdir(znode_t *zp, vattr_t *vap, struct inode **xipp, cred_t *cr)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- znode_t *xzp;
- dmu_tx_t *tx;
- int error;
- zfs_acl_ids_t acl_ids;
- boolean_t fuid_dirtied;
-#ifdef DEBUG
- uint64_t parent;
-#endif
-
- *xipp = NULL;
-
- if ((error = zfs_zaccess(zp, ACE_WRITE_NAMED_ATTRS, 0, B_FALSE, cr)))
- return (error);
-
- if ((error = zfs_acl_ids_create(zp, IS_XATTR, vap, cr, NULL,
- &acl_ids)) != 0)
- return (error);
- if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zp->z_projid)) {
- zfs_acl_ids_free(&acl_ids);
- return (SET_ERROR(EDQUOT));
- }
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
- ZFS_SA_BASE_ATTR_SIZE);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
- dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_abort(tx);
- return (error);
- }
- zfs_mknode(zp, vap, tx, cr, IS_XATTR, &xzp, &acl_ids);
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
-#ifdef DEBUG
- error = sa_lookup(xzp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
- &parent, sizeof (parent));
- ASSERT(error == 0 && parent == zp->z_id);
-#endif
-
- VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &xzp->z_id,
- sizeof (xzp->z_id), tx));
-
- if (!zp->z_unlinked)
- (void) zfs_log_create(zfsvfs->z_log, tx, TX_MKXATTR, zp,
- xzp, "", NULL, acl_ids.z_fuidp, vap);
-
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_commit(tx);
-
- *xipp = ZTOI(xzp);
-
- return (0);
-}
-
-/*
- * Return a znode for the extended attribute directory for zp.
- * ** If the directory does not already exist, it is created **
- *
- * IN: zp - znode to obtain attribute directory from
- * cr - credentials of caller
- * flags - flags from the VOP_LOOKUP call
- *
- * OUT: xipp - pointer to extended attribute znode
- *
- * RETURN: 0 on success
- * error number on failure
- */
-int
-zfs_get_xattrdir(znode_t *zp, struct inode **xipp, cred_t *cr, int flags)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- znode_t *xzp;
- zfs_dirlock_t *dl;
- vattr_t va;
- int error;
-top:
- error = zfs_dirent_lock(&dl, zp, "", &xzp, ZXATTR, NULL, NULL);
- if (error)
- return (error);
-
- if (xzp != NULL) {
- *xipp = ZTOI(xzp);
- zfs_dirent_unlock(dl);
- return (0);
- }
-
- if (!(flags & CREATE_XATTR_DIR)) {
- zfs_dirent_unlock(dl);
- return (SET_ERROR(ENOENT));
- }
-
- if (zfs_is_readonly(zfsvfs)) {
- zfs_dirent_unlock(dl);
- return (SET_ERROR(EROFS));
- }
-
- /*
- * The ability to 'create' files in an attribute
- * directory comes from the write_xattr permission on the base file.
- *
- * The ability to 'search' an attribute directory requires
- * read_xattr permission on the base file.
- *
- * Once in a directory the ability to read/write attributes
- * is controlled by the permissions on the attribute file.
- */
- va.va_mask = ATTR_MODE | ATTR_UID | ATTR_GID;
- va.va_mode = S_IFDIR | S_ISVTX | 0777;
- zfs_fuid_map_ids(zp, cr, &va.va_uid, &va.va_gid);
-
- va.va_dentry = NULL;
- error = zfs_make_xattrdir(zp, &va, xipp, cr);
- zfs_dirent_unlock(dl);
-
- if (error == ERESTART) {
- /* NB: we already did dmu_tx_wait() if necessary */
- goto top;
- }
-
- return (error);
-}
-
-/*
- * Decide whether it is okay to remove within a sticky directory.
- *
- * In sticky directories, write access is not sufficient;
- * you can remove entries from a directory only if:
- *
- * you own the directory,
- * you own the entry,
- * you have write access to the entry,
- * or you are privileged (checked in secpolicy...).
- *
- * The function returns 0 if remove access is granted.
- */
-int
-zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr)
-{
- uid_t uid;
- uid_t downer;
- uid_t fowner;
- zfsvfs_t *zfsvfs = ZTOZSB(zdp);
-
- if (zfsvfs->z_replay)
- return (0);
-
- if ((zdp->z_mode & S_ISVTX) == 0)
- return (0);
-
- downer = zfs_fuid_map_id(zfsvfs, KUID_TO_SUID(ZTOI(zdp)->i_uid),
- cr, ZFS_OWNER);
- fowner = zfs_fuid_map_id(zfsvfs, KUID_TO_SUID(ZTOI(zp)->i_uid),
- cr, ZFS_OWNER);
-
- if ((uid = crgetuid(cr)) == downer || uid == fowner ||
- zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr) == 0)
- return (0);
- else
- return (secpolicy_vnode_remove(cr));
-}
diff --git a/module/zfs/zfs_sysfs.c b/module/zfs/zfs_sysfs.c
deleted file mode 100644
index bb7f3b69a..000000000
--- a/module/zfs/zfs_sysfs.c
+++ /dev/null
@@ -1,661 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2018, 2019 by Delphix. All rights reserved.
- */
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/zfeature.h>
-#include <sys/zfs_ioctl.h>
-#include <sys/zfs_sysfs.h>
-#include <sys/kmem.h>
-#include <sys/fs/zfs.h>
-#include <linux/kobject.h>
-
-#include "zfs_prop.h"
-
-#if !defined(_KERNEL)
-#error kernel builds only
-#endif
-
-/*
- * ZFS Module sysfs support
- *
- * This extends our sysfs '/sys/module/zfs' entry to include feature
- * and property attributes. The primary consumer of this information
- * is user processes, like the zfs CLI, that need to know what the
- * current loaded ZFS module supports. The libzfs binary will consult
- * this information when instantiating the zfs|zpool property tables
- * and the pool features table.
- *
- * The added top-level directories are:
- * /sys/module/zfs
- * ├── features.kernel
- * ├── features.pool
- * ├── properties.dataset
- * └── properties.pool
- *
- * The local interface for the zfs kobjects includes:
- * zfs_kobj_init()
- * zfs_kobj_add()
- * zfs_kobj_release()
- * zfs_kobj_add_attr()
- * zfs_kobj_fini()
- */
-
-/*
- * A zfs_mod_kobj_t represents a zfs kobject under '/sys/module/zfs'
- */
-struct zfs_mod_kobj;
-typedef struct zfs_mod_kobj zfs_mod_kobj_t;
-
-struct zfs_mod_kobj {
- struct kobject zko_kobj;
- struct kobj_type zko_kobj_type;
- struct sysfs_ops zko_sysfs_ops;
- size_t zko_attr_count;
- struct attribute *zko_attr_list; /* allocated */
- struct attribute **zko_default_attrs; /* allocated */
- size_t zko_child_count;
- zfs_mod_kobj_t *zko_children; /* allocated */
-};
-
-#define ATTR_TABLE_SIZE(cnt) (sizeof (struct attribute) * (cnt))
-/* Note +1 for NULL terminator slot */
-#define DEFAULT_ATTR_SIZE(cnt) (sizeof (struct attribute *) * (cnt + 1))
-#define CHILD_TABLE_SIZE(cnt) (sizeof (zfs_mod_kobj_t) * (cnt))
-
-/*
- * These are the top-level kobjects under '/sys/module/zfs/'
- */
-static zfs_mod_kobj_t kernel_features_kobj;
-static zfs_mod_kobj_t pool_features_kobj;
-static zfs_mod_kobj_t dataset_props_kobj;
-static zfs_mod_kobj_t pool_props_kobj;
-
-/*
- * The show function is used to provide the content
- * of an attribute into a PAGE_SIZE buffer.
- */
-typedef ssize_t (*sysfs_show_func)(struct kobject *, struct attribute *,
- char *);
-
-static void
-zfs_kobj_fini(zfs_mod_kobj_t *zkobj)
-{
- /* finalize any child kobjects */
- if (zkobj->zko_child_count != 0) {
- ASSERT(zkobj->zko_children);
- for (int i = 0; i < zkobj->zko_child_count; i++)
- zfs_kobj_fini(&zkobj->zko_children[i]);
- }
-
- /* kobject_put() will call zfs_kobj_release() to release memory */
- kobject_del(&zkobj->zko_kobj);
- kobject_put(&zkobj->zko_kobj);
-}
-
-static void
-zfs_kobj_release(struct kobject *kobj)
-{
- zfs_mod_kobj_t *zkobj = container_of(kobj, zfs_mod_kobj_t, zko_kobj);
-
- if (zkobj->zko_attr_list != NULL) {
- ASSERT3S(zkobj->zko_attr_count, !=, 0);
- kmem_free(zkobj->zko_attr_list,
- ATTR_TABLE_SIZE(zkobj->zko_attr_count));
- zkobj->zko_attr_list = NULL;
- }
-
- if (zkobj->zko_default_attrs != NULL) {
- kmem_free(zkobj->zko_default_attrs,
- DEFAULT_ATTR_SIZE(zkobj->zko_attr_count));
- zkobj->zko_default_attrs = NULL;
- }
-
- if (zkobj->zko_child_count != 0) {
- ASSERT(zkobj->zko_children);
-
- kmem_free(zkobj->zko_children,
- CHILD_TABLE_SIZE(zkobj->zko_child_count));
- zkobj->zko_child_count = 0;
- zkobj->zko_children = NULL;
- }
-
- zkobj->zko_attr_count = 0;
-}
-
-#ifndef sysfs_attr_init
-#define sysfs_attr_init(attr) do {} while (0)
-#endif
-
-static void
-zfs_kobj_add_attr(zfs_mod_kobj_t *zkobj, int attr_num, const char *attr_name)
-{
- VERIFY3U(attr_num, <, zkobj->zko_attr_count);
- ASSERT(zkobj->zko_attr_list);
- ASSERT(zkobj->zko_default_attrs);
-
- zkobj->zko_attr_list[attr_num].name = attr_name;
- zkobj->zko_attr_list[attr_num].mode = 0444;
- zkobj->zko_default_attrs[attr_num] = &zkobj->zko_attr_list[attr_num];
- sysfs_attr_init(&zkobj->zko_attr_list[attr_num]);
-}
-
-static int
-zfs_kobj_init(zfs_mod_kobj_t *zkobj, int attr_cnt, int child_cnt,
- sysfs_show_func show_func)
-{
- /*
- * Initialize object's attributes. Count can be zero.
- */
- if (attr_cnt > 0) {
- zkobj->zko_attr_list = kmem_zalloc(ATTR_TABLE_SIZE(attr_cnt),
- KM_SLEEP);
- if (zkobj->zko_attr_list == NULL)
- return (ENOMEM);
- }
- /* this will always have at least one slot for NULL termination */
- zkobj->zko_default_attrs = kmem_zalloc(DEFAULT_ATTR_SIZE(attr_cnt),
- KM_SLEEP);
- if (zkobj->zko_default_attrs == NULL) {
- if (zkobj->zko_attr_list != NULL) {
- kmem_free(zkobj->zko_attr_list,
- ATTR_TABLE_SIZE(attr_cnt));
- }
- return (ENOMEM);
- }
- zkobj->zko_attr_count = attr_cnt;
- zkobj->zko_kobj_type.default_attrs = zkobj->zko_default_attrs;
-
- if (child_cnt > 0) {
- zkobj->zko_children = kmem_zalloc(CHILD_TABLE_SIZE(child_cnt),
- KM_SLEEP);
- if (zkobj->zko_children == NULL) {
- if (zkobj->zko_default_attrs != NULL) {
- kmem_free(zkobj->zko_default_attrs,
- DEFAULT_ATTR_SIZE(attr_cnt));
- }
- if (zkobj->zko_attr_list != NULL) {
- kmem_free(zkobj->zko_attr_list,
- ATTR_TABLE_SIZE(attr_cnt));
- }
- return (ENOMEM);
- }
- zkobj->zko_child_count = child_cnt;
- }
-
- zkobj->zko_sysfs_ops.show = show_func;
- zkobj->zko_kobj_type.sysfs_ops = &zkobj->zko_sysfs_ops;
- zkobj->zko_kobj_type.release = zfs_kobj_release;
-
- return (0);
-}
-
-static int
-zfs_kobj_add(zfs_mod_kobj_t *zkobj, struct kobject *parent, const char *name)
-{
- /* zko_default_attrs must be NULL terminated */
- ASSERT(zkobj->zko_default_attrs != NULL);
- ASSERT(zkobj->zko_default_attrs[zkobj->zko_attr_count] == NULL);
-
- kobject_init(&zkobj->zko_kobj, &zkobj->zko_kobj_type);
- return (kobject_add(&zkobj->zko_kobj, parent, name));
-}
-
-/*
- * Each zfs property has these common attributes
- */
-static const char *zprop_attrs[] = {
- "type",
- "readonly",
- "setonce",
- "visible",
- "values",
- "default",
- "datasets" /* zfs properties only */
-};
-
-#define ZFS_PROP_ATTR_COUNT ARRAY_SIZE(zprop_attrs)
-#define ZPOOL_PROP_ATTR_COUNT (ZFS_PROP_ATTR_COUNT - 1)
-
-static const char *zprop_types[] = {
- "number",
- "string",
- "index",
-};
-
-typedef struct zfs_type_map {
- zfs_type_t ztm_type;
- const char *ztm_name;
-} zfs_type_map_t;
-
-static zfs_type_map_t type_map[] = {
- {ZFS_TYPE_FILESYSTEM, "filesystem"},
- {ZFS_TYPE_SNAPSHOT, "snapshot"},
- {ZFS_TYPE_VOLUME, "volume"},
- {ZFS_TYPE_BOOKMARK, "bookmark"}
-};
-
-/*
- * Show the content for a zfs property attribute
- */
-static ssize_t
-zprop_sysfs_show(const char *attr_name, const zprop_desc_t *property,
- char *buf, size_t buflen)
-{
- const char *show_str;
- char number[32];
-
- /* For dataset properties list the dataset types that apply */
- if (strcmp(attr_name, "datasets") == 0 &&
- property->pd_types != ZFS_TYPE_POOL) {
- int len = 0;
-
- for (int i = 0; i < ARRAY_SIZE(type_map); i++) {
- if (type_map[i].ztm_type & property->pd_types) {
- len += snprintf(buf + len, buflen - len, "%s ",
- type_map[i].ztm_name);
- }
- }
- len += snprintf(buf + len, buflen - len, "\n");
- return (len);
- }
-
- if (strcmp(attr_name, "type") == 0) {
- show_str = zprop_types[property->pd_proptype];
- } else if (strcmp(attr_name, "readonly") == 0) {
- show_str = property->pd_attr == PROP_READONLY ? "1" : "0";
- } else if (strcmp(attr_name, "setonce") == 0) {
- show_str = property->pd_attr == PROP_ONETIME ? "1" : "0";
- } else if (strcmp(attr_name, "visible") == 0) {
- show_str = property->pd_visible ? "1" : "0";
- } else if (strcmp(attr_name, "values") == 0) {
- show_str = property->pd_values ? property->pd_values : "";
- } else if (strcmp(attr_name, "default") == 0) {
- switch (property->pd_proptype) {
- case PROP_TYPE_NUMBER:
- (void) snprintf(number, sizeof (number), "%llu",
- (u_longlong_t)property->pd_numdefault);
- show_str = number;
- break;
- case PROP_TYPE_STRING:
- show_str = property->pd_strdefault ?
- property->pd_strdefault : "";
- break;
- case PROP_TYPE_INDEX:
- if (zprop_index_to_string(property->pd_propnum,
- property->pd_numdefault, &show_str,
- property->pd_types) != 0) {
- show_str = "";
- }
- break;
- default:
- return (0);
- }
- } else {
- return (0);
- }
-
- return (snprintf(buf, buflen, "%s\n", show_str));
-}
-
-static ssize_t
-dataset_property_show(struct kobject *kobj, struct attribute *attr, char *buf)
-{
- zfs_prop_t prop = zfs_name_to_prop(kobject_name(kobj));
- zprop_desc_t *prop_tbl = zfs_prop_get_table();
- ssize_t len;
-
- ASSERT3U(prop, <, ZFS_NUM_PROPS);
-
- len = zprop_sysfs_show(attr->name, &prop_tbl[prop], buf, PAGE_SIZE);
-
- return (len);
-}
-
-static ssize_t
-pool_property_show(struct kobject *kobj, struct attribute *attr, char *buf)
-{
- zpool_prop_t prop = zpool_name_to_prop(kobject_name(kobj));
- zprop_desc_t *prop_tbl = zpool_prop_get_table();
- ssize_t len;
-
- ASSERT3U(prop, <, ZPOOL_NUM_PROPS);
-
- len = zprop_sysfs_show(attr->name, &prop_tbl[prop], buf, PAGE_SIZE);
-
- return (len);
-}
-
-/*
- * ZFS kernel feature attributes for '/sys/module/zfs/features.kernel'
- *
- * This list is intended for kernel features that don't have a pool feature
- * association or that extend existing user kernel interfaces.
- *
- * A user processes can easily check if the running zfs kernel module
- * supports the new feature.
- */
-static const char *zfs_kernel_features[] = {
- /* --> Add new kernel features here */
- "com.delphix:vdev_initialize",
- "org.zfsonlinux:vdev_trim",
-};
-
-#define KERNEL_FEATURE_COUNT ARRAY_SIZE(zfs_kernel_features)
-
-static ssize_t
-kernel_feature_show(struct kobject *kobj, struct attribute *attr, char *buf)
-{
- if (strcmp(attr->name, "supported") == 0)
- return (snprintf(buf, PAGE_SIZE, "yes\n"));
- return (0);
-}
-
-static void
-kernel_feature_to_kobj(zfs_mod_kobj_t *parent, int slot, const char *name)
-{
- zfs_mod_kobj_t *zfs_kobj = &parent->zko_children[slot];
-
- ASSERT3U(slot, <, KERNEL_FEATURE_COUNT);
- ASSERT(name);
-
- int err = zfs_kobj_init(zfs_kobj, 1, 0, kernel_feature_show);
- if (err)
- return;
-
- zfs_kobj_add_attr(zfs_kobj, 0, "supported");
-
- err = zfs_kobj_add(zfs_kobj, &parent->zko_kobj, name);
- if (err)
- zfs_kobj_release(&zfs_kobj->zko_kobj);
-}
-
-static int
-zfs_kernel_features_init(zfs_mod_kobj_t *zfs_kobj, struct kobject *parent)
-{
- /*
- * Create a parent kobject to host kernel features.
- *
- * '/sys/module/zfs/features.kernel'
- */
- int err = zfs_kobj_init(zfs_kobj, 0, KERNEL_FEATURE_COUNT,
- kernel_feature_show);
- if (err)
- return (err);
- err = zfs_kobj_add(zfs_kobj, parent, ZFS_SYSFS_KERNEL_FEATURES);
- if (err) {
- zfs_kobj_release(&zfs_kobj->zko_kobj);
- return (err);
- }
-
- /*
- * Now create a kobject for each feature.
- *
- * '/sys/module/zfs/features.kernel/<feature>'
- */
- for (int f = 0; f < KERNEL_FEATURE_COUNT; f++)
- kernel_feature_to_kobj(zfs_kobj, f, zfs_kernel_features[f]);
-
- return (0);
-}
-
-/*
- * Each pool feature has these common attributes
- */
-static const char *pool_feature_attrs[] = {
- "description",
- "guid",
- "uname",
- "readonly_compatible",
- "required_for_mos",
- "activate_on_enable",
- "per_dataset"
-};
-
-#define ZPOOL_FEATURE_ATTR_COUNT ARRAY_SIZE(pool_feature_attrs)
-
-/*
- * Show the content for the given zfs pool feature attribute
- */
-static ssize_t
-pool_feature_show(struct kobject *kobj, struct attribute *attr, char *buf)
-{
- spa_feature_t fid;
-
- if (zfeature_lookup_guid(kobject_name(kobj), &fid) != 0)
- return (0);
-
- ASSERT3U(fid, <, SPA_FEATURES);
-
- zfeature_flags_t flags = spa_feature_table[fid].fi_flags;
- const char *show_str = NULL;
-
- if (strcmp(attr->name, "description") == 0) {
- show_str = spa_feature_table[fid].fi_desc;
- } else if (strcmp(attr->name, "guid") == 0) {
- show_str = spa_feature_table[fid].fi_guid;
- } else if (strcmp(attr->name, "uname") == 0) {
- show_str = spa_feature_table[fid].fi_uname;
- } else if (strcmp(attr->name, "readonly_compatible") == 0) {
- show_str = flags & ZFEATURE_FLAG_READONLY_COMPAT ? "1" : "0";
- } else if (strcmp(attr->name, "required_for_mos") == 0) {
- show_str = flags & ZFEATURE_FLAG_MOS ? "1" : "0";
- } else if (strcmp(attr->name, "activate_on_enable") == 0) {
- show_str = flags & ZFEATURE_FLAG_ACTIVATE_ON_ENABLE ? "1" : "0";
- } else if (strcmp(attr->name, "per_dataset") == 0) {
- show_str = flags & ZFEATURE_FLAG_PER_DATASET ? "1" : "0";
- }
- if (show_str == NULL)
- return (0);
-
- return (snprintf(buf, PAGE_SIZE, "%s\n", show_str));
-}
-
-static void
-pool_feature_to_kobj(zfs_mod_kobj_t *parent, spa_feature_t fid,
- const char *name)
-{
- zfs_mod_kobj_t *zfs_kobj = &parent->zko_children[fid];
-
- ASSERT3U(fid, <, SPA_FEATURES);
- ASSERT(name);
-
- int err = zfs_kobj_init(zfs_kobj, ZPOOL_FEATURE_ATTR_COUNT, 0,
- pool_feature_show);
- if (err)
- return;
-
- for (int i = 0; i < ZPOOL_FEATURE_ATTR_COUNT; i++)
- zfs_kobj_add_attr(zfs_kobj, i, pool_feature_attrs[i]);
-
- err = zfs_kobj_add(zfs_kobj, &parent->zko_kobj, name);
- if (err)
- zfs_kobj_release(&zfs_kobj->zko_kobj);
-}
-
-static int
-zfs_pool_features_init(zfs_mod_kobj_t *zfs_kobj, struct kobject *parent)
-{
- /*
- * Create a parent kobject to host pool features.
- *
- * '/sys/module/zfs/features.pool'
- */
- int err = zfs_kobj_init(zfs_kobj, 0, SPA_FEATURES, pool_feature_show);
- if (err)
- return (err);
- err = zfs_kobj_add(zfs_kobj, parent, ZFS_SYSFS_POOL_FEATURES);
- if (err) {
- zfs_kobj_release(&zfs_kobj->zko_kobj);
- return (err);
- }
-
- /*
- * Now create a kobject for each feature.
- *
- * '/sys/module/zfs/features.pool/<feature>'
- */
- for (spa_feature_t i = 0; i < SPA_FEATURES; i++)
- pool_feature_to_kobj(zfs_kobj, i, spa_feature_table[i].fi_guid);
-
- return (0);
-}
-
-typedef struct prop_to_kobj_arg {
- zprop_desc_t *p2k_table;
- zfs_mod_kobj_t *p2k_parent;
- sysfs_show_func p2k_show_func;
- int p2k_attr_count;
-} prop_to_kobj_arg_t;
-
-static int
-zprop_to_kobj(int prop, void *args)
-{
- prop_to_kobj_arg_t *data = args;
- zfs_mod_kobj_t *parent = data->p2k_parent;
- zfs_mod_kobj_t *zfs_kobj = &parent->zko_children[prop];
- const char *name = data->p2k_table[prop].pd_name;
- int err;
-
- ASSERT(name);
-
- err = zfs_kobj_init(zfs_kobj, data->p2k_attr_count, 0,
- data->p2k_show_func);
- if (err)
- return (ZPROP_CONT);
-
- for (int i = 0; i < data->p2k_attr_count; i++)
- zfs_kobj_add_attr(zfs_kobj, i, zprop_attrs[i]);
-
- err = zfs_kobj_add(zfs_kobj, &parent->zko_kobj, name);
- if (err)
- zfs_kobj_release(&zfs_kobj->zko_kobj);
-
- return (ZPROP_CONT);
-}
-
-static int
-zfs_sysfs_properties_init(zfs_mod_kobj_t *zfs_kobj, struct kobject *parent,
- zfs_type_t type)
-{
- prop_to_kobj_arg_t context;
- const char *name;
- int err;
-
- /*
- * Create a parent kobject to host properties.
- *
- * '/sys/module/zfs/properties.<type>'
- */
- if (type == ZFS_TYPE_POOL) {
- name = ZFS_SYSFS_POOL_PROPERTIES;
- context.p2k_table = zpool_prop_get_table();
- context.p2k_attr_count = ZPOOL_PROP_ATTR_COUNT;
- context.p2k_parent = zfs_kobj;
- context.p2k_show_func = pool_property_show;
- err = zfs_kobj_init(zfs_kobj, 0, ZPOOL_NUM_PROPS,
- pool_property_show);
- } else {
- name = ZFS_SYSFS_DATASET_PROPERTIES;
- context.p2k_table = zfs_prop_get_table();
- context.p2k_attr_count = ZFS_PROP_ATTR_COUNT;
- context.p2k_parent = zfs_kobj;
- context.p2k_show_func = dataset_property_show;
- err = zfs_kobj_init(zfs_kobj, 0, ZFS_NUM_PROPS,
- dataset_property_show);
- }
-
- if (err)
- return (err);
-
- err = zfs_kobj_add(zfs_kobj, parent, name);
- if (err) {
- zfs_kobj_release(&zfs_kobj->zko_kobj);
- return (err);
- }
-
- /*
- * Create a kobject for each property.
- *
- * '/sys/module/zfs/properties.<type>/<property>'
- */
- (void) zprop_iter_common(zprop_to_kobj, &context, B_TRUE,
- B_FALSE, type);
-
- return (err);
-}
-
-void
-zfs_sysfs_init(void)
-{
- struct kobject *parent;
-#if defined(CONFIG_ZFS) && !defined(CONFIG_ZFS_MODULE)
- parent = kobject_create_and_add("zfs", fs_kobj);
-#else
- parent = &(((struct module *)(THIS_MODULE))->mkobj).kobj;
-#endif
- int err;
-
- if (parent == NULL)
- return;
-
- err = zfs_kernel_features_init(&kernel_features_kobj, parent);
- if (err)
- return;
-
- err = zfs_pool_features_init(&pool_features_kobj, parent);
- if (err) {
- zfs_kobj_fini(&kernel_features_kobj);
- return;
- }
-
- err = zfs_sysfs_properties_init(&pool_props_kobj, parent,
- ZFS_TYPE_POOL);
- if (err) {
- zfs_kobj_fini(&kernel_features_kobj);
- zfs_kobj_fini(&pool_features_kobj);
- return;
- }
-
- err = zfs_sysfs_properties_init(&dataset_props_kobj, parent,
- ZFS_TYPE_FILESYSTEM);
- if (err) {
- zfs_kobj_fini(&kernel_features_kobj);
- zfs_kobj_fini(&pool_features_kobj);
- zfs_kobj_fini(&pool_props_kobj);
- return;
- }
-}
-
-void
-zfs_sysfs_fini(void)
-{
- /*
- * Remove top-level kobjects; each will remove any children kobjects
- */
- zfs_kobj_fini(&kernel_features_kobj);
- zfs_kobj_fini(&pool_features_kobj);
- zfs_kobj_fini(&dataset_props_kobj);
- zfs_kobj_fini(&pool_props_kobj);
-}
diff --git a/module/zfs/zfs_vfsops.c b/module/zfs/zfs_vfsops.c
deleted file mode 100644
index 0914e4b7d..000000000
--- a/module/zfs/zfs_vfsops.c
+++ /dev/null
@@ -1,2562 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
- */
-
-/* Portions Copyright 2010 Robert Milkowski */
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/sysmacros.h>
-#include <sys/kmem.h>
-#include <sys/pathname.h>
-#include <sys/vnode.h>
-#include <sys/vfs.h>
-#include <sys/mntent.h>
-#include <sys/cmn_err.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_dir.h>
-#include <sys/zil.h>
-#include <sys/fs/zfs.h>
-#include <sys/dmu.h>
-#include <sys/dsl_prop.h>
-#include <sys/dsl_dataset.h>
-#include <sys/dsl_deleg.h>
-#include <sys/spa.h>
-#include <sys/zap.h>
-#include <sys/sa.h>
-#include <sys/sa_impl.h>
-#include <sys/policy.h>
-#include <sys/atomic.h>
-#include <sys/zfs_ioctl.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zfs_fuid.h>
-#include <sys/sunddi.h>
-#include <sys/dmu_objset.h>
-#include <sys/spa_boot.h>
-#include <sys/objlist.h>
-#include <sys/zpl.h>
-#include <linux/vfs_compat.h>
-#include "zfs_comutil.h"
-
-enum {
- TOKEN_RO,
- TOKEN_RW,
- TOKEN_SETUID,
- TOKEN_NOSETUID,
- TOKEN_EXEC,
- TOKEN_NOEXEC,
- TOKEN_DEVICES,
- TOKEN_NODEVICES,
- TOKEN_DIRXATTR,
- TOKEN_SAXATTR,
- TOKEN_XATTR,
- TOKEN_NOXATTR,
- TOKEN_ATIME,
- TOKEN_NOATIME,
- TOKEN_RELATIME,
- TOKEN_NORELATIME,
- TOKEN_NBMAND,
- TOKEN_NONBMAND,
- TOKEN_MNTPOINT,
- TOKEN_LAST,
-};
-
-static const match_table_t zpl_tokens = {
- { TOKEN_RO, MNTOPT_RO },
- { TOKEN_RW, MNTOPT_RW },
- { TOKEN_SETUID, MNTOPT_SETUID },
- { TOKEN_NOSETUID, MNTOPT_NOSETUID },
- { TOKEN_EXEC, MNTOPT_EXEC },
- { TOKEN_NOEXEC, MNTOPT_NOEXEC },
- { TOKEN_DEVICES, MNTOPT_DEVICES },
- { TOKEN_NODEVICES, MNTOPT_NODEVICES },
- { TOKEN_DIRXATTR, MNTOPT_DIRXATTR },
- { TOKEN_SAXATTR, MNTOPT_SAXATTR },
- { TOKEN_XATTR, MNTOPT_XATTR },
- { TOKEN_NOXATTR, MNTOPT_NOXATTR },
- { TOKEN_ATIME, MNTOPT_ATIME },
- { TOKEN_NOATIME, MNTOPT_NOATIME },
- { TOKEN_RELATIME, MNTOPT_RELATIME },
- { TOKEN_NORELATIME, MNTOPT_NORELATIME },
- { TOKEN_NBMAND, MNTOPT_NBMAND },
- { TOKEN_NONBMAND, MNTOPT_NONBMAND },
- { TOKEN_MNTPOINT, MNTOPT_MNTPOINT "=%s" },
- { TOKEN_LAST, NULL },
-};
-
-static void
-zfsvfs_vfs_free(vfs_t *vfsp)
-{
- if (vfsp != NULL) {
- if (vfsp->vfs_mntpoint != NULL)
- strfree(vfsp->vfs_mntpoint);
-
- kmem_free(vfsp, sizeof (vfs_t));
- }
-}
-
-static int
-zfsvfs_parse_option(char *option, int token, substring_t *args, vfs_t *vfsp)
-{
- switch (token) {
- case TOKEN_RO:
- vfsp->vfs_readonly = B_TRUE;
- vfsp->vfs_do_readonly = B_TRUE;
- break;
- case TOKEN_RW:
- vfsp->vfs_readonly = B_FALSE;
- vfsp->vfs_do_readonly = B_TRUE;
- break;
- case TOKEN_SETUID:
- vfsp->vfs_setuid = B_TRUE;
- vfsp->vfs_do_setuid = B_TRUE;
- break;
- case TOKEN_NOSETUID:
- vfsp->vfs_setuid = B_FALSE;
- vfsp->vfs_do_setuid = B_TRUE;
- break;
- case TOKEN_EXEC:
- vfsp->vfs_exec = B_TRUE;
- vfsp->vfs_do_exec = B_TRUE;
- break;
- case TOKEN_NOEXEC:
- vfsp->vfs_exec = B_FALSE;
- vfsp->vfs_do_exec = B_TRUE;
- break;
- case TOKEN_DEVICES:
- vfsp->vfs_devices = B_TRUE;
- vfsp->vfs_do_devices = B_TRUE;
- break;
- case TOKEN_NODEVICES:
- vfsp->vfs_devices = B_FALSE;
- vfsp->vfs_do_devices = B_TRUE;
- break;
- case TOKEN_DIRXATTR:
- vfsp->vfs_xattr = ZFS_XATTR_DIR;
- vfsp->vfs_do_xattr = B_TRUE;
- break;
- case TOKEN_SAXATTR:
- vfsp->vfs_xattr = ZFS_XATTR_SA;
- vfsp->vfs_do_xattr = B_TRUE;
- break;
- case TOKEN_XATTR:
- vfsp->vfs_xattr = ZFS_XATTR_DIR;
- vfsp->vfs_do_xattr = B_TRUE;
- break;
- case TOKEN_NOXATTR:
- vfsp->vfs_xattr = ZFS_XATTR_OFF;
- vfsp->vfs_do_xattr = B_TRUE;
- break;
- case TOKEN_ATIME:
- vfsp->vfs_atime = B_TRUE;
- vfsp->vfs_do_atime = B_TRUE;
- break;
- case TOKEN_NOATIME:
- vfsp->vfs_atime = B_FALSE;
- vfsp->vfs_do_atime = B_TRUE;
- break;
- case TOKEN_RELATIME:
- vfsp->vfs_relatime = B_TRUE;
- vfsp->vfs_do_relatime = B_TRUE;
- break;
- case TOKEN_NORELATIME:
- vfsp->vfs_relatime = B_FALSE;
- vfsp->vfs_do_relatime = B_TRUE;
- break;
- case TOKEN_NBMAND:
- vfsp->vfs_nbmand = B_TRUE;
- vfsp->vfs_do_nbmand = B_TRUE;
- break;
- case TOKEN_NONBMAND:
- vfsp->vfs_nbmand = B_FALSE;
- vfsp->vfs_do_nbmand = B_TRUE;
- break;
- case TOKEN_MNTPOINT:
- vfsp->vfs_mntpoint = match_strdup(&args[0]);
- if (vfsp->vfs_mntpoint == NULL)
- return (SET_ERROR(ENOMEM));
-
- break;
- default:
- break;
- }
-
- return (0);
-}
-
-/*
- * Parse the raw mntopts and return a vfs_t describing the options.
- */
-static int
-zfsvfs_parse_options(char *mntopts, vfs_t **vfsp)
-{
- vfs_t *tmp_vfsp;
- int error;
-
- tmp_vfsp = kmem_zalloc(sizeof (vfs_t), KM_SLEEP);
-
- if (mntopts != NULL) {
- substring_t args[MAX_OPT_ARGS];
- char *tmp_mntopts, *p, *t;
- int token;
-
- tmp_mntopts = t = strdup(mntopts);
- if (tmp_mntopts == NULL)
- return (SET_ERROR(ENOMEM));
-
- while ((p = strsep(&t, ",")) != NULL) {
- if (!*p)
- continue;
-
- args[0].to = args[0].from = NULL;
- token = match_token(p, zpl_tokens, args);
- error = zfsvfs_parse_option(p, token, args, tmp_vfsp);
- if (error) {
- strfree(tmp_mntopts);
- zfsvfs_vfs_free(tmp_vfsp);
- return (error);
- }
- }
-
- strfree(tmp_mntopts);
- }
-
- *vfsp = tmp_vfsp;
-
- return (0);
-}
-
-boolean_t
-zfs_is_readonly(zfsvfs_t *zfsvfs)
-{
- return (!!(zfsvfs->z_sb->s_flags & SB_RDONLY));
-}
-
-/*ARGSUSED*/
-int
-zfs_sync(struct super_block *sb, int wait, cred_t *cr)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
-
- /*
- * Semantically, the only requirement is that the sync be initiated.
- * The DMU syncs out txgs frequently, so there's nothing to do.
- */
- if (!wait)
- return (0);
-
- if (zfsvfs != NULL) {
- /*
- * Sync a specific filesystem.
- */
- dsl_pool_t *dp;
-
- ZFS_ENTER(zfsvfs);
- dp = dmu_objset_pool(zfsvfs->z_os);
-
- /*
- * If the system is shutting down, then skip any
- * filesystems which may exist on a suspended pool.
- */
- if (spa_suspended(dp->dp_spa)) {
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- if (zfsvfs->z_log != NULL)
- zil_commit(zfsvfs->z_log, 0);
-
- ZFS_EXIT(zfsvfs);
- } else {
- /*
- * Sync all ZFS filesystems. This is what happens when you
- * run sync(1M). Unlike other filesystems, ZFS honors the
- * request by waiting for all pools to commit all dirty data.
- */
- spa_sync_allpools();
- }
-
- return (0);
-}
-
-static void
-atime_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
- struct super_block *sb = zfsvfs->z_sb;
-
- if (sb == NULL)
- return;
- /*
- * Update SB_NOATIME bit in VFS super block. Since atime update is
- * determined by atime_needs_update(), atime_needs_update() needs to
- * return false if atime is turned off, and not unconditionally return
- * false if atime is turned on.
- */
- if (newval)
- sb->s_flags &= ~SB_NOATIME;
- else
- sb->s_flags |= SB_NOATIME;
-}
-
-static void
-relatime_changed_cb(void *arg, uint64_t newval)
-{
- ((zfsvfs_t *)arg)->z_relatime = newval;
-}
-
-static void
-xattr_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
-
- if (newval == ZFS_XATTR_OFF) {
- zfsvfs->z_flags &= ~ZSB_XATTR;
- } else {
- zfsvfs->z_flags |= ZSB_XATTR;
-
- if (newval == ZFS_XATTR_SA)
- zfsvfs->z_xattr_sa = B_TRUE;
- else
- zfsvfs->z_xattr_sa = B_FALSE;
- }
-}
-
-static void
-acltype_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
-
- switch (newval) {
- case ZFS_ACLTYPE_OFF:
- zfsvfs->z_acl_type = ZFS_ACLTYPE_OFF;
- zfsvfs->z_sb->s_flags &= ~SB_POSIXACL;
- break;
- case ZFS_ACLTYPE_POSIXACL:
-#ifdef CONFIG_FS_POSIX_ACL
- zfsvfs->z_acl_type = ZFS_ACLTYPE_POSIXACL;
- zfsvfs->z_sb->s_flags |= SB_POSIXACL;
-#else
- zfsvfs->z_acl_type = ZFS_ACLTYPE_OFF;
- zfsvfs->z_sb->s_flags &= ~SB_POSIXACL;
-#endif /* CONFIG_FS_POSIX_ACL */
- break;
- default:
- break;
- }
-}
-
-static void
-blksz_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
- ASSERT3U(newval, <=, spa_maxblocksize(dmu_objset_spa(zfsvfs->z_os)));
- ASSERT3U(newval, >=, SPA_MINBLOCKSIZE);
- ASSERT(ISP2(newval));
-
- zfsvfs->z_max_blksz = newval;
-}
-
-static void
-readonly_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
- struct super_block *sb = zfsvfs->z_sb;
-
- if (sb == NULL)
- return;
-
- if (newval)
- sb->s_flags |= SB_RDONLY;
- else
- sb->s_flags &= ~SB_RDONLY;
-}
-
-static void
-devices_changed_cb(void *arg, uint64_t newval)
-{
-}
-
-static void
-setuid_changed_cb(void *arg, uint64_t newval)
-{
-}
-
-static void
-exec_changed_cb(void *arg, uint64_t newval)
-{
-}
-
-static void
-nbmand_changed_cb(void *arg, uint64_t newval)
-{
- zfsvfs_t *zfsvfs = arg;
- struct super_block *sb = zfsvfs->z_sb;
-
- if (sb == NULL)
- return;
-
- if (newval == TRUE)
- sb->s_flags |= SB_MANDLOCK;
- else
- sb->s_flags &= ~SB_MANDLOCK;
-}
-
-static void
-snapdir_changed_cb(void *arg, uint64_t newval)
-{
- ((zfsvfs_t *)arg)->z_show_ctldir = newval;
-}
-
-static void
-vscan_changed_cb(void *arg, uint64_t newval)
-{
- ((zfsvfs_t *)arg)->z_vscan = newval;
-}
-
-static void
-acl_inherit_changed_cb(void *arg, uint64_t newval)
-{
- ((zfsvfs_t *)arg)->z_acl_inherit = newval;
-}
-
-static int
-zfs_register_callbacks(vfs_t *vfsp)
-{
- struct dsl_dataset *ds = NULL;
- objset_t *os = NULL;
- zfsvfs_t *zfsvfs = NULL;
- int error = 0;
-
- ASSERT(vfsp);
- zfsvfs = vfsp->vfs_data;
- ASSERT(zfsvfs);
- os = zfsvfs->z_os;
-
- /*
- * The act of registering our callbacks will destroy any mount
- * options we may have. In order to enable temporary overrides
- * of mount options, we stash away the current values and
- * restore them after we register the callbacks.
- */
- if (zfs_is_readonly(zfsvfs) || !spa_writeable(dmu_objset_spa(os))) {
- vfsp->vfs_do_readonly = B_TRUE;
- vfsp->vfs_readonly = B_TRUE;
- }
-
- /*
- * Register property callbacks.
- *
- * It would probably be fine to just check for i/o error from
- * the first prop_register(), but I guess I like to go
- * overboard...
- */
- ds = dmu_objset_ds(os);
- dsl_pool_config_enter(dmu_objset_pool(os), FTAG);
- error = dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_ATIME), atime_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_RELATIME), relatime_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_XATTR), xattr_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_RECORDSIZE), blksz_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_READONLY), readonly_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_DEVICES), devices_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_SETUID), setuid_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_EXEC), exec_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_SNAPDIR), snapdir_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_ACLTYPE), acltype_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_ACLINHERIT), acl_inherit_changed_cb,
- zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_VSCAN), vscan_changed_cb, zfsvfs);
- error = error ? error : dsl_prop_register(ds,
- zfs_prop_to_name(ZFS_PROP_NBMAND), nbmand_changed_cb, zfsvfs);
- dsl_pool_config_exit(dmu_objset_pool(os), FTAG);
- if (error)
- goto unregister;
-
- /*
- * Invoke our callbacks to restore temporary mount options.
- */
- if (vfsp->vfs_do_readonly)
- readonly_changed_cb(zfsvfs, vfsp->vfs_readonly);
- if (vfsp->vfs_do_setuid)
- setuid_changed_cb(zfsvfs, vfsp->vfs_setuid);
- if (vfsp->vfs_do_exec)
- exec_changed_cb(zfsvfs, vfsp->vfs_exec);
- if (vfsp->vfs_do_devices)
- devices_changed_cb(zfsvfs, vfsp->vfs_devices);
- if (vfsp->vfs_do_xattr)
- xattr_changed_cb(zfsvfs, vfsp->vfs_xattr);
- if (vfsp->vfs_do_atime)
- atime_changed_cb(zfsvfs, vfsp->vfs_atime);
- if (vfsp->vfs_do_relatime)
- relatime_changed_cb(zfsvfs, vfsp->vfs_relatime);
- if (vfsp->vfs_do_nbmand)
- nbmand_changed_cb(zfsvfs, vfsp->vfs_nbmand);
-
- return (0);
-
-unregister:
- dsl_prop_unregister_all(ds, zfsvfs);
- return (error);
-}
-
-static int
-zfs_space_delta_cb(dmu_object_type_t bonustype, void *data,
- uint64_t *userp, uint64_t *groupp, uint64_t *projectp)
-{
- sa_hdr_phys_t sa;
- sa_hdr_phys_t *sap = data;
- uint64_t flags;
- int hdrsize;
- boolean_t swap = B_FALSE;
-
- /*
- * Is it a valid type of object to track?
- */
- if (bonustype != DMU_OT_ZNODE && bonustype != DMU_OT_SA)
- return (SET_ERROR(ENOENT));
-
- /*
- * If we have a NULL data pointer
- * then assume the id's aren't changing and
- * return EEXIST to the dmu to let it know to
- * use the same ids
- */
- if (data == NULL)
- return (SET_ERROR(EEXIST));
-
- if (bonustype == DMU_OT_ZNODE) {
- znode_phys_t *znp = data;
- *userp = znp->zp_uid;
- *groupp = znp->zp_gid;
- *projectp = ZFS_DEFAULT_PROJID;
- return (0);
- }
-
- if (sap->sa_magic == 0) {
- /*
- * This should only happen for newly created files
- * that haven't had the znode data filled in yet.
- */
- *userp = 0;
- *groupp = 0;
- *projectp = ZFS_DEFAULT_PROJID;
- return (0);
- }
-
- sa = *sap;
- if (sa.sa_magic == BSWAP_32(SA_MAGIC)) {
- sa.sa_magic = SA_MAGIC;
- sa.sa_layout_info = BSWAP_16(sa.sa_layout_info);
- swap = B_TRUE;
- } else {
- VERIFY3U(sa.sa_magic, ==, SA_MAGIC);
- }
-
- hdrsize = sa_hdrsize(&sa);
- VERIFY3U(hdrsize, >=, sizeof (sa_hdr_phys_t));
-
- *userp = *((uint64_t *)((uintptr_t)data + hdrsize + SA_UID_OFFSET));
- *groupp = *((uint64_t *)((uintptr_t)data + hdrsize + SA_GID_OFFSET));
- flags = *((uint64_t *)((uintptr_t)data + hdrsize + SA_FLAGS_OFFSET));
- if (swap)
- flags = BSWAP_64(flags);
-
- if (flags & ZFS_PROJID)
- *projectp = *((uint64_t *)((uintptr_t)data + hdrsize +
- SA_PROJID_OFFSET));
- else
- *projectp = ZFS_DEFAULT_PROJID;
-
- if (swap) {
- *userp = BSWAP_64(*userp);
- *groupp = BSWAP_64(*groupp);
- *projectp = BSWAP_64(*projectp);
- }
- return (0);
-}
-
-static void
-fuidstr_to_sid(zfsvfs_t *zfsvfs, const char *fuidstr,
- char *domainbuf, int buflen, uid_t *ridp)
-{
- uint64_t fuid;
- const char *domain;
-
- fuid = zfs_strtonum(fuidstr, NULL);
-
- domain = zfs_fuid_find_by_idx(zfsvfs, FUID_INDEX(fuid));
- if (domain)
- (void) strlcpy(domainbuf, domain, buflen);
- else
- domainbuf[0] = '\0';
- *ridp = FUID_RID(fuid);
-}
-
-static uint64_t
-zfs_userquota_prop_to_obj(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type)
-{
- switch (type) {
- case ZFS_PROP_USERUSED:
- case ZFS_PROP_USEROBJUSED:
- return (DMU_USERUSED_OBJECT);
- case ZFS_PROP_GROUPUSED:
- case ZFS_PROP_GROUPOBJUSED:
- return (DMU_GROUPUSED_OBJECT);
- case ZFS_PROP_PROJECTUSED:
- case ZFS_PROP_PROJECTOBJUSED:
- return (DMU_PROJECTUSED_OBJECT);
- case ZFS_PROP_USERQUOTA:
- return (zfsvfs->z_userquota_obj);
- case ZFS_PROP_GROUPQUOTA:
- return (zfsvfs->z_groupquota_obj);
- case ZFS_PROP_USEROBJQUOTA:
- return (zfsvfs->z_userobjquota_obj);
- case ZFS_PROP_GROUPOBJQUOTA:
- return (zfsvfs->z_groupobjquota_obj);
- case ZFS_PROP_PROJECTQUOTA:
- return (zfsvfs->z_projectquota_obj);
- case ZFS_PROP_PROJECTOBJQUOTA:
- return (zfsvfs->z_projectobjquota_obj);
- default:
- return (ZFS_NO_OBJECT);
- }
-}
-
-int
-zfs_userspace_many(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type,
- uint64_t *cookiep, void *vbuf, uint64_t *bufsizep)
-{
- int error;
- zap_cursor_t zc;
- zap_attribute_t za;
- zfs_useracct_t *buf = vbuf;
- uint64_t obj;
- int offset = 0;
-
- if (!dmu_objset_userspace_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
-
- if ((type == ZFS_PROP_PROJECTQUOTA || type == ZFS_PROP_PROJECTUSED ||
- type == ZFS_PROP_PROJECTOBJQUOTA ||
- type == ZFS_PROP_PROJECTOBJUSED) &&
- !dmu_objset_projectquota_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
-
- if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
- type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA ||
- type == ZFS_PROP_PROJECTOBJUSED ||
- type == ZFS_PROP_PROJECTOBJQUOTA) &&
- !dmu_objset_userobjspace_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
-
- obj = zfs_userquota_prop_to_obj(zfsvfs, type);
- if (obj == ZFS_NO_OBJECT) {
- *bufsizep = 0;
- return (0);
- }
-
- if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
- type == ZFS_PROP_PROJECTOBJUSED)
- offset = DMU_OBJACCT_PREFIX_LEN;
-
- for (zap_cursor_init_serialized(&zc, zfsvfs->z_os, obj, *cookiep);
- (error = zap_cursor_retrieve(&zc, &za)) == 0;
- zap_cursor_advance(&zc)) {
- if ((uintptr_t)buf - (uintptr_t)vbuf + sizeof (zfs_useracct_t) >
- *bufsizep)
- break;
-
- /*
- * skip object quota (with zap name prefix DMU_OBJACCT_PREFIX)
- * when dealing with block quota and vice versa.
- */
- if ((offset > 0) != (strncmp(za.za_name, DMU_OBJACCT_PREFIX,
- DMU_OBJACCT_PREFIX_LEN) == 0))
- continue;
-
- fuidstr_to_sid(zfsvfs, za.za_name + offset,
- buf->zu_domain, sizeof (buf->zu_domain), &buf->zu_rid);
-
- buf->zu_space = za.za_first_integer;
- buf++;
- }
- if (error == ENOENT)
- error = 0;
-
- ASSERT3U((uintptr_t)buf - (uintptr_t)vbuf, <=, *bufsizep);
- *bufsizep = (uintptr_t)buf - (uintptr_t)vbuf;
- *cookiep = zap_cursor_serialize(&zc);
- zap_cursor_fini(&zc);
- return (error);
-}
-
-/*
- * buf must be big enough (eg, 32 bytes)
- */
-static int
-id_to_fuidstr(zfsvfs_t *zfsvfs, const char *domain, uid_t rid,
- char *buf, boolean_t addok)
-{
- uint64_t fuid;
- int domainid = 0;
-
- if (domain && domain[0]) {
- domainid = zfs_fuid_find_by_domain(zfsvfs, domain, NULL, addok);
- if (domainid == -1)
- return (SET_ERROR(ENOENT));
- }
- fuid = FUID_ENCODE(domainid, rid);
- (void) sprintf(buf, "%llx", (longlong_t)fuid);
- return (0);
-}
-
-int
-zfs_userspace_one(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type,
- const char *domain, uint64_t rid, uint64_t *valp)
-{
- char buf[20 + DMU_OBJACCT_PREFIX_LEN];
- int offset = 0;
- int err;
- uint64_t obj;
-
- *valp = 0;
-
- if (!dmu_objset_userspace_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
-
- if ((type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
- type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA ||
- type == ZFS_PROP_PROJECTOBJUSED ||
- type == ZFS_PROP_PROJECTOBJQUOTA) &&
- !dmu_objset_userobjspace_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
-
- if (type == ZFS_PROP_PROJECTQUOTA || type == ZFS_PROP_PROJECTUSED ||
- type == ZFS_PROP_PROJECTOBJQUOTA ||
- type == ZFS_PROP_PROJECTOBJUSED) {
- if (!dmu_objset_projectquota_present(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
- if (!zpl_is_valid_projid(rid))
- return (SET_ERROR(EINVAL));
- }
-
- obj = zfs_userquota_prop_to_obj(zfsvfs, type);
- if (obj == ZFS_NO_OBJECT)
- return (0);
-
- if (type == ZFS_PROP_USEROBJUSED || type == ZFS_PROP_GROUPOBJUSED ||
- type == ZFS_PROP_PROJECTOBJUSED) {
- strlcpy(buf, DMU_OBJACCT_PREFIX, DMU_OBJACCT_PREFIX_LEN + 1);
- offset = DMU_OBJACCT_PREFIX_LEN;
- }
-
- err = id_to_fuidstr(zfsvfs, domain, rid, buf + offset, B_FALSE);
- if (err)
- return (err);
-
- err = zap_lookup(zfsvfs->z_os, obj, buf, 8, 1, valp);
- if (err == ENOENT)
- err = 0;
- return (err);
-}
-
-int
-zfs_set_userquota(zfsvfs_t *zfsvfs, zfs_userquota_prop_t type,
- const char *domain, uint64_t rid, uint64_t quota)
-{
- char buf[32];
- int err;
- dmu_tx_t *tx;
- uint64_t *objp;
- boolean_t fuid_dirtied;
-
- if (zfsvfs->z_version < ZPL_VERSION_USERSPACE)
- return (SET_ERROR(ENOTSUP));
-
- switch (type) {
- case ZFS_PROP_USERQUOTA:
- objp = &zfsvfs->z_userquota_obj;
- break;
- case ZFS_PROP_GROUPQUOTA:
- objp = &zfsvfs->z_groupquota_obj;
- break;
- case ZFS_PROP_USEROBJQUOTA:
- objp = &zfsvfs->z_userobjquota_obj;
- break;
- case ZFS_PROP_GROUPOBJQUOTA:
- objp = &zfsvfs->z_groupobjquota_obj;
- break;
- case ZFS_PROP_PROJECTQUOTA:
- if (!dmu_objset_projectquota_enabled(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
- if (!zpl_is_valid_projid(rid))
- return (SET_ERROR(EINVAL));
-
- objp = &zfsvfs->z_projectquota_obj;
- break;
- case ZFS_PROP_PROJECTOBJQUOTA:
- if (!dmu_objset_projectquota_enabled(zfsvfs->z_os))
- return (SET_ERROR(ENOTSUP));
- if (!zpl_is_valid_projid(rid))
- return (SET_ERROR(EINVAL));
-
- objp = &zfsvfs->z_projectobjquota_obj;
- break;
- default:
- return (SET_ERROR(EINVAL));
- }
-
- err = id_to_fuidstr(zfsvfs, domain, rid, buf, B_TRUE);
- if (err)
- return (err);
- fuid_dirtied = zfsvfs->z_fuid_dirty;
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_zap(tx, *objp ? *objp : DMU_NEW_OBJECT, B_TRUE, NULL);
- if (*objp == 0) {
- dmu_tx_hold_zap(tx, MASTER_NODE_OBJ, B_TRUE,
- zfs_userquota_prop_prefixes[type]);
- }
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- err = dmu_tx_assign(tx, TXG_WAIT);
- if (err) {
- dmu_tx_abort(tx);
- return (err);
- }
-
- mutex_enter(&zfsvfs->z_lock);
- if (*objp == 0) {
- *objp = zap_create(zfsvfs->z_os, DMU_OT_USERGROUP_QUOTA,
- DMU_OT_NONE, 0, tx);
- VERIFY(0 == zap_add(zfsvfs->z_os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[type], 8, 1, objp, tx));
- }
- mutex_exit(&zfsvfs->z_lock);
-
- if (quota == 0) {
- err = zap_remove(zfsvfs->z_os, *objp, buf, tx);
- if (err == ENOENT)
- err = 0;
- } else {
- err = zap_update(zfsvfs->z_os, *objp, buf, 8, 1, &quota, tx);
- }
- ASSERT(err == 0);
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
- dmu_tx_commit(tx);
- return (err);
-}
-
-boolean_t
-zfs_id_overobjquota(zfsvfs_t *zfsvfs, uint64_t usedobj, uint64_t id)
-{
- char buf[20 + DMU_OBJACCT_PREFIX_LEN];
- uint64_t used, quota, quotaobj;
- int err;
-
- if (!dmu_objset_userobjspace_present(zfsvfs->z_os)) {
- if (dmu_objset_userobjspace_upgradable(zfsvfs->z_os)) {
- dsl_pool_config_enter(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- dmu_objset_id_quota_upgrade(zfsvfs->z_os);
- dsl_pool_config_exit(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- }
- return (B_FALSE);
- }
-
- if (usedobj == DMU_PROJECTUSED_OBJECT) {
- if (!dmu_objset_projectquota_present(zfsvfs->z_os)) {
- if (dmu_objset_projectquota_upgradable(zfsvfs->z_os)) {
- dsl_pool_config_enter(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- dmu_objset_id_quota_upgrade(zfsvfs->z_os);
- dsl_pool_config_exit(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- }
- return (B_FALSE);
- }
- quotaobj = zfsvfs->z_projectobjquota_obj;
- } else if (usedobj == DMU_USERUSED_OBJECT) {
- quotaobj = zfsvfs->z_userobjquota_obj;
- } else if (usedobj == DMU_GROUPUSED_OBJECT) {
- quotaobj = zfsvfs->z_groupobjquota_obj;
- } else {
- return (B_FALSE);
- }
- if (quotaobj == 0 || zfsvfs->z_replay)
- return (B_FALSE);
-
- (void) sprintf(buf, "%llx", (longlong_t)id);
- err = zap_lookup(zfsvfs->z_os, quotaobj, buf, 8, 1, &quota);
- if (err != 0)
- return (B_FALSE);
-
- (void) sprintf(buf, DMU_OBJACCT_PREFIX "%llx", (longlong_t)id);
- err = zap_lookup(zfsvfs->z_os, usedobj, buf, 8, 1, &used);
- if (err != 0)
- return (B_FALSE);
- return (used >= quota);
-}
-
-boolean_t
-zfs_id_overblockquota(zfsvfs_t *zfsvfs, uint64_t usedobj, uint64_t id)
-{
- char buf[20];
- uint64_t used, quota, quotaobj;
- int err;
-
- if (usedobj == DMU_PROJECTUSED_OBJECT) {
- if (!dmu_objset_projectquota_present(zfsvfs->z_os)) {
- if (dmu_objset_projectquota_upgradable(zfsvfs->z_os)) {
- dsl_pool_config_enter(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- dmu_objset_id_quota_upgrade(zfsvfs->z_os);
- dsl_pool_config_exit(
- dmu_objset_pool(zfsvfs->z_os), FTAG);
- }
- return (B_FALSE);
- }
- quotaobj = zfsvfs->z_projectquota_obj;
- } else if (usedobj == DMU_USERUSED_OBJECT) {
- quotaobj = zfsvfs->z_userquota_obj;
- } else if (usedobj == DMU_GROUPUSED_OBJECT) {
- quotaobj = zfsvfs->z_groupquota_obj;
- } else {
- return (B_FALSE);
- }
- if (quotaobj == 0 || zfsvfs->z_replay)
- return (B_FALSE);
-
- (void) sprintf(buf, "%llx", (longlong_t)id);
- err = zap_lookup(zfsvfs->z_os, quotaobj, buf, 8, 1, &quota);
- if (err != 0)
- return (B_FALSE);
-
- err = zap_lookup(zfsvfs->z_os, usedobj, buf, 8, 1, &used);
- if (err != 0)
- return (B_FALSE);
- return (used >= quota);
-}
-
-boolean_t
-zfs_id_overquota(zfsvfs_t *zfsvfs, uint64_t usedobj, uint64_t id)
-{
- return (zfs_id_overblockquota(zfsvfs, usedobj, id) ||
- zfs_id_overobjquota(zfsvfs, usedobj, id));
-}
-
-/*
- * Associate this zfsvfs with the given objset, which must be owned.
- * This will cache a bunch of on-disk state from the objset in the
- * zfsvfs.
- */
-static int
-zfsvfs_init(zfsvfs_t *zfsvfs, objset_t *os)
-{
- int error;
- uint64_t val;
-
- zfsvfs->z_max_blksz = SPA_OLD_MAXBLOCKSIZE;
- zfsvfs->z_show_ctldir = ZFS_SNAPDIR_VISIBLE;
- zfsvfs->z_os = os;
-
- error = zfs_get_zplprop(os, ZFS_PROP_VERSION, &zfsvfs->z_version);
- if (error != 0)
- return (error);
- if (zfsvfs->z_version >
- zfs_zpl_version_map(spa_version(dmu_objset_spa(os)))) {
- (void) printk("Can't mount a version %lld file system "
- "on a version %lld pool\n. Pool must be upgraded to mount "
- "this file system.\n", (u_longlong_t)zfsvfs->z_version,
- (u_longlong_t)spa_version(dmu_objset_spa(os)));
- return (SET_ERROR(ENOTSUP));
- }
- error = zfs_get_zplprop(os, ZFS_PROP_NORMALIZE, &val);
- if (error != 0)
- return (error);
- zfsvfs->z_norm = (int)val;
-
- error = zfs_get_zplprop(os, ZFS_PROP_UTF8ONLY, &val);
- if (error != 0)
- return (error);
- zfsvfs->z_utf8 = (val != 0);
-
- error = zfs_get_zplprop(os, ZFS_PROP_CASE, &val);
- if (error != 0)
- return (error);
- zfsvfs->z_case = (uint_t)val;
-
- if ((error = zfs_get_zplprop(os, ZFS_PROP_ACLTYPE, &val)) != 0)
- return (error);
- zfsvfs->z_acl_type = (uint_t)val;
-
- /*
- * Fold case on file systems that are always or sometimes case
- * insensitive.
- */
- if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE ||
- zfsvfs->z_case == ZFS_CASE_MIXED)
- zfsvfs->z_norm |= U8_TEXTPREP_TOUPPER;
-
- zfsvfs->z_use_fuids = USE_FUIDS(zfsvfs->z_version, zfsvfs->z_os);
- zfsvfs->z_use_sa = USE_SA(zfsvfs->z_version, zfsvfs->z_os);
-
- uint64_t sa_obj = 0;
- if (zfsvfs->z_use_sa) {
- /* should either have both of these objects or none */
- error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_SA_ATTRS, 8, 1,
- &sa_obj);
- if (error != 0)
- return (error);
-
- error = zfs_get_zplprop(os, ZFS_PROP_XATTR, &val);
- if ((error == 0) && (val == ZFS_XATTR_SA))
- zfsvfs->z_xattr_sa = B_TRUE;
- }
-
- error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_ROOT_OBJ, 8, 1,
- &zfsvfs->z_root);
- if (error != 0)
- return (error);
- ASSERT(zfsvfs->z_root != 0);
-
- error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_UNLINKED_SET, 8, 1,
- &zfsvfs->z_unlinkedobj);
- if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_USERQUOTA],
- 8, 1, &zfsvfs->z_userquota_obj);
- if (error == ENOENT)
- zfsvfs->z_userquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_GROUPQUOTA],
- 8, 1, &zfsvfs->z_groupquota_obj);
- if (error == ENOENT)
- zfsvfs->z_groupquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_PROJECTQUOTA],
- 8, 1, &zfsvfs->z_projectquota_obj);
- if (error == ENOENT)
- zfsvfs->z_projectquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA],
- 8, 1, &zfsvfs->z_userobjquota_obj);
- if (error == ENOENT)
- zfsvfs->z_userobjquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA],
- 8, 1, &zfsvfs->z_groupobjquota_obj);
- if (error == ENOENT)
- zfsvfs->z_groupobjquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ,
- zfs_userquota_prop_prefixes[ZFS_PROP_PROJECTOBJQUOTA],
- 8, 1, &zfsvfs->z_projectobjquota_obj);
- if (error == ENOENT)
- zfsvfs->z_projectobjquota_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES, 8, 1,
- &zfsvfs->z_fuid_obj);
- if (error == ENOENT)
- zfsvfs->z_fuid_obj = 0;
- else if (error != 0)
- return (error);
-
- error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_SHARES_DIR, 8, 1,
- &zfsvfs->z_shares_dir);
- if (error == ENOENT)
- zfsvfs->z_shares_dir = 0;
- else if (error != 0)
- return (error);
-
- error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END,
- &zfsvfs->z_attr_table);
- if (error != 0)
- return (error);
-
- if (zfsvfs->z_version >= ZPL_VERSION_SA)
- sa_register_update_callback(os, zfs_sa_upgrade);
-
- return (0);
-}
-
-int
-zfsvfs_create(const char *osname, boolean_t readonly, zfsvfs_t **zfvp)
-{
- objset_t *os;
- zfsvfs_t *zfsvfs;
- int error;
- boolean_t ro = (readonly || (strchr(osname, '@') != NULL));
-
- zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
-
- error = dmu_objset_own(osname, DMU_OST_ZFS, ro, B_TRUE, zfsvfs, &os);
- if (error != 0) {
- kmem_free(zfsvfs, sizeof (zfsvfs_t));
- return (error);
- }
-
- error = zfsvfs_create_impl(zfvp, zfsvfs, os);
- if (error != 0) {
- dmu_objset_disown(os, B_TRUE, zfsvfs);
- }
- return (error);
-}
-
-
-/*
- * Note: zfsvfs is assumed to be malloc'd, and will be freed by this function
- * on a failure. Do not pass in a statically allocated zfsvfs.
- */
-int
-zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os)
-{
- int error;
-
- zfsvfs->z_vfs = NULL;
- zfsvfs->z_sb = NULL;
- zfsvfs->z_parent = zfsvfs;
-
- mutex_init(&zfsvfs->z_znodes_lock, NULL, MUTEX_DEFAULT, NULL);
- mutex_init(&zfsvfs->z_lock, NULL, MUTEX_DEFAULT, NULL);
- list_create(&zfsvfs->z_all_znodes, sizeof (znode_t),
- offsetof(znode_t, z_link_node));
- rrm_init(&zfsvfs->z_teardown_lock, B_FALSE);
- rw_init(&zfsvfs->z_teardown_inactive_lock, NULL, RW_DEFAULT, NULL);
- rw_init(&zfsvfs->z_fuid_lock, NULL, RW_DEFAULT, NULL);
-
- int size = MIN(1 << (highbit64(zfs_object_mutex_size) - 1),
- ZFS_OBJ_MTX_MAX);
- zfsvfs->z_hold_size = size;
- zfsvfs->z_hold_trees = vmem_zalloc(sizeof (avl_tree_t) * size,
- KM_SLEEP);
- zfsvfs->z_hold_locks = vmem_zalloc(sizeof (kmutex_t) * size, KM_SLEEP);
- for (int i = 0; i != size; i++) {
- avl_create(&zfsvfs->z_hold_trees[i], zfs_znode_hold_compare,
- sizeof (znode_hold_t), offsetof(znode_hold_t, zh_node));
- mutex_init(&zfsvfs->z_hold_locks[i], NULL, MUTEX_DEFAULT, NULL);
- }
-
- error = zfsvfs_init(zfsvfs, os);
- if (error != 0) {
- *zfvp = NULL;
- zfsvfs_free(zfsvfs);
- return (error);
- }
-
- zfsvfs->z_drain_task = TASKQID_INVALID;
- zfsvfs->z_draining = B_FALSE;
- zfsvfs->z_drain_cancel = B_TRUE;
-
- *zfvp = zfsvfs;
- return (0);
-}
-
-static int
-zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting)
-{
- int error;
- boolean_t readonly = zfs_is_readonly(zfsvfs);
-
- error = zfs_register_callbacks(zfsvfs->z_vfs);
- if (error)
- return (error);
-
- zfsvfs->z_log = zil_open(zfsvfs->z_os, zfs_get_data);
-
- /*
- * If we are not mounting (ie: online recv), then we don't
- * have to worry about replaying the log as we blocked all
- * operations out since we closed the ZIL.
- */
- if (mounting) {
- ASSERT3P(zfsvfs->z_kstat.dk_kstats, ==, NULL);
- dataset_kstats_create(&zfsvfs->z_kstat, zfsvfs->z_os);
-
- /*
- * During replay we remove the read only flag to
- * allow replays to succeed.
- */
- if (readonly != 0) {
- readonly_changed_cb(zfsvfs, B_FALSE);
- } else {
- zap_stats_t zs;
- if (zap_get_stats(zfsvfs->z_os, zfsvfs->z_unlinkedobj,
- &zs) == 0) {
- dataset_kstats_update_nunlinks_kstat(
- &zfsvfs->z_kstat, zs.zs_num_entries);
- }
- dprintf_ds(zfsvfs->z_os->os_dsl_dataset,
- "num_entries in unlinked set: %llu",
- zs.zs_num_entries);
- zfs_unlinked_drain(zfsvfs);
- }
-
- /*
- * Parse and replay the intent log.
- *
- * Because of ziltest, this must be done after
- * zfs_unlinked_drain(). (Further note: ziltest
- * doesn't use readonly mounts, where
- * zfs_unlinked_drain() isn't called.) This is because
- * ziltest causes spa_sync() to think it's committed,
- * but actually it is not, so the intent log contains
- * many txg's worth of changes.
- *
- * In particular, if object N is in the unlinked set in
- * the last txg to actually sync, then it could be
- * actually freed in a later txg and then reallocated
- * in a yet later txg. This would write a "create
- * object N" record to the intent log. Normally, this
- * would be fine because the spa_sync() would have
- * written out the fact that object N is free, before
- * we could write the "create object N" intent log
- * record.
- *
- * But when we are in ziltest mode, we advance the "open
- * txg" without actually spa_sync()-ing the changes to
- * disk. So we would see that object N is still
- * allocated and in the unlinked set, and there is an
- * intent log record saying to allocate it.
- */
- if (spa_writeable(dmu_objset_spa(zfsvfs->z_os))) {
- if (zil_replay_disable) {
- zil_destroy(zfsvfs->z_log, B_FALSE);
- } else {
- zfsvfs->z_replay = B_TRUE;
- zil_replay(zfsvfs->z_os, zfsvfs,
- zfs_replay_vector);
- zfsvfs->z_replay = B_FALSE;
- }
- }
-
- /* restore readonly bit */
- if (readonly != 0)
- readonly_changed_cb(zfsvfs, B_TRUE);
- }
-
- /*
- * Set the objset user_ptr to track its zfsvfs.
- */
- mutex_enter(&zfsvfs->z_os->os_user_ptr_lock);
- dmu_objset_set_user(zfsvfs->z_os, zfsvfs);
- mutex_exit(&zfsvfs->z_os->os_user_ptr_lock);
-
- return (0);
-}
-
-void
-zfsvfs_free(zfsvfs_t *zfsvfs)
-{
- int i, size = zfsvfs->z_hold_size;
-
- zfs_fuid_destroy(zfsvfs);
-
- mutex_destroy(&zfsvfs->z_znodes_lock);
- mutex_destroy(&zfsvfs->z_lock);
- list_destroy(&zfsvfs->z_all_znodes);
- rrm_destroy(&zfsvfs->z_teardown_lock);
- rw_destroy(&zfsvfs->z_teardown_inactive_lock);
- rw_destroy(&zfsvfs->z_fuid_lock);
- for (i = 0; i != size; i++) {
- avl_destroy(&zfsvfs->z_hold_trees[i]);
- mutex_destroy(&zfsvfs->z_hold_locks[i]);
- }
- vmem_free(zfsvfs->z_hold_trees, sizeof (avl_tree_t) * size);
- vmem_free(zfsvfs->z_hold_locks, sizeof (kmutex_t) * size);
- zfsvfs_vfs_free(zfsvfs->z_vfs);
- dataset_kstats_destroy(&zfsvfs->z_kstat);
- kmem_free(zfsvfs, sizeof (zfsvfs_t));
-}
-
-static void
-zfs_set_fuid_feature(zfsvfs_t *zfsvfs)
-{
- zfsvfs->z_use_fuids = USE_FUIDS(zfsvfs->z_version, zfsvfs->z_os);
- zfsvfs->z_use_sa = USE_SA(zfsvfs->z_version, zfsvfs->z_os);
-}
-
-void
-zfs_unregister_callbacks(zfsvfs_t *zfsvfs)
-{
- objset_t *os = zfsvfs->z_os;
-
- if (!dmu_objset_is_snapshot(os))
- dsl_prop_unregister_all(dmu_objset_ds(os), zfsvfs);
-}
-
-#ifdef HAVE_MLSLABEL
-/*
- * Check that the hex label string is appropriate for the dataset being
- * mounted into the global_zone proper.
- *
- * Return an error if the hex label string is not default or
- * admin_low/admin_high. For admin_low labels, the corresponding
- * dataset must be readonly.
- */
-int
-zfs_check_global_label(const char *dsname, const char *hexsl)
-{
- if (strcasecmp(hexsl, ZFS_MLSLABEL_DEFAULT) == 0)
- return (0);
- if (strcasecmp(hexsl, ADMIN_HIGH) == 0)
- return (0);
- if (strcasecmp(hexsl, ADMIN_LOW) == 0) {
- /* must be readonly */
- uint64_t rdonly;
-
- if (dsl_prop_get_integer(dsname,
- zfs_prop_to_name(ZFS_PROP_READONLY), &rdonly, NULL))
- return (SET_ERROR(EACCES));
- return (rdonly ? 0 : EACCES);
- }
- return (SET_ERROR(EACCES));
-}
-#endif /* HAVE_MLSLABEL */
-
-static int
-zfs_statfs_project(zfsvfs_t *zfsvfs, znode_t *zp, struct kstatfs *statp,
- uint32_t bshift)
-{
- char buf[20 + DMU_OBJACCT_PREFIX_LEN];
- uint64_t offset = DMU_OBJACCT_PREFIX_LEN;
- uint64_t quota;
- uint64_t used;
- int err;
-
- strlcpy(buf, DMU_OBJACCT_PREFIX, DMU_OBJACCT_PREFIX_LEN + 1);
- err = id_to_fuidstr(zfsvfs, NULL, zp->z_projid, buf + offset, B_FALSE);
- if (err)
- return (err);
-
- if (zfsvfs->z_projectquota_obj == 0)
- goto objs;
-
- err = zap_lookup(zfsvfs->z_os, zfsvfs->z_projectquota_obj,
- buf + offset, 8, 1, &quota);
- if (err == ENOENT)
- goto objs;
- else if (err)
- return (err);
-
- err = zap_lookup(zfsvfs->z_os, DMU_PROJECTUSED_OBJECT,
- buf + offset, 8, 1, &used);
- if (unlikely(err == ENOENT)) {
- uint32_t blksize;
- u_longlong_t nblocks;
-
- /*
- * Quota accounting is async, so it is possible race case.
- * There is at least one object with the given project ID.
- */
- sa_object_size(zp->z_sa_hdl, &blksize, &nblocks);
- if (unlikely(zp->z_blksz == 0))
- blksize = zfsvfs->z_max_blksz;
-
- used = blksize * nblocks;
- } else if (err) {
- return (err);
- }
-
- statp->f_blocks = quota >> bshift;
- statp->f_bfree = (quota > used) ? ((quota - used) >> bshift) : 0;
- statp->f_bavail = statp->f_bfree;
-
-objs:
- if (zfsvfs->z_projectobjquota_obj == 0)
- return (0);
-
- err = zap_lookup(zfsvfs->z_os, zfsvfs->z_projectobjquota_obj,
- buf + offset, 8, 1, &quota);
- if (err == ENOENT)
- return (0);
- else if (err)
- return (err);
-
- err = zap_lookup(zfsvfs->z_os, DMU_PROJECTUSED_OBJECT,
- buf, 8, 1, &used);
- if (unlikely(err == ENOENT)) {
- /*
- * Quota accounting is async, so it is possible race case.
- * There is at least one object with the given project ID.
- */
- used = 1;
- } else if (err) {
- return (err);
- }
-
- statp->f_files = quota;
- statp->f_ffree = (quota > used) ? (quota - used) : 0;
-
- return (0);
-}
-
-int
-zfs_statvfs(struct dentry *dentry, struct kstatfs *statp)
-{
- zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
- uint64_t refdbytes, availbytes, usedobjs, availobjs;
- int err = 0;
-
- ZFS_ENTER(zfsvfs);
-
- dmu_objset_space(zfsvfs->z_os,
- &refdbytes, &availbytes, &usedobjs, &availobjs);
-
- uint64_t fsid = dmu_objset_fsid_guid(zfsvfs->z_os);
- /*
- * The underlying storage pool actually uses multiple block
- * size. Under Solaris frsize (fragment size) is reported as
- * the smallest block size we support, and bsize (block size)
- * as the filesystem's maximum block size. Unfortunately,
- * under Linux the fragment size and block size are often used
- * interchangeably. Thus we are forced to report both of them
- * as the filesystem's maximum block size.
- */
- statp->f_frsize = zfsvfs->z_max_blksz;
- statp->f_bsize = zfsvfs->z_max_blksz;
- uint32_t bshift = fls(statp->f_bsize) - 1;
-
- /*
- * The following report "total" blocks of various kinds in
- * the file system, but reported in terms of f_bsize - the
- * "preferred" size.
- */
-
- /* Round up so we never have a filesystem using 0 blocks. */
- refdbytes = P2ROUNDUP(refdbytes, statp->f_bsize);
- statp->f_blocks = (refdbytes + availbytes) >> bshift;
- statp->f_bfree = availbytes >> bshift;
- statp->f_bavail = statp->f_bfree; /* no root reservation */
-
- /*
- * statvfs() should really be called statufs(), because it assumes
- * static metadata. ZFS doesn't preallocate files, so the best
- * we can do is report the max that could possibly fit in f_files,
- * and that minus the number actually used in f_ffree.
- * For f_ffree, report the smaller of the number of objects available
- * and the number of blocks (each object will take at least a block).
- */
- statp->f_ffree = MIN(availobjs, availbytes >> DNODE_SHIFT);
- statp->f_files = statp->f_ffree + usedobjs;
- statp->f_fsid.val[0] = (uint32_t)fsid;
- statp->f_fsid.val[1] = (uint32_t)(fsid >> 32);
- statp->f_type = ZFS_SUPER_MAGIC;
- statp->f_namelen = MAXNAMELEN - 1;
-
- /*
- * We have all of 40 characters to stuff a string here.
- * Is there anything useful we could/should provide?
- */
- bzero(statp->f_spare, sizeof (statp->f_spare));
-
- if (dmu_objset_projectquota_enabled(zfsvfs->z_os) &&
- dmu_objset_projectquota_present(zfsvfs->z_os)) {
- znode_t *zp = ITOZ(dentry->d_inode);
-
- if (zp->z_pflags & ZFS_PROJINHERIT && zp->z_projid &&
- zpl_is_valid_projid(zp->z_projid))
- err = zfs_statfs_project(zfsvfs, zp, statp, bshift);
- }
-
- ZFS_EXIT(zfsvfs);
- return (err);
-}
-
-int
-zfs_root(zfsvfs_t *zfsvfs, struct inode **ipp)
-{
- znode_t *rootzp;
- int error;
-
- ZFS_ENTER(zfsvfs);
-
- error = zfs_zget(zfsvfs, zfsvfs->z_root, &rootzp);
- if (error == 0)
- *ipp = ZTOI(rootzp);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-#ifdef HAVE_D_PRUNE_ALIASES
-/*
- * Linux kernels older than 3.1 do not support a per-filesystem shrinker.
- * To accommodate this we must improvise and manually walk the list of znodes
- * attempting to prune dentries in order to be able to drop the inodes.
- *
- * To avoid scanning the same znodes multiple times they are always rotated
- * to the end of the z_all_znodes list. New znodes are inserted at the
- * end of the list so we're always scanning the oldest znodes first.
- */
-static int
-zfs_prune_aliases(zfsvfs_t *zfsvfs, unsigned long nr_to_scan)
-{
- znode_t **zp_array, *zp;
- int max_array = MIN(nr_to_scan, PAGE_SIZE * 8 / sizeof (znode_t *));
- int objects = 0;
- int i = 0, j = 0;
-
- zp_array = kmem_zalloc(max_array * sizeof (znode_t *), KM_SLEEP);
-
- mutex_enter(&zfsvfs->z_znodes_lock);
- while ((zp = list_head(&zfsvfs->z_all_znodes)) != NULL) {
-
- if ((i++ > nr_to_scan) || (j >= max_array))
- break;
-
- ASSERT(list_link_active(&zp->z_link_node));
- list_remove(&zfsvfs->z_all_znodes, zp);
- list_insert_tail(&zfsvfs->z_all_znodes, zp);
-
- /* Skip active znodes and .zfs entries */
- if (MUTEX_HELD(&zp->z_lock) || zp->z_is_ctldir)
- continue;
-
- if (igrab(ZTOI(zp)) == NULL)
- continue;
-
- zp_array[j] = zp;
- j++;
- }
- mutex_exit(&zfsvfs->z_znodes_lock);
-
- for (i = 0; i < j; i++) {
- zp = zp_array[i];
-
- ASSERT3P(zp, !=, NULL);
- d_prune_aliases(ZTOI(zp));
-
- if (atomic_read(&ZTOI(zp)->i_count) == 1)
- objects++;
-
- iput(ZTOI(zp));
- }
-
- kmem_free(zp_array, max_array * sizeof (znode_t *));
-
- return (objects);
-}
-#endif /* HAVE_D_PRUNE_ALIASES */
-
-/*
- * The ARC has requested that the filesystem drop entries from the dentry
- * and inode caches. This can occur when the ARC needs to free meta data
- * blocks but can't because they are all pinned by entries in these caches.
- */
-int
-zfs_prune(struct super_block *sb, unsigned long nr_to_scan, int *objects)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
- int error = 0;
-#if defined(HAVE_SHRINK) || defined(HAVE_SPLIT_SHRINKER_CALLBACK)
- struct shrinker *shrinker = &sb->s_shrink;
- struct shrink_control sc = {
- .nr_to_scan = nr_to_scan,
- .gfp_mask = GFP_KERNEL,
- };
-#endif
-
- ZFS_ENTER(zfsvfs);
-
-#if defined(HAVE_SPLIT_SHRINKER_CALLBACK) && \
- defined(SHRINK_CONTROL_HAS_NID) && \
- defined(SHRINKER_NUMA_AWARE)
- if (sb->s_shrink.flags & SHRINKER_NUMA_AWARE) {
- *objects = 0;
- for_each_online_node(sc.nid) {
- *objects += (*shrinker->scan_objects)(shrinker, &sc);
- }
- } else {
- *objects = (*shrinker->scan_objects)(shrinker, &sc);
- }
-
-#elif defined(HAVE_SPLIT_SHRINKER_CALLBACK)
- *objects = (*shrinker->scan_objects)(shrinker, &sc);
-#elif defined(HAVE_SHRINK)
- *objects = (*shrinker->shrink)(shrinker, &sc);
-#elif defined(HAVE_D_PRUNE_ALIASES)
-#define D_PRUNE_ALIASES_IS_DEFAULT
- *objects = zfs_prune_aliases(zfsvfs, nr_to_scan);
-#else
-#error "No available dentry and inode cache pruning mechanism."
-#endif
-
-#if defined(HAVE_D_PRUNE_ALIASES) && !defined(D_PRUNE_ALIASES_IS_DEFAULT)
-#undef D_PRUNE_ALIASES_IS_DEFAULT
- /*
- * Fall back to zfs_prune_aliases if the kernel's per-superblock
- * shrinker couldn't free anything, possibly due to the inodes being
- * allocated in a different memcg.
- */
- if (*objects == 0)
- *objects = zfs_prune_aliases(zfsvfs, nr_to_scan);
-#endif
-
- ZFS_EXIT(zfsvfs);
-
- dprintf_ds(zfsvfs->z_os->os_dsl_dataset,
- "pruning, nr_to_scan=%lu objects=%d error=%d\n",
- nr_to_scan, *objects, error);
-
- return (error);
-}
-
-/*
- * Teardown the zfsvfs_t.
- *
- * Note, if 'unmounting' is FALSE, we return with the 'z_teardown_lock'
- * and 'z_teardown_inactive_lock' held.
- */
-static int
-zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting)
-{
- znode_t *zp;
-
- zfs_unlinked_drain_stop_wait(zfsvfs);
-
- /*
- * If someone has not already unmounted this file system,
- * drain the iput_taskq to ensure all active references to the
- * zfsvfs_t have been handled only then can it be safely destroyed.
- */
- if (zfsvfs->z_os) {
- /*
- * If we're unmounting we have to wait for the list to
- * drain completely.
- *
- * If we're not unmounting there's no guarantee the list
- * will drain completely, but iputs run from the taskq
- * may add the parents of dir-based xattrs to the taskq
- * so we want to wait for these.
- *
- * We can safely read z_nr_znodes without locking because the
- * VFS has already blocked operations which add to the
- * z_all_znodes list and thus increment z_nr_znodes.
- */
- int round = 0;
- while (zfsvfs->z_nr_znodes > 0) {
- taskq_wait_outstanding(dsl_pool_iput_taskq(
- dmu_objset_pool(zfsvfs->z_os)), 0);
- if (++round > 1 && !unmounting)
- break;
- }
- }
-
- rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG);
-
- if (!unmounting) {
- /*
- * We purge the parent filesystem's super block as the
- * parent filesystem and all of its snapshots have their
- * inode's super block set to the parent's filesystem's
- * super block. Note, 'z_parent' is self referential
- * for non-snapshots.
- */
- shrink_dcache_sb(zfsvfs->z_parent->z_sb);
- }
-
- /*
- * Close the zil. NB: Can't close the zil while zfs_inactive
- * threads are blocked as zil_close can call zfs_inactive.
- */
- if (zfsvfs->z_log) {
- zil_close(zfsvfs->z_log);
- zfsvfs->z_log = NULL;
- }
-
- rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_WRITER);
-
- /*
- * If we are not unmounting (ie: online recv) and someone already
- * unmounted this file system while we were doing the switcheroo,
- * or a reopen of z_os failed then just bail out now.
- */
- if (!unmounting && (zfsvfs->z_unmounted || zfsvfs->z_os == NULL)) {
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
- rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
- return (SET_ERROR(EIO));
- }
-
- /*
- * At this point there are no VFS ops active, and any new VFS ops
- * will fail with EIO since we have z_teardown_lock for writer (only
- * relevant for forced unmount).
- *
- * Release all holds on dbufs. We also grab an extra reference to all
- * the remaining inodes so that the kernel does not attempt to free
- * any inodes of a suspended fs. This can cause deadlocks since the
- * zfs_resume_fs() process may involve starting threads, which might
- * attempt to free unreferenced inodes to free up memory for the new
- * thread.
- */
- if (!unmounting) {
- mutex_enter(&zfsvfs->z_znodes_lock);
- for (zp = list_head(&zfsvfs->z_all_znodes); zp != NULL;
- zp = list_next(&zfsvfs->z_all_znodes, zp)) {
- if (zp->z_sa_hdl)
- zfs_znode_dmu_fini(zp);
- if (igrab(ZTOI(zp)) != NULL)
- zp->z_suspended = B_TRUE;
-
- }
- mutex_exit(&zfsvfs->z_znodes_lock);
- }
-
- /*
- * If we are unmounting, set the unmounted flag and let new VFS ops
- * unblock. zfs_inactive will have the unmounted behavior, and all
- * other VFS ops will fail with EIO.
- */
- if (unmounting) {
- zfsvfs->z_unmounted = B_TRUE;
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
- rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
- }
-
- /*
- * z_os will be NULL if there was an error in attempting to reopen
- * zfsvfs, so just return as the properties had already been
- *
- * unregistered and cached data had been evicted before.
- */
- if (zfsvfs->z_os == NULL)
- return (0);
-
- /*
- * Unregister properties.
- */
- zfs_unregister_callbacks(zfsvfs);
-
- /*
- * Evict cached data. We must write out any dirty data before
- * disowning the dataset.
- */
- objset_t *os = zfsvfs->z_os;
- boolean_t os_dirty = B_FALSE;
- for (int t = 0; t < TXG_SIZE; t++) {
- if (dmu_objset_is_dirty(os, t)) {
- os_dirty = B_TRUE;
- break;
- }
- }
- if (!zfs_is_readonly(zfsvfs) && os_dirty) {
- txg_wait_synced(dmu_objset_pool(zfsvfs->z_os), 0);
- }
- dmu_objset_evict_dbufs(zfsvfs->z_os);
-
- return (0);
-}
-
-#if !defined(HAVE_2ARGS_BDI_SETUP_AND_REGISTER) && \
- !defined(HAVE_3ARGS_BDI_SETUP_AND_REGISTER)
-atomic_long_t zfs_bdi_seq = ATOMIC_LONG_INIT(0);
-#endif
-
-int
-zfs_domount(struct super_block *sb, zfs_mnt_t *zm, int silent)
-{
- const char *osname = zm->mnt_osname;
- struct inode *root_inode;
- uint64_t recordsize;
- int error = 0;
- zfsvfs_t *zfsvfs = NULL;
- vfs_t *vfs = NULL;
-
- ASSERT(zm);
- ASSERT(osname);
-
- error = zfsvfs_parse_options(zm->mnt_data, &vfs);
- if (error)
- return (error);
-
- error = zfsvfs_create(osname, vfs->vfs_readonly, &zfsvfs);
- if (error) {
- zfsvfs_vfs_free(vfs);
- goto out;
- }
-
- if ((error = dsl_prop_get_integer(osname, "recordsize",
- &recordsize, NULL))) {
- zfsvfs_vfs_free(vfs);
- goto out;
- }
-
- vfs->vfs_data = zfsvfs;
- zfsvfs->z_vfs = vfs;
- zfsvfs->z_sb = sb;
- sb->s_fs_info = zfsvfs;
- sb->s_magic = ZFS_SUPER_MAGIC;
- sb->s_maxbytes = MAX_LFS_FILESIZE;
- sb->s_time_gran = 1;
- sb->s_blocksize = recordsize;
- sb->s_blocksize_bits = ilog2(recordsize);
-
- error = -zpl_bdi_setup(sb, "zfs");
- if (error)
- goto out;
-
- sb->s_bdi->ra_pages = 0;
-
- /* Set callback operations for the file system. */
- sb->s_op = &zpl_super_operations;
- sb->s_xattr = zpl_xattr_handlers;
- sb->s_export_op = &zpl_export_operations;
-#ifdef HAVE_S_D_OP
- sb->s_d_op = &zpl_dentry_operations;
-#endif /* HAVE_S_D_OP */
-
- /* Set features for file system. */
- zfs_set_fuid_feature(zfsvfs);
-
- if (dmu_objset_is_snapshot(zfsvfs->z_os)) {
- uint64_t pval;
-
- atime_changed_cb(zfsvfs, B_FALSE);
- readonly_changed_cb(zfsvfs, B_TRUE);
- if ((error = dsl_prop_get_integer(osname,
- "xattr", &pval, NULL)))
- goto out;
- xattr_changed_cb(zfsvfs, pval);
- if ((error = dsl_prop_get_integer(osname,
- "acltype", &pval, NULL)))
- goto out;
- acltype_changed_cb(zfsvfs, pval);
- zfsvfs->z_issnap = B_TRUE;
- zfsvfs->z_os->os_sync = ZFS_SYNC_DISABLED;
- zfsvfs->z_snap_defer_time = jiffies;
-
- mutex_enter(&zfsvfs->z_os->os_user_ptr_lock);
- dmu_objset_set_user(zfsvfs->z_os, zfsvfs);
- mutex_exit(&zfsvfs->z_os->os_user_ptr_lock);
- } else {
- if ((error = zfsvfs_setup(zfsvfs, B_TRUE)))
- goto out;
- }
-
- /* Allocate a root inode for the filesystem. */
- error = zfs_root(zfsvfs, &root_inode);
- if (error) {
- (void) zfs_umount(sb);
- goto out;
- }
-
- /* Allocate a root dentry for the filesystem */
- sb->s_root = d_make_root(root_inode);
- if (sb->s_root == NULL) {
- (void) zfs_umount(sb);
- error = SET_ERROR(ENOMEM);
- goto out;
- }
-
- if (!zfsvfs->z_issnap)
- zfsctl_create(zfsvfs);
-
- zfsvfs->z_arc_prune = arc_add_prune_callback(zpl_prune_sb, sb);
-out:
- if (error) {
- if (zfsvfs != NULL) {
- dmu_objset_disown(zfsvfs->z_os, B_TRUE, zfsvfs);
- zfsvfs_free(zfsvfs);
- }
- /*
- * make sure we don't have dangling sb->s_fs_info which
- * zfs_preumount will use.
- */
- sb->s_fs_info = NULL;
- }
-
- return (error);
-}
-
-/*
- * Called when an unmount is requested and certain sanity checks have
- * already passed. At this point no dentries or inodes have been reclaimed
- * from their respective caches. We drop the extra reference on the .zfs
- * control directory to allow everything to be reclaimed. All snapshots
- * must already have been unmounted to reach this point.
- */
-void
-zfs_preumount(struct super_block *sb)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
-
- /* zfsvfs is NULL when zfs_domount fails during mount */
- if (zfsvfs) {
- zfs_unlinked_drain_stop_wait(zfsvfs);
- zfsctl_destroy(sb->s_fs_info);
- /*
- * Wait for iput_async before entering evict_inodes in
- * generic_shutdown_super. The reason we must finish before
- * evict_inodes is when lazytime is on, or when zfs_purgedir
- * calls zfs_zget, iput would bump i_count from 0 to 1. This
- * would race with the i_count check in evict_inodes. This means
- * it could destroy the inode while we are still using it.
- *
- * We wait for two passes. xattr directories in the first pass
- * may add xattr entries in zfs_purgedir, so in the second pass
- * we wait for them. We don't use taskq_wait here because it is
- * a pool wide taskq. Other mounted filesystems can constantly
- * do iput_async and there's no guarantee when taskq will be
- * empty.
- */
- taskq_wait_outstanding(dsl_pool_iput_taskq(
- dmu_objset_pool(zfsvfs->z_os)), 0);
- taskq_wait_outstanding(dsl_pool_iput_taskq(
- dmu_objset_pool(zfsvfs->z_os)), 0);
- }
-}
-
-/*
- * Called once all other unmount released tear down has occurred.
- * It is our responsibility to release any remaining infrastructure.
- */
-/*ARGSUSED*/
-int
-zfs_umount(struct super_block *sb)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
- objset_t *os;
-
- if (zfsvfs->z_arc_prune != NULL)
- arc_remove_prune_callback(zfsvfs->z_arc_prune);
- VERIFY(zfsvfs_teardown(zfsvfs, B_TRUE) == 0);
- os = zfsvfs->z_os;
- zpl_bdi_destroy(sb);
-
- /*
- * z_os will be NULL if there was an error in
- * attempting to reopen zfsvfs.
- */
- if (os != NULL) {
- /*
- * Unset the objset user_ptr.
- */
- mutex_enter(&os->os_user_ptr_lock);
- dmu_objset_set_user(os, NULL);
- mutex_exit(&os->os_user_ptr_lock);
-
- /*
- * Finally release the objset
- */
- dmu_objset_disown(os, B_TRUE, zfsvfs);
- }
-
- zfsvfs_free(zfsvfs);
- return (0);
-}
-
-int
-zfs_remount(struct super_block *sb, int *flags, zfs_mnt_t *zm)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
- vfs_t *vfsp;
- boolean_t issnap = dmu_objset_is_snapshot(zfsvfs->z_os);
- int error;
-
- if ((issnap || !spa_writeable(dmu_objset_spa(zfsvfs->z_os))) &&
- !(*flags & SB_RDONLY)) {
- *flags |= SB_RDONLY;
- return (EROFS);
- }
-
- error = zfsvfs_parse_options(zm->mnt_data, &vfsp);
- if (error)
- return (error);
-
- if (!zfs_is_readonly(zfsvfs) && (*flags & SB_RDONLY))
- txg_wait_synced(dmu_objset_pool(zfsvfs->z_os), 0);
-
- zfs_unregister_callbacks(zfsvfs);
- zfsvfs_vfs_free(zfsvfs->z_vfs);
-
- vfsp->vfs_data = zfsvfs;
- zfsvfs->z_vfs = vfsp;
- if (!issnap)
- (void) zfs_register_callbacks(vfsp);
-
- return (error);
-}
-
-int
-zfs_vget(struct super_block *sb, struct inode **ipp, fid_t *fidp)
-{
- zfsvfs_t *zfsvfs = sb->s_fs_info;
- znode_t *zp;
- uint64_t object = 0;
- uint64_t fid_gen = 0;
- uint64_t gen_mask;
- uint64_t zp_gen;
- int i, err;
-
- *ipp = NULL;
-
- if (fidp->fid_len == SHORT_FID_LEN || fidp->fid_len == LONG_FID_LEN) {
- zfid_short_t *zfid = (zfid_short_t *)fidp;
-
- for (i = 0; i < sizeof (zfid->zf_object); i++)
- object |= ((uint64_t)zfid->zf_object[i]) << (8 * i);
-
- for (i = 0; i < sizeof (zfid->zf_gen); i++)
- fid_gen |= ((uint64_t)zfid->zf_gen[i]) << (8 * i);
- } else {
- return (SET_ERROR(EINVAL));
- }
-
- /* LONG_FID_LEN means snapdirs */
- if (fidp->fid_len == LONG_FID_LEN) {
- zfid_long_t *zlfid = (zfid_long_t *)fidp;
- uint64_t objsetid = 0;
- uint64_t setgen = 0;
-
- for (i = 0; i < sizeof (zlfid->zf_setid); i++)
- objsetid |= ((uint64_t)zlfid->zf_setid[i]) << (8 * i);
-
- for (i = 0; i < sizeof (zlfid->zf_setgen); i++)
- setgen |= ((uint64_t)zlfid->zf_setgen[i]) << (8 * i);
-
- if (objsetid != ZFSCTL_INO_SNAPDIRS - object) {
- dprintf("snapdir fid: objsetid (%llu) != "
- "ZFSCTL_INO_SNAPDIRS (%llu) - object (%llu)\n",
- objsetid, ZFSCTL_INO_SNAPDIRS, object);
-
- return (SET_ERROR(EINVAL));
- }
-
- if (fid_gen > 1 || setgen != 0) {
- dprintf("snapdir fid: fid_gen (%llu) and setgen "
- "(%llu)\n", fid_gen, setgen);
- return (SET_ERROR(EINVAL));
- }
-
- return (zfsctl_snapdir_vget(sb, objsetid, fid_gen, ipp));
- }
-
- ZFS_ENTER(zfsvfs);
- /* A zero fid_gen means we are in the .zfs control directories */
- if (fid_gen == 0 &&
- (object == ZFSCTL_INO_ROOT || object == ZFSCTL_INO_SNAPDIR)) {
- *ipp = zfsvfs->z_ctldir;
- ASSERT(*ipp != NULL);
- if (object == ZFSCTL_INO_SNAPDIR) {
- VERIFY(zfsctl_root_lookup(*ipp, "snapshot", ipp,
- 0, kcred, NULL, NULL) == 0);
- } else {
- igrab(*ipp);
- }
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- gen_mask = -1ULL >> (64 - 8 * i);
-
- dprintf("getting %llu [%llu mask %llx]\n", object, fid_gen, gen_mask);
- if ((err = zfs_zget(zfsvfs, object, &zp))) {
- ZFS_EXIT(zfsvfs);
- return (err);
- }
-
- /* Don't export xattr stuff */
- if (zp->z_pflags & ZFS_XATTR) {
- iput(ZTOI(zp));
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOENT));
- }
-
- (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
- sizeof (uint64_t));
- zp_gen = zp_gen & gen_mask;
- if (zp_gen == 0)
- zp_gen = 1;
- if ((fid_gen == 0) && (zfsvfs->z_root == object))
- fid_gen = zp_gen;
- if (zp->z_unlinked || zp_gen != fid_gen) {
- dprintf("znode gen (%llu) != fid gen (%llu)\n", zp_gen,
- fid_gen);
- iput(ZTOI(zp));
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOENT));
- }
-
- *ipp = ZTOI(zp);
- if (*ipp)
- zfs_inode_update(ITOZ(*ipp));
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*
- * Block out VFS ops and close zfsvfs_t
- *
- * Note, if successful, then we return with the 'z_teardown_lock' and
- * 'z_teardown_inactive_lock' write held. We leave ownership of the underlying
- * dataset and objset intact so that they can be atomically handed off during
- * a subsequent rollback or recv operation and the resume thereafter.
- */
-int
-zfs_suspend_fs(zfsvfs_t *zfsvfs)
-{
- int error;
-
- if ((error = zfsvfs_teardown(zfsvfs, B_FALSE)) != 0)
- return (error);
-
- return (0);
-}
-
-/*
- * Rebuild SA and release VOPs. Note that ownership of the underlying dataset
- * is an invariant across any of the operations that can be performed while the
- * filesystem was suspended. Whether it succeeded or failed, the preconditions
- * are the same: the relevant objset and associated dataset are owned by
- * zfsvfs, held, and long held on entry.
- */
-int
-zfs_resume_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds)
-{
- int err, err2;
- znode_t *zp;
-
- ASSERT(RRM_WRITE_HELD(&zfsvfs->z_teardown_lock));
- ASSERT(RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock));
-
- /*
- * We already own this, so just update the objset_t, as the one we
- * had before may have been evicted.
- */
- objset_t *os;
- VERIFY3P(ds->ds_owner, ==, zfsvfs);
- VERIFY(dsl_dataset_long_held(ds));
- VERIFY0(dmu_objset_from_ds(ds, &os));
-
- err = zfsvfs_init(zfsvfs, os);
- if (err != 0)
- goto bail;
-
- VERIFY(zfsvfs_setup(zfsvfs, B_FALSE) == 0);
-
- zfs_set_fuid_feature(zfsvfs);
- zfsvfs->z_rollback_time = jiffies;
-
- /*
- * Attempt to re-establish all the active inodes with their
- * dbufs. If a zfs_rezget() fails, then we unhash the inode
- * and mark it stale. This prevents a collision if a new
- * inode/object is created which must use the same inode
- * number. The stale inode will be be released when the
- * VFS prunes the dentry holding the remaining references
- * on the stale inode.
- */
- mutex_enter(&zfsvfs->z_znodes_lock);
- for (zp = list_head(&zfsvfs->z_all_znodes); zp;
- zp = list_next(&zfsvfs->z_all_znodes, zp)) {
- err2 = zfs_rezget(zp);
- if (err2) {
- remove_inode_hash(ZTOI(zp));
- zp->z_is_stale = B_TRUE;
- }
-
- /* see comment in zfs_suspend_fs() */
- if (zp->z_suspended) {
- zfs_iput_async(ZTOI(zp));
- zp->z_suspended = B_FALSE;
- }
- }
- mutex_exit(&zfsvfs->z_znodes_lock);
-
- if (!zfs_is_readonly(zfsvfs) && !zfsvfs->z_unmounted) {
- /*
- * zfs_suspend_fs() could have interrupted freeing
- * of dnodes. We need to restart this freeing so
- * that we don't "leak" the space.
- */
- zfs_unlinked_drain(zfsvfs);
- }
-
-bail:
- if (err != 0)
- zfsvfs->z_unmounted = B_TRUE;
-
- /* release the VFS ops */
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
- rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
-
- if (err != 0) {
- /*
- * Since we couldn't setup the sa framework, try to force
- * unmount this file system.
- */
- if (zfsvfs->z_os)
- (void) zfs_umount(zfsvfs->z_sb);
- }
- return (err);
-}
-
-/*
- * Release VOPs and unmount a suspended filesystem.
- */
-int
-zfs_end_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds)
-{
- ASSERT(RRM_WRITE_HELD(&zfsvfs->z_teardown_lock));
- ASSERT(RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock));
-
- /*
- * We already own this, so just hold and rele it to update the
- * objset_t, as the one we had before may have been evicted.
- */
- objset_t *os;
- VERIFY3P(ds->ds_owner, ==, zfsvfs);
- VERIFY(dsl_dataset_long_held(ds));
- VERIFY0(dmu_objset_from_ds(ds, &os));
- zfsvfs->z_os = os;
-
- /* release the VOPs */
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
- rrm_exit(&zfsvfs->z_teardown_lock, FTAG);
-
- /*
- * Try to force unmount this file system.
- */
- (void) zfs_umount(zfsvfs->z_sb);
- zfsvfs->z_unmounted = B_TRUE;
- return (0);
-}
-
-int
-zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers)
-{
- int error;
- objset_t *os = zfsvfs->z_os;
- dmu_tx_t *tx;
-
- if (newvers < ZPL_VERSION_INITIAL || newvers > ZPL_VERSION)
- return (SET_ERROR(EINVAL));
-
- if (newvers < zfsvfs->z_version)
- return (SET_ERROR(EINVAL));
-
- if (zfs_spa_version_map(newvers) >
- spa_version(dmu_objset_spa(zfsvfs->z_os)))
- return (SET_ERROR(ENOTSUP));
-
- tx = dmu_tx_create(os);
- dmu_tx_hold_zap(tx, MASTER_NODE_OBJ, B_FALSE, ZPL_VERSION_STR);
- if (newvers >= ZPL_VERSION_SA && !zfsvfs->z_use_sa) {
- dmu_tx_hold_zap(tx, MASTER_NODE_OBJ, B_TRUE,
- ZFS_SA_ATTRS);
- dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
- }
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- return (error);
- }
-
- error = zap_update(os, MASTER_NODE_OBJ, ZPL_VERSION_STR,
- 8, 1, &newvers, tx);
-
- if (error) {
- dmu_tx_commit(tx);
- return (error);
- }
-
- if (newvers >= ZPL_VERSION_SA && !zfsvfs->z_use_sa) {
- uint64_t sa_obj;
-
- ASSERT3U(spa_version(dmu_objset_spa(zfsvfs->z_os)), >=,
- SPA_VERSION_SA);
- sa_obj = zap_create(os, DMU_OT_SA_MASTER_NODE,
- DMU_OT_NONE, 0, tx);
-
- error = zap_add(os, MASTER_NODE_OBJ,
- ZFS_SA_ATTRS, 8, 1, &sa_obj, tx);
- ASSERT0(error);
-
- VERIFY(0 == sa_set_sa_object(os, sa_obj));
- sa_register_update_callback(os, zfs_sa_upgrade);
- }
-
- spa_history_log_internal_ds(dmu_objset_ds(os), "upgrade", tx,
- "from %llu to %llu", zfsvfs->z_version, newvers);
-
- dmu_tx_commit(tx);
-
- zfsvfs->z_version = newvers;
- os->os_version = newvers;
-
- zfs_set_fuid_feature(zfsvfs);
-
- return (0);
-}
-
-/*
- * Read a property stored within the master node.
- */
-int
-zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value)
-{
- uint64_t *cached_copy = NULL;
-
- /*
- * Figure out where in the objset_t the cached copy would live, if it
- * is available for the requested property.
- */
- if (os != NULL) {
- switch (prop) {
- case ZFS_PROP_VERSION:
- cached_copy = &os->os_version;
- break;
- case ZFS_PROP_NORMALIZE:
- cached_copy = &os->os_normalization;
- break;
- case ZFS_PROP_UTF8ONLY:
- cached_copy = &os->os_utf8only;
- break;
- case ZFS_PROP_CASE:
- cached_copy = &os->os_casesensitivity;
- break;
- default:
- break;
- }
- }
- if (cached_copy != NULL && *cached_copy != OBJSET_PROP_UNINITIALIZED) {
- *value = *cached_copy;
- return (0);
- }
-
- /*
- * If the property wasn't cached, look up the file system's value for
- * the property. For the version property, we look up a slightly
- * different string.
- */
- const char *pname;
- int error = ENOENT;
- if (prop == ZFS_PROP_VERSION)
- pname = ZPL_VERSION_STR;
- else
- pname = zfs_prop_to_name(prop);
-
- if (os != NULL) {
- ASSERT3U(os->os_phys->os_type, ==, DMU_OST_ZFS);
- error = zap_lookup(os, MASTER_NODE_OBJ, pname, 8, 1, value);
- }
-
- if (error == ENOENT) {
- /* No value set, use the default value */
- switch (prop) {
- case ZFS_PROP_VERSION:
- *value = ZPL_VERSION;
- break;
- case ZFS_PROP_NORMALIZE:
- case ZFS_PROP_UTF8ONLY:
- *value = 0;
- break;
- case ZFS_PROP_CASE:
- *value = ZFS_CASE_SENSITIVE;
- break;
- case ZFS_PROP_ACLTYPE:
- *value = ZFS_ACLTYPE_OFF;
- break;
- default:
- return (error);
- }
- error = 0;
- }
-
- /*
- * If one of the methods for getting the property value above worked,
- * copy it into the objset_t's cache.
- */
- if (error == 0 && cached_copy != NULL) {
- *cached_copy = *value;
- }
-
- return (error);
-}
-
-/*
- * Return true if the corresponding vfs's unmounted flag is set.
- * Otherwise return false.
- * If this function returns true we know VFS unmount has been initiated.
- */
-boolean_t
-zfs_get_vfs_flag_unmounted(objset_t *os)
-{
- zfsvfs_t *zfvp;
- boolean_t unmounted = B_FALSE;
-
- ASSERT(dmu_objset_type(os) == DMU_OST_ZFS);
-
- mutex_enter(&os->os_user_ptr_lock);
- zfvp = dmu_objset_get_user(os);
- if (zfvp != NULL && zfvp->z_unmounted)
- unmounted = B_TRUE;
- mutex_exit(&os->os_user_ptr_lock);
-
- return (unmounted);
-}
-
-struct objnode {
- avl_node_t node;
- uint64_t obj;
-};
-
-static int
-objnode_compare(const void *o1, const void *o2)
-{
- const struct objnode *obj1 = o1;
- const struct objnode *obj2 = o2;
- if (obj1->obj < obj2->obj)
- return (-1);
- if (obj1->obj > obj2->obj)
- return (1);
- return (0);
-}
-
-objlist_t *
-zfs_get_deleteq(objset_t *os)
-{
- objlist_t *deleteq_objlist = objlist_create();
- uint64_t deleteq_obj;
- zap_cursor_t zc;
- zap_attribute_t za;
- dmu_object_info_t doi;
-
- ASSERT3U(os->os_phys->os_type, ==, DMU_OST_ZFS);
- VERIFY0(dmu_object_info(os, MASTER_NODE_OBJ, &doi));
- ASSERT3U(doi.doi_type, ==, DMU_OT_MASTER_NODE);
-
- VERIFY0(zap_lookup(os, MASTER_NODE_OBJ,
- ZFS_UNLINKED_SET, sizeof (uint64_t), 1, &deleteq_obj));
-
- /*
- * In order to insert objects into the objlist, they must be in sorted
- * order. We don't know what order we'll get them out of the ZAP in, so
- * we insert them into and remove them from an avl_tree_t to sort them.
- */
- avl_tree_t at;
- avl_create(&at, objnode_compare, sizeof (struct objnode),
- offsetof(struct objnode, node));
-
- for (zap_cursor_init(&zc, os, deleteq_obj);
- zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) {
- struct objnode *obj = kmem_zalloc(sizeof (*obj), KM_SLEEP);
- obj->obj = za.za_first_integer;
- avl_add(&at, obj);
- }
- zap_cursor_fini(&zc);
-
- struct objnode *next, *found = avl_first(&at);
- while (found != NULL) {
- next = AVL_NEXT(&at, found);
- objlist_insert(deleteq_objlist, found->obj);
- found = next;
- }
-
- void *cookie = NULL;
- while ((found = avl_destroy_nodes(&at, &cookie)) != NULL)
- kmem_free(found, sizeof (*found));
- avl_destroy(&at);
- return (deleteq_objlist);
-}
-
-
-void
-zfs_init(void)
-{
- zfsctl_init();
- zfs_znode_init();
- dmu_objset_register_type(DMU_OST_ZFS, zfs_space_delta_cb);
- register_filesystem(&zpl_fs_type);
-}
-
-void
-zfs_fini(void)
-{
- /*
- * we don't use outstanding because zpl_posix_acl_free might add more.
- */
- taskq_wait(system_delay_taskq);
- taskq_wait(system_taskq);
- unregister_filesystem(&zpl_fs_type);
- zfs_znode_fini();
- zfsctl_fini();
-}
-
-#if defined(_KERNEL)
-EXPORT_SYMBOL(zfs_suspend_fs);
-EXPORT_SYMBOL(zfs_resume_fs);
-EXPORT_SYMBOL(zfs_userspace_one);
-EXPORT_SYMBOL(zfs_userspace_many);
-EXPORT_SYMBOL(zfs_set_userquota);
-EXPORT_SYMBOL(zfs_id_overblockquota);
-EXPORT_SYMBOL(zfs_id_overobjquota);
-EXPORT_SYMBOL(zfs_id_overquota);
-EXPORT_SYMBOL(zfs_set_version);
-EXPORT_SYMBOL(zfsvfs_create);
-EXPORT_SYMBOL(zfsvfs_free);
-EXPORT_SYMBOL(zfs_is_readonly);
-EXPORT_SYMBOL(zfs_domount);
-EXPORT_SYMBOL(zfs_preumount);
-EXPORT_SYMBOL(zfs_umount);
-EXPORT_SYMBOL(zfs_remount);
-EXPORT_SYMBOL(zfs_statvfs);
-EXPORT_SYMBOL(zfs_vget);
-EXPORT_SYMBOL(zfs_prune);
-#endif
diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c
deleted file mode 100644
index de7b59935..000000000
--- a/module/zfs/zfs_vnops.c
+++ /dev/null
@@ -1,5275 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
- * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
- * Copyright 2017 Nexenta Systems, Inc.
- */
-
-/* Portions Copyright 2007 Jeremy Teo */
-/* Portions Copyright 2010 Robert Milkowski */
-
-
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysmacros.h>
-#include <sys/vfs.h>
-#include <sys/file.h>
-#include <sys/stat.h>
-#include <sys/kmem.h>
-#include <sys/taskq.h>
-#include <sys/uio.h>
-#include <sys/vmsystm.h>
-#include <sys/atomic.h>
-#include <sys/pathname.h>
-#include <sys/cmn_err.h>
-#include <sys/errno.h>
-#include <sys/zfs_dir.h>
-#include <sys/zfs_acl.h>
-#include <sys/zfs_ioctl.h>
-#include <sys/fs/zfs.h>
-#include <sys/dmu.h>
-#include <sys/dmu_objset.h>
-#include <sys/spa.h>
-#include <sys/txg.h>
-#include <sys/dbuf.h>
-#include <sys/zap.h>
-#include <sys/sa.h>
-#include <sys/policy.h>
-#include <sys/sunddi.h>
-#include <sys/sid.h>
-#include <sys/mode.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zfs_fuid.h>
-#include <sys/zfs_sa.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_rlock.h>
-#include <sys/cred.h>
-#include <sys/zpl.h>
-#include <sys/zil.h>
-#include <sys/sa_impl.h>
-
-/*
- * Programming rules.
- *
- * Each vnode op performs some logical unit of work. To do this, the ZPL must
- * properly lock its in-core state, create a DMU transaction, do the work,
- * record this work in the intent log (ZIL), commit the DMU transaction,
- * and wait for the intent log to commit if it is a synchronous operation.
- * Moreover, the vnode ops must work in both normal and log replay context.
- * The ordering of events is important to avoid deadlocks and references
- * to freed memory. The example below illustrates the following Big Rules:
- *
- * (1) A check must be made in each zfs thread for a mounted file system.
- * This is done avoiding races using ZFS_ENTER(zfsvfs).
- * A ZFS_EXIT(zfsvfs) is needed before all returns. Any znodes
- * must be checked with ZFS_VERIFY_ZP(zp). Both of these macros
- * can return EIO from the calling function.
- *
- * (2) iput() should always be the last thing except for zil_commit()
- * (if necessary) and ZFS_EXIT(). This is for 3 reasons:
- * First, if it's the last reference, the vnode/znode
- * can be freed, so the zp may point to freed memory. Second, the last
- * reference will call zfs_zinactive(), which may induce a lot of work --
- * pushing cached pages (which acquires range locks) and syncing out
- * cached atime changes. Third, zfs_zinactive() may require a new tx,
- * which could deadlock the system if you were already holding one.
- * If you must call iput() within a tx then use zfs_iput_async().
- *
- * (3) All range locks must be grabbed before calling dmu_tx_assign(),
- * as they can span dmu_tx_assign() calls.
- *
- * (4) If ZPL locks are held, pass TXG_NOWAIT as the second argument to
- * dmu_tx_assign(). This is critical because we don't want to block
- * while holding locks.
- *
- * If no ZPL locks are held (aside from ZFS_ENTER()), use TXG_WAIT. This
- * reduces lock contention and CPU usage when we must wait (note that if
- * throughput is constrained by the storage, nearly every transaction
- * must wait).
- *
- * Note, in particular, that if a lock is sometimes acquired before
- * the tx assigns, and sometimes after (e.g. z_lock), then failing
- * to use a non-blocking assign can deadlock the system. The scenario:
- *
- * Thread A has grabbed a lock before calling dmu_tx_assign().
- * Thread B is in an already-assigned tx, and blocks for this lock.
- * Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open()
- * forever, because the previous txg can't quiesce until B's tx commits.
- *
- * If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT,
- * then drop all locks, call dmu_tx_wait(), and try again. On subsequent
- * calls to dmu_tx_assign(), pass TXG_NOTHROTTLE in addition to TXG_NOWAIT,
- * to indicate that this operation has already called dmu_tx_wait().
- * This will ensure that we don't retry forever, waiting a short bit
- * each time.
- *
- * (5) If the operation succeeded, generate the intent log entry for it
- * before dropping locks. This ensures that the ordering of events
- * in the intent log matches the order in which they actually occurred.
- * During ZIL replay the zfs_log_* functions will update the sequence
- * number to indicate the zil transaction has replayed.
- *
- * (6) At the end of each vnode op, the DMU tx must always commit,
- * regardless of whether there were any errors.
- *
- * (7) After dropping all locks, invoke zil_commit(zilog, foid)
- * to ensure that synchronous semantics are provided when necessary.
- *
- * In general, this is how things should be ordered in each vnode op:
- *
- * ZFS_ENTER(zfsvfs); // exit if unmounted
- * top:
- * zfs_dirent_lock(&dl, ...) // lock directory entry (may igrab())
- * rw_enter(...); // grab any other locks you need
- * tx = dmu_tx_create(...); // get DMU tx
- * dmu_tx_hold_*(); // hold each object you might modify
- * error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- * if (error) {
- * rw_exit(...); // drop locks
- * zfs_dirent_unlock(dl); // unlock directory entry
- * iput(...); // release held vnodes
- * if (error == ERESTART) {
- * waited = B_TRUE;
- * dmu_tx_wait(tx);
- * dmu_tx_abort(tx);
- * goto top;
- * }
- * dmu_tx_abort(tx); // abort DMU tx
- * ZFS_EXIT(zfsvfs); // finished in zfs
- * return (error); // really out of space
- * }
- * error = do_real_work(); // do whatever this VOP does
- * if (error == 0)
- * zfs_log_*(...); // on success, make ZIL entry
- * dmu_tx_commit(tx); // commit DMU tx -- error or not
- * rw_exit(...); // drop locks
- * zfs_dirent_unlock(dl); // unlock directory entry
- * iput(...); // release held vnodes
- * zil_commit(zilog, foid); // synchronous when necessary
- * ZFS_EXIT(zfsvfs); // finished in zfs
- * return (error); // done, report error
- */
-
-/*
- * Virus scanning is unsupported. It would be possible to add a hook
- * here to performance the required virus scan. This could be done
- * entirely in the kernel or potentially as an update to invoke a
- * scanning utility.
- */
-static int
-zfs_vscan(struct inode *ip, cred_t *cr, int async)
-{
- return (0);
-}
-
-/* ARGSUSED */
-int
-zfs_open(struct inode *ip, int mode, int flag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- /* Honor ZFS_APPENDONLY file attribute */
- if ((mode & FMODE_WRITE) && (zp->z_pflags & ZFS_APPENDONLY) &&
- ((flag & O_APPEND) == 0)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- /* Virus scan eligible files on open */
- if (!zfs_has_ctldir(zp) && zfsvfs->z_vscan && S_ISREG(ip->i_mode) &&
- !(zp->z_pflags & ZFS_AV_QUARANTINED) && zp->z_size > 0) {
- if (zfs_vscan(ip, cr, 0) != 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EACCES));
- }
- }
-
- /* Keep a count of the synchronous opens in the znode */
- if (flag & O_SYNC)
- atomic_inc_32(&zp->z_sync_cnt);
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/* ARGSUSED */
-int
-zfs_close(struct inode *ip, int flag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- /* Decrement the synchronous opens in the znode */
- if (flag & O_SYNC)
- atomic_dec_32(&zp->z_sync_cnt);
-
- if (!zfs_has_ctldir(zp) && zfsvfs->z_vscan && S_ISREG(ip->i_mode) &&
- !(zp->z_pflags & ZFS_AV_QUARANTINED) && zp->z_size > 0)
- VERIFY(zfs_vscan(ip, cr, 1) == 0);
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-#if defined(SEEK_HOLE) && defined(SEEK_DATA)
-/*
- * Lseek support for finding holes (cmd == SEEK_HOLE) and
- * data (cmd == SEEK_DATA). "off" is an in/out parameter.
- */
-static int
-zfs_holey_common(struct inode *ip, int cmd, loff_t *off)
-{
- znode_t *zp = ITOZ(ip);
- uint64_t noff = (uint64_t)*off; /* new offset */
- uint64_t file_sz;
- int error;
- boolean_t hole;
-
- file_sz = zp->z_size;
- if (noff >= file_sz) {
- return (SET_ERROR(ENXIO));
- }
-
- if (cmd == SEEK_HOLE)
- hole = B_TRUE;
- else
- hole = B_FALSE;
-
- error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
-
- if (error == ESRCH)
- return (SET_ERROR(ENXIO));
-
- /* file was dirty, so fall back to using generic logic */
- if (error == EBUSY) {
- if (hole)
- *off = file_sz;
-
- return (0);
- }
-
- /*
- * We could find a hole that begins after the logical end-of-file,
- * because dmu_offset_next() only works on whole blocks. If the
- * EOF falls mid-block, then indicate that the "virtual hole"
- * at the end of the file begins at the logical EOF, rather than
- * at the end of the last block.
- */
- if (noff > file_sz) {
- ASSERT(hole);
- noff = file_sz;
- }
-
- if (noff < *off)
- return (error);
- *off = noff;
- return (error);
-}
-
-int
-zfs_holey(struct inode *ip, int cmd, loff_t *off)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- error = zfs_holey_common(ip, cmd, off);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-#endif /* SEEK_HOLE && SEEK_DATA */
-
-#if defined(_KERNEL)
-/*
- * When a file is memory mapped, we must keep the IO data synchronized
- * between the DMU cache and the memory mapped pages. What this means:
- *
- * On Write: If we find a memory mapped page, we write to *both*
- * the page and the dmu buffer.
- */
-static void
-update_pages(struct inode *ip, int64_t start, int len,
- objset_t *os, uint64_t oid)
-{
- struct address_space *mp = ip->i_mapping;
- struct page *pp;
- uint64_t nbytes;
- int64_t off;
- void *pb;
-
- off = start & (PAGE_SIZE-1);
- for (start &= PAGE_MASK; len > 0; start += PAGE_SIZE) {
- nbytes = MIN(PAGE_SIZE - off, len);
-
- pp = find_lock_page(mp, start >> PAGE_SHIFT);
- if (pp) {
- if (mapping_writably_mapped(mp))
- flush_dcache_page(pp);
-
- pb = kmap(pp);
- (void) dmu_read(os, oid, start+off, nbytes, pb+off,
- DMU_READ_PREFETCH);
- kunmap(pp);
-
- if (mapping_writably_mapped(mp))
- flush_dcache_page(pp);
-
- mark_page_accessed(pp);
- SetPageUptodate(pp);
- ClearPageError(pp);
- unlock_page(pp);
- put_page(pp);
- }
-
- len -= nbytes;
- off = 0;
- }
-}
-
-/*
- * When a file is memory mapped, we must keep the IO data synchronized
- * between the DMU cache and the memory mapped pages. What this means:
- *
- * On Read: We "read" preferentially from memory mapped pages,
- * else we default from the dmu buffer.
- *
- * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
- * the file is memory mapped.
- */
-static int
-mappedread(struct inode *ip, int nbytes, uio_t *uio)
-{
- struct address_space *mp = ip->i_mapping;
- struct page *pp;
- znode_t *zp = ITOZ(ip);
- int64_t start, off;
- uint64_t bytes;
- int len = nbytes;
- int error = 0;
- void *pb;
-
- start = uio->uio_loffset;
- off = start & (PAGE_SIZE-1);
- for (start &= PAGE_MASK; len > 0; start += PAGE_SIZE) {
- bytes = MIN(PAGE_SIZE - off, len);
-
- pp = find_lock_page(mp, start >> PAGE_SHIFT);
- if (pp) {
- ASSERT(PageUptodate(pp));
- unlock_page(pp);
-
- pb = kmap(pp);
- error = uiomove(pb + off, bytes, UIO_READ, uio);
- kunmap(pp);
-
- if (mapping_writably_mapped(mp))
- flush_dcache_page(pp);
-
- mark_page_accessed(pp);
- put_page(pp);
- } else {
- error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
- uio, bytes);
- }
-
- len -= bytes;
- off = 0;
- if (error)
- break;
- }
- return (error);
-}
-#endif /* _KERNEL */
-
-unsigned long zfs_read_chunk_size = 1024 * 1024; /* Tunable */
-unsigned long zfs_delete_blocks = DMU_MAX_DELETEBLKCNT;
-
-/*
- * Read bytes from specified file into supplied buffer.
- *
- * IN: ip - inode of file to be read from.
- * uio - structure supplying read location, range info,
- * and return buffer.
- * ioflag - FSYNC flags; used to provide FRSYNC semantics.
- * O_DIRECT flag; used to bypass page cache.
- * cr - credentials of caller.
- *
- * OUT: uio - updated offset and range, buffer filled.
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Side Effects:
- * inode - atime updated if byte count > 0
- */
-/* ARGSUSED */
-int
-zfs_read(struct inode *ip, uio_t *uio, int ioflag, cred_t *cr)
-{
- int error = 0;
- boolean_t frsync = B_FALSE;
-
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if (zp->z_pflags & ZFS_AV_QUARANTINED) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EACCES));
- }
-
- /*
- * Validate file offset
- */
- if (uio->uio_loffset < (offset_t)0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Fasttrack empty reads
- */
- if (uio->uio_resid == 0) {
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
-#ifdef FRSYNC
- /*
- * If we're in FRSYNC mode, sync out this znode before reading it.
- * Only do this for non-snapshots.
- *
- * Some platforms do not support FRSYNC and instead map it
- * to FSYNC, which results in unnecessary calls to zil_commit. We
- * only honor FRSYNC requests on platforms which support it.
- */
- frsync = !!(ioflag & FRSYNC);
-#endif
- if (zfsvfs->z_log &&
- (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
- zil_commit(zfsvfs->z_log, zp->z_id);
-
- /*
- * Lock the range against changes.
- */
- locked_range_t *lr = rangelock_enter(&zp->z_rangelock,
- uio->uio_loffset, uio->uio_resid, RL_READER);
-
- /*
- * If we are reading past end-of-file we can skip
- * to the end; but we might still need to set atime.
- */
- if (uio->uio_loffset >= zp->z_size) {
- error = 0;
- goto out;
- }
-
- ASSERT(uio->uio_loffset < zp->z_size);
- ssize_t n = MIN(uio->uio_resid, zp->z_size - uio->uio_loffset);
- ssize_t start_resid = n;
-
-#ifdef HAVE_UIO_ZEROCOPY
- xuio_t *xuio = NULL;
- if ((uio->uio_extflg == UIO_XUIO) &&
- (((xuio_t *)uio)->xu_type == UIOTYPE_ZEROCOPY)) {
- int nblk;
- int blksz = zp->z_blksz;
- uint64_t offset = uio->uio_loffset;
-
- xuio = (xuio_t *)uio;
- if ((ISP2(blksz))) {
- nblk = (P2ROUNDUP(offset + n, blksz) - P2ALIGN(offset,
- blksz)) / blksz;
- } else {
- ASSERT(offset + n <= blksz);
- nblk = 1;
- }
- (void) dmu_xuio_init(xuio, nblk);
-
- if (vn_has_cached_data(ip)) {
- /*
- * For simplicity, we always allocate a full buffer
- * even if we only expect to read a portion of a block.
- */
- while (--nblk >= 0) {
- (void) dmu_xuio_add(xuio,
- dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
- blksz), 0, blksz);
- }
- }
- }
-#endif /* HAVE_UIO_ZEROCOPY */
-
- while (n > 0) {
- ssize_t nbytes = MIN(n, zfs_read_chunk_size -
- P2PHASE(uio->uio_loffset, zfs_read_chunk_size));
-
- if (zp->z_is_mapped && !(ioflag & O_DIRECT)) {
- error = mappedread(ip, nbytes, uio);
- } else {
- error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
- uio, nbytes);
- }
-
- if (error) {
- /* convert checksum errors into IO errors */
- if (error == ECKSUM)
- error = SET_ERROR(EIO);
- break;
- }
-
- n -= nbytes;
- }
-
- int64_t nread = start_resid - n;
- dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
- task_io_account_read(nread);
-out:
- rangelock_exit(lr);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Write the bytes to a file.
- *
- * IN: ip - inode of file to be written to.
- * uio - structure supplying write location, range info,
- * and data buffer.
- * ioflag - FAPPEND flag set if in append mode.
- * O_DIRECT flag; used to bypass page cache.
- * cr - credentials of caller.
- *
- * OUT: uio - updated offset and range.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * ip - ctime|mtime updated if byte count > 0
- */
-
-/* ARGSUSED */
-int
-zfs_write(struct inode *ip, uio_t *uio, int ioflag, cred_t *cr)
-{
- int error = 0;
- ssize_t start_resid = uio->uio_resid;
-
- /*
- * Fasttrack empty write
- */
- ssize_t n = start_resid;
- if (n == 0)
- return (0);
-
- rlim64_t limit = uio->uio_limit;
- if (limit == RLIM64_INFINITY || limit > MAXOFFSET_T)
- limit = MAXOFFSET_T;
-
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- sa_bulk_attr_t bulk[4];
- int count = 0;
- uint64_t mtime[2], ctime[2];
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
- &zp->z_size, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, 8);
-
- /*
- * Callers might not be able to detect properly that we are read-only,
- * so check it explicitly here.
- */
- if (zfs_is_readonly(zfsvfs)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EROFS));
- }
-
- /*
- * If immutable or not appending then return EPERM
- */
- if ((zp->z_pflags & (ZFS_IMMUTABLE | ZFS_READONLY)) ||
- ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & FAPPEND) &&
- (uio->uio_loffset < zp->z_size))) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- /*
- * Validate file offset
- */
- offset_t woff = ioflag & FAPPEND ? zp->z_size : uio->uio_loffset;
- if (woff < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- int max_blksz = zfsvfs->z_max_blksz;
- xuio_t *xuio = NULL;
-
- /*
- * Pre-fault the pages to ensure slow (eg NFS) pages
- * don't hold up txg.
- * Skip this if uio contains loaned arc_buf.
- */
-#ifdef HAVE_UIO_ZEROCOPY
- if ((uio->uio_extflg == UIO_XUIO) &&
- (((xuio_t *)uio)->xu_type == UIOTYPE_ZEROCOPY))
- xuio = (xuio_t *)uio;
- else
-#endif
- if (uio_prefaultpages(MIN(n, max_blksz), uio)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EFAULT));
- }
-
- /*
- * If in append mode, set the io offset pointer to eof.
- */
- locked_range_t *lr;
- if (ioflag & FAPPEND) {
- /*
- * Obtain an appending range lock to guarantee file append
- * semantics. We reset the write offset once we have the lock.
- */
- lr = rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
- woff = lr->lr_offset;
- if (lr->lr_length == UINT64_MAX) {
- /*
- * We overlocked the file because this write will cause
- * the file block size to increase.
- * Note that zp_size cannot change with this lock held.
- */
- woff = zp->z_size;
- }
- uio->uio_loffset = woff;
- } else {
- /*
- * Note that if the file block size will change as a result of
- * this write, then this range lock will lock the entire file
- * so that we can re-write the block safely.
- */
- lr = rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
- }
-
- if (woff >= limit) {
- rangelock_exit(lr);
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EFBIG));
- }
-
- if ((woff + n) > limit || woff > (limit - n))
- n = limit - woff;
-
- /* Will this write extend the file length? */
- int write_eof = (woff + n > zp->z_size);
-
- uint64_t end_size = MAX(zp->z_size, woff + n);
- zilog_t *zilog = zfsvfs->z_log;
-#ifdef HAVE_UIO_ZEROCOPY
- int i_iov = 0;
- const iovec_t *iovp = uio->uio_iov;
- ASSERTV(int iovcnt = uio->uio_iovcnt);
-#endif
-
-
- /*
- * Write the file in reasonable size chunks. Each chunk is written
- * in a separate transaction; this keeps the intent log records small
- * and allows us to do more fine-grained space accounting.
- */
- while (n > 0) {
- woff = uio->uio_loffset;
-
- if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT,
- KUID_TO_SUID(ip->i_uid)) ||
- zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT,
- KGID_TO_SGID(ip->i_gid)) ||
- (zp->z_projid != ZFS_DEFAULT_PROJID &&
- zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
- zp->z_projid))) {
- error = SET_ERROR(EDQUOT);
- break;
- }
-
- arc_buf_t *abuf = NULL;
- const iovec_t *aiov = NULL;
- if (xuio) {
-#ifdef HAVE_UIO_ZEROCOPY
- ASSERT(i_iov < iovcnt);
- ASSERT3U(uio->uio_segflg, !=, UIO_BVEC);
- aiov = &iovp[i_iov];
- abuf = dmu_xuio_arcbuf(xuio, i_iov);
- dmu_xuio_clear(xuio, i_iov);
- ASSERT((aiov->iov_base == abuf->b_data) ||
- ((char *)aiov->iov_base - (char *)abuf->b_data +
- aiov->iov_len == arc_buf_size(abuf)));
- i_iov++;
-#endif
- } else if (n >= max_blksz && woff >= zp->z_size &&
- P2PHASE(woff, max_blksz) == 0 &&
- zp->z_blksz == max_blksz) {
- /*
- * This write covers a full block. "Borrow" a buffer
- * from the dmu so that we can fill it before we enter
- * a transaction. This avoids the possibility of
- * holding up the transaction if the data copy hangs
- * up on a pagefault (e.g., from an NFS server mapping).
- */
- size_t cbytes;
-
- abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
- max_blksz);
- ASSERT(abuf != NULL);
- ASSERT(arc_buf_size(abuf) == max_blksz);
- if ((error = uiocopy(abuf->b_data, max_blksz,
- UIO_WRITE, uio, &cbytes))) {
- dmu_return_arcbuf(abuf);
- break;
- }
- ASSERT(cbytes == max_blksz);
- }
-
- /*
- * Start a transaction.
- */
- dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
- DB_DNODE_ENTER(db);
- dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff,
- MIN(n, max_blksz));
- DB_DNODE_EXIT(db);
- zfs_sa_upgrade_txholds(tx, zp);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- if (abuf != NULL)
- dmu_return_arcbuf(abuf);
- break;
- }
-
- /*
- * If rangelock_enter() over-locked we grow the blocksize
- * and then reduce the lock range. This will only happen
- * on the first iteration since rangelock_reduce() will
- * shrink down lr_length to the appropriate size.
- */
- if (lr->lr_length == UINT64_MAX) {
- uint64_t new_blksz;
-
- if (zp->z_blksz > max_blksz) {
- /*
- * File's blocksize is already larger than the
- * "recordsize" property. Only let it grow to
- * the next power of 2.
- */
- ASSERT(!ISP2(zp->z_blksz));
- new_blksz = MIN(end_size,
- 1 << highbit64(zp->z_blksz));
- } else {
- new_blksz = MIN(end_size, max_blksz);
- }
- zfs_grow_blocksize(zp, new_blksz, tx);
- rangelock_reduce(lr, woff, n);
- }
-
- /*
- * XXX - should we really limit each write to z_max_blksz?
- * Perhaps we should use SPA_MAXBLOCKSIZE chunks?
- */
- ssize_t nbytes = MIN(n, max_blksz - P2PHASE(woff, max_blksz));
-
- ssize_t tx_bytes;
- if (abuf == NULL) {
- tx_bytes = uio->uio_resid;
- uio->uio_fault_disable = B_TRUE;
- error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
- uio, nbytes, tx);
- uio->uio_fault_disable = B_FALSE;
- if (error == EFAULT) {
- dmu_tx_commit(tx);
- if (uio_prefaultpages(MIN(n, max_blksz), uio)) {
- break;
- }
- continue;
- } else if (error != 0) {
- dmu_tx_commit(tx);
- break;
- }
- tx_bytes -= uio->uio_resid;
- } else {
- tx_bytes = nbytes;
- ASSERT(xuio == NULL || tx_bytes == aiov->iov_len);
- /*
- * If this is not a full block write, but we are
- * extending the file past EOF and this data starts
- * block-aligned, use assign_arcbuf(). Otherwise,
- * write via dmu_write().
- */
- if (tx_bytes < max_blksz && (!write_eof ||
- aiov->iov_base != abuf->b_data)) {
- ASSERT(xuio);
- dmu_write(zfsvfs->z_os, zp->z_id, woff,
- /* cppcheck-suppress nullPointer */
- aiov->iov_len, aiov->iov_base, tx);
- dmu_return_arcbuf(abuf);
- xuio_stat_wbuf_copied();
- } else {
- ASSERT(xuio || tx_bytes == max_blksz);
- error = dmu_assign_arcbuf_by_dbuf(
- sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
- if (error != 0) {
- dmu_return_arcbuf(abuf);
- dmu_tx_commit(tx);
- break;
- }
- }
- ASSERT(tx_bytes <= uio->uio_resid);
- uioskip(uio, tx_bytes);
- }
- if (tx_bytes && zp->z_is_mapped && !(ioflag & O_DIRECT)) {
- update_pages(ip, woff,
- tx_bytes, zfsvfs->z_os, zp->z_id);
- }
-
- /*
- * If we made no progress, we're done. If we made even
- * partial progress, update the znode and ZIL accordingly.
- */
- if (tx_bytes == 0) {
- (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
- (void *)&zp->z_size, sizeof (uint64_t), tx);
- dmu_tx_commit(tx);
- ASSERT(error != 0);
- break;
- }
-
- /*
- * Clear Set-UID/Set-GID bits on successful write if not
- * privileged and at least one of the execute bits is set.
- *
- * It would be nice to do this after all writes have
- * been done, but that would still expose the ISUID/ISGID
- * to another app after the partial write is committed.
- *
- * Note: we don't call zfs_fuid_map_id() here because
- * user 0 is not an ephemeral uid.
- */
- mutex_enter(&zp->z_acl_lock);
- uint32_t uid = KUID_TO_SUID(ip->i_uid);
- if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) |
- (S_IXUSR >> 6))) != 0 &&
- (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
- secpolicy_vnode_setid_retain(cr,
- ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
- uint64_t newmode;
- zp->z_mode &= ~(S_ISUID | S_ISGID);
- ip->i_mode = newmode = zp->z_mode;
- (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
- (void *)&newmode, sizeof (uint64_t), tx);
- }
- mutex_exit(&zp->z_acl_lock);
-
- zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
-
- /*
- * Update the file size (zp_size) if it has changed;
- * account for possible concurrent updates.
- */
- while ((end_size = zp->z_size) < uio->uio_loffset) {
- (void) atomic_cas_64(&zp->z_size, end_size,
- uio->uio_loffset);
- ASSERT(error == 0);
- }
- /*
- * If we are replaying and eof is non zero then force
- * the file size to the specified eof. Note, there's no
- * concurrency during replay.
- */
- if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
- zp->z_size = zfsvfs->z_replay_eof;
-
- error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
-
- zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag,
- NULL, NULL);
- dmu_tx_commit(tx);
-
- if (error != 0)
- break;
- ASSERT(tx_bytes == nbytes);
- n -= nbytes;
-
- if (!xuio && n > 0) {
- if (uio_prefaultpages(MIN(n, max_blksz), uio)) {
- error = EFAULT;
- break;
- }
- }
- }
-
- zfs_inode_update(zp);
- rangelock_exit(lr);
-
- /*
- * If we're in replay mode, or we made no progress, return error.
- * Otherwise, it's at least a partial write, so it's successful.
- */
- if (zfsvfs->z_replay || uio->uio_resid == start_resid) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (ioflag & (FSYNC | FDSYNC) ||
- zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, zp->z_id);
-
- int64_t nwritten = start_resid - uio->uio_resid;
- dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
- task_io_account_write(nwritten);
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*
- * Drop a reference on the passed inode asynchronously. This ensures
- * that the caller will never drop the last reference on an inode in
- * the current context. Doing so while holding open a tx could result
- * in a deadlock if iput_final() re-enters the filesystem code.
- */
-void
-zfs_iput_async(struct inode *ip)
-{
- objset_t *os = ITOZSB(ip)->z_os;
-
- ASSERT(atomic_read(&ip->i_count) > 0);
- ASSERT(os != NULL);
-
- if (atomic_read(&ip->i_count) == 1)
- VERIFY(taskq_dispatch(dsl_pool_iput_taskq(dmu_objset_pool(os)),
- (task_func_t *)iput, ip, TQ_SLEEP) != TASKQID_INVALID);
- else
- iput(ip);
-}
-
-/* ARGSUSED */
-void
-zfs_get_done(zgd_t *zgd, int error)
-{
- znode_t *zp = zgd->zgd_private;
-
- if (zgd->zgd_db)
- dmu_buf_rele(zgd->zgd_db, zgd);
-
- rangelock_exit(zgd->zgd_lr);
-
- /*
- * Release the vnode asynchronously as we currently have the
- * txg stopped from syncing.
- */
- zfs_iput_async(ZTOI(zp));
-
- kmem_free(zgd, sizeof (zgd_t));
-}
-
-#ifdef DEBUG
-static int zil_fault_io = 0;
-#endif
-
-/*
- * Get data to generate a TX_WRITE intent log record.
- */
-int
-zfs_get_data(void *arg, lr_write_t *lr, char *buf, struct lwb *lwb, zio_t *zio)
-{
- zfsvfs_t *zfsvfs = arg;
- objset_t *os = zfsvfs->z_os;
- znode_t *zp;
- uint64_t object = lr->lr_foid;
- uint64_t offset = lr->lr_offset;
- uint64_t size = lr->lr_length;
- dmu_buf_t *db;
- zgd_t *zgd;
- int error = 0;
-
- ASSERT3P(lwb, !=, NULL);
- ASSERT3P(zio, !=, NULL);
- ASSERT3U(size, !=, 0);
-
- /*
- * Nothing to do if the file has been removed
- */
- if (zfs_zget(zfsvfs, object, &zp) != 0)
- return (SET_ERROR(ENOENT));
- if (zp->z_unlinked) {
- /*
- * Release the vnode asynchronously as we currently have the
- * txg stopped from syncing.
- */
- zfs_iput_async(ZTOI(zp));
- return (SET_ERROR(ENOENT));
- }
-
- zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
- zgd->zgd_lwb = lwb;
- zgd->zgd_private = zp;
-
- /*
- * Write records come in two flavors: immediate and indirect.
- * For small writes it's cheaper to store the data with the
- * log record (immediate); for large writes it's cheaper to
- * sync the data and get a pointer to it (indirect) so that
- * we don't have to write the data twice.
- */
- if (buf != NULL) { /* immediate write */
- zgd->zgd_lr = rangelock_enter(&zp->z_rangelock,
- offset, size, RL_READER);
- /* test for truncation needs to be done while range locked */
- if (offset >= zp->z_size) {
- error = SET_ERROR(ENOENT);
- } else {
- error = dmu_read(os, object, offset, size, buf,
- DMU_READ_NO_PREFETCH);
- }
- ASSERT(error == 0 || error == ENOENT);
- } else { /* indirect write */
- /*
- * Have to lock the whole block to ensure when it's
- * written out and its checksum is being calculated
- * that no one can change the data. We need to re-check
- * blocksize after we get the lock in case it's changed!
- */
- for (;;) {
- uint64_t blkoff;
- size = zp->z_blksz;
- blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
- offset -= blkoff;
- zgd->zgd_lr = rangelock_enter(&zp->z_rangelock,
- offset, size, RL_READER);
- if (zp->z_blksz == size)
- break;
- offset += blkoff;
- rangelock_exit(zgd->zgd_lr);
- }
- /* test for truncation needs to be done while range locked */
- if (lr->lr_offset >= zp->z_size)
- error = SET_ERROR(ENOENT);
-#ifdef DEBUG
- if (zil_fault_io) {
- error = SET_ERROR(EIO);
- zil_fault_io = 0;
- }
-#endif
- if (error == 0)
- error = dmu_buf_hold(os, object, offset, zgd, &db,
- DMU_READ_NO_PREFETCH);
-
- if (error == 0) {
- blkptr_t *bp = &lr->lr_blkptr;
-
- zgd->zgd_db = db;
- zgd->zgd_bp = bp;
-
- ASSERT(db->db_offset == offset);
- ASSERT(db->db_size == size);
-
- error = dmu_sync(zio, lr->lr_common.lrc_txg,
- zfs_get_done, zgd);
- ASSERT(error || lr->lr_length <= size);
-
- /*
- * On success, we need to wait for the write I/O
- * initiated by dmu_sync() to complete before we can
- * release this dbuf. We will finish everything up
- * in the zfs_get_done() callback.
- */
- if (error == 0)
- return (0);
-
- if (error == EALREADY) {
- lr->lr_common.lrc_txtype = TX_WRITE2;
- /*
- * TX_WRITE2 relies on the data previously
- * written by the TX_WRITE that caused
- * EALREADY. We zero out the BP because
- * it is the old, currently-on-disk BP.
- */
- zgd->zgd_bp = NULL;
- BP_ZERO(bp);
- error = 0;
- }
- }
- }
-
- zfs_get_done(zgd, error);
-
- return (error);
-}
-
-/*ARGSUSED*/
-int
-zfs_access(struct inode *ip, int mode, int flag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if (flag & V_ACE_MASK)
- error = zfs_zaccess(zp, mode, flag, B_FALSE, cr);
- else
- error = zfs_zaccess_rwx(zp, mode, flag, cr);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Lookup an entry in a directory, or an extended attribute directory.
- * If it exists, return a held inode reference for it.
- *
- * IN: dip - inode of directory to search.
- * nm - name of entry to lookup.
- * flags - LOOKUP_XATTR set if looking for an attribute.
- * cr - credentials of caller.
- * direntflags - directory lookup flags
- * realpnp - returned pathname.
- *
- * OUT: ipp - inode of located entry, NULL if not found.
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * NA
- */
-/* ARGSUSED */
-int
-zfs_lookup(struct inode *dip, char *nm, struct inode **ipp, int flags,
- cred_t *cr, int *direntflags, pathname_t *realpnp)
-{
- znode_t *zdp = ITOZ(dip);
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- int error = 0;
-
- /*
- * Fast path lookup, however we must skip DNLC lookup
- * for case folding or normalizing lookups because the
- * DNLC code only stores the passed in name. This means
- * creating 'a' and removing 'A' on a case insensitive
- * file system would work, but DNLC still thinks 'a'
- * exists and won't let you create it again on the next
- * pass through fast path.
- */
- if (!(flags & (LOOKUP_XATTR | FIGNORECASE))) {
-
- if (!S_ISDIR(dip->i_mode)) {
- return (SET_ERROR(ENOTDIR));
- } else if (zdp->z_sa_hdl == NULL) {
- return (SET_ERROR(EIO));
- }
-
- if (nm[0] == 0 || (nm[0] == '.' && nm[1] == '\0')) {
- error = zfs_fastaccesschk_execute(zdp, cr);
- if (!error) {
- *ipp = dip;
- igrab(*ipp);
- return (0);
- }
- return (error);
- }
- }
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zdp);
-
- *ipp = NULL;
-
- if (flags & LOOKUP_XATTR) {
- /*
- * We don't allow recursive attributes..
- * Maybe someday we will.
- */
- if (zdp->z_pflags & ZFS_XATTR) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- if ((error = zfs_get_xattrdir(zdp, ipp, cr, flags))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- /*
- * Do we have permission to get into attribute directory?
- */
-
- if ((error = zfs_zaccess(ITOZ(*ipp), ACE_EXECUTE, 0,
- B_FALSE, cr))) {
- iput(*ipp);
- *ipp = NULL;
- }
-
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (!S_ISDIR(dip->i_mode)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOTDIR));
- }
-
- /*
- * Check accessibility of directory.
- */
-
- if ((error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (zfsvfs->z_utf8 && u8_validate(nm, strlen(nm),
- NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
-
- error = zfs_dirlook(zdp, nm, ipp, flags, direntflags, realpnp);
- if ((error == 0) && (*ipp))
- zfs_inode_update(ITOZ(*ipp));
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Attempt to create a new entry in a directory. If the entry
- * already exists, truncate the file if permissible, else return
- * an error. Return the ip of the created or trunc'd file.
- *
- * IN: dip - inode of directory to put new file entry in.
- * name - name of new file entry.
- * vap - attributes of new file.
- * excl - flag indicating exclusive or non-exclusive mode.
- * mode - mode to open file with.
- * cr - credentials of caller.
- * flag - file flag.
- * vsecp - ACL to be set
- *
- * OUT: ipp - inode of created or trunc'd entry.
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * dip - ctime|mtime updated if new entry created
- * ip - ctime|mtime always, atime if new
- */
-
-/* ARGSUSED */
-int
-zfs_create(struct inode *dip, char *name, vattr_t *vap, int excl,
- int mode, struct inode **ipp, cred_t *cr, int flag, vsecattr_t *vsecp)
-{
- znode_t *zp, *dzp = ITOZ(dip);
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- zilog_t *zilog;
- objset_t *os;
- zfs_dirlock_t *dl;
- dmu_tx_t *tx;
- int error;
- uid_t uid;
- gid_t gid;
- zfs_acl_ids_t acl_ids;
- boolean_t fuid_dirtied;
- boolean_t have_acl = B_FALSE;
- boolean_t waited = B_FALSE;
-
- /*
- * If we have an ephemeral id, ACL, or XVATTR then
- * make sure file system is at proper version
- */
-
- gid = crgetgid(cr);
- uid = crgetuid(cr);
-
- if (zfsvfs->z_use_fuids == B_FALSE &&
- (vsecp || IS_EPHEMERAL(uid) || IS_EPHEMERAL(gid)))
- return (SET_ERROR(EINVAL));
-
- if (name == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- os = zfsvfs->z_os;
- zilog = zfsvfs->z_log;
-
- if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
- NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
-
- if (vap->va_mask & ATTR_XVATTR) {
- if ((error = secpolicy_xvattr((xvattr_t *)vap,
- crgetuid(cr), cr, vap->va_mode)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- }
-
-top:
- *ipp = NULL;
- if (*name == '\0') {
- /*
- * Null component name refers to the directory itself.
- */
- igrab(dip);
- zp = dzp;
- dl = NULL;
- error = 0;
- } else {
- /* possible igrab(zp) */
- int zflg = 0;
-
- if (flag & FIGNORECASE)
- zflg |= ZCILOOK;
-
- error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
- NULL, NULL);
- if (error) {
- if (have_acl)
- zfs_acl_ids_free(&acl_ids);
- if (strcmp(name, "..") == 0)
- error = SET_ERROR(EISDIR);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- }
-
- if (zp == NULL) {
- uint64_t txtype;
- uint64_t projid = ZFS_DEFAULT_PROJID;
-
- /*
- * Create a new file object and update the directory
- * to reference it.
- */
- if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) {
- if (have_acl)
- zfs_acl_ids_free(&acl_ids);
- goto out;
- }
-
- /*
- * We only support the creation of regular files in
- * extended attribute directories.
- */
-
- if ((dzp->z_pflags & ZFS_XATTR) && !S_ISREG(vap->va_mode)) {
- if (have_acl)
- zfs_acl_ids_free(&acl_ids);
- error = SET_ERROR(EINVAL);
- goto out;
- }
-
- if (!have_acl && (error = zfs_acl_ids_create(dzp, 0, vap,
- cr, vsecp, &acl_ids)) != 0)
- goto out;
- have_acl = B_TRUE;
-
- if (S_ISREG(vap->va_mode) || S_ISDIR(vap->va_mode))
- projid = zfs_inherit_projid(dzp);
- if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) {
- zfs_acl_ids_free(&acl_ids);
- error = SET_ERROR(EDQUOT);
- goto out;
- }
-
- tx = dmu_tx_create(os);
-
- dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
- ZFS_SA_BASE_ATTR_SIZE);
-
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
- dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
- if (!zfsvfs->z_use_sa &&
- acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
- 0, acl_ids.z_aclp->z_acl_bytes);
- }
-
- error = dmu_tx_assign(tx,
- (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_abort(tx);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);
-
- error = zfs_link_create(dl, zp, tx, ZNEW);
- if (error != 0) {
- /*
- * Since, we failed to add the directory entry for it,
- * delete the newly created dnode.
- */
- zfs_znode_delete(zp, tx);
- remove_inode_hash(ZTOI(zp));
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_commit(tx);
- goto out;
- }
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- txtype = zfs_log_create_txtype(Z_FILE, vsecp, vap);
- if (flag & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_create(zilog, tx, txtype, dzp, zp, name,
- vsecp, acl_ids.z_fuidp, vap);
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_commit(tx);
- } else {
- int aflags = (flag & FAPPEND) ? V_APPEND : 0;
-
- if (have_acl)
- zfs_acl_ids_free(&acl_ids);
- have_acl = B_FALSE;
-
- /*
- * A directory entry already exists for this name.
- */
- /*
- * Can't truncate an existing file if in exclusive mode.
- */
- if (excl) {
- error = SET_ERROR(EEXIST);
- goto out;
- }
- /*
- * Can't open a directory for writing.
- */
- if (S_ISDIR(ZTOI(zp)->i_mode)) {
- error = SET_ERROR(EISDIR);
- goto out;
- }
- /*
- * Verify requested access to file.
- */
- if (mode && (error = zfs_zaccess_rwx(zp, mode, aflags, cr))) {
- goto out;
- }
-
- mutex_enter(&dzp->z_lock);
- dzp->z_seq++;
- mutex_exit(&dzp->z_lock);
-
- /*
- * Truncate regular files if requested.
- */
- if (S_ISREG(ZTOI(zp)->i_mode) &&
- (vap->va_mask & ATTR_SIZE) && (vap->va_size == 0)) {
- /* we can't hold any locks when calling zfs_freesp() */
- if (dl) {
- zfs_dirent_unlock(dl);
- dl = NULL;
- }
- error = zfs_freesp(zp, 0, 0, mode, TRUE);
- }
- }
-out:
-
- if (dl)
- zfs_dirent_unlock(dl);
-
- if (error) {
- if (zp)
- iput(ZTOI(zp));
- } else {
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
- *ipp = ZTOI(zp);
- }
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/* ARGSUSED */
-int
-zfs_tmpfile(struct inode *dip, vattr_t *vap, int excl,
- int mode, struct inode **ipp, cred_t *cr, int flag, vsecattr_t *vsecp)
-{
- znode_t *zp = NULL, *dzp = ITOZ(dip);
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- objset_t *os;
- dmu_tx_t *tx;
- int error;
- uid_t uid;
- gid_t gid;
- zfs_acl_ids_t acl_ids;
- uint64_t projid = ZFS_DEFAULT_PROJID;
- boolean_t fuid_dirtied;
- boolean_t have_acl = B_FALSE;
- boolean_t waited = B_FALSE;
-
- /*
- * If we have an ephemeral id, ACL, or XVATTR then
- * make sure file system is at proper version
- */
-
- gid = crgetgid(cr);
- uid = crgetuid(cr);
-
- if (zfsvfs->z_use_fuids == B_FALSE &&
- (vsecp || IS_EPHEMERAL(uid) || IS_EPHEMERAL(gid)))
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- os = zfsvfs->z_os;
-
- if (vap->va_mask & ATTR_XVATTR) {
- if ((error = secpolicy_xvattr((xvattr_t *)vap,
- crgetuid(cr), cr, vap->va_mode)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- }
-
-top:
- *ipp = NULL;
-
- /*
- * Create a new file object and update the directory
- * to reference it.
- */
- if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) {
- if (have_acl)
- zfs_acl_ids_free(&acl_ids);
- goto out;
- }
-
- if (!have_acl && (error = zfs_acl_ids_create(dzp, 0, vap,
- cr, vsecp, &acl_ids)) != 0)
- goto out;
- have_acl = B_TRUE;
-
- if (S_ISREG(vap->va_mode) || S_ISDIR(vap->va_mode))
- projid = zfs_inherit_projid(dzp);
- if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) {
- zfs_acl_ids_free(&acl_ids);
- error = SET_ERROR(EDQUOT);
- goto out;
- }
-
- tx = dmu_tx_create(os);
-
- dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
- ZFS_SA_BASE_ATTR_SIZE);
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
-
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- if (!zfsvfs->z_use_sa &&
- acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
- 0, acl_ids.z_aclp->z_acl_bytes);
- }
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_abort(tx);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- zfs_mknode(dzp, vap, tx, cr, IS_TMPFILE, &zp, &acl_ids);
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- /* Add to unlinked set */
- zp->z_unlinked = B_TRUE;
- zfs_unlinked_add(zp, tx);
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_commit(tx);
-out:
-
- if (error) {
- if (zp)
- iput(ZTOI(zp));
- } else {
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
- *ipp = ZTOI(zp);
- }
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Remove an entry from a directory.
- *
- * IN: dip - inode of directory to remove entry from.
- * name - name of entry to remove.
- * cr - credentials of caller.
- * flags - case flags.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * dip - ctime|mtime
- * ip - ctime (if nlink > 0)
- */
-
-uint64_t null_xattr = 0;
-
-/*ARGSUSED*/
-int
-zfs_remove(struct inode *dip, char *name, cred_t *cr, int flags)
-{
- znode_t *zp, *dzp = ITOZ(dip);
- znode_t *xzp;
- struct inode *ip;
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- zilog_t *zilog;
- uint64_t acl_obj, xattr_obj;
- uint64_t xattr_obj_unlinked = 0;
- uint64_t obj = 0;
- uint64_t links;
- zfs_dirlock_t *dl;
- dmu_tx_t *tx;
- boolean_t may_delete_now, delete_now = FALSE;
- boolean_t unlinked, toobig = FALSE;
- uint64_t txtype;
- pathname_t *realnmp = NULL;
- pathname_t realnm;
- int error;
- int zflg = ZEXISTS;
- boolean_t waited = B_FALSE;
-
- if (name == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- zilog = zfsvfs->z_log;
-
- if (flags & FIGNORECASE) {
- zflg |= ZCILOOK;
- pn_alloc(&realnm);
- realnmp = &realnm;
- }
-
-top:
- xattr_obj = 0;
- xzp = NULL;
- /*
- * Attempt to lock directory; fail if entry doesn't exist.
- */
- if ((error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
- NULL, realnmp))) {
- if (realnmp)
- pn_free(realnmp);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- ip = ZTOI(zp);
-
- if ((error = zfs_zaccess_delete(dzp, zp, cr))) {
- goto out;
- }
-
- /*
- * Need to use rmdir for removing directories.
- */
- if (S_ISDIR(ip->i_mode)) {
- error = SET_ERROR(EPERM);
- goto out;
- }
-
- mutex_enter(&zp->z_lock);
- may_delete_now = atomic_read(&ip->i_count) == 1 && !(zp->z_is_mapped);
- mutex_exit(&zp->z_lock);
-
- /*
- * We may delete the znode now, or we may put it in the unlinked set;
- * it depends on whether we're the last link, and on whether there are
- * other holds on the inode. So we dmu_tx_hold() the right things to
- * allow for either case.
- */
- obj = zp->z_id;
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
- zfs_sa_upgrade_txholds(tx, dzp);
- if (may_delete_now) {
- toobig = zp->z_size > zp->z_blksz * zfs_delete_blocks;
- /* if the file is too big, only hold_free a token amount */
- dmu_tx_hold_free(tx, zp->z_id, 0,
- (toobig ? DMU_MAX_ACCESS : DMU_OBJECT_END));
- }
-
- /* are there any extended attributes? */
- error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
- &xattr_obj, sizeof (xattr_obj));
- if (error == 0 && xattr_obj) {
- error = zfs_zget(zfsvfs, xattr_obj, &xzp);
- ASSERT0(error);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
- dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
- }
-
- mutex_enter(&zp->z_lock);
- if ((acl_obj = zfs_external_acl(zp)) != 0 && may_delete_now)
- dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
- mutex_exit(&zp->z_lock);
-
- /* charge as an update -- would be nice not to charge at all */
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
-
- /*
- * Mark this transaction as typically resulting in a net free of space
- */
- dmu_tx_mark_netfree(tx);
-
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- iput(ip);
- if (xzp)
- iput(ZTOI(xzp));
- goto top;
- }
- if (realnmp)
- pn_free(realnmp);
- dmu_tx_abort(tx);
- iput(ip);
- if (xzp)
- iput(ZTOI(xzp));
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- /*
- * Remove the directory entry.
- */
- error = zfs_link_destroy(dl, zp, tx, zflg, &unlinked);
-
- if (error) {
- dmu_tx_commit(tx);
- goto out;
- }
-
- if (unlinked) {
- /*
- * Hold z_lock so that we can make sure that the ACL obj
- * hasn't changed. Could have been deleted due to
- * zfs_sa_upgrade().
- */
- mutex_enter(&zp->z_lock);
- (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
- &xattr_obj_unlinked, sizeof (xattr_obj_unlinked));
- delete_now = may_delete_now && !toobig &&
- atomic_read(&ip->i_count) == 1 && !(zp->z_is_mapped) &&
- xattr_obj == xattr_obj_unlinked && zfs_external_acl(zp) ==
- acl_obj;
- }
-
- if (delete_now) {
- if (xattr_obj_unlinked) {
- ASSERT3U(ZTOI(xzp)->i_nlink, ==, 2);
- mutex_enter(&xzp->z_lock);
- xzp->z_unlinked = B_TRUE;
- clear_nlink(ZTOI(xzp));
- links = 0;
- error = sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
- &links, sizeof (links), tx);
- ASSERT3U(error, ==, 0);
- mutex_exit(&xzp->z_lock);
- zfs_unlinked_add(xzp, tx);
-
- if (zp->z_is_sa)
- error = sa_remove(zp->z_sa_hdl,
- SA_ZPL_XATTR(zfsvfs), tx);
- else
- error = sa_update(zp->z_sa_hdl,
- SA_ZPL_XATTR(zfsvfs), &null_xattr,
- sizeof (uint64_t), tx);
- ASSERT0(error);
- }
- /*
- * Add to the unlinked set because a new reference could be
- * taken concurrently resulting in a deferred destruction.
- */
- zfs_unlinked_add(zp, tx);
- mutex_exit(&zp->z_lock);
- } else if (unlinked) {
- mutex_exit(&zp->z_lock);
- zfs_unlinked_add(zp, tx);
- }
-
- txtype = TX_REMOVE;
- if (flags & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_remove(zilog, tx, txtype, dzp, name, obj, unlinked);
-
- dmu_tx_commit(tx);
-out:
- if (realnmp)
- pn_free(realnmp);
-
- zfs_dirent_unlock(dl);
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
-
- if (delete_now)
- iput(ip);
- else
- zfs_iput_async(ip);
-
- if (xzp) {
- zfs_inode_update(xzp);
- zfs_iput_async(ZTOI(xzp));
- }
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Create a new directory and insert it into dip using the name
- * provided. Return a pointer to the inserted directory.
- *
- * IN: dip - inode of directory to add subdir to.
- * dirname - name of new directory.
- * vap - attributes of new directory.
- * cr - credentials of caller.
- * flags - case flags.
- * vsecp - ACL to be set
- *
- * OUT: ipp - inode of created directory.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * dip - ctime|mtime updated
- * ipp - ctime|mtime|atime updated
- */
-/*ARGSUSED*/
-int
-zfs_mkdir(struct inode *dip, char *dirname, vattr_t *vap, struct inode **ipp,
- cred_t *cr, int flags, vsecattr_t *vsecp)
-{
- znode_t *zp, *dzp = ITOZ(dip);
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- zilog_t *zilog;
- zfs_dirlock_t *dl;
- uint64_t txtype;
- dmu_tx_t *tx;
- int error;
- int zf = ZNEW;
- uid_t uid;
- gid_t gid = crgetgid(cr);
- zfs_acl_ids_t acl_ids;
- boolean_t fuid_dirtied;
- boolean_t waited = B_FALSE;
-
- ASSERT(S_ISDIR(vap->va_mode));
-
- /*
- * If we have an ephemeral id, ACL, or XVATTR then
- * make sure file system is at proper version
- */
-
- uid = crgetuid(cr);
- if (zfsvfs->z_use_fuids == B_FALSE &&
- (vsecp || IS_EPHEMERAL(uid) || IS_EPHEMERAL(gid)))
- return (SET_ERROR(EINVAL));
-
- if (dirname == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- zilog = zfsvfs->z_log;
-
- if (dzp->z_pflags & ZFS_XATTR) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- if (zfsvfs->z_utf8 && u8_validate(dirname,
- strlen(dirname), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
- if (flags & FIGNORECASE)
- zf |= ZCILOOK;
-
- if (vap->va_mask & ATTR_XVATTR) {
- if ((error = secpolicy_xvattr((xvattr_t *)vap,
- crgetuid(cr), cr, vap->va_mode)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- }
-
- if ((error = zfs_acl_ids_create(dzp, 0, vap, cr,
- vsecp, &acl_ids)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- /*
- * First make sure the new directory doesn't exist.
- *
- * Existence is checked first to make sure we don't return
- * EACCES instead of EEXIST which can cause some applications
- * to fail.
- */
-top:
- *ipp = NULL;
-
- if ((error = zfs_dirent_lock(&dl, dzp, dirname, &zp, zf,
- NULL, NULL))) {
- zfs_acl_ids_free(&acl_ids);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if ((error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, 0, B_FALSE, cr))) {
- zfs_acl_ids_free(&acl_ids);
- zfs_dirent_unlock(dl);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zfs_inherit_projid(dzp))) {
- zfs_acl_ids_free(&acl_ids);
- zfs_dirent_unlock(dl);
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EDQUOT));
- }
-
- /*
- * Add a new entry to the directory.
- */
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname);
- dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
- acl_ids.z_aclp->z_acl_bytes);
- }
-
- dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
- ZFS_SA_BASE_ATTR_SIZE);
-
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_abort(tx);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- /*
- * Create new node.
- */
- zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);
-
- /*
- * Now put new name in parent dir.
- */
- error = zfs_link_create(dl, zp, tx, ZNEW);
- if (error != 0) {
- zfs_znode_delete(zp, tx);
- remove_inode_hash(ZTOI(zp));
- goto out;
- }
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- *ipp = ZTOI(zp);
-
- txtype = zfs_log_create_txtype(Z_DIR, vsecp, vap);
- if (flags & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_create(zilog, tx, txtype, dzp, zp, dirname, vsecp,
- acl_ids.z_fuidp, vap);
-
-out:
- zfs_acl_ids_free(&acl_ids);
-
- dmu_tx_commit(tx);
-
- zfs_dirent_unlock(dl);
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- if (error != 0) {
- iput(ZTOI(zp));
- } else {
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
- }
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Remove a directory subdir entry. If the current working
- * directory is the same as the subdir to be removed, the
- * remove will fail.
- *
- * IN: dip - inode of directory to remove from.
- * name - name of directory to be removed.
- * cwd - inode of current working directory.
- * cr - credentials of caller.
- * flags - case flags
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * dip - ctime|mtime updated
- */
-/*ARGSUSED*/
-int
-zfs_rmdir(struct inode *dip, char *name, struct inode *cwd, cred_t *cr,
- int flags)
-{
- znode_t *dzp = ITOZ(dip);
- znode_t *zp;
- struct inode *ip;
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- zilog_t *zilog;
- zfs_dirlock_t *dl;
- dmu_tx_t *tx;
- int error;
- int zflg = ZEXISTS;
- boolean_t waited = B_FALSE;
-
- if (name == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- zilog = zfsvfs->z_log;
-
- if (flags & FIGNORECASE)
- zflg |= ZCILOOK;
-top:
- zp = NULL;
-
- /*
- * Attempt to lock directory; fail if entry doesn't exist.
- */
- if ((error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg,
- NULL, NULL))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- ip = ZTOI(zp);
-
- if ((error = zfs_zaccess_delete(dzp, zp, cr))) {
- goto out;
- }
-
- if (!S_ISDIR(ip->i_mode)) {
- error = SET_ERROR(ENOTDIR);
- goto out;
- }
-
- if (ip == cwd) {
- error = SET_ERROR(EINVAL);
- goto out;
- }
-
- /*
- * Grab a lock on the directory to make sure that no one is
- * trying to add (or lookup) entries while we are removing it.
- */
- rw_enter(&zp->z_name_lock, RW_WRITER);
-
- /*
- * Grab a lock on the parent pointer to make sure we play well
- * with the treewalk and directory rename code.
- */
- rw_enter(&zp->z_parent_lock, RW_WRITER);
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
- zfs_sa_upgrade_txholds(tx, zp);
- zfs_sa_upgrade_txholds(tx, dzp);
- dmu_tx_mark_netfree(tx);
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- rw_exit(&zp->z_parent_lock);
- rw_exit(&zp->z_name_lock);
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- iput(ip);
- goto top;
- }
- dmu_tx_abort(tx);
- iput(ip);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- error = zfs_link_destroy(dl, zp, tx, zflg, NULL);
-
- if (error == 0) {
- uint64_t txtype = TX_RMDIR;
- if (flags & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_remove(zilog, tx, txtype, dzp, name, ZFS_NO_OBJECT,
- B_FALSE);
- }
-
- dmu_tx_commit(tx);
-
- rw_exit(&zp->z_parent_lock);
- rw_exit(&zp->z_name_lock);
-out:
- zfs_dirent_unlock(dl);
-
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
- iput(ip);
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Read directory entries from the given directory cursor position and emit
- * name and position for each entry.
- *
- * IN: ip - inode of directory to read.
- * ctx - directory entry context.
- * cr - credentials of caller.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * ip - atime updated
- *
- * Note that the low 4 bits of the cookie returned by zap is always zero.
- * This allows us to use the low range for "special" directory entries:
- * We use 0 for '.', and 1 for '..'. If this is the root of the filesystem,
- * we use the offset 2 for the '.zfs' directory.
- */
-/* ARGSUSED */
-int
-zfs_readdir(struct inode *ip, zpl_dir_context_t *ctx, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- objset_t *os;
- zap_cursor_t zc;
- zap_attribute_t zap;
- int error;
- uint8_t prefetch;
- uint8_t type;
- int done = 0;
- uint64_t parent;
- uint64_t offset; /* must be unsigned; checks for < 1 */
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
- &parent, sizeof (parent))) != 0)
- goto out;
-
- /*
- * Quit if directory has been removed (posix)
- */
- if (zp->z_unlinked)
- goto out;
-
- error = 0;
- os = zfsvfs->z_os;
- offset = ctx->pos;
- prefetch = zp->z_zn_prefetch;
-
- /*
- * Initialize the iterator cursor.
- */
- if (offset <= 3) {
- /*
- * Start iteration from the beginning of the directory.
- */
- zap_cursor_init(&zc, os, zp->z_id);
- } else {
- /*
- * The offset is a serialized cursor.
- */
- zap_cursor_init_serialized(&zc, os, zp->z_id, offset);
- }
-
- /*
- * Transform to file-system independent format
- */
- while (!done) {
- uint64_t objnum;
- /*
- * Special case `.', `..', and `.zfs'.
- */
- if (offset == 0) {
- (void) strcpy(zap.za_name, ".");
- zap.za_normalization_conflict = 0;
- objnum = zp->z_id;
- type = DT_DIR;
- } else if (offset == 1) {
- (void) strcpy(zap.za_name, "..");
- zap.za_normalization_conflict = 0;
- objnum = parent;
- type = DT_DIR;
- } else if (offset == 2 && zfs_show_ctldir(zp)) {
- (void) strcpy(zap.za_name, ZFS_CTLDIR_NAME);
- zap.za_normalization_conflict = 0;
- objnum = ZFSCTL_INO_ROOT;
- type = DT_DIR;
- } else {
- /*
- * Grab next entry.
- */
- if ((error = zap_cursor_retrieve(&zc, &zap))) {
- if (error == ENOENT)
- break;
- else
- goto update;
- }
-
- /*
- * Allow multiple entries provided the first entry is
- * the object id. Non-zpl consumers may safely make
- * use of the additional space.
- *
- * XXX: This should be a feature flag for compatibility
- */
- if (zap.za_integer_length != 8 ||
- zap.za_num_integers == 0) {
- cmn_err(CE_WARN, "zap_readdir: bad directory "
- "entry, obj = %lld, offset = %lld, "
- "length = %d, num = %lld\n",
- (u_longlong_t)zp->z_id,
- (u_longlong_t)offset,
- zap.za_integer_length,
- (u_longlong_t)zap.za_num_integers);
- error = SET_ERROR(ENXIO);
- goto update;
- }
-
- objnum = ZFS_DIRENT_OBJ(zap.za_first_integer);
- type = ZFS_DIRENT_TYPE(zap.za_first_integer);
- }
-
- done = !zpl_dir_emit(ctx, zap.za_name, strlen(zap.za_name),
- objnum, type);
- if (done)
- break;
-
- /* Prefetch znode */
- if (prefetch) {
- dmu_prefetch(os, objnum, 0, 0, 0,
- ZIO_PRIORITY_SYNC_READ);
- }
-
- /*
- * Move to the next entry, fill in the previous offset.
- */
- if (offset > 2 || (offset == 2 && !zfs_show_ctldir(zp))) {
- zap_cursor_advance(&zc);
- offset = zap_cursor_serialize(&zc);
- } else {
- offset += 1;
- }
- ctx->pos = offset;
- }
- zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */
-
-update:
- zap_cursor_fini(&zc);
- if (error == ENOENT)
- error = 0;
-out:
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-ulong_t zfs_fsync_sync_cnt = 4;
-
-int
-zfs_fsync(struct inode *ip, int syncflag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- (void) tsd_set(zfs_fsyncer_key, (void *)zfs_fsync_sync_cnt);
-
- if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
- zil_commit(zfsvfs->z_log, zp->z_id);
- ZFS_EXIT(zfsvfs);
- }
- tsd_set(zfs_fsyncer_key, NULL);
-
- return (0);
-}
-
-
-/*
- * Get the requested file attributes and place them in the provided
- * vattr structure.
- *
- * IN: ip - inode of file.
- * vap - va_mask identifies requested attributes.
- * If ATTR_XVATTR set, then optional attrs are requested
- * flags - ATTR_NOACLCHECK (CIFS server context)
- * cr - credentials of caller.
- *
- * OUT: vap - attribute values.
- *
- * RETURN: 0 (always succeeds)
- */
-/* ARGSUSED */
-int
-zfs_getattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error = 0;
- uint64_t links;
- uint64_t atime[2], mtime[2], ctime[2];
- xvattr_t *xvap = (xvattr_t *)vap; /* vap may be an xvattr_t * */
- xoptattr_t *xoap = NULL;
- boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
- sa_bulk_attr_t bulk[3];
- int count = 0;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- zfs_fuid_map_ids(zp, cr, &vap->va_uid, &vap->va_gid);
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
-
- if ((error = sa_bulk_lookup(zp->z_sa_hdl, bulk, count)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- /*
- * If ACL is trivial don't bother looking for ACE_READ_ATTRIBUTES.
- * Also, if we are the owner don't bother, since owner should
- * always be allowed to read basic attributes of file.
- */
- if (!(zp->z_pflags & ZFS_ACL_TRIVIAL) &&
- (vap->va_uid != crgetuid(cr))) {
- if ((error = zfs_zaccess(zp, ACE_READ_ATTRIBUTES, 0,
- skipaclchk, cr))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- }
-
- /*
- * Return all attributes. It's cheaper to provide the answer
- * than to determine whether we were asked the question.
- */
-
- mutex_enter(&zp->z_lock);
- vap->va_type = vn_mode_to_vtype(zp->z_mode);
- vap->va_mode = zp->z_mode;
- vap->va_fsid = ZTOI(zp)->i_sb->s_dev;
- vap->va_nodeid = zp->z_id;
- if ((zp->z_id == zfsvfs->z_root) && zfs_show_ctldir(zp))
- links = ZTOI(zp)->i_nlink + 1;
- else
- links = ZTOI(zp)->i_nlink;
- vap->va_nlink = MIN(links, ZFS_LINK_MAX);
- vap->va_size = i_size_read(ip);
- vap->va_rdev = ip->i_rdev;
- vap->va_seq = ip->i_generation;
-
- /*
- * Add in any requested optional attributes and the create time.
- * Also set the corresponding bits in the returned attribute bitmap.
- */
- if ((xoap = xva_getxoptattr(xvap)) != NULL && zfsvfs->z_use_fuids) {
- if (XVA_ISSET_REQ(xvap, XAT_ARCHIVE)) {
- xoap->xoa_archive =
- ((zp->z_pflags & ZFS_ARCHIVE) != 0);
- XVA_SET_RTN(xvap, XAT_ARCHIVE);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_READONLY)) {
- xoap->xoa_readonly =
- ((zp->z_pflags & ZFS_READONLY) != 0);
- XVA_SET_RTN(xvap, XAT_READONLY);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_SYSTEM)) {
- xoap->xoa_system =
- ((zp->z_pflags & ZFS_SYSTEM) != 0);
- XVA_SET_RTN(xvap, XAT_SYSTEM);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_HIDDEN)) {
- xoap->xoa_hidden =
- ((zp->z_pflags & ZFS_HIDDEN) != 0);
- XVA_SET_RTN(xvap, XAT_HIDDEN);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
- xoap->xoa_nounlink =
- ((zp->z_pflags & ZFS_NOUNLINK) != 0);
- XVA_SET_RTN(xvap, XAT_NOUNLINK);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
- xoap->xoa_immutable =
- ((zp->z_pflags & ZFS_IMMUTABLE) != 0);
- XVA_SET_RTN(xvap, XAT_IMMUTABLE);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
- xoap->xoa_appendonly =
- ((zp->z_pflags & ZFS_APPENDONLY) != 0);
- XVA_SET_RTN(xvap, XAT_APPENDONLY);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
- xoap->xoa_nodump =
- ((zp->z_pflags & ZFS_NODUMP) != 0);
- XVA_SET_RTN(xvap, XAT_NODUMP);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_OPAQUE)) {
- xoap->xoa_opaque =
- ((zp->z_pflags & ZFS_OPAQUE) != 0);
- XVA_SET_RTN(xvap, XAT_OPAQUE);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
- xoap->xoa_av_quarantined =
- ((zp->z_pflags & ZFS_AV_QUARANTINED) != 0);
- XVA_SET_RTN(xvap, XAT_AV_QUARANTINED);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
- xoap->xoa_av_modified =
- ((zp->z_pflags & ZFS_AV_MODIFIED) != 0);
- XVA_SET_RTN(xvap, XAT_AV_MODIFIED);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) &&
- S_ISREG(ip->i_mode)) {
- zfs_sa_get_scanstamp(zp, xvap);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) {
- uint64_t times[2];
-
- (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_CRTIME(zfsvfs),
- times, sizeof (times));
- ZFS_TIME_DECODE(&xoap->xoa_createtime, times);
- XVA_SET_RTN(xvap, XAT_CREATETIME);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
- xoap->xoa_reparse = ((zp->z_pflags & ZFS_REPARSE) != 0);
- XVA_SET_RTN(xvap, XAT_REPARSE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_GEN)) {
- xoap->xoa_generation = ip->i_generation;
- XVA_SET_RTN(xvap, XAT_GEN);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_OFFLINE)) {
- xoap->xoa_offline =
- ((zp->z_pflags & ZFS_OFFLINE) != 0);
- XVA_SET_RTN(xvap, XAT_OFFLINE);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_SPARSE)) {
- xoap->xoa_sparse =
- ((zp->z_pflags & ZFS_SPARSE) != 0);
- XVA_SET_RTN(xvap, XAT_SPARSE);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
- xoap->xoa_projinherit =
- ((zp->z_pflags & ZFS_PROJINHERIT) != 0);
- XVA_SET_RTN(xvap, XAT_PROJINHERIT);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_PROJID)) {
- xoap->xoa_projid = zp->z_projid;
- XVA_SET_RTN(xvap, XAT_PROJID);
- }
- }
-
- ZFS_TIME_DECODE(&vap->va_atime, atime);
- ZFS_TIME_DECODE(&vap->va_mtime, mtime);
- ZFS_TIME_DECODE(&vap->va_ctime, ctime);
-
- mutex_exit(&zp->z_lock);
-
- sa_object_size(zp->z_sa_hdl, &vap->va_blksize, &vap->va_nblocks);
-
- if (zp->z_blksz == 0) {
- /*
- * Block size hasn't been set; suggest maximal I/O transfers.
- */
- vap->va_blksize = zfsvfs->z_max_blksz;
- }
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*
- * Get the basic file attributes and place them in the provided kstat
- * structure. The inode is assumed to be the authoritative source
- * for most of the attributes. However, the znode currently has the
- * authoritative atime, blksize, and block count.
- *
- * IN: ip - inode of file.
- *
- * OUT: sp - kstat values.
- *
- * RETURN: 0 (always succeeds)
- */
-/* ARGSUSED */
-int
-zfs_getattr_fast(struct inode *ip, struct kstat *sp)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- uint32_t blksize;
- u_longlong_t nblocks;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- mutex_enter(&zp->z_lock);
-
- generic_fillattr(ip, sp);
- /*
- * +1 link count for root inode with visible '.zfs' directory.
- */
- if ((zp->z_id == zfsvfs->z_root) && zfs_show_ctldir(zp))
- if (sp->nlink < ZFS_LINK_MAX)
- sp->nlink++;
-
- sa_object_size(zp->z_sa_hdl, &blksize, &nblocks);
- sp->blksize = blksize;
- sp->blocks = nblocks;
-
- if (unlikely(zp->z_blksz == 0)) {
- /*
- * Block size hasn't been set; suggest maximal I/O transfers.
- */
- sp->blksize = zfsvfs->z_max_blksz;
- }
-
- mutex_exit(&zp->z_lock);
-
- /*
- * Required to prevent NFS client from detecting different inode
- * numbers of snapshot root dentry before and after snapshot mount.
- */
- if (zfsvfs->z_issnap) {
- if (ip->i_sb->s_root->d_inode == ip)
- sp->ino = ZFSCTL_INO_SNAPDIRS -
- dmu_objset_id(zfsvfs->z_os);
- }
-
- ZFS_EXIT(zfsvfs);
-
- return (0);
-}
-
-/*
- * For the operation of changing file's user/group/project, we need to
- * handle not only the main object that is assigned to the file directly,
- * but also the ones that are used by the file via hidden xattr directory.
- *
- * Because the xattr directory may contains many EA entries, as to it may
- * be impossible to change all of them via the transaction of changing the
- * main object's user/group/project attributes. Then we have to change them
- * via other multiple independent transactions one by one. It may be not good
- * solution, but we have no better idea yet.
- */
-static int
-zfs_setattr_dir(znode_t *dzp)
-{
- struct inode *dxip = ZTOI(dzp);
- struct inode *xip = NULL;
- zfsvfs_t *zfsvfs = ITOZSB(dxip);
- objset_t *os = zfsvfs->z_os;
- zap_cursor_t zc;
- zap_attribute_t zap;
- zfs_dirlock_t *dl;
- znode_t *zp;
- dmu_tx_t *tx = NULL;
- uint64_t uid, gid;
- sa_bulk_attr_t bulk[4];
- int count;
- int err;
-
- zap_cursor_init(&zc, os, dzp->z_id);
- while ((err = zap_cursor_retrieve(&zc, &zap)) == 0) {
- count = 0;
- if (zap.za_integer_length != 8 || zap.za_num_integers != 1) {
- err = ENXIO;
- break;
- }
-
- err = zfs_dirent_lock(&dl, dzp, (char *)zap.za_name, &zp,
- ZEXISTS, NULL, NULL);
- if (err == ENOENT)
- goto next;
- if (err)
- break;
-
- xip = ZTOI(zp);
- if (KUID_TO_SUID(xip->i_uid) == KUID_TO_SUID(dxip->i_uid) &&
- KGID_TO_SGID(xip->i_gid) == KGID_TO_SGID(dxip->i_gid) &&
- zp->z_projid == dzp->z_projid)
- goto next;
-
- tx = dmu_tx_create(os);
- if (!(zp->z_pflags & ZFS_PROJID))
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
- else
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
-
- err = dmu_tx_assign(tx, TXG_WAIT);
- if (err)
- break;
-
- mutex_enter(&dzp->z_lock);
-
- if (KUID_TO_SUID(xip->i_uid) != KUID_TO_SUID(dxip->i_uid)) {
- xip->i_uid = dxip->i_uid;
- uid = zfs_uid_read(dxip);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
- &uid, sizeof (uid));
- }
-
- if (KGID_TO_SGID(xip->i_gid) != KGID_TO_SGID(dxip->i_gid)) {
- xip->i_gid = dxip->i_gid;
- gid = zfs_gid_read(dxip);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
- &gid, sizeof (gid));
- }
-
- if (zp->z_projid != dzp->z_projid) {
- if (!(zp->z_pflags & ZFS_PROJID)) {
- zp->z_pflags |= ZFS_PROJID;
- SA_ADD_BULK_ATTR(bulk, count,
- SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags,
- sizeof (zp->z_pflags));
- }
-
- zp->z_projid = dzp->z_projid;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PROJID(zfsvfs),
- NULL, &zp->z_projid, sizeof (zp->z_projid));
- }
-
- mutex_exit(&dzp->z_lock);
-
- if (likely(count > 0)) {
- err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
- dmu_tx_commit(tx);
- } else {
- dmu_tx_abort(tx);
- }
- tx = NULL;
- if (err != 0 && err != ENOENT)
- break;
-
-next:
- if (xip) {
- iput(xip);
- xip = NULL;
- zfs_dirent_unlock(dl);
- }
- zap_cursor_advance(&zc);
- }
-
- if (tx)
- dmu_tx_abort(tx);
- if (xip) {
- iput(xip);
- zfs_dirent_unlock(dl);
- }
- zap_cursor_fini(&zc);
-
- return (err == ENOENT ? 0 : err);
-}
-
-/*
- * Set the file attributes to the values contained in the
- * vattr structure.
- *
- * IN: ip - inode of file to be modified.
- * vap - new attribute values.
- * If ATTR_XVATTR set, then optional attrs are being set
- * flags - ATTR_UTIME set if non-default time values provided.
- * - ATTR_NOACLCHECK (CIFS context only).
- * cr - credentials of caller.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * ip - ctime updated, mtime updated if size changed.
- */
-/* ARGSUSED */
-int
-zfs_setattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- objset_t *os = zfsvfs->z_os;
- zilog_t *zilog;
- dmu_tx_t *tx;
- vattr_t oldva;
- xvattr_t *tmpxvattr;
- uint_t mask = vap->va_mask;
- uint_t saved_mask = 0;
- int trim_mask = 0;
- uint64_t new_mode;
- uint64_t new_kuid = 0, new_kgid = 0, new_uid, new_gid;
- uint64_t xattr_obj;
- uint64_t mtime[2], ctime[2], atime[2];
- uint64_t projid = ZFS_INVALID_PROJID;
- znode_t *attrzp;
- int need_policy = FALSE;
- int err, err2 = 0;
- zfs_fuid_info_t *fuidp = NULL;
- xvattr_t *xvap = (xvattr_t *)vap; /* vap may be an xvattr_t * */
- xoptattr_t *xoap;
- zfs_acl_t *aclp;
- boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
- boolean_t fuid_dirtied = B_FALSE;
- boolean_t handle_eadir = B_FALSE;
- sa_bulk_attr_t *bulk, *xattr_bulk;
- int count = 0, xattr_count = 0, bulks = 8;
-
- if (mask == 0)
- return (0);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- /*
- * If this is a xvattr_t, then get a pointer to the structure of
- * optional attributes. If this is NULL, then we have a vattr_t.
- */
- xoap = xva_getxoptattr(xvap);
- if (xoap != NULL && (mask & ATTR_XVATTR)) {
- if (XVA_ISSET_REQ(xvap, XAT_PROJID)) {
- if (!dmu_objset_projectquota_enabled(os) ||
- (!S_ISREG(ip->i_mode) && !S_ISDIR(ip->i_mode))) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOTSUP));
- }
-
- projid = xoap->xoa_projid;
- if (unlikely(projid == ZFS_INVALID_PROJID)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- if (projid == zp->z_projid && zp->z_pflags & ZFS_PROJID)
- projid = ZFS_INVALID_PROJID;
- else
- need_policy = TRUE;
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT) &&
- (xoap->xoa_projinherit !=
- ((zp->z_pflags & ZFS_PROJINHERIT) != 0)) &&
- (!dmu_objset_projectquota_enabled(os) ||
- (!S_ISREG(ip->i_mode) && !S_ISDIR(ip->i_mode)))) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENOTSUP));
- }
- }
-
- zilog = zfsvfs->z_log;
-
- /*
- * Make sure that if we have ephemeral uid/gid or xvattr specified
- * that file system is at proper version level
- */
-
- if (zfsvfs->z_use_fuids == B_FALSE &&
- (((mask & ATTR_UID) && IS_EPHEMERAL(vap->va_uid)) ||
- ((mask & ATTR_GID) && IS_EPHEMERAL(vap->va_gid)) ||
- (mask & ATTR_XVATTR))) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- if (mask & ATTR_SIZE && S_ISDIR(ip->i_mode)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EISDIR));
- }
-
- if (mask & ATTR_SIZE && !S_ISREG(ip->i_mode) && !S_ISFIFO(ip->i_mode)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- tmpxvattr = kmem_alloc(sizeof (xvattr_t), KM_SLEEP);
- xva_init(tmpxvattr);
-
- bulk = kmem_alloc(sizeof (sa_bulk_attr_t) * bulks, KM_SLEEP);
- xattr_bulk = kmem_alloc(sizeof (sa_bulk_attr_t) * bulks, KM_SLEEP);
-
- /*
- * Immutable files can only alter immutable bit and atime
- */
- if ((zp->z_pflags & ZFS_IMMUTABLE) &&
- ((mask & (ATTR_SIZE|ATTR_UID|ATTR_GID|ATTR_MTIME|ATTR_MODE)) ||
- ((mask & ATTR_XVATTR) && XVA_ISSET_REQ(xvap, XAT_CREATETIME)))) {
- err = SET_ERROR(EPERM);
- goto out3;
- }
-
- if ((mask & ATTR_SIZE) && (zp->z_pflags & ZFS_READONLY)) {
- err = SET_ERROR(EPERM);
- goto out3;
- }
-
- /*
- * Verify timestamps doesn't overflow 32 bits.
- * ZFS can handle large timestamps, but 32bit syscalls can't
- * handle times greater than 2039. This check should be removed
- * once large timestamps are fully supported.
- */
- if (mask & (ATTR_ATIME | ATTR_MTIME)) {
- if (((mask & ATTR_ATIME) &&
- TIMESPEC_OVERFLOW(&vap->va_atime)) ||
- ((mask & ATTR_MTIME) &&
- TIMESPEC_OVERFLOW(&vap->va_mtime))) {
- err = SET_ERROR(EOVERFLOW);
- goto out3;
- }
- }
-
-top:
- attrzp = NULL;
- aclp = NULL;
-
- /* Can this be moved to before the top label? */
- if (zfs_is_readonly(zfsvfs)) {
- err = SET_ERROR(EROFS);
- goto out3;
- }
-
- /*
- * First validate permissions
- */
-
- if (mask & ATTR_SIZE) {
- err = zfs_zaccess(zp, ACE_WRITE_DATA, 0, skipaclchk, cr);
- if (err)
- goto out3;
-
- /*
- * XXX - Note, we are not providing any open
- * mode flags here (like FNDELAY), so we may
- * block if there are locks present... this
- * should be addressed in openat().
- */
- /* XXX - would it be OK to generate a log record here? */
- err = zfs_freesp(zp, vap->va_size, 0, 0, FALSE);
- if (err)
- goto out3;
- }
-
- if (mask & (ATTR_ATIME|ATTR_MTIME) ||
- ((mask & ATTR_XVATTR) && (XVA_ISSET_REQ(xvap, XAT_HIDDEN) ||
- XVA_ISSET_REQ(xvap, XAT_READONLY) ||
- XVA_ISSET_REQ(xvap, XAT_ARCHIVE) ||
- XVA_ISSET_REQ(xvap, XAT_OFFLINE) ||
- XVA_ISSET_REQ(xvap, XAT_SPARSE) ||
- XVA_ISSET_REQ(xvap, XAT_CREATETIME) ||
- XVA_ISSET_REQ(xvap, XAT_SYSTEM)))) {
- need_policy = zfs_zaccess(zp, ACE_WRITE_ATTRIBUTES, 0,
- skipaclchk, cr);
- }
-
- if (mask & (ATTR_UID|ATTR_GID)) {
- int idmask = (mask & (ATTR_UID|ATTR_GID));
- int take_owner;
- int take_group;
-
- /*
- * NOTE: even if a new mode is being set,
- * we may clear S_ISUID/S_ISGID bits.
- */
-
- if (!(mask & ATTR_MODE))
- vap->va_mode = zp->z_mode;
-
- /*
- * Take ownership or chgrp to group we are a member of
- */
-
- take_owner = (mask & ATTR_UID) && (vap->va_uid == crgetuid(cr));
- take_group = (mask & ATTR_GID) &&
- zfs_groupmember(zfsvfs, vap->va_gid, cr);
-
- /*
- * If both ATTR_UID and ATTR_GID are set then take_owner and
- * take_group must both be set in order to allow taking
- * ownership.
- *
- * Otherwise, send the check through secpolicy_vnode_setattr()
- *
- */
-
- if (((idmask == (ATTR_UID|ATTR_GID)) &&
- take_owner && take_group) ||
- ((idmask == ATTR_UID) && take_owner) ||
- ((idmask == ATTR_GID) && take_group)) {
- if (zfs_zaccess(zp, ACE_WRITE_OWNER, 0,
- skipaclchk, cr) == 0) {
- /*
- * Remove setuid/setgid for non-privileged users
- */
- (void) secpolicy_setid_clear(vap, cr);
- trim_mask = (mask & (ATTR_UID|ATTR_GID));
- } else {
- need_policy = TRUE;
- }
- } else {
- need_policy = TRUE;
- }
- }
-
- mutex_enter(&zp->z_lock);
- oldva.va_mode = zp->z_mode;
- zfs_fuid_map_ids(zp, cr, &oldva.va_uid, &oldva.va_gid);
- if (mask & ATTR_XVATTR) {
- /*
- * Update xvattr mask to include only those attributes
- * that are actually changing.
- *
- * the bits will be restored prior to actually setting
- * the attributes so the caller thinks they were set.
- */
- if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
- if (xoap->xoa_appendonly !=
- ((zp->z_pflags & ZFS_APPENDONLY) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_APPENDONLY);
- XVA_SET_REQ(tmpxvattr, XAT_APPENDONLY);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
- if (xoap->xoa_projinherit !=
- ((zp->z_pflags & ZFS_PROJINHERIT) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_PROJINHERIT);
- XVA_SET_REQ(tmpxvattr, XAT_PROJINHERIT);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
- if (xoap->xoa_nounlink !=
- ((zp->z_pflags & ZFS_NOUNLINK) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_NOUNLINK);
- XVA_SET_REQ(tmpxvattr, XAT_NOUNLINK);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
- if (xoap->xoa_immutable !=
- ((zp->z_pflags & ZFS_IMMUTABLE) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_IMMUTABLE);
- XVA_SET_REQ(tmpxvattr, XAT_IMMUTABLE);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
- if (xoap->xoa_nodump !=
- ((zp->z_pflags & ZFS_NODUMP) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_NODUMP);
- XVA_SET_REQ(tmpxvattr, XAT_NODUMP);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
- if (xoap->xoa_av_modified !=
- ((zp->z_pflags & ZFS_AV_MODIFIED) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_AV_MODIFIED);
- XVA_SET_REQ(tmpxvattr, XAT_AV_MODIFIED);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
- if ((!S_ISREG(ip->i_mode) &&
- xoap->xoa_av_quarantined) ||
- xoap->xoa_av_quarantined !=
- ((zp->z_pflags & ZFS_AV_QUARANTINED) != 0)) {
- need_policy = TRUE;
- } else {
- XVA_CLR_REQ(xvap, XAT_AV_QUARANTINED);
- XVA_SET_REQ(tmpxvattr, XAT_AV_QUARANTINED);
- }
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
- mutex_exit(&zp->z_lock);
- err = SET_ERROR(EPERM);
- goto out3;
- }
-
- if (need_policy == FALSE &&
- (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) ||
- XVA_ISSET_REQ(xvap, XAT_OPAQUE))) {
- need_policy = TRUE;
- }
- }
-
- mutex_exit(&zp->z_lock);
-
- if (mask & ATTR_MODE) {
- if (zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr) == 0) {
- err = secpolicy_setid_setsticky_clear(ip, vap,
- &oldva, cr);
- if (err)
- goto out3;
-
- trim_mask |= ATTR_MODE;
- } else {
- need_policy = TRUE;
- }
- }
-
- if (need_policy) {
- /*
- * If trim_mask is set then take ownership
- * has been granted or write_acl is present and user
- * has the ability to modify mode. In that case remove
- * UID|GID and or MODE from mask so that
- * secpolicy_vnode_setattr() doesn't revoke it.
- */
-
- if (trim_mask) {
- saved_mask = vap->va_mask;
- vap->va_mask &= ~trim_mask;
- }
- err = secpolicy_vnode_setattr(cr, ip, vap, &oldva, flags,
- (int (*)(void *, int, cred_t *))zfs_zaccess_unix, zp);
- if (err)
- goto out3;
-
- if (trim_mask)
- vap->va_mask |= saved_mask;
- }
-
- /*
- * secpolicy_vnode_setattr, or take ownership may have
- * changed va_mask
- */
- mask = vap->va_mask;
-
- if ((mask & (ATTR_UID | ATTR_GID)) || projid != ZFS_INVALID_PROJID) {
- handle_eadir = B_TRUE;
- err = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
- &xattr_obj, sizeof (xattr_obj));
-
- if (err == 0 && xattr_obj) {
- err = zfs_zget(ZTOZSB(zp), xattr_obj, &attrzp);
- if (err)
- goto out2;
- }
- if (mask & ATTR_UID) {
- new_kuid = zfs_fuid_create(zfsvfs,
- (uint64_t)vap->va_uid, cr, ZFS_OWNER, &fuidp);
- if (new_kuid != KUID_TO_SUID(ZTOI(zp)->i_uid) &&
- zfs_id_overquota(zfsvfs, DMU_USERUSED_OBJECT,
- new_kuid)) {
- if (attrzp)
- iput(ZTOI(attrzp));
- err = SET_ERROR(EDQUOT);
- goto out2;
- }
- }
-
- if (mask & ATTR_GID) {
- new_kgid = zfs_fuid_create(zfsvfs,
- (uint64_t)vap->va_gid, cr, ZFS_GROUP, &fuidp);
- if (new_kgid != KGID_TO_SGID(ZTOI(zp)->i_gid) &&
- zfs_id_overquota(zfsvfs, DMU_GROUPUSED_OBJECT,
- new_kgid)) {
- if (attrzp)
- iput(ZTOI(attrzp));
- err = SET_ERROR(EDQUOT);
- goto out2;
- }
- }
-
- if (projid != ZFS_INVALID_PROJID &&
- zfs_id_overquota(zfsvfs, DMU_PROJECTUSED_OBJECT, projid)) {
- if (attrzp)
- iput(ZTOI(attrzp));
- err = EDQUOT;
- goto out2;
- }
- }
- tx = dmu_tx_create(os);
-
- if (mask & ATTR_MODE) {
- uint64_t pmode = zp->z_mode;
- uint64_t acl_obj;
- new_mode = (pmode & S_IFMT) | (vap->va_mode & ~S_IFMT);
-
- zfs_acl_chmod_setattr(zp, &aclp, new_mode);
-
- mutex_enter(&zp->z_lock);
- if (!zp->z_is_sa && ((acl_obj = zfs_external_acl(zp)) != 0)) {
- /*
- * Are we upgrading ACL from old V0 format
- * to V1 format?
- */
- if (zfsvfs->z_version >= ZPL_VERSION_FUID &&
- zfs_znode_acl_version(zp) ==
- ZFS_ACL_VERSION_INITIAL) {
- dmu_tx_hold_free(tx, acl_obj, 0,
- DMU_OBJECT_END);
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
- 0, aclp->z_acl_bytes);
- } else {
- dmu_tx_hold_write(tx, acl_obj, 0,
- aclp->z_acl_bytes);
- }
- } else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
- 0, aclp->z_acl_bytes);
- }
- mutex_exit(&zp->z_lock);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
- } else {
- if (((mask & ATTR_XVATTR) &&
- XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) ||
- (projid != ZFS_INVALID_PROJID &&
- !(zp->z_pflags & ZFS_PROJID)))
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
- else
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- }
-
- if (attrzp) {
- dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, B_FALSE);
- }
-
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
-
- zfs_sa_upgrade_txholds(tx, zp);
-
- err = dmu_tx_assign(tx, TXG_WAIT);
- if (err)
- goto out;
-
- count = 0;
- /*
- * Set each attribute requested.
- * We group settings according to the locks they need to acquire.
- *
- * Note: you cannot set ctime directly, although it will be
- * updated as a side-effect of calling this function.
- */
-
- if (projid != ZFS_INVALID_PROJID && !(zp->z_pflags & ZFS_PROJID)) {
- /*
- * For the existed object that is upgraded from old system,
- * its on-disk layout has no slot for the project ID attribute.
- * But quota accounting logic needs to access related slots by
- * offset directly. So we need to adjust old objects' layout
- * to make the project ID to some unified and fixed offset.
- */
- if (attrzp)
- err = sa_add_projid(attrzp->z_sa_hdl, tx, projid);
- if (err == 0)
- err = sa_add_projid(zp->z_sa_hdl, tx, projid);
-
- if (unlikely(err == EEXIST))
- err = 0;
- else if (err != 0)
- goto out;
- else
- projid = ZFS_INVALID_PROJID;
- }
-
- if (mask & (ATTR_UID|ATTR_GID|ATTR_MODE))
- mutex_enter(&zp->z_acl_lock);
- mutex_enter(&zp->z_lock);
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, sizeof (zp->z_pflags));
-
- if (attrzp) {
- if (mask & (ATTR_UID|ATTR_GID|ATTR_MODE))
- mutex_enter(&attrzp->z_acl_lock);
- mutex_enter(&attrzp->z_lock);
- SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
- SA_ZPL_FLAGS(zfsvfs), NULL, &attrzp->z_pflags,
- sizeof (attrzp->z_pflags));
- if (projid != ZFS_INVALID_PROJID) {
- attrzp->z_projid = projid;
- SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
- SA_ZPL_PROJID(zfsvfs), NULL, &attrzp->z_projid,
- sizeof (attrzp->z_projid));
- }
- }
-
- if (mask & (ATTR_UID|ATTR_GID)) {
-
- if (mask & ATTR_UID) {
- ZTOI(zp)->i_uid = SUID_TO_KUID(new_kuid);
- new_uid = zfs_uid_read(ZTOI(zp));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
- &new_uid, sizeof (new_uid));
- if (attrzp) {
- SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
- SA_ZPL_UID(zfsvfs), NULL, &new_uid,
- sizeof (new_uid));
- ZTOI(attrzp)->i_uid = SUID_TO_KUID(new_uid);
- }
- }
-
- if (mask & ATTR_GID) {
- ZTOI(zp)->i_gid = SGID_TO_KGID(new_kgid);
- new_gid = zfs_gid_read(ZTOI(zp));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs),
- NULL, &new_gid, sizeof (new_gid));
- if (attrzp) {
- SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
- SA_ZPL_GID(zfsvfs), NULL, &new_gid,
- sizeof (new_gid));
- ZTOI(attrzp)->i_gid = SGID_TO_KGID(new_kgid);
- }
- }
- if (!(mask & ATTR_MODE)) {
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs),
- NULL, &new_mode, sizeof (new_mode));
- new_mode = zp->z_mode;
- }
- err = zfs_acl_chown_setattr(zp);
- ASSERT(err == 0);
- if (attrzp) {
- err = zfs_acl_chown_setattr(attrzp);
- ASSERT(err == 0);
- }
- }
-
- if (mask & ATTR_MODE) {
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
- &new_mode, sizeof (new_mode));
- zp->z_mode = ZTOI(zp)->i_mode = new_mode;
- ASSERT3P(aclp, !=, NULL);
- err = zfs_aclset_common(zp, aclp, cr, tx);
- ASSERT0(err);
- if (zp->z_acl_cached)
- zfs_acl_free(zp->z_acl_cached);
- zp->z_acl_cached = aclp;
- aclp = NULL;
- }
-
- if ((mask & ATTR_ATIME) || zp->z_atime_dirty) {
- zp->z_atime_dirty = B_FALSE;
- ZFS_TIME_ENCODE(&ip->i_atime, atime);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
- &atime, sizeof (atime));
- }
-
- if (mask & (ATTR_MTIME | ATTR_SIZE)) {
- ZFS_TIME_ENCODE(&vap->va_mtime, mtime);
- ZTOI(zp)->i_mtime = zpl_inode_timespec_trunc(vap->va_mtime,
- ZTOI(zp)->i_sb->s_time_gran);
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
- mtime, sizeof (mtime));
- }
-
- if (mask & (ATTR_CTIME | ATTR_SIZE)) {
- ZFS_TIME_ENCODE(&vap->va_ctime, ctime);
- ZTOI(zp)->i_ctime = zpl_inode_timespec_trunc(vap->va_ctime,
- ZTOI(zp)->i_sb->s_time_gran);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
- ctime, sizeof (ctime));
- }
-
- if (projid != ZFS_INVALID_PROJID) {
- zp->z_projid = projid;
- SA_ADD_BULK_ATTR(bulk, count,
- SA_ZPL_PROJID(zfsvfs), NULL, &zp->z_projid,
- sizeof (zp->z_projid));
- }
-
- if (attrzp && mask) {
- SA_ADD_BULK_ATTR(xattr_bulk, xattr_count,
- SA_ZPL_CTIME(zfsvfs), NULL, &ctime,
- sizeof (ctime));
- }
-
- /*
- * Do this after setting timestamps to prevent timestamp
- * update from toggling bit
- */
-
- if (xoap && (mask & ATTR_XVATTR)) {
-
- /*
- * restore trimmed off masks
- * so that return masks can be set for caller.
- */
-
- if (XVA_ISSET_REQ(tmpxvattr, XAT_APPENDONLY)) {
- XVA_SET_REQ(xvap, XAT_APPENDONLY);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_NOUNLINK)) {
- XVA_SET_REQ(xvap, XAT_NOUNLINK);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_IMMUTABLE)) {
- XVA_SET_REQ(xvap, XAT_IMMUTABLE);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_NODUMP)) {
- XVA_SET_REQ(xvap, XAT_NODUMP);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_AV_MODIFIED)) {
- XVA_SET_REQ(xvap, XAT_AV_MODIFIED);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_AV_QUARANTINED)) {
- XVA_SET_REQ(xvap, XAT_AV_QUARANTINED);
- }
- if (XVA_ISSET_REQ(tmpxvattr, XAT_PROJINHERIT)) {
- XVA_SET_REQ(xvap, XAT_PROJINHERIT);
- }
-
- if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP))
- ASSERT(S_ISREG(ip->i_mode));
-
- zfs_xvattr_set(zp, xvap, tx);
- }
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- if (mask != 0)
- zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask, fuidp);
-
- mutex_exit(&zp->z_lock);
- if (mask & (ATTR_UID|ATTR_GID|ATTR_MODE))
- mutex_exit(&zp->z_acl_lock);
-
- if (attrzp) {
- if (mask & (ATTR_UID|ATTR_GID|ATTR_MODE))
- mutex_exit(&attrzp->z_acl_lock);
- mutex_exit(&attrzp->z_lock);
- }
-out:
- if (err == 0 && xattr_count > 0) {
- err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk,
- xattr_count, tx);
- ASSERT(err2 == 0);
- }
-
- if (aclp)
- zfs_acl_free(aclp);
-
- if (fuidp) {
- zfs_fuid_info_free(fuidp);
- fuidp = NULL;
- }
-
- if (err) {
- dmu_tx_abort(tx);
- if (attrzp)
- iput(ZTOI(attrzp));
- if (err == ERESTART)
- goto top;
- } else {
- if (count > 0)
- err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
- dmu_tx_commit(tx);
- if (attrzp) {
- if (err2 == 0 && handle_eadir)
- err2 = zfs_setattr_dir(attrzp);
- iput(ZTOI(attrzp));
- }
- zfs_inode_update(zp);
- }
-
-out2:
- if (os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
-out3:
- kmem_free(xattr_bulk, sizeof (sa_bulk_attr_t) * bulks);
- kmem_free(bulk, sizeof (sa_bulk_attr_t) * bulks);
- kmem_free(tmpxvattr, sizeof (xvattr_t));
- ZFS_EXIT(zfsvfs);
- return (err);
-}
-
-typedef struct zfs_zlock {
- krwlock_t *zl_rwlock; /* lock we acquired */
- znode_t *zl_znode; /* znode we held */
- struct zfs_zlock *zl_next; /* next in list */
-} zfs_zlock_t;
-
-/*
- * Drop locks and release vnodes that were held by zfs_rename_lock().
- */
-static void
-zfs_rename_unlock(zfs_zlock_t **zlpp)
-{
- zfs_zlock_t *zl;
-
- while ((zl = *zlpp) != NULL) {
- if (zl->zl_znode != NULL)
- zfs_iput_async(ZTOI(zl->zl_znode));
- rw_exit(zl->zl_rwlock);
- *zlpp = zl->zl_next;
- kmem_free(zl, sizeof (*zl));
- }
-}
-
-/*
- * Search back through the directory tree, using the ".." entries.
- * Lock each directory in the chain to prevent concurrent renames.
- * Fail any attempt to move a directory into one of its own descendants.
- * XXX - z_parent_lock can overlap with map or grow locks
- */
-static int
-zfs_rename_lock(znode_t *szp, znode_t *tdzp, znode_t *sdzp, zfs_zlock_t **zlpp)
-{
- zfs_zlock_t *zl;
- znode_t *zp = tdzp;
- uint64_t rootid = ZTOZSB(zp)->z_root;
- uint64_t oidp = zp->z_id;
- krwlock_t *rwlp = &szp->z_parent_lock;
- krw_t rw = RW_WRITER;
-
- /*
- * First pass write-locks szp and compares to zp->z_id.
- * Later passes read-lock zp and compare to zp->z_parent.
- */
- do {
- if (!rw_tryenter(rwlp, rw)) {
- /*
- * Another thread is renaming in this path.
- * Note that if we are a WRITER, we don't have any
- * parent_locks held yet.
- */
- if (rw == RW_READER && zp->z_id > szp->z_id) {
- /*
- * Drop our locks and restart
- */
- zfs_rename_unlock(&zl);
- *zlpp = NULL;
- zp = tdzp;
- oidp = zp->z_id;
- rwlp = &szp->z_parent_lock;
- rw = RW_WRITER;
- continue;
- } else {
- /*
- * Wait for other thread to drop its locks
- */
- rw_enter(rwlp, rw);
- }
- }
-
- zl = kmem_alloc(sizeof (*zl), KM_SLEEP);
- zl->zl_rwlock = rwlp;
- zl->zl_znode = NULL;
- zl->zl_next = *zlpp;
- *zlpp = zl;
-
- if (oidp == szp->z_id) /* We're a descendant of szp */
- return (SET_ERROR(EINVAL));
-
- if (oidp == rootid) /* We've hit the top */
- return (0);
-
- if (rw == RW_READER) { /* i.e. not the first pass */
- int error = zfs_zget(ZTOZSB(zp), oidp, &zp);
- if (error)
- return (error);
- zl->zl_znode = zp;
- }
- (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(ZTOZSB(zp)),
- &oidp, sizeof (oidp));
- rwlp = &zp->z_parent_lock;
- rw = RW_READER;
-
- } while (zp->z_id != sdzp->z_id);
-
- return (0);
-}
-
-/*
- * Move an entry from the provided source directory to the target
- * directory. Change the entry name as indicated.
- *
- * IN: sdip - Source directory containing the "old entry".
- * snm - Old entry name.
- * tdip - Target directory to contain the "new entry".
- * tnm - New entry name.
- * cr - credentials of caller.
- * flags - case flags
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * sdip,tdip - ctime|mtime updated
- */
-/*ARGSUSED*/
-int
-zfs_rename(struct inode *sdip, char *snm, struct inode *tdip, char *tnm,
- cred_t *cr, int flags)
-{
- znode_t *tdzp, *szp, *tzp;
- znode_t *sdzp = ITOZ(sdip);
- zfsvfs_t *zfsvfs = ITOZSB(sdip);
- zilog_t *zilog;
- zfs_dirlock_t *sdl, *tdl;
- dmu_tx_t *tx;
- zfs_zlock_t *zl;
- int cmp, serr, terr;
- int error = 0;
- int zflg = 0;
- boolean_t waited = B_FALSE;
-
- if (snm == NULL || tnm == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(sdzp);
- zilog = zfsvfs->z_log;
-
- tdzp = ITOZ(tdip);
- ZFS_VERIFY_ZP(tdzp);
-
- /*
- * We check i_sb because snapshots and the ctldir must have different
- * super blocks.
- */
- if (tdip->i_sb != sdip->i_sb || zfsctl_is_node(tdip)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EXDEV));
- }
-
- if (zfsvfs->z_utf8 && u8_validate(tnm,
- strlen(tnm), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
-
- if (flags & FIGNORECASE)
- zflg |= ZCILOOK;
-
-top:
- szp = NULL;
- tzp = NULL;
- zl = NULL;
-
- /*
- * This is to prevent the creation of links into attribute space
- * by renaming a linked file into/outof an attribute directory.
- * See the comment in zfs_link() for why this is considered bad.
- */
- if ((tdzp->z_pflags & ZFS_XATTR) != (sdzp->z_pflags & ZFS_XATTR)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Lock source and target directory entries. To prevent deadlock,
- * a lock ordering must be defined. We lock the directory with
- * the smallest object id first, or if it's a tie, the one with
- * the lexically first name.
- */
- if (sdzp->z_id < tdzp->z_id) {
- cmp = -1;
- } else if (sdzp->z_id > tdzp->z_id) {
- cmp = 1;
- } else {
- /*
- * First compare the two name arguments without
- * considering any case folding.
- */
- int nofold = (zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER);
-
- cmp = u8_strcmp(snm, tnm, 0, nofold, U8_UNICODE_LATEST, &error);
- ASSERT(error == 0 || !zfsvfs->z_utf8);
- if (cmp == 0) {
- /*
- * POSIX: "If the old argument and the new argument
- * both refer to links to the same existing file,
- * the rename() function shall return successfully
- * and perform no other action."
- */
- ZFS_EXIT(zfsvfs);
- return (0);
- }
- /*
- * If the file system is case-folding, then we may
- * have some more checking to do. A case-folding file
- * system is either supporting mixed case sensitivity
- * access or is completely case-insensitive. Note
- * that the file system is always case preserving.
- *
- * In mixed sensitivity mode case sensitive behavior
- * is the default. FIGNORECASE must be used to
- * explicitly request case insensitive behavior.
- *
- * If the source and target names provided differ only
- * by case (e.g., a request to rename 'tim' to 'Tim'),
- * we will treat this as a special case in the
- * case-insensitive mode: as long as the source name
- * is an exact match, we will allow this to proceed as
- * a name-change request.
- */
- if ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE ||
- (zfsvfs->z_case == ZFS_CASE_MIXED &&
- flags & FIGNORECASE)) &&
- u8_strcmp(snm, tnm, 0, zfsvfs->z_norm, U8_UNICODE_LATEST,
- &error) == 0) {
- /*
- * case preserving rename request, require exact
- * name matches
- */
- zflg |= ZCIEXACT;
- zflg &= ~ZCILOOK;
- }
- }
-
- /*
- * If the source and destination directories are the same, we should
- * grab the z_name_lock of that directory only once.
- */
- if (sdzp == tdzp) {
- zflg |= ZHAVELOCK;
- rw_enter(&sdzp->z_name_lock, RW_READER);
- }
-
- if (cmp < 0) {
- serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp,
- ZEXISTS | zflg, NULL, NULL);
- terr = zfs_dirent_lock(&tdl,
- tdzp, tnm, &tzp, ZRENAMING | zflg, NULL, NULL);
- } else {
- terr = zfs_dirent_lock(&tdl,
- tdzp, tnm, &tzp, zflg, NULL, NULL);
- serr = zfs_dirent_lock(&sdl,
- sdzp, snm, &szp, ZEXISTS | ZRENAMING | zflg,
- NULL, NULL);
- }
-
- if (serr) {
- /*
- * Source entry invalid or not there.
- */
- if (!terr) {
- zfs_dirent_unlock(tdl);
- if (tzp)
- iput(ZTOI(tzp));
- }
-
- if (sdzp == tdzp)
- rw_exit(&sdzp->z_name_lock);
-
- if (strcmp(snm, "..") == 0)
- serr = EINVAL;
- ZFS_EXIT(zfsvfs);
- return (serr);
- }
- if (terr) {
- zfs_dirent_unlock(sdl);
- iput(ZTOI(szp));
-
- if (sdzp == tdzp)
- rw_exit(&sdzp->z_name_lock);
-
- if (strcmp(tnm, "..") == 0)
- terr = EINVAL;
- ZFS_EXIT(zfsvfs);
- return (terr);
- }
-
- /*
- * If we are using project inheritance, means if the directory has
- * ZFS_PROJINHERIT set, then its descendant directories will inherit
- * not only the project ID, but also the ZFS_PROJINHERIT flag. Under
- * such case, we only allow renames into our tree when the project
- * IDs are the same.
- */
- if (tdzp->z_pflags & ZFS_PROJINHERIT &&
- tdzp->z_projid != szp->z_projid) {
- error = SET_ERROR(EXDEV);
- goto out;
- }
-
- /*
- * Must have write access at the source to remove the old entry
- * and write access at the target to create the new entry.
- * Note that if target and source are the same, this can be
- * done in a single check.
- */
-
- if ((error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr)))
- goto out;
-
- if (S_ISDIR(ZTOI(szp)->i_mode)) {
- /*
- * Check to make sure rename is valid.
- * Can't do a move like this: /usr/a/b to /usr/a/b/c/d
- */
- if ((error = zfs_rename_lock(szp, tdzp, sdzp, &zl)))
- goto out;
- }
-
- /*
- * Does target exist?
- */
- if (tzp) {
- /*
- * Source and target must be the same type.
- */
- if (S_ISDIR(ZTOI(szp)->i_mode)) {
- if (!S_ISDIR(ZTOI(tzp)->i_mode)) {
- error = SET_ERROR(ENOTDIR);
- goto out;
- }
- } else {
- if (S_ISDIR(ZTOI(tzp)->i_mode)) {
- error = SET_ERROR(EISDIR);
- goto out;
- }
- }
- /*
- * POSIX dictates that when the source and target
- * entries refer to the same file object, rename
- * must do nothing and exit without error.
- */
- if (szp->z_id == tzp->z_id) {
- error = 0;
- goto out;
- }
- }
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_zap(tx, sdzp->z_id, FALSE, snm);
- dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm);
- if (sdzp != tdzp) {
- dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, tdzp);
- }
- if (tzp) {
- dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, tzp);
- }
-
- zfs_sa_upgrade_txholds(tx, szp);
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- if (zl != NULL)
- zfs_rename_unlock(&zl);
- zfs_dirent_unlock(sdl);
- zfs_dirent_unlock(tdl);
-
- if (sdzp == tdzp)
- rw_exit(&sdzp->z_name_lock);
-
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- iput(ZTOI(szp));
- if (tzp)
- iput(ZTOI(tzp));
- goto top;
- }
- dmu_tx_abort(tx);
- iput(ZTOI(szp));
- if (tzp)
- iput(ZTOI(tzp));
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (tzp) /* Attempt to remove the existing target */
- error = zfs_link_destroy(tdl, tzp, tx, zflg, NULL);
-
- if (error == 0) {
- error = zfs_link_create(tdl, szp, tx, ZRENAMING);
- if (error == 0) {
- szp->z_pflags |= ZFS_AV_MODIFIED;
- if (tdzp->z_pflags & ZFS_PROJINHERIT)
- szp->z_pflags |= ZFS_PROJINHERIT;
-
- error = sa_update(szp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs),
- (void *)&szp->z_pflags, sizeof (uint64_t), tx);
- ASSERT0(error);
-
- error = zfs_link_destroy(sdl, szp, tx, ZRENAMING, NULL);
- if (error == 0) {
- zfs_log_rename(zilog, tx, TX_RENAME |
- (flags & FIGNORECASE ? TX_CI : 0), sdzp,
- sdl->dl_name, tdzp, tdl->dl_name, szp);
- } else {
- /*
- * At this point, we have successfully created
- * the target name, but have failed to remove
- * the source name. Since the create was done
- * with the ZRENAMING flag, there are
- * complications; for one, the link count is
- * wrong. The easiest way to deal with this
- * is to remove the newly created target, and
- * return the original error. This must
- * succeed; fortunately, it is very unlikely to
- * fail, since we just created it.
- */
- VERIFY3U(zfs_link_destroy(tdl, szp, tx,
- ZRENAMING, NULL), ==, 0);
- }
- } else {
- /*
- * If we had removed the existing target, subsequent
- * call to zfs_link_create() to add back the same entry
- * but, the new dnode (szp) should not fail.
- */
- ASSERT(tzp == NULL);
- }
- }
-
- dmu_tx_commit(tx);
-out:
- if (zl != NULL)
- zfs_rename_unlock(&zl);
-
- zfs_dirent_unlock(sdl);
- zfs_dirent_unlock(tdl);
-
- zfs_inode_update(sdzp);
- if (sdzp == tdzp)
- rw_exit(&sdzp->z_name_lock);
-
- if (sdzp != tdzp)
- zfs_inode_update(tdzp);
-
- zfs_inode_update(szp);
- iput(ZTOI(szp));
- if (tzp) {
- zfs_inode_update(tzp);
- iput(ZTOI(tzp));
- }
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Insert the indicated symbolic reference entry into the directory.
- *
- * IN: dip - Directory to contain new symbolic link.
- * name - Name of directory entry in dip.
- * vap - Attributes of new entry.
- * link - Name for new symlink entry.
- * cr - credentials of caller.
- * flags - case flags
- *
- * OUT: ipp - Inode for new symbolic link.
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * dip - ctime|mtime updated
- */
-/*ARGSUSED*/
-int
-zfs_symlink(struct inode *dip, char *name, vattr_t *vap, char *link,
- struct inode **ipp, cred_t *cr, int flags)
-{
- znode_t *zp, *dzp = ITOZ(dip);
- zfs_dirlock_t *dl;
- dmu_tx_t *tx;
- zfsvfs_t *zfsvfs = ITOZSB(dip);
- zilog_t *zilog;
- uint64_t len = strlen(link);
- int error;
- int zflg = ZNEW;
- zfs_acl_ids_t acl_ids;
- boolean_t fuid_dirtied;
- uint64_t txtype = TX_SYMLINK;
- boolean_t waited = B_FALSE;
-
- ASSERT(S_ISLNK(vap->va_mode));
-
- if (name == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- zilog = zfsvfs->z_log;
-
- if (zfsvfs->z_utf8 && u8_validate(name, strlen(name),
- NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
- if (flags & FIGNORECASE)
- zflg |= ZCILOOK;
-
- if (len > MAXPATHLEN) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENAMETOOLONG));
- }
-
- if ((error = zfs_acl_ids_create(dzp, 0,
- vap, cr, NULL, &acl_ids)) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-top:
- *ipp = NULL;
-
- /*
- * Attempt to lock directory; fail if entry already exists.
- */
- error = zfs_dirent_lock(&dl, dzp, name, &zp, zflg, NULL, NULL);
- if (error) {
- zfs_acl_ids_free(&acl_ids);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) {
- zfs_acl_ids_free(&acl_ids);
- zfs_dirent_unlock(dl);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, ZFS_DEFAULT_PROJID)) {
- zfs_acl_ids_free(&acl_ids);
- zfs_dirent_unlock(dl);
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EDQUOT));
- }
- tx = dmu_tx_create(zfsvfs->z_os);
- fuid_dirtied = zfsvfs->z_fuid_dirty;
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len));
- dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
- dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
- ZFS_SA_BASE_ATTR_SIZE + len);
- dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
- if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) {
- dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0,
- acl_ids.z_aclp->z_acl_bytes);
- }
- if (fuid_dirtied)
- zfs_fuid_txhold(zfsvfs, tx);
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- zfs_acl_ids_free(&acl_ids);
- dmu_tx_abort(tx);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- /*
- * Create a new object for the symlink.
- * for version 4 ZPL datsets the symlink will be an SA attribute
- */
- zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids);
-
- if (fuid_dirtied)
- zfs_fuid_sync(zfsvfs, tx);
-
- mutex_enter(&zp->z_lock);
- if (zp->z_is_sa)
- error = sa_update(zp->z_sa_hdl, SA_ZPL_SYMLINK(zfsvfs),
- link, len, tx);
- else
- zfs_sa_symlink(zp, link, len, tx);
- mutex_exit(&zp->z_lock);
-
- zp->z_size = len;
- (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
- &zp->z_size, sizeof (zp->z_size), tx);
- /*
- * Insert the new object into the directory.
- */
- error = zfs_link_create(dl, zp, tx, ZNEW);
- if (error != 0) {
- zfs_znode_delete(zp, tx);
- remove_inode_hash(ZTOI(zp));
- } else {
- if (flags & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_symlink(zilog, tx, txtype, dzp, zp, name, link);
-
- zfs_inode_update(dzp);
- zfs_inode_update(zp);
- }
-
- zfs_acl_ids_free(&acl_ids);
-
- dmu_tx_commit(tx);
-
- zfs_dirent_unlock(dl);
-
- if (error == 0) {
- *ipp = ZTOI(zp);
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
- } else {
- iput(ZTOI(zp));
- }
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Return, in the buffer contained in the provided uio structure,
- * the symbolic path referred to by ip.
- *
- * IN: ip - inode of symbolic link
- * uio - structure to contain the link path.
- * cr - credentials of caller.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * ip - atime updated
- */
-/* ARGSUSED */
-int
-zfs_readlink(struct inode *ip, uio_t *uio, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- mutex_enter(&zp->z_lock);
- if (zp->z_is_sa)
- error = sa_lookup_uio(zp->z_sa_hdl,
- SA_ZPL_SYMLINK(zfsvfs), uio);
- else
- error = zfs_sa_readlink(zp, uio);
- mutex_exit(&zp->z_lock);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*
- * Insert a new entry into directory tdip referencing sip.
- *
- * IN: tdip - Directory to contain new entry.
- * sip - inode of new entry.
- * name - name of new entry.
- * cr - credentials of caller.
- * flags - case flags.
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * tdip - ctime|mtime updated
- * sip - ctime updated
- */
-/* ARGSUSED */
-int
-zfs_link(struct inode *tdip, struct inode *sip, char *name, cred_t *cr,
- int flags)
-{
- znode_t *dzp = ITOZ(tdip);
- znode_t *tzp, *szp;
- zfsvfs_t *zfsvfs = ITOZSB(tdip);
- zilog_t *zilog;
- zfs_dirlock_t *dl;
- dmu_tx_t *tx;
- int error;
- int zf = ZNEW;
- uint64_t parent;
- uid_t owner;
- boolean_t waited = B_FALSE;
- boolean_t is_tmpfile = 0;
- uint64_t txg;
-#ifdef HAVE_TMPFILE
- is_tmpfile = (sip->i_nlink == 0 && (sip->i_state & I_LINKABLE));
-#endif
- ASSERT(S_ISDIR(tdip->i_mode));
-
- if (name == NULL)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(dzp);
- zilog = zfsvfs->z_log;
-
- /*
- * POSIX dictates that we return EPERM here.
- * Better choices include ENOTSUP or EISDIR.
- */
- if (S_ISDIR(sip->i_mode)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- szp = ITOZ(sip);
- ZFS_VERIFY_ZP(szp);
-
- /*
- * If we are using project inheritance, means if the directory has
- * ZFS_PROJINHERIT set, then its descendant directories will inherit
- * not only the project ID, but also the ZFS_PROJINHERIT flag. Under
- * such case, we only allow hard link creation in our tree when the
- * project IDs are the same.
- */
- if (dzp->z_pflags & ZFS_PROJINHERIT && dzp->z_projid != szp->z_projid) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EXDEV));
- }
-
- /*
- * We check i_sb because snapshots and the ctldir must have different
- * super blocks.
- */
- if (sip->i_sb != tdip->i_sb || zfsctl_is_node(sip)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EXDEV));
- }
-
- /* Prevent links to .zfs/shares files */
-
- if ((error = sa_lookup(szp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
- &parent, sizeof (uint64_t))) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- if (parent == zfsvfs->z_shares_dir) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- if (zfsvfs->z_utf8 && u8_validate(name,
- strlen(name), NULL, U8_VALIDATE_ENTIRE, &error) < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EILSEQ));
- }
- if (flags & FIGNORECASE)
- zf |= ZCILOOK;
-
- /*
- * We do not support links between attributes and non-attributes
- * because of the potential security risk of creating links
- * into "normal" file space in order to circumvent restrictions
- * imposed in attribute space.
- */
- if ((szp->z_pflags & ZFS_XATTR) != (dzp->z_pflags & ZFS_XATTR)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- owner = zfs_fuid_map_id(zfsvfs, KUID_TO_SUID(sip->i_uid),
- cr, ZFS_OWNER);
- if (owner != crgetuid(cr) && secpolicy_basic_link(cr) != 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
-top:
- /*
- * Attempt to lock directory; fail if entry already exists.
- */
- error = zfs_dirent_lock(&dl, dzp, name, &tzp, zf, NULL, NULL);
- if (error) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE);
- dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
- if (is_tmpfile)
- dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
-
- zfs_sa_upgrade_txholds(tx, szp);
- zfs_sa_upgrade_txholds(tx, dzp);
- error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT);
- if (error) {
- zfs_dirent_unlock(dl);
- if (error == ERESTART) {
- waited = B_TRUE;
- dmu_tx_wait(tx);
- dmu_tx_abort(tx);
- goto top;
- }
- dmu_tx_abort(tx);
- ZFS_EXIT(zfsvfs);
- return (error);
- }
- /* unmark z_unlinked so zfs_link_create will not reject */
- if (is_tmpfile)
- szp->z_unlinked = B_FALSE;
- error = zfs_link_create(dl, szp, tx, 0);
-
- if (error == 0) {
- uint64_t txtype = TX_LINK;
- /*
- * tmpfile is created to be in z_unlinkedobj, so remove it.
- * Also, we don't log in ZIL, because all previous file
- * operation on the tmpfile are ignored by ZIL. Instead we
- * always wait for txg to sync to make sure all previous
- * operation are sync safe.
- */
- if (is_tmpfile) {
- VERIFY(zap_remove_int(zfsvfs->z_os,
- zfsvfs->z_unlinkedobj, szp->z_id, tx) == 0);
- } else {
- if (flags & FIGNORECASE)
- txtype |= TX_CI;
- zfs_log_link(zilog, tx, txtype, dzp, szp, name);
- }
- } else if (is_tmpfile) {
- /* restore z_unlinked since when linking failed */
- szp->z_unlinked = B_TRUE;
- }
- txg = dmu_tx_get_txg(tx);
- dmu_tx_commit(tx);
-
- zfs_dirent_unlock(dl);
-
- if (!is_tmpfile && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- if (is_tmpfile)
- txg_wait_synced(dmu_objset_pool(zfsvfs->z_os), txg);
-
- zfs_inode_update(dzp);
- zfs_inode_update(szp);
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-static void
-zfs_putpage_commit_cb(void *arg)
-{
- struct page *pp = arg;
-
- ClearPageError(pp);
- end_page_writeback(pp);
-}
-
-/*
- * Push a page out to disk, once the page is on stable storage the
- * registered commit callback will be run as notification of completion.
- *
- * IN: ip - page mapped for inode.
- * pp - page to push (page is locked)
- * wbc - writeback control data
- *
- * RETURN: 0 if success
- * error code if failure
- *
- * Timestamps:
- * ip - ctime|mtime updated
- */
-/* ARGSUSED */
-int
-zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- loff_t offset;
- loff_t pgoff;
- unsigned int pglen;
- dmu_tx_t *tx;
- caddr_t va;
- int err = 0;
- uint64_t mtime[2], ctime[2];
- sa_bulk_attr_t bulk[3];
- int cnt = 0;
- struct address_space *mapping;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- ASSERT(PageLocked(pp));
-
- pgoff = page_offset(pp); /* Page byte-offset in file */
- offset = i_size_read(ip); /* File length in bytes */
- pglen = MIN(PAGE_SIZE, /* Page length in bytes */
- P2ROUNDUP(offset, PAGE_SIZE)-pgoff);
-
- /* Page is beyond end of file */
- if (pgoff >= offset) {
- unlock_page(pp);
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- /* Truncate page length to end of file */
- if (pgoff + pglen > offset)
- pglen = offset - pgoff;
-
-#if 0
- /*
- * FIXME: Allow mmap writes past its quota. The correct fix
- * is to register a page_mkwrite() handler to count the page
- * against its quota when it is about to be dirtied.
- */
- if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT,
- KUID_TO_SUID(ip->i_uid)) ||
- zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT,
- KGID_TO_SGID(ip->i_gid)) ||
- (zp->z_projid != ZFS_DEFAULT_PROJID &&
- zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
- zp->z_projid))) {
- err = EDQUOT;
- }
-#endif
-
- /*
- * The ordering here is critical and must adhere to the following
- * rules in order to avoid deadlocking in either zfs_read() or
- * zfs_free_range() due to a lock inversion.
- *
- * 1) The page must be unlocked prior to acquiring the range lock.
- * This is critical because zfs_read() calls find_lock_page()
- * which may block on the page lock while holding the range lock.
- *
- * 2) Before setting or clearing write back on a page the range lock
- * must be held in order to prevent a lock inversion with the
- * zfs_free_range() function.
- *
- * This presents a problem because upon entering this function the
- * page lock is already held. To safely acquire the range lock the
- * page lock must be dropped. This creates a window where another
- * process could truncate, invalidate, dirty, or write out the page.
- *
- * Therefore, after successfully reacquiring the range and page locks
- * the current page state is checked. In the common case everything
- * will be as is expected and it can be written out. However, if
- * the page state has changed it must be handled accordingly.
- */
- mapping = pp->mapping;
- redirty_page_for_writepage(wbc, pp);
- unlock_page(pp);
-
- locked_range_t *lr = rangelock_enter(&zp->z_rangelock,
- pgoff, pglen, RL_WRITER);
- lock_page(pp);
-
- /* Page mapping changed or it was no longer dirty, we're done */
- if (unlikely((mapping != pp->mapping) || !PageDirty(pp))) {
- unlock_page(pp);
- rangelock_exit(lr);
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- /* Another process started write block if required */
- if (PageWriteback(pp)) {
- unlock_page(pp);
- rangelock_exit(lr);
-
- if (wbc->sync_mode != WB_SYNC_NONE) {
- if (PageWriteback(pp))
- wait_on_page_bit(pp, PG_writeback);
- }
-
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- /* Clear the dirty flag the required locks are held */
- if (!clear_page_dirty_for_io(pp)) {
- unlock_page(pp);
- rangelock_exit(lr);
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- /*
- * Counterpart for redirty_page_for_writepage() above. This page
- * was in fact not skipped and should not be counted as if it were.
- */
- wbc->pages_skipped--;
- set_page_writeback(pp);
- unlock_page(pp);
-
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_write(tx, zp->z_id, pgoff, pglen);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
-
- err = dmu_tx_assign(tx, TXG_NOWAIT);
- if (err != 0) {
- if (err == ERESTART)
- dmu_tx_wait(tx);
-
- dmu_tx_abort(tx);
- __set_page_dirty_nobuffers(pp);
- ClearPageError(pp);
- end_page_writeback(pp);
- rangelock_exit(lr);
- ZFS_EXIT(zfsvfs);
- return (err);
- }
-
- va = kmap(pp);
- ASSERT3U(pglen, <=, PAGE_SIZE);
- dmu_write(zfsvfs->z_os, zp->z_id, pgoff, pglen, va, tx);
- kunmap(pp);
-
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, 8);
-
- /* Preserve the mtime and ctime provided by the inode */
- ZFS_TIME_ENCODE(&ip->i_mtime, mtime);
- ZFS_TIME_ENCODE(&ip->i_ctime, ctime);
- zp->z_atime_dirty = B_FALSE;
- zp->z_seq++;
-
- err = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx);
-
- zfs_log_write(zfsvfs->z_log, tx, TX_WRITE, zp, pgoff, pglen, 0,
- zfs_putpage_commit_cb, pp);
- dmu_tx_commit(tx);
-
- rangelock_exit(lr);
-
- if (wbc->sync_mode != WB_SYNC_NONE) {
- /*
- * Note that this is rarely called under writepages(), because
- * writepages() normally handles the entire commit for
- * performance reasons.
- */
- zil_commit(zfsvfs->z_log, zp->z_id);
- }
-
- ZFS_EXIT(zfsvfs);
- return (err);
-}
-
-/*
- * Update the system attributes when the inode has been dirtied. For the
- * moment we only update the mode, atime, mtime, and ctime.
- */
-int
-zfs_dirty_inode(struct inode *ip, int flags)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- dmu_tx_t *tx;
- uint64_t mode, atime[2], mtime[2], ctime[2];
- sa_bulk_attr_t bulk[4];
- int error = 0;
- int cnt = 0;
-
- if (zfs_is_readonly(zfsvfs) || dmu_objset_is_snapshot(zfsvfs->z_os))
- return (0);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
-#ifdef I_DIRTY_TIME
- /*
- * This is the lazytime semantic introduced in Linux 4.0
- * This flag will only be called from update_time when lazytime is set.
- * (Note, I_DIRTY_SYNC will also set if not lazytime)
- * Fortunately mtime and ctime are managed within ZFS itself, so we
- * only need to dirty atime.
- */
- if (flags == I_DIRTY_TIME) {
- zp->z_atime_dirty = B_TRUE;
- goto out;
- }
-#endif
-
- tx = dmu_tx_create(zfsvfs->z_os);
-
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
-
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- goto out;
- }
-
- mutex_enter(&zp->z_lock);
- zp->z_atime_dirty = B_FALSE;
-
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8);
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
-
- /* Preserve the mode, mtime and ctime provided by the inode */
- ZFS_TIME_ENCODE(&ip->i_atime, atime);
- ZFS_TIME_ENCODE(&ip->i_mtime, mtime);
- ZFS_TIME_ENCODE(&ip->i_ctime, ctime);
- mode = ip->i_mode;
-
- zp->z_mode = mode;
-
- error = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx);
- mutex_exit(&zp->z_lock);
-
- dmu_tx_commit(tx);
-out:
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*ARGSUSED*/
-void
-zfs_inactive(struct inode *ip)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- uint64_t atime[2];
- int error;
- int need_unlock = 0;
-
- /* Only read lock if we haven't already write locked, e.g. rollback */
- if (!RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock)) {
- need_unlock = 1;
- rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_READER);
- }
- if (zp->z_sa_hdl == NULL) {
- if (need_unlock)
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
- return;
- }
-
- if (zp->z_atime_dirty && zp->z_unlinked == B_FALSE) {
- dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
-
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- } else {
- ZFS_TIME_ENCODE(&ip->i_atime, atime);
- mutex_enter(&zp->z_lock);
- (void) sa_update(zp->z_sa_hdl, SA_ZPL_ATIME(zfsvfs),
- (void *)&atime, sizeof (atime), tx);
- zp->z_atime_dirty = B_FALSE;
- mutex_exit(&zp->z_lock);
- dmu_tx_commit(tx);
- }
- }
-
- zfs_zinactive(zp);
- if (need_unlock)
- rw_exit(&zfsvfs->z_teardown_inactive_lock);
-}
-
-/*
- * Bounds-check the seek operation.
- *
- * IN: ip - inode seeking within
- * ooff - old file offset
- * noffp - pointer to new file offset
- *
- * RETURN: 0 if success
- * EINVAL if new offset invalid
- */
-/* ARGSUSED */
-int
-zfs_seek(struct inode *ip, offset_t ooff, offset_t *noffp)
-{
- if (S_ISDIR(ip->i_mode))
- return (0);
- return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
-}
-
-/*
- * Fill pages with data from the disk.
- */
-static int
-zfs_fillpage(struct inode *ip, struct page *pl[], int nr_pages)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- objset_t *os;
- struct page *cur_pp;
- u_offset_t io_off, total;
- size_t io_len;
- loff_t i_size;
- unsigned page_idx;
- int err;
-
- os = zfsvfs->z_os;
- io_len = nr_pages << PAGE_SHIFT;
- i_size = i_size_read(ip);
- io_off = page_offset(pl[0]);
-
- if (io_off + io_len > i_size)
- io_len = i_size - io_off;
-
- /*
- * Iterate over list of pages and read each page individually.
- */
- page_idx = 0;
- for (total = io_off + io_len; io_off < total; io_off += PAGESIZE) {
- caddr_t va;
-
- cur_pp = pl[page_idx++];
- va = kmap(cur_pp);
- err = dmu_read(os, zp->z_id, io_off, PAGESIZE, va,
- DMU_READ_PREFETCH);
- kunmap(cur_pp);
- if (err) {
- /* convert checksum errors into IO errors */
- if (err == ECKSUM)
- err = SET_ERROR(EIO);
- return (err);
- }
- }
-
- return (0);
-}
-
-/*
- * Uses zfs_fillpage to read data from the file and fill the pages.
- *
- * IN: ip - inode of file to get data from.
- * pl - list of pages to read
- * nr_pages - number of pages to read
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * vp - atime updated
- */
-/* ARGSUSED */
-int
-zfs_getpage(struct inode *ip, struct page *pl[], int nr_pages)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int err;
-
- if (pl == NULL)
- return (0);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- err = zfs_fillpage(ip, pl, nr_pages);
-
- ZFS_EXIT(zfsvfs);
- return (err);
-}
-
-/*
- * Check ZFS specific permissions to memory map a section of a file.
- *
- * IN: ip - inode of the file to mmap
- * off - file offset
- * addrp - start address in memory region
- * len - length of memory region
- * vm_flags- address flags
- *
- * RETURN: 0 if success
- * error code if failure
- */
-/*ARGSUSED*/
-int
-zfs_map(struct inode *ip, offset_t off, caddr_t *addrp, size_t len,
- unsigned long vm_flags)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if ((vm_flags & VM_WRITE) && (zp->z_pflags &
- (ZFS_IMMUTABLE | ZFS_READONLY | ZFS_APPENDONLY))) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EPERM));
- }
-
- if ((vm_flags & (VM_READ | VM_EXEC)) &&
- (zp->z_pflags & ZFS_AV_QUARANTINED)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EACCES));
- }
-
- if (off < 0 || len > MAXOFFSET_T - off) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(ENXIO));
- }
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*
- * convoff - converts the given data (start, whence) to the
- * given whence.
- */
-int
-convoff(struct inode *ip, flock64_t *lckdat, int whence, offset_t offset)
-{
- vattr_t vap;
- int error;
-
- if ((lckdat->l_whence == SEEK_END) || (whence == SEEK_END)) {
- if ((error = zfs_getattr(ip, &vap, 0, CRED())))
- return (error);
- }
-
- switch (lckdat->l_whence) {
- case SEEK_CUR:
- lckdat->l_start += offset;
- break;
- case SEEK_END:
- lckdat->l_start += vap.va_size;
- /* FALLTHRU */
- case SEEK_SET:
- break;
- default:
- return (SET_ERROR(EINVAL));
- }
-
- if (lckdat->l_start < 0)
- return (SET_ERROR(EINVAL));
-
- switch (whence) {
- case SEEK_CUR:
- lckdat->l_start -= offset;
- break;
- case SEEK_END:
- lckdat->l_start -= vap.va_size;
- /* FALLTHRU */
- case SEEK_SET:
- break;
- default:
- return (SET_ERROR(EINVAL));
- }
-
- lckdat->l_whence = (short)whence;
- return (0);
-}
-
-/*
- * Free or allocate space in a file. Currently, this function only
- * supports the `F_FREESP' command. However, this command is somewhat
- * misnamed, as its functionality includes the ability to allocate as
- * well as free space.
- *
- * IN: ip - inode of file to free data in.
- * cmd - action to take (only F_FREESP supported).
- * bfp - section of file to free/alloc.
- * flag - current file open mode flags.
- * offset - current file offset.
- * cr - credentials of caller.
- *
- * RETURN: 0 on success, error code on failure.
- *
- * Timestamps:
- * ip - ctime|mtime updated
- */
-/* ARGSUSED */
-int
-zfs_space(struct inode *ip, int cmd, flock64_t *bfp, int flag,
- offset_t offset, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- uint64_t off, len;
- int error;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if (cmd != F_FREESP) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Callers might not be able to detect properly that we are read-only,
- * so check it explicitly here.
- */
- if (zfs_is_readonly(zfsvfs)) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EROFS));
- }
-
- if ((error = convoff(ip, bfp, SEEK_SET, offset))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- if (bfp->l_len < 0) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- /*
- * Permissions aren't checked on Solaris because on this OS
- * zfs_space() can only be called with an opened file handle.
- * On Linux we can get here through truncate_range() which
- * operates directly on inodes, so we need to check access rights.
- */
- if ((error = zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr))) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- off = bfp->l_start;
- len = bfp->l_len; /* 0 means from off to end of file */
-
- error = zfs_freesp(zp, off, len, flag, TRUE);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-/*ARGSUSED*/
-int
-zfs_fid(struct inode *ip, fid_t *fidp)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- uint32_t gen;
- uint64_t gen64;
- uint64_t object = zp->z_id;
- zfid_short_t *zfid;
- int size, i, error;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs),
- &gen64, sizeof (uint64_t))) != 0) {
- ZFS_EXIT(zfsvfs);
- return (error);
- }
-
- gen = (uint32_t)gen64;
-
- size = SHORT_FID_LEN;
-
- zfid = (zfid_short_t *)fidp;
-
- zfid->zf_len = size;
-
- for (i = 0; i < sizeof (zfid->zf_object); i++)
- zfid->zf_object[i] = (uint8_t)(object >> (8 * i));
-
- /* Must have a non-zero generation number to distinguish from .zfs */
- if (gen == 0)
- gen = 1;
- for (i = 0; i < sizeof (zfid->zf_gen); i++)
- zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));
-
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*ARGSUSED*/
-int
-zfs_getsecattr(struct inode *ip, vsecattr_t *vsecp, int flag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error;
- boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
- error = zfs_getacl(zp, vsecp, skipaclchk, cr);
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-/*ARGSUSED*/
-int
-zfs_setsecattr(struct inode *ip, vsecattr_t *vsecp, int flag, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int error;
- boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
- zilog_t *zilog = zfsvfs->z_log;
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
-
- error = zfs_setacl(zp, vsecp, skipaclchk, cr);
-
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- zil_commit(zilog, 0);
-
- ZFS_EXIT(zfsvfs);
- return (error);
-}
-
-#ifdef HAVE_UIO_ZEROCOPY
-/*
- * The smallest read we may consider to loan out an arcbuf.
- * This must be a power of 2.
- */
-int zcr_blksz_min = (1 << 10); /* 1K */
-/*
- * If set to less than the file block size, allow loaning out of an
- * arcbuf for a partial block read. This must be a power of 2.
- */
-int zcr_blksz_max = (1 << 17); /* 128K */
-
-/*ARGSUSED*/
-static int
-zfs_reqzcbuf(struct inode *ip, enum uio_rw ioflag, xuio_t *xuio, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- int max_blksz = zfsvfs->z_max_blksz;
- uio_t *uio = &xuio->xu_uio;
- ssize_t size = uio->uio_resid;
- offset_t offset = uio->uio_loffset;
- int blksz;
- int fullblk, i;
- arc_buf_t *abuf;
- ssize_t maxsize;
- int preamble, postamble;
-
- if (xuio->xu_type != UIOTYPE_ZEROCOPY)
- return (SET_ERROR(EINVAL));
-
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
- switch (ioflag) {
- case UIO_WRITE:
- /*
- * Loan out an arc_buf for write if write size is bigger than
- * max_blksz, and the file's block size is also max_blksz.
- */
- blksz = max_blksz;
- if (size < blksz || zp->z_blksz != blksz) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
- /*
- * Caller requests buffers for write before knowing where the
- * write offset might be (e.g. NFS TCP write).
- */
- if (offset == -1) {
- preamble = 0;
- } else {
- preamble = P2PHASE(offset, blksz);
- if (preamble) {
- preamble = blksz - preamble;
- size -= preamble;
- }
- }
-
- postamble = P2PHASE(size, blksz);
- size -= postamble;
-
- fullblk = size / blksz;
- (void) dmu_xuio_init(xuio,
- (preamble != 0) + fullblk + (postamble != 0));
-
- /*
- * Have to fix iov base/len for partial buffers. They
- * currently represent full arc_buf's.
- */
- if (preamble) {
- /* data begins in the middle of the arc_buf */
- abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
- blksz);
- ASSERT(abuf);
- (void) dmu_xuio_add(xuio, abuf,
- blksz - preamble, preamble);
- }
-
- for (i = 0; i < fullblk; i++) {
- abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
- blksz);
- ASSERT(abuf);
- (void) dmu_xuio_add(xuio, abuf, 0, blksz);
- }
-
- if (postamble) {
- /* data ends in the middle of the arc_buf */
- abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
- blksz);
- ASSERT(abuf);
- (void) dmu_xuio_add(xuio, abuf, 0, postamble);
- }
- break;
- case UIO_READ:
- /*
- * Loan out an arc_buf for read if the read size is larger than
- * the current file block size. Block alignment is not
- * considered. Partial arc_buf will be loaned out for read.
- */
- blksz = zp->z_blksz;
- if (blksz < zcr_blksz_min)
- blksz = zcr_blksz_min;
- if (blksz > zcr_blksz_max)
- blksz = zcr_blksz_max;
- /* avoid potential complexity of dealing with it */
- if (blksz > max_blksz) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- maxsize = zp->z_size - uio->uio_loffset;
- if (size > maxsize)
- size = maxsize;
-
- if (size < blksz) {
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
- break;
- default:
- ZFS_EXIT(zfsvfs);
- return (SET_ERROR(EINVAL));
- }
-
- uio->uio_extflg = UIO_XUIO;
- XUIO_XUZC_RW(xuio) = ioflag;
- ZFS_EXIT(zfsvfs);
- return (0);
-}
-
-/*ARGSUSED*/
-static int
-zfs_retzcbuf(struct inode *ip, xuio_t *xuio, cred_t *cr)
-{
- int i;
- arc_buf_t *abuf;
- int ioflag = XUIO_XUZC_RW(xuio);
-
- ASSERT(xuio->xu_type == UIOTYPE_ZEROCOPY);
-
- i = dmu_xuio_cnt(xuio);
- while (i-- > 0) {
- abuf = dmu_xuio_arcbuf(xuio, i);
- /*
- * if abuf == NULL, it must be a write buffer
- * that has been returned in zfs_write().
- */
- if (abuf)
- dmu_return_arcbuf(abuf);
- ASSERT(abuf || ioflag == UIO_WRITE);
- }
-
- dmu_xuio_fini(xuio);
- return (0);
-}
-#endif /* HAVE_UIO_ZEROCOPY */
-
-#if defined(_KERNEL)
-EXPORT_SYMBOL(zfs_open);
-EXPORT_SYMBOL(zfs_close);
-EXPORT_SYMBOL(zfs_read);
-EXPORT_SYMBOL(zfs_write);
-EXPORT_SYMBOL(zfs_access);
-EXPORT_SYMBOL(zfs_lookup);
-EXPORT_SYMBOL(zfs_create);
-EXPORT_SYMBOL(zfs_tmpfile);
-EXPORT_SYMBOL(zfs_remove);
-EXPORT_SYMBOL(zfs_mkdir);
-EXPORT_SYMBOL(zfs_rmdir);
-EXPORT_SYMBOL(zfs_readdir);
-EXPORT_SYMBOL(zfs_fsync);
-EXPORT_SYMBOL(zfs_getattr);
-EXPORT_SYMBOL(zfs_getattr_fast);
-EXPORT_SYMBOL(zfs_setattr);
-EXPORT_SYMBOL(zfs_rename);
-EXPORT_SYMBOL(zfs_symlink);
-EXPORT_SYMBOL(zfs_readlink);
-EXPORT_SYMBOL(zfs_link);
-EXPORT_SYMBOL(zfs_inactive);
-EXPORT_SYMBOL(zfs_space);
-EXPORT_SYMBOL(zfs_fid);
-EXPORT_SYMBOL(zfs_getsecattr);
-EXPORT_SYMBOL(zfs_setsecattr);
-EXPORT_SYMBOL(zfs_getpage);
-EXPORT_SYMBOL(zfs_putpage);
-EXPORT_SYMBOL(zfs_dirty_inode);
-EXPORT_SYMBOL(zfs_map);
-
-/* BEGIN CSTYLED */
-module_param(zfs_delete_blocks, ulong, 0644);
-MODULE_PARM_DESC(zfs_delete_blocks, "Delete files larger than N blocks async");
-module_param(zfs_read_chunk_size, ulong, 0644);
-MODULE_PARM_DESC(zfs_read_chunk_size, "Bytes to read per chunk");
-/* END CSTYLED */
-
-#endif
diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c
deleted file mode 100644
index 549c701a0..000000000
--- a/module/zfs/zfs_znode.c
+++ /dev/null
@@ -1,2234 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
- */
-
-/* Portions Copyright 2007 Jeremy Teo */
-
-#ifdef _KERNEL
-#include <sys/types.h>
-#include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysmacros.h>
-#include <sys/mntent.h>
-#include <sys/u8_textprep.h>
-#include <sys/dsl_dataset.h>
-#include <sys/vfs.h>
-#include <sys/vnode.h>
-#include <sys/file.h>
-#include <sys/kmem.h>
-#include <sys/errno.h>
-#include <sys/mode.h>
-#include <sys/atomic.h>
-#include <sys/zfs_dir.h>
-#include <sys/zfs_acl.h>
-#include <sys/zfs_ioctl.h>
-#include <sys/zfs_rlock.h>
-#include <sys/zfs_fuid.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/dnode.h>
-#include <sys/fs/zfs.h>
-#include <sys/zpl.h>
-#endif /* _KERNEL */
-
-#include <sys/dmu.h>
-#include <sys/dmu_objset.h>
-#include <sys/dmu_tx.h>
-#include <sys/refcount.h>
-#include <sys/stat.h>
-#include <sys/zap.h>
-#include <sys/zfs_znode.h>
-#include <sys/sa.h>
-#include <sys/zfs_sa.h>
-#include <sys/zfs_stat.h>
-
-#include "zfs_prop.h"
-#include "zfs_comutil.h"
-
-/*
- * Functions needed for userland (ie: libzpool) are not put under
- * #ifdef_KERNEL; the rest of the functions have dependencies
- * (such as VFS logic) that will not compile easily in userland.
- */
-#ifdef _KERNEL
-
-static kmem_cache_t *znode_cache = NULL;
-static kmem_cache_t *znode_hold_cache = NULL;
-unsigned int zfs_object_mutex_size = ZFS_OBJ_MTX_SZ;
-
-/*
- * This is used by the test suite so that it can delay znodes from being
- * freed in order to inspect the unlinked set.
- */
-int zfs_unlink_suspend_progress = 0;
-
-/*
- * This callback is invoked when acquiring a RL_WRITER or RL_APPEND lock on
- * z_rangelock. It will modify the offset and length of the lock to reflect
- * znode-specific information, and convert RL_APPEND to RL_WRITER. This is
- * called with the rangelock_t's rl_lock held, which avoids races.
- */
-static void
-zfs_rangelock_cb(locked_range_t *new, void *arg)
-{
- znode_t *zp = arg;
-
- /*
- * If in append mode, convert to writer and lock starting at the
- * current end of file.
- */
- if (new->lr_type == RL_APPEND) {
- new->lr_offset = zp->z_size;
- new->lr_type = RL_WRITER;
- }
-
- /*
- * If we need to grow the block size then lock the whole file range.
- */
- uint64_t end_size = MAX(zp->z_size, new->lr_offset + new->lr_length);
- if (end_size > zp->z_blksz && (!ISP2(zp->z_blksz) ||
- zp->z_blksz < ZTOZSB(zp)->z_max_blksz)) {
- new->lr_offset = 0;
- new->lr_length = UINT64_MAX;
- }
-}
-
-/*ARGSUSED*/
-static int
-zfs_znode_cache_constructor(void *buf, void *arg, int kmflags)
-{
- znode_t *zp = buf;
-
- inode_init_once(ZTOI(zp));
- list_link_init(&zp->z_link_node);
-
- mutex_init(&zp->z_lock, NULL, MUTEX_DEFAULT, NULL);
- rw_init(&zp->z_parent_lock, NULL, RW_DEFAULT, NULL);
- rw_init(&zp->z_name_lock, NULL, RW_NOLOCKDEP, NULL);
- mutex_init(&zp->z_acl_lock, NULL, MUTEX_DEFAULT, NULL);
- rw_init(&zp->z_xattr_lock, NULL, RW_DEFAULT, NULL);
-
- rangelock_init(&zp->z_rangelock, zfs_rangelock_cb, zp);
-
- zp->z_dirlocks = NULL;
- zp->z_acl_cached = NULL;
- zp->z_xattr_cached = NULL;
- zp->z_xattr_parent = 0;
- zp->z_moved = B_FALSE;
- return (0);
-}
-
-/*ARGSUSED*/
-static void
-zfs_znode_cache_destructor(void *buf, void *arg)
-{
- znode_t *zp = buf;
-
- ASSERT(!list_link_active(&zp->z_link_node));
- mutex_destroy(&zp->z_lock);
- rw_destroy(&zp->z_parent_lock);
- rw_destroy(&zp->z_name_lock);
- mutex_destroy(&zp->z_acl_lock);
- rw_destroy(&zp->z_xattr_lock);
- rangelock_fini(&zp->z_rangelock);
-
- ASSERT(zp->z_dirlocks == NULL);
- ASSERT(zp->z_acl_cached == NULL);
- ASSERT(zp->z_xattr_cached == NULL);
-}
-
-static int
-zfs_znode_hold_cache_constructor(void *buf, void *arg, int kmflags)
-{
- znode_hold_t *zh = buf;
-
- mutex_init(&zh->zh_lock, NULL, MUTEX_DEFAULT, NULL);
- zfs_refcount_create(&zh->zh_refcount);
- zh->zh_obj = ZFS_NO_OBJECT;
-
- return (0);
-}
-
-static void
-zfs_znode_hold_cache_destructor(void *buf, void *arg)
-{
- znode_hold_t *zh = buf;
-
- mutex_destroy(&zh->zh_lock);
- zfs_refcount_destroy(&zh->zh_refcount);
-}
-
-void
-zfs_znode_init(void)
-{
- /*
- * Initialize zcache. The KMC_SLAB hint is used in order that it be
- * backed by kmalloc() when on the Linux slab in order that any
- * wait_on_bit() operations on the related inode operate properly.
- */
- ASSERT(znode_cache == NULL);
- znode_cache = kmem_cache_create("zfs_znode_cache",
- sizeof (znode_t), 0, zfs_znode_cache_constructor,
- zfs_znode_cache_destructor, NULL, NULL, NULL, KMC_SLAB);
-
- ASSERT(znode_hold_cache == NULL);
- znode_hold_cache = kmem_cache_create("zfs_znode_hold_cache",
- sizeof (znode_hold_t), 0, zfs_znode_hold_cache_constructor,
- zfs_znode_hold_cache_destructor, NULL, NULL, NULL, 0);
-}
-
-void
-zfs_znode_fini(void)
-{
- /*
- * Cleanup zcache
- */
- if (znode_cache)
- kmem_cache_destroy(znode_cache);
- znode_cache = NULL;
-
- if (znode_hold_cache)
- kmem_cache_destroy(znode_hold_cache);
- znode_hold_cache = NULL;
-}
-
-/*
- * The zfs_znode_hold_enter() / zfs_znode_hold_exit() functions are used to
- * serialize access to a znode and its SA buffer while the object is being
- * created or destroyed. This kind of locking would normally reside in the
- * znode itself but in this case that's impossible because the znode and SA
- * buffer may not yet exist. Therefore the locking is handled externally
- * with an array of mutexs and AVLs trees which contain per-object locks.
- *
- * In zfs_znode_hold_enter() a per-object lock is created as needed, inserted
- * in to the correct AVL tree and finally the per-object lock is held. In
- * zfs_znode_hold_exit() the process is reversed. The per-object lock is
- * released, removed from the AVL tree and destroyed if there are no waiters.
- *
- * This scheme has two important properties:
- *
- * 1) No memory allocations are performed while holding one of the z_hold_locks.
- * This ensures evict(), which can be called from direct memory reclaim, will
- * never block waiting on a z_hold_locks which just happens to have hashed
- * to the same index.
- *
- * 2) All locks used to serialize access to an object are per-object and never
- * shared. This minimizes lock contention without creating a large number
- * of dedicated locks.
- *
- * On the downside it does require znode_lock_t structures to be frequently
- * allocated and freed. However, because these are backed by a kmem cache
- * and very short lived this cost is minimal.
- */
-int
-zfs_znode_hold_compare(const void *a, const void *b)
-{
- const znode_hold_t *zh_a = (const znode_hold_t *)a;
- const znode_hold_t *zh_b = (const znode_hold_t *)b;
-
- return (AVL_CMP(zh_a->zh_obj, zh_b->zh_obj));
-}
-
-boolean_t
-zfs_znode_held(zfsvfs_t *zfsvfs, uint64_t obj)
-{
- znode_hold_t *zh, search;
- int i = ZFS_OBJ_HASH(zfsvfs, obj);
- boolean_t held;
-
- search.zh_obj = obj;
-
- mutex_enter(&zfsvfs->z_hold_locks[i]);
- zh = avl_find(&zfsvfs->z_hold_trees[i], &search, NULL);
- held = (zh && MUTEX_HELD(&zh->zh_lock)) ? B_TRUE : B_FALSE;
- mutex_exit(&zfsvfs->z_hold_locks[i]);
-
- return (held);
-}
-
-static znode_hold_t *
-zfs_znode_hold_enter(zfsvfs_t *zfsvfs, uint64_t obj)
-{
- znode_hold_t *zh, *zh_new, search;
- int i = ZFS_OBJ_HASH(zfsvfs, obj);
- boolean_t found = B_FALSE;
-
- zh_new = kmem_cache_alloc(znode_hold_cache, KM_SLEEP);
- zh_new->zh_obj = obj;
- search.zh_obj = obj;
-
- mutex_enter(&zfsvfs->z_hold_locks[i]);
- zh = avl_find(&zfsvfs->z_hold_trees[i], &search, NULL);
- if (likely(zh == NULL)) {
- zh = zh_new;
- avl_add(&zfsvfs->z_hold_trees[i], zh);
- } else {
- ASSERT3U(zh->zh_obj, ==, obj);
- found = B_TRUE;
- }
- zfs_refcount_add(&zh->zh_refcount, NULL);
- mutex_exit(&zfsvfs->z_hold_locks[i]);
-
- if (found == B_TRUE)
- kmem_cache_free(znode_hold_cache, zh_new);
-
- ASSERT(MUTEX_NOT_HELD(&zh->zh_lock));
- ASSERT3S(zfs_refcount_count(&zh->zh_refcount), >, 0);
- mutex_enter(&zh->zh_lock);
-
- return (zh);
-}
-
-static void
-zfs_znode_hold_exit(zfsvfs_t *zfsvfs, znode_hold_t *zh)
-{
- int i = ZFS_OBJ_HASH(zfsvfs, zh->zh_obj);
- boolean_t remove = B_FALSE;
-
- ASSERT(zfs_znode_held(zfsvfs, zh->zh_obj));
- ASSERT3S(zfs_refcount_count(&zh->zh_refcount), >, 0);
- mutex_exit(&zh->zh_lock);
-
- mutex_enter(&zfsvfs->z_hold_locks[i]);
- if (zfs_refcount_remove(&zh->zh_refcount, NULL) == 0) {
- avl_remove(&zfsvfs->z_hold_trees[i], zh);
- remove = B_TRUE;
- }
- mutex_exit(&zfsvfs->z_hold_locks[i]);
-
- if (remove == B_TRUE)
- kmem_cache_free(znode_hold_cache, zh);
-}
-
-static void
-zfs_znode_sa_init(zfsvfs_t *zfsvfs, znode_t *zp,
- dmu_buf_t *db, dmu_object_type_t obj_type, sa_handle_t *sa_hdl)
-{
- ASSERT(zfs_znode_held(zfsvfs, zp->z_id));
-
- mutex_enter(&zp->z_lock);
-
- ASSERT(zp->z_sa_hdl == NULL);
- ASSERT(zp->z_acl_cached == NULL);
- if (sa_hdl == NULL) {
- VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, zp,
- SA_HDL_SHARED, &zp->z_sa_hdl));
- } else {
- zp->z_sa_hdl = sa_hdl;
- sa_set_userp(sa_hdl, zp);
- }
-
- zp->z_is_sa = (obj_type == DMU_OT_SA) ? B_TRUE : B_FALSE;
-
- mutex_exit(&zp->z_lock);
-}
-
-void
-zfs_znode_dmu_fini(znode_t *zp)
-{
- ASSERT(zfs_znode_held(ZTOZSB(zp), zp->z_id) || zp->z_unlinked ||
- RW_WRITE_HELD(&ZTOZSB(zp)->z_teardown_inactive_lock));
-
- sa_handle_destroy(zp->z_sa_hdl);
- zp->z_sa_hdl = NULL;
-}
-
-/*
- * Called by new_inode() to allocate a new inode.
- */
-int
-zfs_inode_alloc(struct super_block *sb, struct inode **ip)
-{
- znode_t *zp;
-
- zp = kmem_cache_alloc(znode_cache, KM_SLEEP);
- *ip = ZTOI(zp);
-
- return (0);
-}
-
-/*
- * Called in multiple places when an inode should be destroyed.
- */
-void
-zfs_inode_destroy(struct inode *ip)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
-
- mutex_enter(&zfsvfs->z_znodes_lock);
- if (list_link_active(&zp->z_link_node)) {
- list_remove(&zfsvfs->z_all_znodes, zp);
- zfsvfs->z_nr_znodes--;
- }
- mutex_exit(&zfsvfs->z_znodes_lock);
-
- if (zp->z_acl_cached) {
- zfs_acl_free(zp->z_acl_cached);
- zp->z_acl_cached = NULL;
- }
-
- if (zp->z_xattr_cached) {
- nvlist_free(zp->z_xattr_cached);
- zp->z_xattr_cached = NULL;
- }
-
- kmem_cache_free(znode_cache, zp);
-}
-
-static void
-zfs_inode_set_ops(zfsvfs_t *zfsvfs, struct inode *ip)
-{
- uint64_t rdev = 0;
-
- switch (ip->i_mode & S_IFMT) {
- case S_IFREG:
- ip->i_op = &zpl_inode_operations;
- ip->i_fop = &zpl_file_operations;
- ip->i_mapping->a_ops = &zpl_address_space_operations;
- break;
-
- case S_IFDIR:
- ip->i_op = &zpl_dir_inode_operations;
- ip->i_fop = &zpl_dir_file_operations;
- ITOZ(ip)->z_zn_prefetch = B_TRUE;
- break;
-
- case S_IFLNK:
- ip->i_op = &zpl_symlink_inode_operations;
- break;
-
- /*
- * rdev is only stored in a SA only for device files.
- */
- case S_IFCHR:
- case S_IFBLK:
- (void) sa_lookup(ITOZ(ip)->z_sa_hdl, SA_ZPL_RDEV(zfsvfs), &rdev,
- sizeof (rdev));
- /*FALLTHROUGH*/
- case S_IFIFO:
- case S_IFSOCK:
- init_special_inode(ip, ip->i_mode, rdev);
- ip->i_op = &zpl_special_inode_operations;
- break;
-
- default:
- zfs_panic_recover("inode %llu has invalid mode: 0x%x\n",
- (u_longlong_t)ip->i_ino, ip->i_mode);
-
- /* Assume the inode is a file and attempt to continue */
- ip->i_mode = S_IFREG | 0644;
- ip->i_op = &zpl_inode_operations;
- ip->i_fop = &zpl_file_operations;
- ip->i_mapping->a_ops = &zpl_address_space_operations;
- break;
- }
-}
-
-void
-zfs_set_inode_flags(znode_t *zp, struct inode *ip)
-{
- /*
- * Linux and Solaris have different sets of file attributes, so we
- * restrict this conversion to the intersection of the two.
- */
-#ifdef HAVE_INODE_SET_FLAGS
- unsigned int flags = 0;
- if (zp->z_pflags & ZFS_IMMUTABLE)
- flags |= S_IMMUTABLE;
- if (zp->z_pflags & ZFS_APPENDONLY)
- flags |= S_APPEND;
-
- inode_set_flags(ip, flags, S_IMMUTABLE|S_APPEND);
-#else
- if (zp->z_pflags & ZFS_IMMUTABLE)
- ip->i_flags |= S_IMMUTABLE;
- else
- ip->i_flags &= ~S_IMMUTABLE;
-
- if (zp->z_pflags & ZFS_APPENDONLY)
- ip->i_flags |= S_APPEND;
- else
- ip->i_flags &= ~S_APPEND;
-#endif
-}
-
-/*
- * Update the embedded inode given the znode. We should work toward
- * eliminating this function as soon as possible by removing values
- * which are duplicated between the znode and inode. If the generic
- * inode has the correct field it should be used, and the ZFS code
- * updated to access the inode. This can be done incrementally.
- */
-void
-zfs_inode_update(znode_t *zp)
-{
- zfsvfs_t *zfsvfs;
- struct inode *ip;
- uint32_t blksize;
- u_longlong_t i_blocks;
-
- ASSERT(zp != NULL);
- zfsvfs = ZTOZSB(zp);
- ip = ZTOI(zp);
-
- /* Skip .zfs control nodes which do not exist on disk. */
- if (zfsctl_is_node(ip))
- return;
-
- dmu_object_size_from_db(sa_get_db(zp->z_sa_hdl), &blksize, &i_blocks);
-
- spin_lock(&ip->i_lock);
- ip->i_blocks = i_blocks;
- i_size_write(ip, zp->z_size);
- spin_unlock(&ip->i_lock);
-}
-
-
-/*
- * Construct a znode+inode and initialize.
- *
- * This does not do a call to dmu_set_user() that is
- * up to the caller to do, in case you don't want to
- * return the znode
- */
-static znode_t *
-zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
- dmu_object_type_t obj_type, sa_handle_t *hdl)
-{
- znode_t *zp;
- struct inode *ip;
- uint64_t mode;
- uint64_t parent;
- uint64_t tmp_gen;
- uint64_t links;
- uint64_t z_uid, z_gid;
- uint64_t atime[2], mtime[2], ctime[2];
- uint64_t projid = ZFS_DEFAULT_PROJID;
- sa_bulk_attr_t bulk[11];
- int count = 0;
-
- ASSERT(zfsvfs != NULL);
-
- ip = new_inode(zfsvfs->z_sb);
- if (ip == NULL)
- return (NULL);
-
- zp = ITOZ(ip);
- ASSERT(zp->z_dirlocks == NULL);
- ASSERT3P(zp->z_acl_cached, ==, NULL);
- ASSERT3P(zp->z_xattr_cached, ==, NULL);
- zp->z_unlinked = B_FALSE;
- zp->z_atime_dirty = B_FALSE;
- zp->z_moved = B_FALSE;
- zp->z_is_mapped = B_FALSE;
- zp->z_is_ctldir = B_FALSE;
- zp->z_is_stale = B_FALSE;
- zp->z_suspended = B_FALSE;
- zp->z_sa_hdl = NULL;
- zp->z_mapcnt = 0;
- zp->z_id = db->db_object;
- zp->z_blksz = blksz;
- zp->z_seq = 0x7A4653;
- zp->z_sync_cnt = 0;
-
- zfs_znode_sa_init(zfsvfs, zp, db, obj_type, hdl);
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL, &tmp_gen, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
- &zp->z_size, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, &links, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
- &parent, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL, &z_uid, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL, &z_gid, 8);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
-
- if (sa_bulk_lookup(zp->z_sa_hdl, bulk, count) != 0 || tmp_gen == 0 ||
- (dmu_objset_projectquota_enabled(zfsvfs->z_os) &&
- (zp->z_pflags & ZFS_PROJID) &&
- sa_lookup(zp->z_sa_hdl, SA_ZPL_PROJID(zfsvfs), &projid, 8) != 0)) {
- if (hdl == NULL)
- sa_handle_destroy(zp->z_sa_hdl);
- zp->z_sa_hdl = NULL;
- goto error;
- }
-
- zp->z_projid = projid;
- zp->z_mode = ip->i_mode = mode;
- ip->i_generation = (uint32_t)tmp_gen;
- ip->i_blkbits = SPA_MINBLOCKSHIFT;
- set_nlink(ip, (uint32_t)links);
- zfs_uid_write(ip, z_uid);
- zfs_gid_write(ip, z_gid);
- zfs_set_inode_flags(zp, ip);
-
- /* Cache the xattr parent id */
- if (zp->z_pflags & ZFS_XATTR)
- zp->z_xattr_parent = parent;
-
- ZFS_TIME_DECODE(&ip->i_atime, atime);
- ZFS_TIME_DECODE(&ip->i_mtime, mtime);
- ZFS_TIME_DECODE(&ip->i_ctime, ctime);
-
- ip->i_ino = zp->z_id;
- zfs_inode_update(zp);
- zfs_inode_set_ops(zfsvfs, ip);
-
- /*
- * The only way insert_inode_locked() can fail is if the ip->i_ino
- * number is already hashed for this super block. This can never
- * happen because the inode numbers map 1:1 with the object numbers.
- *
- * The one exception is rolling back a mounted file system, but in
- * this case all the active inode are unhashed during the rollback.
- */
- VERIFY3S(insert_inode_locked(ip), ==, 0);
-
- mutex_enter(&zfsvfs->z_znodes_lock);
- list_insert_tail(&zfsvfs->z_all_znodes, zp);
- zfsvfs->z_nr_znodes++;
- membar_producer();
- mutex_exit(&zfsvfs->z_znodes_lock);
-
- unlock_new_inode(ip);
- return (zp);
-
-error:
- iput(ip);
- return (NULL);
-}
-
-/*
- * Safely mark an inode dirty. Inodes which are part of a read-only
- * file system or snapshot may not be dirtied.
- */
-void
-zfs_mark_inode_dirty(struct inode *ip)
-{
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- if (zfs_is_readonly(zfsvfs) || dmu_objset_is_snapshot(zfsvfs->z_os))
- return;
-
- mark_inode_dirty(ip);
-}
-
-static uint64_t empty_xattr;
-static uint64_t pad[4];
-static zfs_acl_phys_t acl_phys;
-/*
- * Create a new DMU object to hold a zfs znode.
- *
- * IN: dzp - parent directory for new znode
- * vap - file attributes for new znode
- * tx - dmu transaction id for zap operations
- * cr - credentials of caller
- * flag - flags:
- * IS_ROOT_NODE - new object will be root
- * IS_TMPFILE - new object is of O_TMPFILE
- * IS_XATTR - new object is an attribute
- * acl_ids - ACL related attributes
- *
- * OUT: zpp - allocated znode (set to dzp if IS_ROOT_NODE)
- *
- */
-void
-zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
- uint_t flag, znode_t **zpp, zfs_acl_ids_t *acl_ids)
-{
- uint64_t crtime[2], atime[2], mtime[2], ctime[2];
- uint64_t mode, size, links, parent, pflags;
- uint64_t projid = ZFS_DEFAULT_PROJID;
- uint64_t rdev = 0;
- zfsvfs_t *zfsvfs = ZTOZSB(dzp);
- dmu_buf_t *db;
- inode_timespec_t now;
- uint64_t gen, obj;
- int bonuslen;
- int dnodesize;
- sa_handle_t *sa_hdl;
- dmu_object_type_t obj_type;
- sa_bulk_attr_t *sa_attrs;
- int cnt = 0;
- zfs_acl_locator_cb_t locate = { 0 };
- znode_hold_t *zh;
-
- if (zfsvfs->z_replay) {
- obj = vap->va_nodeid;
- now = vap->va_ctime; /* see zfs_replay_create() */
- gen = vap->va_nblocks; /* ditto */
- dnodesize = vap->va_fsid; /* ditto */
- } else {
- obj = 0;
- gethrestime(&now);
- gen = dmu_tx_get_txg(tx);
- dnodesize = dmu_objset_dnodesize(zfsvfs->z_os);
- }
-
- if (dnodesize == 0)
- dnodesize = DNODE_MIN_SIZE;
-
- obj_type = zfsvfs->z_use_sa ? DMU_OT_SA : DMU_OT_ZNODE;
-
- bonuslen = (obj_type == DMU_OT_SA) ?
- DN_BONUS_SIZE(dnodesize) : ZFS_OLD_ZNODE_PHYS_SIZE;
-
- /*
- * Create a new DMU object.
- */
- /*
- * There's currently no mechanism for pre-reading the blocks that will
- * be needed to allocate a new object, so we accept the small chance
- * that there will be an i/o error and we will fail one of the
- * assertions below.
- */
- if (S_ISDIR(vap->va_mode)) {
- if (zfsvfs->z_replay) {
- VERIFY0(zap_create_claim_norm_dnsize(zfsvfs->z_os, obj,
- zfsvfs->z_norm, DMU_OT_DIRECTORY_CONTENTS,
- obj_type, bonuslen, dnodesize, tx));
- } else {
- obj = zap_create_norm_dnsize(zfsvfs->z_os,
- zfsvfs->z_norm, DMU_OT_DIRECTORY_CONTENTS,
- obj_type, bonuslen, dnodesize, tx);
- }
- } else {
- if (zfsvfs->z_replay) {
- VERIFY0(dmu_object_claim_dnsize(zfsvfs->z_os, obj,
- DMU_OT_PLAIN_FILE_CONTENTS, 0,
- obj_type, bonuslen, dnodesize, tx));
- } else {
- obj = dmu_object_alloc_dnsize(zfsvfs->z_os,
- DMU_OT_PLAIN_FILE_CONTENTS, 0,
- obj_type, bonuslen, dnodesize, tx);
- }
- }
-
- zh = zfs_znode_hold_enter(zfsvfs, obj);
- VERIFY0(sa_buf_hold(zfsvfs->z_os, obj, NULL, &db));
-
- /*
- * If this is the root, fix up the half-initialized parent pointer
- * to reference the just-allocated physical data area.
- */
- if (flag & IS_ROOT_NODE) {
- dzp->z_id = obj;
- }
-
- /*
- * If parent is an xattr, so am I.
- */
- if (dzp->z_pflags & ZFS_XATTR) {
- flag |= IS_XATTR;
- }
-
- if (zfsvfs->z_use_fuids)
- pflags = ZFS_ARCHIVE | ZFS_AV_MODIFIED;
- else
- pflags = 0;
-
- if (S_ISDIR(vap->va_mode)) {
- size = 2; /* contents ("." and "..") */
- links = 2;
- } else {
- size = 0;
- links = (flag & IS_TMPFILE) ? 0 : 1;
- }
-
- if (S_ISBLK(vap->va_mode) || S_ISCHR(vap->va_mode))
- rdev = vap->va_rdev;
-
- parent = dzp->z_id;
- mode = acl_ids->z_mode;
- if (flag & IS_XATTR)
- pflags |= ZFS_XATTR;
-
- if (S_ISREG(vap->va_mode) || S_ISDIR(vap->va_mode)) {
- /*
- * With ZFS_PROJID flag, we can easily know whether there is
- * project ID stored on disk or not. See zfs_space_delta_cb().
- */
- if (obj_type != DMU_OT_ZNODE &&
- dmu_objset_projectquota_enabled(zfsvfs->z_os))
- pflags |= ZFS_PROJID;
-
- /*
- * Inherit project ID from parent if required.
- */
- projid = zfs_inherit_projid(dzp);
- if (dzp->z_pflags & ZFS_PROJINHERIT)
- pflags |= ZFS_PROJINHERIT;
- }
-
- /*
- * No execs denied will be determined when zfs_mode_compute() is called.
- */
- pflags |= acl_ids->z_aclp->z_hints &
- (ZFS_ACL_TRIVIAL|ZFS_INHERIT_ACE|ZFS_ACL_AUTO_INHERIT|
- ZFS_ACL_DEFAULTED|ZFS_ACL_PROTECTED);
-
- ZFS_TIME_ENCODE(&now, crtime);
- ZFS_TIME_ENCODE(&now, ctime);
-
- if (vap->va_mask & ATTR_ATIME) {
- ZFS_TIME_ENCODE(&vap->va_atime, atime);
- } else {
- ZFS_TIME_ENCODE(&now, atime);
- }
-
- if (vap->va_mask & ATTR_MTIME) {
- ZFS_TIME_ENCODE(&vap->va_mtime, mtime);
- } else {
- ZFS_TIME_ENCODE(&now, mtime);
- }
-
- /* Now add in all of the "SA" attributes */
- VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, NULL, SA_HDL_SHARED,
- &sa_hdl));
-
- /*
- * Setup the array of attributes to be replaced/set on the new file
- *
- * order for DMU_OT_ZNODE is critical since it needs to be constructed
- * in the old znode_phys_t format. Don't change this ordering
- */
- sa_attrs = kmem_alloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP);
-
- if (obj_type == DMU_OT_ZNODE) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zfsvfs),
- NULL, &atime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MTIME(zfsvfs),
- NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CTIME(zfsvfs),
- NULL, &ctime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CRTIME(zfsvfs),
- NULL, &crtime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GEN(zfsvfs),
- NULL, &gen, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MODE(zfsvfs),
- NULL, &mode, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_SIZE(zfsvfs),
- NULL, &size, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PARENT(zfsvfs),
- NULL, &parent, 8);
- } else {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MODE(zfsvfs),
- NULL, &mode, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_SIZE(zfsvfs),
- NULL, &size, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GEN(zfsvfs),
- NULL, &gen, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_UID(zfsvfs),
- NULL, &acl_ids->z_fuid, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GID(zfsvfs),
- NULL, &acl_ids->z_fgid, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PARENT(zfsvfs),
- NULL, &parent, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_FLAGS(zfsvfs),
- NULL, &pflags, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zfsvfs),
- NULL, &atime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MTIME(zfsvfs),
- NULL, &mtime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CTIME(zfsvfs),
- NULL, &ctime, 16);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CRTIME(zfsvfs),
- NULL, &crtime, 16);
- }
-
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_LINKS(zfsvfs), NULL, &links, 8);
-
- if (obj_type == DMU_OT_ZNODE) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_XATTR(zfsvfs), NULL,
- &empty_xattr, 8);
- } else if (dmu_objset_projectquota_enabled(zfsvfs->z_os) &&
- pflags & ZFS_PROJID) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PROJID(zfsvfs),
- NULL, &projid, 8);
- }
- if (obj_type == DMU_OT_ZNODE ||
- (S_ISBLK(vap->va_mode) || S_ISCHR(vap->va_mode))) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_RDEV(zfsvfs),
- NULL, &rdev, 8);
- }
- if (obj_type == DMU_OT_ZNODE) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_FLAGS(zfsvfs),
- NULL, &pflags, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_UID(zfsvfs), NULL,
- &acl_ids->z_fuid, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GID(zfsvfs), NULL,
- &acl_ids->z_fgid, 8);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PAD(zfsvfs), NULL, pad,
- sizeof (uint64_t) * 4);
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ZNODE_ACL(zfsvfs), NULL,
- &acl_phys, sizeof (zfs_acl_phys_t));
- } else if (acl_ids->z_aclp->z_version >= ZFS_ACL_VERSION_FUID) {
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_DACL_COUNT(zfsvfs), NULL,
- &acl_ids->z_aclp->z_acl_count, 8);
- locate.cb_aclp = acl_ids->z_aclp;
- SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_DACL_ACES(zfsvfs),
- zfs_acl_data_locator, &locate,
- acl_ids->z_aclp->z_acl_bytes);
- mode = zfs_mode_compute(mode, acl_ids->z_aclp, &pflags,
- acl_ids->z_fuid, acl_ids->z_fgid);
- }
-
- VERIFY(sa_replace_all_by_template(sa_hdl, sa_attrs, cnt, tx) == 0);
-
- if (!(flag & IS_ROOT_NODE)) {
- /*
- * The call to zfs_znode_alloc() may fail if memory is low
- * via the call path: alloc_inode() -> inode_init_always() ->
- * security_inode_alloc() -> inode_alloc_security(). Since
- * the existing code is written such that zfs_mknode() can
- * not fail retry until sufficient memory has been reclaimed.
- */
- do {
- *zpp = zfs_znode_alloc(zfsvfs, db, 0, obj_type, sa_hdl);
- } while (*zpp == NULL);
-
- VERIFY(*zpp != NULL);
- VERIFY(dzp != NULL);
- } else {
- /*
- * If we are creating the root node, the "parent" we
- * passed in is the znode for the root.
- */
- *zpp = dzp;
-
- (*zpp)->z_sa_hdl = sa_hdl;
- }
-
- (*zpp)->z_pflags = pflags;
- (*zpp)->z_mode = ZTOI(*zpp)->i_mode = mode;
- (*zpp)->z_dnodesize = dnodesize;
- (*zpp)->z_projid = projid;
-
- if (obj_type == DMU_OT_ZNODE ||
- acl_ids->z_aclp->z_version < ZFS_ACL_VERSION_FUID) {
- VERIFY0(zfs_aclset_common(*zpp, acl_ids->z_aclp, cr, tx));
- }
- kmem_free(sa_attrs, sizeof (sa_bulk_attr_t) * ZPL_END);
- zfs_znode_hold_exit(zfsvfs, zh);
-}
-
-/*
- * Update in-core attributes. It is assumed the caller will be doing an
- * sa_bulk_update to push the changes out.
- */
-void
-zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx)
-{
- xoptattr_t *xoap;
- boolean_t update_inode = B_FALSE;
-
- xoap = xva_getxoptattr(xvap);
- ASSERT(xoap);
-
- if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) {
- uint64_t times[2];
- ZFS_TIME_ENCODE(&xoap->xoa_createtime, times);
- (void) sa_update(zp->z_sa_hdl, SA_ZPL_CRTIME(ZTOZSB(zp)),
- &times, sizeof (times), tx);
- XVA_SET_RTN(xvap, XAT_CREATETIME);
- }
- if (XVA_ISSET_REQ(xvap, XAT_READONLY)) {
- ZFS_ATTR_SET(zp, ZFS_READONLY, xoap->xoa_readonly,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_READONLY);
- }
- if (XVA_ISSET_REQ(xvap, XAT_HIDDEN)) {
- ZFS_ATTR_SET(zp, ZFS_HIDDEN, xoap->xoa_hidden,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_HIDDEN);
- }
- if (XVA_ISSET_REQ(xvap, XAT_SYSTEM)) {
- ZFS_ATTR_SET(zp, ZFS_SYSTEM, xoap->xoa_system,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_SYSTEM);
- }
- if (XVA_ISSET_REQ(xvap, XAT_ARCHIVE)) {
- ZFS_ATTR_SET(zp, ZFS_ARCHIVE, xoap->xoa_archive,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_ARCHIVE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) {
- ZFS_ATTR_SET(zp, ZFS_IMMUTABLE, xoap->xoa_immutable,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_IMMUTABLE);
-
- update_inode = B_TRUE;
- }
- if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) {
- ZFS_ATTR_SET(zp, ZFS_NOUNLINK, xoap->xoa_nounlink,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_NOUNLINK);
- }
- if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) {
- ZFS_ATTR_SET(zp, ZFS_APPENDONLY, xoap->xoa_appendonly,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_APPENDONLY);
-
- update_inode = B_TRUE;
- }
- if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) {
- ZFS_ATTR_SET(zp, ZFS_NODUMP, xoap->xoa_nodump,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_NODUMP);
- }
- if (XVA_ISSET_REQ(xvap, XAT_OPAQUE)) {
- ZFS_ATTR_SET(zp, ZFS_OPAQUE, xoap->xoa_opaque,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_OPAQUE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) {
- ZFS_ATTR_SET(zp, ZFS_AV_QUARANTINED,
- xoap->xoa_av_quarantined, zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_AV_QUARANTINED);
- }
- if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) {
- ZFS_ATTR_SET(zp, ZFS_AV_MODIFIED, xoap->xoa_av_modified,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_AV_MODIFIED);
- }
- if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) {
- zfs_sa_set_scanstamp(zp, xvap, tx);
- XVA_SET_RTN(xvap, XAT_AV_SCANSTAMP);
- }
- if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) {
- ZFS_ATTR_SET(zp, ZFS_REPARSE, xoap->xoa_reparse,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_REPARSE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_OFFLINE)) {
- ZFS_ATTR_SET(zp, ZFS_OFFLINE, xoap->xoa_offline,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_OFFLINE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_SPARSE)) {
- ZFS_ATTR_SET(zp, ZFS_SPARSE, xoap->xoa_sparse,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_SPARSE);
- }
- if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) {
- ZFS_ATTR_SET(zp, ZFS_PROJINHERIT, xoap->xoa_projinherit,
- zp->z_pflags, tx);
- XVA_SET_RTN(xvap, XAT_PROJINHERIT);
- }
-
- if (update_inode)
- zfs_set_inode_flags(zp, ZTOI(zp));
-}
-
-int
-zfs_zget(zfsvfs_t *zfsvfs, uint64_t obj_num, znode_t **zpp)
-{
- dmu_object_info_t doi;
- dmu_buf_t *db;
- znode_t *zp;
- znode_hold_t *zh;
- int err;
- sa_handle_t *hdl;
-
- *zpp = NULL;
-
-again:
- zh = zfs_znode_hold_enter(zfsvfs, obj_num);
-
- err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db);
- if (err) {
- zfs_znode_hold_exit(zfsvfs, zh);
- return (err);
- }
-
- dmu_object_info_from_db(db, &doi);
- if (doi.doi_bonus_type != DMU_OT_SA &&
- (doi.doi_bonus_type != DMU_OT_ZNODE ||
- (doi.doi_bonus_type == DMU_OT_ZNODE &&
- doi.doi_bonus_size < sizeof (znode_phys_t)))) {
- sa_buf_rele(db, NULL);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (SET_ERROR(EINVAL));
- }
-
- hdl = dmu_buf_get_user(db);
- if (hdl != NULL) {
- zp = sa_get_userdata(hdl);
-
-
- /*
- * Since "SA" does immediate eviction we
- * should never find a sa handle that doesn't
- * know about the znode.
- */
-
- ASSERT3P(zp, !=, NULL);
-
- mutex_enter(&zp->z_lock);
- ASSERT3U(zp->z_id, ==, obj_num);
- /*
- * If igrab() returns NULL the VFS has independently
- * determined the inode should be evicted and has
- * called iput_final() to start the eviction process.
- * The SA handle is still valid but because the VFS
- * requires that the eviction succeed we must drop
- * our locks and references to allow the eviction to
- * complete. The zfs_zget() may then be retried.
- *
- * This unlikely case could be optimized by registering
- * a sops->drop_inode() callback. The callback would
- * need to detect the active SA hold thereby informing
- * the VFS that this inode should not be evicted.
- */
- if (igrab(ZTOI(zp)) == NULL) {
- mutex_exit(&zp->z_lock);
- sa_buf_rele(db, NULL);
- zfs_znode_hold_exit(zfsvfs, zh);
- /* inode might need this to finish evict */
- cond_resched();
- goto again;
- }
- *zpp = zp;
- err = 0;
- mutex_exit(&zp->z_lock);
- sa_buf_rele(db, NULL);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (err);
- }
-
- /*
- * Not found create new znode/vnode but only if file exists.
- *
- * There is a small window where zfs_vget() could
- * find this object while a file create is still in
- * progress. This is checked for in zfs_znode_alloc()
- *
- * if zfs_znode_alloc() fails it will drop the hold on the
- * bonus buffer.
- */
- zp = zfs_znode_alloc(zfsvfs, db, doi.doi_data_block_size,
- doi.doi_bonus_type, NULL);
- if (zp == NULL) {
- err = SET_ERROR(ENOENT);
- } else {
- *zpp = zp;
- }
- zfs_znode_hold_exit(zfsvfs, zh);
- return (err);
-}
-
-int
-zfs_rezget(znode_t *zp)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- dmu_object_info_t doi;
- dmu_buf_t *db;
- uint64_t obj_num = zp->z_id;
- uint64_t mode;
- uint64_t links;
- sa_bulk_attr_t bulk[10];
- int err;
- int count = 0;
- uint64_t gen;
- uint64_t z_uid, z_gid;
- uint64_t atime[2], mtime[2], ctime[2];
- uint64_t projid = ZFS_DEFAULT_PROJID;
- znode_hold_t *zh;
-
- /*
- * skip ctldir, otherwise they will always get invalidated. This will
- * cause funny behaviour for the mounted snapdirs. Especially for
- * Linux >= 3.18, d_invalidate will detach the mountpoint and prevent
- * anyone automount it again as long as someone is still using the
- * detached mount.
- */
- if (zp->z_is_ctldir)
- return (0);
-
- zh = zfs_znode_hold_enter(zfsvfs, obj_num);
-
- mutex_enter(&zp->z_acl_lock);
- if (zp->z_acl_cached) {
- zfs_acl_free(zp->z_acl_cached);
- zp->z_acl_cached = NULL;
- }
- mutex_exit(&zp->z_acl_lock);
-
- rw_enter(&zp->z_xattr_lock, RW_WRITER);
- if (zp->z_xattr_cached) {
- nvlist_free(zp->z_xattr_cached);
- zp->z_xattr_cached = NULL;
- }
- rw_exit(&zp->z_xattr_lock);
-
- ASSERT(zp->z_sa_hdl == NULL);
- err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db);
- if (err) {
- zfs_znode_hold_exit(zfsvfs, zh);
- return (err);
- }
-
- dmu_object_info_from_db(db, &doi);
- if (doi.doi_bonus_type != DMU_OT_SA &&
- (doi.doi_bonus_type != DMU_OT_ZNODE ||
- (doi.doi_bonus_type == DMU_OT_ZNODE &&
- doi.doi_bonus_size < sizeof (znode_phys_t)))) {
- sa_buf_rele(db, NULL);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (SET_ERROR(EINVAL));
- }
-
- zfs_znode_sa_init(zfsvfs, zp, db, doi.doi_bonus_type, NULL);
-
- /* reload cached values */
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL,
- &gen, sizeof (gen));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
- &zp->z_size, sizeof (zp->z_size));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL,
- &links, sizeof (links));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
- &zp->z_pflags, sizeof (zp->z_pflags));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
- &z_uid, sizeof (z_uid));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
- &z_gid, sizeof (z_gid));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
- &mode, sizeof (mode));
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
- &atime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
- &mtime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
- &ctime, 16);
-
- if (sa_bulk_lookup(zp->z_sa_hdl, bulk, count)) {
- zfs_znode_dmu_fini(zp);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (SET_ERROR(EIO));
- }
-
- if (dmu_objset_projectquota_enabled(zfsvfs->z_os)) {
- err = sa_lookup(zp->z_sa_hdl, SA_ZPL_PROJID(zfsvfs),
- &projid, 8);
- if (err != 0 && err != ENOENT) {
- zfs_znode_dmu_fini(zp);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (SET_ERROR(err));
- }
- }
-
- zp->z_projid = projid;
- zp->z_mode = ZTOI(zp)->i_mode = mode;
- zfs_uid_write(ZTOI(zp), z_uid);
- zfs_gid_write(ZTOI(zp), z_gid);
-
- ZFS_TIME_DECODE(&ZTOI(zp)->i_atime, atime);
- ZFS_TIME_DECODE(&ZTOI(zp)->i_mtime, mtime);
- ZFS_TIME_DECODE(&ZTOI(zp)->i_ctime, ctime);
-
- if ((uint32_t)gen != ZTOI(zp)->i_generation) {
- zfs_znode_dmu_fini(zp);
- zfs_znode_hold_exit(zfsvfs, zh);
- return (SET_ERROR(EIO));
- }
-
- set_nlink(ZTOI(zp), (uint32_t)links);
- zfs_set_inode_flags(zp, ZTOI(zp));
-
- zp->z_blksz = doi.doi_data_block_size;
- zp->z_atime_dirty = B_FALSE;
- zfs_inode_update(zp);
-
- /*
- * If the file has zero links, then it has been unlinked on the send
- * side and it must be in the received unlinked set.
- * We call zfs_znode_dmu_fini() now to prevent any accesses to the
- * stale data and to prevent automatic removal of the file in
- * zfs_zinactive(). The file will be removed either when it is removed
- * on the send side and the next incremental stream is received or
- * when the unlinked set gets processed.
- */
- zp->z_unlinked = (ZTOI(zp)->i_nlink == 0);
- if (zp->z_unlinked)
- zfs_znode_dmu_fini(zp);
-
- zfs_znode_hold_exit(zfsvfs, zh);
-
- return (0);
-}
-
-void
-zfs_znode_delete(znode_t *zp, dmu_tx_t *tx)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- objset_t *os = zfsvfs->z_os;
- uint64_t obj = zp->z_id;
- uint64_t acl_obj = zfs_external_acl(zp);
- znode_hold_t *zh;
-
- zh = zfs_znode_hold_enter(zfsvfs, obj);
- if (acl_obj) {
- VERIFY(!zp->z_is_sa);
- VERIFY(0 == dmu_object_free(os, acl_obj, tx));
- }
- VERIFY(0 == dmu_object_free(os, obj, tx));
- zfs_znode_dmu_fini(zp);
- zfs_znode_hold_exit(zfsvfs, zh);
-}
-
-void
-zfs_zinactive(znode_t *zp)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- uint64_t z_id = zp->z_id;
- znode_hold_t *zh;
-
- ASSERT(zp->z_sa_hdl);
-
- /*
- * Don't allow a zfs_zget() while were trying to release this znode.
- */
- zh = zfs_znode_hold_enter(zfsvfs, z_id);
-
- mutex_enter(&zp->z_lock);
-
- /*
- * If this was the last reference to a file with no links, remove
- * the file from the file system unless the file system is mounted
- * read-only. That can happen, for example, if the file system was
- * originally read-write, the file was opened, then unlinked and
- * the file system was made read-only before the file was finally
- * closed. The file will remain in the unlinked set.
- */
- if (zp->z_unlinked) {
- ASSERT(!zfsvfs->z_issnap);
- if (!zfs_is_readonly(zfsvfs) && !zfs_unlink_suspend_progress) {
- mutex_exit(&zp->z_lock);
- zfs_znode_hold_exit(zfsvfs, zh);
- zfs_rmnode(zp);
- return;
- }
- }
-
- mutex_exit(&zp->z_lock);
- zfs_znode_dmu_fini(zp);
-
- zfs_znode_hold_exit(zfsvfs, zh);
-}
-
-#if defined(HAVE_INODE_TIMESPEC64_TIMES)
-#define zfs_compare_timespec timespec64_compare
-#else
-#define zfs_compare_timespec timespec_compare
-#endif
-
-/*
- * Determine whether the znode's atime must be updated. The logic mostly
- * duplicates the Linux kernel's relatime_need_update() functionality.
- * This function is only called if the underlying filesystem actually has
- * atime updates enabled.
- */
-boolean_t
-zfs_relatime_need_update(const struct inode *ip)
-{
- inode_timespec_t now;
-
- gethrestime(&now);
- /*
- * In relatime mode, only update the atime if the previous atime
- * is earlier than either the ctime or mtime or if at least a day
- * has passed since the last update of atime.
- */
- if (zfs_compare_timespec(&ip->i_mtime, &ip->i_atime) >= 0)
- return (B_TRUE);
-
- if (zfs_compare_timespec(&ip->i_ctime, &ip->i_atime) >= 0)
- return (B_TRUE);
-
- if ((hrtime_t)now.tv_sec - (hrtime_t)ip->i_atime.tv_sec >= 24*60*60)
- return (B_TRUE);
-
- return (B_FALSE);
-}
-
-/*
- * Prepare to update znode time stamps.
- *
- * IN: zp - znode requiring timestamp update
- * flag - ATTR_MTIME, ATTR_CTIME flags
- *
- * OUT: zp - z_seq
- * mtime - new mtime
- * ctime - new ctime
- *
- * Note: We don't update atime here, because we rely on Linux VFS to do
- * atime updating.
- */
-void
-zfs_tstamp_update_setup(znode_t *zp, uint_t flag, uint64_t mtime[2],
- uint64_t ctime[2])
-{
- inode_timespec_t now;
-
- gethrestime(&now);
-
- zp->z_seq++;
-
- if (flag & ATTR_MTIME) {
- ZFS_TIME_ENCODE(&now, mtime);
- ZFS_TIME_DECODE(&(ZTOI(zp)->i_mtime), mtime);
- if (ZTOZSB(zp)->z_use_fuids) {
- zp->z_pflags |= (ZFS_ARCHIVE |
- ZFS_AV_MODIFIED);
- }
- }
-
- if (flag & ATTR_CTIME) {
- ZFS_TIME_ENCODE(&now, ctime);
- ZFS_TIME_DECODE(&(ZTOI(zp)->i_ctime), ctime);
- if (ZTOZSB(zp)->z_use_fuids)
- zp->z_pflags |= ZFS_ARCHIVE;
- }
-}
-
-/*
- * Grow the block size for a file.
- *
- * IN: zp - znode of file to free data in.
- * size - requested block size
- * tx - open transaction.
- *
- * NOTE: this function assumes that the znode is write locked.
- */
-void
-zfs_grow_blocksize(znode_t *zp, uint64_t size, dmu_tx_t *tx)
-{
- int error;
- u_longlong_t dummy;
-
- if (size <= zp->z_blksz)
- return;
- /*
- * If the file size is already greater than the current blocksize,
- * we will not grow. If there is more than one block in a file,
- * the blocksize cannot change.
- */
- if (zp->z_blksz && zp->z_size > zp->z_blksz)
- return;
-
- error = dmu_object_set_blocksize(ZTOZSB(zp)->z_os, zp->z_id,
- size, 0, tx);
-
- if (error == ENOTSUP)
- return;
- ASSERT0(error);
-
- /* What blocksize did we actually get? */
- dmu_object_size_from_db(sa_get_db(zp->z_sa_hdl), &zp->z_blksz, &dummy);
-}
-
-/*
- * Increase the file length
- *
- * IN: zp - znode of file to free data in.
- * end - new end-of-file
- *
- * RETURN: 0 on success, error code on failure
- */
-static int
-zfs_extend(znode_t *zp, uint64_t end)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- dmu_tx_t *tx;
- locked_range_t *lr;
- uint64_t newblksz;
- int error;
-
- /*
- * We will change zp_size, lock the whole file.
- */
- lr = rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
-
- /*
- * Nothing to do if file already at desired length.
- */
- if (end <= zp->z_size) {
- rangelock_exit(lr);
- return (0);
- }
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
- if (end > zp->z_blksz &&
- (!ISP2(zp->z_blksz) || zp->z_blksz < zfsvfs->z_max_blksz)) {
- /*
- * We are growing the file past the current block size.
- */
- if (zp->z_blksz > ZTOZSB(zp)->z_max_blksz) {
- /*
- * File's blocksize is already larger than the
- * "recordsize" property. Only let it grow to
- * the next power of 2.
- */
- ASSERT(!ISP2(zp->z_blksz));
- newblksz = MIN(end, 1 << highbit64(zp->z_blksz));
- } else {
- newblksz = MIN(end, ZTOZSB(zp)->z_max_blksz);
- }
- dmu_tx_hold_write(tx, zp->z_id, 0, newblksz);
- } else {
- newblksz = 0;
- }
-
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- rangelock_exit(lr);
- return (error);
- }
-
- if (newblksz)
- zfs_grow_blocksize(zp, newblksz, tx);
-
- zp->z_size = end;
-
- VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(ZTOZSB(zp)),
- &zp->z_size, sizeof (zp->z_size), tx));
-
- rangelock_exit(lr);
-
- dmu_tx_commit(tx);
-
- return (0);
-}
-
-/*
- * zfs_zero_partial_page - Modeled after update_pages() but
- * with different arguments and semantics for use by zfs_freesp().
- *
- * Zeroes a piece of a single page cache entry for zp at offset
- * start and length len.
- *
- * Caller must acquire a range lock on the file for the region
- * being zeroed in order that the ARC and page cache stay in sync.
- */
-static void
-zfs_zero_partial_page(znode_t *zp, uint64_t start, uint64_t len)
-{
- struct address_space *mp = ZTOI(zp)->i_mapping;
- struct page *pp;
- int64_t off;
- void *pb;
-
- ASSERT((start & PAGE_MASK) == ((start + len - 1) & PAGE_MASK));
-
- off = start & (PAGE_SIZE - 1);
- start &= PAGE_MASK;
-
- pp = find_lock_page(mp, start >> PAGE_SHIFT);
- if (pp) {
- if (mapping_writably_mapped(mp))
- flush_dcache_page(pp);
-
- pb = kmap(pp);
- bzero(pb + off, len);
- kunmap(pp);
-
- if (mapping_writably_mapped(mp))
- flush_dcache_page(pp);
-
- mark_page_accessed(pp);
- SetPageUptodate(pp);
- ClearPageError(pp);
- unlock_page(pp);
- put_page(pp);
- }
-}
-
-/*
- * Free space in a file.
- *
- * IN: zp - znode of file to free data in.
- * off - start of section to free.
- * len - length of section to free.
- *
- * RETURN: 0 on success, error code on failure
- */
-static int
-zfs_free_range(znode_t *zp, uint64_t off, uint64_t len)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- locked_range_t *lr;
- int error;
-
- /*
- * Lock the range being freed.
- */
- lr = rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER);
-
- /*
- * Nothing to do if file already at desired length.
- */
- if (off >= zp->z_size) {
- rangelock_exit(lr);
- return (0);
- }
-
- if (off + len > zp->z_size)
- len = zp->z_size - off;
-
- error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, off, len);
-
- /*
- * Zero partial page cache entries. This must be done under a
- * range lock in order to keep the ARC and page cache in sync.
- */
- if (zp->z_is_mapped) {
- loff_t first_page, last_page, page_len;
- loff_t first_page_offset, last_page_offset;
-
- /* first possible full page in hole */
- first_page = (off + PAGE_SIZE - 1) >> PAGE_SHIFT;
- /* last page of hole */
- last_page = (off + len) >> PAGE_SHIFT;
-
- /* offset of first_page */
- first_page_offset = first_page << PAGE_SHIFT;
- /* offset of last_page */
- last_page_offset = last_page << PAGE_SHIFT;
-
- /* truncate whole pages */
- if (last_page_offset > first_page_offset) {
- truncate_inode_pages_range(ZTOI(zp)->i_mapping,
- first_page_offset, last_page_offset - 1);
- }
-
- /* truncate sub-page ranges */
- if (first_page > last_page) {
- /* entire punched area within a single page */
- zfs_zero_partial_page(zp, off, len);
- } else {
- /* beginning of punched area at the end of a page */
- page_len = first_page_offset - off;
- if (page_len > 0)
- zfs_zero_partial_page(zp, off, page_len);
-
- /* end of punched area at the beginning of a page */
- page_len = off + len - last_page_offset;
- if (page_len > 0)
- zfs_zero_partial_page(zp, last_page_offset,
- page_len);
- }
- }
- rangelock_exit(lr);
-
- return (error);
-}
-
-/*
- * Truncate a file
- *
- * IN: zp - znode of file to free data in.
- * end - new end-of-file.
- *
- * RETURN: 0 on success, error code on failure
- */
-static int
-zfs_trunc(znode_t *zp, uint64_t end)
-{
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- dmu_tx_t *tx;
- locked_range_t *lr;
- int error;
- sa_bulk_attr_t bulk[2];
- int count = 0;
-
- /*
- * We will change zp_size, lock the whole file.
- */
- lr = rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER);
-
- /*
- * Nothing to do if file already at desired length.
- */
- if (end >= zp->z_size) {
- rangelock_exit(lr);
- return (0);
- }
-
- error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, end,
- DMU_OBJECT_END);
- if (error) {
- rangelock_exit(lr);
- return (error);
- }
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
- dmu_tx_mark_netfree(tx);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- rangelock_exit(lr);
- return (error);
- }
-
- zp->z_size = end;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs),
- NULL, &zp->z_size, sizeof (zp->z_size));
-
- if (end == 0) {
- zp->z_pflags &= ~ZFS_SPARSE;
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
- NULL, &zp->z_pflags, 8);
- }
- VERIFY(sa_bulk_update(zp->z_sa_hdl, bulk, count, tx) == 0);
-
- dmu_tx_commit(tx);
- rangelock_exit(lr);
-
- return (0);
-}
-
-/*
- * Free space in a file
- *
- * IN: zp - znode of file to free data in.
- * off - start of range
- * len - end of range (0 => EOF)
- * flag - current file open mode flags.
- * log - TRUE if this action should be logged
- *
- * RETURN: 0 on success, error code on failure
- */
-int
-zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log)
-{
- dmu_tx_t *tx;
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- zilog_t *zilog = zfsvfs->z_log;
- uint64_t mode;
- uint64_t mtime[2], ctime[2];
- sa_bulk_attr_t bulk[3];
- int count = 0;
- int error;
-
- if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), &mode,
- sizeof (mode))) != 0)
- return (error);
-
- if (off > zp->z_size) {
- error = zfs_extend(zp, off+len);
- if (error == 0 && log)
- goto log;
- goto out;
- }
-
- if (len == 0) {
- error = zfs_trunc(zp, off);
- } else {
- if ((error = zfs_free_range(zp, off, len)) == 0 &&
- off + len > zp->z_size)
- error = zfs_extend(zp, off+len);
- }
- if (error || !log)
- goto out;
-log:
- tx = dmu_tx_create(zfsvfs->z_os);
- dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
- zfs_sa_upgrade_txholds(tx, zp);
- error = dmu_tx_assign(tx, TXG_WAIT);
- if (error) {
- dmu_tx_abort(tx);
- goto out;
- }
-
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, mtime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, ctime, 16);
- SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
- NULL, &zp->z_pflags, 8);
- zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
- error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
- ASSERT(error == 0);
-
- zfs_log_truncate(zilog, tx, TX_TRUNCATE, zp, off, len);
-
- dmu_tx_commit(tx);
-
- zfs_inode_update(zp);
- error = 0;
-
-out:
- /*
- * Truncate the page cache - for file truncate operations, use
- * the purpose-built API for truncations. For punching operations,
- * the truncation is handled under a range lock in zfs_free_range.
- */
- if (len == 0)
- truncate_setsize(ZTOI(zp), off);
- return (error);
-}
-
-void
-zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
-{
- struct super_block *sb;
- zfsvfs_t *zfsvfs;
- uint64_t moid, obj, sa_obj, version;
- uint64_t sense = ZFS_CASE_SENSITIVE;
- uint64_t norm = 0;
- nvpair_t *elem;
- int size;
- int error;
- int i;
- znode_t *rootzp = NULL;
- vattr_t vattr;
- znode_t *zp;
- zfs_acl_ids_t acl_ids;
-
- /*
- * First attempt to create master node.
- */
- /*
- * In an empty objset, there are no blocks to read and thus
- * there can be no i/o errors (which we assert below).
- */
- moid = MASTER_NODE_OBJ;
- error = zap_create_claim(os, moid, DMU_OT_MASTER_NODE,
- DMU_OT_NONE, 0, tx);
- ASSERT(error == 0);
-
- /*
- * Set starting attributes.
- */
- version = zfs_zpl_version_map(spa_version(dmu_objset_spa(os)));
- elem = NULL;
- while ((elem = nvlist_next_nvpair(zplprops, elem)) != NULL) {
- /* For the moment we expect all zpl props to be uint64_ts */
- uint64_t val;
- char *name;
-
- ASSERT(nvpair_type(elem) == DATA_TYPE_UINT64);
- VERIFY(nvpair_value_uint64(elem, &val) == 0);
- name = nvpair_name(elem);
- if (strcmp(name, zfs_prop_to_name(ZFS_PROP_VERSION)) == 0) {
- if (val < version)
- version = val;
- } else {
- error = zap_update(os, moid, name, 8, 1, &val, tx);
- }
- ASSERT(error == 0);
- if (strcmp(name, zfs_prop_to_name(ZFS_PROP_NORMALIZE)) == 0)
- norm = val;
- else if (strcmp(name, zfs_prop_to_name(ZFS_PROP_CASE)) == 0)
- sense = val;
- }
- ASSERT(version != 0);
- error = zap_update(os, moid, ZPL_VERSION_STR, 8, 1, &version, tx);
-
- /*
- * Create zap object used for SA attribute registration
- */
-
- if (version >= ZPL_VERSION_SA) {
- sa_obj = zap_create(os, DMU_OT_SA_MASTER_NODE,
- DMU_OT_NONE, 0, tx);
- error = zap_add(os, moid, ZFS_SA_ATTRS, 8, 1, &sa_obj, tx);
- ASSERT(error == 0);
- } else {
- sa_obj = 0;
- }
- /*
- * Create a delete queue.
- */
- obj = zap_create(os, DMU_OT_UNLINKED_SET, DMU_OT_NONE, 0, tx);
-
- error = zap_add(os, moid, ZFS_UNLINKED_SET, 8, 1, &obj, tx);
- ASSERT(error == 0);
-
- /*
- * Create root znode. Create minimal znode/inode/zfsvfs/sb
- * to allow zfs_mknode to work.
- */
- vattr.va_mask = ATTR_MODE|ATTR_UID|ATTR_GID;
- vattr.va_mode = S_IFDIR|0755;
- vattr.va_uid = crgetuid(cr);
- vattr.va_gid = crgetgid(cr);
-
- rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP);
- rootzp->z_unlinked = B_FALSE;
- rootzp->z_atime_dirty = B_FALSE;
- rootzp->z_moved = B_FALSE;
- rootzp->z_is_sa = USE_SA(version, os);
- rootzp->z_pflags = 0;
-
- zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
- zfsvfs->z_os = os;
- zfsvfs->z_parent = zfsvfs;
- zfsvfs->z_version = version;
- zfsvfs->z_use_fuids = USE_FUIDS(version, os);
- zfsvfs->z_use_sa = USE_SA(version, os);
- zfsvfs->z_norm = norm;
-
- sb = kmem_zalloc(sizeof (struct super_block), KM_SLEEP);
- sb->s_fs_info = zfsvfs;
-
- ZTOI(rootzp)->i_sb = sb;
-
- error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END,
- &zfsvfs->z_attr_table);
-
- ASSERT(error == 0);
-
- /*
- * Fold case on file systems that are always or sometimes case
- * insensitive.
- */
- if (sense == ZFS_CASE_INSENSITIVE || sense == ZFS_CASE_MIXED)
- zfsvfs->z_norm |= U8_TEXTPREP_TOUPPER;
-
- mutex_init(&zfsvfs->z_znodes_lock, NULL, MUTEX_DEFAULT, NULL);
- list_create(&zfsvfs->z_all_znodes, sizeof (znode_t),
- offsetof(znode_t, z_link_node));
-
- size = MIN(1 << (highbit64(zfs_object_mutex_size)-1), ZFS_OBJ_MTX_MAX);
- zfsvfs->z_hold_size = size;
- zfsvfs->z_hold_trees = vmem_zalloc(sizeof (avl_tree_t) * size,
- KM_SLEEP);
- zfsvfs->z_hold_locks = vmem_zalloc(sizeof (kmutex_t) * size, KM_SLEEP);
- for (i = 0; i != size; i++) {
- avl_create(&zfsvfs->z_hold_trees[i], zfs_znode_hold_compare,
- sizeof (znode_hold_t), offsetof(znode_hold_t, zh_node));
- mutex_init(&zfsvfs->z_hold_locks[i], NULL, MUTEX_DEFAULT, NULL);
- }
-
- VERIFY(0 == zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr,
- cr, NULL, &acl_ids));
- zfs_mknode(rootzp, &vattr, tx, cr, IS_ROOT_NODE, &zp, &acl_ids);
- ASSERT3P(zp, ==, rootzp);
- error = zap_add(os, moid, ZFS_ROOT_OBJ, 8, 1, &rootzp->z_id, tx);
- ASSERT(error == 0);
- zfs_acl_ids_free(&acl_ids);
-
- atomic_set(&ZTOI(rootzp)->i_count, 0);
- sa_handle_destroy(rootzp->z_sa_hdl);
- kmem_cache_free(znode_cache, rootzp);
-
- for (i = 0; i != size; i++) {
- avl_destroy(&zfsvfs->z_hold_trees[i]);
- mutex_destroy(&zfsvfs->z_hold_locks[i]);
- }
-
- mutex_destroy(&zfsvfs->z_znodes_lock);
-
- vmem_free(zfsvfs->z_hold_trees, sizeof (avl_tree_t) * size);
- vmem_free(zfsvfs->z_hold_locks, sizeof (kmutex_t) * size);
- kmem_free(sb, sizeof (struct super_block));
- kmem_free(zfsvfs, sizeof (zfsvfs_t));
-}
-#endif /* _KERNEL */
-
-static int
-zfs_sa_setup(objset_t *osp, sa_attr_type_t **sa_table)
-{
- uint64_t sa_obj = 0;
- int error;
-
- error = zap_lookup(osp, MASTER_NODE_OBJ, ZFS_SA_ATTRS, 8, 1, &sa_obj);
- if (error != 0 && error != ENOENT)
- return (error);
-
- error = sa_setup(osp, sa_obj, zfs_attr_table, ZPL_END, sa_table);
- return (error);
-}
-
-static int
-zfs_grab_sa_handle(objset_t *osp, uint64_t obj, sa_handle_t **hdlp,
- dmu_buf_t **db, void *tag)
-{
- dmu_object_info_t doi;
- int error;
-
- if ((error = sa_buf_hold(osp, obj, tag, db)) != 0)
- return (error);
-
- dmu_object_info_from_db(*db, &doi);
- if ((doi.doi_bonus_type != DMU_OT_SA &&
- doi.doi_bonus_type != DMU_OT_ZNODE) ||
- (doi.doi_bonus_type == DMU_OT_ZNODE &&
- doi.doi_bonus_size < sizeof (znode_phys_t))) {
- sa_buf_rele(*db, tag);
- return (SET_ERROR(ENOTSUP));
- }
-
- error = sa_handle_get(osp, obj, NULL, SA_HDL_PRIVATE, hdlp);
- if (error != 0) {
- sa_buf_rele(*db, tag);
- return (error);
- }
-
- return (0);
-}
-
-void
-zfs_release_sa_handle(sa_handle_t *hdl, dmu_buf_t *db, void *tag)
-{
- sa_handle_destroy(hdl);
- sa_buf_rele(db, tag);
-}
-
-/*
- * Given an object number, return its parent object number and whether
- * or not the object is an extended attribute directory.
- */
-static int
-zfs_obj_to_pobj(objset_t *osp, sa_handle_t *hdl, sa_attr_type_t *sa_table,
- uint64_t *pobjp, int *is_xattrdir)
-{
- uint64_t parent;
- uint64_t pflags;
- uint64_t mode;
- uint64_t parent_mode;
- sa_bulk_attr_t bulk[3];
- sa_handle_t *sa_hdl;
- dmu_buf_t *sa_db;
- int count = 0;
- int error;
-
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_PARENT], NULL,
- &parent, sizeof (parent));
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_FLAGS], NULL,
- &pflags, sizeof (pflags));
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_MODE], NULL,
- &mode, sizeof (mode));
-
- if ((error = sa_bulk_lookup(hdl, bulk, count)) != 0)
- return (error);
-
- /*
- * When a link is removed its parent pointer is not changed and will
- * be invalid. There are two cases where a link is removed but the
- * file stays around, when it goes to the delete queue and when there
- * are additional links.
- */
- error = zfs_grab_sa_handle(osp, parent, &sa_hdl, &sa_db, FTAG);
- if (error != 0)
- return (error);
-
- error = sa_lookup(sa_hdl, ZPL_MODE, &parent_mode, sizeof (parent_mode));
- zfs_release_sa_handle(sa_hdl, sa_db, FTAG);
- if (error != 0)
- return (error);
-
- *is_xattrdir = ((pflags & ZFS_XATTR) != 0) && S_ISDIR(mode);
-
- /*
- * Extended attributes can be applied to files, directories, etc.
- * Otherwise the parent must be a directory.
- */
- if (!*is_xattrdir && !S_ISDIR(parent_mode))
- return (SET_ERROR(EINVAL));
-
- *pobjp = parent;
-
- return (0);
-}
-
-/*
- * Given an object number, return some zpl level statistics
- */
-static int
-zfs_obj_to_stats_impl(sa_handle_t *hdl, sa_attr_type_t *sa_table,
- zfs_stat_t *sb)
-{
- sa_bulk_attr_t bulk[4];
- int count = 0;
-
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_MODE], NULL,
- &sb->zs_mode, sizeof (sb->zs_mode));
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_GEN], NULL,
- &sb->zs_gen, sizeof (sb->zs_gen));
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_LINKS], NULL,
- &sb->zs_links, sizeof (sb->zs_links));
- SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_CTIME], NULL,
- &sb->zs_ctime, sizeof (sb->zs_ctime));
-
- return (sa_bulk_lookup(hdl, bulk, count));
-}
-
-static int
-zfs_obj_to_path_impl(objset_t *osp, uint64_t obj, sa_handle_t *hdl,
- sa_attr_type_t *sa_table, char *buf, int len)
-{
- sa_handle_t *sa_hdl;
- sa_handle_t *prevhdl = NULL;
- dmu_buf_t *prevdb = NULL;
- dmu_buf_t *sa_db = NULL;
- char *path = buf + len - 1;
- int error;
-
- *path = '\0';
- sa_hdl = hdl;
-
- uint64_t deleteq_obj;
- VERIFY0(zap_lookup(osp, MASTER_NODE_OBJ,
- ZFS_UNLINKED_SET, sizeof (uint64_t), 1, &deleteq_obj));
- error = zap_lookup_int(osp, deleteq_obj, obj);
- if (error == 0) {
- return (ESTALE);
- } else if (error != ENOENT) {
- return (error);
- }
- error = 0;
-
- for (;;) {
- uint64_t pobj = 0;
- char component[MAXNAMELEN + 2];
- size_t complen;
- int is_xattrdir = 0;
-
- if (prevdb)
- zfs_release_sa_handle(prevhdl, prevdb, FTAG);
-
- if ((error = zfs_obj_to_pobj(osp, sa_hdl, sa_table, &pobj,
- &is_xattrdir)) != 0)
- break;
-
- if (pobj == obj) {
- if (path[0] != '/')
- *--path = '/';
- break;
- }
-
- component[0] = '/';
- if (is_xattrdir) {
- (void) sprintf(component + 1, "<xattrdir>");
- } else {
- error = zap_value_search(osp, pobj, obj,
- ZFS_DIRENT_OBJ(-1ULL), component + 1);
- if (error != 0)
- break;
- }
-
- complen = strlen(component);
- path -= complen;
- ASSERT(path >= buf);
- bcopy(component, path, complen);
- obj = pobj;
-
- if (sa_hdl != hdl) {
- prevhdl = sa_hdl;
- prevdb = sa_db;
- }
- error = zfs_grab_sa_handle(osp, obj, &sa_hdl, &sa_db, FTAG);
- if (error != 0) {
- sa_hdl = prevhdl;
- sa_db = prevdb;
- break;
- }
- }
-
- if (sa_hdl != NULL && sa_hdl != hdl) {
- ASSERT(sa_db != NULL);
- zfs_release_sa_handle(sa_hdl, sa_db, FTAG);
- }
-
- if (error == 0)
- (void) memmove(buf, path, buf + len - path);
-
- return (error);
-}
-
-int
-zfs_obj_to_path(objset_t *osp, uint64_t obj, char *buf, int len)
-{
- sa_attr_type_t *sa_table;
- sa_handle_t *hdl;
- dmu_buf_t *db;
- int error;
-
- error = zfs_sa_setup(osp, &sa_table);
- if (error != 0)
- return (error);
-
- error = zfs_grab_sa_handle(osp, obj, &hdl, &db, FTAG);
- if (error != 0)
- return (error);
-
- error = zfs_obj_to_path_impl(osp, obj, hdl, sa_table, buf, len);
-
- zfs_release_sa_handle(hdl, db, FTAG);
- return (error);
-}
-
-int
-zfs_obj_to_stats(objset_t *osp, uint64_t obj, zfs_stat_t *sb,
- char *buf, int len)
-{
- char *path = buf + len - 1;
- sa_attr_type_t *sa_table;
- sa_handle_t *hdl;
- dmu_buf_t *db;
- int error;
-
- *path = '\0';
-
- error = zfs_sa_setup(osp, &sa_table);
- if (error != 0)
- return (error);
-
- error = zfs_grab_sa_handle(osp, obj, &hdl, &db, FTAG);
- if (error != 0)
- return (error);
-
- error = zfs_obj_to_stats_impl(hdl, sa_table, sb);
- if (error != 0) {
- zfs_release_sa_handle(hdl, db, FTAG);
- return (error);
- }
-
- error = zfs_obj_to_path_impl(osp, obj, hdl, sa_table, buf, len);
-
- zfs_release_sa_handle(hdl, db, FTAG);
- return (error);
-}
-
-#if defined(_KERNEL)
-EXPORT_SYMBOL(zfs_create_fs);
-EXPORT_SYMBOL(zfs_obj_to_path);
-
-/* CSTYLED */
-module_param(zfs_object_mutex_size, uint, 0644);
-MODULE_PARM_DESC(zfs_object_mutex_size, "Size of znode hold array");
-module_param(zfs_unlink_suspend_progress, int, 0644);
-MODULE_PARM_DESC(zfs_unlink_suspend_progress, "Set to prevent async unlinks "
-"(debug - leaks space into the unlinked set)");
-#endif
diff --git a/module/zfs/zio_crypt.c b/module/zfs/zio_crypt.c
deleted file mode 100644
index 7cf20f413..000000000
--- a/module/zfs/zio_crypt.c
+++ /dev/null
@@ -1,2036 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * This file and its contents are supplied under the terms of the
- * Common Development and Distribution License ("CDDL"), version 1.0.
- * You may only use this file in accordance with the terms of version
- * 1.0 of the CDDL.
- *
- * A full copy of the text of the CDDL should have accompanied this
- * source. A copy of the CDDL is also available via the Internet at
- * http://www.illumos.org/license/CDDL.
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright (c) 2017, Datto, Inc. All rights reserved.
- */
-
-#include <sys/zio_crypt.h>
-#include <sys/dmu.h>
-#include <sys/dmu_objset.h>
-#include <sys/dnode.h>
-#include <sys/fs/zfs.h>
-#include <sys/zio.h>
-#include <sys/zil.h>
-#include <sys/sha2.h>
-#include <sys/hkdf.h>
-#include "qat.h"
-
-/*
- * This file is responsible for handling all of the details of generating
- * encryption parameters and performing encryption and authentication.
- *
- * BLOCK ENCRYPTION PARAMETERS:
- * Encryption /Authentication Algorithm Suite (crypt):
- * The encryption algorithm, mode, and key length we are going to use. We
- * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
- * keys. All authentication is currently done with SHA512-HMAC.
- *
- * Plaintext:
- * The unencrypted data that we want to encrypt.
- *
- * Initialization Vector (IV):
- * An initialization vector for the encryption algorithms. This is used to
- * "tweak" the encryption algorithms so that two blocks of the same data are
- * encrypted into different ciphertext outputs, thus obfuscating block patterns.
- * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
- * never reused with the same encryption key. This value is stored unencrypted
- * and must simply be provided to the decryption function. We use a 96 bit IV
- * (as recommended by NIST) for all block encryption. For non-dedup blocks we
- * derive the IV randomly. The first 64 bits of the IV are stored in the second
- * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
- * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
- * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
- * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
- * level 0 blocks is the number of allocated dnodes in that block. The on-disk
- * format supports at most 2^15 slots per L0 dnode block, because the maximum
- * block size is 16MB (2^24). In either case, for level 0 blocks this number
- * will still be smaller than UINT32_MAX so it is safe to store the IV in the
- * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
- * for the dnode code.
- *
- * Master key:
- * This is the most important secret data of an encrypted dataset. It is used
- * along with the salt to generate that actual encryption keys via HKDF. We
- * do not use the master key to directly encrypt any data because there are
- * theoretical limits on how much data can actually be safely encrypted with
- * any encryption mode. The master key is stored encrypted on disk with the
- * user's wrapping key. Its length is determined by the encryption algorithm.
- * For details on how this is stored see the block comment in dsl_crypt.c
- *
- * Salt:
- * Used as an input to the HKDF function, along with the master key. We use a
- * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
- * can be used for encrypting many blocks, so we cache the current salt and the
- * associated derived key in zio_crypt_t so we do not need to derive it again
- * needlessly.
- *
- * Encryption Key:
- * A secret binary key, generated from an HKDF function used to encrypt and
- * decrypt data.
- *
- * Message Authentication Code (MAC)
- * The MAC is an output of authenticated encryption modes such as AES-GCM and
- * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
- * data on disk and return garbage to the application. Effectively, it is a
- * checksum that can not be reproduced by an attacker. We store the MAC in the
- * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
- * regular checksum of the ciphertext which can be used for scrubbing.
- *
- * OBJECT AUTHENTICATION:
- * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
- * they contain some info that always needs to be readable. To prevent this
- * data from being altered, we authenticate this data using SHA512-HMAC. This
- * will produce a MAC (similar to the one produced via encryption) which can
- * be used to verify the object was not modified. HMACs do not require key
- * rotation or IVs, so we can keep up to the full 3 copies of authenticated
- * data.
- *
- * ZIL ENCRYPTION:
- * ZIL blocks have their bp written to disk ahead of the associated data, so we
- * cannot store the MAC there as we normally do. For these blocks the MAC is
- * stored in the embedded checksum within the zil_chain_t header. The salt and
- * IV are generated for the block on bp allocation instead of at encryption
- * time. In addition, ZIL blocks have some pieces that must be left in plaintext
- * for claiming even though all of the sensitive user data still needs to be
- * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
- * pieces of the block need to be encrypted. All data that is not encrypted is
- * authenticated using the AAD mechanisms that the supported encryption modes
- * provide for. In order to preserve the semantics of the ZIL for encrypted
- * datasets, the ZIL is not protected at the objset level as described below.
- *
- * DNODE ENCRYPTION:
- * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
- * in plaintext for scrubbing and claiming, but the bonus buffers might contain
- * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
- * which which pieces of the block need to be encrypted. For more details about
- * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
- *
- * OBJECT SET AUTHENTICATION:
- * Up to this point, everything we have encrypted and authenticated has been
- * at level 0 (or -2 for the ZIL). If we did not do any further work the
- * on-disk format would be susceptible to attacks that deleted or rearranged
- * the order of level 0 blocks. Ideally, the cleanest solution would be to
- * maintain a tree of authentication MACs going up the bp tree. However, this
- * presents a problem for raw sends. Send files do not send information about
- * indirect blocks so there would be no convenient way to transfer the MACs and
- * they cannot be recalculated on the receive side without the master key which
- * would defeat one of the purposes of raw sends in the first place. Instead,
- * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
- * from the level below. We also include some portable fields from blk_prop such
- * as the lsize and compression algorithm to prevent the data from being
- * misinterpreted.
- *
- * At the objset level, we maintain 2 separate 256 bit MACs in the
- * objset_phys_t. The first one is "portable" and is the logical root of the
- * MAC tree maintained in the metadnode's bps. The second, is "local" and is
- * used as the root MAC for the user accounting objects, which are also not
- * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
- * of the send file. The useraccounting code ensures that the useraccounting
- * info is not present upon a receive, so the local MAC can simply be cleared
- * out at that time. For more info about objset_phys_t authentication, see
- * zio_crypt_do_objset_hmacs().
- *
- * CONSIDERATIONS FOR DEDUP:
- * In order for dedup to work, blocks that we want to dedup with one another
- * need to use the same IV and encryption key, so that they will have the same
- * ciphertext. Normally, one should never reuse an IV with the same encryption
- * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
- * blocks. In this case, however, since we are using the same plaintext as
- * well all that we end up with is a duplicate of the original ciphertext we
- * already had. As a result, an attacker with read access to the raw disk will
- * be able to tell which blocks are the same but this information is given away
- * by dedup anyway. In order to get the same IVs and encryption keys for
- * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC
- * here so that a reproducible checksum of the plaintext is never available to
- * the attacker. The HMAC key is kept alongside the master key, encrypted on
- * disk. The first 64 bits of the HMAC are used in place of the random salt, and
- * the next 96 bits are used as the IV. As a result of this mechanism, dedup
- * will only work within a clone family since encrypted dedup requires use of
- * the same master and HMAC keys.
- */
-
-/*
- * After encrypting many blocks with the same key we may start to run up
- * against the theoretical limits of how much data can securely be encrypted
- * with a single key using the supported encryption modes. The most obvious
- * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
- * the more IVs we generate (which both GCM and CCM modes strictly forbid).
- * This risk actually grows surprisingly quickly over time according to the
- * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
- * generated n IVs with a cryptographically secure RNG, the approximate
- * probability p(n) of a collision is given as:
- *
- * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
- *
- * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
- *
- * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
- * we must not write more than 398,065,730 blocks with the same encryption key.
- * Therefore, we rotate our keys after 400,000,000 blocks have been written by
- * generating a new random 64 bit salt for our HKDF encryption key generation
- * function.
- */
-#define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000
-#define ZFS_CURRENT_MAX_SALT_USES \
- (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
-unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT;
-
-typedef struct blkptr_auth_buf {
- uint64_t bab_prop; /* blk_prop - portable mask */
- uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */
- uint64_t bab_pad; /* reserved for future use */
-} blkptr_auth_buf_t;
-
-zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
- {"", ZC_TYPE_NONE, 0, "inherit"},
- {"", ZC_TYPE_NONE, 0, "on"},
- {"", ZC_TYPE_NONE, 0, "off"},
- {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"},
- {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"},
- {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"},
- {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"},
- {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"},
- {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"}
-};
-
-void
-zio_crypt_key_destroy(zio_crypt_key_t *key)
-{
- rw_destroy(&key->zk_salt_lock);
-
- /* free crypto templates */
- crypto_destroy_ctx_template(key->zk_current_tmpl);
- crypto_destroy_ctx_template(key->zk_hmac_tmpl);
-
- /* zero out sensitive data */
- bzero(key, sizeof (zio_crypt_key_t));
-}
-
-int
-zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
-{
- int ret;
- crypto_mechanism_t mech;
- uint_t keydata_len;
-
- ASSERT(key != NULL);
- ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
-
- keydata_len = zio_crypt_table[crypt].ci_keylen;
- bzero(key, sizeof (zio_crypt_key_t));
-
- /* fill keydata buffers and salt with random data */
- ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t));
- if (ret != 0)
- goto error;
-
- ret = random_get_bytes(key->zk_master_keydata, keydata_len);
- if (ret != 0)
- goto error;
-
- ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN);
- if (ret != 0)
- goto error;
-
- ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
- if (ret != 0)
- goto error;
-
- /* derive the current key from the master key */
- ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
- key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
- keydata_len);
- if (ret != 0)
- goto error;
-
- /* initialize keys for the ICP */
- key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
- key->zk_current_key.ck_data = key->zk_current_keydata;
- key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
-
- key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
- key->zk_hmac_key.ck_data = &key->zk_hmac_key;
- key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
-
- /*
- * Initialize the crypto templates. It's ok if this fails because
- * this is just an optimization.
- */
- mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
- ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
- &key->zk_current_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_current_tmpl = NULL;
-
- mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
- ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
- &key->zk_hmac_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_hmac_tmpl = NULL;
-
- key->zk_crypt = crypt;
- key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION;
- key->zk_salt_count = 0;
- rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
-
- return (0);
-
-error:
- zio_crypt_key_destroy(key);
- return (ret);
-}
-
-static int
-zio_crypt_key_change_salt(zio_crypt_key_t *key)
-{
- int ret = 0;
- uint8_t salt[ZIO_DATA_SALT_LEN];
- crypto_mechanism_t mech;
- uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen;
-
- /* generate a new salt */
- ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN);
- if (ret != 0)
- goto error;
-
- rw_enter(&key->zk_salt_lock, RW_WRITER);
-
- /* someone beat us to the salt rotation, just unlock and return */
- if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES)
- goto out_unlock;
-
- /* derive the current key from the master key and the new salt */
- ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
- salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len);
- if (ret != 0)
- goto out_unlock;
-
- /* assign the salt and reset the usage count */
- bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN);
- key->zk_salt_count = 0;
-
- /* destroy the old context template and create the new one */
- crypto_destroy_ctx_template(key->zk_current_tmpl);
- ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
- &key->zk_current_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_current_tmpl = NULL;
-
- rw_exit(&key->zk_salt_lock);
-
- return (0);
-
-out_unlock:
- rw_exit(&key->zk_salt_lock);
-error:
- return (ret);
-}
-
-/* See comment above zfs_key_max_salt_uses definition for details */
-int
-zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt)
-{
- int ret;
- boolean_t salt_change;
-
- rw_enter(&key->zk_salt_lock, RW_READER);
-
- bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN);
- salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >=
- ZFS_CURRENT_MAX_SALT_USES);
-
- rw_exit(&key->zk_salt_lock);
-
- if (salt_change) {
- ret = zio_crypt_key_change_salt(key);
- if (ret != 0)
- goto error;
- }
-
- return (0);
-
-error:
- return (ret);
-}
-
-/*
- * This function handles all encryption and decryption in zfs. When
- * encrypting it expects puio to reference the plaintext and cuio to
- * reference the ciphertext. cuio must have enough space for the
- * ciphertext + room for a MAC. datalen should be the length of the
- * plaintext / ciphertext alone.
- */
-static int
-zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
- crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
- uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
-{
- int ret;
- crypto_data_t plaindata, cipherdata;
- CK_AES_CCM_PARAMS ccmp;
- CK_AES_GCM_PARAMS gcmp;
- crypto_mechanism_t mech;
- zio_crypt_info_t crypt_info;
- uint_t plain_full_len, maclen;
-
- ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
- ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW);
-
- /* lookup the encryption info */
- crypt_info = zio_crypt_table[crypt];
-
- /* the mac will always be the last iovec_t in the cipher uio */
- maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len;
-
- ASSERT(maclen <= ZIO_DATA_MAC_LEN);
-
- /* setup encryption mechanism (same as crypt) */
- mech.cm_type = crypto_mech2id(crypt_info.ci_mechname);
-
- /*
- * Strangely, the ICP requires that plain_full_len must include
- * the MAC length when decrypting, even though the UIO does not
- * need to have the extra space allocated.
- */
- if (encrypt) {
- plain_full_len = datalen;
- } else {
- plain_full_len = datalen + maclen;
- }
-
- /*
- * setup encryption params (currently only AES CCM and AES GCM
- * are supported)
- */
- if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) {
- ccmp.ulNonceSize = ZIO_DATA_IV_LEN;
- ccmp.ulAuthDataSize = auth_len;
- ccmp.authData = authbuf;
- ccmp.ulMACSize = maclen;
- ccmp.nonce = ivbuf;
- ccmp.ulDataSize = plain_full_len;
-
- mech.cm_param = (char *)(&ccmp);
- mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
- } else {
- gcmp.ulIvLen = ZIO_DATA_IV_LEN;
- gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
- gcmp.ulAADLen = auth_len;
- gcmp.pAAD = authbuf;
- gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
- gcmp.pIv = ivbuf;
-
- mech.cm_param = (char *)(&gcmp);
- mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
- }
-
- /* populate the cipher and plain data structs. */
- plaindata.cd_format = CRYPTO_DATA_UIO;
- plaindata.cd_offset = 0;
- plaindata.cd_uio = puio;
- plaindata.cd_miscdata = NULL;
- plaindata.cd_length = plain_full_len;
-
- cipherdata.cd_format = CRYPTO_DATA_UIO;
- cipherdata.cd_offset = 0;
- cipherdata.cd_uio = cuio;
- cipherdata.cd_miscdata = NULL;
- cipherdata.cd_length = datalen + maclen;
-
- /* perform the actual encryption */
- if (encrypt) {
- ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata,
- NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
- } else {
- ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata,
- NULL);
- if (ret != CRYPTO_SUCCESS) {
- ASSERT3U(ret, ==, CRYPTO_INVALID_MAC);
- ret = SET_ERROR(ECKSUM);
- goto error;
- }
- }
-
- return (0);
-
-error:
- return (ret);
-}
-
-int
-zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
- uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
-{
- int ret;
- uio_t puio, cuio;
- uint64_t aad[3];
- iovec_t plain_iovecs[2], cipher_iovecs[3];
- uint64_t crypt = key->zk_crypt;
- uint_t enc_len, keydata_len, aad_len;
-
- ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
- ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
-
- keydata_len = zio_crypt_table[crypt].ci_keylen;
-
- /* generate iv for wrapping the master and hmac key */
- ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN);
- if (ret != 0)
- goto error;
-
- /* initialize uio_ts */
- plain_iovecs[0].iov_base = key->zk_master_keydata;
- plain_iovecs[0].iov_len = keydata_len;
- plain_iovecs[1].iov_base = key->zk_hmac_keydata;
- plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
-
- cipher_iovecs[0].iov_base = keydata_out;
- cipher_iovecs[0].iov_len = keydata_len;
- cipher_iovecs[1].iov_base = hmac_keydata_out;
- cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
- cipher_iovecs[2].iov_base = mac;
- cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
-
- /*
- * Although we don't support writing to the old format, we do
- * support rewrapping the key so that the user can move and
- * quarantine datasets on the old format.
- */
- if (key->zk_version == 0) {
- aad_len = sizeof (uint64_t);
- aad[0] = LE_64(key->zk_guid);
- } else {
- ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
- aad_len = sizeof (uint64_t) * 3;
- aad[0] = LE_64(key->zk_guid);
- aad[1] = LE_64(crypt);
- aad[2] = LE_64(key->zk_version);
- }
-
- enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN;
- puio.uio_iov = plain_iovecs;
- puio.uio_iovcnt = 2;
- puio.uio_segflg = UIO_SYSSPACE;
- cuio.uio_iov = cipher_iovecs;
- cuio.uio_iovcnt = 3;
- cuio.uio_segflg = UIO_SYSSPACE;
-
- /* encrypt the keys and store the resulting ciphertext and mac */
- ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
- &puio, &cuio, (uint8_t *)aad, aad_len);
- if (ret != 0)
- goto error;
-
- return (0);
-
-error:
- return (ret);
-}
-
-int
-zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
- uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
- uint8_t *mac, zio_crypt_key_t *key)
-{
- int ret;
- crypto_mechanism_t mech;
- uio_t puio, cuio;
- uint64_t aad[3];
- iovec_t plain_iovecs[2], cipher_iovecs[3];
- uint_t enc_len, keydata_len, aad_len;
-
- ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
- ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
-
- rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
-
- keydata_len = zio_crypt_table[crypt].ci_keylen;
-
- /* initialize uio_ts */
- plain_iovecs[0].iov_base = key->zk_master_keydata;
- plain_iovecs[0].iov_len = keydata_len;
- plain_iovecs[1].iov_base = key->zk_hmac_keydata;
- plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
-
- cipher_iovecs[0].iov_base = keydata;
- cipher_iovecs[0].iov_len = keydata_len;
- cipher_iovecs[1].iov_base = hmac_keydata;
- cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
- cipher_iovecs[2].iov_base = mac;
- cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
-
- if (version == 0) {
- aad_len = sizeof (uint64_t);
- aad[0] = LE_64(guid);
- } else {
- ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
- aad_len = sizeof (uint64_t) * 3;
- aad[0] = LE_64(guid);
- aad[1] = LE_64(crypt);
- aad[2] = LE_64(version);
- }
-
- enc_len = keydata_len + SHA512_HMAC_KEYLEN;
- puio.uio_iov = plain_iovecs;
- puio.uio_segflg = UIO_SYSSPACE;
- puio.uio_iovcnt = 2;
- cuio.uio_iov = cipher_iovecs;
- cuio.uio_iovcnt = 3;
- cuio.uio_segflg = UIO_SYSSPACE;
-
- /* decrypt the keys and store the result in the output buffers */
- ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
- &puio, &cuio, (uint8_t *)aad, aad_len);
- if (ret != 0)
- goto error;
-
- /* generate a fresh salt */
- ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
- if (ret != 0)
- goto error;
-
- /* derive the current key from the master key */
- ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
- key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
- keydata_len);
- if (ret != 0)
- goto error;
-
- /* initialize keys for ICP */
- key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
- key->zk_current_key.ck_data = key->zk_current_keydata;
- key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
-
- key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
- key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
- key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
-
- /*
- * Initialize the crypto templates. It's ok if this fails because
- * this is just an optimization.
- */
- mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
- ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
- &key->zk_current_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_current_tmpl = NULL;
-
- mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
- ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
- &key->zk_hmac_tmpl, KM_SLEEP);
- if (ret != CRYPTO_SUCCESS)
- key->zk_hmac_tmpl = NULL;
-
- key->zk_crypt = crypt;
- key->zk_version = version;
- key->zk_guid = guid;
- key->zk_salt_count = 0;
-
- return (0);
-
-error:
- zio_crypt_key_destroy(key);
- return (ret);
-}
-
-int
-zio_crypt_generate_iv(uint8_t *ivbuf)
-{
- int ret;
-
- /* randomly generate the IV */
- ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN);
- if (ret != 0)
- goto error;
-
- return (0);
-
-error:
- bzero(ivbuf, ZIO_DATA_IV_LEN);
- return (ret);
-}
-
-int
-zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
- uint8_t *digestbuf, uint_t digestlen)
-{
- int ret;
- crypto_mechanism_t mech;
- crypto_data_t in_data, digest_data;
- uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
-
- ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
-
- /* initialize sha512-hmac mechanism and crypto data */
- mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
- mech.cm_param = NULL;
- mech.cm_param_len = 0;
-
- /* initialize the crypto data */
- in_data.cd_format = CRYPTO_DATA_RAW;
- in_data.cd_offset = 0;
- in_data.cd_length = datalen;
- in_data.cd_raw.iov_base = (char *)data;
- in_data.cd_raw.iov_len = in_data.cd_length;
-
- digest_data.cd_format = CRYPTO_DATA_RAW;
- digest_data.cd_offset = 0;
- digest_data.cd_length = SHA512_DIGEST_LENGTH;
- digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
- digest_data.cd_raw.iov_len = digest_data.cd_length;
-
- /* generate the hmac */
- ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl,
- &digest_data, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- bcopy(raw_digestbuf, digestbuf, digestlen);
-
- return (0);
-
-error:
- bzero(digestbuf, digestlen);
- return (ret);
-}
-
-int
-zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
- uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
-{
- int ret;
- uint8_t digestbuf[SHA512_DIGEST_LENGTH];
-
- ret = zio_crypt_do_hmac(key, data, datalen,
- digestbuf, SHA512_DIGEST_LENGTH);
- if (ret != 0)
- return (ret);
-
- bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN);
- bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN);
-
- return (0);
-}
-
-/*
- * The following functions are used to encode and decode encryption parameters
- * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
- * byte strings, which normally means that these strings would not need to deal
- * with byteswapping at all. However, both blkptr_t and zil_header_t may be
- * byteswapped by lower layers and so we must "undo" that byteswap here upon
- * decoding and encoding in a non-native byteorder. These functions require
- * that the byteorder bit is correct before being called.
- */
-void
-zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv)
-{
- uint64_t val64;
- uint32_t val32;
-
- ASSERT(BP_IS_ENCRYPTED(bp));
-
- if (!BP_SHOULD_BYTESWAP(bp)) {
- bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t));
- bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t));
- bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
- BP_SET_IV2(bp, val32);
- } else {
- bcopy(salt, &val64, sizeof (uint64_t));
- bp->blk_dva[2].dva_word[0] = BSWAP_64(val64);
-
- bcopy(iv, &val64, sizeof (uint64_t));
- bp->blk_dva[2].dva_word[1] = BSWAP_64(val64);
-
- bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
- BP_SET_IV2(bp, BSWAP_32(val32));
- }
-}
-
-void
-zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv)
-{
- uint64_t val64;
- uint32_t val32;
-
- ASSERT(BP_IS_PROTECTED(bp));
-
- /* for convenience, so callers don't need to check */
- if (BP_IS_AUTHENTICATED(bp)) {
- bzero(salt, ZIO_DATA_SALT_LEN);
- bzero(iv, ZIO_DATA_IV_LEN);
- return;
- }
-
- if (!BP_SHOULD_BYTESWAP(bp)) {
- bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t));
- bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t));
-
- val32 = (uint32_t)BP_GET_IV2(bp);
- bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
- } else {
- val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]);
- bcopy(&val64, salt, sizeof (uint64_t));
-
- val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]);
- bcopy(&val64, iv, sizeof (uint64_t));
-
- val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp));
- bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
- }
-}
-
-void
-zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac)
-{
- uint64_t val64;
-
- ASSERT(BP_USES_CRYPT(bp));
- ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET);
-
- if (!BP_SHOULD_BYTESWAP(bp)) {
- bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t));
- bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3],
- sizeof (uint64_t));
- } else {
- bcopy(mac, &val64, sizeof (uint64_t));
- bp->blk_cksum.zc_word[2] = BSWAP_64(val64);
-
- bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t));
- bp->blk_cksum.zc_word[3] = BSWAP_64(val64);
- }
-}
-
-void
-zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac)
-{
- uint64_t val64;
-
- ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp));
-
- /* for convenience, so callers don't need to check */
- if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
- bzero(mac, ZIO_DATA_MAC_LEN);
- return;
- }
-
- if (!BP_SHOULD_BYTESWAP(bp)) {
- bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t));
- bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t),
- sizeof (uint64_t));
- } else {
- val64 = BSWAP_64(bp->blk_cksum.zc_word[2]);
- bcopy(&val64, mac, sizeof (uint64_t));
-
- val64 = BSWAP_64(bp->blk_cksum.zc_word[3]);
- bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t));
- }
-}
-
-void
-zio_crypt_encode_mac_zil(void *data, uint8_t *mac)
-{
- zil_chain_t *zilc = data;
-
- bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t));
- bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3],
- sizeof (uint64_t));
-}
-
-void
-zio_crypt_decode_mac_zil(const void *data, uint8_t *mac)
-{
- /*
- * The ZIL MAC is embedded in the block it protects, which will
- * not have been byteswapped by the time this function has been called.
- * As a result, we don't need to worry about byteswapping the MAC.
- */
- const zil_chain_t *zilc = data;
-
- bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t));
- bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t),
- sizeof (uint64_t));
-}
-
-/*
- * This routine takes a block of dnodes (src_abd) and copies only the bonus
- * buffers to the same offsets in the dst buffer. datalen should be the size
- * of both the src_abd and the dst buffer (not just the length of the bonus
- * buffers).
- */
-void
-zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
-{
- uint_t i, max_dnp = datalen >> DNODE_SHIFT;
- uint8_t *src;
- dnode_phys_t *dnp, *sdnp, *ddnp;
-
- src = abd_borrow_buf_copy(src_abd, datalen);
-
- sdnp = (dnode_phys_t *)src;
- ddnp = (dnode_phys_t *)dst;
-
- for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
- dnp = &sdnp[i];
- if (dnp->dn_type != DMU_OT_NONE &&
- DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
- dnp->dn_bonuslen != 0) {
- bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]),
- DN_MAX_BONUS_LEN(dnp));
- }
- }
-
- abd_return_buf(src_abd, src, datalen);
-}
-
-/*
- * This function decides what fields from blk_prop are included in
- * the on-disk various MAC algorithms.
- */
-static void
-zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version)
-{
- /*
- * Version 0 did not properly zero out all non-portable fields
- * as it should have done. We maintain this code so that we can
- * do read-only imports of pools on this version.
- */
- if (version == 0) {
- BP_SET_DEDUP(bp, 0);
- BP_SET_CHECKSUM(bp, 0);
- BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
- return;
- }
-
- ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
-
- /*
- * The hole_birth feature might set these fields even if this bp
- * is a hole. We zero them out here to guarantee that raw sends
- * will function with or without the feature.
- */
- if (BP_IS_HOLE(bp)) {
- bp->blk_prop = 0ULL;
- return;
- }
-
- /*
- * At L0 we want to verify these fields to ensure that data blocks
- * can not be reinterpreted. For instance, we do not want an attacker
- * to trick us into returning raw lz4 compressed data to the user
- * by modifying the compression bits. At higher levels, we cannot
- * enforce this policy since raw sends do not convey any information
- * about indirect blocks, so these values might be different on the
- * receive side. Fortunately, this does not open any new attack
- * vectors, since any alterations that can be made to a higher level
- * bp must still verify the correct order of the layer below it.
- */
- if (BP_GET_LEVEL(bp) != 0) {
- BP_SET_BYTEORDER(bp, 0);
- BP_SET_COMPRESS(bp, 0);
-
- /*
- * psize cannot be set to zero or it will trigger
- * asserts, but the value doesn't really matter as
- * long as it is constant.
- */
- BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
- }
-
- BP_SET_DEDUP(bp, 0);
- BP_SET_CHECKSUM(bp, 0);
-}
-
-static void
-zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp,
- blkptr_auth_buf_t *bab, uint_t *bab_len)
-{
- blkptr_t tmpbp = *bp;
-
- if (should_bswap)
- byteswap_uint64_array(&tmpbp, sizeof (blkptr_t));
-
- ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp));
- ASSERT0(BP_IS_EMBEDDED(&tmpbp));
-
- zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac);
-
- /*
- * We always MAC blk_prop in LE to ensure portability. This
- * must be done after decoding the mac, since the endianness
- * will get zero'd out here.
- */
- zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version);
- bab->bab_prop = LE_64(tmpbp.blk_prop);
- bab->bab_pad = 0ULL;
-
- /* version 0 did not include the padding */
- *bab_len = sizeof (blkptr_auth_buf_t);
- if (version == 0)
- *bab_len -= sizeof (uint64_t);
-}
-
-static int
-zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version,
- boolean_t should_bswap, blkptr_t *bp)
-{
- int ret;
- uint_t bab_len;
- blkptr_auth_buf_t bab;
- crypto_data_t cd;
-
- zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
- cd.cd_format = CRYPTO_DATA_RAW;
- cd.cd_offset = 0;
- cd.cd_length = bab_len;
- cd.cd_raw.iov_base = (char *)&bab;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_update(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- return (0);
-
-error:
- return (ret);
-}
-
-static void
-zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version,
- boolean_t should_bswap, blkptr_t *bp)
-{
- uint_t bab_len;
- blkptr_auth_buf_t bab;
-
- zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
- SHA2Update(ctx, &bab, bab_len);
-}
-
-static void
-zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version,
- boolean_t should_bswap, blkptr_t *bp)
-{
- uint_t bab_len;
- blkptr_auth_buf_t bab;
-
- zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
- bcopy(&bab, *aadp, bab_len);
- *aadp += bab_len;
- *aad_len += bab_len;
-}
-
-static int
-zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version,
- boolean_t should_bswap, dnode_phys_t *dnp)
-{
- int ret, i;
- dnode_phys_t *adnp;
- boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
- crypto_data_t cd;
- uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)];
-
- cd.cd_format = CRYPTO_DATA_RAW;
- cd.cd_offset = 0;
-
- /* authenticate the core dnode (masking out non-portable bits) */
- bcopy(dnp, tmp_dncore, sizeof (tmp_dncore));
- adnp = (dnode_phys_t *)tmp_dncore;
- if (le_bswap) {
- adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec);
- adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen);
- adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid);
- adnp->dn_used = BSWAP_64(adnp->dn_used);
- }
- adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
- adnp->dn_used = 0;
-
- cd.cd_length = sizeof (tmp_dncore);
- cd.cd_raw.iov_base = (char *)adnp;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_update(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- for (i = 0; i < dnp->dn_nblkptr; i++) {
- ret = zio_crypt_bp_do_hmac_updates(ctx, version,
- should_bswap, &dnp->dn_blkptr[i]);
- if (ret != 0)
- goto error;
- }
-
- if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
- ret = zio_crypt_bp_do_hmac_updates(ctx, version,
- should_bswap, DN_SPILL_BLKPTR(dnp));
- if (ret != 0)
- goto error;
- }
-
- return (0);
-
-error:
- return (ret);
-}
-
-/*
- * objset_phys_t blocks introduce a number of exceptions to the normal
- * authentication process. objset_phys_t's contain 2 separate HMACS for
- * protecting the integrity of their data. The portable_mac protects the
- * metadnode. This MAC can be sent with a raw send and protects against
- * reordering of data within the metadnode. The local_mac protects the user
- * accounting objects which are not sent from one system to another.
- *
- * In addition, objset blocks are the only blocks that can be modified and
- * written to disk without the key loaded under certain circumstances. During
- * zil_claim() we need to be able to update the zil_header_t to complete
- * claiming log blocks and during raw receives we need to write out the
- * portable_mac from the send file. Both of these actions are possible
- * because these fields are not protected by either MAC so neither one will
- * need to modify the MACs without the key. However, when the modified blocks
- * are written out they will be byteswapped into the host machine's native
- * endianness which will modify fields protected by the MAC. As a result, MAC
- * calculation for objset blocks works slightly differently from other block
- * types. Where other block types MAC the data in whatever endianness is
- * written to disk, objset blocks always MAC little endian version of their
- * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
- * and le_bswap indicates whether a byteswap is needed to get this block
- * into little endian format.
- */
-int
-zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
- boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac)
-{
- int ret;
- crypto_mechanism_t mech;
- crypto_context_t ctx;
- crypto_data_t cd;
- objset_phys_t *osp = data;
- uint64_t intval;
- boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
- uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
- uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
-
- /* initialize HMAC mechanism */
- mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
- mech.cm_param = NULL;
- mech.cm_param_len = 0;
-
- cd.cd_format = CRYPTO_DATA_RAW;
- cd.cd_offset = 0;
-
- /* calculate the portable MAC from the portable fields and metadnode */
- ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- /* add in the os_type */
- intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type);
- cd.cd_length = sizeof (uint64_t);
- cd.cd_raw.iov_base = (char *)&intval;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_update(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- /* add in the portable os_flags */
- intval = osp->os_flags;
- if (should_bswap)
- intval = BSWAP_64(intval);
- intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
- if (!ZFS_HOST_BYTEORDER)
- intval = BSWAP_64(intval);
-
- cd.cd_length = sizeof (uint64_t);
- cd.cd_raw.iov_base = (char *)&intval;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_update(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- /* add in fields from the metadnode */
- ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
- should_bswap, &osp->os_meta_dnode);
- if (ret)
- goto error;
-
- /* store the final digest in a temporary buffer and copy what we need */
- cd.cd_length = SHA512_DIGEST_LENGTH;
- cd.cd_raw.iov_base = (char *)raw_portable_mac;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_final(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN);
-
- /*
- * The local MAC protects the user, group and project accounting.
- * If these objects are not present, the local MAC is zeroed out.
- */
- if ((datalen >= OBJSET_PHYS_SIZE_V3 &&
- osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
- osp->os_groupused_dnode.dn_type == DMU_OT_NONE &&
- osp->os_projectused_dnode.dn_type == DMU_OT_NONE) ||
- (datalen >= OBJSET_PHYS_SIZE_V2 &&
- osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
- osp->os_groupused_dnode.dn_type == DMU_OT_NONE) ||
- (datalen <= OBJSET_PHYS_SIZE_V1)) {
- bzero(local_mac, ZIO_OBJSET_MAC_LEN);
- return (0);
- }
-
- /* calculate the local MAC from the userused and groupused dnodes */
- ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- /* add in the non-portable os_flags */
- intval = osp->os_flags;
- if (should_bswap)
- intval = BSWAP_64(intval);
- intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
- if (!ZFS_HOST_BYTEORDER)
- intval = BSWAP_64(intval);
-
- cd.cd_length = sizeof (uint64_t);
- cd.cd_raw.iov_base = (char *)&intval;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_update(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- /* add in fields from the user accounting dnodes */
- if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) {
- ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
- should_bswap, &osp->os_userused_dnode);
- if (ret)
- goto error;
- }
-
- if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) {
- ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
- should_bswap, &osp->os_groupused_dnode);
- if (ret)
- goto error;
- }
-
- if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE &&
- datalen >= OBJSET_PHYS_SIZE_V3) {
- ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
- should_bswap, &osp->os_projectused_dnode);
- if (ret)
- goto error;
- }
-
- /* store the final digest in a temporary buffer and copy what we need */
- cd.cd_length = SHA512_DIGEST_LENGTH;
- cd.cd_raw.iov_base = (char *)raw_local_mac;
- cd.cd_raw.iov_len = cd.cd_length;
-
- ret = crypto_mac_final(ctx, &cd, NULL);
- if (ret != CRYPTO_SUCCESS) {
- ret = SET_ERROR(EIO);
- goto error;
- }
-
- bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN);
-
- return (0);
-
-error:
- bzero(portable_mac, ZIO_OBJSET_MAC_LEN);
- bzero(local_mac, ZIO_OBJSET_MAC_LEN);
- return (ret);
-}
-
-static void
-zio_crypt_destroy_uio(uio_t *uio)
-{
- if (uio->uio_iov)
- kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t));
-}
-
-/*
- * This function parses an uncompressed indirect block and returns a checksum
- * of all the portable fields from all of the contained bps. The portable
- * fields are the MAC and all of the fields from blk_prop except for the dedup,
- * checksum, and psize bits. For an explanation of the purpose of this, see
- * the comment block on object set authentication.
- */
-static int
-zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf,
- uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum)
-{
- blkptr_t *bp;
- int i, epb = datalen >> SPA_BLKPTRSHIFT;
- SHA2_CTX ctx;
- uint8_t digestbuf[SHA512_DIGEST_LENGTH];
-
- /* checksum all of the MACs from the layer below */
- SHA2Init(SHA512, &ctx);
- for (i = 0, bp = buf; i < epb; i++, bp++) {
- zio_crypt_bp_do_indrect_checksum_updates(&ctx, version,
- byteswap, bp);
- }
- SHA2Final(digestbuf, &ctx);
-
- if (generate) {
- bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN);
- return (0);
- }
-
- if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0)
- return (SET_ERROR(ECKSUM));
-
- return (0);
-}
-
-int
-zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
- uint_t datalen, boolean_t byteswap, uint8_t *cksum)
-{
- int ret;
-
- /*
- * Unfortunately, callers of this function will not always have
- * easy access to the on-disk format version. This info is
- * normally found in the DSL Crypto Key, but the checksum-of-MACs
- * is expected to be verifiable even when the key isn't loaded.
- * Here, instead of doing a ZAP lookup for the version for each
- * zio, we simply try both existing formats.
- */
- ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf,
- datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum);
- if (ret == ECKSUM) {
- ASSERT(!generate);
- ret = zio_crypt_do_indirect_mac_checksum_impl(generate,
- buf, datalen, 0, byteswap, cksum);
- }
-
- return (ret);
-}
-
-int
-zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
- uint_t datalen, boolean_t byteswap, uint8_t *cksum)
-{
- int ret;
- void *buf;
-
- buf = abd_borrow_buf_copy(abd, datalen);
- ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen,
- byteswap, cksum);
- abd_return_buf(abd, buf, datalen);
-
- return (ret);
-}
-
-/*
- * Special case handling routine for encrypting / decrypting ZIL blocks.
- * We do not check for the older ZIL chain because the encryption feature
- * was not available before the newer ZIL chain was introduced. The goal
- * here is to encrypt everything except the blkptr_t of a lr_write_t and
- * the zil_chain_t header. Everything that is not encrypted is authenticated.
- */
-static int
-zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
- uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio,
- uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len,
- boolean_t *no_crypt)
-{
- int ret;
- uint64_t txtype, lr_len;
- uint_t nr_src, nr_dst, crypt_len;
- uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
- iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
- uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
- zil_chain_t *zilc;
- lr_t *lr;
- uint8_t *aadbuf = zio_buf_alloc(datalen);
-
- /* cipherbuf always needs an extra iovec for the MAC */
- if (encrypt) {
- src = plainbuf;
- dst = cipherbuf;
- nr_src = 0;
- nr_dst = 1;
- } else {
- src = cipherbuf;
- dst = plainbuf;
- nr_src = 1;
- nr_dst = 0;
- }
-
- /* find the start and end record of the log block */
- zilc = (zil_chain_t *)src;
- slrp = src + sizeof (zil_chain_t);
- aadp = aadbuf;
- blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused);
-
- /* calculate the number of encrypted iovecs we will need */
- for (; slrp < blkend; slrp += lr_len) {
- lr = (lr_t *)slrp;
-
- if (!byteswap) {
- txtype = lr->lrc_txtype;
- lr_len = lr->lrc_reclen;
- } else {
- txtype = BSWAP_64(lr->lrc_txtype);
- lr_len = BSWAP_64(lr->lrc_reclen);
- }
-
- nr_iovecs++;
- if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t))
- nr_iovecs++;
- }
-
- nr_src += nr_iovecs;
- nr_dst += nr_iovecs;
-
- /* allocate the iovec arrays */
- if (nr_src != 0) {
- src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
- if (src_iovecs == NULL) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
- }
-
- if (nr_dst != 0) {
- dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
- if (dst_iovecs == NULL) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
- }
-
- /*
- * Copy the plain zil header over and authenticate everything except
- * the checksum that will store our MAC. If we are writing the data
- * the embedded checksum will not have been calculated yet, so we don't
- * authenticate that.
- */
- bcopy(src, dst, sizeof (zil_chain_t));
- bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t));
- aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t);
- aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t);
-
- /* loop over records again, filling in iovecs */
- nr_iovecs = 0;
- slrp = src + sizeof (zil_chain_t);
- dlrp = dst + sizeof (zil_chain_t);
-
- for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) {
- lr = (lr_t *)slrp;
-
- if (!byteswap) {
- txtype = lr->lrc_txtype;
- lr_len = lr->lrc_reclen;
- } else {
- txtype = BSWAP_64(lr->lrc_txtype);
- lr_len = BSWAP_64(lr->lrc_reclen);
- }
-
- /* copy the common lr_t */
- bcopy(slrp, dlrp, sizeof (lr_t));
- bcopy(slrp, aadp, sizeof (lr_t));
- aadp += sizeof (lr_t);
- aad_len += sizeof (lr_t);
-
- ASSERT3P(src_iovecs, !=, NULL);
- ASSERT3P(dst_iovecs, !=, NULL);
-
- /*
- * If this is a TX_WRITE record we want to encrypt everything
- * except the bp if exists. If the bp does exist we want to
- * authenticate it.
- */
- if (txtype == TX_WRITE) {
- crypt_len = sizeof (lr_write_t) -
- sizeof (lr_t) - sizeof (blkptr_t);
- src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t);
- src_iovecs[nr_iovecs].iov_len = crypt_len;
- dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t);
- dst_iovecs[nr_iovecs].iov_len = crypt_len;
-
- /* copy the bp now since it will not be encrypted */
- bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
- dlrp + sizeof (lr_write_t) - sizeof (blkptr_t),
- sizeof (blkptr_t));
- bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
- aadp, sizeof (blkptr_t));
- aadp += sizeof (blkptr_t);
- aad_len += sizeof (blkptr_t);
- nr_iovecs++;
- total_len += crypt_len;
-
- if (lr_len != sizeof (lr_write_t)) {
- crypt_len = lr_len - sizeof (lr_write_t);
- src_iovecs[nr_iovecs].iov_base =
- slrp + sizeof (lr_write_t);
- src_iovecs[nr_iovecs].iov_len = crypt_len;
- dst_iovecs[nr_iovecs].iov_base =
- dlrp + sizeof (lr_write_t);
- dst_iovecs[nr_iovecs].iov_len = crypt_len;
- nr_iovecs++;
- total_len += crypt_len;
- }
- } else {
- crypt_len = lr_len - sizeof (lr_t);
- src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t);
- src_iovecs[nr_iovecs].iov_len = crypt_len;
- dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t);
- dst_iovecs[nr_iovecs].iov_len = crypt_len;
- nr_iovecs++;
- total_len += crypt_len;
- }
- }
-
- *no_crypt = (nr_iovecs == 0);
- *enc_len = total_len;
- *authbuf = aadbuf;
- *auth_len = aad_len;
-
- if (encrypt) {
- puio->uio_iov = src_iovecs;
- puio->uio_iovcnt = nr_src;
- cuio->uio_iov = dst_iovecs;
- cuio->uio_iovcnt = nr_dst;
- } else {
- puio->uio_iov = dst_iovecs;
- puio->uio_iovcnt = nr_dst;
- cuio->uio_iov = src_iovecs;
- cuio->uio_iovcnt = nr_src;
- }
-
- return (0);
-
-error:
- zio_buf_free(aadbuf, datalen);
- if (src_iovecs != NULL)
- kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
- if (dst_iovecs != NULL)
- kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
-
- *enc_len = 0;
- *authbuf = NULL;
- *auth_len = 0;
- *no_crypt = B_FALSE;
- puio->uio_iov = NULL;
- puio->uio_iovcnt = 0;
- cuio->uio_iov = NULL;
- cuio->uio_iovcnt = 0;
- return (ret);
-}
-
-/*
- * Special case handling routine for encrypting / decrypting dnode blocks.
- */
-static int
-zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version,
- uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
- uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
- uint_t *auth_len, boolean_t *no_crypt)
-{
- int ret;
- uint_t nr_src, nr_dst, crypt_len;
- uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
- uint_t i, j, max_dnp = datalen >> DNODE_SHIFT;
- iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
- uint8_t *src, *dst, *aadp;
- dnode_phys_t *dnp, *adnp, *sdnp, *ddnp;
- uint8_t *aadbuf = zio_buf_alloc(datalen);
-
- if (encrypt) {
- src = plainbuf;
- dst = cipherbuf;
- nr_src = 0;
- nr_dst = 1;
- } else {
- src = cipherbuf;
- dst = plainbuf;
- nr_src = 1;
- nr_dst = 0;
- }
-
- sdnp = (dnode_phys_t *)src;
- ddnp = (dnode_phys_t *)dst;
- aadp = aadbuf;
-
- /*
- * Count the number of iovecs we will need to do the encryption by
- * counting the number of bonus buffers that need to be encrypted.
- */
- for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
- /*
- * This block may still be byteswapped. However, all of the
- * values we use are either uint8_t's (for which byteswapping
- * is a noop) or a * != 0 check, which will work regardless
- * of whether or not we byteswap.
- */
- if (sdnp[i].dn_type != DMU_OT_NONE &&
- DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) &&
- sdnp[i].dn_bonuslen != 0) {
- nr_iovecs++;
- }
- }
-
- nr_src += nr_iovecs;
- nr_dst += nr_iovecs;
-
- if (nr_src != 0) {
- src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
- if (src_iovecs == NULL) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
- }
-
- if (nr_dst != 0) {
- dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
- if (dst_iovecs == NULL) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
- }
-
- nr_iovecs = 0;
-
- /*
- * Iterate through the dnodes again, this time filling in the uios
- * we allocated earlier. We also concatenate any data we want to
- * authenticate onto aadbuf.
- */
- for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
- dnp = &sdnp[i];
-
- /* copy over the core fields and blkptrs (kept as plaintext) */
- bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp);
-
- if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
- bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]),
- sizeof (blkptr_t));
- }
-
- /*
- * Handle authenticated data. We authenticate everything in
- * the dnode that can be brought over when we do a raw send.
- * This includes all of the core fields as well as the MACs
- * stored in the bp checksums and all of the portable bits
- * from blk_prop. We include the dnode padding here in case it
- * ever gets used in the future. Some dn_flags and dn_used are
- * not portable so we mask those out values out of the
- * authenticated data.
- */
- crypt_len = offsetof(dnode_phys_t, dn_blkptr);
- bcopy(dnp, aadp, crypt_len);
- adnp = (dnode_phys_t *)aadp;
- adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
- adnp->dn_used = 0;
- aadp += crypt_len;
- aad_len += crypt_len;
-
- for (j = 0; j < dnp->dn_nblkptr; j++) {
- zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
- version, byteswap, &dnp->dn_blkptr[j]);
- }
-
- if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
- zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
- version, byteswap, DN_SPILL_BLKPTR(dnp));
- }
-
- /*
- * If this bonus buffer needs to be encrypted, we prepare an
- * iovec_t. The encryption / decryption functions will fill
- * this in for us with the encrypted or decrypted data.
- * Otherwise we add the bonus buffer to the authenticated
- * data buffer and copy it over to the destination. The
- * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
- * we can guarantee alignment with the AES block size
- * (128 bits).
- */
- crypt_len = DN_MAX_BONUS_LEN(dnp);
- if (dnp->dn_type != DMU_OT_NONE &&
- DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
- dnp->dn_bonuslen != 0) {
- ASSERT3U(nr_iovecs, <, nr_src);
- ASSERT3U(nr_iovecs, <, nr_dst);
- ASSERT3P(src_iovecs, !=, NULL);
- ASSERT3P(dst_iovecs, !=, NULL);
- src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
- src_iovecs[nr_iovecs].iov_len = crypt_len;
- dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
- dst_iovecs[nr_iovecs].iov_len = crypt_len;
-
- nr_iovecs++;
- total_len += crypt_len;
- } else {
- bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len);
- bcopy(DN_BONUS(dnp), aadp, crypt_len);
- aadp += crypt_len;
- aad_len += crypt_len;
- }
- }
-
- *no_crypt = (nr_iovecs == 0);
- *enc_len = total_len;
- *authbuf = aadbuf;
- *auth_len = aad_len;
-
- if (encrypt) {
- puio->uio_iov = src_iovecs;
- puio->uio_iovcnt = nr_src;
- cuio->uio_iov = dst_iovecs;
- cuio->uio_iovcnt = nr_dst;
- } else {
- puio->uio_iov = dst_iovecs;
- puio->uio_iovcnt = nr_dst;
- cuio->uio_iov = src_iovecs;
- cuio->uio_iovcnt = nr_src;
- }
-
- return (0);
-
-error:
- zio_buf_free(aadbuf, datalen);
- if (src_iovecs != NULL)
- kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
- if (dst_iovecs != NULL)
- kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
-
- *enc_len = 0;
- *authbuf = NULL;
- *auth_len = 0;
- *no_crypt = B_FALSE;
- puio->uio_iov = NULL;
- puio->uio_iovcnt = 0;
- cuio->uio_iov = NULL;
- cuio->uio_iovcnt = 0;
- return (ret);
-}
-
-static int
-zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf,
- uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio,
- uint_t *enc_len)
-{
- int ret;
- uint_t nr_plain = 1, nr_cipher = 2;
- iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL;
-
- /* allocate the iovecs for the plain and cipher data */
- plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t),
- KM_SLEEP);
- if (!plain_iovecs) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
-
- cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t),
- KM_SLEEP);
- if (!cipher_iovecs) {
- ret = SET_ERROR(ENOMEM);
- goto error;
- }
-
- plain_iovecs[0].iov_base = plainbuf;
- plain_iovecs[0].iov_len = datalen;
- cipher_iovecs[0].iov_base = cipherbuf;
- cipher_iovecs[0].iov_len = datalen;
-
- *enc_len = datalen;
- puio->uio_iov = plain_iovecs;
- puio->uio_iovcnt = nr_plain;
- cuio->uio_iov = cipher_iovecs;
- cuio->uio_iovcnt = nr_cipher;
-
- return (0);
-
-error:
- if (plain_iovecs != NULL)
- kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t));
- if (cipher_iovecs != NULL)
- kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t));
-
- *enc_len = 0;
- puio->uio_iov = NULL;
- puio->uio_iovcnt = 0;
- cuio->uio_iov = NULL;
- cuio->uio_iovcnt = 0;
- return (ret);
-}
-
-/*
- * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
- * that they can be used for encryption and decryption by zio_do_crypt_uio().
- * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
- * requiring special handling to parse out pieces that are to be encrypted. The
- * authbuf is used by these special cases to store additional authenticated
- * data (AAD) for the encryption modes.
- */
-static int
-zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot,
- uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
- uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
- uint_t *auth_len, boolean_t *no_crypt)
-{
- int ret;
- iovec_t *mac_iov;
-
- ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE);
-
- /* route to handler */
- switch (ot) {
- case DMU_OT_INTENT_LOG:
- ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf,
- datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len,
- no_crypt);
- break;
- case DMU_OT_DNODE:
- ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf,
- cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf,
- auth_len, no_crypt);
- break;
- default:
- ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf,
- datalen, puio, cuio, enc_len);
- *authbuf = NULL;
- *auth_len = 0;
- *no_crypt = B_FALSE;
- break;
- }
-
- if (ret != 0)
- goto error;
-
- /* populate the uios */
- puio->uio_segflg = UIO_SYSSPACE;
- cuio->uio_segflg = UIO_SYSSPACE;
-
- mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]);
- mac_iov->iov_base = mac;
- mac_iov->iov_len = ZIO_DATA_MAC_LEN;
-
- return (0);
-
-error:
- return (ret);
-}
-
-/*
- * Primary encryption / decryption entrypoint for zio data.
- */
-int
-zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
- dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
- uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
- boolean_t *no_crypt)
-{
- int ret;
- boolean_t locked = B_FALSE;
- uint64_t crypt = key->zk_crypt;
- uint_t keydata_len = zio_crypt_table[crypt].ci_keylen;
- uint_t enc_len, auth_len;
- uio_t puio, cuio;
- uint8_t enc_keydata[MASTER_KEY_MAX_LEN];
- crypto_key_t tmp_ckey, *ckey = NULL;
- crypto_ctx_template_t tmpl;
- uint8_t *authbuf = NULL;
-
- /*
- * If the needed key is the current one, just use it. Otherwise we
- * need to generate a temporary one from the given salt + master key.
- * If we are encrypting, we must return a copy of the current salt
- * so that it can be stored in the blkptr_t.
- */
- rw_enter(&key->zk_salt_lock, RW_READER);
- locked = B_TRUE;
-
- if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) {
- ckey = &key->zk_current_key;
- tmpl = key->zk_current_tmpl;
- } else {
- rw_exit(&key->zk_salt_lock);
- locked = B_FALSE;
-
- ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
- salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len);
- if (ret != 0)
- goto error;
-
- tmp_ckey.ck_format = CRYPTO_KEY_RAW;
- tmp_ckey.ck_data = enc_keydata;
- tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
-
- ckey = &tmp_ckey;
- tmpl = NULL;
- }
-
- /*
- * Attempt to use QAT acceleration if we can. We currently don't
- * do this for metadnode and ZIL blocks, since they have a much
- * more involved buffer layout and the qat_crypt() function only
- * works in-place.
- */
- if (qat_crypt_use_accel(datalen) &&
- ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) {
- uint8_t *srcbuf, *dstbuf;
-
- if (encrypt) {
- srcbuf = plainbuf;
- dstbuf = cipherbuf;
- } else {
- srcbuf = cipherbuf;
- dstbuf = plainbuf;
- }
-
- ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf,
- dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen);
- if (ret == CPA_STATUS_SUCCESS) {
- if (locked) {
- rw_exit(&key->zk_salt_lock);
- locked = B_FALSE;
- }
-
- return (0);
- }
- /* If the hardware implementation fails fall back to software */
- }
-
- bzero(&puio, sizeof (uio_t));
- bzero(&cuio, sizeof (uio_t));
-
- /* create uios for encryption */
- ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf,
- cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len,
- &authbuf, &auth_len, no_crypt);
- if (ret != 0)
- goto error;
-
- /* perform the encryption / decryption in software */
- ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
- &puio, &cuio, authbuf, auth_len);
- if (ret != 0)
- goto error;
-
- if (locked) {
- rw_exit(&key->zk_salt_lock);
- locked = B_FALSE;
- }
-
- if (authbuf != NULL)
- zio_buf_free(authbuf, datalen);
- if (ckey == &tmp_ckey)
- bzero(enc_keydata, keydata_len);
- zio_crypt_destroy_uio(&puio);
- zio_crypt_destroy_uio(&cuio);
-
- return (0);
-
-error:
- if (locked)
- rw_exit(&key->zk_salt_lock);
- if (authbuf != NULL)
- zio_buf_free(authbuf, datalen);
- if (ckey == &tmp_ckey)
- bzero(enc_keydata, keydata_len);
- zio_crypt_destroy_uio(&puio);
- zio_crypt_destroy_uio(&cuio);
-
- return (ret);
-}
-
-/*
- * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
- * linear buffers.
- */
-int
-zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
- boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
- uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
-{
- int ret;
- void *ptmp, *ctmp;
-
- if (encrypt) {
- ptmp = abd_borrow_buf_copy(pabd, datalen);
- ctmp = abd_borrow_buf(cabd, datalen);
- } else {
- ptmp = abd_borrow_buf(pabd, datalen);
- ctmp = abd_borrow_buf_copy(cabd, datalen);
- }
-
- ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
- datalen, ptmp, ctmp, no_crypt);
- if (ret != 0)
- goto error;
-
- if (encrypt) {
- abd_return_buf(pabd, ptmp, datalen);
- abd_return_buf_copy(cabd, ctmp, datalen);
- } else {
- abd_return_buf_copy(pabd, ptmp, datalen);
- abd_return_buf(cabd, ctmp, datalen);
- }
-
- return (0);
-
-error:
- if (encrypt) {
- abd_return_buf(pabd, ptmp, datalen);
- abd_return_buf_copy(cabd, ctmp, datalen);
- } else {
- abd_return_buf_copy(pabd, ptmp, datalen);
- abd_return_buf(cabd, ctmp, datalen);
- }
-
- return (ret);
-}
-
-#if defined(_KERNEL)
-/* BEGIN CSTYLED */
-module_param(zfs_key_max_salt_uses, ulong, 0644);
-MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value "
- "can be used for generating encryption keys before it is rotated");
-/* END CSTYLED */
-#endif
diff --git a/module/zfs/zpl_ctldir.c b/module/zfs/zpl_ctldir.c
deleted file mode 100644
index 6df367b81..000000000
--- a/module/zfs/zpl_ctldir.c
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
- * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- * LLNL-CODE-403049.
- * Rewritten for Linux by:
- * Rohan Puri <[email protected]>
- * Brian Behlendorf <[email protected]>
- */
-
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zpl.h>
-
-/*
- * Common open routine. Disallow any write access.
- */
-/* ARGSUSED */
-static int
-zpl_common_open(struct inode *ip, struct file *filp)
-{
- if (filp->f_mode & FMODE_WRITE)
- return (-EACCES);
-
- return (generic_file_open(ip, filp));
-}
-
-/*
- * Get root directory contents.
- */
-static int
-zpl_root_iterate(struct file *filp, zpl_dir_context_t *ctx)
-{
- zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
- int error = 0;
-
- ZFS_ENTER(zfsvfs);
-
- if (!zpl_dir_emit_dots(filp, ctx))
- goto out;
-
- if (ctx->pos == 2) {
- if (!zpl_dir_emit(ctx, ZFS_SNAPDIR_NAME,
- strlen(ZFS_SNAPDIR_NAME), ZFSCTL_INO_SNAPDIR, DT_DIR))
- goto out;
-
- ctx->pos++;
- }
-
- if (ctx->pos == 3) {
- if (!zpl_dir_emit(ctx, ZFS_SHAREDIR_NAME,
- strlen(ZFS_SHAREDIR_NAME), ZFSCTL_INO_SHARES, DT_DIR))
- goto out;
-
- ctx->pos++;
- }
-out:
- ZFS_EXIT(zfsvfs);
-
- return (error);
-}
-
-#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
-static int
-zpl_root_readdir(struct file *filp, void *dirent, filldir_t filldir)
-{
- zpl_dir_context_t ctx =
- ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
- int error;
-
- error = zpl_root_iterate(filp, &ctx);
- filp->f_pos = ctx.pos;
-
- return (error);
-}
-#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */
-
-/*
- * Get root directory attributes.
- */
-/* ARGSUSED */
-static int
-zpl_root_getattr_impl(const struct path *path, struct kstat *stat,
- u32 request_mask, unsigned int query_flags)
-{
- struct inode *ip = path->dentry->d_inode;
-
- generic_fillattr(ip, stat);
- stat->atime = current_time(ip);
-
- return (0);
-}
-ZPL_GETATTR_WRAPPER(zpl_root_getattr);
-
-static struct dentry *
-#ifdef HAVE_LOOKUP_NAMEIDATA
-zpl_root_lookup(struct inode *dip, struct dentry *dentry, struct nameidata *nd)
-#else
-zpl_root_lookup(struct inode *dip, struct dentry *dentry, unsigned int flags)
-#endif
-{
- cred_t *cr = CRED();
- struct inode *ip;
- int error;
-
- crhold(cr);
- error = -zfsctl_root_lookup(dip, dname(dentry), &ip, 0, cr, NULL, NULL);
- ASSERT3S(error, <=, 0);
- crfree(cr);
-
- if (error) {
- if (error == -ENOENT)
- return (d_splice_alias(NULL, dentry));
- else
- return (ERR_PTR(error));
- }
-
- return (d_splice_alias(ip, dentry));
-}
-
-/*
- * The '.zfs' control directory file and inode operations.
- */
-const struct file_operations zpl_fops_root = {
- .open = zpl_common_open,
- .llseek = generic_file_llseek,
- .read = generic_read_dir,
-#ifdef HAVE_VFS_ITERATE_SHARED
- .iterate_shared = zpl_root_iterate,
-#elif defined(HAVE_VFS_ITERATE)
- .iterate = zpl_root_iterate,
-#else
- .readdir = zpl_root_readdir,
-#endif
-};
-
-const struct inode_operations zpl_ops_root = {
- .lookup = zpl_root_lookup,
- .getattr = zpl_root_getattr,
-};
-
-#ifdef HAVE_AUTOMOUNT
-static struct vfsmount *
-zpl_snapdir_automount(struct path *path)
-{
- int error;
-
- error = -zfsctl_snapshot_mount(path, 0);
- if (error)
- return (ERR_PTR(error));
-
- /*
- * Rather than returning the new vfsmount for the snapshot we must
- * return NULL to indicate a mount collision. This is done because
- * the user space mount calls do_add_mount() which adds the vfsmount
- * to the name space. If we returned the new mount here it would be
- * added again to the vfsmount list resulting in list corruption.
- */
- return (NULL);
-}
-#endif /* HAVE_AUTOMOUNT */
-
-/*
- * Negative dentries must always be revalidated so newly created snapshots
- * can be detected and automounted. Normal dentries should be kept because
- * as of the 3.18 kernel revaliding the mountpoint dentry will result in
- * the snapshot being immediately unmounted.
- */
-static int
-#ifdef HAVE_D_REVALIDATE_NAMEIDATA
-zpl_snapdir_revalidate(struct dentry *dentry, struct nameidata *i)
-#else
-zpl_snapdir_revalidate(struct dentry *dentry, unsigned int flags)
-#endif
-{
- return (!!dentry->d_inode);
-}
-
-dentry_operations_t zpl_dops_snapdirs = {
-/*
- * Auto mounting of snapshots is only supported for 2.6.37 and
- * newer kernels. Prior to this kernel the ops->follow_link()
- * callback was used as a hack to trigger the mount. The
- * resulting vfsmount was then explicitly grafted in to the
- * name space. While it might be possible to add compatibility
- * code to accomplish this it would require considerable care.
- */
-#ifdef HAVE_AUTOMOUNT
- .d_automount = zpl_snapdir_automount,
-#endif /* HAVE_AUTOMOUNT */
- .d_revalidate = zpl_snapdir_revalidate,
-};
-
-static struct dentry *
-#ifdef HAVE_LOOKUP_NAMEIDATA
-zpl_snapdir_lookup(struct inode *dip, struct dentry *dentry,
- struct nameidata *nd)
-#else
-zpl_snapdir_lookup(struct inode *dip, struct dentry *dentry,
- unsigned int flags)
-#endif
-
-{
- fstrans_cookie_t cookie;
- cred_t *cr = CRED();
- struct inode *ip = NULL;
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfsctl_snapdir_lookup(dip, dname(dentry), &ip,
- 0, cr, NULL, NULL);
- ASSERT3S(error, <=, 0);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- if (error && error != -ENOENT)
- return (ERR_PTR(error));
-
- ASSERT(error == 0 || ip == NULL);
- d_clear_d_op(dentry);
- d_set_d_op(dentry, &zpl_dops_snapdirs);
-#ifdef HAVE_AUTOMOUNT
- dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
-#endif
-
- return (d_splice_alias(ip, dentry));
-}
-
-static int
-zpl_snapdir_iterate(struct file *filp, zpl_dir_context_t *ctx)
-{
- zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
- fstrans_cookie_t cookie;
- char snapname[MAXNAMELEN];
- boolean_t case_conflict;
- uint64_t id, pos;
- int error = 0;
-
- ZFS_ENTER(zfsvfs);
- cookie = spl_fstrans_mark();
-
- if (!zpl_dir_emit_dots(filp, ctx))
- goto out;
-
- pos = ctx->pos;
- while (error == 0) {
- dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG);
- error = -dmu_snapshot_list_next(zfsvfs->z_os, MAXNAMELEN,
- snapname, &id, &pos, &case_conflict);
- dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG);
- if (error)
- goto out;
-
- if (!zpl_dir_emit(ctx, snapname, strlen(snapname),
- ZFSCTL_INO_SHARES - id, DT_DIR))
- goto out;
-
- ctx->pos = pos;
- }
-out:
- spl_fstrans_unmark(cookie);
- ZFS_EXIT(zfsvfs);
-
- if (error == -ENOENT)
- return (0);
-
- return (error);
-}
-
-#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
-static int
-zpl_snapdir_readdir(struct file *filp, void *dirent, filldir_t filldir)
-{
- zpl_dir_context_t ctx =
- ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
- int error;
-
- error = zpl_snapdir_iterate(filp, &ctx);
- filp->f_pos = ctx.pos;
-
- return (error);
-}
-#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */
-
-static int
-zpl_snapdir_rename2(struct inode *sdip, struct dentry *sdentry,
- struct inode *tdip, struct dentry *tdentry, unsigned int flags)
-{
- cred_t *cr = CRED();
- int error;
-
- /* We probably don't want to support renameat2(2) in ctldir */
- if (flags)
- return (-EINVAL);
-
- crhold(cr);
- error = -zfsctl_snapdir_rename(sdip, dname(sdentry),
- tdip, dname(tdentry), cr, 0);
- ASSERT3S(error, <=, 0);
- crfree(cr);
-
- return (error);
-}
-
-#ifndef HAVE_RENAME_WANTS_FLAGS
-static int
-zpl_snapdir_rename(struct inode *sdip, struct dentry *sdentry,
- struct inode *tdip, struct dentry *tdentry)
-{
- return (zpl_snapdir_rename2(sdip, sdentry, tdip, tdentry, 0));
-}
-#endif
-
-static int
-zpl_snapdir_rmdir(struct inode *dip, struct dentry *dentry)
-{
- cred_t *cr = CRED();
- int error;
-
- crhold(cr);
- error = -zfsctl_snapdir_remove(dip, dname(dentry), cr, 0);
- ASSERT3S(error, <=, 0);
- crfree(cr);
-
- return (error);
-}
-
-static int
-zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, zpl_umode_t mode)
-{
- cred_t *cr = CRED();
- vattr_t *vap;
- struct inode *ip;
- int error;
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dip, mode | S_IFDIR, cr);
-
- error = -zfsctl_snapdir_mkdir(dip, dname(dentry), vap, &ip, cr, 0);
- if (error == 0) {
- d_clear_d_op(dentry);
- d_set_d_op(dentry, &zpl_dops_snapdirs);
- d_instantiate(dentry, ip);
- }
-
- kmem_free(vap, sizeof (vattr_t));
- ASSERT3S(error, <=, 0);
- crfree(cr);
-
- return (error);
-}
-
-/*
- * Get snapshot directory attributes.
- */
-/* ARGSUSED */
-static int
-zpl_snapdir_getattr_impl(const struct path *path, struct kstat *stat,
- u32 request_mask, unsigned int query_flags)
-{
- struct inode *ip = path->dentry->d_inode;
- zfsvfs_t *zfsvfs = ITOZSB(ip);
-
- ZFS_ENTER(zfsvfs);
- generic_fillattr(ip, stat);
-
- stat->nlink = stat->size = 2;
- stat->ctime = stat->mtime = dmu_objset_snap_cmtime(zfsvfs->z_os);
- stat->atime = current_time(ip);
- ZFS_EXIT(zfsvfs);
-
- return (0);
-}
-ZPL_GETATTR_WRAPPER(zpl_snapdir_getattr);
-
-/*
- * The '.zfs/snapshot' directory file operations. These mainly control
- * generating the list of available snapshots when doing an 'ls' in the
- * directory. See zpl_snapdir_readdir().
- */
-const struct file_operations zpl_fops_snapdir = {
- .open = zpl_common_open,
- .llseek = generic_file_llseek,
- .read = generic_read_dir,
-#ifdef HAVE_VFS_ITERATE_SHARED
- .iterate_shared = zpl_snapdir_iterate,
-#elif defined(HAVE_VFS_ITERATE)
- .iterate = zpl_snapdir_iterate,
-#else
- .readdir = zpl_snapdir_readdir,
-#endif
-
-};
-
-/*
- * The '.zfs/snapshot' directory inode operations. These mainly control
- * creating an inode for a snapshot directory and initializing the needed
- * infrastructure to automount the snapshot. See zpl_snapdir_lookup().
- */
-const struct inode_operations zpl_ops_snapdir = {
- .lookup = zpl_snapdir_lookup,
- .getattr = zpl_snapdir_getattr,
-#ifdef HAVE_RENAME_WANTS_FLAGS
- .rename = zpl_snapdir_rename2,
-#else
- .rename = zpl_snapdir_rename,
-#endif
- .rmdir = zpl_snapdir_rmdir,
- .mkdir = zpl_snapdir_mkdir,
-};
-
-static struct dentry *
-#ifdef HAVE_LOOKUP_NAMEIDATA
-zpl_shares_lookup(struct inode *dip, struct dentry *dentry,
- struct nameidata *nd)
-#else
-zpl_shares_lookup(struct inode *dip, struct dentry *dentry,
- unsigned int flags)
-#endif
-{
- fstrans_cookie_t cookie;
- cred_t *cr = CRED();
- struct inode *ip = NULL;
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfsctl_shares_lookup(dip, dname(dentry), &ip,
- 0, cr, NULL, NULL);
- ASSERT3S(error, <=, 0);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- if (error) {
- if (error == -ENOENT)
- return (d_splice_alias(NULL, dentry));
- else
- return (ERR_PTR(error));
- }
-
- return (d_splice_alias(ip, dentry));
-}
-
-static int
-zpl_shares_iterate(struct file *filp, zpl_dir_context_t *ctx)
-{
- fstrans_cookie_t cookie;
- cred_t *cr = CRED();
- zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
- znode_t *dzp;
- int error = 0;
-
- ZFS_ENTER(zfsvfs);
- cookie = spl_fstrans_mark();
-
- if (zfsvfs->z_shares_dir == 0) {
- zpl_dir_emit_dots(filp, ctx);
- goto out;
- }
-
- error = -zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp);
- if (error)
- goto out;
-
- crhold(cr);
- error = -zfs_readdir(ZTOI(dzp), ctx, cr);
- crfree(cr);
-
- iput(ZTOI(dzp));
-out:
- spl_fstrans_unmark(cookie);
- ZFS_EXIT(zfsvfs);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
-static int
-zpl_shares_readdir(struct file *filp, void *dirent, filldir_t filldir)
-{
- zpl_dir_context_t ctx =
- ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
- int error;
-
- error = zpl_shares_iterate(filp, &ctx);
- filp->f_pos = ctx.pos;
-
- return (error);
-}
-#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */
-
-/* ARGSUSED */
-static int
-zpl_shares_getattr_impl(const struct path *path, struct kstat *stat,
- u32 request_mask, unsigned int query_flags)
-{
- struct inode *ip = path->dentry->d_inode;
- zfsvfs_t *zfsvfs = ITOZSB(ip);
- znode_t *dzp;
- int error;
-
- ZFS_ENTER(zfsvfs);
-
- if (zfsvfs->z_shares_dir == 0) {
- generic_fillattr(path->dentry->d_inode, stat);
- stat->nlink = stat->size = 2;
- stat->atime = current_time(ip);
- ZFS_EXIT(zfsvfs);
- return (0);
- }
-
- error = -zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp);
- if (error == 0) {
- error = -zfs_getattr_fast(ZTOI(dzp), stat);
- iput(ZTOI(dzp));
- }
-
- ZFS_EXIT(zfsvfs);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-ZPL_GETATTR_WRAPPER(zpl_shares_getattr);
-
-/*
- * The '.zfs/shares' directory file operations.
- */
-const struct file_operations zpl_fops_shares = {
- .open = zpl_common_open,
- .llseek = generic_file_llseek,
- .read = generic_read_dir,
-#ifdef HAVE_VFS_ITERATE_SHARED
- .iterate_shared = zpl_shares_iterate,
-#elif defined(HAVE_VFS_ITERATE)
- .iterate = zpl_shares_iterate,
-#else
- .readdir = zpl_shares_readdir,
-#endif
-
-};
-
-/*
- * The '.zfs/shares' directory inode operations.
- */
-const struct inode_operations zpl_ops_shares = {
- .lookup = zpl_shares_lookup,
- .getattr = zpl_shares_getattr,
-};
diff --git a/module/zfs/zpl_export.c b/module/zfs/zpl_export.c
deleted file mode 100644
index a264d664c..000000000
--- a/module/zfs/zpl_export.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2011 Gunnar Beutner
- * Copyright (c) 2012 Cyril Plisko. All rights reserved.
- */
-
-
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zpl.h>
-
-
-static int
-#ifdef HAVE_ENCODE_FH_WITH_INODE
-zpl_encode_fh(struct inode *ip, __u32 *fh, int *max_len, struct inode *parent)
-{
-#else
-zpl_encode_fh(struct dentry *dentry, __u32 *fh, int *max_len, int connectable)
-{
- /* CSTYLED */
- struct inode *ip = dentry->d_inode;
-#endif /* HAVE_ENCODE_FH_WITH_INODE */
- fstrans_cookie_t cookie;
- fid_t *fid = (fid_t *)fh;
- int len_bytes, rc;
-
- len_bytes = *max_len * sizeof (__u32);
-
- if (len_bytes < offsetof(fid_t, fid_data))
- return (255);
-
- fid->fid_len = len_bytes - offsetof(fid_t, fid_data);
- cookie = spl_fstrans_mark();
-
- if (zfsctl_is_node(ip))
- rc = zfsctl_fid(ip, fid);
- else
- rc = zfs_fid(ip, fid);
-
- spl_fstrans_unmark(cookie);
- len_bytes = offsetof(fid_t, fid_data) + fid->fid_len;
- *max_len = roundup(len_bytes, sizeof (__u32)) / sizeof (__u32);
-
- return (rc == 0 ? FILEID_INO32_GEN : 255);
-}
-
-static struct dentry *
-zpl_dentry_obtain_alias(struct inode *ip)
-{
- struct dentry *result;
-
-#ifdef HAVE_D_OBTAIN_ALIAS
- result = d_obtain_alias(ip);
-#else
- result = d_alloc_anon(ip);
-
- if (result == NULL) {
- iput(ip);
- result = ERR_PTR(-ENOMEM);
- }
-#endif /* HAVE_D_OBTAIN_ALIAS */
-
- return (result);
-}
-
-static struct dentry *
-zpl_fh_to_dentry(struct super_block *sb, struct fid *fh,
- int fh_len, int fh_type)
-{
- fid_t *fid = (fid_t *)fh;
- fstrans_cookie_t cookie;
- struct inode *ip;
- int len_bytes, rc;
-
- len_bytes = fh_len * sizeof (__u32);
-
- if (fh_type != FILEID_INO32_GEN ||
- len_bytes < offsetof(fid_t, fid_data) ||
- len_bytes < offsetof(fid_t, fid_data) + fid->fid_len)
- return (ERR_PTR(-EINVAL));
-
- cookie = spl_fstrans_mark();
- rc = zfs_vget(sb, &ip, fid);
- spl_fstrans_unmark(cookie);
-
- if (rc) {
- /*
- * If we see ENOENT it might mean that an NFSv4 * client
- * is using a cached inode value in a file handle and
- * that the sought after file has had its inode changed
- * by a third party. So change the error to ESTALE
- * which will trigger a full lookup by the client and
- * will find the new filename/inode pair if it still
- * exists.
- */
- if (rc == ENOENT)
- rc = ESTALE;
-
- return (ERR_PTR(-rc));
- }
-
- ASSERT((ip != NULL) && !IS_ERR(ip));
-
- return (zpl_dentry_obtain_alias(ip));
-}
-
-static struct dentry *
-zpl_get_parent(struct dentry *child)
-{
- cred_t *cr = CRED();
- fstrans_cookie_t cookie;
- struct inode *ip;
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_lookup(child->d_inode, "..", &ip, 0, cr, NULL, NULL);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- if (error)
- return (ERR_PTR(error));
-
- return (zpl_dentry_obtain_alias(ip));
-}
-
-#ifdef HAVE_COMMIT_METADATA
-static int
-zpl_commit_metadata(struct inode *inode)
-{
- cred_t *cr = CRED();
- fstrans_cookie_t cookie;
- int error;
-
- if (zfsctl_is_node(inode))
- return (0);
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_fsync(inode, 0, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-#endif /* HAVE_COMMIT_METADATA */
-
-const struct export_operations zpl_export_operations = {
- .encode_fh = zpl_encode_fh,
- .fh_to_dentry = zpl_fh_to_dentry,
- .get_parent = zpl_get_parent,
-#ifdef HAVE_COMMIT_METADATA
- .commit_metadata = zpl_commit_metadata,
-#endif /* HAVE_COMMIT_METADATA */
-};
diff --git a/module/zfs/zpl_file.c b/module/zfs/zpl_file.c
deleted file mode 100644
index acad4670d..000000000
--- a/module/zfs/zpl_file.c
+++ /dev/null
@@ -1,1075 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
- * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
- */
-
-
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-#endif
-#include <sys/file.h>
-#include <sys/dmu_objset.h>
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_project.h>
-
-
-static int
-zpl_open(struct inode *ip, struct file *filp)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- error = generic_file_open(ip, filp);
- if (error)
- return (error);
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_open(ip, filp->f_mode, filp->f_flags, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_release(struct inode *ip, struct file *filp)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- if (ITOZ(ip)->z_atime_dirty)
- zfs_mark_inode_dirty(ip);
-
- crhold(cr);
- error = -zfs_close(ip, filp->f_flags, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_iterate(struct file *filp, zpl_dir_context_t *ctx)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_readdir(file_inode(filp), ctx, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#if !defined(HAVE_VFS_ITERATE) && !defined(HAVE_VFS_ITERATE_SHARED)
-static int
-zpl_readdir(struct file *filp, void *dirent, filldir_t filldir)
-{
- zpl_dir_context_t ctx =
- ZPL_DIR_CONTEXT_INIT(dirent, filldir, filp->f_pos);
- int error;
-
- error = zpl_iterate(filp, &ctx);
- filp->f_pos = ctx.pos;
-
- return (error);
-}
-#endif /* !HAVE_VFS_ITERATE && !HAVE_VFS_ITERATE_SHARED */
-
-#if defined(HAVE_FSYNC_WITH_DENTRY)
-/*
- * Linux 2.6.x - 2.6.34 API,
- * Through 2.6.34 the nfsd kernel server would pass a NULL 'file struct *'
- * to the fops->fsync() hook. For this reason, we must be careful not to
- * use filp unconditionally.
- */
-static int
-zpl_fsync(struct file *filp, struct dentry *dentry, int datasync)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_fsync(dentry->d_inode, datasync, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifdef HAVE_FILE_AIO_FSYNC
-static int
-zpl_aio_fsync(struct kiocb *kiocb, int datasync)
-{
- struct file *filp = kiocb->ki_filp;
- return (zpl_fsync(filp, file_dentry(filp), datasync));
-}
-#endif
-
-#elif defined(HAVE_FSYNC_WITHOUT_DENTRY)
-/*
- * Linux 2.6.35 - 3.0 API,
- * As of 2.6.35 the dentry argument to the fops->fsync() hook was deemed
- * redundant. The dentry is still accessible via filp->f_path.dentry,
- * and we are guaranteed that filp will never be NULL.
- */
-static int
-zpl_fsync(struct file *filp, int datasync)
-{
- struct inode *inode = filp->f_mapping->host;
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_fsync(inode, datasync, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifdef HAVE_FILE_AIO_FSYNC
-static int
-zpl_aio_fsync(struct kiocb *kiocb, int datasync)
-{
- return (zpl_fsync(kiocb->ki_filp, datasync));
-}
-#endif
-
-#elif defined(HAVE_FSYNC_RANGE)
-/*
- * Linux 3.1 - 3.x API,
- * As of 3.1 the responsibility to call filemap_write_and_wait_range() has
- * been pushed down in to the .fsync() vfs hook. Additionally, the i_mutex
- * lock is no longer held by the caller, for zfs we don't require the lock
- * to be held so we don't acquire it.
- */
-static int
-zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
-{
- struct inode *inode = filp->f_mapping->host;
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- error = filemap_write_and_wait_range(inode->i_mapping, start, end);
- if (error)
- return (error);
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_fsync(inode, datasync, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifdef HAVE_FILE_AIO_FSYNC
-static int
-zpl_aio_fsync(struct kiocb *kiocb, int datasync)
-{
- return (zpl_fsync(kiocb->ki_filp, kiocb->ki_pos, -1, datasync));
-}
-#endif
-
-#else
-#error "Unsupported fops->fsync() implementation"
-#endif
-
-static inline int
-zfs_io_flags(struct kiocb *kiocb)
-{
- int flags = 0;
-
-#if defined(IOCB_DSYNC)
- if (kiocb->ki_flags & IOCB_DSYNC)
- flags |= FDSYNC;
-#endif
-#if defined(IOCB_SYNC)
- if (kiocb->ki_flags & IOCB_SYNC)
- flags |= FSYNC;
-#endif
-#if defined(IOCB_APPEND)
- if (kiocb->ki_flags & IOCB_APPEND)
- flags |= FAPPEND;
-#endif
-#if defined(IOCB_DIRECT)
- if (kiocb->ki_flags & IOCB_DIRECT)
- flags |= FDIRECT;
-#endif
- return (flags);
-}
-
-static ssize_t
-zpl_read_common_iovec(struct inode *ip, const struct iovec *iovp, size_t count,
- unsigned long nr_segs, loff_t *ppos, uio_seg_t segment, int flags,
- cred_t *cr, size_t skip)
-{
- ssize_t read;
- uio_t uio = { { 0 }, 0 };
- int error;
- fstrans_cookie_t cookie;
-
- uio.uio_iov = iovp;
- uio.uio_iovcnt = nr_segs;
- uio.uio_loffset = *ppos;
- uio.uio_segflg = segment;
- uio.uio_limit = MAXOFFSET_T;
- uio.uio_resid = count;
- uio.uio_skip = skip;
-
- cookie = spl_fstrans_mark();
- error = -zfs_read(ip, &uio, flags, cr);
- spl_fstrans_unmark(cookie);
- if (error < 0)
- return (error);
-
- read = count - uio.uio_resid;
- *ppos += read;
-
- return (read);
-}
-
-inline ssize_t
-zpl_read_common(struct inode *ip, const char *buf, size_t len, loff_t *ppos,
- uio_seg_t segment, int flags, cred_t *cr)
-{
- struct iovec iov;
-
- iov.iov_base = (void *)buf;
- iov.iov_len = len;
-
- return (zpl_read_common_iovec(ip, &iov, len, 1, ppos, segment,
- flags, cr, 0));
-}
-
-static ssize_t
-zpl_iter_read_common(struct kiocb *kiocb, const struct iovec *iovp,
- unsigned long nr_segs, size_t count, uio_seg_t seg, size_t skip)
-{
- cred_t *cr = CRED();
- struct file *filp = kiocb->ki_filp;
- struct inode *ip = filp->f_mapping->host;
- zfsvfs_t *zfsvfs = ZTOZSB(ITOZ(ip));
- ssize_t read;
- unsigned int f_flags = filp->f_flags;
-
- f_flags |= zfs_io_flags(kiocb);
- crhold(cr);
- read = zpl_read_common_iovec(filp->f_mapping->host, iovp, count,
- nr_segs, &kiocb->ki_pos, seg, f_flags, cr, skip);
- crfree(cr);
-
- /*
- * If relatime is enabled, call file_accessed() only if
- * zfs_relatime_need_update() is true. This is needed since datasets
- * with inherited "relatime" property aren't necessarily mounted with
- * MNT_RELATIME flag (e.g. after `zfs set relatime=...`), which is what
- * relatime test in VFS by relatime_need_update() is based on.
- */
- if (!IS_NOATIME(ip) && zfsvfs->z_relatime) {
- if (zfs_relatime_need_update(ip))
- file_accessed(filp);
- } else {
- file_accessed(filp);
- }
-
- return (read);
-}
-
-#if defined(HAVE_VFS_RW_ITERATE)
-static ssize_t
-zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to)
-{
- ssize_t ret;
- uio_seg_t seg = UIO_USERSPACE;
- if (to->type & ITER_KVEC)
- seg = UIO_SYSSPACE;
- if (to->type & ITER_BVEC)
- seg = UIO_BVEC;
- ret = zpl_iter_read_common(kiocb, to->iov, to->nr_segs,
- iov_iter_count(to), seg, to->iov_offset);
- if (ret > 0)
- iov_iter_advance(to, ret);
- return (ret);
-}
-#else
-static ssize_t
-zpl_aio_read(struct kiocb *kiocb, const struct iovec *iovp,
- unsigned long nr_segs, loff_t pos)
-{
- ssize_t ret;
- size_t count;
-
- ret = generic_segment_checks(iovp, &nr_segs, &count, VERIFY_WRITE);
- if (ret)
- return (ret);
-
- return (zpl_iter_read_common(kiocb, iovp, nr_segs, count,
- UIO_USERSPACE, 0));
-}
-#endif /* HAVE_VFS_RW_ITERATE */
-
-static ssize_t
-zpl_write_common_iovec(struct inode *ip, const struct iovec *iovp, size_t count,
- unsigned long nr_segs, loff_t *ppos, uio_seg_t segment, int flags,
- cred_t *cr, size_t skip)
-{
- ssize_t wrote;
- uio_t uio = { { 0 }, 0 };
- int error;
- fstrans_cookie_t cookie;
-
- if (flags & O_APPEND)
- *ppos = i_size_read(ip);
-
- uio.uio_iov = iovp;
- uio.uio_iovcnt = nr_segs;
- uio.uio_loffset = *ppos;
- uio.uio_segflg = segment;
- uio.uio_limit = MAXOFFSET_T;
- uio.uio_resid = count;
- uio.uio_skip = skip;
-
- cookie = spl_fstrans_mark();
- error = -zfs_write(ip, &uio, flags, cr);
- spl_fstrans_unmark(cookie);
- if (error < 0)
- return (error);
-
- wrote = count - uio.uio_resid;
- *ppos += wrote;
-
- return (wrote);
-}
-
-inline ssize_t
-zpl_write_common(struct inode *ip, const char *buf, size_t len, loff_t *ppos,
- uio_seg_t segment, int flags, cred_t *cr)
-{
- struct iovec iov;
-
- iov.iov_base = (void *)buf;
- iov.iov_len = len;
-
- return (zpl_write_common_iovec(ip, &iov, len, 1, ppos, segment,
- flags, cr, 0));
-}
-
-static ssize_t
-zpl_iter_write_common(struct kiocb *kiocb, const struct iovec *iovp,
- unsigned long nr_segs, size_t count, uio_seg_t seg, size_t skip)
-{
- cred_t *cr = CRED();
- struct file *filp = kiocb->ki_filp;
- ssize_t wrote;
- unsigned int f_flags = filp->f_flags;
-
- f_flags |= zfs_io_flags(kiocb);
- crhold(cr);
- wrote = zpl_write_common_iovec(filp->f_mapping->host, iovp, count,
- nr_segs, &kiocb->ki_pos, seg, f_flags, cr, skip);
- crfree(cr);
-
- return (wrote);
-}
-
-#if defined(HAVE_VFS_RW_ITERATE)
-static ssize_t
-zpl_iter_write(struct kiocb *kiocb, struct iov_iter *from)
-{
- size_t count;
- ssize_t ret;
- uio_seg_t seg = UIO_USERSPACE;
-
-#ifndef HAVE_GENERIC_WRITE_CHECKS_KIOCB
- struct file *file = kiocb->ki_filp;
- struct address_space *mapping = file->f_mapping;
- struct inode *ip = mapping->host;
- int isblk = S_ISBLK(ip->i_mode);
-
- count = iov_iter_count(from);
- ret = generic_write_checks(file, &kiocb->ki_pos, &count, isblk);
- if (ret)
- return (ret);
-#else
- /*
- * XXX - ideally this check should be in the same lock region with
- * write operations, so that there's no TOCTTOU race when doing
- * append and someone else grow the file.
- */
- ret = generic_write_checks(kiocb, from);
- if (ret <= 0)
- return (ret);
- count = ret;
-#endif
-
- if (from->type & ITER_KVEC)
- seg = UIO_SYSSPACE;
- if (from->type & ITER_BVEC)
- seg = UIO_BVEC;
-
- ret = zpl_iter_write_common(kiocb, from->iov, from->nr_segs,
- count, seg, from->iov_offset);
- if (ret > 0)
- iov_iter_advance(from, ret);
-
- return (ret);
-}
-#else
-static ssize_t
-zpl_aio_write(struct kiocb *kiocb, const struct iovec *iovp,
- unsigned long nr_segs, loff_t pos)
-{
- struct file *file = kiocb->ki_filp;
- struct address_space *mapping = file->f_mapping;
- struct inode *ip = mapping->host;
- int isblk = S_ISBLK(ip->i_mode);
- size_t count;
- ssize_t ret;
-
- ret = generic_segment_checks(iovp, &nr_segs, &count, VERIFY_READ);
- if (ret)
- return (ret);
-
- ret = generic_write_checks(file, &pos, &count, isblk);
- if (ret)
- return (ret);
-
- return (zpl_iter_write_common(kiocb, iovp, nr_segs, count,
- UIO_USERSPACE, 0));
-}
-#endif /* HAVE_VFS_RW_ITERATE */
-
-#if defined(HAVE_VFS_RW_ITERATE)
-static ssize_t
-zpl_direct_IO_impl(int rw, struct kiocb *kiocb, struct iov_iter *iter)
-{
- if (rw == WRITE)
- return (zpl_iter_write(kiocb, iter));
- else
- return (zpl_iter_read(kiocb, iter));
-}
-#if defined(HAVE_VFS_DIRECT_IO_ITER)
-static ssize_t
-zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter)
-{
- return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
-}
-#elif defined(HAVE_VFS_DIRECT_IO_ITER_OFFSET)
-static ssize_t
-zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
-{
- ASSERT3S(pos, ==, kiocb->ki_pos);
- return (zpl_direct_IO_impl(iov_iter_rw(iter), kiocb, iter));
-}
-#elif defined(HAVE_VFS_DIRECT_IO_ITER_RW_OFFSET)
-static ssize_t
-zpl_direct_IO(int rw, struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
-{
- ASSERT3S(pos, ==, kiocb->ki_pos);
- return (zpl_direct_IO_impl(rw, kiocb, iter));
-}
-#else
-#error "Unknown direct IO interface"
-#endif
-
-#else
-
-#if defined(HAVE_VFS_DIRECT_IO_IOVEC)
-static ssize_t
-zpl_direct_IO(int rw, struct kiocb *kiocb, const struct iovec *iovp,
- loff_t pos, unsigned long nr_segs)
-{
- if (rw == WRITE)
- return (zpl_aio_write(kiocb, iovp, nr_segs, pos));
- else
- return (zpl_aio_read(kiocb, iovp, nr_segs, pos));
-}
-#else
-#error "Unknown direct IO interface"
-#endif
-
-#endif /* HAVE_VFS_RW_ITERATE */
-
-static loff_t
-zpl_llseek(struct file *filp, loff_t offset, int whence)
-{
-#if defined(SEEK_HOLE) && defined(SEEK_DATA)
- fstrans_cookie_t cookie;
-
- if (whence == SEEK_DATA || whence == SEEK_HOLE) {
- struct inode *ip = filp->f_mapping->host;
- loff_t maxbytes = ip->i_sb->s_maxbytes;
- loff_t error;
-
- spl_inode_lock_shared(ip);
- cookie = spl_fstrans_mark();
- error = -zfs_holey(ip, whence, &offset);
- spl_fstrans_unmark(cookie);
- if (error == 0)
- error = lseek_execute(filp, ip, offset, maxbytes);
- spl_inode_unlock_shared(ip);
-
- return (error);
- }
-#endif /* SEEK_HOLE && SEEK_DATA */
-
- return (generic_file_llseek(filp, offset, whence));
-}
-
-/*
- * It's worth taking a moment to describe how mmap is implemented
- * for zfs because it differs considerably from other Linux filesystems.
- * However, this issue is handled the same way under OpenSolaris.
- *
- * The issue is that by design zfs bypasses the Linux page cache and
- * leaves all caching up to the ARC. This has been shown to work
- * well for the common read(2)/write(2) case. However, mmap(2)
- * is problem because it relies on being tightly integrated with the
- * page cache. To handle this we cache mmap'ed files twice, once in
- * the ARC and a second time in the page cache. The code is careful
- * to keep both copies synchronized.
- *
- * When a file with an mmap'ed region is written to using write(2)
- * both the data in the ARC and existing pages in the page cache
- * are updated. For a read(2) data will be read first from the page
- * cache then the ARC if needed. Neither a write(2) or read(2) will
- * will ever result in new pages being added to the page cache.
- *
- * New pages are added to the page cache only via .readpage() which
- * is called when the vfs needs to read a page off disk to back the
- * virtual memory region. These pages may be modified without
- * notifying the ARC and will be written out periodically via
- * .writepage(). This will occur due to either a sync or the usual
- * page aging behavior. Note because a read(2) of a mmap'ed file
- * will always check the page cache first even when the ARC is out
- * of date correct data will still be returned.
- *
- * While this implementation ensures correct behavior it does have
- * have some drawbacks. The most obvious of which is that it
- * increases the required memory footprint when access mmap'ed
- * files. It also adds additional complexity to the code keeping
- * both caches synchronized.
- *
- * Longer term it may be possible to cleanly resolve this wart by
- * mapping page cache pages directly on to the ARC buffers. The
- * Linux address space operations are flexible enough to allow
- * selection of which pages back a particular index. The trick
- * would be working out the details of which subsystem is in
- * charge, the ARC, the page cache, or both. It may also prove
- * helpful to move the ARC buffers to a scatter-gather lists
- * rather than a vmalloc'ed region.
- */
-static int
-zpl_mmap(struct file *filp, struct vm_area_struct *vma)
-{
- struct inode *ip = filp->f_mapping->host;
- znode_t *zp = ITOZ(ip);
- int error;
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- error = -zfs_map(ip, vma->vm_pgoff, (caddr_t *)vma->vm_start,
- (size_t)(vma->vm_end - vma->vm_start), vma->vm_flags);
- spl_fstrans_unmark(cookie);
- if (error)
- return (error);
-
- error = generic_file_mmap(filp, vma);
- if (error)
- return (error);
-
- mutex_enter(&zp->z_lock);
- zp->z_is_mapped = B_TRUE;
- mutex_exit(&zp->z_lock);
-
- return (error);
-}
-
-/*
- * Populate a page with data for the Linux page cache. This function is
- * only used to support mmap(2). There will be an identical copy of the
- * data in the ARC which is kept up to date via .write() and .writepage().
- *
- * Current this function relies on zpl_read_common() and the O_DIRECT
- * flag to read in a page. This works but the more correct way is to
- * update zfs_fillpage() to be Linux friendly and use that interface.
- */
-static int
-zpl_readpage(struct file *filp, struct page *pp)
-{
- struct inode *ip;
- struct page *pl[1];
- int error = 0;
- fstrans_cookie_t cookie;
-
- ASSERT(PageLocked(pp));
- ip = pp->mapping->host;
- pl[0] = pp;
-
- cookie = spl_fstrans_mark();
- error = -zfs_getpage(ip, pl, 1);
- spl_fstrans_unmark(cookie);
-
- if (error) {
- SetPageError(pp);
- ClearPageUptodate(pp);
- } else {
- ClearPageError(pp);
- SetPageUptodate(pp);
- flush_dcache_page(pp);
- }
-
- unlock_page(pp);
- return (error);
-}
-
-/*
- * Populate a set of pages with data for the Linux page cache. This
- * function will only be called for read ahead and never for demand
- * paging. For simplicity, the code relies on read_cache_pages() to
- * correctly lock each page for IO and call zpl_readpage().
- */
-static int
-zpl_readpages(struct file *filp, struct address_space *mapping,
- struct list_head *pages, unsigned nr_pages)
-{
- return (read_cache_pages(mapping, pages,
- (filler_t *)zpl_readpage, filp));
-}
-
-int
-zpl_putpage(struct page *pp, struct writeback_control *wbc, void *data)
-{
- struct address_space *mapping = data;
- fstrans_cookie_t cookie;
-
- ASSERT(PageLocked(pp));
- ASSERT(!PageWriteback(pp));
-
- cookie = spl_fstrans_mark();
- (void) zfs_putpage(mapping->host, pp, wbc);
- spl_fstrans_unmark(cookie);
-
- return (0);
-}
-
-static int
-zpl_writepages(struct address_space *mapping, struct writeback_control *wbc)
-{
- znode_t *zp = ITOZ(mapping->host);
- zfsvfs_t *zfsvfs = ITOZSB(mapping->host);
- enum writeback_sync_modes sync_mode;
- int result;
-
- ZFS_ENTER(zfsvfs);
- if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
- wbc->sync_mode = WB_SYNC_ALL;
- ZFS_EXIT(zfsvfs);
- sync_mode = wbc->sync_mode;
-
- /*
- * We don't want to run write_cache_pages() in SYNC mode here, because
- * that would make putpage() wait for a single page to be committed to
- * disk every single time, resulting in atrocious performance. Instead
- * we run it once in non-SYNC mode so that the ZIL gets all the data,
- * and then we commit it all in one go.
- */
- wbc->sync_mode = WB_SYNC_NONE;
- result = write_cache_pages(mapping, wbc, zpl_putpage, mapping);
- if (sync_mode != wbc->sync_mode) {
- ZFS_ENTER(zfsvfs);
- ZFS_VERIFY_ZP(zp);
- if (zfsvfs->z_log != NULL)
- zil_commit(zfsvfs->z_log, zp->z_id);
- ZFS_EXIT(zfsvfs);
-
- /*
- * We need to call write_cache_pages() again (we can't just
- * return after the commit) because the previous call in
- * non-SYNC mode does not guarantee that we got all the dirty
- * pages (see the implementation of write_cache_pages() for
- * details). That being said, this is a no-op in most cases.
- */
- wbc->sync_mode = sync_mode;
- result = write_cache_pages(mapping, wbc, zpl_putpage, mapping);
- }
- return (result);
-}
-
-/*
- * Write out dirty pages to the ARC, this function is only required to
- * support mmap(2). Mapped pages may be dirtied by memory operations
- * which never call .write(). These dirty pages are kept in sync with
- * the ARC buffers via this hook.
- */
-static int
-zpl_writepage(struct page *pp, struct writeback_control *wbc)
-{
- if (ITOZSB(pp->mapping->host)->z_os->os_sync == ZFS_SYNC_ALWAYS)
- wbc->sync_mode = WB_SYNC_ALL;
-
- return (zpl_putpage(pp, wbc, pp->mapping));
-}
-
-/*
- * The only flag combination which matches the behavior of zfs_space()
- * is FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE. The FALLOC_FL_PUNCH_HOLE
- * flag was introduced in the 2.6.38 kernel.
- */
-#if defined(HAVE_FILE_FALLOCATE) || defined(HAVE_INODE_FALLOCATE)
-long
-zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len)
-{
- int error = -EOPNOTSUPP;
-
-#if defined(FALLOC_FL_PUNCH_HOLE) && defined(FALLOC_FL_KEEP_SIZE)
- cred_t *cr = CRED();
- flock64_t bf;
- loff_t olen;
- fstrans_cookie_t cookie;
-
- if (mode != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
- return (error);
-
- if (offset < 0 || len <= 0)
- return (-EINVAL);
-
- spl_inode_lock(ip);
- olen = i_size_read(ip);
-
- if (offset > olen) {
- spl_inode_unlock(ip);
- return (0);
- }
- if (offset + len > olen)
- len = olen - offset;
- bf.l_type = F_WRLCK;
- bf.l_whence = SEEK_SET;
- bf.l_start = offset;
- bf.l_len = len;
- bf.l_pid = 0;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_space(ip, F_FREESP, &bf, FWRITE, offset, cr);
- spl_fstrans_unmark(cookie);
- spl_inode_unlock(ip);
-
- crfree(cr);
-#endif /* defined(FALLOC_FL_PUNCH_HOLE) && defined(FALLOC_FL_KEEP_SIZE) */
-
- ASSERT3S(error, <=, 0);
- return (error);
-}
-#endif /* defined(HAVE_FILE_FALLOCATE) || defined(HAVE_INODE_FALLOCATE) */
-
-#ifdef HAVE_FILE_FALLOCATE
-static long
-zpl_fallocate(struct file *filp, int mode, loff_t offset, loff_t len)
-{
- return zpl_fallocate_common(file_inode(filp),
- mode, offset, len);
-}
-#endif /* HAVE_FILE_FALLOCATE */
-
-#define ZFS_FL_USER_VISIBLE (FS_FL_USER_VISIBLE | ZFS_PROJINHERIT_FL)
-#define ZFS_FL_USER_MODIFIABLE (FS_FL_USER_MODIFIABLE | ZFS_PROJINHERIT_FL)
-
-static uint32_t
-__zpl_ioctl_getflags(struct inode *ip)
-{
- uint64_t zfs_flags = ITOZ(ip)->z_pflags;
- uint32_t ioctl_flags = 0;
-
- if (zfs_flags & ZFS_IMMUTABLE)
- ioctl_flags |= FS_IMMUTABLE_FL;
-
- if (zfs_flags & ZFS_APPENDONLY)
- ioctl_flags |= FS_APPEND_FL;
-
- if (zfs_flags & ZFS_NODUMP)
- ioctl_flags |= FS_NODUMP_FL;
-
- if (zfs_flags & ZFS_PROJINHERIT)
- ioctl_flags |= ZFS_PROJINHERIT_FL;
-
- return (ioctl_flags & ZFS_FL_USER_VISIBLE);
-}
-
-/*
- * Map zfs file z_pflags (xvattr_t) to linux file attributes. Only file
- * attributes common to both Linux and Solaris are mapped.
- */
-static int
-zpl_ioctl_getflags(struct file *filp, void __user *arg)
-{
- uint32_t flags;
- int err;
-
- flags = __zpl_ioctl_getflags(file_inode(filp));
- err = copy_to_user(arg, &flags, sizeof (flags));
-
- return (err);
-}
-
-/*
- * fchange() is a helper macro to detect if we have been asked to change a
- * flag. This is ugly, but the requirement that we do this is a consequence of
- * how the Linux file attribute interface was designed. Another consequence is
- * that concurrent modification of files suffers from a TOCTOU race. Neither
- * are things we can fix without modifying the kernel-userland interface, which
- * is outside of our jurisdiction.
- */
-
-#define fchange(f0, f1, b0, b1) (!((f0) & (b0)) != !((f1) & (b1)))
-
-static int
-__zpl_ioctl_setflags(struct inode *ip, uint32_t ioctl_flags, xvattr_t *xva)
-{
- uint64_t zfs_flags = ITOZ(ip)->z_pflags;
- xoptattr_t *xoap;
-
- if (ioctl_flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL |
- ZFS_PROJINHERIT_FL))
- return (-EOPNOTSUPP);
-
- if (ioctl_flags & ~ZFS_FL_USER_MODIFIABLE)
- return (-EACCES);
-
- if ((fchange(ioctl_flags, zfs_flags, FS_IMMUTABLE_FL, ZFS_IMMUTABLE) ||
- fchange(ioctl_flags, zfs_flags, FS_APPEND_FL, ZFS_APPENDONLY)) &&
- !capable(CAP_LINUX_IMMUTABLE))
- return (-EACCES);
-
- if (!zpl_inode_owner_or_capable(ip))
- return (-EACCES);
-
- xva_init(xva);
- xoap = xva_getxoptattr(xva);
-
- XVA_SET_REQ(xva, XAT_IMMUTABLE);
- if (ioctl_flags & FS_IMMUTABLE_FL)
- xoap->xoa_immutable = B_TRUE;
-
- XVA_SET_REQ(xva, XAT_APPENDONLY);
- if (ioctl_flags & FS_APPEND_FL)
- xoap->xoa_appendonly = B_TRUE;
-
- XVA_SET_REQ(xva, XAT_NODUMP);
- if (ioctl_flags & FS_NODUMP_FL)
- xoap->xoa_nodump = B_TRUE;
-
- XVA_SET_REQ(xva, XAT_PROJINHERIT);
- if (ioctl_flags & ZFS_PROJINHERIT_FL)
- xoap->xoa_projinherit = B_TRUE;
-
- return (0);
-}
-
-static int
-zpl_ioctl_setflags(struct file *filp, void __user *arg)
-{
- struct inode *ip = file_inode(filp);
- uint32_t flags;
- cred_t *cr = CRED();
- xvattr_t xva;
- int err;
- fstrans_cookie_t cookie;
-
- if (copy_from_user(&flags, arg, sizeof (flags)))
- return (-EFAULT);
-
- err = __zpl_ioctl_setflags(ip, flags, &xva);
- if (err)
- return (err);
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- err = -zfs_setattr(ip, (vattr_t *)&xva, 0, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- return (err);
-}
-
-static int
-zpl_ioctl_getxattr(struct file *filp, void __user *arg)
-{
- zfsxattr_t fsx = { 0 };
- struct inode *ip = file_inode(filp);
- int err;
-
- fsx.fsx_xflags = __zpl_ioctl_getflags(ip);
- fsx.fsx_projid = ITOZ(ip)->z_projid;
- err = copy_to_user(arg, &fsx, sizeof (fsx));
-
- return (err);
-}
-
-static int
-zpl_ioctl_setxattr(struct file *filp, void __user *arg)
-{
- struct inode *ip = file_inode(filp);
- zfsxattr_t fsx;
- cred_t *cr = CRED();
- xvattr_t xva;
- xoptattr_t *xoap;
- int err;
- fstrans_cookie_t cookie;
-
- if (copy_from_user(&fsx, arg, sizeof (fsx)))
- return (-EFAULT);
-
- if (!zpl_is_valid_projid(fsx.fsx_projid))
- return (-EINVAL);
-
- err = __zpl_ioctl_setflags(ip, fsx.fsx_xflags, &xva);
- if (err)
- return (err);
-
- xoap = xva_getxoptattr(&xva);
- XVA_SET_REQ(&xva, XAT_PROJID);
- xoap->xoa_projid = fsx.fsx_projid;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- err = -zfs_setattr(ip, (vattr_t *)&xva, 0, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- return (err);
-}
-
-static long
-zpl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
- switch (cmd) {
- case FS_IOC_GETFLAGS:
- return (zpl_ioctl_getflags(filp, (void *)arg));
- case FS_IOC_SETFLAGS:
- return (zpl_ioctl_setflags(filp, (void *)arg));
- case ZFS_IOC_FSGETXATTR:
- return (zpl_ioctl_getxattr(filp, (void *)arg));
- case ZFS_IOC_FSSETXATTR:
- return (zpl_ioctl_setxattr(filp, (void *)arg));
- default:
- return (-ENOTTY);
- }
-}
-
-#ifdef CONFIG_COMPAT
-static long
-zpl_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
- switch (cmd) {
- case FS_IOC32_GETFLAGS:
- cmd = FS_IOC_GETFLAGS;
- break;
- case FS_IOC32_SETFLAGS:
- cmd = FS_IOC_SETFLAGS;
- break;
- default:
- return (-ENOTTY);
- }
- return (zpl_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)));
-}
-#endif /* CONFIG_COMPAT */
-
-
-const struct address_space_operations zpl_address_space_operations = {
- .readpages = zpl_readpages,
- .readpage = zpl_readpage,
- .writepage = zpl_writepage,
- .writepages = zpl_writepages,
- .direct_IO = zpl_direct_IO,
-};
-
-const struct file_operations zpl_file_operations = {
- .open = zpl_open,
- .release = zpl_release,
- .llseek = zpl_llseek,
-#ifdef HAVE_VFS_RW_ITERATE
-#ifdef HAVE_NEW_SYNC_READ
- .read = new_sync_read,
- .write = new_sync_write,
-#endif
- .read_iter = zpl_iter_read,
- .write_iter = zpl_iter_write,
-#else
- .read = do_sync_read,
- .write = do_sync_write,
- .aio_read = zpl_aio_read,
- .aio_write = zpl_aio_write,
-#endif
- .mmap = zpl_mmap,
- .fsync = zpl_fsync,
-#ifdef HAVE_FILE_AIO_FSYNC
- .aio_fsync = zpl_aio_fsync,
-#endif
-#ifdef HAVE_FILE_FALLOCATE
- .fallocate = zpl_fallocate,
-#endif /* HAVE_FILE_FALLOCATE */
- .unlocked_ioctl = zpl_ioctl,
-#ifdef CONFIG_COMPAT
- .compat_ioctl = zpl_compat_ioctl,
-#endif
-};
-
-const struct file_operations zpl_dir_file_operations = {
- .llseek = generic_file_llseek,
- .read = generic_read_dir,
-#if defined(HAVE_VFS_ITERATE_SHARED)
- .iterate_shared = zpl_iterate,
-#elif defined(HAVE_VFS_ITERATE)
- .iterate = zpl_iterate,
-#else
- .readdir = zpl_readdir,
-#endif
- .fsync = zpl_fsync,
- .unlocked_ioctl = zpl_ioctl,
-#ifdef CONFIG_COMPAT
- .compat_ioctl = zpl_compat_ioctl,
-#endif
-};
diff --git a/module/zfs/zpl_inode.c b/module/zfs/zpl_inode.c
deleted file mode 100644
index 3f3b2e2dc..000000000
--- a/module/zfs/zpl_inode.c
+++ /dev/null
@@ -1,826 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
- * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
- */
-
-
-#include <sys/zfs_ctldir.h>
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/dmu_objset.h>
-#include <sys/vfs.h>
-#include <sys/zpl.h>
-#include <sys/file.h>
-
-
-static struct dentry *
-#ifdef HAVE_LOOKUP_NAMEIDATA
-zpl_lookup(struct inode *dir, struct dentry *dentry, struct nameidata *nd)
-#else
-zpl_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
-#endif
-{
- cred_t *cr = CRED();
- struct inode *ip;
- int error;
- fstrans_cookie_t cookie;
- pathname_t *ppn = NULL;
- pathname_t pn;
- int zfs_flags = 0;
- zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
-
- if (dlen(dentry) >= ZAP_MAXNAMELEN)
- return (ERR_PTR(-ENAMETOOLONG));
-
- crhold(cr);
- cookie = spl_fstrans_mark();
-
- /* If we are a case insensitive fs, we need the real name */
- if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE) {
- zfs_flags = FIGNORECASE;
- pn_alloc(&pn);
- ppn = &pn;
- }
-
- error = -zfs_lookup(dir, dname(dentry), &ip, zfs_flags, cr, NULL, ppn);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
- crfree(cr);
-
- spin_lock(&dentry->d_lock);
- dentry->d_time = jiffies;
-#ifndef HAVE_S_D_OP
- d_set_d_op(dentry, &zpl_dentry_operations);
-#endif /* HAVE_S_D_OP */
- spin_unlock(&dentry->d_lock);
-
- if (error) {
- /*
- * If we have a case sensitive fs, we do not want to
- * insert negative entries, so return NULL for ENOENT.
- * Fall through if the error is not ENOENT. Also free memory.
- */
- if (ppn) {
- pn_free(ppn);
- if (error == -ENOENT)
- return (NULL);
- }
-
- if (error == -ENOENT)
- return (d_splice_alias(NULL, dentry));
- else
- return (ERR_PTR(error));
- }
-
- /*
- * If we are case insensitive, call the correct function
- * to install the name.
- */
- if (ppn) {
- struct dentry *new_dentry;
- struct qstr ci_name;
-
- if (strcmp(dname(dentry), pn.pn_buf) == 0) {
- new_dentry = d_splice_alias(ip, dentry);
- } else {
- ci_name.name = pn.pn_buf;
- ci_name.len = strlen(pn.pn_buf);
- new_dentry = d_add_ci(dentry, ip, &ci_name);
- }
- pn_free(ppn);
- return (new_dentry);
- } else {
- return (d_splice_alias(ip, dentry));
- }
-}
-
-void
-zpl_vap_init(vattr_t *vap, struct inode *dir, zpl_umode_t mode, cred_t *cr)
-{
- vap->va_mask = ATTR_MODE;
- vap->va_mode = mode;
- vap->va_uid = crgetfsuid(cr);
-
- if (dir && dir->i_mode & S_ISGID) {
- vap->va_gid = KGID_TO_SGID(dir->i_gid);
- if (S_ISDIR(mode))
- vap->va_mode |= S_ISGID;
- } else {
- vap->va_gid = crgetfsgid(cr);
- }
-}
-
-static int
-#ifdef HAVE_CREATE_NAMEIDATA
-zpl_create(struct inode *dir, struct dentry *dentry, zpl_umode_t mode,
- struct nameidata *nd)
-#else
-zpl_create(struct inode *dir, struct dentry *dentry, zpl_umode_t mode,
- bool flag)
-#endif
-{
- cred_t *cr = CRED();
- struct inode *ip;
- vattr_t *vap;
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dir, mode, cr);
-
- cookie = spl_fstrans_mark();
- error = -zfs_create(dir, dname(dentry), vap, 0, mode, &ip, cr, 0, NULL);
- if (error == 0) {
- d_instantiate(dentry, ip);
-
- error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
- if (error == 0)
- error = zpl_init_acl(ip, dir);
-
- if (error)
- (void) zfs_remove(dir, dname(dentry), cr, 0);
- }
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_mknod(struct inode *dir, struct dentry *dentry, zpl_umode_t mode,
- dev_t rdev)
-{
- cred_t *cr = CRED();
- struct inode *ip;
- vattr_t *vap;
- int error;
- fstrans_cookie_t cookie;
-
- /*
- * We currently expect Linux to supply rdev=0 for all sockets
- * and fifos, but we want to know if this behavior ever changes.
- */
- if (S_ISSOCK(mode) || S_ISFIFO(mode))
- ASSERT(rdev == 0);
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dir, mode, cr);
- vap->va_rdev = rdev;
-
- cookie = spl_fstrans_mark();
- error = -zfs_create(dir, dname(dentry), vap, 0, mode, &ip, cr, 0, NULL);
- if (error == 0) {
- d_instantiate(dentry, ip);
-
- error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
- if (error == 0)
- error = zpl_init_acl(ip, dir);
-
- if (error)
- (void) zfs_remove(dir, dname(dentry), cr, 0);
- }
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifdef HAVE_TMPFILE
-static int
-zpl_tmpfile(struct inode *dir, struct dentry *dentry, zpl_umode_t mode)
-{
- cred_t *cr = CRED();
- struct inode *ip;
- vattr_t *vap;
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dir, mode, cr);
-
- cookie = spl_fstrans_mark();
- error = -zfs_tmpfile(dir, vap, 0, mode, &ip, cr, 0, NULL);
- if (error == 0) {
- /* d_tmpfile will do drop_nlink, so we should set it first */
- set_nlink(ip, 1);
- d_tmpfile(dentry, ip);
-
- error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
- if (error == 0)
- error = zpl_init_acl(ip, dir);
- /*
- * don't need to handle error here, file is already in
- * unlinked set.
- */
- }
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-#endif
-
-static int
-zpl_unlink(struct inode *dir, struct dentry *dentry)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
- zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_remove(dir, dname(dentry), cr, 0);
-
- /*
- * For a CI FS we must invalidate the dentry to prevent the
- * creation of negative entries.
- */
- if (error == 0 && zfsvfs->z_case == ZFS_CASE_INSENSITIVE)
- d_invalidate(dentry);
-
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_mkdir(struct inode *dir, struct dentry *dentry, zpl_umode_t mode)
-{
- cred_t *cr = CRED();
- vattr_t *vap;
- struct inode *ip;
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dir, mode | S_IFDIR, cr);
-
- cookie = spl_fstrans_mark();
- error = -zfs_mkdir(dir, dname(dentry), vap, &ip, cr, 0, NULL);
- if (error == 0) {
- d_instantiate(dentry, ip);
-
- error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
- if (error == 0)
- error = zpl_init_acl(ip, dir);
-
- if (error)
- (void) zfs_rmdir(dir, dname(dentry), NULL, cr, 0);
- }
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_rmdir(struct inode *dir, struct dentry *dentry)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
- zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_rmdir(dir, dname(dentry), NULL, cr, 0);
-
- /*
- * For a CI FS we must invalidate the dentry to prevent the
- * creation of negative entries.
- */
- if (error == 0 && zfsvfs->z_case == ZFS_CASE_INSENSITIVE)
- d_invalidate(dentry);
-
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask,
- unsigned int query_flags)
-{
- int error;
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
-
- /*
- * XXX request_mask and query_flags currently ignored.
- */
-
- error = -zfs_getattr_fast(path->dentry->d_inode, stat);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-ZPL_GETATTR_WRAPPER(zpl_getattr);
-
-static int
-zpl_setattr(struct dentry *dentry, struct iattr *ia)
-{
- struct inode *ip = dentry->d_inode;
- cred_t *cr = CRED();
- vattr_t *vap;
- int error;
- fstrans_cookie_t cookie;
-
- error = setattr_prepare(dentry, ia);
- if (error)
- return (error);
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- vap->va_mask = ia->ia_valid & ATTR_IATTR_MASK;
- vap->va_mode = ia->ia_mode;
- vap->va_uid = KUID_TO_SUID(ia->ia_uid);
- vap->va_gid = KGID_TO_SGID(ia->ia_gid);
- vap->va_size = ia->ia_size;
- vap->va_atime = ia->ia_atime;
- vap->va_mtime = ia->ia_mtime;
- vap->va_ctime = ia->ia_ctime;
-
- if (vap->va_mask & ATTR_ATIME) {
- ip->i_atime = zpl_inode_timespec_trunc(ia->ia_atime,
- ip->i_sb->s_time_gran);
- }
-
- cookie = spl_fstrans_mark();
- error = -zfs_setattr(ip, vap, 0, cr);
- if (!error && (ia->ia_valid & ATTR_MODE))
- error = zpl_chmod_acl(ip);
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_rename2(struct inode *sdip, struct dentry *sdentry,
- struct inode *tdip, struct dentry *tdentry, unsigned int flags)
-{
- cred_t *cr = CRED();
- int error;
- fstrans_cookie_t cookie;
-
- /* We don't have renameat2(2) support */
- if (flags)
- return (-EINVAL);
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_rename(sdip, dname(sdentry), tdip, dname(tdentry), cr, 0);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifndef HAVE_RENAME_WANTS_FLAGS
-static int
-zpl_rename(struct inode *sdip, struct dentry *sdentry,
- struct inode *tdip, struct dentry *tdentry)
-{
- return (zpl_rename2(sdip, sdentry, tdip, tdentry, 0));
-}
-#endif
-
-static int
-zpl_symlink(struct inode *dir, struct dentry *dentry, const char *name)
-{
- cred_t *cr = CRED();
- vattr_t *vap;
- struct inode *ip;
- int error;
- fstrans_cookie_t cookie;
-
- crhold(cr);
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- zpl_vap_init(vap, dir, S_IFLNK | S_IRWXUGO, cr);
-
- cookie = spl_fstrans_mark();
- error = -zfs_symlink(dir, dname(dentry), vap, (char *)name, &ip, cr, 0);
- if (error == 0) {
- d_instantiate(dentry, ip);
-
- error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
- if (error)
- (void) zfs_remove(dir, dname(dentry), cr, 0);
- }
-
- spl_fstrans_unmark(cookie);
- kmem_free(vap, sizeof (vattr_t));
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#if defined(HAVE_PUT_LINK_COOKIE)
-static void
-zpl_put_link(struct inode *unused, void *cookie)
-{
- kmem_free(cookie, MAXPATHLEN);
-}
-#elif defined(HAVE_PUT_LINK_NAMEIDATA)
-static void
-zpl_put_link(struct dentry *dentry, struct nameidata *nd, void *ptr)
-{
- const char *link = nd_get_link(nd);
-
- if (!IS_ERR(link))
- kmem_free(link, MAXPATHLEN);
-}
-#elif defined(HAVE_PUT_LINK_DELAYED)
-static void
-zpl_put_link(void *ptr)
-{
- kmem_free(ptr, MAXPATHLEN);
-}
-#endif
-
-static int
-zpl_get_link_common(struct dentry *dentry, struct inode *ip, char **link)
-{
- fstrans_cookie_t cookie;
- cred_t *cr = CRED();
- struct iovec iov;
- uio_t uio = { { 0 }, 0 };
- int error;
-
- crhold(cr);
- *link = NULL;
- iov.iov_len = MAXPATHLEN;
- iov.iov_base = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
-
- uio.uio_iov = &iov;
- uio.uio_iovcnt = 1;
- uio.uio_segflg = UIO_SYSSPACE;
- uio.uio_resid = (MAXPATHLEN - 1);
-
- cookie = spl_fstrans_mark();
- error = -zfs_readlink(ip, &uio, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- if (error)
- kmem_free(iov.iov_base, MAXPATHLEN);
- else
- *link = iov.iov_base;
-
- return (error);
-}
-
-#if defined(HAVE_GET_LINK_DELAYED)
-const char *
-zpl_get_link(struct dentry *dentry, struct inode *inode,
- struct delayed_call *done)
-{
- char *link = NULL;
- int error;
-
- if (!dentry)
- return (ERR_PTR(-ECHILD));
-
- error = zpl_get_link_common(dentry, inode, &link);
- if (error)
- return (ERR_PTR(error));
-
- set_delayed_call(done, zpl_put_link, link);
-
- return (link);
-}
-#elif defined(HAVE_GET_LINK_COOKIE)
-const char *
-zpl_get_link(struct dentry *dentry, struct inode *inode, void **cookie)
-{
- char *link = NULL;
- int error;
-
- if (!dentry)
- return (ERR_PTR(-ECHILD));
-
- error = zpl_get_link_common(dentry, inode, &link);
- if (error)
- return (ERR_PTR(error));
-
- return (*cookie = link);
-}
-#elif defined(HAVE_FOLLOW_LINK_COOKIE)
-const char *
-zpl_follow_link(struct dentry *dentry, void **cookie)
-{
- char *link = NULL;
- int error;
-
- error = zpl_get_link_common(dentry, dentry->d_inode, &link);
- if (error)
- return (ERR_PTR(error));
-
- return (*cookie = link);
-}
-#elif defined(HAVE_FOLLOW_LINK_NAMEIDATA)
-static void *
-zpl_follow_link(struct dentry *dentry, struct nameidata *nd)
-{
- char *link = NULL;
- int error;
-
- error = zpl_get_link_common(dentry, dentry->d_inode, &link);
- if (error)
- nd_set_link(nd, ERR_PTR(error));
- else
- nd_set_link(nd, link);
-
- return (NULL);
-}
-#endif
-
-static int
-zpl_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
-{
- cred_t *cr = CRED();
- struct inode *ip = old_dentry->d_inode;
- int error;
- fstrans_cookie_t cookie;
-
- if (ip->i_nlink >= ZFS_LINK_MAX)
- return (-EMLINK);
-
- crhold(cr);
- ip->i_ctime = current_time(ip);
- igrab(ip); /* Use ihold() if available */
-
- cookie = spl_fstrans_mark();
- error = -zfs_link(dir, ip, dname(dentry), cr, 0);
- if (error) {
- iput(ip);
- goto out;
- }
-
- d_instantiate(dentry, ip);
-out:
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-#ifdef HAVE_INODE_TRUNCATE_RANGE
-static void
-zpl_truncate_range(struct inode *ip, loff_t start, loff_t end)
-{
- cred_t *cr = CRED();
- flock64_t bf;
- fstrans_cookie_t cookie;
-
- ASSERT3S(start, <=, end);
-
- /*
- * zfs_freesp() will interpret (len == 0) as meaning "truncate until
- * the end of the file". We don't want that.
- */
- if (start == end)
- return;
-
- crhold(cr);
-
- bf.l_type = F_WRLCK;
- bf.l_whence = SEEK_SET;
- bf.l_start = start;
- bf.l_len = end - start;
- bf.l_pid = 0;
- cookie = spl_fstrans_mark();
- zfs_space(ip, F_FREESP, &bf, FWRITE, start, cr);
- spl_fstrans_unmark(cookie);
-
- crfree(cr);
-}
-#endif /* HAVE_INODE_TRUNCATE_RANGE */
-
-#ifdef HAVE_INODE_FALLOCATE
-static long
-zpl_fallocate(struct inode *ip, int mode, loff_t offset, loff_t len)
-{
- return (zpl_fallocate_common(ip, mode, offset, len));
-}
-#endif /* HAVE_INODE_FALLOCATE */
-
-static int
-#ifdef HAVE_D_REVALIDATE_NAMEIDATA
-zpl_revalidate(struct dentry *dentry, struct nameidata *nd)
-{
- unsigned int flags = (nd ? nd->flags : 0);
-#else
-zpl_revalidate(struct dentry *dentry, unsigned int flags)
-{
-#endif /* HAVE_D_REVALIDATE_NAMEIDATA */
- /* CSTYLED */
- zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
- int error;
-
- if (flags & LOOKUP_RCU)
- return (-ECHILD);
-
- /*
- * Automounted snapshots rely on periodic dentry revalidation
- * to defer snapshots from being automatically unmounted.
- */
- if (zfsvfs->z_issnap) {
- if (time_after(jiffies, zfsvfs->z_snap_defer_time +
- MAX(zfs_expire_snapshot * HZ / 2, HZ))) {
- zfsvfs->z_snap_defer_time = jiffies;
- zfsctl_snapshot_unmount_delay(zfsvfs->z_os->os_spa,
- dmu_objset_id(zfsvfs->z_os), zfs_expire_snapshot);
- }
- }
-
- /*
- * After a rollback negative dentries created before the rollback
- * time must be invalidated. Otherwise they can obscure files which
- * are only present in the rolled back dataset.
- */
- if (dentry->d_inode == NULL) {
- spin_lock(&dentry->d_lock);
- error = time_before(dentry->d_time, zfsvfs->z_rollback_time);
- spin_unlock(&dentry->d_lock);
-
- if (error)
- return (0);
- }
-
- /*
- * The dentry may reference a stale inode if a mounted file system
- * was rolled back to a point in time where the object didn't exist.
- */
- if (dentry->d_inode && ITOZ(dentry->d_inode)->z_is_stale)
- return (0);
-
- return (1);
-}
-
-const struct inode_operations zpl_inode_operations = {
- .setattr = zpl_setattr,
- .getattr = zpl_getattr,
-#ifdef HAVE_GENERIC_SETXATTR
- .setxattr = generic_setxattr,
- .getxattr = generic_getxattr,
- .removexattr = generic_removexattr,
-#endif
- .listxattr = zpl_xattr_list,
-#ifdef HAVE_INODE_TRUNCATE_RANGE
- .truncate_range = zpl_truncate_range,
-#endif /* HAVE_INODE_TRUNCATE_RANGE */
-#ifdef HAVE_INODE_FALLOCATE
- .fallocate = zpl_fallocate,
-#endif /* HAVE_INODE_FALLOCATE */
-#if defined(CONFIG_FS_POSIX_ACL)
-#if defined(HAVE_SET_ACL)
- .set_acl = zpl_set_acl,
-#endif
-#if defined(HAVE_GET_ACL)
- .get_acl = zpl_get_acl,
-#elif defined(HAVE_CHECK_ACL)
- .check_acl = zpl_check_acl,
-#elif defined(HAVE_PERMISSION)
- .permission = zpl_permission,
-#endif /* HAVE_GET_ACL | HAVE_CHECK_ACL | HAVE_PERMISSION */
-#endif /* CONFIG_FS_POSIX_ACL */
-};
-
-const struct inode_operations zpl_dir_inode_operations = {
- .create = zpl_create,
- .lookup = zpl_lookup,
- .link = zpl_link,
- .unlink = zpl_unlink,
- .symlink = zpl_symlink,
- .mkdir = zpl_mkdir,
- .rmdir = zpl_rmdir,
- .mknod = zpl_mknod,
-#ifdef HAVE_RENAME_WANTS_FLAGS
- .rename = zpl_rename2,
-#else
- .rename = zpl_rename,
-#endif
-#ifdef HAVE_TMPFILE
- .tmpfile = zpl_tmpfile,
-#endif
- .setattr = zpl_setattr,
- .getattr = zpl_getattr,
-#ifdef HAVE_GENERIC_SETXATTR
- .setxattr = generic_setxattr,
- .getxattr = generic_getxattr,
- .removexattr = generic_removexattr,
-#endif
- .listxattr = zpl_xattr_list,
-#if defined(CONFIG_FS_POSIX_ACL)
-#if defined(HAVE_SET_ACL)
- .set_acl = zpl_set_acl,
-#endif
-#if defined(HAVE_GET_ACL)
- .get_acl = zpl_get_acl,
-#elif defined(HAVE_CHECK_ACL)
- .check_acl = zpl_check_acl,
-#elif defined(HAVE_PERMISSION)
- .permission = zpl_permission,
-#endif /* HAVE_GET_ACL | HAVE_CHECK_ACL | HAVE_PERMISSION */
-#endif /* CONFIG_FS_POSIX_ACL */
-};
-
-const struct inode_operations zpl_symlink_inode_operations = {
-#ifdef HAVE_GENERIC_READLINK
- .readlink = generic_readlink,
-#endif
-#if defined(HAVE_GET_LINK_DELAYED) || defined(HAVE_GET_LINK_COOKIE)
- .get_link = zpl_get_link,
-#elif defined(HAVE_FOLLOW_LINK_COOKIE) || defined(HAVE_FOLLOW_LINK_NAMEIDATA)
- .follow_link = zpl_follow_link,
-#endif
-#if defined(HAVE_PUT_LINK_COOKIE) || defined(HAVE_PUT_LINK_NAMEIDATA)
- .put_link = zpl_put_link,
-#endif
- .setattr = zpl_setattr,
- .getattr = zpl_getattr,
-#ifdef HAVE_GENERIC_SETXATTR
- .setxattr = generic_setxattr,
- .getxattr = generic_getxattr,
- .removexattr = generic_removexattr,
-#endif
- .listxattr = zpl_xattr_list,
-};
-
-const struct inode_operations zpl_special_inode_operations = {
- .setattr = zpl_setattr,
- .getattr = zpl_getattr,
-#ifdef HAVE_GENERIC_SETXATTR
- .setxattr = generic_setxattr,
- .getxattr = generic_getxattr,
- .removexattr = generic_removexattr,
-#endif
- .listxattr = zpl_xattr_list,
-#if defined(CONFIG_FS_POSIX_ACL)
-#if defined(HAVE_SET_ACL)
- .set_acl = zpl_set_acl,
-#endif
-#if defined(HAVE_GET_ACL)
- .get_acl = zpl_get_acl,
-#elif defined(HAVE_CHECK_ACL)
- .check_acl = zpl_check_acl,
-#elif defined(HAVE_PERMISSION)
- .permission = zpl_permission,
-#endif /* HAVE_GET_ACL | HAVE_CHECK_ACL | HAVE_PERMISSION */
-#endif /* CONFIG_FS_POSIX_ACL */
-};
-
-dentry_operations_t zpl_dentry_operations = {
- .d_revalidate = zpl_revalidate,
-};
diff --git a/module/zfs/zpl_super.c b/module/zfs/zpl_super.c
deleted file mode 100644
index 810ab2898..000000000
--- a/module/zfs/zpl_super.c
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
- */
-
-
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/zfs_ctldir.h>
-#include <sys/zpl.h>
-
-
-static struct inode *
-zpl_inode_alloc(struct super_block *sb)
-{
- struct inode *ip;
-
- VERIFY3S(zfs_inode_alloc(sb, &ip), ==, 0);
- inode_set_iversion(ip, 1);
-
- return (ip);
-}
-
-static void
-zpl_inode_destroy(struct inode *ip)
-{
- ASSERT(atomic_read(&ip->i_count) == 0);
- zfs_inode_destroy(ip);
-}
-
-/*
- * Called from __mark_inode_dirty() to reflect that something in the
- * inode has changed. We use it to ensure the znode system attributes
- * are always strictly update to date with respect to the inode.
- */
-#ifdef HAVE_DIRTY_INODE_WITH_FLAGS
-static void
-zpl_dirty_inode(struct inode *ip, int flags)
-{
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- zfs_dirty_inode(ip, flags);
- spl_fstrans_unmark(cookie);
-}
-#else
-static void
-zpl_dirty_inode(struct inode *ip)
-{
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- zfs_dirty_inode(ip, 0);
- spl_fstrans_unmark(cookie);
-}
-#endif /* HAVE_DIRTY_INODE_WITH_FLAGS */
-
-/*
- * When ->drop_inode() is called its return value indicates if the
- * inode should be evicted from the inode cache. If the inode is
- * unhashed and has no links the default policy is to evict it
- * immediately.
- *
- * Prior to 2.6.36 this eviction was accomplished by the vfs calling
- * ->delete_inode(). It was ->delete_inode()'s responsibility to
- * truncate the inode pages and call clear_inode(). The call to
- * clear_inode() synchronously invalidates all the buffers and
- * calls ->clear_inode(). It was ->clear_inode()'s responsibility
- * to cleanup and filesystem specific data before freeing the inode.
- *
- * This elaborate mechanism was replaced by ->evict_inode() which
- * does the job of both ->delete_inode() and ->clear_inode(). It
- * will be called exactly once, and when it returns the inode must
- * be in a state where it can simply be freed.i
- *
- * The ->evict_inode() callback must minimally truncate the inode pages,
- * and call clear_inode(). For 2.6.35 and later kernels this will
- * simply update the inode state, with the sync occurring before the
- * truncate in evict(). For earlier kernels clear_inode() maps to
- * end_writeback() which is responsible for completing all outstanding
- * write back. In either case, once this is done it is safe to cleanup
- * any remaining inode specific data via zfs_inactive().
- * remaining filesystem specific data.
- */
-#ifdef HAVE_EVICT_INODE
-static void
-zpl_evict_inode(struct inode *ip)
-{
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- truncate_setsize(ip, 0);
- clear_inode(ip);
- zfs_inactive(ip);
- spl_fstrans_unmark(cookie);
-}
-
-#else
-
-static void
-zpl_drop_inode(struct inode *ip)
-{
- generic_delete_inode(ip);
-}
-
-static void
-zpl_clear_inode(struct inode *ip)
-{
- fstrans_cookie_t cookie;
-
- cookie = spl_fstrans_mark();
- zfs_inactive(ip);
- spl_fstrans_unmark(cookie);
-}
-
-static void
-zpl_inode_delete(struct inode *ip)
-{
- truncate_setsize(ip, 0);
- clear_inode(ip);
-}
-#endif /* HAVE_EVICT_INODE */
-
-static void
-zpl_put_super(struct super_block *sb)
-{
- fstrans_cookie_t cookie;
- int error;
-
- cookie = spl_fstrans_mark();
- error = -zfs_umount(sb);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
-}
-
-static int
-zpl_sync_fs(struct super_block *sb, int wait)
-{
- fstrans_cookie_t cookie;
- cred_t *cr = CRED();
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- error = -zfs_sync(sb, wait, cr);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_statfs(struct dentry *dentry, struct kstatfs *statp)
-{
- fstrans_cookie_t cookie;
- int error;
-
- cookie = spl_fstrans_mark();
- error = -zfs_statvfs(dentry, statp);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
-
- /*
- * If required by a 32-bit system call, dynamically scale the
- * block size up to 16MiB and decrease the block counts. This
- * allows for a maximum size of 64EiB to be reported. The file
- * counts must be artificially capped at 2^32-1.
- */
- if (unlikely(zpl_is_32bit_api())) {
- while (statp->f_blocks > UINT32_MAX &&
- statp->f_bsize < SPA_MAXBLOCKSIZE) {
- statp->f_frsize <<= 1;
- statp->f_bsize <<= 1;
-
- statp->f_blocks >>= 1;
- statp->f_bfree >>= 1;
- statp->f_bavail >>= 1;
- }
-
- uint64_t usedobjs = statp->f_files - statp->f_ffree;
- statp->f_ffree = MIN(statp->f_ffree, UINT32_MAX - usedobjs);
- statp->f_files = statp->f_ffree + usedobjs;
- }
-
- return (error);
-}
-
-static int
-zpl_remount_fs(struct super_block *sb, int *flags, char *data)
-{
- zfs_mnt_t zm = { .mnt_osname = NULL, .mnt_data = data };
- fstrans_cookie_t cookie;
- int error;
-
- cookie = spl_fstrans_mark();
- error = -zfs_remount(sb, flags, &zm);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-__zpl_show_options(struct seq_file *seq, zfsvfs_t *zfsvfs)
-{
- seq_printf(seq, ",%s",
- zfsvfs->z_flags & ZSB_XATTR ? "xattr" : "noxattr");
-
-#ifdef CONFIG_FS_POSIX_ACL
- switch (zfsvfs->z_acl_type) {
- case ZFS_ACLTYPE_POSIXACL:
- seq_puts(seq, ",posixacl");
- break;
- default:
- seq_puts(seq, ",noacl");
- break;
- }
-#endif /* CONFIG_FS_POSIX_ACL */
-
- return (0);
-}
-
-#ifdef HAVE_SHOW_OPTIONS_WITH_DENTRY
-static int
-zpl_show_options(struct seq_file *seq, struct dentry *root)
-{
- return (__zpl_show_options(seq, root->d_sb->s_fs_info));
-}
-#else
-static int
-zpl_show_options(struct seq_file *seq, struct vfsmount *vfsp)
-{
- return (__zpl_show_options(seq, vfsp->mnt_sb->s_fs_info));
-}
-#endif /* HAVE_SHOW_OPTIONS_WITH_DENTRY */
-
-static int
-zpl_fill_super(struct super_block *sb, void *data, int silent)
-{
- zfs_mnt_t *zm = (zfs_mnt_t *)data;
- fstrans_cookie_t cookie;
- int error;
-
- cookie = spl_fstrans_mark();
- error = -zfs_domount(sb, zm, silent);
- spl_fstrans_unmark(cookie);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_test_super(struct super_block *s, void *data)
-{
- zfsvfs_t *zfsvfs = s->s_fs_info;
- objset_t *os = data;
-
- if (zfsvfs == NULL)
- return (0);
-
- return (os == zfsvfs->z_os);
-}
-
-static struct super_block *
-zpl_mount_impl(struct file_system_type *fs_type, int flags, zfs_mnt_t *zm)
-{
- struct super_block *s;
- objset_t *os;
- int err;
-
- err = dmu_objset_hold(zm->mnt_osname, FTAG, &os);
- if (err)
- return (ERR_PTR(-err));
-
- /*
- * The dsl pool lock must be released prior to calling sget().
- * It is possible sget() may block on the lock in grab_super()
- * while deactivate_super() holds that same lock and waits for
- * a txg sync. If the dsl_pool lock is held over sget()
- * this can prevent the pool sync and cause a deadlock.
- */
- dsl_pool_rele(dmu_objset_pool(os), FTAG);
- s = zpl_sget(fs_type, zpl_test_super, set_anon_super, flags, os);
- dsl_dataset_rele(dmu_objset_ds(os), FTAG);
-
- if (IS_ERR(s))
- return (ERR_CAST(s));
-
- if (s->s_root == NULL) {
- err = zpl_fill_super(s, zm, flags & SB_SILENT ? 1 : 0);
- if (err) {
- deactivate_locked_super(s);
- return (ERR_PTR(err));
- }
- s->s_flags |= SB_ACTIVE;
- } else if ((flags ^ s->s_flags) & SB_RDONLY) {
- deactivate_locked_super(s);
- return (ERR_PTR(-EBUSY));
- }
-
- return (s);
-}
-
-#ifdef HAVE_FST_MOUNT
-static struct dentry *
-zpl_mount(struct file_system_type *fs_type, int flags,
- const char *osname, void *data)
-{
- zfs_mnt_t zm = { .mnt_osname = osname, .mnt_data = data };
-
- struct super_block *sb = zpl_mount_impl(fs_type, flags, &zm);
- if (IS_ERR(sb))
- return (ERR_CAST(sb));
-
- return (dget(sb->s_root));
-}
-#else
-static int
-zpl_get_sb(struct file_system_type *fs_type, int flags,
- const char *osname, void *data, struct vfsmount *mnt)
-{
- zfs_mnt_t zm = { .mnt_osname = osname, .mnt_data = data };
-
- struct super_block *sb = zpl_mount_impl(fs_type, flags, &zm);
- if (IS_ERR(sb))
- return (PTR_ERR(sb));
-
- (void) simple_set_mnt(mnt, sb);
-
- return (0);
-}
-#endif /* HAVE_FST_MOUNT */
-
-static void
-zpl_kill_sb(struct super_block *sb)
-{
- zfs_preumount(sb);
- kill_anon_super(sb);
-
-#ifdef HAVE_S_INSTANCES_LIST_HEAD
- sb->s_instances.next = &(zpl_fs_type.fs_supers);
-#endif /* HAVE_S_INSTANCES_LIST_HEAD */
-}
-
-void
-zpl_prune_sb(int64_t nr_to_scan, void *arg)
-{
- struct super_block *sb = (struct super_block *)arg;
- int objects = 0;
-
- (void) -zfs_prune(sb, nr_to_scan, &objects);
-}
-
-#ifdef HAVE_NR_CACHED_OBJECTS
-static int
-zpl_nr_cached_objects(struct super_block *sb)
-{
- return (0);
-}
-#endif /* HAVE_NR_CACHED_OBJECTS */
-
-#ifdef HAVE_FREE_CACHED_OBJECTS
-static void
-zpl_free_cached_objects(struct super_block *sb, int nr_to_scan)
-{
- /* noop */
-}
-#endif /* HAVE_FREE_CACHED_OBJECTS */
-
-const struct super_operations zpl_super_operations = {
- .alloc_inode = zpl_inode_alloc,
- .destroy_inode = zpl_inode_destroy,
- .dirty_inode = zpl_dirty_inode,
- .write_inode = NULL,
-#ifdef HAVE_EVICT_INODE
- .evict_inode = zpl_evict_inode,
-#else
- .drop_inode = zpl_drop_inode,
- .clear_inode = zpl_clear_inode,
- .delete_inode = zpl_inode_delete,
-#endif /* HAVE_EVICT_INODE */
- .put_super = zpl_put_super,
- .sync_fs = zpl_sync_fs,
- .statfs = zpl_statfs,
- .remount_fs = zpl_remount_fs,
- .show_options = zpl_show_options,
- .show_stats = NULL,
-#ifdef HAVE_NR_CACHED_OBJECTS
- .nr_cached_objects = zpl_nr_cached_objects,
-#endif /* HAVE_NR_CACHED_OBJECTS */
-#ifdef HAVE_FREE_CACHED_OBJECTS
- .free_cached_objects = zpl_free_cached_objects,
-#endif /* HAVE_FREE_CACHED_OBJECTS */
-};
-
-struct file_system_type zpl_fs_type = {
- .owner = THIS_MODULE,
- .name = ZFS_DRIVER,
-#ifdef HAVE_FST_MOUNT
- .mount = zpl_mount,
-#else
- .get_sb = zpl_get_sb,
-#endif /* HAVE_FST_MOUNT */
- .kill_sb = zpl_kill_sb,
-};
diff --git a/module/zfs/zpl_xattr.c b/module/zfs/zpl_xattr.c
deleted file mode 100644
index 95523f28e..000000000
--- a/module/zfs/zpl_xattr.c
+++ /dev/null
@@ -1,1548 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
- *
- * Extended attributes (xattr) on Solaris are implemented as files
- * which exist in a hidden xattr directory. These extended attributes
- * can be accessed using the attropen() system call which opens
- * the extended attribute. It can then be manipulated just like
- * a standard file descriptor. This has a couple advantages such
- * as practically no size limit on the file, and the extended
- * attributes permissions may differ from those of the parent file.
- * This interface is really quite clever, but it's also completely
- * different than what is supported on Linux. It also comes with a
- * steep performance penalty when accessing small xattrs because they
- * are not stored with the parent file.
- *
- * Under Linux extended attributes are manipulated by the system
- * calls getxattr(2), setxattr(2), and listxattr(2). They consider
- * extended attributes to be name/value pairs where the name is a
- * NULL terminated string. The name must also include one of the
- * following namespace prefixes:
- *
- * user - No restrictions and is available to user applications.
- * trusted - Restricted to kernel and root (CAP_SYS_ADMIN) use.
- * system - Used for access control lists (system.nfs4_acl, etc).
- * security - Used by SELinux to store a files security context.
- *
- * The value under Linux to limited to 65536 bytes of binary data.
- * In practice, individual xattrs tend to be much smaller than this
- * and are typically less than 100 bytes. A good example of this
- * are the security.selinux xattrs which are less than 100 bytes and
- * exist for every file when xattr labeling is enabled.
- *
- * The Linux xattr implementation has been written to take advantage of
- * this typical usage. When the dataset property 'xattr=sa' is set,
- * then xattrs will be preferentially stored as System Attributes (SA).
- * This allows tiny xattrs (~100 bytes) to be stored with the dnode and
- * up to 64k of xattrs to be stored in the spill block. If additional
- * xattr space is required, which is unlikely under Linux, they will
- * be stored using the traditional directory approach.
- *
- * This optimization results in roughly a 3x performance improvement
- * when accessing xattrs because it avoids the need to perform a seek
- * for every xattr value. When multiple xattrs are stored per-file
- * the performance improvements are even greater because all of the
- * xattrs stored in the spill block will be cached.
- *
- * However, by default SA based xattrs are disabled in the Linux port
- * to maximize compatibility with other implementations. If you do
- * enable SA based xattrs then they will not be visible on platforms
- * which do not support this feature.
- *
- * NOTE: One additional consequence of the xattr directory implementation
- * is that when an extended attribute is manipulated an inode is created.
- * This inode will exist in the Linux inode cache but there will be no
- * associated entry in the dentry cache which references it. This is
- * safe but it may result in some confusion. Enabling SA based xattrs
- * largely avoids the issue except in the overflow case.
- */
-
-#include <sys/zfs_vfsops.h>
-#include <sys/zfs_vnops.h>
-#include <sys/zfs_znode.h>
-#include <sys/zap.h>
-#include <sys/vfs.h>
-#include <sys/zpl.h>
-
-typedef struct xattr_filldir {
- size_t size;
- size_t offset;
- char *buf;
- struct dentry *dentry;
-} xattr_filldir_t;
-
-static const struct xattr_handler *zpl_xattr_handler(const char *);
-
-static int
-zpl_xattr_permission(xattr_filldir_t *xf, const char *name, int name_len)
-{
- static const struct xattr_handler *handler;
- struct dentry *d = xf->dentry;
-
- handler = zpl_xattr_handler(name);
- if (!handler)
- return (0);
-
- if (handler->list) {
-#if defined(HAVE_XATTR_LIST_SIMPLE)
- if (!handler->list(d))
- return (0);
-#elif defined(HAVE_XATTR_LIST_DENTRY)
- if (!handler->list(d, NULL, 0, name, name_len, 0))
- return (0);
-#elif defined(HAVE_XATTR_LIST_HANDLER)
- if (!handler->list(handler, d, NULL, 0, name, name_len))
- return (0);
-#elif defined(HAVE_XATTR_LIST_INODE)
- if (!handler->list(d->d_inode, NULL, 0, name, name_len))
- return (0);
-#endif
- }
-
- return (1);
-}
-
-/*
- * Determine is a given xattr name should be visible and if so copy it
- * in to the provided buffer (xf->buf).
- */
-static int
-zpl_xattr_filldir(xattr_filldir_t *xf, const char *name, int name_len)
-{
- /* Check permissions using the per-namespace list xattr handler. */
- if (!zpl_xattr_permission(xf, name, name_len))
- return (0);
-
- /* When xf->buf is NULL only calculate the required size. */
- if (xf->buf) {
- if (xf->offset + name_len + 1 > xf->size)
- return (-ERANGE);
-
- memcpy(xf->buf + xf->offset, name, name_len);
- xf->buf[xf->offset + name_len] = '\0';
- }
-
- xf->offset += (name_len + 1);
-
- return (0);
-}
-
-/*
- * Read as many directory entry names as will fit in to the provided buffer,
- * or when no buffer is provided calculate the required buffer size.
- */
-int
-zpl_xattr_readdir(struct inode *dxip, xattr_filldir_t *xf)
-{
- zap_cursor_t zc;
- zap_attribute_t zap;
- int error;
-
- zap_cursor_init(&zc, ITOZSB(dxip)->z_os, ITOZ(dxip)->z_id);
-
- while ((error = -zap_cursor_retrieve(&zc, &zap)) == 0) {
-
- if (zap.za_integer_length != 8 || zap.za_num_integers != 1) {
- error = -ENXIO;
- break;
- }
-
- error = zpl_xattr_filldir(xf, zap.za_name, strlen(zap.za_name));
- if (error)
- break;
-
- zap_cursor_advance(&zc);
- }
-
- zap_cursor_fini(&zc);
-
- if (error == -ENOENT)
- error = 0;
-
- return (error);
-}
-
-static ssize_t
-zpl_xattr_list_dir(xattr_filldir_t *xf, cred_t *cr)
-{
- struct inode *ip = xf->dentry->d_inode;
- struct inode *dxip = NULL;
- int error;
-
- /* Lookup the xattr directory */
- error = -zfs_lookup(ip, NULL, &dxip, LOOKUP_XATTR, cr, NULL, NULL);
- if (error) {
- if (error == -ENOENT)
- error = 0;
-
- return (error);
- }
-
- error = zpl_xattr_readdir(dxip, xf);
- iput(dxip);
-
- return (error);
-}
-
-static ssize_t
-zpl_xattr_list_sa(xattr_filldir_t *xf)
-{
- znode_t *zp = ITOZ(xf->dentry->d_inode);
- nvpair_t *nvp = NULL;
- int error = 0;
-
- mutex_enter(&zp->z_lock);
- if (zp->z_xattr_cached == NULL)
- error = -zfs_sa_get_xattr(zp);
- mutex_exit(&zp->z_lock);
-
- if (error)
- return (error);
-
- ASSERT(zp->z_xattr_cached);
-
- while ((nvp = nvlist_next_nvpair(zp->z_xattr_cached, nvp)) != NULL) {
- ASSERT3U(nvpair_type(nvp), ==, DATA_TYPE_BYTE_ARRAY);
-
- error = zpl_xattr_filldir(xf, nvpair_name(nvp),
- strlen(nvpair_name(nvp)));
- if (error)
- return (error);
- }
-
- return (0);
-}
-
-ssize_t
-zpl_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size)
-{
- znode_t *zp = ITOZ(dentry->d_inode);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- xattr_filldir_t xf = { buffer_size, 0, buffer, dentry };
- cred_t *cr = CRED();
- fstrans_cookie_t cookie;
- int error = 0;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- ZPL_ENTER(zfsvfs);
- ZPL_VERIFY_ZP(zp);
- rw_enter(&zp->z_xattr_lock, RW_READER);
-
- if (zfsvfs->z_use_sa && zp->z_is_sa) {
- error = zpl_xattr_list_sa(&xf);
- if (error)
- goto out;
- }
-
- error = zpl_xattr_list_dir(&xf, cr);
- if (error)
- goto out;
-
- error = xf.offset;
-out:
-
- rw_exit(&zp->z_xattr_lock);
- ZPL_EXIT(zfsvfs);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- return (error);
-}
-
-static int
-zpl_xattr_get_dir(struct inode *ip, const char *name, void *value,
- size_t size, cred_t *cr)
-{
- struct inode *dxip = NULL;
- struct inode *xip = NULL;
- loff_t pos = 0;
- int error;
-
- /* Lookup the xattr directory */
- error = -zfs_lookup(ip, NULL, &dxip, LOOKUP_XATTR, cr, NULL, NULL);
- if (error)
- goto out;
-
- /* Lookup a specific xattr name in the directory */
- error = -zfs_lookup(dxip, (char *)name, &xip, 0, cr, NULL, NULL);
- if (error)
- goto out;
-
- if (!size) {
- error = i_size_read(xip);
- goto out;
- }
-
- if (size < i_size_read(xip)) {
- error = -ERANGE;
- goto out;
- }
-
- error = zpl_read_common(xip, value, size, &pos, UIO_SYSSPACE, 0, cr);
-out:
- if (xip)
- iput(xip);
-
- if (dxip)
- iput(dxip);
-
- return (error);
-}
-
-static int
-zpl_xattr_get_sa(struct inode *ip, const char *name, void *value, size_t size)
-{
- znode_t *zp = ITOZ(ip);
- uchar_t *nv_value;
- uint_t nv_size;
- int error = 0;
-
- ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));
-
- mutex_enter(&zp->z_lock);
- if (zp->z_xattr_cached == NULL)
- error = -zfs_sa_get_xattr(zp);
- mutex_exit(&zp->z_lock);
-
- if (error)
- return (error);
-
- ASSERT(zp->z_xattr_cached);
- error = -nvlist_lookup_byte_array(zp->z_xattr_cached, name,
- &nv_value, &nv_size);
- if (error)
- return (error);
-
- if (size == 0 || value == NULL)
- return (nv_size);
-
- if (size < nv_size)
- return (-ERANGE);
-
- memcpy(value, nv_value, nv_size);
-
- return (nv_size);
-}
-
-static int
-__zpl_xattr_get(struct inode *ip, const char *name, void *value, size_t size,
- cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- int error;
-
- ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));
-
- if (zfsvfs->z_use_sa && zp->z_is_sa) {
- error = zpl_xattr_get_sa(ip, name, value, size);
- if (error != -ENOENT)
- goto out;
- }
-
- error = zpl_xattr_get_dir(ip, name, value, size, cr);
-out:
- if (error == -ENOENT)
- error = -ENODATA;
-
- return (error);
-}
-
-#define XATTR_NOENT 0x0
-#define XATTR_IN_SA 0x1
-#define XATTR_IN_DIR 0x2
-/* check where the xattr resides */
-static int
-__zpl_xattr_where(struct inode *ip, const char *name, int *where, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- int error;
-
- ASSERT(where);
- ASSERT(RW_LOCK_HELD(&zp->z_xattr_lock));
-
- *where = XATTR_NOENT;
- if (zfsvfs->z_use_sa && zp->z_is_sa) {
- error = zpl_xattr_get_sa(ip, name, NULL, 0);
- if (error >= 0)
- *where |= XATTR_IN_SA;
- else if (error != -ENOENT)
- return (error);
- }
-
- error = zpl_xattr_get_dir(ip, name, NULL, 0, cr);
- if (error >= 0)
- *where |= XATTR_IN_DIR;
- else if (error != -ENOENT)
- return (error);
-
- if (*where == (XATTR_IN_SA|XATTR_IN_DIR))
- cmn_err(CE_WARN, "ZFS: inode %p has xattr \"%s\""
- " in both SA and dir", ip, name);
- if (*where == XATTR_NOENT)
- error = -ENODATA;
- else
- error = 0;
- return (error);
-}
-
-static int
-zpl_xattr_get(struct inode *ip, const char *name, void *value, size_t size)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- cred_t *cr = CRED();
- fstrans_cookie_t cookie;
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- ZPL_ENTER(zfsvfs);
- ZPL_VERIFY_ZP(zp);
- rw_enter(&zp->z_xattr_lock, RW_READER);
- error = __zpl_xattr_get(ip, name, value, size, cr);
- rw_exit(&zp->z_xattr_lock);
- ZPL_EXIT(zfsvfs);
- spl_fstrans_unmark(cookie);
- crfree(cr);
-
- return (error);
-}
-
-static int
-zpl_xattr_set_dir(struct inode *ip, const char *name, const void *value,
- size_t size, int flags, cred_t *cr)
-{
- struct inode *dxip = NULL;
- struct inode *xip = NULL;
- vattr_t *vap = NULL;
- ssize_t wrote;
- int lookup_flags, error;
- const int xattr_mode = S_IFREG | 0644;
- loff_t pos = 0;
-
- /*
- * Lookup the xattr directory. When we're adding an entry pass
- * CREATE_XATTR_DIR to ensure the xattr directory is created.
- * When removing an entry this flag is not passed to avoid
- * unnecessarily creating a new xattr directory.
- */
- lookup_flags = LOOKUP_XATTR;
- if (value != NULL)
- lookup_flags |= CREATE_XATTR_DIR;
-
- error = -zfs_lookup(ip, NULL, &dxip, lookup_flags, cr, NULL, NULL);
- if (error)
- goto out;
-
- /* Lookup a specific xattr name in the directory */
- error = -zfs_lookup(dxip, (char *)name, &xip, 0, cr, NULL, NULL);
- if (error && (error != -ENOENT))
- goto out;
-
- error = 0;
-
- /* Remove a specific name xattr when value is set to NULL. */
- if (value == NULL) {
- if (xip)
- error = -zfs_remove(dxip, (char *)name, cr, 0);
-
- goto out;
- }
-
- /* Lookup failed create a new xattr. */
- if (xip == NULL) {
- vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
- vap->va_mode = xattr_mode;
- vap->va_mask = ATTR_MODE;
- vap->va_uid = crgetfsuid(cr);
- vap->va_gid = crgetfsgid(cr);
-
- error = -zfs_create(dxip, (char *)name, vap, 0, 0644, &xip,
- cr, 0, NULL);
- if (error)
- goto out;
- }
-
- ASSERT(xip != NULL);
-
- error = -zfs_freesp(ITOZ(xip), 0, 0, xattr_mode, TRUE);
- if (error)
- goto out;
-
- wrote = zpl_write_common(xip, value, size, &pos, UIO_SYSSPACE, 0, cr);
- if (wrote < 0)
- error = wrote;
-
-out:
-
- if (error == 0) {
- ip->i_ctime = current_time(ip);
- zfs_mark_inode_dirty(ip);
- }
-
- if (vap)
- kmem_free(vap, sizeof (vattr_t));
-
- if (xip)
- iput(xip);
-
- if (dxip)
- iput(dxip);
-
- if (error == -ENOENT)
- error = -ENODATA;
-
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_xattr_set_sa(struct inode *ip, const char *name, const void *value,
- size_t size, int flags, cred_t *cr)
-{
- znode_t *zp = ITOZ(ip);
- nvlist_t *nvl;
- size_t sa_size;
- int error = 0;
-
- mutex_enter(&zp->z_lock);
- if (zp->z_xattr_cached == NULL)
- error = -zfs_sa_get_xattr(zp);
- mutex_exit(&zp->z_lock);
-
- if (error)
- return (error);
-
- ASSERT(zp->z_xattr_cached);
- nvl = zp->z_xattr_cached;
-
- if (value == NULL) {
- error = -nvlist_remove(nvl, name, DATA_TYPE_BYTE_ARRAY);
- if (error == -ENOENT)
- error = zpl_xattr_set_dir(ip, name, NULL, 0, flags, cr);
- } else {
- /* Limited to 32k to keep nvpair memory allocations small */
- if (size > DXATTR_MAX_ENTRY_SIZE)
- return (-EFBIG);
-
- /* Prevent the DXATTR SA from consuming the entire SA region */
- error = -nvlist_size(nvl, &sa_size, NV_ENCODE_XDR);
- if (error)
- return (error);
-
- if (sa_size > DXATTR_MAX_SA_SIZE)
- return (-EFBIG);
-
- error = -nvlist_add_byte_array(nvl, name,
- (uchar_t *)value, size);
- }
-
- /*
- * Update the SA for additions, modifications, and removals. On
- * error drop the inconsistent cached version of the nvlist, it
- * will be reconstructed from the ARC when next accessed.
- */
- if (error == 0)
- error = -zfs_sa_set_xattr(zp);
-
- if (error) {
- nvlist_free(nvl);
- zp->z_xattr_cached = NULL;
- }
-
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-static int
-zpl_xattr_set(struct inode *ip, const char *name, const void *value,
- size_t size, int flags)
-{
- znode_t *zp = ITOZ(ip);
- zfsvfs_t *zfsvfs = ZTOZSB(zp);
- cred_t *cr = CRED();
- fstrans_cookie_t cookie;
- int where;
- int error;
-
- crhold(cr);
- cookie = spl_fstrans_mark();
- ZPL_ENTER(zfsvfs);
- ZPL_VERIFY_ZP(zp);
- rw_enter(&ITOZ(ip)->z_xattr_lock, RW_WRITER);
-
- /*
- * Before setting the xattr check to see if it already exists.
- * This is done to ensure the following optional flags are honored.
- *
- * XATTR_CREATE: fail if xattr already exists
- * XATTR_REPLACE: fail if xattr does not exist
- *
- * We also want to know if it resides in sa or dir, so we can make
- * sure we don't end up with duplicate in both places.
- */
- error = __zpl_xattr_where(ip, name, &where, cr);
- if (error < 0) {
- if (error != -ENODATA)
- goto out;
- if (flags & XATTR_REPLACE)
- goto out;
-
- /* The xattr to be removed already doesn't exist */
- error = 0;
- if (value == NULL)
- goto out;
- } else {
- error = -EEXIST;
- if (flags & XATTR_CREATE)
- goto out;
- }
-
- /* Preferentially store the xattr as a SA for better performance */
- if (zfsvfs->z_use_sa && zp->z_is_sa &&
- (zfsvfs->z_xattr_sa || (value == NULL && where & XATTR_IN_SA))) {
- error = zpl_xattr_set_sa(ip, name, value, size, flags, cr);
- if (error == 0) {
- /*
- * Successfully put into SA, we need to clear the one
- * in dir.
- */
- if (where & XATTR_IN_DIR)
- zpl_xattr_set_dir(ip, name, NULL, 0, 0, cr);
- goto out;
- }
- }
-
- error = zpl_xattr_set_dir(ip, name, value, size, flags, cr);
- /*
- * Successfully put into dir, we need to clear the one in SA.
- */
- if (error == 0 && (where & XATTR_IN_SA))
- zpl_xattr_set_sa(ip, name, NULL, 0, 0, cr);
-out:
- rw_exit(&ITOZ(ip)->z_xattr_lock);
- ZPL_EXIT(zfsvfs);
- spl_fstrans_unmark(cookie);
- crfree(cr);
- ASSERT3S(error, <=, 0);
-
- return (error);
-}
-
-/*
- * Extended user attributes
- *
- * "Extended user attributes may be assigned to files and directories for
- * storing arbitrary additional information such as the mime type,
- * character set or encoding of a file. The access permissions for user
- * attributes are defined by the file permission bits: read permission
- * is required to retrieve the attribute value, and writer permission is
- * required to change it.
- *
- * The file permission bits of regular files and directories are
- * interpreted differently from the file permission bits of special
- * files and symbolic links. For regular files and directories the file
- * permission bits define access to the file's contents, while for
- * device special files they define access to the device described by
- * the special file. The file permissions of symbolic links are not
- * used in access checks. These differences would allow users to
- * consume filesystem resources in a way not controllable by disk quotas
- * for group or world writable special files and directories.
- *
- * For this reason, extended user attributes are allowed only for
- * regular files and directories, and access to extended user attributes
- * is restricted to the owner and to users with appropriate capabilities
- * for directories with the sticky bit set (see the chmod(1) manual page
- * for an explanation of the sticky bit)." - xattr(7)
- *
- * ZFS allows extended user attributes to be disabled administratively
- * by setting the 'xattr=off' property on the dataset.
- */
-static int
-__zpl_xattr_user_list(struct inode *ip, char *list, size_t list_size,
- const char *name, size_t name_len)
-{
- return (ITOZSB(ip)->z_flags & ZSB_XATTR);
-}
-ZPL_XATTR_LIST_WRAPPER(zpl_xattr_user_list);
-
-static int
-__zpl_xattr_user_get(struct inode *ip, const char *name,
- void *value, size_t size)
-{
- char *xattr_name;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- if (!(ITOZSB(ip)->z_flags & ZSB_XATTR))
- return (-EOPNOTSUPP);
-
- xattr_name = kmem_asprintf("%s%s", XATTR_USER_PREFIX, name);
- error = zpl_xattr_get(ip, xattr_name, value, size);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_GET_WRAPPER(zpl_xattr_user_get);
-
-static int
-__zpl_xattr_user_set(struct inode *ip, const char *name,
- const void *value, size_t size, int flags)
-{
- char *xattr_name;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- if (!(ITOZSB(ip)->z_flags & ZSB_XATTR))
- return (-EOPNOTSUPP);
-
- xattr_name = kmem_asprintf("%s%s", XATTR_USER_PREFIX, name);
- error = zpl_xattr_set(ip, xattr_name, value, size, flags);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_SET_WRAPPER(zpl_xattr_user_set);
-
-xattr_handler_t zpl_xattr_user_handler =
-{
- .prefix = XATTR_USER_PREFIX,
- .list = zpl_xattr_user_list,
- .get = zpl_xattr_user_get,
- .set = zpl_xattr_user_set,
-};
-
-/*
- * Trusted extended attributes
- *
- * "Trusted extended attributes are visible and accessible only to
- * processes that have the CAP_SYS_ADMIN capability. Attributes in this
- * class are used to implement mechanisms in user space (i.e., outside
- * the kernel) which keep information in extended attributes to which
- * ordinary processes should not have access." - xattr(7)
- */
-static int
-__zpl_xattr_trusted_list(struct inode *ip, char *list, size_t list_size,
- const char *name, size_t name_len)
-{
- return (capable(CAP_SYS_ADMIN));
-}
-ZPL_XATTR_LIST_WRAPPER(zpl_xattr_trusted_list);
-
-static int
-__zpl_xattr_trusted_get(struct inode *ip, const char *name,
- void *value, size_t size)
-{
- char *xattr_name;
- int error;
-
- if (!capable(CAP_SYS_ADMIN))
- return (-EACCES);
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- xattr_name = kmem_asprintf("%s%s", XATTR_TRUSTED_PREFIX, name);
- error = zpl_xattr_get(ip, xattr_name, value, size);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_GET_WRAPPER(zpl_xattr_trusted_get);
-
-static int
-__zpl_xattr_trusted_set(struct inode *ip, const char *name,
- const void *value, size_t size, int flags)
-{
- char *xattr_name;
- int error;
-
- if (!capable(CAP_SYS_ADMIN))
- return (-EACCES);
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- xattr_name = kmem_asprintf("%s%s", XATTR_TRUSTED_PREFIX, name);
- error = zpl_xattr_set(ip, xattr_name, value, size, flags);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_SET_WRAPPER(zpl_xattr_trusted_set);
-
-xattr_handler_t zpl_xattr_trusted_handler =
-{
- .prefix = XATTR_TRUSTED_PREFIX,
- .list = zpl_xattr_trusted_list,
- .get = zpl_xattr_trusted_get,
- .set = zpl_xattr_trusted_set,
-};
-
-/*
- * Extended security attributes
- *
- * "The security attribute namespace is used by kernel security modules,
- * such as Security Enhanced Linux, and also to implement file
- * capabilities (see capabilities(7)). Read and write access
- * permissions to security attributes depend on the policy implemented
- * for each security attribute by the security module. When no security
- * module is loaded, all processes have read access to extended security
- * attributes, and write access is limited to processes that have the
- * CAP_SYS_ADMIN capability." - xattr(7)
- */
-static int
-__zpl_xattr_security_list(struct inode *ip, char *list, size_t list_size,
- const char *name, size_t name_len)
-{
- return (1);
-}
-ZPL_XATTR_LIST_WRAPPER(zpl_xattr_security_list);
-
-static int
-__zpl_xattr_security_get(struct inode *ip, const char *name,
- void *value, size_t size)
-{
- char *xattr_name;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- xattr_name = kmem_asprintf("%s%s", XATTR_SECURITY_PREFIX, name);
- error = zpl_xattr_get(ip, xattr_name, value, size);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_GET_WRAPPER(zpl_xattr_security_get);
-
-static int
-__zpl_xattr_security_set(struct inode *ip, const char *name,
- const void *value, size_t size, int flags)
-{
- char *xattr_name;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") == 0)
- return (-EINVAL);
-#endif
- xattr_name = kmem_asprintf("%s%s", XATTR_SECURITY_PREFIX, name);
- error = zpl_xattr_set(ip, xattr_name, value, size, flags);
- strfree(xattr_name);
-
- return (error);
-}
-ZPL_XATTR_SET_WRAPPER(zpl_xattr_security_set);
-
-#ifdef HAVE_CALLBACK_SECURITY_INODE_INIT_SECURITY
-static int
-__zpl_xattr_security_init(struct inode *ip, const struct xattr *xattrs,
- void *fs_info)
-{
- const struct xattr *xattr;
- int error = 0;
-
- for (xattr = xattrs; xattr->name != NULL; xattr++) {
- error = __zpl_xattr_security_set(ip,
- xattr->name, xattr->value, xattr->value_len, 0);
-
- if (error < 0)
- break;
- }
-
- return (error);
-}
-
-int
-zpl_xattr_security_init(struct inode *ip, struct inode *dip,
- const struct qstr *qstr)
-{
- return security_inode_init_security(ip, dip, qstr,
- &__zpl_xattr_security_init, NULL);
-}
-
-#else
-int
-zpl_xattr_security_init(struct inode *ip, struct inode *dip,
- const struct qstr *qstr)
-{
- int error;
- size_t len;
- void *value;
- char *name;
-
- error = zpl_security_inode_init_security(ip, dip, qstr,
- &name, &value, &len);
- if (error) {
- if (error == -EOPNOTSUPP)
- return (0);
-
- return (error);
- }
-
- error = __zpl_xattr_security_set(ip, name, value, len, 0);
-
- kfree(name);
- kfree(value);
-
- return (error);
-}
-#endif /* HAVE_CALLBACK_SECURITY_INODE_INIT_SECURITY */
-
-/*
- * Security xattr namespace handlers.
- */
-xattr_handler_t zpl_xattr_security_handler = {
- .prefix = XATTR_SECURITY_PREFIX,
- .list = zpl_xattr_security_list,
- .get = zpl_xattr_security_get,
- .set = zpl_xattr_security_set,
-};
-
-/*
- * Extended system attributes
- *
- * "Extended system attributes are used by the kernel to store system
- * objects such as Access Control Lists. Read and write access permissions
- * to system attributes depend on the policy implemented for each system
- * attribute implemented by filesystems in the kernel." - xattr(7)
- */
-#ifdef CONFIG_FS_POSIX_ACL
-int
-zpl_set_acl(struct inode *ip, struct posix_acl *acl, int type)
-{
- char *name, *value = NULL;
- int error = 0;
- size_t size = 0;
-
- if (S_ISLNK(ip->i_mode))
- return (-EOPNOTSUPP);
-
- switch (type) {
- case ACL_TYPE_ACCESS:
- name = XATTR_NAME_POSIX_ACL_ACCESS;
- if (acl) {
- zpl_equivmode_t mode = ip->i_mode;
- error = posix_acl_equiv_mode(acl, &mode);
- if (error < 0) {
- return (error);
- } else {
- /*
- * The mode bits will have been set by
- * ->zfs_setattr()->zfs_acl_chmod_setattr()
- * using the ZFS ACL conversion. If they
- * differ from the Posix ACL conversion dirty
- * the inode to write the Posix mode bits.
- */
- if (ip->i_mode != mode) {
- ip->i_mode = mode;
- ip->i_ctime = current_time(ip);
- zfs_mark_inode_dirty(ip);
- }
-
- if (error == 0)
- acl = NULL;
- }
- }
- break;
-
- case ACL_TYPE_DEFAULT:
- name = XATTR_NAME_POSIX_ACL_DEFAULT;
- if (!S_ISDIR(ip->i_mode))
- return (acl ? -EACCES : 0);
- break;
-
- default:
- return (-EINVAL);
- }
-
- if (acl) {
- size = posix_acl_xattr_size(acl->a_count);
- value = kmem_alloc(size, KM_SLEEP);
-
- error = zpl_acl_to_xattr(acl, value, size);
- if (error < 0) {
- kmem_free(value, size);
- return (error);
- }
- }
-
- error = zpl_xattr_set(ip, name, value, size, 0);
- if (value)
- kmem_free(value, size);
-
- if (!error) {
- if (acl)
- zpl_set_cached_acl(ip, type, acl);
- else
- zpl_forget_cached_acl(ip, type);
- }
-
- return (error);
-}
-
-struct posix_acl *
-zpl_get_acl(struct inode *ip, int type)
-{
- struct posix_acl *acl;
- void *value = NULL;
- char *name;
- int size;
-
- /*
- * As of Linux 3.14, the kernel get_acl will check this for us.
- * Also as of Linux 4.7, comparing against ACL_NOT_CACHED is wrong
- * as the kernel get_acl will set it to temporary sentinel value.
- */
-#ifndef HAVE_KERNEL_GET_ACL_HANDLE_CACHE
- acl = get_cached_acl(ip, type);
- if (acl != ACL_NOT_CACHED)
- return (acl);
-#endif
-
- switch (type) {
- case ACL_TYPE_ACCESS:
- name = XATTR_NAME_POSIX_ACL_ACCESS;
- break;
- case ACL_TYPE_DEFAULT:
- name = XATTR_NAME_POSIX_ACL_DEFAULT;
- break;
- default:
- return (ERR_PTR(-EINVAL));
- }
-
- size = zpl_xattr_get(ip, name, NULL, 0);
- if (size > 0) {
- value = kmem_alloc(size, KM_SLEEP);
- size = zpl_xattr_get(ip, name, value, size);
- }
-
- if (size > 0) {
- acl = zpl_acl_from_xattr(value, size);
- } else if (size == -ENODATA || size == -ENOSYS) {
- acl = NULL;
- } else {
- acl = ERR_PTR(-EIO);
- }
-
- if (size > 0)
- kmem_free(value, size);
-
- /* As of Linux 4.7, the kernel get_acl will set this for us */
-#ifndef HAVE_KERNEL_GET_ACL_HANDLE_CACHE
- if (!IS_ERR(acl))
- zpl_set_cached_acl(ip, type, acl);
-#endif
-
- return (acl);
-}
-
-#if !defined(HAVE_GET_ACL)
-static int
-__zpl_check_acl(struct inode *ip, int mask)
-{
- struct posix_acl *acl;
- int error;
-
- acl = zpl_get_acl(ip, ACL_TYPE_ACCESS);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
-
- if (acl) {
- error = posix_acl_permission(ip, acl, mask);
- zpl_posix_acl_release(acl);
- return (error);
- }
-
- return (-EAGAIN);
-}
-
-#if defined(HAVE_CHECK_ACL_WITH_FLAGS)
-int
-zpl_check_acl(struct inode *ip, int mask, unsigned int flags)
-{
- return (__zpl_check_acl(ip, mask));
-}
-#elif defined(HAVE_CHECK_ACL)
-int
-zpl_check_acl(struct inode *ip, int mask)
-{
- return (__zpl_check_acl(ip, mask));
-}
-#elif defined(HAVE_PERMISSION_WITH_NAMEIDATA)
-int
-zpl_permission(struct inode *ip, int mask, struct nameidata *nd)
-{
- return (generic_permission(ip, mask, __zpl_check_acl));
-}
-#elif defined(HAVE_PERMISSION)
-int
-zpl_permission(struct inode *ip, int mask)
-{
- return (generic_permission(ip, mask, __zpl_check_acl));
-}
-#endif /* HAVE_CHECK_ACL | HAVE_PERMISSION */
-#endif /* !HAVE_GET_ACL */
-
-int
-zpl_init_acl(struct inode *ip, struct inode *dir)
-{
- struct posix_acl *acl = NULL;
- int error = 0;
-
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (0);
-
- if (!S_ISLNK(ip->i_mode)) {
- acl = zpl_get_acl(dir, ACL_TYPE_DEFAULT);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
- if (!acl) {
- ip->i_mode &= ~current_umask();
- ip->i_ctime = current_time(ip);
- zfs_mark_inode_dirty(ip);
- return (0);
- }
- }
-
- if (acl) {
- umode_t mode;
-
- if (S_ISDIR(ip->i_mode)) {
- error = zpl_set_acl(ip, acl, ACL_TYPE_DEFAULT);
- if (error)
- goto out;
- }
-
- mode = ip->i_mode;
- error = __posix_acl_create(&acl, GFP_KERNEL, &mode);
- if (error >= 0) {
- ip->i_mode = mode;
- zfs_mark_inode_dirty(ip);
- if (error > 0)
- error = zpl_set_acl(ip, acl, ACL_TYPE_ACCESS);
- }
- }
-out:
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-
-int
-zpl_chmod_acl(struct inode *ip)
-{
- struct posix_acl *acl;
- int error;
-
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (0);
-
- if (S_ISLNK(ip->i_mode))
- return (-EOPNOTSUPP);
-
- acl = zpl_get_acl(ip, ACL_TYPE_ACCESS);
- if (IS_ERR(acl) || !acl)
- return (PTR_ERR(acl));
-
- error = __posix_acl_chmod(&acl, GFP_KERNEL, ip->i_mode);
- if (!error)
- error = zpl_set_acl(ip, acl, ACL_TYPE_ACCESS);
-
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-
-static int
-__zpl_xattr_acl_list_access(struct inode *ip, char *list, size_t list_size,
- const char *name, size_t name_len)
-{
- char *xattr_name = XATTR_NAME_POSIX_ACL_ACCESS;
- size_t xattr_size = sizeof (XATTR_NAME_POSIX_ACL_ACCESS);
-
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (0);
-
- if (list && xattr_size <= list_size)
- memcpy(list, xattr_name, xattr_size);
-
- return (xattr_size);
-}
-ZPL_XATTR_LIST_WRAPPER(zpl_xattr_acl_list_access);
-
-static int
-__zpl_xattr_acl_list_default(struct inode *ip, char *list, size_t list_size,
- const char *name, size_t name_len)
-{
- char *xattr_name = XATTR_NAME_POSIX_ACL_DEFAULT;
- size_t xattr_size = sizeof (XATTR_NAME_POSIX_ACL_DEFAULT);
-
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (0);
-
- if (list && xattr_size <= list_size)
- memcpy(list, xattr_name, xattr_size);
-
- return (xattr_size);
-}
-ZPL_XATTR_LIST_WRAPPER(zpl_xattr_acl_list_default);
-
-static int
-__zpl_xattr_acl_get_access(struct inode *ip, const char *name,
- void *buffer, size_t size)
-{
- struct posix_acl *acl;
- int type = ACL_TYPE_ACCESS;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") != 0)
- return (-EINVAL);
-#endif
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (-EOPNOTSUPP);
-
- acl = zpl_get_acl(ip, type);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
- if (acl == NULL)
- return (-ENODATA);
-
- error = zpl_acl_to_xattr(acl, buffer, size);
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-ZPL_XATTR_GET_WRAPPER(zpl_xattr_acl_get_access);
-
-static int
-__zpl_xattr_acl_get_default(struct inode *ip, const char *name,
- void *buffer, size_t size)
-{
- struct posix_acl *acl;
- int type = ACL_TYPE_DEFAULT;
- int error;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") != 0)
- return (-EINVAL);
-#endif
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (-EOPNOTSUPP);
-
- acl = zpl_get_acl(ip, type);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
- if (acl == NULL)
- return (-ENODATA);
-
- error = zpl_acl_to_xattr(acl, buffer, size);
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-ZPL_XATTR_GET_WRAPPER(zpl_xattr_acl_get_default);
-
-static int
-__zpl_xattr_acl_set_access(struct inode *ip, const char *name,
- const void *value, size_t size, int flags)
-{
- struct posix_acl *acl;
- int type = ACL_TYPE_ACCESS;
- int error = 0;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") != 0)
- return (-EINVAL);
-#endif
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (-EOPNOTSUPP);
-
- if (!zpl_inode_owner_or_capable(ip))
- return (-EPERM);
-
- if (value) {
- acl = zpl_acl_from_xattr(value, size);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
- else if (acl) {
- error = zpl_posix_acl_valid(ip, acl);
- if (error) {
- zpl_posix_acl_release(acl);
- return (error);
- }
- }
- } else {
- acl = NULL;
- }
-
- error = zpl_set_acl(ip, acl, type);
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-ZPL_XATTR_SET_WRAPPER(zpl_xattr_acl_set_access);
-
-static int
-__zpl_xattr_acl_set_default(struct inode *ip, const char *name,
- const void *value, size_t size, int flags)
-{
- struct posix_acl *acl;
- int type = ACL_TYPE_DEFAULT;
- int error = 0;
- /* xattr_resolve_name will do this for us if this is defined */
-#ifndef HAVE_XATTR_HANDLER_NAME
- if (strcmp(name, "") != 0)
- return (-EINVAL);
-#endif
- if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIXACL)
- return (-EOPNOTSUPP);
-
- if (!zpl_inode_owner_or_capable(ip))
- return (-EPERM);
-
- if (value) {
- acl = zpl_acl_from_xattr(value, size);
- if (IS_ERR(acl))
- return (PTR_ERR(acl));
- else if (acl) {
- error = zpl_posix_acl_valid(ip, acl);
- if (error) {
- zpl_posix_acl_release(acl);
- return (error);
- }
- }
- } else {
- acl = NULL;
- }
-
- error = zpl_set_acl(ip, acl, type);
- zpl_posix_acl_release(acl);
-
- return (error);
-}
-ZPL_XATTR_SET_WRAPPER(zpl_xattr_acl_set_default);
-
-/*
- * ACL access xattr namespace handlers.
- *
- * Use .name instead of .prefix when available. xattr_resolve_name will match
- * whole name and reject anything that has .name only as prefix.
- */
-xattr_handler_t zpl_xattr_acl_access_handler =
-{
-#ifdef HAVE_XATTR_HANDLER_NAME
- .name = XATTR_NAME_POSIX_ACL_ACCESS,
-#else
- .prefix = XATTR_NAME_POSIX_ACL_ACCESS,
-#endif
- .list = zpl_xattr_acl_list_access,
- .get = zpl_xattr_acl_get_access,
- .set = zpl_xattr_acl_set_access,
-#if defined(HAVE_XATTR_LIST_SIMPLE) || \
- defined(HAVE_XATTR_LIST_DENTRY) || \
- defined(HAVE_XATTR_LIST_HANDLER)
- .flags = ACL_TYPE_ACCESS,
-#endif
-};
-
-/*
- * ACL default xattr namespace handlers.
- *
- * Use .name instead of .prefix when available. xattr_resolve_name will match
- * whole name and reject anything that has .name only as prefix.
- */
-xattr_handler_t zpl_xattr_acl_default_handler =
-{
-#ifdef HAVE_XATTR_HANDLER_NAME
- .name = XATTR_NAME_POSIX_ACL_DEFAULT,
-#else
- .prefix = XATTR_NAME_POSIX_ACL_DEFAULT,
-#endif
- .list = zpl_xattr_acl_list_default,
- .get = zpl_xattr_acl_get_default,
- .set = zpl_xattr_acl_set_default,
-#if defined(HAVE_XATTR_LIST_SIMPLE) || \
- defined(HAVE_XATTR_LIST_DENTRY) || \
- defined(HAVE_XATTR_LIST_HANDLER)
- .flags = ACL_TYPE_DEFAULT,
-#endif
-};
-
-#endif /* CONFIG_FS_POSIX_ACL */
-
-xattr_handler_t *zpl_xattr_handlers[] = {
- &zpl_xattr_security_handler,
- &zpl_xattr_trusted_handler,
- &zpl_xattr_user_handler,
-#ifdef CONFIG_FS_POSIX_ACL
- &zpl_xattr_acl_access_handler,
- &zpl_xattr_acl_default_handler,
-#endif /* CONFIG_FS_POSIX_ACL */
- NULL
-};
-
-static const struct xattr_handler *
-zpl_xattr_handler(const char *name)
-{
- if (strncmp(name, XATTR_USER_PREFIX,
- XATTR_USER_PREFIX_LEN) == 0)
- return (&zpl_xattr_user_handler);
-
- if (strncmp(name, XATTR_TRUSTED_PREFIX,
- XATTR_TRUSTED_PREFIX_LEN) == 0)
- return (&zpl_xattr_trusted_handler);
-
- if (strncmp(name, XATTR_SECURITY_PREFIX,
- XATTR_SECURITY_PREFIX_LEN) == 0)
- return (&zpl_xattr_security_handler);
-
-#ifdef CONFIG_FS_POSIX_ACL
- if (strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS,
- sizeof (XATTR_NAME_POSIX_ACL_ACCESS)) == 0)
- return (&zpl_xattr_acl_access_handler);
-
- if (strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT,
- sizeof (XATTR_NAME_POSIX_ACL_DEFAULT)) == 0)
- return (&zpl_xattr_acl_default_handler);
-#endif /* CONFIG_FS_POSIX_ACL */
-
- return (NULL);
-}
-
-#if !defined(HAVE_POSIX_ACL_RELEASE) || defined(HAVE_POSIX_ACL_RELEASE_GPL_ONLY)
-struct acl_rel_struct {
- struct acl_rel_struct *next;
- struct posix_acl *acl;
- clock_t time;
-};
-
-#define ACL_REL_GRACE (60*HZ)
-#define ACL_REL_WINDOW (1*HZ)
-#define ACL_REL_SCHED (ACL_REL_GRACE+ACL_REL_WINDOW)
-
-/*
- * Lockless multi-producer single-consumer fifo list.
- * Nodes are added to tail and removed from head. Tail pointer is our
- * synchronization point. It always points to the next pointer of the last
- * node, or head if list is empty.
- */
-static struct acl_rel_struct *acl_rel_head = NULL;
-static struct acl_rel_struct **acl_rel_tail = &acl_rel_head;
-
-static void
-zpl_posix_acl_free(void *arg)
-{
- struct acl_rel_struct *freelist = NULL;
- struct acl_rel_struct *a;
- clock_t new_time;
- boolean_t refire = B_FALSE;
-
- ASSERT3P(acl_rel_head, !=, NULL);
- while (acl_rel_head) {
- a = acl_rel_head;
- if (ddi_get_lbolt() - a->time >= ACL_REL_GRACE) {
- /*
- * If a is the last node we need to reset tail, but we
- * need to use cmpxchg to make sure it is still the
- * last node.
- */
- if (acl_rel_tail == &a->next) {
- acl_rel_head = NULL;
- if (cmpxchg(&acl_rel_tail, &a->next,
- &acl_rel_head) == &a->next) {
- ASSERT3P(a->next, ==, NULL);
- a->next = freelist;
- freelist = a;
- break;
- }
- }
- /*
- * a is not last node, make sure next pointer is set
- * by the adder and advance the head.
- */
- while (READ_ONCE(a->next) == NULL)
- cpu_relax();
- acl_rel_head = a->next;
- a->next = freelist;
- freelist = a;
- } else {
- /*
- * a is still in grace period. We are responsible to
- * reschedule the free task, since adder will only do
- * so if list is empty.
- */
- new_time = a->time + ACL_REL_SCHED;
- refire = B_TRUE;
- break;
- }
- }
-
- if (refire)
- taskq_dispatch_delay(system_delay_taskq, zpl_posix_acl_free,
- NULL, TQ_SLEEP, new_time);
-
- while (freelist) {
- a = freelist;
- freelist = a->next;
- kfree(a->acl);
- kmem_free(a, sizeof (struct acl_rel_struct));
- }
-}
-
-void
-zpl_posix_acl_release_impl(struct posix_acl *acl)
-{
- struct acl_rel_struct *a, **prev;
-
- a = kmem_alloc(sizeof (struct acl_rel_struct), KM_SLEEP);
- a->next = NULL;
- a->acl = acl;
- a->time = ddi_get_lbolt();
- /* atomically points tail to us and get the previous tail */
- prev = xchg(&acl_rel_tail, &a->next);
- ASSERT3P(*prev, ==, NULL);
- *prev = a;
- /* if it was empty before, schedule the free task */
- if (prev == &acl_rel_head)
- taskq_dispatch_delay(system_delay_taskq, zpl_posix_acl_free,
- NULL, TQ_SLEEP, ddi_get_lbolt() + ACL_REL_SCHED);
-}
-#endif