summaryrefslogtreecommitdiffstats
path: root/module/zfs/zio.c
diff options
context:
space:
mode:
authorNed Bass <[email protected]>2010-08-26 11:38:38 -0700
committerBrian Behlendorf <[email protected]>2010-08-31 08:38:50 -0700
commitda6b4005c92ce02359aabae6549f1031ad1fe41c (patch)
treef6702e0655ab0dece5a98666260dc13ed68f32c5 /module/zfs/zio.c
parentc776b317e44a64d53217d34c3fa61d36fd5a32d5 (diff)
Fix stack zio_execute()
Implement zio_execute() as a wrapper around the static function __zio_execute() so that we can force __zio_execute() to be inlined. This reduces stack overhead which is important because __zio_execute() is called recursively in several zio code paths. zio_execute() itself cannot be inlined because it is externally visible. Signed-off-by: Brian Behlendorf <[email protected]>
Diffstat (limited to 'module/zfs/zio.c')
-rw-r--r--module/zfs/zio.c26
1 files changed, 22 insertions, 4 deletions
diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index fb60bd8a0..c7eb66103 100644
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -93,6 +93,8 @@ int zio_buf_debug_limit = 16384;
int zio_buf_debug_limit = 0;
#endif
+static inline void __zio_execute(zio_t *zio);
+
void
zio_init(void)
{
@@ -451,7 +453,7 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
if (--*countp == 0 && pio->io_stall == countp) {
pio->io_stall = NULL;
mutex_exit(&pio->io_lock);
- zio_execute(pio);
+ __zio_execute(pio);
} else {
mutex_exit(&pio->io_lock);
}
@@ -1122,9 +1124,24 @@ zio_interrupt(zio_t *zio)
*/
static zio_pipe_stage_t *zio_pipeline[];
+/*
+ * zio_execute() is a wrapper around the static function
+ * __zio_execute() so that we can force __zio_execute() to be
+ * inlined. This reduces stack overhead which is important
+ * because __zio_execute() is called recursively in several zio
+ * code paths. zio_execute() itself cannot be inlined because
+ * it is externally visible.
+ */
void
zio_execute(zio_t *zio)
{
+ __zio_execute(zio);
+}
+
+__attribute__((always_inline))
+static inline void
+__zio_execute(zio_t *zio)
+{
zio->io_executor = curthread;
while (zio->io_stage < ZIO_STAGE_DONE) {
@@ -1169,6 +1186,7 @@ zio_execute(zio_t *zio)
}
}
+
/*
* ==========================================================================
* Initiate I/O, either sync or async
@@ -1184,7 +1202,7 @@ zio_wait(zio_t *zio)
zio->io_waiter = curthread;
- zio_execute(zio);
+ __zio_execute(zio);
mutex_enter(&zio->io_lock);
while (zio->io_executor != NULL)
@@ -1214,7 +1232,7 @@ zio_nowait(zio_t *zio)
zio_add_child(spa->spa_async_zio_root, zio);
}
- zio_execute(zio);
+ __zio_execute(zio);
}
/*
@@ -1269,7 +1287,7 @@ zio_reexecute(zio_t *pio)
* responsibility of the caller to wait on him.
*/
if (!(pio->io_flags & ZIO_FLAG_GODFATHER))
- zio_execute(pio);
+ __zio_execute(pio);
}
void