illumos/illumos-gate: usr/src/uts/common/fs/zfs/zio.c annotate

annotate usr/src/uts/common/fs/zfs/zio.c @ 13765:9410cf539b11

backout 3006: causes 3046 (panics after mounting root)

author	Richard Lowe <richlowe@richlowe.net>
date	Wed, 01 Aug 2012 16:40:39 -0400
parents	38b4aca480b3
children	ac6eff781c67

rev	line source
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2 * CDDL HEADER START
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	3 *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	4 * The contents of this file are subject to the terms of the
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	5 * Common Development and Distribution License (the "License").
938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	6 * You may not use this file except in compliance with the License.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	7 *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	9 * or http://www.opensolaris.org/os/licensing.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	10 * See the License for the specific language governing permissions
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	11 * and limitations under the License.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	12 *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	13 * When distributing Covered Code, include this CDDL HEADER in each
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	15 * If applicable, add the following below this CDDL HEADER, with the
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	16 * fields enclosed by brackets "[]" replaced with your own identifying
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	17 * information: Portions Copyright [yyyy] [name of copyright owner]
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	18 *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	19 * CDDL HEADER END
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	20 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	21 /*
12296 7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
13572 85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	23 * Copyright (c) 2012 by Delphix. All rights reserved.
13414 b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	25 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	26
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	27 #include <sys/zfs_context.h>
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	28 #include <sys/fm/fs/zfs.h>
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	29 #include <sys/spa.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	30 #include <sys/txg.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	31 #include <sys/spa_impl.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	32 #include <sys/vdev_impl.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	33 #include <sys/zio_impl.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	34 #include <sys/zio_compress.h>
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	35 #include <sys/zio_checksum.h>
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	36 #include <sys/dmu_objset.h>
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	37 #include <sys/arc.h>
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	38 #include <sys/ddt.h>
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	39
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	40 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	41 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	42 * I/O priority table
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	43 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	44 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	45 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	46 0, /* ZIO_PRIORITY_NOW */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	47 0, /* ZIO_PRIORITY_SYNC_READ */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	48 0, /* ZIO_PRIORITY_SYNC_WRITE */
11146 7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	49 0, /* ZIO_PRIORITY_LOG_WRITE */
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	50 1, /* ZIO_PRIORITY_CACHE_FILL */
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	51 1, /* ZIO_PRIORITY_AGG */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	52 4, /* ZIO_PRIORITY_FREE */
11146 7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	53 4, /* ZIO_PRIORITY_ASYNC_WRITE */
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	54 6, /* ZIO_PRIORITY_ASYNC_READ */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	55 10, /* ZIO_PRIORITY_RESILVER */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	56 20, /* ZIO_PRIORITY_SCRUB */
12450 c77e20e4e046 6938089 dedup-induced latency causes FC initiator logouts/FC port resets George Wilson <George.Wilson@Sun.COM> parents: 12351 diff changeset	57 2, /* ZIO_PRIORITY_DDT_PREFETCH */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	58 };
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	59
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	60 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	61 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	62 * I/O type descriptions
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	63 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	64 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	65 char *zio_type_name[ZIO_TYPES] = {
11146 7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	66 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim",
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	67 "zio_ioctl"
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	68 };
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	69
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	70 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	71 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	72 * I/O kmem caches
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	73 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	74 */
4055 9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	75 kmem_cache_t *zio_cache;
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	76 kmem_cache_t *zio_link_cache;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	77 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	78 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	79
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	80 #ifdef _KERNEL
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	81 extern vmem_t *zio_alloc_arena;
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	82 #endif
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	83 extern int zfs_mg_alloc_failures;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	84
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	85 /*
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	86 * An allocating zio is one that either currently has the DVA allocate
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	87 * stage set or will have it later in its lifetime.
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	88 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	89 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	90
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	91 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE;
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	92
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	93 #ifdef ZFS_DEBUG
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	94 int zio_buf_debug_limit = 16384;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	95 #else
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	96 int zio_buf_debug_limit = 0;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	97 #endif
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	98
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	99 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	100 zio_init(void)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	101 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	102 size_t c;
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	103 vmem_t *data_alloc_arena = NULL;
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	104
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	105 #ifdef _KERNEL
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	106 data_alloc_arena = zio_alloc_arena;
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	107 #endif
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	108 zio_cache = kmem_cache_create("zio_cache",
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	109 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	110 zio_link_cache = kmem_cache_create("zio_link_cache",
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	111 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
4055 9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	112
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	113 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	114 * For small buffers, we want a cache for each multiple of
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	115 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	116 * for each quarter-power of 2. For large buffers, we want
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	117 * a cache for each multiple of PAGESIZE.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	118 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	119 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	120 size_t size = (c + 1) << SPA_MINBLOCKSHIFT;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	121 size_t p2 = size;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	122 size_t align = 0;
13177 f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	123 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	124
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	125 while (p2 & (p2 - 1))
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	126 p2 &= p2 - 1;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	127
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	128 if (size <= 4 * SPA_MINBLOCKSIZE) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	129 align = SPA_MINBLOCKSIZE;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	130 } else if (P2PHASE(size, PAGESIZE) == 0) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	131 align = PAGESIZE;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	132 } else if (P2PHASE(size, p2 >> 2) == 0) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	133 align = p2 >> 2;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	134 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	135
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	136 if (align != 0) {
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	137 char name[36];
2856 6f4d5ee1906a 6463348 ZFS code could be more portable nd150628 parents: 2082 diff changeset	138 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	139 zio_buf_cache[c] = kmem_cache_create(name, size,
13177 f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	140 align, NULL, NULL, NULL, NULL, NULL, cflags);
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	141
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	142 /*
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	143 * Since zio_data bufs do not appear in crash dumps, we
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	144 * pass KMC_NOTOUCH so that no allocator metadata is
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	145 * stored with the buffers.
f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	146 */
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	147 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	148 zio_data_buf_cache[c] = kmem_cache_create(name, size,
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	149 align, NULL, NULL, NULL, NULL, data_alloc_arena,
13177 f3dd638d3435 6863750 kmem metadata in zfs_file_data impedes debugging non-debug crash dumps Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 13061 diff changeset	150 cflags \| KMC_NOTOUCH);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	151 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	152 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	153
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	154 while (--c != 0) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	155 ASSERT(zio_buf_cache[c] != NULL);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	156 if (zio_buf_cache[c - 1] == NULL)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	157 zio_buf_cache[c - 1] = zio_buf_cache[c];
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	158
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	159 ASSERT(zio_data_buf_cache[c] != NULL);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	160 if (zio_data_buf_cache[c - 1] == NULL)
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	161 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c];
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	162 }
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	163
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	164 /*
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	165 * The zio write taskqs have 1 thread per cpu, allow 1/2 of the taskqs
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	166 * to fail 3 times per txg or 8 failures, whichever is greater.
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	167 */
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	168 zfs_mg_alloc_failures = MAX((3 * max_ncpus / 2), 8);
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	169
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	170 zio_inject_init();
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	171 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	172
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	173 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	174 zio_fini(void)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	175 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	176 size_t c;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	177 kmem_cache_t *last_cache = NULL;
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	178 kmem_cache_t *last_data_cache = NULL;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	179
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	180 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	181 if (zio_buf_cache[c] != last_cache) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	182 last_cache = zio_buf_cache[c];
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	183 kmem_cache_destroy(zio_buf_cache[c]);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	184 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	185 zio_buf_cache[c] = NULL;
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	186
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	187 if (zio_data_buf_cache[c] != last_data_cache) {
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	188 last_data_cache = zio_data_buf_cache[c];
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	189 kmem_cache_destroy(zio_data_buf_cache[c]);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	190 }
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	191 zio_data_buf_cache[c] = NULL;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	192 }
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	193
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	194 kmem_cache_destroy(zio_link_cache);
4055 9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	195 kmem_cache_destroy(zio_cache);
9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	196
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	197 zio_inject_fini();
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	198 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	199
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	200 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	201 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	202 * Allocate and free I/O buffers
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	203 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	204 */
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	205
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	206 /*
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	207 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	208 * crashdump if the kernel panics, so use it judiciously. Obviously, it's
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	209 * useful to inspect ZFS metadata, but if possible, we should avoid keeping
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	210 * excess / transient data in-core during a crashdump.
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	211 */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	212 void *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	213 zio_buf_alloc(size_t size)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	214 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	215 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	216
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	217 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	218
6245 1a2a7cfb9f26 6429205 each zpool needs to monitor its throughput and throttle heavy writers maybee parents: 5688 diff changeset	219 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	220 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	221
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	222 /*
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	223 * Use zio_data_buf_alloc to allocate data. The data will not appear in a
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	224 * crashdump if the kernel panics. This exists so that we will limit the amount
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	225 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	226 * of kernel heap dumped to disk when the kernel panics)
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	227 */
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	228 void *
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	229 zio_data_buf_alloc(size_t size)
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	230 {
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	231 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	232
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	233 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	234
6245 1a2a7cfb9f26 6429205 each zpool needs to monitor its throughput and throttle heavy writers maybee parents: 5688 diff changeset	235 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE));
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	236 }
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	237
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	238 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	239 zio_buf_free(void *buf, size_t size)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	240 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	241 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	242
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	243 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	244
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	245 kmem_cache_free(zio_buf_cache[c], buf);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	246 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	247
3290 256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	248 void
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	249 zio_data_buf_free(void *buf, size_t size)
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	250 {
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	251 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	252
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	253 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	254
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	255 kmem_cache_free(zio_data_buf_cache[c], buf);
256464cbb73c 4894692 caching data in heap inflates crash dump johansen parents: 3063 diff changeset	256 }
3463 5007f15d5674 6480626 contention on spa_config_lock on niagra ahrens parents: 3459 diff changeset	257
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	258 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	259 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	260 * Push and pop I/O transform buffers
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	261 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	262 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	263 static void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	264 zio_push_transform(zio_t zio, void data, uint64_t size, uint64_t bufsize,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	265 zio_transform_func_t *transform)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	266 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	267 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	268
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	269 zt->zt_orig_data = zio->io_data;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	270 zt->zt_orig_size = zio->io_size;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	271 zt->zt_bufsize = bufsize;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	272 zt->zt_transform = transform;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	273
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	274 zt->zt_next = zio->io_transform_stack;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	275 zio->io_transform_stack = zt;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	276
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	277 zio->io_data = data;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	278 zio->io_size = size;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	279 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	280
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	281 static void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	282 zio_pop_transforms(zio_t *zio)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	283 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	284 zio_transform_t *zt;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	285
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	286 while ((zt = zio->io_transform_stack) != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	287 if (zt->zt_transform != NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	288 zt->zt_transform(zio,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	289 zt->zt_orig_data, zt->zt_orig_size);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	290
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	291 if (zt->zt_bufsize != 0)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	292 zio_buf_free(zio->io_data, zt->zt_bufsize);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	293
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	294 zio->io_data = zt->zt_orig_data;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	295 zio->io_size = zt->zt_orig_size;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	296 zio->io_transform_stack = zt->zt_next;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	297
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	298 kmem_free(zt, sizeof (zio_transform_t));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	299 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	300 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	301
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	302 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	303 * ==========================================================================
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	304 * I/O transform callbacks for subblocks and decompression
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	305 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	306 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	307 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	308 zio_subblock(zio_t zio, void data, uint64_t size)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	309 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	310 ASSERT(zio->io_size > size);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	311
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	312 if (zio->io_type == ZIO_TYPE_READ)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	313 bcopy(zio->io_data, data, size);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	314 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	315
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	316 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	317 zio_decompress(zio_t zio, void data, uint64_t size)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	318 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	319 if (zio->io_error == 0 &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	320 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	321 zio->io_data, data, zio->io_size, size) != 0)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	322 zio->io_error = EIO;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	323 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	324
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	325 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	326 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	327 * I/O parent/child relationships and pipeline interlocks
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	328 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	329 */
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	330 /*
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	331 * NOTE - Callers to zio_walk_parents() and zio_walk_children must
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	332 * continue calling these functions until they return NULL.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	333 * Otherwise, the next caller will pick up the list walk in
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	334 * some indeterminate state. (Otherwise every caller would
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	335 * have to pass in a cookie to keep the state represented by
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	336 * io_walk_link, which gets annoying.)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	337 */
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	338 zio_t *
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	339 zio_walk_parents(zio_t *cio)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	340 {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	341 zio_link_t *zl = cio->io_walk_link;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	342 list_t *pl = &cio->io_parent_list;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	343
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	344 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	345 cio->io_walk_link = zl;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	346
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	347 if (zl == NULL)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	348 return (NULL);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	349
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	350 ASSERT(zl->zl_child == cio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	351 return (zl->zl_parent);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	352 }
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	353
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	354 zio_t *
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	355 zio_walk_children(zio_t *pio)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	356 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	357 zio_link_t *zl = pio->io_walk_link;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	358 list_t *cl = &pio->io_child_list;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	359
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	360 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	361 pio->io_walk_link = zl;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	362
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	363 if (zl == NULL)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	364 return (NULL);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	365
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	366 ASSERT(zl->zl_parent == pio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	367 return (zl->zl_child);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	368 }
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	369
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	370 zio_t *
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	371 zio_unique_parent(zio_t *cio)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	372 {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	373 zio_t *pio = zio_walk_parents(cio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	374
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	375 VERIFY(zio_walk_parents(cio) == NULL);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	376 return (pio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	377 }
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	378
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	379 void
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	380 zio_add_child(zio_t pio, zio_t cio)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	381 {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	382 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	383
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	384 /*
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	385 * Logical I/Os can have logical, gang, or vdev children.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	386 * Gang I/Os can have gang or vdev children.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	387 * Vdev I/Os can only have vdev children.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	388 * The following ASSERT captures all of these constraints.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	389 */
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	390 ASSERT(cio->io_child_type <= pio->io_child_type);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	391
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	392 zl->zl_parent = pio;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	393 zl->zl_child = cio;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	394
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	395 mutex_enter(&cio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	396 mutex_enter(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	397
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	398 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	399
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	400 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	401 pio->io_children[cio->io_child_type][w] += !cio->io_state[w];
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	402
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	403 list_insert_head(&pio->io_child_list, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	404 list_insert_head(&cio->io_parent_list, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	405
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	406 pio->io_child_count++;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	407 cio->io_parent_count++;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	408
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	409 mutex_exit(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	410 mutex_exit(&cio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	411 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	412
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	413 static void
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	414 zio_remove_child(zio_t pio, zio_t cio, zio_link_t *zl)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	415 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	416 ASSERT(zl->zl_parent == pio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	417 ASSERT(zl->zl_child == cio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	418
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	419 mutex_enter(&cio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	420 mutex_enter(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	421
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	422 list_remove(&pio->io_child_list, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	423 list_remove(&cio->io_parent_list, zl);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	424
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	425 pio->io_child_count--;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	426 cio->io_parent_count--;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	427
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	428 mutex_exit(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	429 mutex_exit(&cio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	430
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	431 kmem_cache_free(zio_link_cache, zl);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	432 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	433
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	434 static boolean_t
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	435 zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	436 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	437 uint64_t *countp = &zio->io_children[child][wait];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	438 boolean_t waiting = B_FALSE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	439
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	440 mutex_enter(&zio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	441 ASSERT(zio->io_stall == NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	442 if (*countp != 0) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	443 zio->io_stage >>= 1;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	444 zio->io_stall = countp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	445 waiting = B_TRUE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	446 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	447 mutex_exit(&zio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	448
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	449 return (waiting);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	450 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	451
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	452 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	453 zio_notify_parent(zio_t pio, zio_t zio, enum zio_wait_type wait)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	454 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	455 uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	456 int *errorp = &pio->io_child_error[zio->io_child_type];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	457
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	458 mutex_enter(&pio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	459 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	460 errorp = zio_worst_error(errorp, zio->io_error);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	461 pio->io_reexecute \|= zio->io_reexecute;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	462 ASSERT3U(*countp, >, 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	463 if (--*countp == 0 && pio->io_stall == countp) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	464 pio->io_stall = NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	465 mutex_exit(&pio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	466 zio_execute(pio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	467 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	468 mutex_exit(&pio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	469 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	470 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	471
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	472 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	473 zio_inherit_child_errors(zio_t *zio, enum zio_child c)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	474 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	475 if (zio->io_child_error[c] != 0 && zio->io_error == 0)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	476 zio->io_error = zio->io_child_error[c];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	477 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	478
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	479 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	480 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	481 * Create the various types of I/O (read, write, free, etc)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	482 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	483 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	484 static zio_t *
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	485 zio_create(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	486 void data, uint64_t size, zio_done_func_t done, void *private,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	487 zio_type_t type, int priority, enum zio_flag flags,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	488 vdev_t vd, uint64_t offset, const zbookmark_t zb,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	489 enum zio_stage stage, enum zio_stage pipeline)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	490 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	491 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	492
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	493 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	494 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	495 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	496
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	497 ASSERT(!vd \|\| spa_config_held(spa, SCL_STATE_ALL, RW_READER));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	498 ASSERT(!bp \|\| !(flags & ZIO_FLAG_CONFIG_WRITER));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	499 ASSERT(vd \|\| stage == ZIO_STAGE_OPEN);
7046 361307ae060d 6343667 scrub/resilver has to start over when a snapshot is taken ahrens parents: 7030 diff changeset	500
4055 9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	501 zio = kmem_cache_alloc(zio_cache, KM_SLEEP);
9b8dd5af941d 6536445 want ::zio to show zio tree eschrock parents: 3882 diff changeset	502 bzero(zio, sizeof (zio_t));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	503
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	504 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	505 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	506
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	507 list_create(&zio->io_parent_list, sizeof (zio_link_t),
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	508 offsetof(zio_link_t, zl_parent_node));
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	509 list_create(&zio->io_child_list, sizeof (zio_link_t),
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	510 offsetof(zio_link_t, zl_child_node));
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	511
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	512 if (vd != NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	513 zio->io_child_type = ZIO_CHILD_VDEV;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	514 else if (flags & ZIO_FLAG_GANG_CHILD)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	515 zio->io_child_type = ZIO_CHILD_GANG;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	516 else if (flags & ZIO_FLAG_DDT_CHILD)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	517 zio->io_child_type = ZIO_CHILD_DDT;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	518 else
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	519 zio->io_child_type = ZIO_CHILD_LOGICAL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	520
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	521 if (bp != NULL) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	522 zio->io_bp = (blkptr_t *)bp;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	523 zio->io_bp_copy = *bp;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	524 zio->io_bp_orig = *bp;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	525 if (type != ZIO_TYPE_WRITE \|\|
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	526 zio->io_child_type == ZIO_CHILD_DDT)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	527 zio->io_bp = &zio->io_bp_copy; /* so caller can free */
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	528 if (zio->io_child_type == ZIO_CHILD_LOGICAL)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	529 zio->io_logical = zio;
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	530 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	531 pipeline \|= ZIO_GANG_STAGES;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	532 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	533
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	534 zio->io_spa = spa;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	535 zio->io_txg = txg;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	536 zio->io_done = done;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	537 zio->io_private = private;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	538 zio->io_type = type;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	539 zio->io_priority = priority;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	540 zio->io_vd = vd;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	541 zio->io_offset = offset;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	542 zio->io_orig_data = zio->io_data = data;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	543 zio->io_orig_size = zio->io_size = size;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	544 zio->io_orig_flags = zio->io_flags = flags;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	545 zio->io_orig_stage = zio->io_stage = stage;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	546 zio->io_orig_pipeline = zio->io_pipeline = pipeline;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	547
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	548 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	549 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	550
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	551 if (zb != NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	552 zio->io_bookmark = *zb;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	553
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	554 if (pio != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	555 if (zio->io_logical == NULL)
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	556 zio->io_logical = pio->io_logical;
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	557 if (zio->io_child_type == ZIO_CHILD_GANG)
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	558 zio->io_gang_leader = pio->io_gang_leader;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	559 zio_add_child(pio, zio);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	560 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	561
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	562 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	563 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	564
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	565 static void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	566 zio_destroy(zio_t *zio)
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	567 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	568 list_destroy(&zio->io_parent_list);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	569 list_destroy(&zio->io_child_list);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	570 mutex_destroy(&zio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	571 cv_destroy(&zio->io_cv);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	572 kmem_cache_free(zio_cache, zio);
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	573 }
33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	574
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	575 zio_t *
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	576 zio_null(zio_t pio, spa_t spa, vdev_t vd, zio_done_func_t done,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	577 void *private, enum zio_flag flags)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	578 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	579 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	580
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	581 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private,
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	582 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	583 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	584
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	585 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	586 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	587
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	588 zio_t *
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	589 zio_root(spa_t spa, zio_done_func_t done, void *private, enum zio_flag flags)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	590 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	591 return (zio_null(NULL, spa, NULL, done, private, flags));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	592 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	593
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	594 zio_t *
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	595 zio_read(zio_t pio, spa_t spa, const blkptr_t *bp,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	596 void data, uint64_t size, zio_done_func_t done, void *private,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	597 int priority, enum zio_flag flags, const zbookmark_t *zb)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	598 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	599 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	600
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	601 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp,
7046 361307ae060d 6343667 scrub/resilver has to start over when a snapshot is taken ahrens parents: 7030 diff changeset	602 data, size, done, private,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	603 ZIO_TYPE_READ, priority, flags, NULL, 0, zb,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	604 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	605 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	606
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	607 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	608 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	609
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	610 zio_t *
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	611 zio_write(zio_t pio, spa_t spa, uint64_t txg, blkptr_t *bp,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	612 void data, uint64_t size, const zio_prop_t zp,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	613 zio_done_func_t ready, zio_done_func_t done, void *private,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	614 int priority, enum zio_flag flags, const zbookmark_t *zb)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	615 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	616 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	617
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	618 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	619 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	620 zp->zp_compress >= ZIO_COMPRESS_OFF &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	621 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS &&
13700 2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	622 DMU_OT_IS_VALID(zp->zp_type) &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	623 zp->zp_level < 32 &&
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	624 zp->zp_copies > 0 &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	625 zp->zp_copies <= spa_max_replication(spa) &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	626 zp->zp_dedup <= 1 &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	627 zp->zp_dedup_verify <= 1);
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	628
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	629 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	630 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	631 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	632 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	633
3547 e396e0a440b1 6512391 DMU should leverage ZIO dependencies to achieve greater parallelism maybee parents: 3463 diff changeset	634 zio->io_ready = ready;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	635 zio->io_prop = *zp;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	636
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	637 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	638 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	639
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	640 zio_t *
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	641 zio_rewrite(zio_t pio, spa_t spa, uint64_t txg, blkptr_t bp, void data,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	642 uint64_t size, zio_done_func_t done, void private, int priority,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	643 enum zio_flag flags, zbookmark_t *zb)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	644 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	645 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	646
7181 8d299641aa23 6719934 assertion failed: ((&zh->zh_log)->blk_birth == 0), file: ../../common/fs/zfs/zil.c, line: 1336 perrin parents: 7046 diff changeset	647 zio = zio_create(pio, spa, txg, bp, data, size, done, private,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	648 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	649 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	650
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	651 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	652 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	653
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	654 void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	655 zio_write_override(zio_t zio, blkptr_t bp, int copies)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	656 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	657 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	658 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	659 ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	660 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	661
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	662 zio->io_prop.zp_copies = copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	663 zio->io_bp_override = bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	664 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	665
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	666 void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	667 zio_free(spa_t spa, uint64_t txg, const blkptr_t bp)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	668 {
12470 54258108784b 6948890 snapshot deletion can induce pathologically long spa_sync() times Matthew Ahrens <Matthew.Ahrens@Sun.COM> parents: 12450 diff changeset	669 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	670 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	671
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	672 zio_t *
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	673 zio_free_sync(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	674 enum zio_flag flags)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	675 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	676 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	677
12296 7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	678 dprintf_bp(bp, "freeing in txg %llu, pass %u",
7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	679 (longlong_t)txg, spa->spa_sync_pass);
7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	680
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	681 ASSERT(!BP_IS_HOLE(bp));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	682 ASSERT(spa_syncing_txg(spa) == txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	683 ASSERT(spa_sync_pass(spa) <= SYNC_PASS_DEFERRED_FREE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	684
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	685 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	686 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, flags,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	687 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	688
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	689 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	690 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	691
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	692 zio_t *
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	693 zio_claim(zio_t pio, spa_t spa, uint64_t txg, const blkptr_t *bp,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	694 zio_done_func_t done, void private, enum zio_flag flags)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	695 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	696 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	697
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	698 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	699 * A claim is an allocation of a specific block. Claims are needed
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	700 * to support immediate writes in the intent log. The issue is that
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	701 * immediate writes contain committed data, but in a txg that was
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	702 * not committed. Upon opening the pool after an unclean shutdown,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	703 * the intent log claims all blocks that contain immediate write data
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	704 * so that the SPA knows they're in use.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	705 *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	706 * All claims must be resolved in the first txg -- before the SPA
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	707 * starts allocating blocks -- so that nothing is allocated twice.
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	708 * If txg == 0 we just verify that the block is claimable.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	709 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	710 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	711 ASSERT(txg == spa_first_txg(spa) \|\| txg == 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	712 ASSERT(!BP_GET_DEDUP(bp) \|\| !spa_writeable(spa)); /* zdb(1M) */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	713
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	714 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	715 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	716 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	717
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	718 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	719 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	720
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	721 zio_t *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	722 zio_ioctl(zio_t pio, spa_t spa, vdev_t *vd, int cmd,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	723 zio_done_func_t done, void private, int priority, enum zio_flag flags)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	724 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	725 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	726 int c;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	727
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	728 if (vd->vdev_children == 0) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	729 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	730 ZIO_TYPE_IOCTL, priority, flags, vd, 0, NULL,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	731 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	732
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	733 zio->io_cmd = cmd;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	734 } else {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	735 zio = zio_null(pio, spa, NULL, NULL, NULL, flags);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	736
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	737 for (c = 0; c < vd->vdev_children; c++)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	738 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	739 done, private, priority, flags));
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	740 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	741
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	742 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	743 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	744
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	745 zio_t *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	746 zio_read_phys(zio_t pio, vdev_t vd, uint64_t offset, uint64_t size,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	747 void data, int checksum, zio_done_func_t done, void *private,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	748 int priority, enum zio_flag flags, boolean_t labels)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	749 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	750 zio_t *zio;
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	751
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	752 ASSERT(vd->vdev_children == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	753 ASSERT(!labels \|\| offset + size <= VDEV_LABEL_START_SIZE \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	754 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	755 ASSERT3U(offset + size, <=, vd->vdev_psize);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	756
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	757 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	758 ZIO_TYPE_READ, priority, flags, vd, offset, NULL,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	759 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	760
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	761 zio->io_prop.zp_checksum = checksum;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	762
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	763 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	764 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	765
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	766 zio_t *
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	767 zio_write_phys(zio_t pio, vdev_t vd, uint64_t offset, uint64_t size,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	768 void data, int checksum, zio_done_func_t done, void *private,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	769 int priority, enum zio_flag flags, boolean_t labels)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	770 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	771 zio_t *zio;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	772
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	773 ASSERT(vd->vdev_children == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	774 ASSERT(!labels \|\| offset + size <= VDEV_LABEL_START_SIZE \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	775 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	776 ASSERT3U(offset + size, <=, vd->vdev_psize);
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	777
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	778 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	779 ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	780 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	781
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	782 zio->io_prop.zp_checksum = checksum;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	783
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	784 if (zio_checksum_table[checksum].ci_eck) {
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	785 /*
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	786 * zec checksums are necessarily destructive -- they modify
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	787 * the end of the write buffer to hold the verifier/checksum.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	788 * Therefore, we must make a local copy in case the data is
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	789 * being written to multiple places in parallel.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	790 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	791 void *wbuf = zio_buf_alloc(size);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	792 bcopy(data, wbuf, size);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	793 zio_push_transform(zio, wbuf, size, size, NULL);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	794 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	795
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	796 return (zio);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	797 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	798
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	799 /*
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	800 * Create a child I/O to do some work for us.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	801 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	802 zio_t *
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	803 zio_vdev_child_io(zio_t pio, blkptr_t bp, vdev_t *vd, uint64_t offset,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	804 void *data, uint64_t size, int type, int priority, enum zio_flag flags,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	805 zio_done_func_t done, void private)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	806 {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	807 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	808 zio_t *zio;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	809
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	810 ASSERT(vd->vdev_parent ==
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	811 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	812
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	813 if (type == ZIO_TYPE_READ && bp != NULL) {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	814 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	815 * If we have the bp, then the child should perform the
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	816 * checksum and the parent need not. This pushes error
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	817 * detection as close to the leaves as possible and
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	818 * eliminates redundant checksums in the interior nodes.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	819 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	820 pipeline \|= ZIO_STAGE_CHECKSUM_VERIFY;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	821 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	822 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	823
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	824 if (vd->vdev_children == 0)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	825 offset += VDEV_LABEL_START_SIZE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	826
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	827 flags \|= ZIO_VDEV_CHILD_FLAGS(pio) \| ZIO_FLAG_DONT_PROPAGATE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	828
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	829 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	830 * If we've decided to do a repair, the write is not speculative --
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	831 * even if the original read was.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	832 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	833 if (flags & ZIO_FLAG_IO_REPAIR)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	834 flags &= ~ZIO_FLAG_SPECULATIVE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	835
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	836 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	837 done, private, type, priority, flags, vd, offset, &pio->io_bookmark,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	838 ZIO_STAGE_VDEV_IO_START >> 1, pipeline);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	839
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	840 return (zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	841 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	842
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	843 zio_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	844 zio_vdev_delegated_io(vdev_t vd, uint64_t offset, void data, uint64_t size,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	845 int type, int priority, enum zio_flag flags,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	846 zio_done_func_t done, void private)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	847 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	848 zio_t *zio;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	849
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	850 ASSERT(vd->vdev_ops->vdev_op_leaf);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	851
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	852 zio = zio_create(NULL, vd->vdev_spa, 0, NULL,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	853 data, size, done, private, type, priority,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	854 flags \| ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_RETRY,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	855 vd, offset, NULL,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	856 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	857
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	858 return (zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	859 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	860
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	861 void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	862 zio_flush(zio_t zio, vdev_t vd)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	863 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	864 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	865 NULL, NULL, ZIO_PRIORITY_NOW,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	866 ZIO_FLAG_CANFAIL \| ZIO_FLAG_DONT_PROPAGATE \| ZIO_FLAG_DONT_RETRY));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	867 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	868
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	869 void
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	870 zio_shrink(zio_t *zio, uint64_t size)
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	871 {
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	872 ASSERT(zio->io_executor == NULL);
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	873 ASSERT(zio->io_orig_size == zio->io_size);
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	874 ASSERT(size <= zio->io_size);
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	875
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	876 /*
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	877 * We don't shrink for raidz because of problems with the
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	878 * reconstruction when reading back less than the block size.
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	879 * Note, BP_IS_RAIDZ() assumes no compression.
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	880 */
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	881 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF);
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	882 if (!BP_IS_RAIDZ(zio->io_bp))
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	883 zio->io_orig_size = zio->io_size = size;
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	884 }
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	885
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	886 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	887 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	888 * Prepare to read and write logical blocks
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	889 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	890 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	891
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	892 static int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	893 zio_read_bp_init(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	894 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	895 blkptr_t *bp = zio->io_bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	896
8274 846b39508aff 6713916 scrub/resilver needlessly decompress data Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 8241 diff changeset	897 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	898 zio->io_child_type == ZIO_CHILD_LOGICAL &&
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	899 !(zio->io_flags & ZIO_FLAG_RAW)) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	900 uint64_t psize = BP_GET_PSIZE(bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	901 void *cbuf = zio_buf_alloc(psize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	902
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	903 zio_push_transform(zio, cbuf, psize, psize, zio_decompress);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	904 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	905
13700 2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	906 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	907 zio->io_flags \|= ZIO_FLAG_DONT_CACHE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	908
11125 fca3e6d28599 6899923 vdev_offline/vdev_add deadlock Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 11026 diff changeset	909 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP)
fca3e6d28599 6899923 vdev_offline/vdev_add deadlock Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 11026 diff changeset	910 zio->io_flags \|= ZIO_FLAG_DONT_CACHE;
fca3e6d28599 6899923 vdev_offline/vdev_add deadlock Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 11026 diff changeset	911
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	912 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	913 zio->io_pipeline = ZIO_DDT_READ_PIPELINE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	914
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	915 return (ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	916 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	917
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	918 static int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	919 zio_write_bp_init(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	920 {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	921 spa_t *spa = zio->io_spa;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	922 zio_prop_t *zp = &zio->io_prop;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	923 enum zio_compress compress = zp->zp_compress;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	924 blkptr_t *bp = zio->io_bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	925 uint64_t lsize = zio->io_size;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	926 uint64_t psize = lsize;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	927 int pass = 1;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	928
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	929 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	930 * If our children haven't all reached the ready stage,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	931 * wait for them and then repeat this pipeline stage.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	932 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	933 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	934 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	935 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	936
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	937 if (!IO_IS_ALLOCATING(zio))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	938 return (ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	939
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	940 ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	941
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	942 if (zio->io_bp_override) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	943 ASSERT(bp->blk_birth != zio->io_txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	944 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	945
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	946 bp = zio->io_bp_override;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	947 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	948
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	949 if (BP_IS_HOLE(bp) \|\| !zp->zp_dedup)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	950 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	951
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	952 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup \|\|
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	953 zp->zp_dedup_verify);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	954
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	955 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	956 BP_SET_DEDUP(bp, 1);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	957 zio->io_pipeline \|= ZIO_STAGE_DDT_WRITE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	958 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	959 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	960 zio->io_bp_override = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	961 BP_ZERO(bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	962 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	963
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	964 if (bp->blk_birth == zio->io_txg) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	965 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	966 * We're rewriting an existing block, which means we're
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	967 * working on behalf of spa_sync(). For spa_sync() to
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	968 * converge, it must eventually be the case that we don't
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	969 * have to allocate new blocks. But compression changes
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	970 * the blocksize, which forces a reallocate, and makes
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	971 * convergence take longer. Therefore, after the first
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	972 * few passes, stop compressing to ensure convergence.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	973 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	974 pass = spa_sync_pass(spa);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	975
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	976 ASSERT(zio->io_txg == spa_syncing_txg(spa));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	977 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	978 ASSERT(!BP_GET_DEDUP(bp));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	979
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	980 if (pass > SYNC_PASS_DONT_COMPRESS)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	981 compress = ZIO_COMPRESS_OFF;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	982
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	983 /* Make sure someone doesn't change their mind on overwrites */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	984 ASSERT(MIN(zp->zp_copies + BP_IS_GANG(bp),
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	985 spa_max_replication(spa)) == BP_GET_NDVAS(bp));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	986 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	987
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	988 if (compress != ZIO_COMPRESS_OFF) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	989 void *cbuf = zio_buf_alloc(lsize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	990 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	991 if (psize == 0 \|\| psize == lsize) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	992 compress = ZIO_COMPRESS_OFF;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	993 zio_buf_free(cbuf, lsize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	994 } else {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	995 ASSERT(psize < lsize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	996 zio_push_transform(zio, cbuf, psize, lsize, NULL);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	997 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	998 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	999
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1000 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1001 * The final pass of spa_sync() must be all rewrites, but the first
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1002 * few passes offer a trade-off: allocating blocks defers convergence,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1003 * but newly allocated blocks are sequential, so they can be written
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1004 * to disk faster. Therefore, we allow the first few passes of
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1005 * spa_sync() to allocate new blocks, but force rewrites after that.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1006 * There should only be a handful of blocks after pass 1 in any case.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1007 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1008 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == psize &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1009 pass > SYNC_PASS_REWRITE) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1010 ASSERT(psize != 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1011 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1012 zio->io_pipeline = ZIO_REWRITE_PIPELINE \| gang_stages;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1013 zio->io_flags \|= ZIO_FLAG_IO_REWRITE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1014 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1015 BP_ZERO(bp);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1016 zio->io_pipeline = ZIO_WRITE_PIPELINE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1017 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1018
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1019 if (psize == 0) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1020 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1021 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1022 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1023 BP_SET_LSIZE(bp, lsize);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1024 BP_SET_PSIZE(bp, psize);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1025 BP_SET_COMPRESS(bp, compress);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1026 BP_SET_CHECKSUM(bp, zp->zp_checksum);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1027 BP_SET_TYPE(bp, zp->zp_type);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1028 BP_SET_LEVEL(bp, zp->zp_level);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1029 BP_SET_DEDUP(bp, zp->zp_dedup);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1030 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1031 if (zp->zp_dedup) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1032 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1033 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1034 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1035 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1036 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1037
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1038 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1039 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1040
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1041 static int
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1042 zio_free_bp_init(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1043 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1044 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1045
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1046 if (zio->io_child_type == ZIO_CHILD_LOGICAL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1047 if (BP_GET_DEDUP(bp))
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1048 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1049 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1050
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1051 return (ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1052 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1053
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1054 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1055 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1056 * Execute the I/O pipeline
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1057 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1058 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1059
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1060 static void
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1061 zio_taskq_dispatch(zio_t *zio, enum zio_taskq_type q, boolean_t cutinline)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1062 {
11146 7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1063 spa_t *spa = zio->io_spa;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1064 zio_type_t t = zio->io_type;
13414 b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1065 int flags = (cutinline ? TQ_FRONT : 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1066
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1067 /*
9722 e3866bad4e96 6844069 vdev_probe() starvation brings txg train to a screeching halt George Wilson <George.Wilson@Sun.COM> parents: 9470 diff changeset	1068 * If we're a config writer or a probe, the normal issue and
e3866bad4e96 6844069 vdev_probe() starvation brings txg train to a screeching halt George Wilson <George.Wilson@Sun.COM> parents: 9470 diff changeset	1069 * interrupt threads may all be blocked waiting for the config lock.
e3866bad4e96 6844069 vdev_probe() starvation brings txg train to a screeching halt George Wilson <George.Wilson@Sun.COM> parents: 9470 diff changeset	1070 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1071 */
9722 e3866bad4e96 6844069 vdev_probe() starvation brings txg train to a screeching halt George Wilson <George.Wilson@Sun.COM> parents: 9470 diff changeset	1072 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER \| ZIO_FLAG_PROBE))
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1073 t = ZIO_TYPE_NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1074
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1075 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1076 * A similar issue exists for the L2ARC write thread until L2ARC 2.0.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1077 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1078 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1079 t = ZIO_TYPE_NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1080
11146 7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1081 /*
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1082 * If this is a high priority I/O, then use the high priority taskq.
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1083 */
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1084 if (zio->io_priority == ZIO_PRIORITY_NOW &&
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1085 spa->spa_zio_taskq[t][q + 1] != NULL)
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1086 q++;
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1087
7e58f40bcb1c 6826241 Sync write IOPS drops dramatically during TXG sync George Wilson <George.Wilson@Sun.COM> parents: 11125 diff changeset	1088 ASSERT3U(q, <, ZIO_TASKQ_TYPES);
13414 b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1089
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1090 /*
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1091 * NB: We are assuming that the zio can only be dispatched
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1092 * to a single taskq at a time. It would be a grievous error
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1093 * to dispatch the zio to another taskq at the same time.
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1094 */
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1095 ASSERT(zio->io_tqent.tqent_next == NULL);
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1096 taskq_dispatch_ent(spa->spa_zio_taskq[t][q],
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	1097 (task_func_t *)zio_execute, zio, flags, &zio->io_tqent);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1098 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1099
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1100 static boolean_t
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1101 zio_taskq_member(zio_t *zio, enum zio_taskq_type q)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1102 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1103 kthread_t *executor = zio->io_executor;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1104 spa_t *spa = zio->io_spa;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1105
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1106 for (zio_type_t t = 0; t < ZIO_TYPES; t++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1107 if (taskq_member(spa->spa_zio_taskq[t][q], executor))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1108 return (B_TRUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1109
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1110 return (B_FALSE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1111 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1112
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1113 static int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1114 zio_issue_async(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1115 {
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1116 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1117
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1118 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1119 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1120
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1121 void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1122 zio_interrupt(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1123 {
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1124 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1125 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1126
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1127 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1128 * Execute the I/O pipeline until one of the following occurs:
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1129 * (1) the I/O completes; (2) the pipeline stalls waiting for
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1130 * dependent child I/Os; (3) the I/O issues, so we're waiting
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1131 * for an I/O completion interrupt; (4) the I/O is delegated by
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1132 * vdev-level caching or aggregation; (5) the I/O is deferred
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1133 * due to vdev-level queueing; (6) the I/O is handed off to
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1134 * another thread. In all cases, the pipeline stops whenever
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1135 * there's no CPU work; it never burns a thread in cv_wait().
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1136 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1137 * There's no locking on io_stage because there's no legitimate way
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1138 * for multiple threads to be attempting to process the same I/O.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1139 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1140 static zio_pipe_stage_t *zio_pipeline[];
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1141
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1142 void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1143 zio_execute(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1144 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1145 zio->io_executor = curthread;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1146
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1147 while (zio->io_stage < ZIO_STAGE_DONE) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1148 enum zio_stage pipeline = zio->io_pipeline;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1149 enum zio_stage stage = zio->io_stage;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1150 int rv;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1151
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1152 ASSERT(!MUTEX_HELD(&zio->io_lock));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1153 ASSERT(ISP2(stage));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1154 ASSERT(zio->io_stall == NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1155
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1156 do {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1157 stage <<= 1;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1158 } while ((stage & pipeline) == 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1159
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1160 ASSERT(stage <= ZIO_STAGE_DONE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1161
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1162 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1163 * If we are in interrupt context and this pipeline stage
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1164 * will grab a config lock that is held across I/O,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1165 * or may wait for an I/O that needs an interrupt thread
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1166 * to complete, issue async to avoid deadlock.
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1167 *
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1168 * For VDEV_IO_START, we cut in line so that the io will
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1169 * be sent to disk promptly.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1170 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1171 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1172 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) {
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1173 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1174 zio_requeue_io_start_cut_in_line : B_FALSE;
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1175 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1176 return;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1177 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1178
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1179 zio->io_stage = stage;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1180 rv = zio_pipeline[highbit(stage) - 1](zio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1181
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1182 if (rv == ZIO_PIPELINE_STOP)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1183 return;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1184
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1185 ASSERT(rv == ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1186 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1187 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1188
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1189 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1190 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1191 * Initiate I/O, either sync or async
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1192 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1193 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1194 int
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1195 zio_wait(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1196 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1197 int error;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1198
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1199 ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1200 ASSERT(zio->io_executor == NULL);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1201
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1202 zio->io_waiter = curthread;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1203
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1204 zio_execute(zio);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1205
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1206 mutex_enter(&zio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1207 while (zio->io_executor != NULL)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1208 cv_wait(&zio->io_cv, &zio->io_lock);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1209 mutex_exit(&zio->io_lock);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1210
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1211 error = zio->io_error;
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1212 zio_destroy(zio);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1213
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1214 return (error);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1215 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1216
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1217 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1218 zio_nowait(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1219 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1220 ASSERT(zio->io_executor == NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1221
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1222 if (zio->io_child_type == ZIO_CHILD_LOGICAL &&
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1223 zio_unique_parent(zio) == NULL) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1224 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1225 * This is a logical async I/O with no parent to wait for it.
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1226 * We add it to the spa_async_root_zio "Godfather" I/O which
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1227 * will ensure they complete prior to unloading the pool.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1228 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1229 spa_t *spa = zio->io_spa;
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1230
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1231 zio_add_child(spa->spa_async_zio_root, zio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1232 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1233
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1234 zio_execute(zio);
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1235 }
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1236
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1237 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1238 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1239 * Reexecute or suspend/resume failed I/O
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1240 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1241 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1242
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1243 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1244 zio_reexecute(zio_t *pio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1245 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1246 zio_t cio, cio_next;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1247
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1248 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1249 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1250 ASSERT(pio->io_gang_leader == NULL);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1251 ASSERT(pio->io_gang_tree == NULL);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1252
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1253 pio->io_flags = pio->io_orig_flags;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1254 pio->io_stage = pio->io_orig_stage;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1255 pio->io_pipeline = pio->io_orig_pipeline;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1256 pio->io_reexecute = 0;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1257 pio->io_error = 0;
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1258 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1259 pio->io_state[w] = 0;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1260 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1261 pio->io_child_error[c] = 0;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1262
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1263 if (IO_IS_ALLOCATING(pio))
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1264 BP_ZERO(pio->io_bp);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1265
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1266 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1267 * As we reexecute pio's children, new children could be created.
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1268 * New children go to the head of pio's io_child_list, however,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1269 * so we will (correctly) not reexecute them. The key is that
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1270 * the remainder of pio's io_child_list, from 'cio_next' onward,
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1271 * cannot be affected by any side effects of reexecuting 'cio'.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1272 */
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1273 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1274 cio_next = zio_walk_children(pio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1275 mutex_enter(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1276 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1277 pio->io_children[cio->io_child_type][w]++;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1278 mutex_exit(&pio->io_lock);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1279 zio_reexecute(cio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1280 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1281
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1282 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1283 * Now that all children have been reexecuted, execute the parent.
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1284 * We don't reexecute "The Godfather" I/O here as it's the
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1285 * responsibility of the caller to wait on him.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1286 */
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1287 if (!(pio->io_flags & ZIO_FLAG_GODFATHER))
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1288 zio_execute(pio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1289 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1290
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1291 void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1292 zio_suspend(spa_t spa, zio_t zio)
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1293 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1294 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1295 fm_panic("Pool '%s' has encountered an uncorrectable I/O "
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1296 "failure and the failure mode property for this pool "
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1297 "is set to panic.", spa_name(spa));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1298
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1299 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1300
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1301 mutex_enter(&spa->spa_suspend_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1302
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1303 if (spa->spa_suspend_zio_root == NULL)
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1304 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL,
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1305 ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE \|
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1306 ZIO_FLAG_GODFATHER);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1307
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1308 spa->spa_suspended = B_TRUE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1309
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1310 if (zio != NULL) {
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1311 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1312 ASSERT(zio != spa->spa_suspend_zio_root);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1313 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1314 ASSERT(zio_unique_parent(zio) == NULL);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1315 ASSERT(zio->io_stage == ZIO_STAGE_DONE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1316 zio_add_child(spa->spa_suspend_zio_root, zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1317 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1318
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1319 mutex_exit(&spa->spa_suspend_lock);
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1320 }
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1321
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1322 int
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1323 zio_resume(spa_t *spa)
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1324 {
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1325 zio_t *pio;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1326
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1327 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1328 * Reexecute all previously suspended i/o.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1329 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1330 mutex_enter(&spa->spa_suspend_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1331 spa->spa_suspended = B_FALSE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1332 cv_broadcast(&spa->spa_suspend_cv);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1333 pio = spa->spa_suspend_zio_root;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1334 spa->spa_suspend_zio_root = NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1335 mutex_exit(&spa->spa_suspend_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1336
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1337 if (pio == NULL)
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1338 return (0);
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1339
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1340 zio_reexecute(pio);
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	1341 return (zio_wait(pio));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1342 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1343
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1344 void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1345 zio_resume_wait(spa_t *spa)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1346 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1347 mutex_enter(&spa->spa_suspend_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1348 while (spa_suspended(spa))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1349 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1350 mutex_exit(&spa->spa_suspend_lock);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1351 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1352
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1353 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1354 * ==========================================================================
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1355 * Gang blocks.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1356 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1357 * A gang block is a collection of small blocks that looks to the DMU
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1358 * like one large block. When zio_dva_allocate() cannot find a block
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1359 * of the requested size, due to either severe fragmentation or the pool
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1360 * being nearly full, it calls zio_write_gang_block() to construct the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1361 * block from smaller fragments.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1362 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1363 * A gang block consists of a gang header (zio_gbh_phys_t) and up to
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1364 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1365 * an indirect block: it's an array of block pointers. It consumes
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1366 * only one sector and hence is allocatable regardless of fragmentation.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1367 * The gang header's bps point to its gang members, which hold the data.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1368 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1369 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg>
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1370 * as the verifier to ensure uniqueness of the SHA256 checksum.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1371 * Critically, the gang block bp's blk_cksum is the checksum of the data,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1372 * not the gang header. This ensures that data block signatures (needed for
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1373 * deduplication) are independent of how the block is physically stored.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1374 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1375 * Gang blocks can be nested: a gang member may itself be a gang block.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1376 * Thus every gang block is a tree in which root and all interior nodes are
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1377 * gang headers, and the leaves are normal blocks that contain user data.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1378 * The root of the gang tree is called the gang leader.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1379 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1380 * To perform any operation (read, rewrite, free, claim) on a gang block,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1381 * zio_gang_assemble() first assembles the gang tree (minus data leaves)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1382 * in the io_gang_tree field of the original logical i/o by recursively
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1383 * reading the gang leader and all gang headers below it. This yields
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1384 * an in-core tree containing the contents of every gang header and the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1385 * bps for every constituent of the gang block.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1386 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1387 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1388 * and invokes a callback on each bp. To free a gang block, zio_gang_issue()
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1389 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1390 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim().
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1391 * zio_read_gang() is a wrapper around zio_read() that omits reading gang
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1392 * headers, since we already have those in io_gang_tree. zio_rewrite_gang()
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1393 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite()
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1394 * of the gang header plus zio_checksum_compute() of the data to update the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1395 * gang header's blk_cksum as described above.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1396 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1397 * The two-phase assemble/issue model solves the problem of partial failure --
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1398 * what if you'd freed part of a gang block but then couldn't read the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1399 * gang header for another part? Assembling the entire gang tree first
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1400 * ensures that all the necessary gang header I/O has succeeded before
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1401 * starting the actual work of free, claim, or write. Once the gang tree
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1402 * is assembled, free and claim are in-memory operations that cannot fail.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1403 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1404 * In the event that a gang write fails, zio_dva_unallocate() walks the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1405 * gang tree to immediately free (i.e. insert back into the space map)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1406 * everything we've allocated. This ensures that we don't get ENOSPC
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1407 * errors during repeated suspend/resume cycles due to a flaky device.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1408 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1409 * Gang rewrites only happen during sync-to-convergence. If we can't assemble
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1410 * the gang tree, we won't modify the block, so we can safely defer the free
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1411 * (knowing that the block is still intact). If we can assemble the gang
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1412 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1413 * each constituent bp and we can allocate a new block on the next sync pass.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1414 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1415 * In all cases, the gang tree allows complete recovery from partial failure.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1416 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1417 */
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1418
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1419 static zio_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1420 zio_read_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, void data)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1421 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1422 if (gn != NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1423 return (pio);
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1424
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1425 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1426 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1427 &pio->io_bookmark));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1428 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1429
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1430 zio_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1431 zio_rewrite_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, void data)
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1432 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1433 zio_t *zio;
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1434
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1435 if (gn != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1436 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1437 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1438 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1439 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1440 * As we rewrite each gang header, the pipeline will compute
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1441 * a new gang block header checksum for it; but no one will
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1442 * compute a new data checksum, so we do that here. The one
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1443 * exception is the gang leader: the pipeline already computed
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1444 * its data checksum because that stage precedes gang assembly.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1445 * (Presently, nothing actually uses interior data checksums;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1446 * this is just good hygiene.)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1447 */
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1448 if (gn != pio->io_gang_leader->io_gang_tree) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1449 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1450 data, BP_GET_PSIZE(bp));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1451 }
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1452 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1453 * If we are here to damage data for testing purposes,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1454 * leave the GBH alone so that we can detect the damage.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1455 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1456 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1457 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1458 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1459 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1460 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1461 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1462 }
c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1463
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1464 return (zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1465 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1466
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1467 /* ARGSUSED */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1468 zio_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1469 zio_free_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, void data)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1470 {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1471 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1472 ZIO_GANG_CHILD_FLAGS(pio)));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1473 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1474
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1475 /* ARGSUSED */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1476 zio_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1477 zio_claim_gang(zio_t pio, blkptr_t bp, zio_gang_node_t gn, void data)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1478 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1479 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1480 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio)));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1481 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1482
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1483 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1484 NULL,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1485 zio_read_gang,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1486 zio_rewrite_gang,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1487 zio_free_gang,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1488 zio_claim_gang,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1489 NULL
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1490 };
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1491
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1492 static void zio_gang_tree_assemble_done(zio_t *zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1493
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1494 static zio_gang_node_t *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1495 zio_gang_node_alloc(zio_gang_node_t **gnpp)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1496 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1497 zio_gang_node_t *gn;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1498
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1499 ASSERT(*gnpp == NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1500
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1501 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1502 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1503 *gnpp = gn;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1504
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1505 return (gn);
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1506 }
c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1507
c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1508 static void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1509 zio_gang_node_free(zio_gang_node_t **gnpp)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1510 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1511 zio_gang_node_t gn = gnpp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1512
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1513 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1514 ASSERT(gn->gn_child[g] == NULL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1515
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1516 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1517 kmem_free(gn, sizeof (*gn));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1518 *gnpp = NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1519 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1520
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1521 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1522 zio_gang_tree_free(zio_gang_node_t **gnpp)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1523 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1524 zio_gang_node_t gn = gnpp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1525
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1526 if (gn == NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1527 return;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1528
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1529 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1530 zio_gang_tree_free(&gn->gn_child[g]);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1531
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1532 zio_gang_node_free(gnpp);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1533 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1534
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1535 static void
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1536 zio_gang_tree_assemble(zio_t gio, blkptr_t bp, zio_gang_node_t **gnpp)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1537 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1538 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1539
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1540 ASSERT(gio->io_gang_leader == gio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1541 ASSERT(BP_IS_GANG(bp));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1542
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1543 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1544 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn,
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1545 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1546 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1547
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1548 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1549 zio_gang_tree_assemble_done(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1550 {
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1551 zio_t *gio = zio->io_gang_leader;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1552 zio_gang_node_t *gn = zio->io_private;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1553 blkptr_t *bp = zio->io_bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1554
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1555 ASSERT(gio == zio_unique_parent(zio));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1556 ASSERT(zio->io_child_count == 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1557
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1558 if (zio->io_error)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1559 return;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1560
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1561 if (BP_SHOULD_BYTESWAP(bp))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1562 byteswap_uint64_array(zio->io_data, zio->io_size);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1563
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1564 ASSERT(zio->io_data == gn->gn_gbh);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1565 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE);
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	1566 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1567
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1568 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1569 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1570 if (!BP_IS_GANG(gbp))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1571 continue;
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1572 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1573 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1574 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1575
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1576 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1577 zio_gang_tree_issue(zio_t pio, zio_gang_node_t gn, blkptr_t bp, void data)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1578 {
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1579 zio_t *gio = pio->io_gang_leader;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1580 zio_t *zio;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1581
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1582 ASSERT(BP_IS_GANG(bp) == !!gn);
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1583 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp));
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1584 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) \|\| gn == gio->io_gang_tree);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1585
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1586 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1587 * If you're a gang header, your data is in gn->gn_gbh.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1588 * If you're a gang member, your data is in 'data' and gn == NULL.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1589 */
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1590 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1591
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1592 if (gn != NULL) {
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	1593 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1594
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1595 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1596 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1597 if (BP_IS_HOLE(gbp))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1598 continue;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1599 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1600 data = (char *)data + BP_GET_PSIZE(gbp);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1601 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1602 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1603
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1604 if (gn == gio->io_gang_tree)
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1605 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1606
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1607 if (zio != pio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1608 zio_nowait(zio);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1609 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1610
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1611 static int
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1612 zio_gang_assemble(zio_t *zio)
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	1613 {
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1614 blkptr_t *bp = zio->io_bp;
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1615
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1616 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1617 ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1618
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1619 zio->io_gang_leader = zio;
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1620
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1621 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1622
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1623 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1624 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1625
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1626 static int
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1627 zio_gang_issue(zio_t *zio)
6523 c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1628 {
c1d2a7f04573 6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful ek110237 parents: 6245 diff changeset	1629 blkptr_t *bp = zio->io_bp;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1630
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1631 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1632 return (ZIO_PIPELINE_STOP);
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	1633
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1634 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1635 ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1636
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1637 if (zio->io_child_error[ZIO_CHILD_GANG] == 0)
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1638 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1639 else
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1640 zio_gang_tree_free(&zio->io_gang_tree);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1641
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1642 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1643
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1644 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1645 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1646
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1647 static void
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1648 zio_write_gang_member_ready(zio_t *zio)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1649 {
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	1650 zio_t *pio = zio_unique_parent(zio);
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1651 zio_t *gio = zio->io_gang_leader;
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1652 dva_t *cdva = zio->io_bp->blk_dva;
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1653 dva_t *pdva = pio->io_bp->blk_dva;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1654 uint64_t asize;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1655
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1656 if (BP_IS_HOLE(zio->io_bp))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1657 return;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1658
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1659 ASSERT(BP_IS_HOLE(&zio->io_bp_orig));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1660
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1661 ASSERT(zio->io_child_type == ZIO_CHILD_GANG);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1662 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1663 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1664 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp));
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1665 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp));
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1666
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1667 mutex_enter(&pio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1668 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) {
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1669 ASSERT(DVA_GET_GANG(&pdva[d]));
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1670 asize = DVA_GET_ASIZE(&pdva[d]);
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1671 asize += DVA_GET_ASIZE(&cdva[d]);
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1672 DVA_SET_ASIZE(&pdva[d], asize);
e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1673 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1674 mutex_exit(&pio->io_lock);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1675 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1676
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	1677 static int
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1678 zio_write_gang_block(zio_t *pio)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1679 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1680 spa_t *spa = pio->io_spa;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1681 blkptr_t *bp = pio->io_bp;
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1682 zio_t *gio = pio->io_gang_leader;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1683 zio_t *zio;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1684 zio_gang_node_t gn, *gnpp;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1685 zio_gbh_phys_t *gbh;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1686 uint64_t txg = pio->io_txg;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1687 uint64_t resid = pio->io_size;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1688 uint64_t lsize;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1689 int copies = gio->io_prop.zp_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1690 int gbh_copies = MIN(copies + 1, spa_max_replication(spa));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1691 zio_prop_t zp;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1692 int error;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1693
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1694 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1695 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1696 METASLAB_HINTBP_FAVOR \| METASLAB_GANG_HEADER);
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1697 if (error) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1698 pio->io_error = error;
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1699 return (ZIO_PIPELINE_CONTINUE);
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1700 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1701
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1702 if (pio == gio) {
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1703 gnpp = &gio->io_gang_tree;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1704 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1705 gnpp = pio->io_private;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1706 ASSERT(pio->io_ready == zio_write_gang_member_ready);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1707 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1708
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1709 gn = zio_gang_node_alloc(gnpp);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1710 gbh = gn->gn_gbh;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1711 bzero(gbh, SPA_GANGBLOCKSIZE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1712
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1713 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1714 * Create the gang header.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1715 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1716 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1717 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	1718
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1719 /*
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1720 * Create and nowait the gang children.
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	1721 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1722 for (int g = 0; resid != 0; resid -= lsize, g++) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1723 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1724 SPA_MINBLOCKSIZE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1725 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1726
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	1727 zp.zp_checksum = gio->io_prop.zp_checksum;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1728 zp.zp_compress = ZIO_COMPRESS_OFF;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1729 zp.zp_type = DMU_OT_NONE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1730 zp.zp_level = 0;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1731 zp.zp_copies = gio->io_prop.zp_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1732 zp.zp_dedup = 0;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1733 zp.zp_dedup_verify = 0;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1734
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1735 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1736 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1737 zio_write_gang_member_ready, NULL, &gn->gn_child[g],
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1738 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1739 &pio->io_bookmark));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1740 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1741
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1742 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1743 * Set pio's pipeline to just wait for zio to finish.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1744 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1745 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1746
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1747 zio_nowait(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1748
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	1749 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1750 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1751
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1752 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	1753 * ==========================================================================
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1754 * Dedup
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1755 * ==========================================================================
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1756 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1757 static void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1758 zio_ddt_child_read_done(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1759 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1760 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1761 ddt_entry_t *dde = zio->io_private;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1762 ddt_phys_t *ddp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1763 zio_t *pio = zio_unique_parent(zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1764
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1765 mutex_enter(&pio->io_lock);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1766 ddp = ddt_phys_select(dde, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1767 if (zio->io_error == 0)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1768 ddt_phys_clear(ddp); /* this ddp doesn't need repair */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1769 if (zio->io_error == 0 && dde->dde_repair_data == NULL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1770 dde->dde_repair_data = zio->io_data;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1771 else
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1772 zio_buf_free(zio->io_data, zio->io_size);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1773 mutex_exit(&pio->io_lock);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1774 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1775
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1776 static int
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1777 zio_ddt_read_start(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1778 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1779 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1780
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1781 ASSERT(BP_GET_DEDUP(bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1782 ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1783 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1784
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1785 if (zio->io_child_error[ZIO_CHILD_DDT]) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1786 ddt_t *ddt = ddt_select(zio->io_spa, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1787 ddt_entry_t *dde = ddt_repair_start(ddt, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1788 ddt_phys_t *ddp = dde->dde_phys;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1789 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1790 blkptr_t blk;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1791
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1792 ASSERT(zio->io_vsd == NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1793 zio->io_vsd = dde;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1794
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1795 if (ddp_self == NULL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1796 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1797
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1798 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1799 if (ddp->ddp_phys_birth == 0 \|\| ddp == ddp_self)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1800 continue;
11125 fca3e6d28599 6899923 vdev_offline/vdev_add deadlock Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 11026 diff changeset	1801 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp,
fca3e6d28599 6899923 vdev_offline/vdev_add deadlock Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 11026 diff changeset	1802 &blk);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1803 zio_nowait(zio_read(zio, zio->io_spa, &blk,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1804 zio_buf_alloc(zio->io_size), zio->io_size,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1805 zio_ddt_child_read_done, dde, zio->io_priority,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1806 ZIO_DDT_CHILD_FLAGS(zio) \| ZIO_FLAG_DONT_PROPAGATE,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1807 &zio->io_bookmark));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1808 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1809 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1810 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1811
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1812 zio_nowait(zio_read(zio, zio->io_spa, bp,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1813 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1814 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1815
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1816 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1817 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1818
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1819 static int
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1820 zio_ddt_read_done(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1821 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1822 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1823
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1824 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE))
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1825 return (ZIO_PIPELINE_STOP);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1826
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1827 ASSERT(BP_GET_DEDUP(bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1828 ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1829 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1830
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1831 if (zio->io_child_error[ZIO_CHILD_DDT]) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1832 ddt_t *ddt = ddt_select(zio->io_spa, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1833 ddt_entry_t *dde = zio->io_vsd;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1834 if (ddt == NULL) {
11147 74e8c05021f1 6678033 resilver code should prefetch George Wilson <George.Wilson@Sun.COM> parents: 11146 diff changeset	1835 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1836 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1837 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1838 if (dde == NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1839 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	1840 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1841 return (ZIO_PIPELINE_STOP);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1842 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1843 if (dde->dde_repair_data != NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1844 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1845 zio->io_child_error[ZIO_CHILD_DDT] = 0;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1846 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1847 ddt_repair_done(ddt, dde);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1848 zio->io_vsd = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1849 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1850
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1851 ASSERT(zio->io_vsd == NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1852
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1853 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1854 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1855
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1856 static boolean_t
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1857 zio_ddt_collision(zio_t zio, ddt_t ddt, ddt_entry_t *dde)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1858 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1859 spa_t *spa = zio->io_spa;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1860
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1861 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1862 * Note: we compare the original data, not the transformed data,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1863 * because when zio->io_bp is an override bp, we will not have
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1864 * pushed the I/O transforms. That's an important optimization
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1865 * because otherwise we'd compress/encrypt all dmu_sync() data twice.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1866 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1867 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1868 zio_t *lio = dde->dde_lead_zio[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1869
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1870 if (lio != NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1871 return (lio->io_orig_size != zio->io_orig_size \|\|
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1872 bcmp(zio->io_orig_data, lio->io_orig_data,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1873 zio->io_orig_size) != 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1874 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1875 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1876
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1877 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1878 ddt_phys_t *ddp = &dde->dde_phys[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1879
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1880 if (ddp->ddp_phys_birth != 0) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1881 arc_buf_t *abuf = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1882 uint32_t aflags = ARC_WAIT;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1883 blkptr_t blk = *zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1884 int error;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1885
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1886 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1887
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1888 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1889
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1890 error = arc_read_nolock(NULL, spa, &blk,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1891 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1892 ZIO_FLAG_CANFAIL \| ZIO_FLAG_SPECULATIVE,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1893 &aflags, &zio->io_bookmark);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1894
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1895 if (error == 0) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1896 if (arc_buf_size(abuf) != zio->io_orig_size \|\|
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1897 bcmp(abuf->b_data, zio->io_orig_data,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1898 zio->io_orig_size) != 0)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1899 error = EEXIST;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1900 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1901 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1902
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1903 ddt_enter(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1904 return (error != 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1905 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1906 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1907
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1908 return (B_FALSE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1909 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1910
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1911 static void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1912 zio_ddt_child_write_ready(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1913 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1914 int p = zio->io_prop.zp_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1915 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1916 ddt_entry_t *dde = zio->io_private;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1917 ddt_phys_t *ddp = &dde->dde_phys[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1918 zio_t *pio;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1919
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1920 if (zio->io_error)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1921 return;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1922
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1923 ddt_enter(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1924
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1925 ASSERT(dde->dde_lead_zio[p] == zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1926
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1927 ddt_phys_fill(ddp, zio->io_bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1928
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1929 while ((pio = zio_walk_parents(zio)) != NULL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1930 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1931
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1932 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1933 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1934
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1935 static void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1936 zio_ddt_child_write_done(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1937 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1938 int p = zio->io_prop.zp_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1939 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1940 ddt_entry_t *dde = zio->io_private;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1941 ddt_phys_t *ddp = &dde->dde_phys[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1942
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1943 ddt_enter(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1944
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1945 ASSERT(ddp->ddp_refcnt == 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1946 ASSERT(dde->dde_lead_zio[p] == zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1947 dde->dde_lead_zio[p] = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1948
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1949 if (zio->io_error == 0) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1950 while (zio_walk_parents(zio) != NULL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1951 ddt_phys_addref(ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1952 } else {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1953 ddt_phys_clear(ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1954 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1955
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1956 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1957 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1958
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1959 static void
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1960 zio_ddt_ditto_write_done(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1961 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1962 int p = DDT_PHYS_DITTO;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1963 zio_prop_t *zp = &zio->io_prop;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1964 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1965 ddt_t *ddt = ddt_select(zio->io_spa, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1966 ddt_entry_t *dde = zio->io_private;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1967 ddt_phys_t *ddp = &dde->dde_phys[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1968 ddt_key_t *ddk = &dde->dde_key;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1969
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1970 ddt_enter(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1971
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1972 ASSERT(ddp->ddp_refcnt == 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1973 ASSERT(dde->dde_lead_zio[p] == zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1974 dde->dde_lead_zio[p] = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1975
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1976 if (zio->io_error == 0) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1977 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1978 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1979 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1980 if (ddp->ddp_phys_birth != 0)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1981 ddt_phys_free(ddt, ddk, ddp, zio->io_txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1982 ddt_phys_fill(ddp, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1983 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1984
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1985 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1986 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1987
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1988 static int
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1989 zio_ddt_write(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1990 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1991 spa_t *spa = zio->io_spa;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1992 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1993 uint64_t txg = zio->io_txg;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1994 zio_prop_t *zp = &zio->io_prop;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1995 int p = zp->zp_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1996 int ditto_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1997 zio_t *cio = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1998 zio_t *dio = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	1999 ddt_t *ddt = ddt_select(spa, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2000 ddt_entry_t *dde;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2001 ddt_phys_t *ddp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2002
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2003 ASSERT(BP_GET_DEDUP(bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2004 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2005 ASSERT(BP_IS_HOLE(bp) \|\| zio->io_bp_override);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2006
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2007 ddt_enter(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2008 dde = ddt_lookup(ddt, bp, B_TRUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2009 ddp = &dde->dde_phys[p];
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2010
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2011 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2012 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2013 * If we're using a weak checksum, upgrade to a strong checksum
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2014 * and try again. If we're already using a strong checksum,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2015 * we can't resolve it, so just convert to an ordinary write.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2016 * (And automatically e-mail a paper to Nature?)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2017 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2018 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2019 zp->zp_checksum = spa_dedup_checksum(spa);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2020 zio_pop_transforms(zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2021 zio->io_stage = ZIO_STAGE_OPEN;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2022 BP_ZERO(bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2023 } else {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2024 zp->zp_dedup = 0;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2025 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2026 zio->io_pipeline = ZIO_WRITE_PIPELINE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2027 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2028 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2029 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2030
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2031 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2032 ASSERT(ditto_copies < SPA_DVAS_PER_BP);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2033
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2034 if (ditto_copies > ddt_ditto_copies_present(dde) &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2035 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2036 zio_prop_t czp = *zp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2037
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2038 czp.zp_copies = ditto_copies;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2039
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2040 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2041 * If we arrived here with an override bp, we won't have run
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2042 * the transform stack, so we won't have the data we need to
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2043 * generate a child i/o. So, toss the override bp and restart.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2044 * This is safe, because using the override bp is just an
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2045 * optimization; and it's rare, so the cost doesn't matter.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2046 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2047 if (zio->io_bp_override) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2048 zio_pop_transforms(zio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2049 zio->io_stage = ZIO_STAGE_OPEN;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2050 zio->io_pipeline = ZIO_WRITE_PIPELINE;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2051 zio->io_bp_override = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2052 BP_ZERO(bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2053 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2054 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2055 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2056
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2057 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2058 zio->io_orig_size, &czp, NULL,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2059 zio_ddt_ditto_write_done, dde, zio->io_priority,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2060 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2061
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2062 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2063 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2064 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2065
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2066 if (ddp->ddp_phys_birth != 0 \|\| dde->dde_lead_zio[p] != NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2067 if (ddp->ddp_phys_birth != 0)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2068 ddt_bp_fill(ddp, bp, txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2069 if (dde->dde_lead_zio[p] != NULL)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2070 zio_add_child(zio, dde->dde_lead_zio[p]);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2071 else
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2072 ddt_phys_addref(ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2073 } else if (zio->io_bp_override) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2074 ASSERT(bp->blk_birth == txg);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2075 ASSERT(BP_EQUAL(bp, zio->io_bp_override));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2076 ddt_phys_fill(ddp, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2077 ddt_phys_addref(ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2078 } else {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2079 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2080 zio->io_orig_size, zp, zio_ddt_child_write_ready,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2081 zio_ddt_child_write_done, dde, zio->io_priority,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2082 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2083
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2084 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2085 dde->dde_lead_zio[p] = cio;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2086 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2087
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2088 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2089
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2090 if (cio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2091 zio_nowait(cio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2092 if (dio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2093 zio_nowait(dio);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2094
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2095 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2096 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2097
12296 7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	2098 ddt_entry_t freedde; / for debugging */
7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	2099
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2100 static int
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2101 zio_ddt_free(zio_t *zio)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2102 {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2103 spa_t *spa = zio->io_spa;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2104 blkptr_t *bp = zio->io_bp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2105 ddt_t *ddt = ddt_select(spa, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2106 ddt_entry_t *dde;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2107 ddt_phys_t *ddp;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2108
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2109 ASSERT(BP_GET_DEDUP(bp));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2110 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2111
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2112 ddt_enter(ddt);
12296 7cf402a7f374 6675946 'zpool status' should show the progress of resilvering for individual disk. Lin Ling <Lin.Ling@Sun.COM> parents: 11670 diff changeset	2113 freedde = dde = ddt_lookup(ddt, bp, B_TRUE);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2114 ddp = ddt_phys_select(dde, bp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2115 ddt_phys_decref(ddp);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2116 ddt_exit(ddt);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2117
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2118 return (ZIO_PIPELINE_CONTINUE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2119 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2120
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2121 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2122 * ==========================================================================
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2123 * Allocate and free blocks
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2124 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2125 */
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2126 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2127 zio_dva_allocate(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2128 {
4527 5d5b6ba91b17 PSARC 2007/171 ZFS Separate Intent Log perrin parents: 4469 diff changeset	2129 spa_t *spa = zio->io_spa;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2130 metaslab_class_t *mc = spa_normal_class(spa);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2131 blkptr_t *bp = zio->io_bp;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2132 int error;
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2133 int flags = 0;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2134
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2135 if (zio->io_gang_leader == NULL) {
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2136 ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2137 zio->io_gang_leader = zio;
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2138 }
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2139
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2140 ASSERT(BP_IS_HOLE(bp));
13765 9410cf539b11 backout 3006: causes 3046 (panics after mounting root) Richard Lowe <richlowe@richlowe.net> parents: 13764 diff changeset	2141 ASSERT3U(BP_GET_NDVAS(bp), ==, 0);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2142 ASSERT3U(zio->io_prop.zp_copies, >, 0);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2143 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2144 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp));
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2145
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2146 /*
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2147 * The dump device does not support gang blocks so allocation on
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2148 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2149 * the "fast" gang feature.
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2150 */
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2151 flags \|= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0;
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2152 flags \|= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ?
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2153 METASLAB_GANG_CHILD : 0;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2154 error = metaslab_alloc(spa, mc, zio->io_size, bp,
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2155 zio->io_prop.zp_copies, zio->io_txg, NULL, flags);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2156
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2157 if (error) {
13379 4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2158 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, "
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2159 "size %llu, error %d", spa_name(spa), zio, zio->io_size,
4df42cc92254 1051 zfs should handle imbalanced luns George Wilson <George.Wilson@delphix.com> parents: 13177 diff changeset	2160 error);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2161 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2162 return (zio_write_gang_block(zio));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2163 zio->io_error = error;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2164 }
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2165
4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2166 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2167 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2168
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2169 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2170 zio_dva_free(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2171 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2172 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2173
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2174 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2175 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2176
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2177 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2178 zio_dva_claim(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2179 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2180 int error;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2181
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2182 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2183 if (error)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2184 zio->io_error = error;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2185
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2186 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2187 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2188
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2189 /*
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2190 * Undo an allocation. This is used by zio_done() when an I/O fails
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2191 * and we want to give back the block we just allocated.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2192 * This handles both normal blocks and gang blocks.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2193 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2194 static void
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2195 zio_dva_unallocate(zio_t zio, zio_gang_node_t gn, blkptr_t *bp)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2196 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2197 ASSERT(bp->blk_birth == zio->io_txg \|\| BP_IS_HOLE(bp));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2198 ASSERT(zio->io_bp_override == NULL);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2199
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2200 if (!BP_IS_HOLE(bp))
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2201 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2202
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2203 if (gn != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2204 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2205 zio_dva_unallocate(zio, gn->gn_child[g],
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2206 &gn->gn_gbh->zg_blkptr[g]);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2207 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2208 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2209 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2210
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2211 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2212 * Try to allocate an intent log block. Return 0 on success, errno on failure.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2213 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2214 int
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2215 zio_alloc_zil(spa_t spa, uint64_t txg, blkptr_t new_bp, blkptr_t *old_bp,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2216 uint64_t size, boolean_t use_slog)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2217 {
10310 ba87b3315737 PSARC 2009/423 ZFS logbias property Neil Perrin <Neil.Perrin@Sun.COM> parents: 9725 diff changeset	2218 int error = 1;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2219
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2220 ASSERT(txg > spa_syncing_txg(spa));
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2221
13572 85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2222 /*
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2223 * ZIL blocks are always contiguous (i.e. not gang blocks) so we
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2224 * set the METASLAB_GANG_AVOID flag so that they don't "fast gang"
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2225 * when allocating them.
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2226 */
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2227 if (use_slog) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2228 error = metaslab_alloc(spa, spa_log_class(spa), size,
13572 85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2229 new_bp, 1, txg, old_bp,
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2230 METASLAB_HINTBP_AVOID \| METASLAB_GANG_AVOID);
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2231 }
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2232
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2233 if (error) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2234 error = metaslab_alloc(spa, spa_normal_class(spa), size,
13572 85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2235 new_bp, 1, txg, old_bp,
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2236 METASLAB_HINTBP_AVOID \| METASLAB_GANG_AVOID);
85c66b89d5f2 1909 disk sync write perf regression when slog is used post oi_148 George Wilson <george.wilson@delphix.com> parents: 13414 diff changeset	2237 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2238
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2239 if (error == 0) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2240 BP_SET_LSIZE(new_bp, size);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2241 BP_SET_PSIZE(new_bp, size);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2242 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF);
11670 1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	2243 BP_SET_CHECKSUM(new_bp,
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	2244 spa_version(spa) >= SPA_VERSION_SLIM_ZIL
1d964fb5d948 6595532 ZIL is too talkative Neil Perrin <Neil.Perrin@Sun.COM> parents: 11173 diff changeset	2245 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2246 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2247 BP_SET_LEVEL(new_bp, 0);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2248 BP_SET_DEDUP(new_bp, 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2249 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2250 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2251
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2252 return (error);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2253 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2254
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2255 /*
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2256 * Free an intent log block.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2257 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2258 void
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2259 zio_free_zil(spa_t spa, uint64_t txg, blkptr_t bp)
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2260 {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2261 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2262 ASSERT(!BP_IS_GANG(bp));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2263
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2264 zio_free(spa, txg, bp);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2265 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2266
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2267 /*
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2268 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2269 * Read and write to physical devices
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2270 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2271 */
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2272 static int
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	2273 zio_vdev_io_start(zio_t *zio)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2274 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2275 vdev_t *vd = zio->io_vd;
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	2276 uint64_t align;
5329 33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	2277 spa_t *spa = zio->io_spa;
33cb98223b2d PSARC 2007/567 zpool failmode property gw25295 parents: 4831 diff changeset	2278
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2279 ASSERT(zio->io_error == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2280 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2281
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2282 if (vd == NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2283 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2284 spa_config_enter(spa, SCL_ZIO, zio, RW_READER);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2285
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2286 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2287 * The mirror_ops handle multiple DVAs in a single BP.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2288 */
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2289 return (vdev_mirror_ops.vdev_op_io_start(zio));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2290 }
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	2291
12586 b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2292 /*
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2293 * We keep track of time-sensitive I/Os so that the scan thread
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2294 * can quickly react to certain workloads. In particular, we care
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2295 * about non-scrubbing, top-level reads and writes with the following
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2296 * characteristics:
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2297 * - synchronous writes of user data to non-slog devices
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2298 * - any reads of user data
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2299 * When these conditions are met, adjust the timestamp of spa_last_io
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2300 * which allows the scan thread to adjust its workload accordingly.
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2301 */
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2302 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL &&
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2303 vd == vd->vdev_top && !vd->vdev_islog &&
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2304 zio->io_bookmark.zb_objset != DMU_META_OBJSET &&
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2305 zio->io_txg != spa_syncing_txg(spa)) {
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2306 uint64_t old = spa->spa_last_io;
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2307 uint64_t new = ddi_get_lbolt64();
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2308 if (old != new)
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2309 (void) atomic_cas_64(&spa->spa_last_io, old, new);
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2310 }
b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2311
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2312 align = 1ULL << vd->vdev_top->vdev_ashift;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2313
1732 9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2314 if (P2PHASE(zio->io_size, align) != 0) {
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2315 uint64_t asize = P2ROUNDUP(zio->io_size, align);
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2316 char *abuf = zio_buf_alloc(asize);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2317 ASSERT(vd == vd->vdev_top);
1732 9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2318 if (zio->io_type == ZIO_TYPE_WRITE) {
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2319 bcopy(zio->io_data, abuf, zio->io_size);
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2320 bzero(abuf + zio->io_size, asize - zio->io_size);
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2321 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2322 zio_push_transform(zio, abuf, asize, asize, zio_subblock);
1732 9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2323 }
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2324
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2325 ASSERT(P2PHASE(zio->io_offset, align) == 0);
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2326 ASSERT(P2PHASE(zio->io_size, align) == 0);
13061 bda0decf867b PSARC 2010/306 Read-only ZFS pools George Wilson <George.Wilson@Sun.COM> parents: 12586 diff changeset	2327 VERIFY(zio->io_type != ZIO_TYPE_WRITE \|\| spa_writeable(spa));
8241 5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2328
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2329 /*
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2330 * If this is a repair I/O, and there's no self-healing involved --
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2331 * that is, we're just resilvering what we expect to resilver --
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2332 * then don't do the I/O unless zio's txg is actually in vd's DTL.
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2333 * This prevents spurious resilvering with nested replication.
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2334 * For example, given a mirror of mirrors, (A+B)+(C+D), if only
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2335 * A is out of date, we'll read from C+D, then use the data to
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2336 * resilver A+B -- but we don't actually want to resilver B, just A.
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2337 * The top-level mirror has no way to know this, so instead we just
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2338 * discard unnecessary repairs as we work our way down the vdev tree.
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2339 * The same logic applies to any form of nested replication:
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2340 * ditto + mirror, RAID-Z + replacing, etc. This covers them all.
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2341 */
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2342 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2343 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) &&
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2344 zio->io_txg != 0 && /* not a delegated i/o */
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2345 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2346 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2347 zio_vdev_io_bypass(zio);
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2348 return (ZIO_PIPELINE_CONTINUE);
5a60f16123ba 6328632 zpool offline is a bit too conservative Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7872 diff changeset	2349 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2350
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2351 if (vd->vdev_ops->vdev_op_leaf &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2352 (zio->io_type == ZIO_TYPE_READ \|\| zio->io_type == ZIO_TYPE_WRITE)) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2353
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2354 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio) == 0)
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2355 return (ZIO_PIPELINE_CONTINUE);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2356
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2357 if ((zio = vdev_queue_io(zio)) == NULL)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2358 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2359
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2360 if (!vdev_accessible(vd, zio)) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2361 zio->io_error = ENXIO;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2362 zio_interrupt(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2363 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2364 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2365 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2366
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2367 return (vd->vdev_ops->vdev_op_io_start(zio));
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2368 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2369
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2370 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2371 zio_vdev_io_done(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2372 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2373 vdev_t *vd = zio->io_vd;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2374 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2375 boolean_t unexpected_error = B_FALSE;
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2376
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2377 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2378 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2379
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2380 ASSERT(zio->io_type == ZIO_TYPE_READ \|\| zio->io_type == ZIO_TYPE_WRITE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2381
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2382 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2383
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2384 vdev_queue_io_done(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2385
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2386 if (zio->io_type == ZIO_TYPE_WRITE)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2387 vdev_cache_write(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2388
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2389 if (zio_injection_enabled && zio->io_error == 0)
9725 0bf7402e8022 6843014 ZFS B_FAILFAST handling is broken Eric Schrock <Eric.Schrock@Sun.COM> parents: 9722 diff changeset	2390 zio->io_error = zio_handle_device_injection(vd,
0bf7402e8022 6843014 ZFS B_FAILFAST handling is broken Eric Schrock <Eric.Schrock@Sun.COM> parents: 9722 diff changeset	2391 zio, EIO);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2392
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2393 if (zio_injection_enabled && zio->io_error == 0)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2394 zio->io_error = zio_handle_label_injection(zio, EIO);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2395
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2396 if (zio->io_error) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2397 if (!vdev_accessible(vd, zio)) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2398 zio->io_error = ENXIO;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2399 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2400 unexpected_error = B_TRUE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2401 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2402 }
6976 cae5f06df471 PSARC 2008/388 Short circuit for vdev probe failure eschrock parents: 6523 diff changeset	2403 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2404
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2405 ops->vdev_op_io_done(zio);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2406
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2407 if (unexpected_error)
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2408 VERIFY(vdev_probe(vd, zio) == NULL);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2409
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2410 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2411 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2412
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2413 /*
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2414 * For non-raidz ZIOs, we can just copy aside the bad data read from the
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2415 * disk, and use that to finish the checksum ereport later.
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2416 */
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2417 static void
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2418 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr,
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2419 const void *good_buf)
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2420 {
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2421 /* no processing needed */
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2422 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE);
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2423 }
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2424
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2425 /ARGSUSED/
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2426 void
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2427 zio_vsd_default_cksum_report(zio_t zio, zio_cksum_report_t zcr, void *ignored)
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2428 {
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2429 void *buf = zio_buf_alloc(zio->io_size);
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2430
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2431 bcopy(zio->io_data, buf, zio->io_size);
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2432
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2433 zcr->zcr_cbinfo = zio->io_size;
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2434 zcr->zcr_cbdata = buf;
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2435 zcr->zcr_finish = zio_vsd_default_cksum_finish;
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2436 zcr->zcr_free = zio_buf_free;
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2437 }
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2438
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2439 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2440 zio_vdev_io_assess(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2441 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2442 vdev_t *vd = zio->io_vd;
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2443
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2444 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2445 return (ZIO_PIPELINE_STOP);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2446
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2447 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2448 spa_config_exit(zio->io_spa, SCL_ZIO, zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2449
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2450 if (zio->io_vsd != NULL) {
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2451 zio->io_vsd_ops->vsd_free(zio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2452 zio->io_vsd = NULL;
1732 9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2453 }
9e3ae798af31 6280668 pluggable block allocation policy bonwick parents: 1544 diff changeset	2454
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2455 if (zio_injection_enabled && zio->io_error == 0)
1544 938876158511 PSARC 2006/077 zpool clear eschrock parents: 896 diff changeset	2456 zio->io_error = zio_handle_fault_injection(zio, EIO);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2457
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2458 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2459 * If the I/O failed, determine whether we should attempt to retry it.
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	2460 *
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	2461 * On retry, we cut in line in the issue queue, since we don't want
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	2462 * compression/checksumming/etc. work to prevent our (cheap) IO reissue.
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2463 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2464 if (zio->io_error && vd == NULL &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2465 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY \| ZIO_FLAG_IO_RETRY))) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2466 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2467 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2468 zio->io_error = 0;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2469 zio->io_flags \|= ZIO_FLAG_IO_RETRY \|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2470 ZIO_FLAG_DONT_CACHE \| ZIO_FLAG_DONT_AGGREGATE;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2471 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;
11173 87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	2472 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE,
87f3734e64df 6881015 ZFS write activity prevents other threads from running in a timely manner Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 11147 diff changeset	2473 zio_requeue_io_start_cut_in_line);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2474 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2475 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2476
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2477 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2478 * If we got an error on a leaf device, convert it to ENXIO
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2479 * if the device is not accessible at all.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2480 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2481 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2482 !vdev_accessible(vd, zio))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2483 zio->io_error = ENXIO;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2484
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2485 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2486 * If we can't write to an interior vdev (mirror or RAID-Z),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2487 * set vdev_cant_write so that we stop trying to allocate from it.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2488 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2489 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE &&
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2490 vd != NULL && !vd->vdev_ops->vdev_op_leaf)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2491 vd->vdev_cant_write = B_TRUE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2492
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2493 if (zio->io_error)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2494 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2495
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2496 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2497 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2498
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2499 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2500 zio_vdev_io_reissue(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2501 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2502 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2503 ASSERT(zio->io_error == 0);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2504
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2505 zio->io_stage >>= 1;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2506 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2507
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2508 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2509 zio_vdev_io_redone(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2510 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2511 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2512
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2513 zio->io_stage >>= 1;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2514 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2515
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2516 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2517 zio_vdev_io_bypass(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2518 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2519 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2520 ASSERT(zio->io_error == 0);
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2521
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2522 zio->io_flags \|= ZIO_FLAG_IO_BYPASS;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2523 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2524 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2525
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2526 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2527 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2528 * Generate and verify checksums
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2529 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2530 */
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2531 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2532 zio_checksum_generate(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2533 {
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2534 blkptr_t *bp = zio->io_bp;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2535 enum zio_checksum checksum;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2536
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2537 if (bp == NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2538 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2539 * This is zio_write_phys().
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2540 * We're either generating a label checksum, or none at all.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2541 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2542 checksum = zio->io_prop.zp_checksum;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2543
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2544 if (checksum == ZIO_CHECKSUM_OFF)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2545 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2546
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2547 ASSERT(checksum == ZIO_CHECKSUM_LABEL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2548 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2549 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2550 ASSERT(!IO_IS_ALLOCATING(zio));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2551 checksum = ZIO_CHECKSUM_GANG_HEADER;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2552 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2553 checksum = BP_GET_CHECKSUM(bp);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2554 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2555 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2556
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2557 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2558
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2559 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2560 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2561
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2562 static int
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2563 zio_checksum_verify(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2564 {
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2565 zio_bad_cksum_t info;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2566 blkptr_t *bp = zio->io_bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2567 int error;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2568
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2569 ASSERT(zio->io_vd != NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2570
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2571 if (bp == NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2572 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2573 * This is zio_read_phys().
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2574 * We're either verifying a label checksum, or nothing at all.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2575 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2576 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2577 return (ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2578
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2579 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2580 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2581
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2582 if ((error = zio_checksum_error(zio, &info)) != 0) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2583 zio->io_error = error;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2584 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2585 zfs_ereport_start_checksum(zio->io_spa,
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2586 zio->io_vd, zio, zio->io_offset,
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2587 zio->io_size, NULL, &info);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2588 }
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2589 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2590
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2591 return (ZIO_PIPELINE_CONTINUE);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2592 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2593
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2594 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2595 * Called by RAID-Z to ensure we don't compute the checksum twice.
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2596 */
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2597 void
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2598 zio_checksum_verified(zio_t *zio)
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2599 {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2600 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2601 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2602
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2603 /*
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2604 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2605 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2606 * An error of 0 indictes success. ENXIO indicates whole-device failure,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2607 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2608 * indicate errors that are specific to one I/O, and most likely permanent.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2609 * Any other error is presumed to be worse because we weren't expecting it.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2610 * ==========================================================================
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2611 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2612 int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2613 zio_worst_error(int e1, int e2)
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2614 {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2615 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO };
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2616 int r1, r2;
1775 e51e26b432c0 6410698 ZFS metadata needs to be more highly replicated (ditto blocks) billm parents: 1732 diff changeset	2617
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2618 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2619 if (e1 == zio_error_rank[r1])
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2620 break;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2621
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2622 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2623 if (e2 == zio_error_rank[r2])
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2624 break;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2625
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2626 return (r1 > r2 ? e1 : e2);
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2627 }
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2628
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2629 /*
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2630 * ==========================================================================
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2631 * I/O completion
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2632 * ==========================================================================
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2633 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2634 static int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2635 zio_ready(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2636 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2637 blkptr_t *bp = zio->io_bp;
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2638 zio_t pio, pio_next;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2639
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2640 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) \|\|
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2641 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY))
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2642 return (ZIO_PIPELINE_STOP);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2643
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2644 if (zio->io_ready) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2645 ASSERT(IO_IS_ALLOCATING(zio));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2646 ASSERT(bp->blk_birth == zio->io_txg \|\| BP_IS_HOLE(bp));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2647 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2648
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2649 zio->io_ready(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2650 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2651
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2652 if (bp != NULL && bp != &zio->io_bp_copy)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2653 zio->io_bp_copy = *bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2654
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2655 if (zio->io_error)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2656 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2657
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2658 mutex_enter(&zio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2659 zio->io_state[ZIO_WAIT_READY] = 1;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2660 pio = zio_walk_parents(zio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2661 mutex_exit(&zio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2662
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2663 /*
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2664 * As we notify zio's parents, new parents could be added.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2665 * New parents go to the head of zio's io_parent_list, however,
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2666 * so we will (correctly) not notify them. The remainder of zio's
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2667 * io_parent_list, from 'pio_next' onward, cannot change because
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2668 * all parents must wait for us to be done before they can be done.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2669 */
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2670 for (; pio != NULL; pio = pio_next) {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2671 pio_next = zio_walk_parents(zio);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2672 zio_notify_parent(pio, zio, ZIO_WAIT_READY);
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2673 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2674
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2675 if (zio->io_flags & ZIO_FLAG_NODATA) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2676 if (BP_IS_GANG(bp)) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2677 zio->io_flags &= ~ZIO_FLAG_NODATA;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2678 } else {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2679 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2680 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2681 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2682 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2683
11026 e8e10df16a8f 6899159 injection isn't trashing pools Tim Haley <Tim.Haley@Sun.COM> parents: 10922 diff changeset	2684 if (zio_injection_enabled &&
e8e10df16a8f 6899159 injection isn't trashing pools Tim Haley <Tim.Haley@Sun.COM> parents: 10922 diff changeset	2685 zio->io_spa->spa_syncing_txg == zio->io_txg)
e8e10df16a8f 6899159 injection isn't trashing pools Tim Haley <Tim.Haley@Sun.COM> parents: 10922 diff changeset	2686 zio_handle_ignored_writes(zio);
e8e10df16a8f 6899159 injection isn't trashing pools Tim Haley <Tim.Haley@Sun.COM> parents: 10922 diff changeset	2687
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2688 return (ZIO_PIPELINE_CONTINUE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2689 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2690
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2691 static int
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2692 zio_done(zio_t *zio)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2693 {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2694 spa_t *spa = zio->io_spa;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2695 zio_t *lio = zio->io_logical;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2696 blkptr_t *bp = zio->io_bp;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2697 vdev_t *vd = zio->io_vd;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2698 uint64_t psize = zio->io_size;
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2699 zio_t pio, pio_next;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2700
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2701 /*
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2702 * If our children haven't all completed,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2703 * wait for them and then repeat this pipeline stage.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2704 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2705 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2706 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) \|\|
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2707 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) \|\|
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2708 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2709 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2710
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2711 for (int c = 0; c < ZIO_CHILD_TYPES; c++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2712 for (int w = 0; w < ZIO_WAIT_TYPES; w++)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2713 ASSERT(zio->io_children[c][w] == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2714
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2715 if (bp != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2716 ASSERT(bp->blk_pad[0] == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2717 ASSERT(bp->blk_pad[1] == 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2718 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 \|\|
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2719 (bp == zio_unique_parent(zio)->io_bp));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2720 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2721 zio->io_bp_override == NULL &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2722 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2723 ASSERT(!BP_SHOULD_BYTESWAP(bp));
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2724 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2725 ASSERT(BP_COUNT_GANG(bp) == 0 \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2726 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp)));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2727 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2728 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2729
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2730 /*
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2731 * If there were child vdev/gang/ddt errors, they apply to us now.
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2732 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2733 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2734 zio_inherit_child_errors(zio, ZIO_CHILD_GANG);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2735 zio_inherit_child_errors(zio, ZIO_CHILD_DDT);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2736
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2737 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2738 * If the I/O on the transformed data was successful, generate any
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2739 * checksum reports now while we still have the transformed data.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2740 */
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2741 if (zio->io_error == 0) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2742 while (zio->io_cksum_report != NULL) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2743 zio_cksum_report_t *zcr = zio->io_cksum_report;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2744 uint64_t align = zcr->zcr_align;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2745 uint64_t asize = P2ROUNDUP(psize, align);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2746 char *abuf = zio->io_data;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2747
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2748 if (asize != psize) {
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2749 abuf = zio_buf_alloc(asize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2750 bcopy(zio->io_data, abuf, psize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2751 bzero(abuf + psize, asize - psize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2752 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2753
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2754 zio->io_cksum_report = zcr->zcr_next;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2755 zcr->zcr_next = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2756 zcr->zcr_finish(zcr, abuf);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2757 zfs_ereport_free_checksum(zcr);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2758
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2759 if (asize != psize)
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2760 zio_buf_free(abuf, asize);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2761 }
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2762 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2763
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2764 zio_pop_transforms(zio); /* note: may set zio->io_error */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2765
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2766 vdev_stat_update(zio, psize);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2767
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2768 if (zio->io_error) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2769 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2770 * If this I/O is attached to a particular vdev,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2771 * generate an error message describing the I/O failure
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2772 * at the block level. We ignore these errors if the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2773 * device is currently unavailable.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2774 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2775 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd))
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2776 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2777
10685 931790026ac6 6846163 ZFS continues to use faulted logzilla, bringing system to a crawl George Wilson <George.Wilson@Sun.COM> parents: 10614 diff changeset	2778 if ((zio->io_error == EIO \|\| !(zio->io_flags &
931790026ac6 6846163 ZFS continues to use faulted logzilla, bringing system to a crawl George Wilson <George.Wilson@Sun.COM> parents: 10614 diff changeset	2779 (ZIO_FLAG_SPECULATIVE \| ZIO_FLAG_DONT_PROPAGATE))) &&
931790026ac6 6846163 ZFS continues to use faulted logzilla, bringing system to a crawl George Wilson <George.Wilson@Sun.COM> parents: 10614 diff changeset	2780 zio == lio) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2781 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2782 * For logical I/O requests, tell the SPA to log the
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2783 * error and generate a logical data ereport.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2784 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2785 spa_log_error(spa, zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2786 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2787 0, 0);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2788 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2789 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2790
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2791 if (zio->io_error && zio == lio) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2792 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2793 * Determine whether zio should be reexecuted. This will
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2794 * propagate all the way to the root via zio_notify_parent().
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2795 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2796 ASSERT(vd == NULL && bp != NULL);
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2797 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2798
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2799 if (IO_IS_ALLOCATING(zio) &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2800 !(zio->io_flags & ZIO_FLAG_CANFAIL)) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2801 if (zio->io_error != ENOSPC)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2802 zio->io_reexecute \|= ZIO_REEXECUTE_NOW;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2803 else
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2804 zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2805 }
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2806
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2807 if ((zio->io_type == ZIO_TYPE_READ \|\|
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2808 zio->io_type == ZIO_TYPE_FREE) &&
12586 b118bbd65be9 6494473 ZFS needs a way to slow down resilvering George Wilson <George.Wilson@Sun.COM> parents: 12470 diff changeset	2809 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2810 zio->io_error == ENXIO &&
11147 74e8c05021f1 6678033 resilver code should prefetch George Wilson <George.Wilson@Sun.COM> parents: 11146 diff changeset	2811 spa_load_state(spa) == SPA_LOAD_NONE &&
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2812 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2813 zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2814
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2815 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2816 zio->io_reexecute \|= ZIO_REEXECUTE_SUSPEND;
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2817
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2818 /*
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2819 * Here is a possibly good place to attempt to do
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2820 * either combinatorial reconstruction or error correction
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2821 * based on checksums. It also might be a good place
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2822 * to send out preliminary ereports before we suspend
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2823 * processing.
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2824 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2825 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2826
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2827 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2828 * If there were logical child errors, they apply to us now.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2829 * We defer this until now to avoid conflating logical child
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2830 * errors with errors that happened to the zio itself when
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2831 * updating vdev stats and reporting FMA events above.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2832 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2833 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2834
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2835 if ((zio->io_error \|\| zio->io_reexecute) &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2836 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2837 !(zio->io_flags & ZIO_FLAG_IO_REWRITE))
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2838 zio_dva_unallocate(zio, zio->io_gang_tree, bp);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2839
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2840 zio_gang_tree_free(&zio->io_gang_tree);
2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2841
9470 6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2842 /*
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2843 * Godfather I/Os should never suspend.
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2844 */
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2845 if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2846 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND))
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2847 zio->io_reexecute = 0;
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2848
6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2849 if (zio->io_reexecute) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2850 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2851 * This is a logical I/O that wants to reexecute.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2852 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2853 * Reexecute is top-down. When an i/o fails, if it's not
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2854 * the root, it simply notifies its parent and sticks around.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2855 * The parent, seeing that it still has children in zio_done(),
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2856 * does the same. This percolates all the way up to the root.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2857 * The root i/o will reexecute or suspend the entire tree.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2858 *
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2859 * This approach ensures that zio_reexecute() honors
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2860 * all the original i/o dependency relationships, e.g.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2861 * parents not executing until children are ready.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2862 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2863 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2864
9443 2a96d8478e95 6833711 gang leaders shouldn't have to be logical Bill Moore <Bill.Moore@Sun.COM> parents: 9396 diff changeset	2865 zio->io_gang_leader = NULL;
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2866
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2867 mutex_enter(&zio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2868 zio->io_state[ZIO_WAIT_DONE] = 1;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2869 mutex_exit(&zio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2870
9234 bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2871 /*
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2872 * "The Godfather" I/O monitors its children but is
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2873 * not a true parent to them. It will track them through
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2874 * the pipeline but severs its ties whenever they get into
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2875 * trouble (e.g. suspended). This allows "The Godfather"
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2876 * I/O to return status without blocking.
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2877 */
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2878 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) {
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2879 zio_link_t *zl = zio->io_walk_link;
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2880 pio_next = zio_walk_parents(zio);
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2881
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2882 if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2883 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2884 zio_remove_child(pio, zio, zl);
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2885 zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2886 }
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2887 }
bffdc4fc05c4 6792139 recovering from a suspended pool needs some work George Wilson <George.Wilson@Sun.COM> parents: 8632 diff changeset	2888
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2889 if ((pio = zio_unique_parent(zio)) != NULL) {
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2890 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2891 * We're not a root i/o, so there's nothing to do
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2892 * but notify our parent. Don't propagate errors
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2893 * upward since we haven't permanently failed yet.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2894 */
9470 6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2895 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2896 zio->io_flags \|= ZIO_FLAG_DONT_PROPAGATE;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2897 zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2898 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2899 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2900 * We'd fail again if we reexecuted now, so suspend
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2901 * until conditions improve (e.g. device comes online).
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2902 */
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2903 zio_suspend(spa, zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2904 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2905 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2906 * Reexecution is potentially a huge amount of work.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2907 * Hand it off to the otherwise-unused claim taskq.
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2908 */
13414 b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	2909 ASSERT(zio->io_tqent.tqent_next == NULL);
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	2910 (void) taskq_dispatch_ent(
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2911 spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
13414 b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	2912 (task_func_t *)zio_reexecute, zio, 0,
b42c1f0432b6 734 taskq_dispatch_prealloc() desired Garrett D'Amore <garrett@nexenta.com> parents: 13379 diff changeset	2913 &zio->io_tqent);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2914 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2915 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2916 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2917
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2918 ASSERT(zio->io_child_count == 0);
9470 6d827468d7b5 6834217 godfather I/O should reexecute George Wilson <George.Wilson@Sun.COM> parents: 9443 diff changeset	2919 ASSERT(zio->io_reexecute == 0);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2920 ASSERT(zio->io_error == 0 \|\| (zio->io_flags & ZIO_FLAG_CANFAIL));
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2921
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2922 /*
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2923 * Report any checksum errors, since the I/O is complete.
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2924 */
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2925 while (zio->io_cksum_report != NULL) {
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2926 zio_cksum_report_t *zcr = zio->io_cksum_report;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2927 zio->io_cksum_report = zcr->zcr_next;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2928 zcr->zcr_next = NULL;
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2929 zcr->zcr_finish(zcr, NULL);
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2930 zfs_ereport_free_checksum(zcr);
10614 4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2931 }
4f397871da47 PSARC 2009/497 zfs checksum ereport payload additions Jonathan Adams <Jonathan.Adams@Sun.COM> parents: 10310 diff changeset	2932
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2933 /*
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2934 * It is the responsibility of the done callback to ensure that this
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2935 * particular zio is no longer discoverable for adoption, and as
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2936 * such, cannot acquire any new parents.
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2937 */
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2938 if (zio->io_done)
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2939 zio->io_done(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2940
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2941 mutex_enter(&zio->io_lock);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2942 zio->io_state[ZIO_WAIT_DONE] = 1;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2943 mutex_exit(&zio->io_lock);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2944
8632 36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2945 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) {
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2946 zio_link_t *zl = zio->io_walk_link;
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2947 pio_next = zio_walk_parents(zio);
36ef517870a3 6798384 It can take a village to raise a zio Bill Moore <Bill.Moore@Sun.COM> parents: 8274 diff changeset	2948 zio_remove_child(pio, zio, zl);
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2949 zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2950 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2951
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2952 if (zio->io_waiter != NULL) {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2953 mutex_enter(&zio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2954 zio->io_executor = NULL;
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2955 cv_broadcast(&zio->io_cv);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2956 mutex_exit(&zio->io_lock);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2957 } else {
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2958 zio_destroy(zio);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2959 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2960
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2961 return (ZIO_PIPELINE_STOP);
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2962 }
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2963
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2964 /*
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2965 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2966 * I/O pipeline definition
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2967 * ==========================================================================
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2968 */
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2969 static zio_pipe_stage_t *zio_pipeline[] = {
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2970 NULL,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2971 zio_read_bp_init,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2972 zio_free_bp_init,
5530 4ed96167d864 6354519 stack overflow in zfs due to zio pipeline bonwick parents: 5450 diff changeset	2973 zio_issue_async,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2974 zio_write_bp_init,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2975 zio_checksum_generate,
10922 e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2976 zio_ddt_read_start,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2977 zio_ddt_read_done,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2978 zio_ddt_write,
e2081f502306 PSARC 2009/571 ZFS Deduplication Properties Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 10921 diff changeset	2979 zio_ddt_free,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2980 zio_gang_assemble,
b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2981 zio_gang_issue,
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2982 zio_dva_allocate,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2983 zio_dva_free,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2984 zio_dva_claim,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2985 zio_ready,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2986 zio_vdev_io_start,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2987 zio_vdev_io_done,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2988 zio_vdev_io_assess,
b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2989 zio_checksum_verify,
7754 b80e4842ad54 6754011 SPA 3.0: lock breakup, i/o pipeline refactoring, device failure handling Jeff Bonwick <Jeff.Bonwick@Sun.COM> parents: 7181 diff changeset	2990 zio_done
789 b348f31ed315 PSARC 2002/240 ZFS ahrens parents: diff changeset	2991 };
13700 2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2992
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2993 /* dnp is the dnode for zb1->zb_object */
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2994 boolean_t
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2995 zbookmark_is_before(const dnode_phys_t dnp, const zbookmark_t zb1,
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2996 const zbookmark_t *zb2)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2997 {
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2998 uint64_t zb1nextL0, zb2thisobj;
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	2999
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3000 ASSERT(zb1->zb_objset == zb2->zb_objset);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3001 ASSERT(zb2->zb_level == 0);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3002
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3003 /*
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3004 * A bookmark in the deadlist is considered to be after
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3005 * everything else.
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3006 */
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3007 if (zb2->zb_object == DMU_DEADLIST_OBJECT)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3008 return (B_TRUE);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3009
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3010 /* The objset_phys_t isn't before anything. */
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3011 if (dnp == NULL)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3012 return (B_FALSE);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3013
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3014 zb1nextL0 = (zb1->zb_blkid + 1) <<
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3015 ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT));
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3016
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3017 zb2thisobj = zb2->zb_object ? zb2->zb_object :
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3018 zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3019
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3020 if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3021 uint64_t nextobj = zb1nextL0 *
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3022 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT;
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3023 return (nextobj <= zb2thisobj);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3024 }
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3025
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3026 if (zb1->zb_object < zb2thisobj)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3027 return (B_TRUE);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3028 if (zb1->zb_object > zb2thisobj)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3029 return (B_FALSE);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3030 if (zb2->zb_object == DMU_META_DNODE_OBJECT)
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3031 return (B_FALSE);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3032 return (zb1nextL0 <= zb2->zb_blkid);
2889e2596bd6 2619 asynchronous destruction of ZFS file systems Christopher Siden <chris.siden@delphix.com> parents: 13572 diff changeset	3033 }

Mercurial > illumos > illumos-gate

annotate usr/src/uts/common/fs/zfs/zio.c @ 13765:9410cf539b11