Mercurial > illumos > git > illumos-omnios
changeset 21387:61706a713008
10952 defer new resilvers and misc. resilver-related fixes
Portions contributed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Portions contributed by: Brian Behlendorf <behlendorf1@llnl.gov>
Portions contributed by: Arkadiusz Bubała <arkadiusz.bubala@open-e.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed by: George Melikov <mail@gmelikov.ru>
Reviewed by: Tony Hutter <hutter2@llnl.gov>
Reviewed by: Don Brady <don.brady@delphix.com>
Reviewed by: Serapheim Dimitropoulos <serapheim.dimitro@delphix.com>
Reviewed by: Giuseppe Di Natale <guss80@gmail.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Kody Kantor <kody.kantor@joyent.com>
Approved by: Gordon Ross <gwr@nexenta.com>
line wrap: on
line diff
--- a/usr/src/cmd/zpool/zpool_main.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/cmd/zpool/zpool_main.c Fri May 17 14:34:57 2019 +0000 @@ -89,6 +89,7 @@ static int zpool_do_initialize(int, char **); static int zpool_do_scrub(int, char **); +static int zpool_do_resilver(int, char **); static int zpool_do_import(int, char **); static int zpool_do_export(int, char **); @@ -141,6 +142,7 @@ HELP_REMOVE, HELP_INITIALIZE, HELP_SCRUB, + HELP_RESILVER, HELP_STATUS, HELP_UPGRADE, HELP_GET, @@ -193,6 +195,7 @@ { "split", zpool_do_split, HELP_SPLIT }, { NULL }, { "initialize", zpool_do_initialize, HELP_INITIALIZE }, + { "resilver", zpool_do_resilver, HELP_RESILVER }, { "scrub", zpool_do_scrub, HELP_SCRUB }, { NULL }, { "import", zpool_do_import, HELP_IMPORT }, @@ -274,6 +277,8 @@ return (gettext("\tinitialize [-cs] <pool> [<device> ...]\n")); case HELP_SCRUB: return (gettext("\tscrub [-s | -p] <pool> ...\n")); + case HELP_RESILVER: + return (gettext("\tresilver <pool> ...\n")); case HELP_STATUS: return (gettext("\tstatus [-DgLPvx] [-T d|u] [pool] ... " "[interval [count]]\n")); @@ -1692,11 +1697,14 @@ (void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c); - if (ps != NULL && ps->pss_state == DSS_SCANNING && - vs->vs_scan_processed != 0 && children == 0) { - (void) printf(gettext(" (%s)"), - (ps->pss_func == POOL_SCAN_RESILVER) ? - "resilvering" : "repairing"); + if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) { + if (vs->vs_scan_processed != 0) { + (void) printf(gettext(" (%s)"), + (ps->pss_func == POOL_SCAN_RESILVER) ? + "resilvering" : "repairing"); + } else if (vs->vs_resilver_deferred) { + (void) printf(gettext(" (awaiting resilver)")); + } } if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE || @@ -4519,7 +4527,7 @@ * Ignore faulted pools. */ if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) { - (void) fprintf(stderr, gettext("cannot scrub '%s': pool is " + (void) fprintf(stderr, gettext("cannot scan '%s': pool is " "currently unavailable\n"), zpool_get_name(zhp)); return (1); } @@ -4587,6 +4595,43 @@ return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb)); } +/* + * zpool resilver <pool> ... + * + * Restarts any in-progress resilver + */ +int +zpool_do_resilver(int argc, char **argv) +{ + int c; + scrub_cbdata_t cb; + + cb.cb_type = POOL_SCAN_RESILVER; + cb.cb_scrub_cmd = POOL_SCRUB_NORMAL; + cb.cb_argc = argc; + cb.cb_argv = argv; + + /* check options */ + while ((c = getopt(argc, argv, "")) != -1) { + switch (c) { + case '?': + (void) fprintf(stderr, gettext("invalid option '%c'\n"), + optopt); + usage(B_FALSE); + } + } + + argc -= optind; + argv += optind; + + if (argc < 1) { + (void) fprintf(stderr, gettext("missing pool name argument\n")); + usage(B_FALSE); + } + + return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb)); +} + static void zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res) {
--- a/usr/src/common/zfs/zfeature_common.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/common/zfs/zfeature_common.c Fri May 17 14:34:57 2019 +0000 @@ -315,10 +315,13 @@ "freed or remapped.", ZFEATURE_FLAG_READONLY_COMPAT, obsolete_counts_deps); - { zfeature_register(SPA_FEATURE_ALLOCATION_CLASSES, "org.zfsonlinux:allocation_classes", "allocation_classes", "Support for separate allocation classes.", ZFEATURE_FLAG_READONLY_COMPAT, NULL); - } + + zfeature_register(SPA_FEATURE_RESILVER_DEFER, + "com.datto:resilver_defer", "resilver_defer", + "Support for defering new resilvers when one is already running.", + ZFEATURE_FLAG_READONLY_COMPAT, NULL); }
--- a/usr/src/common/zfs/zfeature_common.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/common/zfs/zfeature_common.h Fri May 17 14:34:57 2019 +0000 @@ -63,6 +63,7 @@ SPA_FEATURE_POOL_CHECKPOINT, SPA_FEATURE_SPACEMAP_V2, SPA_FEATURE_ALLOCATION_CLASSES, + SPA_FEATURE_RESILVER_DEFER, SPA_FEATURES } spa_feature_t;
--- a/usr/src/lib/libzfs/common/libzfs.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/lib/libzfs/common/libzfs.h Fri May 17 14:34:57 2019 +0000 @@ -140,6 +140,7 @@ EZFS_TOOMANY, /* argument list too long */ EZFS_INITIALIZING, /* currently initializing */ EZFS_NO_INITIALIZE, /* no active initialize */ + EZFS_NO_RESILVER_DEFER, /* pool doesn't support resilver_defer */ EZFS_UNKNOWN } zfs_error_t;
--- a/usr/src/lib/libzfs/common/libzfs_pool.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/lib/libzfs/common/libzfs_pool.c Fri May 17 14:34:57 2019 +0000 @@ -2035,6 +2035,10 @@ (void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN, "cannot scrub %s"), zc.zc_name); } + } else if (func == POOL_SCAN_RESILVER) { + assert(cmd == POOL_SCRUB_NORMAL); + (void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN, + "cannot restart resilver on %s"), zc.zc_name); } else if (func == POOL_SCAN_NONE) { (void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN, "cannot cancel scrubbing %s"), @@ -2062,6 +2066,8 @@ } } else if (err == ENOENT) { return (zfs_error(hdl, EZFS_NO_SCRUB, msg)); + } else if (err == ENOTSUP && func == POOL_SCAN_RESILVER) { + return (zfs_error(hdl, EZFS_NO_RESILVER_DEFER, msg)); } else { return (zpool_standard_error(hdl, err, msg)); }
--- a/usr/src/lib/libzfs/common/libzfs_util.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/lib/libzfs/common/libzfs_util.c Fri May 17 14:34:57 2019 +0000 @@ -260,6 +260,9 @@ case EZFS_NO_INITIALIZE: return (dgettext(TEXT_DOMAIN, "there is no active " "initialization")); + case EZFS_NO_RESILVER_DEFER: + return (dgettext(TEXT_DOMAIN, "this action requires the " + "resilver_defer feature")); case EZFS_UNKNOWN: return (dgettext(TEXT_DOMAIN, "unknown error")); default:
--- a/usr/src/man/man1m/zpool.1m Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/man/man1m/zpool.1m Fri May 17 14:34:57 2019 +0000 @@ -26,7 +26,7 @@ .\" Copyright (c) 2017 George Melikov. All Rights Reserved. .\" Copyright 2019 Joyent, Inc. .\" -.Dd April 27, 2018 +.Dd May 15, 2019 .Dt ZPOOL 1M .Os .Sh NAME @@ -156,6 +156,9 @@ .Op Fl f .Ar pool Ar device Op Ar new_device .Nm +.Cm resilver +.Ar pool Ns ... +.Nm .Cm scrub .Op Fl s | Fl p .Ar pool Ns ... @@ -1776,6 +1779,19 @@ .El .It Xo .Nm +.Cm resilver +.Ar pool Ns ... +.Xc +Starts a resilver. +If an existing resilver is already running it will be restarted from the +beginning. +Any drives that were scheduled for a deferred resilver will be added to the +new one. +This requires the +.Sy resilver_defer +feature. +.It Xo +.Nm .Cm scrub .Op Fl s | Fl p .Ar pool Ns ...
--- a/usr/src/man/man5/zpool-features.5 Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/man/man5/zpool-features.5 Fri May 17 14:34:57 2019 +0000 @@ -15,7 +15,7 @@ .\" CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your .\" own identifying information: .\" Portions Copyright [yyyy] [name of copyright owner] -.TH ZPOOL-FEATURES 5 "Apr 19, 2019" +.TH ZPOOL-FEATURES 5 "May 15, 2019" .SH NAME zpool\-features \- ZFS pool feature descriptions .SH DESCRIPTION @@ -683,5 +683,26 @@ removal, it can be returned to the \fBenabled\fR state if all the top-level vdevs from an allocation class are removed. +.RE +.sp +.ne 2 +.na +\fB\fBresilver_defer\fR\fR +.ad +.RS 4n +.TS +l l . +GUID com.datto:resilver_defer +READ\-ONLY COMPATIBLE yes +DEPENDENCIES none +.TE + +This feature allows zfs to postpone new resilvers if an existing one is already +in progress. Without this feature, any new resilvers will cause the currently +running one to be immediately restarted from the beginning. + +This feature becomes \fBactive\fR once a resilver has been deferred, and +returns to being \fBenabled\fR when the deferred resilver begins. + .SH "SEE ALSO" \fBzfs\fR(1M), \fBzpool\fR(1M)
--- a/usr/src/pkg/manifests/system-test-zfstest.mf Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/pkg/manifests/system-test-zfstest.mf Fri May 17 14:34:57 2019 +0000 @@ -88,7 +88,9 @@ dir path=opt/zfs-tests/tests/functional/cli_root/zpool_offline dir path=opt/zfs-tests/tests/functional/cli_root/zpool_online dir path=opt/zfs-tests/tests/functional/cli_root/zpool_remove +dir path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen dir path=opt/zfs-tests/tests/functional/cli_root/zpool_replace +dir path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver dir path=opt/zfs-tests/tests/functional/cli_root/zpool_scrub dir path=opt/zfs-tests/tests/functional/cli_root/zpool_set dir path=opt/zfs-tests/tests/functional/cli_root/zpool_status @@ -1727,6 +1729,36 @@ file \ path=opt/zfs-tests/tests/functional/cli_root/zpool_remove/zpool_remove_003_pos \ mode=0555 +file path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/cleanup \ + mode=0555 +file path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/setup mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.cfg \ + mode=0444 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib \ + mode=0444 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_001_pos \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_002_pos \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_003_pos \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_005_pos \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_006_neg \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_007_pos \ + mode=0555 file path=opt/zfs-tests/tests/functional/cli_root/zpool_replace/cleanup \ mode=0555 file path=opt/zfs-tests/tests/functional/cli_root/zpool_replace/setup \ @@ -1734,6 +1766,19 @@ file \ path=opt/zfs-tests/tests/functional/cli_root/zpool_replace/zpool_replace_001_neg \ mode=0555 +file path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver/cleanup \ + mode=0555 +file path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver/setup \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver.cfg \ + mode=0444 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver_bad_args \ + mode=0555 +file \ + path=opt/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver_restart \ + mode=0555 file path=opt/zfs-tests/tests/functional/cli_root/zpool_scrub/cleanup \ mode=0555 file path=opt/zfs-tests/tests/functional/cli_root/zpool_scrub/setup mode=0555
--- a/usr/src/test/zfs-tests/runfiles/delphix.run Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/runfiles/delphix.run Fri May 17 14:34:57 2019 +0000 @@ -339,9 +339,14 @@ [/opt/zfs-tests/tests/functional/cli_root/zpool_replace] tests = ['zpool_replace_001_neg'] +[/opt/zfs-tests/tests/functional/cli_root/zpool_resilver] +tests = ['zpool_resilver_bad_args', 'zpool_resilver_restart'] +tags = ['functional', 'cli_root', 'zpool_resilver'] + [/opt/zfs-tests/tests/functional/cli_root/zpool_scrub] tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos', - 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_multiple_copies'] + 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_print_repairing', + 'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies'] [/opt/zfs-tests/tests/functional/cli_root/zpool_set] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg']
--- a/usr/src/test/zfs-tests/runfiles/omnios.run Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/runfiles/omnios.run Fri May 17 14:34:57 2019 +0000 @@ -340,9 +340,14 @@ [/opt/zfs-tests/tests/functional/cli_root/zpool_replace] tests = ['zpool_replace_001_neg'] +[/opt/zfs-tests/tests/functional/cli_root/zpool_resilver] +tests = ['zpool_resilver_bad_args', 'zpool_resilver_restart'] +tags = ['functional', 'cli_root', 'zpool_resilver'] + [/opt/zfs-tests/tests/functional/cli_root/zpool_scrub] tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos', - 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_multiple_copies'] + 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_print_repairing', + 'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies'] [/opt/zfs-tests/tests/functional/cli_root/zpool_set] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg']
--- a/usr/src/test/zfs-tests/runfiles/openindiana.run Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/runfiles/openindiana.run Fri May 17 14:34:57 2019 +0000 @@ -340,9 +340,14 @@ [/opt/zfs-tests/tests/functional/cli_root/zpool_replace] tests = ['zpool_replace_001_neg'] +[/opt/zfs-tests/tests/functional/cli_root/zpool_resilver] +tests = ['zpool_resilver_bad_args', 'zpool_resilver_restart'] +tags = ['functional', 'cli_root', 'zpool_resilver'] + [/opt/zfs-tests/tests/functional/cli_root/zpool_scrub] tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos', - 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_multiple_copies'] + 'zpool_scrub_004_pos', 'zpool_scrub_005_pos', 'zpool_scrub_print_repairing', + 'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies'] [/opt/zfs-tests/tests/functional/cli_root/zpool_set] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg']
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg Fri May 17 14:34:57 2019 +0000 @@ -78,4 +78,5 @@ "feature@zpool_checkpoint" "feature@spacemap_v2" "feature@allocation_classes" + "feature@resilver_defer" )
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/cleanup.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/cleanup.ksh Fri May 17 14:34:57 2019 +0000 @@ -34,6 +34,8 @@ verify_runnable "global" +log_must set_tunable32 zfs_scan_suspend_progress 0 + for pool in "$TESTPOOL" "$TESTPOOL1"; do datasetexists $pool/$TESTFS && \ log_must zfs destroy -Rf $pool/$TESTFS
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/import_cachefile_device_replaced.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/import_cachefile_device_replaced.ksh Fri May 17 14:34:57 2019 +0000 @@ -42,10 +42,10 @@ # each sync. # 2. Add data to pool # 3. Re-import the pool so that data isn't cached -# 4. Use zinject to slow down device I/O +# 4. Use zfs_scan_suspend_progress to ensure resilvers don't progress # 5. Trigger the resilvering # 6. Use spa freeze to stop writing to the pool. -# 7. Clear zinject events (needed to export the pool) +# 7. Re-enable scan progress # 8. Export the pool # @@ -58,7 +58,7 @@ # Revert zfs_txg_timeout to defaults [[ -n ZFS_TXG_TIMEOUT ]] && log_must set_zfs_txg_timeout $ZFS_TXG_TIMEOUT - + log_must set_tunable32 zfs_scan_suspend_progress 0 cleanup } @@ -85,22 +85,16 @@ log_must zpool export $TESTPOOL1 log_must cp $CPATHBKP $CPATH log_must zpool import -c $CPATH -o cachefile=$CPATH $TESTPOOL1 - typeset device - for device in $zinjectdevices ; do - log_must zinject -d $device -D 200:1 $TESTPOOL1 > /dev/null - done + log_must set_tunable32 zfs_scan_suspend_progress 1 log_must zpool replace $TESTPOOL1 $replacevdev $replaceby # Cachefile: pool in resilvering state log_must cp $CPATH $CPATHBKP2 - # We must disable zinject in order to export the pool, so we freeze - # it first to prevent writing out subsequent resilvering progress. - log_must zpool freeze $TESTPOOL1 # Confirm pool is still replacing log_must pool_is_replacing $TESTPOOL1 - log_must zinject -c all > /dev/null log_must zpool export $TESTPOOL1 + log_must set_tunable32 zfs_scan_suspend_progress 0 ( $earlyremove ) && log_must rm $replacevdev
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_device_replaced.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_device_replaced.ksh Fri May 17 14:34:57 2019 +0000 @@ -63,6 +63,7 @@ [[ -n ZFS_TXG_TIMEOUT ]] && log_must set_zfs_txg_timeout $ZFS_TXG_TIMEOUT log_must rm -rf $BACKUP_DEVICE_DIR + log_must set_tunable32 zfs_scan_suspend_progress 0 cleanup } @@ -96,22 +97,17 @@ # This should not free original data. log_must overwrite_data $TESTPOOL1 "" - # Steps to insure resilvering happens very slowly. log_must zpool export $TESTPOOL1 log_must zpool import -d $DEVICE_DIR $TESTPOOL1 - typeset device - for device in $zinjectdevices ; do - log_must zinject -d $device -D 200:1 $TESTPOOL1 > /dev/null - done + + # Ensure resilvering doesn't complete. + log_must set_tunable32 zfs_scan_suspend_progress 1 log_must zpool replace $TESTPOOL1 $replacevdev $replaceby - # We must disable zinject in order to export the pool, so we freeze - # it first to prevent writing out subsequent resilvering progress. - log_must zpool freeze $TESTPOOL1 # Confirm pool is still replacing log_must pool_is_replacing $TESTPOOL1 - log_must zinject -c all > /dev/null log_must zpool export $TESTPOOL1 + log_must set_tunable32 zfs_scan_suspend_progress 0 ############################################################ # Test 1: rewind while device is resilvering.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/Makefile Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,21 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2019 Joyent, Inc. +# + +include $(SRC)/Makefile.master + +ROOTOPTPKG = $(ROOT)/opt/zfs-tests +TARGETDIR = $(ROOTOPTPKG)/tests/functional/cli_root/zpool_reopen + +include $(SRC)/test/zfs-tests/Makefile.com
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/cleanup.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,36 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved. +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +verify_runnable "global" + +cleanup_devices $DISKS + +# Unplug the disk and remove scsi_debug module +case "$(uname)" in +Linux) + for SDDEVICE in $(get_debug_device); do + remove_disk $SDDEVICE + done + unload_scsi_debug + ;; +esac + +log_pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/setup.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,34 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved. +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.cfg + +verify_runnable "global" + +# Create scsi_debug devices for the reopen tests +case "$(uname)" in +Linux) + load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS '512b' + ;; +SunOS) + log_unsupported "scsi debug module unsupported" + ;; +esac + +log_pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.cfg Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,49 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved. +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +export DISK_ARRAY_NUM=$(echo ${DISKS} | nawk '{print NF}') +export DISKSARRAY=$DISKS +export SMALL_FILE_SIZE=10 +export LARGE_FILE_SIZE=80 +export MAXTIMEOUT=40 + +export SDSIZE=256 +export SDHOSTS=1 +export SDTGTS=1 +export SDLUNS=1 + +export DISK1=$(echo $DISKS | nawk '{print $1}') +export DISK2=$(echo $DISKS | nawk '{print $2}') +export DISK3=$(echo $DISKS | nawk '{print $3}') + +case "$(uname)" in +Linux) + set_slice_prefix + set_device_dir + devs_id[0]=$(get_persistent_disk_name $DISK1) + devs_id[1]=$(get_persistent_disk_name $DISK2) + devs_id[2]=$(get_persistent_disk_name $DISK3) + export devs_id + ;; +SunOS) + DEV_DSKDIR="/dev" + ;; +esac
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,124 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.cfg + +# +# Clear labels on the given disks +# +function clear_labels #disks +{ + for disk in $@; do + if ( is_loop_device $disk ) || ( is_mpath_device $disk ); then + zpool labelclear -f /dev/$disk + else + zpool labelclear -f /dev/${disk}1 + fi + done +} + +# +# Set the REMOVED_DISK and REMOVED_DISK_ID constants for device +# used for re-plugging. When the disk is loop device use the +# scsi_debug emulated drive. Otherwise use the real drive. +# +function set_removed_disk +{ + if is_loop_device $DISK1; then + export REMOVED_DISK=$(get_debug_device) + export REMOVED_DISK_ID=$(get_persistent_disk_name $REMOVED_DISK) + elif ( is_real_device $DISK1 ) || ( is_mpath_device $DISK1 ); then + export REMOVED_DISK="$DISK1" + export REMOVED_DISK_ID=${devs_id[0]} + else + log_fail "No drives that supports removal" + fi +} + +# +# Generate random file of the given size in MiB +# +function generate_random_file #path size_mb +{ + typeset path=$1 + typeset -i size_mb=$2 + file_write -o create -f $path -b 1048576 -s0 -c $size_mb -d R +} + +# +# Wait until specific event or timeout occur. +# +# The passed function is executed with pool name as argument +# with an interval of 1 second until it succeeds or until the +# timeout occurs. +# It returns 1 on timeout or 0 otherwise. +# +function wait_for_action #pool timeout function +{ + typeset pool=$1 + typeset -i timeout=$2 + typeset func=$3 + + while [ $timeout -gt 0 ]; do + (( --timeout )) + if ( $func $pool ); then + return 0 + fi + sleep 1 + done + + return 1 +} + +# +# Helpers for wait_for_action function: +# wait_for_resilver_start - wait until resilver is started +# wait_for_resilver_end - wait until resilver is finished +# wait_for_scrub_end - wait until scrub is finished +# +function wait_for_resilver_start #pool timeout +{ + wait_for_action $1 $2 is_pool_resilvering + return $? +} + +function wait_for_resilver_end #pool timeout +{ + wait_for_action $1 $2 is_pool_resilvered + return $? +} + +function wait_for_scrub_end #pool timeout +{ + wait_for_action $1 $2 is_pool_scrubbed + return $? +} + +# +# Check if scan action has been restarted on the given pool +# + +function is_scan_restarted #pool +{ + typeset pool=$1 + zpool history -i $pool | grep -q "scan aborted, restarting" + return $? +} + +function is_deferred_scan_started #pool +{ + typeset pool=$1 + zpool history -i $pool | grep -q "starting deferred resilver" + return $? +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_001_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,70 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test if zpool reopen with no arguments works correctly. +# +# STRATEGY: +# 1. Create a pool. +# 2. Remove a disk. +# 3. Reopen a pool and verify if removed disk is marked as unavailable. +# 4. "Plug back" disk. +# 5. Reopen a pool and verify if removed disk is marked online again. +# 6. Check if reopen caused resilver start. +# + +verify_runnable "global" + +function cleanup +{ + # bring back removed disk online for further tests + insert_disk $REMOVED_DISK $scsi_host + poolexists $TESTPOOL && destroy_pool $TESTPOOL + clear_labels $REMOVED_DISK $DISK2 +} + +log_assert "Testing zpool reopen with no arguments" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a pool. +default_mirror_setup_noexit $REMOVED_DISK_ID $DISK2 +# 2. Remove a disk. +remove_disk $REMOVED_DISK +# 3. Reopen a pool and verify if removed disk is marked as unavailable. +log_must zpool reopen +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail" +# Write some data to the pool +log_must generate_random_file /$TESTPOOL/data $SMALL_FILE_SIZE +# 4. "Plug back" disk. +insert_disk $REMOVED_DISK $scsi_host +# 5. Reopen a pool and verify if removed disk is marked online again. +log_must zpool reopen +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "online" +# 6. Check if reopen caused resilver start. +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT + +# clean up +log_must zpool destroy $TESTPOOL +clear_labels $REMOVED_DISK $DISK2 + +log_pass "Zpool reopen with no arguments test passed"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_002_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,70 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test if zpool reopen with pool name as argument works correctly. +# +# STRATEGY: +# 1. Create a pool. +# 2. Remove a disk. +# 3. Reopen a pool and verify if removed disk is marked as unavailable. +# 4. "Plug back" disk. +# 5. Reopen a pool and verify if removed disk is marked online again. +# 6. Check if reopen caused resilver start. +# + +verify_runnable "global" + +function cleanup +{ + # bring back removed disk online for further tests + insert_disk $REMOVED_DISK $scsi_host + poolexists $TESTPOOL && destroy_pool $TESTPOOL + clear_labels $REMOVED_DISK $DISK2 +} + +log_assert "Testing zpool reopen with no arguments" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a pool. +default_mirror_setup_noexit $REMOVED_DISK_ID $DISK2 +# 2. Remove a disk. +remove_disk $REMOVED_DISK +# 3. Reopen a pool and verify if removed disk is marked as unavailable. +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail" +# Write some data to the pool +log_must generate_random_file /$TESTPOOL/data $SMALL_FILE_SIZE +# 4. "Plug back" disk. +insert_disk $REMOVED_DISK $scsi_host +# 5. Reopen a pool and verify if removed disk is marked online again. +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "online" +# 6. Check if reopen caused resilver start. +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT + +# clean up +log_must zpool destroy $TESTPOOL +clear_labels $REMOVED_DISK $DISK2 + +log_pass "Zpool reopen with no arguments test passed"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_003_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,101 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test zpool reopen while scrub is running. +# Checks if re-plugged device is fully resilvered. +# +# STRATEGY: +# 1. Create a pool +# 2. Remove a disk. +# 3. Write a test file to the pool and calculate its checksum. +# 4. Execute scrub. +# 5. "Plug back" disk. +# 6. Reopen a pool. +# 7. Check if scrub scan is replaced by resilver. +# 8. Put another device offline and check if the test file checksum is correct. +# +# NOTES: +# A 250ms delay is added to make sure that the scrub is running while +# the reopen kicks the resilver. +# + +verify_runnable "global" + +function cleanup +{ + log_must zinject -c all + rm -f $TESTFILE_MD5 2>/dev/null + # bring back removed disk online for further tests + insert_disk $REMOVED_DISK $scsi_host + poolexists $TESTPOOL && destroy_pool $TESTPOOL +} + +log_assert "Testing zpool reopen with pool name as argument" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a pool +default_mirror_setup_noexit $REMOVED_DISK_ID $DISK2 +# 2. Remove a disk. +remove_disk $REMOVED_DISK + +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail" + +# 3. Write a test file to the pool and calculate its checksum. +TESTFILE=/$TESTPOOL/data +TESTFILE_MD5=$(mktemp --tmpdir=/var/tmp) +log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE +log_must md5sum $TESTFILE > $TESTFILE_MD5 + +# 4. Execute scrub. +# add delay to I/O requests for remaining disk in pool +log_must zinject -d $DISK2 -D250:1 $TESTPOOL +log_must zpool scrub $TESTPOOL + +# 5. "Plug back" disk. +insert_disk $REMOVED_DISK $scsi_host +# 6. Reopen a pool. +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "online" +# 7. Check if scrub scan is replaced by resilver. +# the scrub operation has to be running while reopen is executed +log_must is_pool_scrubbing $TESTPOOL true +# remove delay from disk +log_must zinject -c all +# the scrub will be replaced by resilver, wait until it ends +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT +# check if the scrub scan has been interrupted by resilver +log_must is_scan_restarted $TESTPOOL + +# 8. Put another device offline and check if the test file checksum is correct. +log_must zpool offline $TESTPOOL $DISK2 +log_must md5sum -c $TESTFILE_MD5 +log_must zpool online $TESTPOOL $DISK2 +sleep 1 + +# clean up +rm -f $TESTFILE_MD5 2>/dev/null +log_must zpool destroy $TESTPOOL + +log_pass "Zpool reopen test successful"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,88 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test zpool reopen -n while scrub is running. +# Checks if re-plugged device is NOT resilvered. +# +# STRATEGY: +# 1. Create a pool +# 2. Remove a disk. +# 3. Write test file to pool. +# 4. Execute scrub. +# 5. "Plug back" disk. +# 6. Reopen a pool with an -n flag. +# 7. Check if resilver was deferred. +# 8. Check if trying to put device to offline fails because of no valid +# replicas. +# +# NOTES: +# A 125ms delay is added to make sure that the scrub is running while +# the reopen is invoked. +# + +verify_runnable "global" + +function cleanup +{ + log_must zinject -c all + # bring back removed disk online for further tests + insert_disk $REMOVED_DISK $scsi_host + poolexists $TESTPOOL && destroy_pool $TESTPOOL +} + +log_assert "Testing zpool reopen with pool name as argument" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a pool +default_mirror_setup_noexit $REMOVED_DISK_ID $DISK2 +# 2. Remove a disk. +remove_disk $REMOVED_DISK +log_must zpool reopen -n $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail" +# 3. Write test file to pool. +log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE +# 4. Execute scrub. +# add delay to I/O requests for remaining disk in pool +log_must zinject -d $DISK2 -D125:1 $TESTPOOL +log_must zpool scrub $TESTPOOL +# 5. "Plug back" disk. +insert_disk $REMOVED_DISK $scsi_host +# 6. Reopen a pool with an -n flag. +log_must zpool reopen -n $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "online" +# remove delay from disk +log_must zinject -c all +# 7. Check if scrub scan is NOT replaced by resilver. +log_must wait_for_scrub_end $TESTPOOL $MAXTIMEOUT +log_must is_deferred_scan_started $TESTPOOL + +# 8. Check if trying to put device to offline fails because of no valid +# replicas. +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT +log_must zpool offline $TESTPOOL $DISK2 + +# clean up +log_must zpool destroy $TESTPOOL + +log_pass "Zpool reopen test successful"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_005_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,86 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test zpool reopen -n while resilver is running. +# Checks if the resilver is restarted. +# +# STRATEGY: +# 1. Create a pool +# 2. Remove a disk. +# 3. Write test file to pool. +# 4. "Plug back" disk. +# 5. Reopen a pool and wait until resilvering is started. +# 6. Reopen a pool again with -n flag. +# 7. Wait until resilvering is finished and check if it was restarted. +# +# NOTES: +# A 25ms delay is added to make sure that the resilver is running while +# the reopen is invoked. +# + +verify_runnable "global" + +function cleanup +{ + log_must zinject -c all + insert_disk $REMOVED_DISK $scsi_host + poolexists $TESTPOOL && destroy_pool $TESTPOOL +} + +log_assert "Testing zpool reopen with pool name as argument" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a pool +default_mirror_setup_noexit $REMOVED_DISK_ID $DISK2 +# 2. Remove a disk. +remove_disk $REMOVED_DISK + +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail" +# 3. Write test file to pool. +log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE +# 4. "Plug back" disk. +insert_disk $REMOVED_DISK $scsi_host + +# 5. Reopen a pool and wait until resilvering is started. +log_must zpool reopen $TESTPOOL +log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "online" +# add delay to I/O requests for the reopened disk +log_must zinject -d $REMOVED_DISK_ID -D25:1 $TESTPOOL +# wait until resilver starts +log_must wait_for_resilver_start $TESTPOOL $MAXTIMEOUT + +# 6. Reopen a pool again with -n flag. +log_must zpool reopen -n $TESTPOOL + +# 7. Wait until resilvering is finished and check if it was restarted. +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT +# remove delay from disk +log_must zinject -c all +log_mustnot is_scan_restarted $TESTPOOL + +# clean up +log_must zpool destroy $TESTPOOL + +log_pass "Zpool reopen test successful"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_006_neg.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,43 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 Open-E, Inc. All Rights Reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Wrong arguments passed to zpool reopen should cause an error. +# +# STRATEGY: +# 1. Create an array with bad 'zpool reopen' arguments. +# 2. For each argument execute the 'zpool reopen' command and verify +# if it returns an error. +# + +verify_runnable "global" + +# 1. Create an array with bad 'zpool reopen' arguments. +typeset -a args=("!" "1" "-s" "--n" "-1" "-" "-c" "-f" "-d 2" "-abc" "-na") + +log_assert "Test 'zpool reopen' with invalid arguments." + +# 2. For each argument execute the 'zpool reopen' command and verify +# if it returns an error. +for arg in ${args[@]}; do + log_mustnot zpool reopen $arg +done + +log_pass "Passing invalid arguments to 'zpool reopen' failed as expected."
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_007_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,67 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib + +# +# DESCRIPTION: +# Test zpool reopen while performing IO to the pool. +# Verify that no IO errors of any kind of reported. +# +# STRATEGY: +# 1. Create a non-redundant pool. +# 2. Repeat: +# a. Write files to the pool. +# b. Execute 'zpool reopen'. +# 3. Verify that no errors are reported by 'zpool status'. + +verify_runnable "global" + +function cleanup +{ + poolexists $TESTPOOL && destroy_pool $TESTPOOL +} + +log_assert "Testing zpool reopen with concurrent user IO" +log_onexit cleanup + +set_removed_disk +scsi_host=$(get_scsi_host $REMOVED_DISK) + +# 1. Create a non-redundant pool. +log_must zpool create $TESTPOOL $DISK1 $DISK2 $DISK3 + +for i in $(seq 10); do + # 3a. Write files in the background to the pool. + mkfile 64m /$TESTPOOL/data.$i & + + # 3b. Execute 'zpool reopen'. + log_must zpool reopen $TESTPOOL + + for disk in $DISK1 $DISK2 $DISK3; do + zpool status -P -v $TESTPOOL | grep $disk | \ + read -r name state rd wr cksum + log_must [ $state = "ONLINE" ] + log_must [ $rd -eq 0 ] + log_must [ $wr -eq 0 ] + log_must [ $cksum -eq 0 ] + done +done + +wait + +log_pass "Zpool reopen with concurrent user IO successful"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/Makefile Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,21 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2019 Joyent, Inc. +# + +include $(SRC)/Makefile.master + +ROOTOPTPKG = $(ROOT)/opt/zfs-tests +TARGETDIR = $(ROOTOPTPKG)/tests/functional/cli_root/zpool_resilver + +include $(SRC)/test/zfs-tests/Makefile.com
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/cleanup.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,33 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_scrub/zpool_scrub.cfg + +verify_runnable "global" + +destroy_mirrors
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/setup.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,39 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_resilver/zpool_resilver.cfg + +verify_runnable "global" +verify_disk_count "$DISKS" 3 + +default_mirror_setup_noexit $DISK1 $DISK2 $DISK3 + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) + +# Create 256M of data +log_must file_write -b 1048576 -c 256 -o create -d 0 -f $mntpnt/bigfile +log_pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver.cfg Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,30 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto. All rights reserved. +# + +export DISK1=$(echo $DISKS | nawk '{print $1}') +export DISK2=$(echo $DISKS | nawk '{print $2}') +export DISK3=$(echo $DISKS | nawk '{print $3}') + +export MAXTIMEOUT=80
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver_bad_args.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,71 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# A badly formed parameter passed to 'zpool resilver' should +# return an error. +# +# STRATEGY: +# 1. Create an array containing bad 'zpool reilver' parameters. +# 2. For each element, execute the sub-command. +# 3. Verify it returns an error. +# 4. Confirm the sub-command returns an error if the resilver_defer +# feature isn't active. +# + +verify_runnable "global" + +set -A args "" "-?" "blah blah" "-%" "--?" "-*" "-=" \ + "-a" "-b" "-c" "-d" "-e" "-f" "-g" "-h" "-i" "-j" "-k" "-l" \ + "-m" "-n" "-o" "-p" "-q" "-r" "-s" "-t" "-u" "-v" "-w" "-x" "-y" "-z" \ + "-A" "-B" "-C" "-D" "-E" "-F" "-G" "-H" "-I" "-J" "-K" "-L" \ + "-M" "-N" "-O" "-P" "-Q" "-R" "-S" "-T" "-U" "-V" "-W" "-X" "-W" "-Z" + +function cleanup +{ + log_must destroy_pool $TESTPOOL2 + log_must rm -f $TEST_BASE_DIR/zpool_resilver.dat +} + +log_onexit cleanup + +log_assert "Execute 'zpool resilver' using invalid parameters." + +typeset -i i=0 +while [[ $i -lt ${#args[*]} ]]; do + log_mustnot zpool resilver ${args[i]} + + ((i = i + 1)) +done + +log_must mkfile $MINVDEVSIZE $TEST_BASE_DIR/zpool_resilver.dat +log_must zpool create -d $TESTPOOL2 $TEST_BASE_DIR/zpool_resilver.dat +log_mustnot zpool resilver $TESTPOOL2 + +log_pass "Badly formed 'zpool resilver' parameters fail as expected."
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_resilver/zpool_resilver_restart.ksh Fri May 17 14:34:57 2019 +0000 @@ -0,0 +1,86 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 Datto Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_reopen/zpool_reopen.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_resilver/zpool_resilver.cfg + +# +# DESCRIPTION: +# "Verify 'zpool resilver' restarts in-progress resilvers" +# +# STRATEGY: +# 1. Write some data and detatch the first drive so it has resilver +# work to do +# 2. Repeat the process with a second disk +# 3. Reattach the drives, causing the second drive's resilver to be +# deferred +# 4. Manually restart the resilver with all drives +# + +verify_runnable "global" + +function cleanup +{ + log_must set_tunable32 zfs_scan_suspend_progress 0 + log_must rm -f $mntpnt/biggerfile1 + log_must rm -f $mntpnt/biggerfile2 +} + +log_onexit cleanup + +log_assert "Verify 'zpool resilver' restarts in-progress resilvers" + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) + +# 1. Write some data and detatch the first drive so it has resilver work to do +log_must file_write -b 524288 -c 1024 -o create -d 0 -f $mntpnt/biggerfile1 +log_must sync +log_must zpool detach $TESTPOOL $DISK2 + +# 2. Repeat the process with a second disk +log_must file_write -b 524288 -c 1024 -o create -d 0 -f $mntpnt/biggerfile2 +log_must sync +log_must zpool detach $TESTPOOL $DISK3 + +# 3. Reattach the drives, causing the second drive's resilver to be deferred +log_must set_tunable32 zfs_scan_suspend_progress 1 + +log_must zpool attach $TESTPOOL $DISK1 $DISK2 +log_must is_pool_resilvering $TESTPOOL true + +log_must zpool attach $TESTPOOL $DISK1 $DISK3 +log_must is_pool_resilvering $TESTPOOL true + +# 4. Manually restart the resilver with all drives +log_must zpool resilver $TESTPOOL +log_must is_deferred_scan_started $TESTPOOL +log_must set_tunable32 zfs_scan_suspend_progress 0 +log_must wait_for_resilver_end $TESTPOOL $MAXTIMEOUT +log_must check_state $TESTPOOL "$DISK2" "online" +log_must check_state $TESTPOOL "$DISK3" "online" + +log_pass "Verified 'zpool resilver' restarts in-progress resilvers"
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/cleanup.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/cleanup.ksh Fri May 17 14:34:57 2019 +0000 @@ -30,5 +30,5 @@ verify_runnable "global" -log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT +log_must set_tunable32 zfs_scan_suspend_progress 0 destroy_mirrors
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_002_pos.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_002_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -45,18 +45,12 @@ # 5. Resume the paused scrub and verify scrub is again being performed. # 6. Verify zpool scrub -s succeed when the system is scrubbing. # -# NOTES: -# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit -# low and adding a 50ms zio delay in order to ensure that the scrub does -# not complete early. -# verify_runnable "global" function cleanup { - log_must zinject -c all - log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT + log_must set_tunable32 zfs_scan_suspend_progress 0 log_must rm -f $mntpnt/biggerfile } @@ -69,8 +63,7 @@ log_must file_write -b 1048576 -c 1024 -o create -d 0 -f $mntpnt/biggerfile log_must sync -log_must zinject -d $DISK1 -D50:1 $TESTPOOL -log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW +log_must set_tunable32 zfs_scan_suspend_progress 1 log_must zpool scrub $TESTPOOL log_must is_pool_scrubbing $TESTPOOL true log_must zpool scrub -p $TESTPOOL
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_003_pos.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_003_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -42,23 +42,19 @@ # 2. Kick off a scrub # 2. Kick off a second scrub and verify it fails # -# NOTES: -# Artificially limit the scrub speed by setting the zfs_scan_vdev_limit -# low in order to ensure that the scrub does not complete early. -# verify_runnable "global" function cleanup { - log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT + log_must set_tunable32 zfs_scan_suspend_progress 0 } log_onexit cleanup log_assert "Scrub command fails when there is already a scrub in progress" -log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW +log_must set_tunable32 zfs_scan_suspend_progress 1 log_must zpool scrub $TESTPOOL log_must is_pool_scrubbing $TESTPOOL true log_mustnot zpool scrub $TESTPOOL
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_004_pos.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_004_pos.ksh Fri May 17 14:34:57 2019 +0000 @@ -39,24 +39,40 @@ # STRATEGY: # 1. Setup a mirror pool and filled with data. # 2. Detach one of devices -# 3. Verify scrub failed until the resilver completed +# 3. Create a file for the resilver to work on so it takes some time +# 4. Export/import the pool to ensure the cache is dropped +# 5. Verify scrub failed until the resilver completed # function cleanup { - log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_DEFAULT + log_must set_tunable32 zfs_scan_suspend_progress 0 + rm -f $mntpnt/extra } verify_runnable "global" +log_onexit cleanup + log_assert "Resilver prevent scrub from starting until the resilver completes" -log_must set_tunable64 zfs_scan_vdev_limit $ZFS_SCAN_VDEV_LIMIT_SLOW -log_must zpool detach $TESTPOOL $DISK2 -log_must zpool attach $TESTPOOL $DISK1 $DISK2 +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) + +# Temporarily prevent scan progress so our test doesn't race +log_must set_tunable32 zfs_scan_suspend_progress 1 + +while ! is_pool_resilvering $TESTPOOL; do + log_must zpool detach $TESTPOOL $DISK2 + log_must file_write -b 1048576 -c 128 -o create -d 0 -f $mntpnt/extra + log_must zpool export $TESTPOOL + log_must zpool import $TESTPOOL + log_must zpool attach $TESTPOOL $DISK1 $DISK2 +done + log_must is_pool_resilvering $TESTPOOL log_mustnot zpool scrub $TESTPOOL +log_must set_tunable32 zfs_scan_suspend_progress 0 while ! is_pool_resilvered $TESTPOOL; do sleep 1 done
--- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_offline_device.ksh Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_offline_device.ksh Fri May 17 14:34:57 2019 +0000 @@ -22,6 +22,7 @@ # # Copyright 2017, loli10K <ezomori.nozomu@gmail.com>. All rights reserved. +# Copyright 2019 Joyent, Inc. # . $STF_SUITE/include/libtest.shlib @@ -125,7 +126,8 @@ zpool_scrub_sync $TESTPOOL2 # 7. Verify data integrity -cksum=$(zpool status $TESTPOOL2 | awk 'L{print $NF;L=0} /CKSUM$/{L=1}') +cksum=$(zpool status $TESTPOOL2 | awk '{if ($NF == "CKSUM") {fnd=1; next} \ + if (fnd && NF < 1) {fnd=0; next} if (fnd) csum += $NF} END {print csum}') if [[ $cksum != 0 ]]; then log_fail "Unexpected CKSUM errors found on $TESTPOOL2 ($cksum)" fi
--- a/usr/src/test/zfs-tests/tests/functional/removal/removal.kshlib Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/test/zfs-tests/tests/functional/removal/removal.kshlib Fri May 17 14:34:57 2019 +0000 @@ -62,7 +62,7 @@ typeset callback=$3 shift 3 - mdb_ctf_set_int zfs_remove_max_bytes_pause 0t0 + mdb_ctf_set_int zfs_removal_suspend_progress 0t1 log_must zpool remove $pool $disk @@ -81,7 +81,7 @@ # log_must is_pool_removing $pool - mdb_ctf_set_int zfs_remove_max_bytes_pause -0t1 + mdb_ctf_set_int zfs_removal_suspend_progress 0t0 log_must wait_for_removal $pool log_mustnot vdevs_in_pool $pool $disk
--- a/usr/src/uts/common/fs/zfs/dsl_scan.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/dsl_scan.c Fri May 17 14:34:57 2019 +0000 @@ -183,12 +183,15 @@ unsigned int zfs_obsolete_min_time_ms = 500; /* min millisecs to resilver per txg */ unsigned int zfs_resilver_min_time_ms = 3000; +int zfs_scan_suspend_progress = 0; /* set to prevent scans from progressing */ boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; /* max number of blocks to free in a single TXG */ uint64_t zfs_async_block_max_blocks = UINT64_MAX; +int zfs_resilver_disable_defer = 0; /* set to disable resilver deferring */ + /* * We wait a few txgs after importing a pool to begin scanning so that * the import / mounting code isn't held up by scrub / resilver IO. @@ -455,7 +458,6 @@ scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, SPA_FEATURE_ASYNC_DESTROY); - bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); avl_create(&scn->scn_queue, scan_ds_queue_compare, sizeof (scan_ds_t), offsetof(scan_ds_t, sds_node)); avl_create(&scn->scn_prefetch_queue, scan_prefetch_queue_compare, @@ -513,6 +515,8 @@ } } + bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); + /* reload the queue into the in-core state */ if (scn->scn_phys.scn_queue_obj != 0) { zap_cursor_t zc; @@ -751,6 +755,11 @@ spa->spa_scrub_reopen = B_FALSE; (void) spa_vdev_state_exit(spa, NULL, 0); + if (func == POOL_SCAN_RESILVER) { + dsl_resilver_restart(spa->spa_dsl_pool, 0); + return (0); + } + if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) { /* got scrub start cmd, resume paused scrub */ int err = dsl_scrub_set_pause_resume(scn->scn_dp, @@ -766,6 +775,41 @@ dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED)); } +/* + * Sets the resilver defer flag to B_FALSE on all leaf devs under vd. Returns + * B_TRUE if we have devices that need to be resilvered and are available to + * accept resilver I/Os. + */ +static boolean_t +dsl_scan_clear_deferred(vdev_t *vd, dmu_tx_t *tx) +{ + boolean_t resilver_needed = B_FALSE; + spa_t *spa = vd->vdev_spa; + + for (int c = 0; c < vd->vdev_children; c++) { + resilver_needed |= + dsl_scan_clear_deferred(vd->vdev_child[c], tx); + } + + if (vd == spa->spa_root_vdev && + spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) { + spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx); + vdev_config_dirty(vd); + spa->spa_resilver_deferred = B_FALSE; + return (resilver_needed); + } + + if (!vdev_is_concrete(vd) || vd->vdev_aux || + !vd->vdev_ops->vdev_op_leaf) + return (resilver_needed); + + if (vd->vdev_resilver_deferred) + vd->vdev_resilver_deferred = B_FALSE; + + return (!vdev_is_dead(vd) && !vd->vdev_offline && + vdev_resilver_needed(vd, NULL, NULL)); +} + /* ARGSUSED */ static void dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) @@ -865,6 +909,25 @@ * Let the async thread assess this and handle the detach. */ spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); + + /* + * Clear any deferred_resilver flags in the config. + * If there are drives that need resilvering, kick + * off an asynchronous request to start resilver. + * dsl_scan_clear_deferred() may update the config + * before the resilver can restart. In the event of + * a crash during this period, the spa loading code + * will find the drives that need to be resilvered + * when the machine reboots and start the resilver then. + */ + boolean_t resilver_needed = + dsl_scan_clear_deferred(spa->spa_root_vdev, tx); + if (resilver_needed) { + spa_history_log_internal(spa, + "starting deferred resilver", tx, + "errors=%llu", spa_get_errlog_size(spa)); + spa_async_request(spa, SPA_ASYNC_RESILVER); + } } scn->scn_phys.scn_end_time = gethrestime_sec(); @@ -935,6 +998,7 @@ /* can't pause a scrub when there is no in-progress scrub */ spa->spa_scan_pass_scrub_pause = gethrestime_sec(); scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED; + scn->scn_phys_cached.scn_flags |= DSF_SCRUB_PAUSED; dsl_scan_sync_state(scn, tx, SYNC_CACHED); spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_PAUSED); } else { @@ -949,6 +1013,7 @@ gethrestime_sec() - spa->spa_scan_pass_scrub_pause; spa->spa_scan_pass_scrub_pause = 0; scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; + scn->scn_phys_cached.scn_flags &= ~DSF_SCRUB_PAUSED; dsl_scan_sync_state(scn, tx, SYNC_CACHED); } } @@ -2335,6 +2400,20 @@ if (scn->scn_phys.scn_state != DSS_SCANNING) return; + /* + * This function is special because it is the only thing + * that can add scan_io_t's to the vdev scan queues from + * outside dsl_scan_sync(). For the most part this is ok + * as long as it is called from within syncing context. + * However, dsl_scan_sync() expects that no new sio's will + * be added between when all the work for a scan is done + * and the next txg when the scan is actually marked as + * completed. This check ensures we do not issue new sio's + * during this period. + */ + if (scn->scn_done_txg != 0) + return; + for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { if (ddp->ddp_phys_birth == 0 || ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) @@ -2986,6 +3065,26 @@ } static boolean_t +dsl_scan_check_deferred(vdev_t *vd) +{ + boolean_t need_resilver = B_FALSE; + + for (int c = 0; c < vd->vdev_children; c++) { + need_resilver |= + dsl_scan_check_deferred(vd->vdev_child[c]); + } + + if (!vdev_is_concrete(vd) || vd->vdev_aux || + !vd->vdev_ops->vdev_op_leaf) + return (need_resilver); + + if (!vd->vdev_resilver_deferred) + need_resilver = B_TRUE; + + return (need_resilver); +} + +static boolean_t dsl_scan_need_resilver(spa_t *spa, const dva_t *dva, size_t psize, uint64_t phys_birth) { @@ -3032,6 +3131,13 @@ if (!vdev_dtl_need_resilver(vd, DVA_GET_OFFSET(dva), psize)) return (B_FALSE); + /* + * Check that this top-level vdev has a device under it which + * is resilvering and is not deferred. + */ + if (!dsl_scan_check_deferred(vd)) + return (B_FALSE); + return (B_TRUE); } @@ -3193,12 +3299,19 @@ int err = 0; state_sync_type_t sync_type = SYNC_OPTIONAL; + if (spa->spa_resilver_deferred && + !spa_feature_is_active(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)) + spa_feature_incr(spa, SPA_FEATURE_RESILVER_DEFER, tx); + /* * Check for scn_restart_txg before checking spa_load_state, so * that we can restart an old-style scan while the pool is being - * imported (see dsl_scan_init). + * imported (see dsl_scan_init). We also restart scans if there + * is a deferred resilver and the user has manually disabled + * deferred resilvers via the tunable. */ - if (dsl_scan_restarting(scn, tx)) { + if (dsl_scan_restarting(scn, tx) || + (spa->spa_resilver_deferred && zfs_resilver_disable_defer)) { pool_scan_func_t func = POOL_SCAN_SCRUB; dsl_scan_done(scn, B_FALSE, tx); if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) @@ -3265,6 +3378,27 @@ return; /* + * zfs_scan_suspend_progress can be set to disable scan progress. + * We don't want to spin the txg_sync thread, so we add a delay + * here to simulate the time spent doing a scan. This is mostly + * useful for testing and debugging. + */ + if (zfs_scan_suspend_progress) { + uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time; + int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? + zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; + + while (zfs_scan_suspend_progress && + !txg_sync_waiting(scn->scn_dp) && + !spa_shutting_down(scn->scn_dp->dp_spa) && + NSEC2MSEC(scan_time_ns) < mintime) { + delay(hz); + scan_time_ns = gethrtime() - scn->scn_sync_start_time; + } + return; + } + + /* * It is possible to switch from unsorted to sorted at any time, * but afterwards the scan will remain sorted unless reloaded from * a checkpoint after a reboot. @@ -3393,6 +3527,8 @@ (longlong_t)tx->tx_txg); } } else if (scn->scn_is_sorted && scn->scn_bytes_pending != 0) { + ASSERT(scn->scn_clearing); + /* need to issue scrubbing IOs from per-vdev queues */ scn->scn_zio_root = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
--- a/usr/src/uts/common/fs/zfs/spa.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/spa.c Fri May 17 14:34:57 2019 +0000 @@ -6103,9 +6103,14 @@ /* * Schedule the resilver to restart in the future. We do this to * ensure that dmu_sync-ed blocks have been stitched into the - * respective datasets. - */ - dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg); + * respective datasets. We do not do this if resilvers have been + * deferred. + */ + if (dsl_scan_resilvering(spa_get_dsl(spa)) && + spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) + vdev_set_deferred_resilver(spa, newvd); + else + dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg); if (spa->spa_bootfs) spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); @@ -6996,6 +7001,10 @@ if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) return (SET_ERROR(ENOTSUP)); + if (func == POOL_SCAN_RESILVER && + !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) + return (SET_ERROR(ENOTSUP)); + /* * If a resilver was requested, but there is no DTL on a * writeable leaf device, we have nothing to do. @@ -7087,6 +7096,7 @@ spa_async_thread(void *arg) { spa_t *spa = (spa_t *)arg; + dsl_pool_t *dp = spa->spa_dsl_pool; int tasks; ASSERT(spa->spa_sync_on); @@ -7162,8 +7172,10 @@ /* * Kick off a resilver. */ - if (tasks & SPA_ASYNC_RESILVER) - dsl_resilver_restart(spa->spa_dsl_pool, 0); + if (tasks & SPA_ASYNC_RESILVER && + (!dsl_scan_resilvering(dp) || + !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) + dsl_resilver_restart(dp, 0); if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { mutex_enter(&spa_namespace_lock);
--- a/usr/src/uts/common/fs/zfs/sys/spa_impl.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/sys/spa_impl.h Fri May 17 14:34:57 2019 +0000 @@ -279,6 +279,13 @@ uint64_t spa_scan_pass_scrub_spent_paused; /* total paused */ uint64_t spa_scan_pass_exam; /* examined bytes per pass */ uint64_t spa_scan_pass_issued; /* issued bytes per pass */ + + /* + * We are in the middle of a resilver, and another resilver + * is needed once this one completes. This is set iff any + * vdev_resilver_deferred is set. + */ + boolean_t spa_resilver_deferred; kmutex_t spa_async_lock; /* protect async state */ kthread_t *spa_async_thread; /* thread doing async task */ int spa_async_suspended; /* async tasks suspended */
--- a/usr/src/uts/common/fs/zfs/sys/vdev.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/sys/vdev.h Fri May 17 14:34:57 2019 +0000 @@ -149,6 +149,8 @@ extern void vdev_state_dirty(vdev_t *vd); extern void vdev_state_clean(vdev_t *vd); +extern void vdev_set_deferred_resilver(spa_t *spa, vdev_t *vd); + typedef enum vdev_config_flag { VDEV_CONFIG_SPARE = 1 << 0, VDEV_CONFIG_L2CACHE = 1 << 1,
--- a/usr/src/uts/common/fs/zfs/sys/vdev_impl.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/sys/vdev_impl.h Fri May 17 14:34:57 2019 +0000 @@ -346,6 +346,7 @@ boolean_t vdev_cant_write; /* vdev is failing all writes */ boolean_t vdev_isspare; /* was a hot spare */ boolean_t vdev_isl2cache; /* was a l2cache device */ + boolean_t vdev_resilver_deferred; /* resilver deferred */ vdev_queue_t vdev_queue; /* I/O deadline schedule queue */ vdev_cache_t vdev_cache; /* physical block cache */ spa_aux_vdev_t *vdev_aux; /* for l2cache and spares vdevs */
--- a/usr/src/uts/common/fs/zfs/vdev.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/vdev.c Fri May 17 14:34:57 2019 +0000 @@ -759,6 +759,9 @@ (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_RESILVER_TXG, &vd->vdev_resilver_txg); + if (nvlist_exists(nv, ZPOOL_CONFIG_RESILVER_DEFER)) + vdev_set_deferred_resilver(spa, vd); + /* * When importing a pool, we want to ignore the persistent fault * state, as the diagnosis made on another system may not be @@ -1732,8 +1735,13 @@ * since this would just restart the scrub we are already doing. */ if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen && - vdev_resilver_needed(vd, NULL, NULL)) - spa_async_request(spa, SPA_ASYNC_RESILVER); + vdev_resilver_needed(vd, NULL, NULL)) { + if (dsl_scan_resilvering(spa->spa_dsl_pool) && + spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) + vdev_set_deferred_resilver(spa, vd); + else + spa_async_request(spa, SPA_ASYNC_RESILVER); + } return (0); } @@ -2440,6 +2448,9 @@ if (vd->vdev_state < VDEV_STATE_DEGRADED) return (B_FALSE); + if (vd->vdev_resilver_deferred) + return (B_FALSE); + if (vd->vdev_resilver_txg == 0 || range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) return (B_TRUE); @@ -3473,8 +3484,14 @@ if (vd != rvd && vdev_writeable(vd->vdev_top)) vdev_state_dirty(vd->vdev_top); - if (vd->vdev_aux == NULL && !vdev_is_dead(vd)) - spa_async_request(spa, SPA_ASYNC_RESILVER); + if (vd->vdev_aux == NULL && !vdev_is_dead(vd)) { + if (dsl_scan_resilvering(spa->spa_dsl_pool) && + spa_feature_is_enabled(spa, + SPA_FEATURE_RESILVER_DEFER)) + vdev_set_deferred_resilver(spa, vd); + else + spa_async_request(spa, SPA_ASYNC_RESILVER); + } spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_CLEAR); } @@ -3617,6 +3634,8 @@ vs->vs_fragmentation = (vd->vdev_mg != NULL) ? vd->vdev_mg->mg_fragmentation : 0; } + if (vd->vdev_ops->vdev_op_leaf) + vs->vs_resilver_deferred = vd->vdev_resilver_deferred; /* * If we're getting stats on the root vdev, aggregate the I/O counts @@ -4330,3 +4349,18 @@ mutex_exit(&vq->vq_lock); } } + +void +vdev_set_deferred_resilver(spa_t *spa, vdev_t *vd) +{ + for (uint64_t i = 0; i < vd->vdev_children; i++) + vdev_set_deferred_resilver(spa, vd->vdev_child[i]); + + if (!vd->vdev_ops->vdev_op_leaf || !vdev_writeable(vd) || + range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) { + return; + } + + vd->vdev_resilver_deferred = B_TRUE; + spa->spa_resilver_deferred = B_TRUE; +}
--- a/usr/src/uts/common/fs/zfs/vdev_indirect.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/vdev_indirect.c Fri May 17 14:34:57 2019 +0000 @@ -1239,6 +1239,8 @@ { indirect_vsd_t *iv = zio->io_vsd; + ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); + for (indirect_split_t *is = list_head(&iv->iv_splits); is != NULL; is = list_next(&iv->iv_splits, is)) { for (int i = 0; i < is->is_children; i++) { @@ -1321,7 +1323,8 @@ vdev_indirect_child_io_done, zio)); } else { iv->iv_split_block = B_TRUE; - if (zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)) { + if (zio->io_type == ZIO_TYPE_READ && + zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)) { /* * Read all copies. Note that for simplicity, * we don't bother consulting the DTL in the @@ -1330,13 +1333,17 @@ vdev_indirect_read_all(zio); } else { /* - * Read one copy of each split segment, from the - * top-level vdev. Since we don't know the - * checksum of each split individually, the child - * zio can't ensure that we get the right data. - * E.g. if it's a mirror, it will just read from a - * random (healthy) leaf vdev. We have to verify - * the checksum in vdev_indirect_io_done(). + * If this is a read zio, we read one copy of each + * split segment, from the top-level vdev. Since + * we don't know the checksum of each split + * individually, the child zio can't ensure that + * we get the right data. E.g. if it's a mirror, + * it will just read from a random (healthy) leaf + * vdev. We have to verify the checksum in + * vdev_indirect_io_done(). + * + * For write zios, the vdev code will ensure we write + * to all children. */ for (indirect_split_t *is = list_head(&iv->iv_splits); is != NULL; is = list_next(&iv->iv_splits, is)) {
--- a/usr/src/uts/common/fs/zfs/vdev_label.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/vdev_label.c Fri May 17 14:34:57 2019 +0000 @@ -377,6 +377,12 @@ fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_TOP_ZAP, vd->vdev_top_zap); } + + if (vd->vdev_resilver_deferred) { + ASSERT(vd->vdev_ops->vdev_op_leaf); + ASSERT(spa->spa_resilver_deferred); + fnvlist_add_boolean(nv, ZPOOL_CONFIG_RESILVER_DEFER); + } } if (getstats) {
--- a/usr/src/uts/common/fs/zfs/vdev_removal.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/vdev_removal.c Fri May 17 14:34:57 2019 +0000 @@ -127,7 +127,7 @@ * This is used by the test suite so that it can ensure that certain * actions happen while in the middle of a removal. */ -uint64_t zfs_remove_max_bytes_pause = UINT64_MAX; +int zfs_removal_suspend_progress = 0; #define VDEV_REMOVAL_ZAP_OBJS "lzap" @@ -1433,14 +1433,14 @@ /* * This delay will pause the removal around the point - * specified by zfs_remove_max_bytes_pause. We do this + * specified by zfs_removal_suspend_progress. We do this * solely from the test suite or during debugging. */ uint64_t bytes_copied = spa->spa_removing_phys.sr_copied; for (int i = 0; i < TXG_SIZE; i++) bytes_copied += svr->svr_bytes_done[i]; - while (zfs_remove_max_bytes_pause <= bytes_copied && + while (zfs_removal_suspend_progress && !svr->svr_thread_exit) delay(hz);
--- a/usr/src/uts/common/fs/zfs/zil.c Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/fs/zfs/zil.c Fri May 17 14:34:57 2019 +0000 @@ -1252,7 +1252,7 @@ * root zios). This is required because of how we can * defer the DKIOCFLUSHWRITECACHE commands for each lwb. * - * When the DKIOCFLUSHWRITECACHE commands are defered, + * When the DKIOCFLUSHWRITECACHE commands are deferred, * the previous lwb will rely on this lwb to flush the * vdevs written to by that previous lwb. Thus, we need * to ensure this lwb doesn't issue the flush until
--- a/usr/src/uts/common/sys/fs/zfs.h Wed May 22 20:34:50 2019 +0300 +++ b/usr/src/uts/common/sys/fs/zfs.h Fri May 17 14:34:57 2019 +0000 @@ -597,6 +597,7 @@ #define ZPOOL_CONFIG_VDEV_TOP_ZAP "com.delphix:vdev_zap_top" #define ZPOOL_CONFIG_VDEV_LEAF_ZAP "com.delphix:vdev_zap_leaf" #define ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS "com.delphix:has_per_vdev_zaps" +#define ZPOOL_CONFIG_RESILVER_DEFER "com.datto:resilver_defer" #define ZPOOL_CONFIG_CACHEFILE "cachefile" /* not stored on disk */ #define ZPOOL_CONFIG_MMP_STATE "mmp_state" /* not stored on disk */ #define ZPOOL_CONFIG_MMP_TXG "mmp_txg" /* not stored on disk */ @@ -895,6 +896,7 @@ uint64_t vs_initialize_state; /* vdev_initialzing_state_t */ uint64_t vs_initialize_action_time; /* time_t */ uint64_t vs_checkpoint_space; /* checkpoint-consumed space */ + uint64_t vs_resilver_deferred; /* resilver deferred */ } vdev_stat_t; /*