diff options
Diffstat (limited to 'scripts/zloop.sh')
-rwxr-xr-x | scripts/zloop.sh | 358 |
1 files changed, 0 insertions, 358 deletions
diff --git a/scripts/zloop.sh b/scripts/zloop.sh deleted file mode 100755 index 04640396ce75..000000000000 --- a/scripts/zloop.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: CDDL-1.0 - -# -# CDDL HEADER START -# -# This file and its contents are supplied under the terms of the -# Common Development and Distribution License ("CDDL"), version 1.0. -# You may only use this file in accordance with the terms of version -# 1.0 of the CDDL. -# -# A full copy of the text of the CDDL should have accompanied this -# source. A copy of the CDDL is also available via the Internet at -# http://www.illumos.org/license/CDDL. -# -# CDDL HEADER END -# - -# -# Copyright (c) 2015 by Delphix. All rights reserved. -# Copyright (C) 2016 Lawrence Livermore National Security, LLC. -# Copyright (c) 2017, Intel Corporation. -# - -BASE_DIR=${0%/*} -SCRIPT_COMMON=common.sh -if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then - . "${BASE_DIR}/${SCRIPT_COMMON}" -else - echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 -fi - -# shellcheck disable=SC2034 -PROG=zloop.sh -GDB=${GDB:-gdb} - -DEFAULTWORKDIR=/var/tmp -DEFAULTCOREDIR=/var/tmp/zloop - -function usage -{ - cat >&2 <<EOF - -$0 [-hl] [-c <dump directory>] [-f <vdev directory>] - [-m <max core dumps>] [-s <vdev size>] [-t <timeout>] - [-I <max iterations>] [-- [extra ztest parameters]] - - This script runs ztest repeatedly with randomized arguments. - If a crash is encountered, the ztest logs, any associated - vdev files, and core file (if one exists) are moved to the - output directory ($DEFAULTCOREDIR by default). Any options - after the -- end-of-options marker will be passed to ztest. - - Options: - -c Specify a core dump directory to use. - -f Specify working directory for ztest vdev files. - -h Print this help message. - -l Create 'ztest.core.N' symlink to core directory. - -m Max number of core dumps to allow before exiting. - -s Size of vdev devices. - -t Total time to loop for, in seconds. If not provided, - zloop runs forever. - -I Max number of iterations to loop before exiting. - -EOF -} - -function or_die -{ - if ! "$@"; then - echo "Command failed: $*" - exit 1 - fi -} - -case $(uname) in -FreeBSD) - coreglob="z*.core" - ;; -Linux) - # core file helpers - read -r origcorepattern </proc/sys/kernel/core_pattern - coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" - - if [[ $coreglob = "*" ]]; then - echo "Setting core file pattern..." - echo "core" > /proc/sys/kernel/core_pattern - coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ - /proc/sys/kernel/core_pattern)*" - fi - ;; -*) - exit 1 - ;; -esac - -function core_file -{ - # shellcheck disable=SC2012,SC2086 - ls -tr1 $coreglob 2>/dev/null | head -1 -} - -function core_prog -{ - # shellcheck disable=SC2154 - prog=$ZTEST - core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ - tr \' ' ') - if [[ "$core_id" == *"zdb "* ]]; then - # shellcheck disable=SC2154 - prog=$ZDB - fi - printf "%s" "$prog" -} - -function store_core -{ - core="$(core_file)" - if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then - df -h "$workdir" >>ztest.out - coreid=$(date "+zloop-%y%m%d-%H%M%S") - foundcrashes=$((foundcrashes + 1)) - - # zdb debugging - zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" - zdbdebug=$($zdbcmd 2>&1) - echo -e "$zdbcmd\n" >>ztest.zdb - echo "$zdbdebug" >>ztest.zdb - - dest=$coredir/$coreid - or_die mkdir -p "$dest/vdev" - - if [[ $symlink -ne 0 ]]; then - or_die ln -sf "$dest" "ztest.core.${foundcrashes}" - fi - - echo "*** ztest crash found - moving logs to $dest" - - or_die mv ztest.history ztest.zdb ztest.out "$dest/" - or_die mv "$workdir/"ztest* "$dest/vdev/" - - if [[ -e "$workdir/zpool.cache" ]]; then - or_die mv "$workdir/zpool.cache" "$dest/vdev/" - fi - - # check for core - if [[ -f "$core" ]]; then - coreprog=$(core_prog "$core") - coredebug=$($GDB --batch --quiet \ - -ex "set print thread-events off" \ - -ex "printf \"*\n* Backtrace \n*\n\"" \ - -ex "bt" \ - -ex "printf \"*\n* Libraries \n*\n\"" \ - -ex "info sharedlib" \ - -ex "printf \"*\n* Threads (full) \n*\n\"" \ - -ex "info threads" \ - -ex "printf \"*\n* Backtraces \n*\n\"" \ - -ex "thread apply all bt" \ - -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ - -ex "thread apply all bt full" \ - -ex "quit" "$coreprog" "$core" 2>&1 | \ - grep -v "New LWP") - - # Dump core + logs to stored directory - echo "$coredebug" >>"$dest/ztest.gdb" - or_die mv "$core" "$dest/" - - # Record info in cores logfile - echo "*** core @ $coredir/$coreid/$core:" | \ - tee -a ztest.cores - fi - - if [[ $coremax -gt 0 ]] && - [[ $foundcrashes -ge $coremax ]]; then - echo "exiting... max $coremax allowed cores" - exit 1 - else - echo "continuing..." - fi - fi -} - -# parse arguments -# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] -coredir=$DEFAULTCOREDIR -basedir=$DEFAULTWORKDIR -rundir="zloop-run" -timeout=0 -size="512m" -coremax=0 -symlink=0 -iterations=0 -while getopts ":ht:m:I:s:c:f:l" opt; do - case $opt in - t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; - m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; - I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;; - s ) [[ -n $OPTARG ]] && size=$OPTARG ;; - c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;; - f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; - l ) symlink=1 ;; - h ) usage - exit 2 - ;; - * ) echo "Invalid argument: -$OPTARG"; - usage - exit 1 - esac -done -# pass remaining arguments on to ztest -shift $((OPTIND - 1)) - -# enable core dumps -ulimit -c unlimited -export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true -export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true - -if [[ -f "$(core_file)" ]]; then - echo -n "There's a core dump here you might want to look at first... " - core_file - echo - exit 1 -fi - -if [[ ! -d $coredir ]]; then - echo "core dump directory ($coredir) does not exist, creating it." - or_die mkdir -p "$coredir" -fi - -if [[ ! -w $coredir ]]; then - echo "core dump directory ($coredir) is not writable." - exit 1 -fi - -or_die rm -f ztest.history ztest.zdb ztest.cores - -ztrc=0 # ztest return value -foundcrashes=0 # number of crashes found so far -starttime=$(date +%s) -curtime=$starttime -iteration=0 - -# if no timeout was specified, loop forever. -while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do - if (( iterations > 0 )) && (( iteration++ == iterations )); then - break - fi - - zopt="-G -VVVVV" - - # start each run with an empty directory - workdir="$basedir/$rundir" - or_die rm -rf "$workdir" - or_die mkdir "$workdir" - - # ashift range 9 - 15 - align=$(((RANDOM % 2) * 3 + 9)) - - # choose parity value - parity=$(((RANDOM % 3) + 1)) - - draid_data=0 - draid_spares=0 - - # randomly use special classes - class="special=random" - - # choose between four types of configs - # (basic, raidz mix, raidz expansion, and draid mix) - case $((RANDOM % 4)) in - - # basic mirror configuration - 0) parity=1 - mirrors=2 - raid_children=0 - vdevs=2 - raid_type="raidz" - ;; - - # fully randomized mirror/raidz (sans dRAID) - 1) mirrors=$(((RANDOM % 3) * 1)) - raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) - vdevs=$(((RANDOM % 3) + 3)) - raid_type="raidz" - ;; - - # randomized raidz expansion (one top-level raidz vdev) - 2) mirrors=0 - vdevs=1 - # derive initial raidz disk count based on parity choice - # P1: 3 - 7 disks - # P2: 5 - 9 disks - # P3: 7 - 11 disks - raid_children=$(((RANDOM % 5) + (parity * 2) + 1)) - - # 1/3 of the time use a dedicated '-X' raidz expansion test - if [[ $((RANDOM % 3)) -eq 0 ]]; then - zopt="$zopt -X -t 16" - raid_type="raidz" - else - raid_type="eraidz" - fi - ;; - - # fully randomized dRAID (sans mirror/raidz) - 3) mirrors=0 - draid_data=$(((RANDOM % 8) + 3)) - draid_spares=$(((RANDOM % 2) + parity)) - stripe=$((draid_data + parity)) - extra=$((draid_spares + (RANDOM % 4))) - raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) - vdevs=$((RANDOM % 3)) - raid_type="draid" - ;; - *) - # avoid shellcheck SC2249 - ;; - esac - - zopt="$zopt -K $raid_type" - zopt="$zopt -m $mirrors" - zopt="$zopt -r $raid_children" - zopt="$zopt -D $draid_data" - zopt="$zopt -S $draid_spares" - zopt="$zopt -R $parity" - zopt="$zopt -v $vdevs" - zopt="$zopt -a $align" - zopt="$zopt -C $class" - zopt="$zopt -s $size" - zopt="$zopt -f $workdir" - - cmd="$ZTEST $zopt $*" - echo "$(date '+%m/%d %T') $cmd" | tee -a ztest.history ztest.out - $cmd >>ztest.out 2>&1 - ztrc=$? - grep -E '===|WARNING' ztest.out >>ztest.history - - store_core - - curtime=$(date +%s) -done - -echo "zloop finished, $foundcrashes crashes found" - -# restore core pattern. -case $(uname) in -Linux) - echo "$origcorepattern" > /proc/sys/kernel/core_pattern - ;; -*) - ;; -esac - -uptime >>ztest.out - -if [[ $foundcrashes -gt 0 ]]; then - exit 1 -fi |