checkutils.sh 15.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
# Copyright (c) 2013 University of Utah and the Flux Group.
# 
# {{{EMULAB-LICENSE
# 
# This file is part of the Emulab network testbed software.
# 
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
# 
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this file.  If not, see <http://www.gnu.org/licenses/>.
# 
# }}}
#

Dan Reading's avatar
Dan Reading committed
24
25
26
27
28
29
30
31
32
33
34
35
36

#exit on unbound var
set -u
#exit on any error
set -e

#only source this file once
if [ "${checkutils+"beenhere"}" == "beenhere" ] ; then
    return 0
else
    checkutils="sourced"
fi

37
38
39
40
if [ -z "${BINDIR-""}" -a -f "/etc/emulab/paths.sh" ]; then
    source /etc/emulab/paths.sh
fi

Dan Reading's avatar
Dan Reading committed
41
declare mfsmode="" #are we running in a MFS
Dan Reading's avatar
Dan Reading committed
42
43
44
45
46
47
48
49
if [ -f /etc/emulab/ismfs ] ; then
    mfsmode=1
else
    mfsmode=0
fi

declare errext_val # used?

50
# Global Vars
Dan Reading's avatar
Dan Reading committed
51
52
53
54
55
declare NOSM="echo" #do nothing command
declare host       #emulab hostname
declare failed=""  #major falure to be commicated to user
declare os=""      #[Linux|FreeBSD] for now
declare -a todo_exit
56
57
declare -A hwinv  # hwinv from tmcc
declare -A hwinvcopy  # a copy of hwinv from tmcc
Dan Reading's avatar
Dan Reading committed
58
59
60
61
62
63
64


#declare -A tcm_out # hwinv for output
#declare -A tcm_inv # what we have discovered
# declare -p todo_exit

# any command causes exit if -e option set
Dan Reading's avatar
Dan Reading committed
65
66
# including a grep just used so see if some string is in a file
# have a way to save current state and restore
Dan Reading's avatar
Dan Reading committed
67
68
69
70
71
72
73
save_e() {
    x=$-
    [[ "${x/e}" == "${x}" ]] &&	errexit_val=off || errexit_val=on
}
restore_e() {
    [[ $errexit_val == "on" ]] && set -e || set +e
}
Dan Reading's avatar
Dan Reading committed
74

Dan Reading's avatar
Dan Reading committed
75
76
77
78
79
80
81
82
83
# give some indication of exit on ERR trap
err_report() {
    echo "TRAP ERR at $1"
}
#trap 'err_report $FUNCNAME:$LINENO' ERR
trap 'err_report $LINENO' ERR


# read info from tmcc no args uses the globel array hwinv
84
# if $1 then use that for a input file else use tmcc
Dan Reading's avatar
Dan Reading committed
85
86
87
88
89
90
91
92
93
94
readtmcinfo() {
    local -A ina
    local keyword
    local -i dcnt=0
    local -i ncnt=0
    local ifile 
    local itmp

    ifile=${1+$1} # use $1 if set otherwise use nothing
    if [ -z "$ifile" ] ; then
95
	# noinput file then use a tmp file to hold data from tmcc
Dan Reading's avatar
Dan Reading committed
96
97
	itmp="y"
	ifile=/tmp/.$$tmcchwinv
98
	$($BINDIR/tmcc hwinfo > $ifile)
Dan Reading's avatar
Dan Reading committed
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    else
	itmp=""
    fi

    # initalize array
    if [ -z "${hwinv["hwinvidx"]+${hwinv["hwinvidx"]}}" ] ; then
	hwinv["hwinvidx"]="" #start the array
    else	
	for i in ${hwinv["hwinvidx"]} ; do
	    unset hwinv[$i]
	done
	hwinv["hwinvidx"]="" #restart the array
    fi

    # handle mult-line  input for disks and nets
    while read -r in ; do
	keyword=${in%% *}
	case $keyword in
	    DISKUNIT ) 
		keyword+="$dcnt"
		((++dcnt))
		;;
	    NETUNIT ) 
		keyword+="$ncnt"
		((++ncnt))
		;;
125
	    \#* ) continue ;; 
Dan Reading's avatar
Dan Reading committed
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
	esac
	hwinv["hwinvidx"]+="$keyword " # keeping the keyword list preserves order
	hwinv[$keyword]=$in
    done < $ifile
    [ -n "$itmp" ] && rm $ifile || : # the colon just stops a error being caught by -e
}

# copy assoctive array hwinv into hwinvcopy
# this is a little stupid but since I can't pass array I use globals
copytmcinfo () {
    # initalize array
    if [ -z "${hwinvcopy["hwinvidx"]+${hwinvcopy["hwinvidx"]}}" ] ; then
	hwinvcopy["hwinvidx"]="" #start the array
    else	
	for i in ${hwinvcopy["hwinvidx"]} ; do
	    unset hwinvcopy[$i]
	done
	hwinvcopy["hwinvidx"]="" #restart the array
    fi
    # copy index from old array
    hwinvcopy["hwinvidx"]=${hwinv["hwinvidx"]} 
    for i in ${hwinv["hwinvidx"]} ; do
	hwinvcopy[$i]=${hwinv[$i]}
    done
}

# compare arrays hwinv and copyhwinv arg1=outputfile
comparetmcinfo() {
    local fileout=$1
    # need to handle differing order with disks and nic addresses
    local localidx="${hwinv["hwinvidx"]}"
    local tbdbidx="${hwinvcopy["hwinvidx"]}"
    local localnics="" tbdbnics="" netunit=""
    local -i a b
    local x addr

    rm -f ${fileout} ${fileout}_local ${fileout}_tbdb
    retval=$(compareunits NET ${fileout}_local ${fileout}_tbdb)
    retval2=$(compareunits DISK ${fileout}_local ${fileout}_tbdb)

    # take NETUNIT out
    localidx=${localidx//NETUNIT[[:digit:]][[:digit:]]/}
    localidx=${localidx//NETUNIT[[:digit:]]/}
    tbdbidx=${tbdbidx//NETUNIT[[:digit:]][[:digit:]]/}
    tbdbidx=${tbdbidx//NETUNIT[[:digit:]]/}

    # take DISKUNIT out
    localidx=${localidx//DISKUNIT[[:digit:]][[:digit:]]/}
    localidx=${localidx//DISKUNIT[[:digit:]]/}
    tbdbidx=${tbdbidx//DISKUNIT[[:digit:]][[:digit:]]/}
    tbdbidx=${tbdbidx//DISKUNIT[[:digit:]]/}

    arrayidx="$localidx $tbdbidx"
    arrayidx=$(uniqstr $arrayidx)

    # step through the local index, looking only for one copy
    for i in ${localidx} ; do
	# following bash syntax: "${a+$a}" says use $a if exists else use nothing
	if [ -z "${hwinvcopy[$i]+${hwinvcopy[$i]}}" ] ; then
	    # localidx has it - hwinvcopy does not
	    printf "\n%s\n" "Local only ${hwinv[$i]}" >> ${fileout}
	    arrayidx=${arrayidx/$i} # nothing to compare with
	fi
    done
    # step through the testbed index, looking only for one copy
    for i in ${tbdbidx} ; do
	# followin bash syntax: "${a+$a}" says use $a if exists else use nothing
	if [ -z "${hwinv[$i]+${hwinv[$i]}}" ] ; then
	    printf "\n%s\n" "Testbest db only"  >> ${fileout}
	    arrayidx=${arrayidx/$i} 
	fi
    done
    
    #compare what is left
    for i in $arrayidx ; do
	if [ "${hwinv[$i]}" != "${hwinvcopy[$i]}" ] ; then
	    echo "" >> $fileout
	    echo "$i does not match" >> $fileout
	    echo "local ${hwinv[$i]}" >> $fileout
	    echo "tbdb ${hwinvcopy[$i]}" >> $fileout
	fi
    done

    [[ -f ${fileout}_local ]] && cat ${fileout}_local >> ${fileout}
    [[ -f ${fileout}_tbdb ]] && cat ${fileout}_tbdb >> ${fileout}
    rm -f ${fileout}_local ${fileout}_tbdb

    return 0
}

# multi-line units arg1=unittype arg2=localonlyfile arg3=tbdbonlyfile
compareunits() {
    local unittype=$1
    local localonly=$2
    local tbdbonly=$3
    # need to handle differing order with disks and nic addresses
    local localidx="${hwinv["hwinvidx"]}"
    local tbdbidx="${hwinvcopy["hwinvidx"]}"
    local localunits="" tbdbunits="" devunit=""
    local -i a b
    local x addr

    # How are things different between unit types
    case $unittype in
	NET )
	    unitinfoidx_str="NETINFO"
	    unitinfo_strip="NETINFO UNITS="
	    unit_str="NETUNIT"
	    unit_pre_strip="*ID=\""
	    unit_post_strip="\"*"
	    unit_human_output="NIC"
237
	    unit_human_case="lower"
Dan Reading's avatar
Dan Reading committed
238
239
240
241
242
243
244
245
	    ;;
	DISK )
	    unitinfoidx_str="DISKINFO"
	    unitinfo_strip="DISKINFO UNITS="
	    unit_str="DISKUNIT"
	    unit_pre_strip="*SN=\""
	    unit_post_strip="\"*"
	    unit_human_output="DISK"
246
	    unit_human_case="upper"
Dan Reading's avatar
Dan Reading committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
	    ;;
	* )
	    echo "Error in compareunits don't now type $unittype. Giving up."
	    exit 1
	    ;;
    esac


    # Pull the units out and checkthem
    # 
    # if any UNITs test - note at this point the same # of UNITS are on both lists
    if [ -n "${hwinv["${unitinfoidx_str}"]+${hwinv["${unitinfoidx_str}"]}}" ] ; then
	    x=${hwinv["${unitinfoidx_str}"]}
	    a=${x/${unitinfo_strip}}
    else
	a=0
    fi
    if [ -n "${hwinvcopy["${unitinfoidx_str}"]+${hwinvcopy["${unitinfoidx_str}"]}}" ] ; then
	x=${hwinvcopy["${unitinfoidx_str}"]}
	b=${x/${unitinfo_strip}}
    else
	b=0
    fi
    [[ $a > $b ]] && maxunits=$a || maxunits=$b 
    for ((i=0; i<$maxunits; i++)) ; do
	# gather just the units addresses 
	devunit="${unit_str}${i}"
        # following bash syntax: "${a+$a}" says use $a if exists else use nothing
	if [ -n "${hwinv[$devunit]+${hwinv[$devunit]}}" ] ; then
	    # add just the address
	    addr=${hwinv[$devunit]}
	    addr=${addr#${unit_pre_strip}}
279
	    addr=${addr%%${unit_post_strip}}
Dan Reading's avatar
Dan Reading committed
280
281
282
283
284
285
	    localunits+="$addr "
	    localidx=${localidx/$devunit}
	fi
	if [ -n "${hwinvcopy[$devunit]+${hwinvcopy[$devunit]}}" ] ; then
	    addr=${hwinvcopy[$devunit]}
	    addr=${addr#${unit_pre_strip}}
286
	    addr=${addr%%${unit_post_strip}}
Dan Reading's avatar
Dan Reading committed
287
288
289
290
291
	    tbdbunits+="$addr "
	    tbdbidx=${tbdbidx/$devunit}
	fi
    done

292
293
294
295
296
297
298
299
300
301
302
303
    # Adjust the case in both strings to the case we want
    if [ "$unit_human_case" == "upper" ] ; then
	localunits=${localunits^^}
	tbdbunits=${tbdbunits^^}
    else
	localunits=${localunits,,}
	tbdbunits=${tbdbunits,,}
    fi

    # remove from the lists all matching words
    x=$localunits
    for i in $x ; do
Dan Reading's avatar
Dan Reading committed
304
305
306
307
308
	if [ "${tbdbunits/$i}" != "${tbdbunits}" ]; then
	    tbdbunits=${tbdbunits/$i}
	    localunits=${localunits/$i}
	fi
    done
309
310
311
    # same but swap arrays
    x=$tbdbunits
    for i in $x ; do
Dan Reading's avatar
Dan Reading committed
312
313
314
315
316
317
	if [ "${localunits/$i}" != "${localunits}" ]; then
	    localunits=${localunits/$i}
	    tbdbunits=${tbdbunits/$i}
	fi
    done
    #remove extra spaces
318
319
    save_e
    set +e
Dan Reading's avatar
Dan Reading committed
320
    read -rd '' tbdbunits <<< "$tbdbunits"
321
322
    read -rd '' localunits <<< "$localunits"
    restore_e
Dan Reading's avatar
Dan Reading committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337

    # any mismatches would be in localunits and tbdbunits
    if [ -n "${localunits}" ]; then
	if [ ! -f $localonly ] ; then
	    printf "\n%s\n" "Found only locally" > $localonly
	fi
	printf "%s%s %s\n" "${unit_human_output}" "s:" "$localunits"  >> $localonly
    fi
    if [ -n "${tbdbunits}" ]; then
	if [ ! -f $tbdbonly ] ; then
	    printf "\n%s\n" "In testbed db but not found" > $tbdbonly
	fi
	printf "%s%s %s\n" "${unit_human_output}" "s:" "$tbdbunits" >> $tbdbonly
    fi

338
    return 0
Dan Reading's avatar
Dan Reading committed
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
}


# take a string make the words in it uniq
uniqstr() {
    local instr="$@"
    local outstr=""
    for i in $instr ; do
	if [ "${outstr/$i}" ==  "$outstr" ] ; then
	    # $i not in outstr, add it
	    outstr+="$i "
	fi
    done
    echo $outstr
}

# no args uses the globel arrays hwinvv, tcm_in, tcm_out
#mergetmcinfo() {
#    for i in ${hwinv["hwinvidx"]} ; do
#	hwinv[$i]+=" ADD"
#    done
#}

# arg $1 is the file to write uses the globel tcm_out array
#writetmcinfo() {
#:
#}


# print only the testbed data table
printtmcinfo() {
    local -i hdunits=0 nicunits=0
    for i in ${hwinv["hwinvidx"]} ; do
	case $i in 
	    CPUINFO ) printf "%s\n" "${hwinv[$i]}" ;;
	    MEMINFO ) printf "%s\n" "${hwinv[$i]}" ;;
	    DISKINFO ) 
		printf "%s\n" "${hwinv[$i]}" 
		x=${hwinv[$i]}
		hdunits=${x/#DISKINFO UNITS=/}

		# for HD need also check that we have a valid value
		# we collect more info then the testbed data base wants
		for ((n=0; n<$hdunits; n++)) ; do
		    # grab diskunitline
                    s=${hwinv[DISKUNIT$n]}
		    # turn space seperated string into array
		    unset -v d ; declare -a d=(${s// / })
		    numelm=${#d[*]}
		    echo -n "${d[0]} " #that is the word DISKUNIT
		    
		    for ((elm=1; elm<$numelm; elm++)) ; do
		        # must have form obj=value (where val can be blank) to work
			objval=${d[$elm]}
			[[ -z $objval ]] && continue  # that's bad no tupil
			obj=${objval%%=*}
			val=${objval##*=}
			[[ -z $val ]] && continue # bad also no value (or empty string)
			u=${val,,} #lower case
			[[ $u == ${u/unk} ]] || continue # the value has the UNKNOWN value
			[[ $u == ${u/na} ]] || continue # the value has the NA
			[[ $u == ${u/not} ]] || continue # the value has the LinuxNot
			[[ $u == ${u/bad} ]] || continue # the value bad_dd
		        # out put the stuff the database wants
		        # skip the stuff the database does not want
			case $obj in
			    SN | TYPE | SECSIZE | SECTORS | WSPEED | RSPEED )
				echo -n "$objval " ;;
			esac
		    done
		    echo "" # end the line
		done
		;;
            NETINFO ) printf "%s\n" "${hwinv[$i]}" 
		x=${hwinv[$i]}
		nicunits=${x/#NETINFO UNITS=/}
		for ((i=0; i<$nicunits; i++)); do printf "%s\n" "${hwinv[NETUNIT$i]}"; done ;;
	esac
    done
}

# print all hwinv
printhwinv() {
    for i in ${hwinv["hwinvidx"]} ; do
	printf "%s\n" "${hwinv[$i]}"
    done
}

# which is not in busybox and not a bash builtin
which() {
429
    mypath=$PATH
Dan Reading's avatar
Dan Reading committed
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
    mypath=${mypath//:/ }
    for i in $mypath ; do
	if [ -e $i/$1 ] ; then
	    echo $i/$1
	    return 0
	fi
    done
    return 1
}

inithostname() {
    os=$(uname -s)
    if [ -z $os ] ; then
	echo "ERROR uname messed up"
	exit 1
    fi
446
447
    if [ -e "$BINDIR/tmcc" ] ; then
	host=$($BINDIR/tmcc nodeid)
Dan Reading's avatar
Dan Reading committed
448
    else
449
	echo "ERROR no tmcc command"
Dan Reading's avatar
Dan Reading committed
450
451
452
453
454
455
456
457
458
	# maybe its just time to give up
	if [ -z "$host" ] ; then 
	    if [ -e "$BOOTDIR/realname" ] ; then
		host=$(cat $BOOTDIR/realname)
	    elif [ -e "$BOOTDIR/nodeid" ] ; then
		host=$(cat $BOOTDIR/nodeid)
	    else
		host=$(hostname)
	    fi
Dan Reading's avatar
Dan Reading committed
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
	fi
    fi
    return 0
}

findSmartctl() {
    local findit=$(which smartctl)
    if [ -z "${findit}" ] ; then
	if [ -x "/usr/sbin/smartctl" ]; then
	    findit="/usr/sbin/smartctl"
	else
	    findit=$NOSM
	fi
    fi
    echo $findit
    return 0
}

Dan Reading's avatar
Dan Reading committed
477
478
# Array of command to be run at exit time
on_exit() {
Dan Reading's avatar
Dan Reading committed
479
480
481
482
483
484
    for i in "${todo_exit[@]}" ; do
#echo "EXIT TODO doing: $i"
        $($i)
    done
    return 0
}
Dan Reading's avatar
Dan Reading committed
485
add_on_exit() {
Dan Reading's avatar
Dan Reading committed
486
487
488
489
490
491
492
493
494
495
496
    local nex=${#todo_exit[*]}
#echo "add on exit called B4n=${#todo_exit[*]} SHELL=$$ |$@|++++++++++++++++++++"
    todo_exit[$nex]="$@"
    if [[ $nex -eq 0 ]]; then
        trap on_exit EXIT
    fi
    return 0
}

# setup logging
initlogs () {
Dan Reading's avatar
Dan Reading committed
497
498
499
    # the following syntax lets us test if a positional arg is set before we try and use it
    # need if running with -u set. 
    # It means use $1 if set else use a default path
Dan Reading's avatar
Dan Reading committed
500
    logfile=${1-"/tmp/nodecheck.log"}
Dan Reading's avatar
Dan Reading committed
501
502

    # this file is only used in gather mode
503
504
505
506
    # and should have been created in gatherinv
    # set the name so it can be tested for
    logfile4tb=${2-".$$no4tb"}

Dan Reading's avatar
Dan Reading committed
507
508
509
    tmplog=/tmp/.$$tmp.log ; cat /dev/null > ${tmplog} # create and truncate
    add_on_exit "rm -f $tmplog"

510
    logout=/tmp/.$$logout.log ; cp /dev/null ${logout} # make it exist
Dan Reading's avatar
Dan Reading committed
511
    add_on_exit "rm -f $logout"
512
    tmpout=/tmp/.$$tmpout.log ; cp /dev/null ${tmpout}
Dan Reading's avatar
Dan Reading committed
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
    add_on_exit "rm -f $tmpout"
#    tmpfunctionout=/tmp/.$$tmpfunctionout.log 

    return 0
}

getdrivenames() {
    # use smartctl if exits
    # use scan of disk devices
    # use / 
    # truncate all together and then make uniq

    local os=$(uname -s)
    local sm=$(findSmartctl)
    local drivelist=""
    local drives="" 
    local x elm

#    if [ "$sm" != "${sm/smartctl}" ] ; then
#	x=$($sm --scan-open | awk '{print $1}')
#	if [ "$x" != "${x/dev}" ] ; then
#	    for elm in $x ; do
#		x=${x/"/dev/pass2"/} # FreeBSD not a HD, Tape?
#		drivelist+="$x "
#	    done
#	fi
#    fi


    case $os in
	Linux )
	    list="a b c d e f g h i j k l m n o p"
	    for i in $list
	    do
		if [ -b /dev/sd${i} ] ; then
		    drivelist+="/dev/sd${i} "
		fi
	    done
	    ;;
	FreeBSD )
	    list="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15"
	    for i in $list
	    do
		[[ -c /dev/ad${i} ]] && drivelist+="/dev/ad${i} " 
557
558
559
		[[ -c /dev/da${i} ]] && drivelist+="/dev/da${i} " 
		[[ -c /dev/ar${i} ]] && drivelist+="/dev/ar${i} " 
		[[ -c /dev/aacd${i} ]] && drivelist+="/dev/aacd${i} " 
Dan Reading's avatar
Dan Reading committed
560
561
		[[ -c /dev/amrd${i} ]] && drivelist+="/dev/amrd${i} " 
		[[ -c /dev/mfid${i} ]] && drivelist+="/dev/mfid${i} " 
562
		[[ -c /dev/mfisyspd${i} ]] && drivelist+="/dev/mfisyspd${i} " 
Dan Reading's avatar
Dan Reading committed
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
	    done
	    ;;
	* )
	    echo "${FUNCNAME[0]}:${LINENO} Internal error"
	    exit
	    ;;
    esac
    
#echo "drivelist||$drivelist||" >&2
#
#    while (( ${#drivelist} ))  ; do
#	elm=${drivelist%% *}
#	drives+="$elm"
#	drivelist=${drivelist//$elm/}
#echo "drivelist||$drivelist|| drives||$drives||" >&2
#    done
#    echo $drives
    echo $drivelist
#echo "checkutils drives:||$drives||"
    return 0
}


# The timesys function terminates its script unless it terminates earlier on its own
# args: max_time output_file command command_args
timesys() {
    maxtime="$1"; shift;
    out="$1" ; shift;
    command="$1"; shift;
    args="$*"
    me_pid=$$;
    sleep $maxtime &
    timer_pid=$!
{
    $command $args &
    command_pid=$!
    wait ${timer_pid}
    siglist="2 15 9"
    for i in $siglist ; do
	running=$(ps -a | grep $command_pid | grep dd)
	[[ "$running" ]] || break
	kill -${i} ${command_pid}
    done
} > $out 2>&1
}