gen_sql 20.6 KB
Newer Older
1
2
#! /usr/local/bin/bash

Dan Reading's avatar
Dan Reading committed
3
4
# let the checkutils.sh script know we are running offline
declare -i offline=1
5
declare -i reporting=0
6
7
8
source checkutils.sh


9
checkdrift_main() {
10
    declare -i countsame=1 filecount=0 hostcount=0 
Dan Reading's avatar
Dan Reading committed
11
12
    declare -i tellNoSN=0 tellNonMFS=0; 
    declare print_blank=no
13
    for host in $allnodes ; do
Dan Reading's avatar
Dan Reading committed
14
	print_blank=yes
Dan Reading's avatar
Dan Reading committed
15
	((hostcount++))
16
17
	if [ -d $projdir/$host/.tbdb ] ; then
	    cd $projdir/$host/.tbdb
18
	    allchecks=$(ls | sort -r | paste -s -d \\\  -) # paste command used to convert into space seperated words.
19
	    allchecks+=' ' # grrr my string matching, expects space at end
20

21
	    # set newest file and take off list
22
23
	    newest=${allchecks%%\ *}
	    allchecks=${allchecks#* } # pop the list
24
	    [[ ! $allchecks ]] && continue # ran out of files for this node
Dan Reading's avatar
Dan Reading committed
25

26
	    #only compare inventory computed in mfsmode=1
27
            # from the top of list skip file that have mfsmode=0
Dan Reading's avatar
Dan Reading committed
28
29
	    while (grep -c 'ismfs=0' $newest &> /dev/null ) ; do
		newest=${allchecks%%\ *} # take off the top
30
		allchecks=${allchecks#* } # pop the list
31
		[[ ! $allchecks ]] && break # ran out of files, break from loop
32
	    done
33
	    [[ ! $allchecks ]] && continue # make sure we didn't consume all the file for node
Dan Reading's avatar
Dan Reading committed
34
	    if (( $tellNoSN )); then
35
36
	    # X X X Hackitly Hackit Hack
	    # drop those files that have empty SN, but report it
Dan Reading's avatar
Dan Reading committed
37
38
39
40
41
	    if [ "$(grep 'DISKUNIT TYPE' $newest)" -o "$(grep 'DISKUNIT SECSIZE' $newest)" ] ; then
		echo "===> Missing SN in $host $(pwd)/$newest"
# pop or not to pop - that is the question (ignore !SN file or process)
#::		newest=${allchecks%%\ *} # take off the top
#::		allchecks=${allchecks#* } # pop the list
42
43
		[[ ! $allchecks ]] && continue # ran out of files
	    fi
Dan Reading's avatar
Dan Reading committed
44
	    fi
45

Dan Reading's avatar
Dan Reading committed
46
47
	    # read $newset into the hwinvcopy array, hwinvcopy array has
	    # been declared in the checkutils.sh script
48
49
	    readtmcinfo $newest hwinvcopy
	    for tocheck in $allchecks ; do
Dan Reading's avatar
Dan Reading committed
50
51
52
		if (( ! $tellNonMFS )) ; then
		# if not in ismfs mode then don't check
 		[[ $(grep 'ismfs=0' $tocheck) ]] && continue 
53
		fi
54
55
56
		readtmcinfo $tocheck hwinv
		# note: will check against self for sanity
		comparetmcinfo /tmp/.$$checkdiff
Dan Reading's avatar
Dan Reading committed
57
		((filecount++))
58
		if [ -s /tmp/.$$checkdiff ] ; then
Dan Reading's avatar
Dan Reading committed
59
60
61
		    [[ "$print_blank" == "yes" ]] && { echo ""; print_blank=no; }
		    echo "$host generated tbdb $newest $tocheck are not the same"
		    echo "==============================================================="
62
63
64
		    cat /tmp/.$$checkdiff
#		    echo "diff $tocheck $newest"
#		    diff $tocheck $newest
65
		    # reset compare file
66
67
		    newest=$tocheck
		    readtmcinfo $newest hwinvcopy
68
69
70
71
72
73
74
		    # XXX - start
		    # XXX Do not enable this code unless you want
		    # the files moved out of the way
#		    [[ ! -d $projdir/$host/.tbdb/.notsame ]] && sudo mkdir -p $projdir/$host/.tbdb/.notsame 
#		    sudo mv $tocheck $projdir/$host/.tbdb/.notsame
		    # XXX - end
		else
Dan Reading's avatar
Dan Reading committed
75
# : echo "$host $newest $tocheck SAME"
76
77
78
79
		    ((countsame++))
		fi
		rm -f /tmp/.$$checkdiff
	    done
Dan Reading's avatar
Dan Reading committed
80
# :[[ $countsame -gt 1 ]] && echo "$host $countsame inventory files are the same"
81
# :	    echo -n .
82
83
84
85
	else
	    echo "No record of node id \"$host\"."
	fi
    done
86
87

    [[ $hostcount -gt 1 ]] && { [[ $reporting -eq 0 ]] && echo "$filecount files in $hostcount nodes checked"; }
88
89
}

90
checkwce_main() {
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
    for host in $allnodes ; do
	if [ -f $projdir/$host/full ] ; then

	#    wcestate=$(grep $projdir/$host/full$)
	    readtmcinfo $projdir/$host/full hwinv
	    numberofdrives=${hwinv["DISKINFO"]}
	    numberofdrives=${numberofdrives/DISKINFO UNITS=/}
	    
            for ((idx=0; idx<${numberofdrives}; idx++)) ; do
		unset -v d ; declare -a d=(${hwinv["DISKUNIT$idx"]})
                wce=${d[2]}
		if [ "$wce" != "${wce/enabled}" ] ; then
		    :
		elif [ "$wce" != "${wce/disabled}" ] ; then
		    x=${d[1]}
		    x=${x##*SN=}; x=${x%% *}; x=${x//\"/}
		    echo "$host ${d[8]} Write Cache Disabled bsidx=$(get_bsidx_from_sn $x)"
		else
		    :
		fi
            done

113
114
115
116
117
118
	else
	    echo "No record of node id \"$host\"."
	fi
    done
}

119
120
121
122
123
cleanup_main() {
    declare -i f=0
    savepwd=$(pwd)
    sorton=".diff"
    toclean=".tbdb .tmcc .full .diff"
124
125
    cd $projdir
#    echo "Must be root to run correctly"
126
127
128
129
130
    for host in $allnodes ; do
	f=0
	# for each node sort the ${sorton} directory by date
	# save the two newest file and delete all the rest
	# in ${sorton} ${toclean} directories
131
132
	[[ ! -d $projdir/$host/${sorton} ]] && continue
	cd $projdir/$host/${sorton}
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
	sortedls=$(ls -t | paste -s -d \\\  -)
	# pop the top 2 newest off
	sortedls=${sortedls#*\ }
	[[ -z "$sortedls" ]] && continue
	sortedls+=' '
	[[ -z "$sortedls" ]] && continue
	sortedls=${sortedls#*\ }
	[[ -z "$sortedls" ]] && continue
	sortedls+=' '
	# print something
	for i in $sortedls ; do
	    ((f++))
	done
	((f--))
	echo "node:$host removing $f files from each ${toclean} directories"

	# take the next file for timestamp
	sortedls=${sortedls%%\ *}
	[[ -z "$sortedls" ]] && continue
152
	dated=$projdir/$host/${sorton}/${sortedls}
153
	# using find delete older file
154
	cd $projdir/$host
155
156
	find ${toclean} ! -newer ${dated} ! -name ${dated} -type f -exec rm '{}' ';'
    done
157
    cd $savepwd
158
159
}

Dan Reading's avatar
Dan Reading committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
#   README
#   to get HD info from serial number
# select b.node_id,b.bs_id,b.bsidx,a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$sn';
#   to get HD info from node_id
# select b.node_id,b.bs_id,b.bsidx,a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.node_id='pc510';
#
#   to get bsidx from serial number
# db_bsidx=$(mysql -B -N -e "select b.bsidx from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$sn';" tbdb)
#   to get serial number from bsidx
# db_sn=$(mysql -B -N -e "select a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.bsidx=$bsidx;" tbdb);

get_bsidx_from_sn() {
    echo $(mysql -B -N -e "select b.bsidx from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$1';" tbdb)
}
get_sn_from_bsidx() {
    echo $(mysql -B -N -e "select a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.bsidx=$1;" tbdb);
}
get_host_from_bsidx() {
    echo $(mysql -B -N -e "select node_id from blockstores where bsidx=$1;" tbdb);
}
get_drive_from_bsidx() {
    echo $(mysql -B -N -e "select bs_id from blockstores where bsidx=$1;" tbdb);
}
183
184
185
get_bsidx_from_hostAnddrive() {
    echo $(mysql -B -N -e "select bsidx from blockstores where node_id=\"$1\" and bs_id=\"disk${2}\";" tbdb)
}
Dan Reading's avatar
Dan Reading committed
186

187
gentbsql_main() {
Dan Reading's avatar
Dan Reading committed
188
    local sn="UNK" hdtype="UNK" secsize="UNK" sectors="UNK" wspeed="UNK" rspeed="UNK"
189
    # get current unique BlockStore number from tbdb
Dan Reading's avatar
Dan Reading committed
190
    bsidx_base=$(mysql -B -N -e "select idx from emulab_indicies where name='next_bsidx';" tbdb)
191

Dan Reading's avatar
Dan Reading committed
192
    keep_bsidx_base=$bsidx_base
Dan Reading's avatar
Dan Reading committed
193
194
    # XXX
    # try to fill gaps above 1000, good idea?
195
    [[ $bsidx_base -gt 200 ]] && bsidx_base=200
Dan Reading's avatar
Dan Reading committed
196
    
197
    for host in $allnodes ; do
198
199
	if [ -d $projdir/$host ] ; then
	    cd $projdir/$host
Dan Reading's avatar
Dan Reading committed
200
	    sn="UNK" hdtype="UNK" secsize="UNK" sectors="UNK" wspeed="UNK" rspeed="UNK"
201
#	    [[ -f "diff" ]] || { echo "# $host: No changes needed"; continue; }
202
	    # need the diff and node files to continue. No diff then nothing to update
203
	    [[ -f "diff" ]] || continue;
Dan Reading's avatar
Dan Reading committed
204
	    [[ -f "node" ]] || { echo "# $host: Missing node file"; continue; }
205
	    # check for old version of file
206
	    if [ -z "$(grep "Diff Report" diff)" ] ; then
Dan Reading's avatar
Dan Reading committed
207
		echo "# $host Inventory Report to old to use"
208
209
		continue
	    fi
210

211
212
213
	    # we need a diff file that has SN in it, check the .diff directory
	    # for the info
	    # so try and find the SN in any .full/* file with serial numbers
214
# debugging	    echo "# $(pwd)"
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
	    dotDiffFiles=$(ls -t .diff)
	    for dDF in $dotDiffFiles ; do
		SNinfo=$(grep 'DISKs:' .diff/$dDF | grep -v UNKNOWN)
		[[ $SNinfo ]] && break # found, stop
	    done
	    # if we failed to find a file then default to the sym link
	    # else we found it 
	    if [ "$SNinfo" ] ; then
		indexF=$dDF
		diffFile=.diff/$dDF
		nodeFile=.tbdb/$dDF
	    else
		indexF=$(pwd)
		diffFile=diff
		nodeFile=node
	    fi
	    listofdisks=$(grep 'DISKs:' $diffFile)
	    listofdisks=${listofdisks//DISKs:}  # take string out
Dan Reading's avatar
Dan Reading committed
233
	    hdnum=0
234
235
236
237
238
	    # if 'DISKs:' is empty then try OUT OF ORDER message
	    if [ -z "$listofdisks" ]; then
		local_listofdisks=$(grep 'OUT OF ORDER' $diffFile)
		# remove up to first 'local['
		local_listofdisks=${local_listofdisks#*local\[}
Dan Reading's avatar
Dan Reading committed
239
		# remove from ']' to end
240
		local_listofdisks=${local_listofdisks%%]*}
Dan Reading's avatar
Dan Reading committed
241
242
243
                # have a second case where "ERROR DISK OUT OF ORDER"
                # remove up to 'from tbdb'
		local_listofdisks=${local_listofdisks#*from\ tbdb\ }
244
                for i in $local_listofdisks ; do
245
246
		    # or we just didn't find anything
		    [[ -z "$local_listofdisks" ]] && continue
247
		    [[ $i == "UNKNOWN" ]] && continue
248
		    echo "# Local $host disks out of order - rewrite order this run. Run $0 again after more inventory captured"
249
250
		    echo "mysql -e \"delete from blockstore_attributes where attrvalue='$i';\" tbdb"
		    listofdisks="$listofdisks $i"
251
# echo "### not enabled - sudo rm $projdir/$host/diff"
252
253
                    done
	    fi
254
	    for i in $listofdisks ; do
255

256
		toadd=$(grep $i $nodeFile)
257
258
		if [ -n "$toadd" ] ; then
		    toadd=${toadd/DISKUNIT}
Dan Reading's avatar
Dan Reading committed
259
260
261
262
263
264
		    if [ "${toadd/SN=}" != "$toadd" ]; then
			sn=${toadd##*SN=}
			sn=${sn%% *}
			sn=${sn//\"/}
			# check and see if serial number is already in database
			bsidx=$(get_bsidx_from_sn $sn)
265
			[[ "$bsidx" ]] && continue
Dan Reading's avatar
Dan Reading committed
266
267
		    fi

268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
		    # /* by Mike's orders
		    #  type		"class"	"protocol"	"rot-speed"	nodes with this type
		    # 
		    #  pata-generic	local	PATA		7200		pc600,pc850,pc2000
		    #  scsi-generic	local	SCSI		10000		pc3000,pc6000
		    #  sata-generic	local	SATA		7200		pc2400w,d710,d820,pcivy
		    #  sas-generic	local	SAS		10000		d820
		    #  ssd-generic	local	SATA		0		gpuhost,d820(pc601)
		    #  c2100-sas	local	SAS		15000		c2100
		    # */
		    nodetype=$(mysql -B -N -e "select type from nodes where node_id='$host';" tbdb)
		    case $nodetype in
			pc600 | pc850 | pc2000 ) hdtype="pata-generic" ;;
			pc3000 | pc6000 ) hdtype="scsi-generic" ;;
			pc2400w | d710 | d820 | pcivy ) hdtype="sata-generic" ;;
			d820 ) hdtype="sas-generic" ;;
			gpuhost ) hdtype="ssd-generic" ;;
			c2100 ) hdtype="c2100-sas" ;;
			* ) 
			    hdtype=${toadd##*TYPE=}
			    hdtype=${hdtype%% *}
			    ;;
		    esac
		    # translate type
		    [[ "$hdtype" == "SATA" ]] && hdtype="sata-generic"
		    [[ "$hdtype" == "PATA" ]] && hdtype="pata-generic"
		    [[ "$hdtype" == "SAS" ]] && hdtype="sas-generic"
		    # XXX XXX XXX special case
		    if [ "$host" == "pc601" ] ; then
			hdtype="ssd-generic"
		    fi
		    # XXX XXX XXX special case
Dan Reading's avatar
Dan Reading committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
		    if [ "${toadd/SECSIZE=}" != "$toadd" ]; then
			secsize=${toadd##*SECSIZE=}
			secsize=${secsize%% *}
		    fi
		    if [ "${toadd/SECTORS=}" != "$toadd" ]; then
			sectors=${toadd##*SECTORS=}
			sectors=${sectors%% *}
		    fi
		    if [ "${toadd/RSPEED=}" != "$toadd" ]; then
			rspeed=${toadd##*RSPEED=}
			rspeed=${rspeed%% *}
		    fi
		    if [ "${toadd/WSPEED=}" != "$toadd" ]; then
			wspeed=${toadd##*WSPEED=}
			wspeed=${wspeed%% *}
		    fi
		    size=$(($sectors / 2048))
		    # check and see if bsidx already used
		    while [ "$(mysql -B -N -e "select bsidx from blockstores where bsidx=$bsidx_base;" tbdb)" ] ; do
319
#			echo "# bsidx=$bsidx_base already used, bumping bsidx_base again"
Dan Reading's avatar
Dan Reading committed
320
321
322
			((++bsidx_base))
		    done
                    # check and see if drive already used
323
324
		    found_blockstores_idx=''
		    found_blockstoreattr_idx=''
Dan Reading's avatar
Dan Reading committed
325
		    while [ "$(mysql -B -N -e "select bs_id from blockstores where node_id=\"$host\" and bs_id=\"disk${hdnum}\";" tbdb)" ] ; do
326
327
328
			# found a driveX on this host - see what we can find in the db
			found_blockstores_idx=''
			found_blockstoreattr_idx=''
329
			found_blockstores_idx=$(get_bsidx_from_hostAnddrive $host ${hdnum})
330
331
332
			found_blockstoreattr_idx=$(get_sn_from_bsidx $found_blockstores_idx)

			if [ "$found_blockstores_idx" ] ; then
333
			    echo -n "# $host drive${hdnum} already used @ bsidx=$found_blockstores_idx"
334
335
336
337
338
			    break
			else
			    echo "# drive${hdnum} already used, bumping hdnum again"
			    ((++hdnum))
			fi
Dan Reading's avatar
Dan Reading committed
339
		    done
340
341
342
343
344
345
346
347
348
349
350
351
		    if [ "$found_blockstores_idx" ] ; then
			# have a blockstores idx, check if we are missing blockstore_attrs
			if [ ! "$found_blockstoreattr_idx" ] ; then
			    # create one
			    echo " BUT blockstore_attributes missing"
			    printf "mysql -e \"insert into blockstore_attributes values (%d, '%s', '%s', '%s');\" tbdb\n" "$found_blockstores_idx" "serialnum" "$sn" "string" 
			    # assume we don't have to update blockstores, but display entry
			    echo "# MATCHING BLOCKSTORES @ $(mysql -B -N -e "select bsidx,node_id,bs_id,total_size  from blockstores where bsidx=$found_blockstores_idx;" tbdb);"
			else
			    echo ""
			fi
		    else
352
353
354
355
356
357
358
359
360
			# just make sure we don't have an entry
			justcheck=$(mysql -B -N -e "select * from blockstore_attributes where attrvalue='$sn';" tbdb );
			if [ "$justcheck" ] ; then
			    echo "ERROR: tbdb Inconsistance  EXIT"
			    echo "bsidx=$bsidx_base but found $sn in blockstore_attributes"
			    mysql -e "select * from blockstore_attributes where attrvalue='$sn';" tbdb
			    exit
			fi

361
362
363
364
365
366
367
			printf "mysql -e \"insert into blockstores values (%d, '%s', 'disk%d', 0, '%s', 'element', %d, 1, now());\" tbdb\n" "$bsidx_base" "$host" "$hdnum" "$hdtype" "$size" 
			# now the second table
			printf "mysql -e \"insert into blockstore_attributes values (%d, '%s', '%s', '%s');\" tbdb\n" "$bsidx_base" "serialnum" "$sn" "string" 
			# used up a bsidx, incr bsidx
			((++bsidx_base))
		    fi
		    # FINALLY onto the next drive
Dan Reading's avatar
Dan Reading committed
368
		    ((++hdnum))
369
		else
Dan Reading's avatar
Dan Reading committed
370
		    if [ "$i" == "UNKNOWN" ] ; then
371
372
			x=$(get_bsidx_from_hostAnddrive $host ${hdnum})
			if [ -n "$x" ] ; then
373
			    echo -n "# Can not locate Serial Number for disk${hdnum} on $host"
374
375
376
377
378
379
			    # so try and find the SN in any .full/* file with serial numbers
			    dotFullFiles=$(ls .full)
			    for dFF in $dotFullFiles ; do
				SNinfo=$(grep -v 'SN="UNKNOWN"' .full/$dFF | grep DISKUNIT)
				[[ $SNinfo ]] && break # found some stop
			    done
380
			    echo " but found $(get_drive_from_bsidx $x) $(get_host_from_bsidx $x) @ bsidx=$x SN=$(get_sn_from_bsidx $x)" 
381
382
			else
			    echo ""
383
			fi
Dan Reading's avatar
Dan Reading committed
384
385
			((++hdnum))
		    else
386
			toadd="SN='$i'"
387
			if [ "$sn" != "UNK" ] ; then
388
389
390
391
392
393
			    x_bsidx=$(get_bsidx_from_sn $i)
			    [[ $x_bsidx ]] && y_hostid=$(get_host_from_bsidx $x_bsidx) || y_hostid=""
			    if [ -n "$x_bsidx" -a -n "$y_hostid" ] ; then
				printf "mysql -e \"delete from blockstores where node_id='%s' and bsidx='%s';\" tbdb\n" "$host" "$x_bsidx"
				printf "mysql -e \"delete from blockstore_attributes where attrkey='%s';\" tbdb\n" "$i"
			    else
Dan Reading's avatar
Dan Reading committed
394
				echo "#Discovery error node_id $host, full:has SN BUT node:does not"
395
396
397
				      # Have only seen this when the "full" file for host lists a hard drive, including SN
				      # but the "node" file does not have serial number for for the HD
			    fi
398
			fi
Dan Reading's avatar
Dan Reading committed
399
		    fi
400
		fi
401
402
	    done
	    listofnics=$(grep 'NICs:' diff)
403
	    listofnics=${listofnics//NICs:/}
404
405
	    for i in $listofnics ; do
		toadd=$(grep $i node)
406
		if [ -n "$toadd" ] ; then
Dan Reading's avatar
Dan Reading committed
407
408
409
410
		    toadd=${toadd/*ID=\"}
		    toadd=${toadd/\"}
		    uuid=$(uuidgen)
		    printf "#BYHAND mysql -e \"insert into interfaces set node_id='%s',mac='%s',card=X,port=X,interface_type='?',iface='ethX',role='?',uuid='$uuid';\" tbdb\n" "$host" "$toadd"
411
412
		else
		    toadd="ID=\"$i\""
Dan Reading's avatar
Dan Reading committed
413
		    printf "#BYHAND mysql -e \"delete from interfaces where node_id='%s' and %s;\" tbdb\n" "$host" "$toadd"
414
		fi
415
	    done
416
	else
417
	    echo "# No record of node id \"$host\"."
418
	fi
419
    done
Dan Reading's avatar
Dan Reading committed
420
    if [ $keep_bsidx_base -lt $bsidx_base ] ; then
421
	echo "# orginal bsidx=$keep_bsidx_base : new bsidx=$bsidx_base"
Dan Reading's avatar
Dan Reading committed
422
	printf "mysql -e \"update emulab_indicies set idx=%d where name='next_bsidx';\" tbdb\n" "$bsidx_base"
423
    else
424
	[[ $reporting -eq 0 ]] && echo "# orginal bsidx=$keep_bsidx_base : new bsidx=$bsidx_base == no update"
Dan Reading's avatar
Dan Reading committed
425
    fi
426
427
428
429
430
431
432
433
434
}

setdirstructure() {
    # start XXX
    # decided to change names again, rename if old names -- XXX remove this after all node have run the new code
    # take into consideration if old directory structure was added to to a new directory structure 
    set -u
    projdir=/proj/emulab-ops/nodecheck
    cd $projdir
435
    allnodes=$(ls -d pc* pg* dbox* gpu* 2> /dev/null)
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
    dirs2workon=""
    for i in $allnodes; do [[ -L $i/$i.diff ]] && dirs2workon+="$i "; done
    owd=$PWD

    for host in $dirs2workon ; do
    	if [ -d ${projdir}/$host ] ; then
	    cd ${projdir}/$host
	    [[ -L $host.full ]] && rm $host.full
	    [[ -L $host.diff ]] && rm $host.diff
	    [[ -L $host ]] && rm $host
	    for i in full diff tbdb ; do
		if [ -d $i ] ; then
		    [[ ! -d .$i ]] && (mkdir .$i ; chmod g+x .$i)
		    list=$(ls -A $i)
		    for j in $list ; do
			mv $i/$j .$i
		    done
		    rmdir $i
		fi
	    done
	    cd $owd
	fi
    done
    
    # put links in place
    for i in $dirs2workon ; do
	cd $projdir/$i
	links="full diff node"
	for link in $links ; do
	    case $link in
		full | diff )
		    # make the symlinks
		    if [ ! -e ${link} -a -d .${link} ] ; then
			linkto=$(ls -t .${link} | head -1)
			if [ -n "$linkto" ] ; then
 			    ln -s .${link}/${linkto} $link
			fi
		    fi
		    ;;
		node )
		    if [ ! -e ${link} -a -d .tbdb ] ; then
			linkto=$(ls -t .tbdb | head -1)
			if [ -n "$linkto" ] ; then
 			    ln -s .tbdb/${linkto} $link
			    ln -s .tbdb/${linkto} $i
			fi
		    fi
		    ;;
	    esac
	done
    done
}

Dan Reading's avatar
Dan Reading committed
489
490
491
492
493
494
declare h4="no"

ph4() {
    [[ ! $h4 ]] && { echo "================================= $1 =================================" >> /tmp/Report_CheckNode; h4="no"; }
}

495
496
genreport_main() {
    reporting=1
Dan Reading's avatar
Dan Reading committed
497
498
499
500
501
502
503
    [[ -z "${CHECKNODE_CRONJOB-}" ]] && declare CHECKNODE_CRONJOB="NO"
    [[ -z "${CHECKNODE_MTA-}" ]] && declare CHECKNODE_MTA="sendmail -t"
    [[ -z "${CHECKNODE_MAILTO-}" ]] && declare CHECKNODE_MAILTO=""
    if [ "${CHECKNODE_CRONJOB}" != "NO" ] ; then
	# if MAILTO is empty then switch to cat-ing the info, cron can do with it what is wants
	# i.e. use the standard cron MAILTO var
	[[ -z "${CHECKNODE_MAILTO}" ]] && CHECKNODE_MTA="cat"
504
505
	echo "Subject: CheckNode report for $(hostname)" > /tmp/Report_CheckNode
	echo "From: CheckNode" >> /tmp/Report_CheckNode
Dan Reading's avatar
Dan Reading committed
506
	echo "To: ${CHECKNODE_MAILTO}" >> /tmp/Report_CheckNode
507
    else
508
	cat /dev/null > /tmp/Report_CheckNode
509
510
511
512
513
514
    fi

    rm -f /tmp/genreport /tmp/driftreport
    nodes2do=$allnodes
    for node in $nodes2do ; do
	allnodes=$node
Dan Reading's avatar
Dan Reading committed
515
516
	h4=''
#	echo "================================= $node =================================" >> /tmp/Report_CheckNode
517
	if [ -s /proj/emulab-ops/nodecheck/$node/diff ] ; then
Dan Reading's avatar
Dan Reading committed
518
519
520
521
522
523
524
525
526
527
528
	    # chop top off diff report and remove UNKNOWN lines
	    tail -n +5 /proj/emulab-ops/nodecheck/$node/diff | grep -v UNKNOWN | grep -v MISSING > /tmp/diffchop
	    # do we still have something to report?
	    if [ -s /tmp/diffchop ] ; then
		diffDate=$(stat -f %Sm -t "%H:%M %d%b%g" /proj/emulab-ops/nodecheck/$node/diff)
		ph4 $node
		echo "----------------------- $diffDate inconsistency with db -------------" >> /tmp/Report_CheckNode
		# remove pesky empty line
		grep -v '^$' /tmp/diffchop >> /tmp/Report_CheckNode
		rm -f /tmp/diffchop
	    fi
529
530
531
532
	fi

	gentbsql_main $node > /tmp/genreport
	if [ -s /tmp/genreport ] ; then
Dan Reading's avatar
Dan Reading committed
533
534
535
536
	    ph4 $node
	    echo ">>>> Run gen_sql for suggested db updates" >> /tmp/Report_CheckNode
#	    echo "--------------------------- suggested db updates  -----------------------" >> /tmp/Report_CheckNode
#	    cat /tmp/genreport >> /tmp/Report_CheckNode
537
538
	fi

Dan Reading's avatar
Dan Reading committed
539
	checkdrift_main $node | grep -v '^$' > /tmp/driftreport
540
	if [ -s /tmp/driftreport ] ; then
Dan Reading's avatar
Dan Reading committed
541
	    ph4 $node
542
543
544
545
546
547
548
	    echo "--------------------------- changes over time ---------------------------" >> /tmp/Report_CheckNode
	    cat /tmp/driftreport >> /tmp/Report_CheckNode
	fi

	rm -f /tmp/genreport /tmp/driftreport
    done

Dan Reading's avatar
Dan Reading committed
549
550
    if [ "${CHECKNODE_CRONJOB}" != "NO" ] ; then
	cat /tmp/Report_CheckNode | ${CHECKNODE_MTA}
551
552
553
    else
	cat /tmp/Report_CheckNode
    fi
Dan Reading's avatar
Dan Reading committed
554
    rm -f /tmp/Report_CheckNode
555
556
}

557
558
# start here
set -u
559
kdir=$(pwd)
560
projdir=/proj/emulab-ops/nodecheck
561
# if projdir is automounted this should mount it
562
563
564
565
566
567
568
569
cd $projdir

if (( $# )) ; then
    allnodes="$@"
else
    nodes=$(ls */tmcc)
    allnodes=${nodes//\/tmcc/}
fi
570
cd $kdir
571

572
if [ "$0" != "${0/setdir}" ] ; then
573
    setdirstructure
574
elif [ "$0" != "${0/sql}" ] ; then
575
    gentbsql_main $@
576
577
elif [ "$0" != "${0/drift}" ] ; then
    checkdrift_main $@
578
579
elif [ "$0" != "${0/cleanup}" ] ; then
    cleanup_main $@
580
581
elif [ "$0" != "${0/report}" ] ; then
    genreport_main $@
582
583
elif [ "$0" != "wce" ] ; then
    checkwce_main $@
584
585
else
    "Print useage help"
586
fi