gen_sql 20.6 KB
Newer Older
1
2
#! /usr/local/bin/bash

Dan Reading's avatar
Dan Reading committed
3
4
# let the checkutils.sh script know we are running offline
declare -i offline=1
5
declare -i reporting=0
6
7
8
source checkutils.sh


9
checkdrift_main() {
10
    declare -i countsame=1 filecount=0 hostcount=0 
Dan Reading's avatar
Dan Reading committed
11
12
    declare -i tellNoSN=0 tellNonMFS=0; 
    declare print_blank=no
13
    for host in $allnodes ; do
Dan Reading's avatar
Dan Reading committed
14
	print_blank=yes
Dan Reading's avatar
Dan Reading committed
15
	((hostcount++))
16
17
	if [ -d $projdir/$host/.tbdb ] ; then
	    cd $projdir/$host/.tbdb
18
	    allchecks=$(ls | sort -r | paste -s -d \\\  -) # paste command used to convert into space seperated words.
19
	    allchecks+=' ' # grrr my string matching, expects space at end
20

21
	    # set newest file and take off list
22
23
	    newest=${allchecks%%\ *}
	    allchecks=${allchecks#* } # pop the list
24
	    [[ ! $allchecks ]] && continue # ran out of files for this node
Dan Reading's avatar
Dan Reading committed
25

26
	    #only compare inventory computed in mfsmode=1
27
            # from the top of list skip file that have mfsmode=0
Dan Reading's avatar
Dan Reading committed
28
29
	    while (grep -c 'ismfs=0' $newest &> /dev/null ) ; do
		newest=${allchecks%%\ *} # take off the top
30
		allchecks=${allchecks#* } # pop the list
31
		[[ ! $allchecks ]] && break # ran out of files, break from loop
32
	    done
33
	    [[ ! $allchecks ]] && continue # make sure we didn't consume all the file for node
Dan Reading's avatar
Dan Reading committed
34
	    if (( $tellNoSN )); then
35
36
	    # X X X Hackitly Hackit Hack
	    # drop those files that have empty SN, but report it
Dan Reading's avatar
Dan Reading committed
37
38
39
40
41
	    if [ "$(grep 'DISKUNIT TYPE' $newest)" -o "$(grep 'DISKUNIT SECSIZE' $newest)" ] ; then
		echo "===> Missing SN in $host $(pwd)/$newest"
# pop or not to pop - that is the question (ignore !SN file or process)
#::		newest=${allchecks%%\ *} # take off the top
#::		allchecks=${allchecks#* } # pop the list
42
43
		[[ ! $allchecks ]] && continue # ran out of files
	    fi
Dan Reading's avatar
Dan Reading committed
44
	    fi
45

Dan Reading's avatar
Dan Reading committed
46
47
	    # read $newset into the hwinvcopy array, hwinvcopy array has
	    # been declared in the checkutils.sh script
48
49
	    readtmcinfo $newest hwinvcopy
	    for tocheck in $allchecks ; do
Dan Reading's avatar
Dan Reading committed
50
51
52
		if (( ! $tellNonMFS )) ; then
		# if not in ismfs mode then don't check
 		[[ $(grep 'ismfs=0' $tocheck) ]] && continue 
53
		fi
54
55
56
		readtmcinfo $tocheck hwinv
		# note: will check against self for sanity
		comparetmcinfo /tmp/.$$checkdiff
Dan Reading's avatar
Dan Reading committed
57
		((filecount++))
58
		if [ -s /tmp/.$$checkdiff ] ; then
Dan Reading's avatar
Dan Reading committed
59
60
61
		    [[ "$print_blank" == "yes" ]] && { echo ""; print_blank=no; }
		    echo "$host generated tbdb $newest $tocheck are not the same"
		    echo "==============================================================="
62
63
64
		    cat /tmp/.$$checkdiff
#		    echo "diff $tocheck $newest"
#		    diff $tocheck $newest
65
		    # reset compare file
66
67
		    newest=$tocheck
		    readtmcinfo $newest hwinvcopy
68
69
70
71
72
73
74
		    # XXX - start
		    # XXX Do not enable this code unless you want
		    # the files moved out of the way
#		    [[ ! -d $projdir/$host/.tbdb/.notsame ]] && sudo mkdir -p $projdir/$host/.tbdb/.notsame 
#		    sudo mv $tocheck $projdir/$host/.tbdb/.notsame
		    # XXX - end
		else
Dan Reading's avatar
Dan Reading committed
75
# : echo "$host $newest $tocheck SAME"
76
77
78
79
		    ((countsame++))
		fi
		rm -f /tmp/.$$checkdiff
	    done
Dan Reading's avatar
Dan Reading committed
80
# :[[ $countsame -gt 1 ]] && echo "$host $countsame inventory files are the same"
81
# :	    echo -n .
82
83
84
85
	else
	    echo "No record of node id \"$host\"."
	fi
    done
86
87

    [[ $hostcount -gt 1 ]] && { [[ $reporting -eq 0 ]] && echo "$filecount files in $hostcount nodes checked"; }
88
89
}

90
checkwce_main() {
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
    for host in $allnodes ; do
	if [ -f $projdir/$host/full ] ; then

	#    wcestate=$(grep $projdir/$host/full$)
	    readtmcinfo $projdir/$host/full hwinv
	    numberofdrives=${hwinv["DISKINFO"]}
	    numberofdrives=${numberofdrives/DISKINFO UNITS=/}
	    
            for ((idx=0; idx<${numberofdrives}; idx++)) ; do
		unset -v d ; declare -a d=(${hwinv["DISKUNIT$idx"]})
                wce=${d[2]}
		if [ "$wce" != "${wce/enabled}" ] ; then
		    :
		elif [ "$wce" != "${wce/disabled}" ] ; then
		    x=${d[1]}
		    x=${x##*SN=}; x=${x%% *}; x=${x//\"/}
		    echo "$host ${d[8]} Write Cache Disabled bsidx=$(get_bsidx_from_sn $x)"
		else
		    :
		fi
            done

113
114
115
116
117
118
	else
	    echo "No record of node id \"$host\"."
	fi
    done
}

119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
cleanup_main() {
    declare -i f=0
    savepwd=$(pwd)
    sorton=".diff"
    toclean=".tbdb .tmcc .full .diff"
    echo "Must be root to run correctly"
    for host in $allnodes ; do
	f=0
	# for each node sort the ${sorton} directory by date
	# save the two newest file and delete all the rest
	# in ${sorton} ${toclean} directories
	[[ ! -d $savepwd/$host/${sorton} ]] && continue
	cd $savepwd/$host/${sorton}
	sortedls=$(ls -t | paste -s -d \\\  -)
	# pop the top 2 newest off
	sortedls=${sortedls#*\ }
	[[ -z "$sortedls" ]] && continue
	sortedls+=' '
	[[ -z "$sortedls" ]] && continue
	sortedls=${sortedls#*\ }
	[[ -z "$sortedls" ]] && continue
	sortedls+=' '
	# print something
	for i in $sortedls ; do
	    ((f++))
	done
	((f--))
	echo "node:$host removing $f files from each ${toclean} directories"

	# take the next file for timestamp
	sortedls=${sortedls%%\ *}
	[[ -z "$sortedls" ]] && continue
	dated=$savepwd/$host/${sorton}/${sortedls}
	# using find delete older file
	cd $savepwd/$host
	find ${toclean} ! -newer ${dated} ! -name ${dated} -type f -exec rm '{}' ';'
    done
}

Dan Reading's avatar
Dan Reading committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
#   README
#   to get HD info from serial number
# select b.node_id,b.bs_id,b.bsidx,a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$sn';
#   to get HD info from node_id
# select b.node_id,b.bs_id,b.bsidx,a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.node_id='pc510';
#
#   to get bsidx from serial number
# db_bsidx=$(mysql -B -N -e "select b.bsidx from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$sn';" tbdb)
#   to get serial number from bsidx
# db_sn=$(mysql -B -N -e "select a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.bsidx=$bsidx;" tbdb);

get_bsidx_from_sn() {
    echo $(mysql -B -N -e "select b.bsidx from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and a.attrvalue='$1';" tbdb)
}
get_sn_from_bsidx() {
    echo $(mysql -B -N -e "select a.attrvalue from blockstores as b,blockstore_attributes as a where b.bsidx=a.bsidx and b.role='element' and a.attrkey='serialnum' and b.bsidx=$1;" tbdb);
}
get_host_from_bsidx() {
    echo $(mysql -B -N -e "select node_id from blockstores where bsidx=$1;" tbdb);
}
get_drive_from_bsidx() {
    echo $(mysql -B -N -e "select bs_id from blockstores where bsidx=$1;" tbdb);
}
181
182
183
get_bsidx_from_hostAnddrive() {
    echo $(mysql -B -N -e "select bsidx from blockstores where node_id=\"$1\" and bs_id=\"disk${2}\";" tbdb)
}
Dan Reading's avatar
Dan Reading committed
184

185
gentbsql_main() {
Dan Reading's avatar
Dan Reading committed
186
    local sn="UNK" hdtype="UNK" secsize="UNK" sectors="UNK" wspeed="UNK" rspeed="UNK"
187
    # get current unique BlockStore number from tbdb
Dan Reading's avatar
Dan Reading committed
188
    bsidx_base=$(mysql -B -N -e "select idx from emulab_indicies where name='next_bsidx';" tbdb)
189

Dan Reading's avatar
Dan Reading committed
190
    keep_bsidx_base=$bsidx_base
Dan Reading's avatar
Dan Reading committed
191
192
    # XXX
    # try to fill gaps above 1000, good idea?
193
    [[ $bsidx_base -gt 200 ]] && bsidx_base=200
Dan Reading's avatar
Dan Reading committed
194
    
195
    for host in $allnodes ; do
196
197
	if [ -d $projdir/$host ] ; then
	    cd $projdir/$host
Dan Reading's avatar
Dan Reading committed
198
	    sn="UNK" hdtype="UNK" secsize="UNK" sectors="UNK" wspeed="UNK" rspeed="UNK"
199
#	    [[ -f "diff" ]] || { echo "# $host: No changes needed"; continue; }
200
	    # need the diff and node files to continue. No diff then nothing to update
201
	    [[ -f "diff" ]] || continue;
Dan Reading's avatar
Dan Reading committed
202
	    [[ -f "node" ]] || { echo "# $host: Missing node file"; continue; }
203
	    # check for old version of file
204
	    if [ -z "$(grep "Diff Report" diff)" ] ; then
Dan Reading's avatar
Dan Reading committed
205
		echo "# $host Inventory Report to old to use"
206
207
		continue
	    fi
208

209
210
211
	    # we need a diff file that has SN in it, check the .diff directory
	    # for the info
	    # so try and find the SN in any .full/* file with serial numbers
212
# debugging	    echo "# $(pwd)"
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
	    dotDiffFiles=$(ls -t .diff)
	    for dDF in $dotDiffFiles ; do
		SNinfo=$(grep 'DISKs:' .diff/$dDF | grep -v UNKNOWN)
		[[ $SNinfo ]] && break # found, stop
	    done
	    # if we failed to find a file then default to the sym link
	    # else we found it 
	    if [ "$SNinfo" ] ; then
		indexF=$dDF
		diffFile=.diff/$dDF
		nodeFile=.tbdb/$dDF
	    else
		indexF=$(pwd)
		diffFile=diff
		nodeFile=node
	    fi
	    listofdisks=$(grep 'DISKs:' $diffFile)
	    listofdisks=${listofdisks//DISKs:}  # take string out
Dan Reading's avatar
Dan Reading committed
231
	    hdnum=0
232
233
234
235
236
	    # if 'DISKs:' is empty then try OUT OF ORDER message
	    if [ -z "$listofdisks" ]; then
		local_listofdisks=$(grep 'OUT OF ORDER' $diffFile)
		# remove up to first 'local['
		local_listofdisks=${local_listofdisks#*local\[}
Dan Reading's avatar
Dan Reading committed
237
		# remove from ']' to end
238
		local_listofdisks=${local_listofdisks%%]*}
Dan Reading's avatar
Dan Reading committed
239
240
241
                # have a second case where "ERROR DISK OUT OF ORDER"
                # remove up to 'from tbdb'
		local_listofdisks=${local_listofdisks#*from\ tbdb\ }
242
                for i in $local_listofdisks ; do
243
244
		    # or we just didn't find anything
		    [[ -z "$local_listofdisks" ]] && continue
245
		    [[ $i == "UNKNOWN" ]] && continue
246
		    echo "# Local $host disks out of order - rewrite order this run. Run $0 again after more inventory captured"
247
248
		    echo "mysql -e \"delete from blockstore_attributes where attrvalue='$i';\" tbdb"
		    listofdisks="$listofdisks $i"
249
# echo "### not enabled - sudo rm $projdir/$host/diff"
250
251
                    done
	    fi
252
	    for i in $listofdisks ; do
253

254
		toadd=$(grep $i $nodeFile)
255
256
		if [ -n "$toadd" ] ; then
		    toadd=${toadd/DISKUNIT}
Dan Reading's avatar
Dan Reading committed
257
258
259
260
261
262
		    if [ "${toadd/SN=}" != "$toadd" ]; then
			sn=${toadd##*SN=}
			sn=${sn%% *}
			sn=${sn//\"/}
			# check and see if serial number is already in database
			bsidx=$(get_bsidx_from_sn $sn)
263
			[[ "$bsidx" ]] && continue
Dan Reading's avatar
Dan Reading committed
264
265
		    fi

266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
		    # /* by Mike's orders
		    #  type		"class"	"protocol"	"rot-speed"	nodes with this type
		    # 
		    #  pata-generic	local	PATA		7200		pc600,pc850,pc2000
		    #  scsi-generic	local	SCSI		10000		pc3000,pc6000
		    #  sata-generic	local	SATA		7200		pc2400w,d710,d820,pcivy
		    #  sas-generic	local	SAS		10000		d820
		    #  ssd-generic	local	SATA		0		gpuhost,d820(pc601)
		    #  c2100-sas	local	SAS		15000		c2100
		    # */
		    nodetype=$(mysql -B -N -e "select type from nodes where node_id='$host';" tbdb)
		    case $nodetype in
			pc600 | pc850 | pc2000 ) hdtype="pata-generic" ;;
			pc3000 | pc6000 ) hdtype="scsi-generic" ;;
			pc2400w | d710 | d820 | pcivy ) hdtype="sata-generic" ;;
			d820 ) hdtype="sas-generic" ;;
			gpuhost ) hdtype="ssd-generic" ;;
			c2100 ) hdtype="c2100-sas" ;;
			* ) 
			    hdtype=${toadd##*TYPE=}
			    hdtype=${hdtype%% *}
			    ;;
		    esac
		    # translate type
		    [[ "$hdtype" == "SATA" ]] && hdtype="sata-generic"
		    [[ "$hdtype" == "PATA" ]] && hdtype="pata-generic"
		    [[ "$hdtype" == "SAS" ]] && hdtype="sas-generic"
		    # XXX XXX XXX special case
		    if [ "$host" == "pc601" ] ; then
			hdtype="ssd-generic"
		    fi
		    # XXX XXX XXX special case
Dan Reading's avatar
Dan Reading committed
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
		    if [ "${toadd/SECSIZE=}" != "$toadd" ]; then
			secsize=${toadd##*SECSIZE=}
			secsize=${secsize%% *}
		    fi
		    if [ "${toadd/SECTORS=}" != "$toadd" ]; then
			sectors=${toadd##*SECTORS=}
			sectors=${sectors%% *}
		    fi
		    if [ "${toadd/RSPEED=}" != "$toadd" ]; then
			rspeed=${toadd##*RSPEED=}
			rspeed=${rspeed%% *}
		    fi
		    if [ "${toadd/WSPEED=}" != "$toadd" ]; then
			wspeed=${toadd##*WSPEED=}
			wspeed=${wspeed%% *}
		    fi
		    size=$(($sectors / 2048))
		    # check and see if bsidx already used
		    while [ "$(mysql -B -N -e "select bsidx from blockstores where bsidx=$bsidx_base;" tbdb)" ] ; do
317
#			echo "# bsidx=$bsidx_base already used, bumping bsidx_base again"
Dan Reading's avatar
Dan Reading committed
318
319
320
			((++bsidx_base))
		    done
                    # check and see if drive already used
321
322
		    found_blockstores_idx=''
		    found_blockstoreattr_idx=''
Dan Reading's avatar
Dan Reading committed
323
		    while [ "$(mysql -B -N -e "select bs_id from blockstores where node_id=\"$host\" and bs_id=\"disk${hdnum}\";" tbdb)" ] ; do
324
325
326
			# found a driveX on this host - see what we can find in the db
			found_blockstores_idx=''
			found_blockstoreattr_idx=''
327
			found_blockstores_idx=$(get_bsidx_from_hostAnddrive $host ${hdnum})
328
329
330
			found_blockstoreattr_idx=$(get_sn_from_bsidx $found_blockstores_idx)

			if [ "$found_blockstores_idx" ] ; then
331
			    echo -n "# $host drive${hdnum} already used @ bsidx=$found_blockstores_idx"
332
333
334
335
336
			    break
			else
			    echo "# drive${hdnum} already used, bumping hdnum again"
			    ((++hdnum))
			fi
Dan Reading's avatar
Dan Reading committed
337
		    done
338
339
340
341
342
343
344
345
346
347
348
349
		    if [ "$found_blockstores_idx" ] ; then
			# have a blockstores idx, check if we are missing blockstore_attrs
			if [ ! "$found_blockstoreattr_idx" ] ; then
			    # create one
			    echo " BUT blockstore_attributes missing"
			    printf "mysql -e \"insert into blockstore_attributes values (%d, '%s', '%s', '%s');\" tbdb\n" "$found_blockstores_idx" "serialnum" "$sn" "string" 
			    # assume we don't have to update blockstores, but display entry
			    echo "# MATCHING BLOCKSTORES @ $(mysql -B -N -e "select bsidx,node_id,bs_id,total_size  from blockstores where bsidx=$found_blockstores_idx;" tbdb);"
			else
			    echo ""
			fi
		    else
350
351
352
353
354
355
356
357
358
			# just make sure we don't have an entry
			justcheck=$(mysql -B -N -e "select * from blockstore_attributes where attrvalue='$sn';" tbdb );
			if [ "$justcheck" ] ; then
			    echo "ERROR: tbdb Inconsistance  EXIT"
			    echo "bsidx=$bsidx_base but found $sn in blockstore_attributes"
			    mysql -e "select * from blockstore_attributes where attrvalue='$sn';" tbdb
			    exit
			fi

359
360
361
362
363
364
365
			printf "mysql -e \"insert into blockstores values (%d, '%s', 'disk%d', 0, '%s', 'element', %d, 1, now());\" tbdb\n" "$bsidx_base" "$host" "$hdnum" "$hdtype" "$size" 
			# now the second table
			printf "mysql -e \"insert into blockstore_attributes values (%d, '%s', '%s', '%s');\" tbdb\n" "$bsidx_base" "serialnum" "$sn" "string" 
			# used up a bsidx, incr bsidx
			((++bsidx_base))
		    fi
		    # FINALLY onto the next drive
Dan Reading's avatar
Dan Reading committed
366
		    ((++hdnum))
367
		else
Dan Reading's avatar
Dan Reading committed
368
		    if [ "$i" == "UNKNOWN" ] ; then
369
370
			x=$(get_bsidx_from_hostAnddrive $host ${hdnum})
			if [ -n "$x" ] ; then
371
			    echo -n "# Can not locate Serial Number for disk${hdnum} on $host"
372
373
374
375
376
377
			    # so try and find the SN in any .full/* file with serial numbers
			    dotFullFiles=$(ls .full)
			    for dFF in $dotFullFiles ; do
				SNinfo=$(grep -v 'SN="UNKNOWN"' .full/$dFF | grep DISKUNIT)
				[[ $SNinfo ]] && break # found some stop
			    done
378
			    echo " but found $(get_drive_from_bsidx $x) $(get_host_from_bsidx $x) @ bsidx=$x SN=$(get_sn_from_bsidx $x)" 
379
380
			else
			    echo ""
381
			fi
Dan Reading's avatar
Dan Reading committed
382
383
			((++hdnum))
		    else
384
			toadd="SN='$i'"
385
			if [ "$sn" != "UNK" ] ; then
386
387
388
389
390
391
			    x_bsidx=$(get_bsidx_from_sn $i)
			    [[ $x_bsidx ]] && y_hostid=$(get_host_from_bsidx $x_bsidx) || y_hostid=""
			    if [ -n "$x_bsidx" -a -n "$y_hostid" ] ; then
				printf "mysql -e \"delete from blockstores where node_id='%s' and bsidx='%s';\" tbdb\n" "$host" "$x_bsidx"
				printf "mysql -e \"delete from blockstore_attributes where attrkey='%s';\" tbdb\n" "$i"
			    else
Dan Reading's avatar
Dan Reading committed
392
				echo "#Discovery error node_id $host, full:has SN BUT node:does not"
393
394
395
				      # Have only seen this when the "full" file for host lists a hard drive, including SN
				      # but the "node" file does not have serial number for for the HD
			    fi
396
			fi
Dan Reading's avatar
Dan Reading committed
397
		    fi
398
		fi
399
400
	    done
	    listofnics=$(grep 'NICs:' diff)
401
	    listofnics=${listofnics//NICs:/}
402
403
	    for i in $listofnics ; do
		toadd=$(grep $i node)
404
		if [ -n "$toadd" ] ; then
Dan Reading's avatar
Dan Reading committed
405
406
407
408
		    toadd=${toadd/*ID=\"}
		    toadd=${toadd/\"}
		    uuid=$(uuidgen)
		    printf "#BYHAND mysql -e \"insert into interfaces set node_id='%s',mac='%s',card=X,port=X,interface_type='?',iface='ethX',role='?',uuid='$uuid';\" tbdb\n" "$host" "$toadd"
409
410
		else
		    toadd="ID=\"$i\""
Dan Reading's avatar
Dan Reading committed
411
		    printf "#BYHAND mysql -e \"delete from interfaces where node_id='%s' and %s;\" tbdb\n" "$host" "$toadd"
412
		fi
413
	    done
414
	else
415
	    echo "# No record of node id \"$host\"."
416
	fi
417
    done
Dan Reading's avatar
Dan Reading committed
418
    if [ $keep_bsidx_base -lt $bsidx_base ] ; then
419
	echo "# orginal bsidx=$keep_bsidx_base : new bsidx=$bsidx_base"
Dan Reading's avatar
Dan Reading committed
420
	printf "mysql -e \"update emulab_indicies set idx=%d where name='next_bsidx';\" tbdb\n" "$bsidx_base"
421
    else
422
	[[ $reporting -eq 0 ]] && echo "# orginal bsidx=$keep_bsidx_base : new bsidx=$bsidx_base == no update"
Dan Reading's avatar
Dan Reading committed
423
    fi
424
425
426
427
428
429
430
431
432
}

setdirstructure() {
    # start XXX
    # decided to change names again, rename if old names -- XXX remove this after all node have run the new code
    # take into consideration if old directory structure was added to to a new directory structure 
    set -u
    projdir=/proj/emulab-ops/nodecheck
    cd $projdir
433
    allnodes=$(ls -d pc* pg* dbox* gpu* 2> /dev/null)
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
    dirs2workon=""
    for i in $allnodes; do [[ -L $i/$i.diff ]] && dirs2workon+="$i "; done
    owd=$PWD

    for host in $dirs2workon ; do
    	if [ -d ${projdir}/$host ] ; then
	    cd ${projdir}/$host
	    [[ -L $host.full ]] && rm $host.full
	    [[ -L $host.diff ]] && rm $host.diff
	    [[ -L $host ]] && rm $host
	    for i in full diff tbdb ; do
		if [ -d $i ] ; then
		    [[ ! -d .$i ]] && (mkdir .$i ; chmod g+x .$i)
		    list=$(ls -A $i)
		    for j in $list ; do
			mv $i/$j .$i
		    done
		    rmdir $i
		fi
	    done
	    cd $owd
	fi
    done
    
    # put links in place
    for i in $dirs2workon ; do
	cd $projdir/$i
	links="full diff node"
	for link in $links ; do
	    case $link in
		full | diff )
		    # make the symlinks
		    if [ ! -e ${link} -a -d .${link} ] ; then
			linkto=$(ls -t .${link} | head -1)
			if [ -n "$linkto" ] ; then
 			    ln -s .${link}/${linkto} $link
			fi
		    fi
		    ;;
		node )
		    if [ ! -e ${link} -a -d .tbdb ] ; then
			linkto=$(ls -t .tbdb | head -1)
			if [ -n "$linkto" ] ; then
 			    ln -s .tbdb/${linkto} $link
			    ln -s .tbdb/${linkto} $i
			fi
		    fi
		    ;;
	    esac
	done
    done
}

Dan Reading's avatar
Dan Reading committed
487
488
489
490
491
492
declare h4="no"

ph4() {
    [[ ! $h4 ]] && { echo "================================= $1 =================================" >> /tmp/Report_CheckNode; h4="no"; }
}

493
494
genreport_main() {
    reporting=1
Dan Reading's avatar
Dan Reading committed
495
496
497
498
499
500
501
    [[ -z "${CHECKNODE_CRONJOB-}" ]] && declare CHECKNODE_CRONJOB="NO"
    [[ -z "${CHECKNODE_MTA-}" ]] && declare CHECKNODE_MTA="sendmail -t"
    [[ -z "${CHECKNODE_MAILTO-}" ]] && declare CHECKNODE_MAILTO=""
    if [ "${CHECKNODE_CRONJOB}" != "NO" ] ; then
	# if MAILTO is empty then switch to cat-ing the info, cron can do with it what is wants
	# i.e. use the standard cron MAILTO var
	[[ -z "${CHECKNODE_MAILTO}" ]] && CHECKNODE_MTA="cat"
502
503
	echo "Subject: CheckNode report for $(hostname)" > /tmp/Report_CheckNode
	echo "From: CheckNode" >> /tmp/Report_CheckNode
Dan Reading's avatar
Dan Reading committed
504
	echo "To: ${CHECKNODE_MAILTO}" >> /tmp/Report_CheckNode
505
506
507
508
509
510
511
512
    else
	echo "" > /tmp/Report_CheckNode
    fi

    rm -f /tmp/genreport /tmp/driftreport
    nodes2do=$allnodes
    for node in $nodes2do ; do
	allnodes=$node
Dan Reading's avatar
Dan Reading committed
513
514
	h4=''
#	echo "================================= $node =================================" >> /tmp/Report_CheckNode
515
	if [ -s /proj/emulab-ops/nodecheck/$node/diff ] ; then
Dan Reading's avatar
Dan Reading committed
516
517
518
519
520
521
522
523
524
525
526
	    # chop top off diff report and remove UNKNOWN lines
	    tail -n +5 /proj/emulab-ops/nodecheck/$node/diff | grep -v UNKNOWN | grep -v MISSING > /tmp/diffchop
	    # do we still have something to report?
	    if [ -s /tmp/diffchop ] ; then
		diffDate=$(stat -f %Sm -t "%H:%M %d%b%g" /proj/emulab-ops/nodecheck/$node/diff)
		ph4 $node
		echo "----------------------- $diffDate inconsistency with db -------------" >> /tmp/Report_CheckNode
		# remove pesky empty line
		grep -v '^$' /tmp/diffchop >> /tmp/Report_CheckNode
		rm -f /tmp/diffchop
	    fi
527
528
529
530
	fi

	gentbsql_main $node > /tmp/genreport
	if [ -s /tmp/genreport ] ; then
Dan Reading's avatar
Dan Reading committed
531
532
533
534
	    ph4 $node
	    echo ">>>> Run gen_sql for suggested db updates" >> /tmp/Report_CheckNode
#	    echo "--------------------------- suggested db updates  -----------------------" >> /tmp/Report_CheckNode
#	    cat /tmp/genreport >> /tmp/Report_CheckNode
535
536
	fi

Dan Reading's avatar
Dan Reading committed
537
	checkdrift_main $node | grep -v '^$' > /tmp/driftreport
538
	if [ -s /tmp/driftreport ] ; then
Dan Reading's avatar
Dan Reading committed
539
	    ph4 $node
540
541
542
543
544
545
546
	    echo "--------------------------- changes over time ---------------------------" >> /tmp/Report_CheckNode
	    cat /tmp/driftreport >> /tmp/Report_CheckNode
	fi

	rm -f /tmp/genreport /tmp/driftreport
    done

Dan Reading's avatar
Dan Reading committed
547
548
    if [ "${CHECKNODE_CRONJOB}" != "NO" ] ; then
	cat /tmp/Report_CheckNode | ${CHECKNODE_MTA}
549
550
551
    else
	cat /tmp/Report_CheckNode
    fi
Dan Reading's avatar
Dan Reading committed
552
    rm -f /tmp/Report_CheckNode
553
554
}

555
556
# start here
set -u
557
kdir=$(pwd)
558
projdir=/proj/emulab-ops/nodecheck
559
# if projdir is automounted this should mount it
560
561
562
563
564
565
566
567
cd $projdir

if (( $# )) ; then
    allnodes="$@"
else
    nodes=$(ls */tmcc)
    allnodes=${nodes//\/tmcc/}
fi
568
cd $kdir
569

570
if [ "$0" != "${0/setdir}" ] ; then
571
    setdirstructure
572
elif [ "$0" != "${0/sql}" ] ; then
573
    gentbsql_main $@
574
575
elif [ "$0" != "${0/drift}" ] ; then
    checkdrift_main $@
576
577
elif [ "$0" != "${0/cleanup}" ] ; then
    cleanup_main $@
578
579
elif [ "$0" != "${0/report}" ] ; then
    genreport_main $@
580
581
elif [ "$0" != "wce" ] ; then
    checkwce_main $@
582
583
else
    "Print useage help"
584
fi