Commit cfad00c0 authored by Dan Reading's avatar Dan Reading

changes to report different ordering of HDs

parent e51b5aaa
...@@ -132,7 +132,7 @@ err_report() { ...@@ -132,7 +132,7 @@ err_report() {
# read info from tmcc or a file. Copy into one of the three global arrays # read info from tmcc or a file. Copy into one of the three global arrays
# hwinv, hwinvcopy, tmccinfo # hwinv, hwinvcopy or tmccinfo
# $1 is the source $2 is the output array # $1 is the source $2 is the output array
# error if not both set # error if not both set
readtmcinfo() { readtmcinfo() {
...@@ -153,6 +153,7 @@ readtmcinfo() { ...@@ -153,6 +153,7 @@ readtmcinfo() {
fi fi
if [ "$source" = "tmcc" ] ; then if [ "$source" = "tmcc" ] ; then
# need temp file to hold tmcc hwinv output
rmtmp="y" # remove tmp file rmtmp="y" # remove tmp file
source=/tmp/.$$tmcchwinv source=/tmp/.$$tmcchwinv
$($BINDIR/tmcc hwinfo > $source) $($BINDIR/tmcc hwinfo > $source)
...@@ -316,7 +317,7 @@ comparetmcinfo() { ...@@ -316,7 +317,7 @@ comparetmcinfo() {
for i in $arrayidx ; do for i in $arrayidx ; do
if [ "${hwinv[$i]}" != "${hwinvcopy[$i]}" ] ; then if [ "${hwinv[$i]}" != "${hwinvcopy[$i]}" ] ; then
if [ ! -f $fileout ] ; then if [ ! -f $fileout ] ; then
echo "Differences found locally compared with testbed database" > $fileout echo "Differences found locally compared with testbed database" >> $fileout
fi fi
echo "$i does not match" >> $fileout echo "$i does not match" >> $fileout
echo "local ${hwinv[$i]}" >> $fileout echo "local ${hwinv[$i]}" >> $fileout
...@@ -348,7 +349,7 @@ compareunits() { ...@@ -348,7 +349,7 @@ compareunits() {
local localidx="${hwinv["hwinvidx"]}" local localidx="${hwinv["hwinvidx"]}"
local tbdbidx="${hwinvcopy["hwinvidx"]}" local tbdbidx="${hwinvcopy["hwinvidx"]}"
local localunits="" tbdbunits="" devunit="" local localunits="" tbdbunits="" devunit=""
local -i a b local -i a b disregard_order
local x addr ckaddr local x addr ckaddr
# How are things different between unit types, only NET and DISK right now # How are things different between unit types, only NET and DISK right now
...@@ -361,6 +362,7 @@ compareunits() { ...@@ -361,6 +362,7 @@ compareunits() {
unit_post_strip="\"*" unit_post_strip="\"*"
unit_human_output="NIC" unit_human_output="NIC"
unit_human_case="lower" unit_human_case="lower"
disregard_order=1
;; ;;
DISK ) DISK )
unitinfoidx_str="DISKINFO" unitinfoidx_str="DISKINFO"
...@@ -370,6 +372,7 @@ compareunits() { ...@@ -370,6 +372,7 @@ compareunits() {
unit_post_strip="\"*" unit_post_strip="\"*"
unit_human_output="DISK" unit_human_output="DISK"
unit_human_case="upper" unit_human_case="upper"
disregard_order=0
;; ;;
* ) * )
echo "Error in compareunits don't now type $unittype. Giving up." echo "Error in compareunits don't now type $unittype. Giving up."
...@@ -429,22 +432,27 @@ compareunits() { ...@@ -429,22 +432,27 @@ compareunits() {
tbdbunits=${tbdbunits,,} tbdbunits=${tbdbunits,,}
fi fi
# remove from the lists all matching words if (( $disregard_order )) ; then
x=$localunits # remove from both lists all words matched in the other list
for i in $x ; do # any thing left-over is non-matching
if [ "${tbdbunits/$i}" != "${tbdbunits}" ]; then x=$localunits
tbdbunits=${tbdbunits/$i} for i in $x ; do
localunits=${localunits/$i} if [ "${tbdbunits/$i}" != "${tbdbunits}" ]; then
fi # same, take it out of both lists
done tbdbunits=${tbdbunits/$i}
# same but swap arrays localunits=${localunits/$i}
x=$tbdbunits fi
for i in $x ; do done
if [ "${localunits/$i}" != "${localunits}" ]; then # same but swap arrays
localunits=${localunits/$i} x=$tbdbunits
tbdbunits=${tbdbunits/$i} for i in $x ; do
fi if [ "${localunits/$i}" != "${localunits}" ]; then
done localunits=${localunits/$i}
tbdbunits=${tbdbunits/$i}
fi
done
fi
#remove extra spaces #remove extra spaces
save_e save_e
set +e set +e
...@@ -452,15 +460,26 @@ compareunits() { ...@@ -452,15 +460,26 @@ compareunits() {
read -rd '' localunits <<< "$localunits" read -rd '' localunits <<< "$localunits"
restore_e restore_e
# if the two strings are the same then zero strings
# any mismatches would be in ether localunits or tbdbunits if (( $disregard_order )) ; then
if [ -n "${localunits}" ]; then # any strings would be mismatches in ether localunits or tbdbunits
printf "%s%s %s\n" "${unit_human_output}" "s:" "$localunits" >> $localonly if [ -n "${localunits}" ]; then
fi printf "%s%s %s\n" "${unit_human_output}" "s:" "$localunits" >> $localonly
if [ -n "${tbdbunits}" ]; then fi
printf "%s%s %s\n" "${unit_human_output}" "s:" "$tbdbunits" >> $tbdbonly if [ -n "${tbdbunits}" ]; then
printf "%s%s %s\n" "${unit_human_output}" "s:" "$tbdbunits" >> $tbdbonly
fi
else
if [ "$tbdbunits" != "$localunits" ] ; then
# we care about order report it
[[ -z "${offline-}" ]] && declare -i offline=0 # if set from gen_sql
(( ! $offline )) && printf "ERROR %s%s OUT OF ORDER found %s from tbdb %s\n" "${unit_human_output}" "s:" "$localunits" "$tbdbunits"
(( ! $offline )) && ( printf "ERROR %s OUT OF ORDER found %s from tbdb %s\n" "${unit_human_output}" "$localunits" "$tbdbunits" >> $fileout ) || ( printf "WARNING %s%s ORDER '%s' compared to '%s'\n" "${unit_human_output}" "s:" "$localunits" "$tbdbunits" >> $fileout )
fi
fi fi
return 0 return 0
} }
...@@ -478,19 +497,6 @@ uniqstr() { ...@@ -478,19 +497,6 @@ uniqstr() {
echo $outstr echo $outstr
} }
# no args uses the globel arrays hwinvv, tcm_in, tcm_out
#mergetmcinfo() {
# for i in ${hwinv["hwinvidx"]} ; do
# hwinv[$i]+=" ADD"
# done
#}
# arg $1 is the file to write uses the globel tcm_out array
#writetmcinfo() {
#:
#}
# print only the testbed data table # print only the testbed data table
printtmcinfo() { printtmcinfo() {
local -i hdunits=0 nicunits=0 local -i hdunits=0 nicunits=0
......
...@@ -237,7 +237,6 @@ if (( $check_flag )) ; then ...@@ -237,7 +237,6 @@ if (( $check_flag )) ; then
tbinfo=${tbinfo% } # get rid of trailing space tbinfo=${tbinfo% } # get rid of trailing space
# echo driveinfo:$driveinfo: tbinfo:$tbinfo: # echo driveinfo:$driveinfo: tbinfo:$tbinfo:
{ {
#save for both output to stdout and log file #save for both output to stdout and log file
if [ -z "$tbinfo" ] ; then if [ -z "$tbinfo" ] ; then
...@@ -248,53 +247,57 @@ if (( $check_flag )) ; then ...@@ -248,53 +247,57 @@ if (( $check_flag )) ; then
#upper case #upper case
tbinfo=${tbinfo^^} tbinfo=${tbinfo^^}
driveinfo=${driveinfo^^} driveinfo=${driveinfo^^}
# turn space seperated string into array # turn space seperated string into array
unset -v tb; declare -a tb=(${tbinfo// / }) unset -v tb; declare -a tb=(${tbinfo// / })
unset -v have; declare -a have=(${driveinfo// / }) unset -v have; declare -a have=(${driveinfo// / })
havecnt=${#have[*]} havecnt=${#have[*]}
tbcnt=${#tb[*]} tbcnt=${#tb[*]}
numserial=${tbcnt} #-# numserial=${tbcnt}
maxcnt=${tbcnt} #-# maxcnt=${tbcnt}
#-#
if [[ ${tbcnt} -ne ${havecnt} ]] ; then if [[ ${tbcnt} -ne ${havecnt} ]] ; then
failed="TBmiss" failed="TBmiss "
[[ ${tbcnt} -gt ${havecnt} ]] && maxcnt=${tbcnt} || maxcnt=${havecnt} #-# [[ ${tbcnt} -gt ${havecnt} ]] && maxcnt=${tbcnt} || maxcnt=${havecnt}
fi fi
#-#
for ((idx=0; idx<$maxcnt; idx++)) ; do #-# for ((idx=0; idx<$maxcnt; idx++)) ; do
for ((ifi=0; ifi<$havecnt; ifi++)) ; do #-# for ((ifi=0; ifi<$havecnt; ifi++)) ; do
for ((tbi=0; tbi<$tbcnt; tbi++)) ; do #-# for ((tbi=0; tbi<$tbcnt; tbi++)) ; do
if [ "${have[$ifi]}" = "${tb[$tbi]}" ] ; then #-# if [ "${have[$ifi]}" = "${tb[$tbi]}" ] ; then
have[$ifi]='' #-# have[$ifi]=''
tb[$tbi]='' #-# tb[$tbi]=''
fi #-# fi
done #-# done
done #-# done
done #-# done
#turn array into string #-# #turn array into string
haveresult=${have[@]-""} haveresult=${have[@]-""}
tbresult=${tb[@]-""} tbresult=${tb[@]-""}
# comment out the next if statment and uncommet the #-# lines for drive
if [[ "$haveresult" || "$tbresult" ]] ; then # ordering not to matter when comparing drives found with tbdb
if [[ $haveresult ]] ; then if [[ "$haveresult" || "$tbresult" ]] ; then
failed="${failed}: NotIn TB $haveresult" failed="$failed found $haveresult but tbdb says $tbresult "
fi fi
if [[ $tbresult ]] ; then #-#
failed="${failed}: TB Claims $tbresult" #-# if [[ "$haveresult" || "$tbresult" ]] ; then
fi #-# if [[ $haveresult ]] ; then
fi #-# failed="${failed}: NotIn TB $haveresult"
#-# fi
#-# if [[ $tbresult ]] ; then
#-# failed="${failed}: TB Claims $tbresult"
#-# fi
#-# fi
echo -n "Have $havecnt drive" echo -n "Have $havecnt drive"
[[ ${havecnt} -gt 1 ]] && echo -n "s" [[ ${havecnt} -gt 1 ]] && echo -n "s"
[[ -z ${failed} ]] && echo " OK" || echo " $failed" FAILED [[ -z ${failed} ]] && echo " OK" || echo " $failed" FAILED
} > ${tmplog} 2>&1 } > ${tmplog} 2>&1
cat ${tmplog} >> $logfile cat ${tmplog} >> $logfile
cat ${tmplog} cat ${tmplog}
fi # check_flag fi # if (( $check_flag ))
} }
getdriveinfo () { getdriveinfo () {
......
...@@ -5,7 +5,7 @@ source checkutils.sh ...@@ -5,7 +5,7 @@ source checkutils.sh
checkdrift_main() { checkdrift_main() {
set -u set -u
declare -i countsame=1 filecount=0 hostcount=0 declare -i countsame=1 filecount=0 hostcount=0
projdir=/proj/emulab-ops/nodecheck projdir=/proj/emulab-ops/nodecheck
cd $projdir cd $projdir
if (( $# )) ; then if (( $# )) ; then
...@@ -19,22 +19,38 @@ checkdrift_main() { ...@@ -19,22 +19,38 @@ checkdrift_main() {
if [ -d $projdir/$host/.tbdb ] ; then if [ -d $projdir/$host/.tbdb ] ; then
cd $projdir/$host/.tbdb cd $projdir/$host/.tbdb
allchecks=$(ls | sort -r | paste -s -d \\\ -) # paste command used to convert into space seperated words. allchecks=$(ls | sort -r | paste -s -d \\\ -) # paste command used to convert into space seperated words.
allchecks+=' ' # grr string matching, expect space at end allchecks+=' ' # grrr my string matching, expects space at end
# check that the first file is has a mfsmode=1 # set newest file and take off list
newest=${allchecks%%\ *} newest=${allchecks%%\ *}
allchecks=${allchecks#* } # pop the list allchecks=${allchecks#* } # pop the list
[[ ! $allchecks ]] && continue [[ ! $allchecks ]] && continue # ran out of files for this node
#only compare inventory computed in mfsmode
# from the top of list skip file that have mfsmode=0
while (grep -c 'ismfs=0' $newest &> /dev/null ) ; do while (grep -c 'ismfs=0' $newest &> /dev/null ) ; do
newest=${allchecks%%\ *} # take off the top newest=${allchecks%%\ *} # take off the top
allchecks=${allchecks#* } # pop the list allchecks=${allchecks#* } # pop the list
[[ ! $allchecks ]] && break [[ ! $allchecks ]] && break # ran out of files, break from loop
done done
[[ ! $allchecks ]] && continue # make sure we didn't consume all the file for node
# X X X Hackitly Hackit Hack
# drop those files that have empty SN, but report it
if [ "$(grep 'DISKUNIT TYPE' $newest)" ] ; then
echo "Missing SN in $host $(pwd)/$newest"
newest=${allchecks%%\ *} # take off the top
allchecks=${allchecks#* } # pop the list
[[ ! $allchecks ]] && continue # ran out of files
fi
readtmcinfo $newest hwinvcopy readtmcinfo $newest hwinvcopy
for tocheck in $allchecks ; do for tocheck in $allchecks ; do
[[ $(grep 'ismfs=0' $tocheck) ]] && continue # if not in ismfs mode then don't check [[ $(grep 'ismfs=0' $tocheck) ]] && continue # if not in ismfs mode then don't check
if [ "$(grep 'DISKUNIT TYPE' $tocheck)" ] ; then
echo "Missing SN in $host $(pwd)/$tocheck"
continue
fi
readtmcinfo $tocheck hwinv readtmcinfo $tocheck hwinv
# note: will check against self for sanity # note: will check against self for sanity
comparetmcinfo /tmp/.$$checkdiff comparetmcinfo /tmp/.$$checkdiff
...@@ -43,8 +59,9 @@ checkdrift_main() { ...@@ -43,8 +59,9 @@ checkdrift_main() {
echo "" echo ""
echo "$host $newest $tocheck are not the same" echo "$host $newest $tocheck are not the same"
echo "=====================================================" echo "====================================================="
echo "diff $tocheck $newest" cat /tmp/.$$checkdiff
diff $tocheck $newest # echo "diff $tocheck $newest"
# diff $tocheck $newest
# reset compare file # reset compare file
newest=$tocheck newest=$tocheck
readtmcinfo $newest hwinvcopy readtmcinfo $newest hwinvcopy
...@@ -66,7 +83,6 @@ checkdrift_main() { ...@@ -66,7 +83,6 @@ checkdrift_main() {
echo "No record of node id \"$host\"." echo "No record of node id \"$host\"."
fi fi
done done
echo ""
# : echo "$filecount files in $hostcount nodes checked" # : echo "$filecount files in $hostcount nodes checked"
} }
checkwce_main() { checkwce_main() {
...@@ -164,6 +180,7 @@ gentbsql_main() { ...@@ -164,6 +180,7 @@ gentbsql_main() {
listofdisks=$(grep 'DISKs:' diff) listofdisks=$(grep 'DISKs:' diff)
listofdisks=${listofdisks//DISKs:} listofdisks=${listofdisks//DISKs:}
hdnum=0 hdnum=0
for i in $listofdisks ; do for i in $listofdisks ; do
toadd=$(grep $i node) toadd=$(grep $i node)
if [ -n "$toadd" ] ; then if [ -n "$toadd" ] ; then
...@@ -242,12 +259,10 @@ gentbsql_main() { ...@@ -242,12 +259,10 @@ gentbsql_main() {
((++hdnum)) ((++hdnum))
else else
if [ "$i" == "UNKNOWN" ] ; then if [ "$i" == "UNKNOWN" ] ; then
echo -n "# Missing Serial Number for disk${hdnum} on $host"
x=$(get_bsidx_from_hostAnddrive $host ${hdnum}) x=$(get_bsidx_from_hostAnddrive $host ${hdnum})
if [ -n "$x" ] ; then if [ -n "$x" ] ; then
echo -n "# Missing Serial Number for disk${hdnum} on $host"
echo " but found $(get_drive_from_bsidx $x) $(get_host_from_bsidx $x) @ bsidx=$x SN=$(get_sn_from_bsidx $x)" echo " but found $(get_drive_from_bsidx $x) $(get_host_from_bsidx $x) @ bsidx=$x SN=$(get_sn_from_bsidx $x)"
else
echo ""
fi fi
((++hdnum)) ((++hdnum))
else else
...@@ -260,7 +275,6 @@ gentbsql_main() { ...@@ -260,7 +275,6 @@ gentbsql_main() {
printf "mysql -e \"delete from blockstore_attributes where attrkey='%s';\" tbdb\n" "$i" printf "mysql -e \"delete from blockstore_attributes where attrkey='%s';\" tbdb\n" "$i"
else else
echo "#Discovery error node_id $host" echo "#Discovery error node_id $host"
# Discovery error
# Have only seen this when the "full" file for host lists a hard drive, including SN # Have only seen this when the "full" file for host lists a hard drive, including SN
# but the "node" file does not have serial number for for the HD # but the "node" file does not have serial number for for the HD
fi fi
...@@ -358,6 +372,7 @@ if [ "$0" != "${0/setdir}" ] ; then ...@@ -358,6 +372,7 @@ if [ "$0" != "${0/setdir}" ] ; then
elif [ "$0" != "${0/sql}" ] ; then elif [ "$0" != "${0/sql}" ] ; then
gentbsql_main $@ gentbsql_main $@
elif [ "$0" != "${0/drift}" ] ; then elif [ "$0" != "${0/drift}" ] ; then
declare -i offline=1
checkdrift_main $@ checkdrift_main $@
elif [ "$0" != "wce" ] ; then elif [ "$0" != "wce" ] ; then
checkwce_main $@ checkwce_main $@
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment