os_setup.in 10.9 KB
Newer Older
1
2
#!/usr/bin/perl -wT
use English;
3
use Getopt::Std;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
4
require 'ctime.pl';
5

6
#
7
8
9
10
# Reboot the nodes in an experiment. The nodes table will already contain
# all the information. This script deals with possible disk reloading,
# rebooting, and waiting for nodes to come back alive before allowing
# experiment creation to continue.
11
#
12
# TODO: Reload disk images.
13
# 
14
# usage: os_setup <pid> <eid>
15
#
16
17
18
19
20
21
sub usage()
{
    print STDOUT "Usage: os_setup <pid> <eid>\n";
    exit(-1);
}
my  $optlist = "";
22
23
24
25
26
27

#
# Configure variables
#
my $TB		= "@prefix@";
my $DBNAME	= "@TBDBNAME@";
28
my $TBOPS       = "@TBOPSEMAIL@";
29
my $TESTMODE    = @TESTMODE@;
30
my $TFTP	= "/tftpboot";
31

32
33
34
35
36
37
38
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;

39
my $nodereboot	= "$TB/bin/node_reboot";
40
my $ping	= "/sbin/ping";
41
my $dbg		= 0;
42
my @nodes       = ();
43
my %osids       = ();
44
my %waitfor     = ();
45
my %canfail     = ();
46
my $db_result;
47
my @row;
48

49
50
#
# This stuff is BOGUS! Quick hack for paper deadline to make Jay happy.
51
# If Frisbee works, this might be appropriate.
52
#
53
my $doreloading = 0;
54
my $forcereload = 0;
55
56
my %reload      = ();
    
57
58
59
60
61
62
# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

$| = 1; #Turn off line buffering on output

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 2) {
    usage();
}
my $pid = $ARGV[0];
my $eid = $ARGV[1];

#
# Untaint args.
#
if ($pid =~ /^([-\@\w]+)$/) {
    $pid = $1;
}
else {
    die("Bad data in pid: $pid.");
}
if ($eid =~ /^([-\@\w]+)$/) {
    $eid = $1;
}
else {
    die("Bad data in eid: $eid.");
}

93
#
94
# Figure out who called us. Only root, people with admin status
95
96
# in the DB, or the owner of the experiment can run this script.
#
97
if ($UID && !TBAdmin($UID)) {
98
99
100
    my ($me) = getpwuid($UID)
	or die "$UID not in passwd file";

101
102
103
104
    my $leader = ExpLeader($pid, $eid);

    if ($me ne $leader) {
	die("os_setup: You must be root or a TB administrator\n");
105
106
107
108
    }
}

#
109
# Get the set of nodes, as well as the nodes table information for them.
110
#
111
112
113
114
115
$db_result =
    DBQueryFatal("select * from nodes left join reserved on ".
		 "nodes.node_id=reserved.node_id ".
		 "where reserved.pid='$pid' and reserved.eid='$eid'");

116
117
if ($db_result->numrows < 1) {	
    die("There are no nodes assigned to experiment '$eid' in project '$pid'.");
118
119
}

120
for ($i = 0; $i < $db_result->numrows; $i++) {
121
122
123
124
    my %row      = $db_result->fetchhash();
    my $node     = $row{'node_id'};
    my $osid     = $row{'def_boot_osid'};
    my $bootpath = 0;
125
126

    push(@nodes, $node);
127
    $osids{$node} = $osid;
128

129
    #
130
131
132
    # Make sure the files specified in the paths exist. We mount the
    # user tftp directory on boss node, so we can ignore the IP address,
    # and just check the path directly. 
133
134
135
136
    #
    if (defined($row{'def_boot_path'})) {
	my $path = $row{'def_boot_path'};

137
138
139
140
141
142
143
144
145
146
147
	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
148
		die("*** File $path for node $node must reside in $TFTP\n");
149
150
	    }

151
	    if (! -f $path) {
152
		die("*** File $path for node $node does not exist!");
153
	    }
154
	    $bootpath = 1;
155
	}
156
157
158
159
    }
    if (defined($row{'next_boot_path'})) {
	my $path = $row{'next_boot_path'};

160
161
162
163
164
165
166
167
168
169
170
	if ($path ne "") {
	    my $ip   = 0;

	    # Split out IP address if it exists.
	    if ($path =~ /^([0-9\.]+):(\/.*)$/) {
		$ip   = $1;
		$path = $2;
	    }

	    # Path must begin with $TFTP
	    if (! ($path =~ /^\/$TFTP\//)) {
171
		die("*** File $path for node $node must reside in $TFTP\n");
172
173
	    }

174
	    if (! -f $path) {
175
		die("*** File $path for node $node does not exist!");
176
177
	    }
	}
178
179
    }

180
181
182
183
184
185
186
    #
    # XXX - Check for existence of the delta files. We do this here
    # cause its easier than looking for a failure later, when the node
    # tries to install the delta. Not a general solution though. Needs
    # more thought.
    #
    foreach my $delta (split(":", $row{'deltas'})) {
187
	if (! -f $delta) {
188
	    die("*** Delta file $delta for node $node does not exist!");
189
190
191
192
193
194
	}
    }
    #
    # XXX - Ditto for RPMs.
    #
    foreach my $rpm (split(":", $row{'rpms'})) {
195
	if (! -f $rpm) {
196
	    die("*** RPM $rpm for node $node does not exist!");
197
198
199
	}
    }
    
200
201
202
203
204
205
    #
    # XXX - Ditto for tarfiles.
    #
    foreach my $tarspec (split(":", $row{'tarballs'})) {
	my ($dir, $tar) = split(" ", $tarspec);
	
206
	if (! -f $tar) {
207
	    die("*** Tarfile $tar for node $node does not exist!");
208
209
	}
    }
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240

    #
    # If there is a path specified, then we don't worry anymore about it.
    # The user must know what is going on. The OSID might have a path
    # associated with it, which means the same thing; we don't worry about
    # it. 
    #
    if (! $bootpath) {
	#
	# These checks are not necessary if the front end and web page
	# are doing the right thing, but lets be careful anyway.
	# 
	if (! $osid) {
	    die("*** $node has no bootpath and no def_boot_osid set!\n");
	}

	#
	# Grab the info for this OSID. This is part of the image check.
	#
	my $osid_result =
	    DBQueryFatal("select * from os_info where osid='$osid'");
	
	if ($osid_result->numrows == 0) {
	    die("*** No such OSID $osid is defined!\n");
	}
	
	my %osid_row   = $osid_result->fetchhash();

	#
	# If there is an actual path, its an OSKit kernel not an image.
	# 
241
	if (! defined($osid_row{'path'}) || $osid_row{'path'} eq "") {
242
	    #
243
244
	    # Not an OSKit kernel.
	    # Make sure this OSID is actually loaded on the machine. 
245
	    #
246
247
248
	    my $p_result =
		DBQueryFatal("select * from partitions ".
			     "where node_id='$node' and osid='$osid'");
249
250

	    #
251
252
	    # If not loaded, then see if the user was looking for the generic
	    # name of the OS that is loaded. 
253
	    # 
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
	    if ($p_result->numrows == 0) {
		#
		# If its a specific Version, and its not loaded on the machine,
		# its currently an error. Later we might reload.
		#
		if (defined($osid_row{'version'}) &&
		    $osid_row{'version'} ne "") {
		    die("*** OSID $osid is not currently loaded on $node!\n");
		}

		#
		# A non-specific version. Try to map it.
		# 
		my $o_result =
		    DBQueryFatal("select o1.* from os_info as o1 ".
				 "left join partitions as p ".
				 "  on o1.osid=p.osid ".
				 "left join os_info as o2 ".
				 "  on o2.OS=o1.OS ".
				 "where p.node_id='$node' ".
				 "  and o2.osid='$osid'");

		if ($o_result->numrows == 0) {
		    die("*** $0:\n".
			"    No mapping can be made for $osid on $node!\n".
			"    Perhaps the disk needs reloading?\n");
		}
		else {
		    my %o_row  = $o_result->fetchhash();
		    my $n_osid = $o_row{'osid'};

		    print "Mapping $osid on $node to $n_osid.\n";
		    DBQueryFatal("update nodes set def_boot_osid='$n_osid' ".
				 "where node_id='$node'");
		    $osids{$node} = $n_osid;
		}
290
291
292
	    }
	}
    }
293
    
294
    #
295
    # If pingable, then the node is "waitable".
296
    #
297
    if (OSFeatureSupported($osids{$node}, "ping")) {
298
	$waitfor{$node} = 1;
299
300
    }
    else {
301
	$waitfor{$node} = 0;
302
    }
303
304
305
306
307

    #
    # Set the canfail bit. Currently, sharks are always canfail=1.
    # Will come from DB at some point.
    #
308
    if ($row{'type'} eq "dnard") {
309
310
311
312
313
314
	$canfail{$node} = 1;
    }
    else {
	$canfail{$node} = 0;
    }
    
315
    print STDOUT "$node - $osids{$node} - $waitfor{$node} - $canfail{$node}\n"
316
	if $dbg;
317
}
318

319
#
320
321
322
# Fire off a mass reboot. The reboot script does this in parallel, so
# no need to create any new children here. We just wait until it exits,
# which means all the nodes are actually rebooting.
323
#
324
325
if (!$TESTMODE) {
  if (system("$nodereboot @nodes")) {
326
    die("*** Failed to reboot some nodes!");
327
  }
328
329
}

330
331
print STDOUT "Waiting for testbed nodes to finish rebooting ...\n";

332
333
my $waitstart = time;

334
335
336
#
# Now lets wait for them to come back alive.
#
337
foreach my $node ( @nodes ) {
338
339
    my $failmesg;
    
340
341
342
343
    #
    # Don't bother to wait for nodes that are running foreign OSs since
    # we are not going to deal with them anyway later in the process.
    #
344
345
    if ($waitfor{$node} == 0) {
	print STDOUT "Not waiting for $node to come alive. Foreign OS.\n";
346
	SetNodeBootStatus($node, NODEBOOTSTATUS_UNKNOWN);
347
348
349
	next;
    }	

350
351
    if (WaitTillAlive($node) == 0) {
	print STDOUT "$node is alive and well\n";
352
	SetNodeBootStatus($node, NODEBOOTSTATUS_OKAY);
353
354
	next;
    }
355
    SetNodeBootStatus($node, NODEBOOTSTATUS_FAILED);
356

357
358
    print STDOUT "*** Oops, $node may be down. ".
	         "This has been reported to testbed-ops.\n";
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377

    if ($canfail{$node}) {
	# Send mail to testbed-ops and to the user about it.
	my ($user) = getpwuid($UID);
	
	SENDMAIL($user, "TESTBED: Node $node is down",
		 "Node $node in pid/eid $pid/$eid appears to be dead.\n\n".
		 "Your experiment will continue to run since this failure\n".
		 "is nonfatal, although you might encounter other problems\n".
		 "if your experiment depends explicitly on this node.\n".
		 "You should terminate this experiment if it cannot ".
		 "tolerate this failure.\n\n".
		 "Testbed Operations has also been notified so they can ".
		 "investigate.\n\n".
		 "Thanks\n".
		 "Testbed Operations\n",
		 0,
		 "Cc: $TBOPS");

378
	print STDERR "*** Continuing with experiment setup anyway ...\n";
379
380
381
382
383
384
385
386
387
388
389
390
391
    }
    else {
	# Reserve it to down experiment.
	MarkNodeDown($node);

	# Send mail to testbed-ops about it
	SENDMAIL($TBOPS, "TESTBED: Node $node is down",
		 "Node $node in pid/eid $pid/$eid appears to be dead.\n\n".
		 "Please look into this matter. $node has been reserved\n".
		 "by the Testbed until this matter has been resolved.\n\n".
		 "Thanks\n".
		 "Testbed Operations\n");

392
	die("*** Experiment will be now be terminated automatically.");
393
    }
394
395
396
397
398
}

print STDOUT "OS Setup Done!\n";
exit 0;

399
400
401
#
# Wait for a node to come back alive.
# 
402
sub WaitTillAlive {
403
    my ($pc) = @_;
404

405
406
407
    print STDERR "Waiting for $pc to come alive\n" if $dbg;
    
    #
408
    # Seems like a long time to wait, but it ain't!
409
    # 
410
    my $maxwait = (60 * 6);
411
    if ($reload{$pc}) {
412
	$maxwait += (60 * 5);
413
414
    }

415
416
417
418
419
420
421
    #
    # Start a counter going, relative to the time we rebooted the first
    # node. 
    # 
    my $waittime  = 0;
    my $minutes   = 0;

422
423
424
425
426
    #
    # Sigh, a long ping results in the script waiting until all the
    # packets are sent from all the pings, before it will exit. So,
    # loop doing a bunch of shorter pings.
    #
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
    while (1) {
	system("$ping -q -c 4 -t 4 $pc >/dev/null 2>&1");
	$status = $? >> 8;

	#
	# Returns 0 if any packets are returned. Returns 2 if pingable
	# but no packets are returned. Other non-zero error codes indicate
	# other problems.  Any non-zero return indicates "not pingable" to us.
	# 
	if (! $status) {
	    print STDERR "$pc alive and well\n" if $dbg;
	    return 0;
	}
	$waittime = time - $waitstart;
	if ($waittime > $maxwait) {
	    print "$pc appears dead; its been ",
	    (int ($waittime / 60))," minutes since reload started.\n";
	    return 1;
445
	}
446
447
448
	if (int($waittime / 60) > $minutes) {
	    $minutes = int($waittime / 60);
	    print "Still waiting for $pc - its been $minutes minute(s)\n";
449
	}
450
451
    }
}