os_load.in 13 KB
Newer Older
1
#!/usr/bin/perl -wT
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
3
4

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
6
7
8
# All rights reserved.
#

9
use English;
10
use Getopt::Std;
11
use File::stat;
12

13
14
15
16
# Be careful not to exit on transient error - this script is in the 'critical
# path' for reload_daemon, so we want to give it the same retries as the daemon
$libdb::DBQUERY_MAXTRIES = 30;

17
# XXX boss.emulab.net and users.emulab.net wired in.
18
#     wd0 wired in. Should come from node_types table in DB
19

20
21
# Load an image onto a disk. The image must be in the DB images table,
# which defines how/where to load, and what partitions are affected.
22
# The nodes and partitions tables are updated appropriately.
23
24
sub usage()
{
Leigh B. Stoller's avatar
Leigh B. Stoller committed
25
    print STDOUT
26
27
	"Usage: os_load [-s] [[-p <pid>] -i <imagename>] <node> [node ...]\n".
	"       os_load [-s] [[-p <pid>] -i <imagename>] -e pid,eid\n".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
28
	"       os_load -l\n".
29
30
	"Use -i to specify an image name. Use node default otherwise.\n".
	"Use -m to specify an image ID (internal name, TB admins only!).\n".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
31
	"Use -s to start reload, but do not wait for it to complete.\n".
32
	"Use -r to supress rebooting nodes - you'll need to to it yourself\n".
33
34
	"Use -e to reload all the nodes in an experiment.\n" .
        "Use -l to get a list of images you are permitted to load.\n";
35
36
    exit(-1);
}
37
my  $optlist = "sldi:e:p:m:r";
38
39
40

# Configure variables
my $TB		= "@prefix@";
41
my $TESTMODE    = @TESTMODE@;
42
my $TBOPS       = "@TBOPSEMAIL@";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
43
44
45
my $BOSSADDR	= "@BOSSNODE@";
my $USERADDR	= "@USERNODE@";

Leigh B. Stoller's avatar
Leigh B. Stoller committed
46
47
48
49
# Max number of retries (per node) before its deemed fatal. This allows
# for the occasional pxeboot failure.
my $MAXRETRIES  = 1;

50
# Load the Testbed support stuff.
51
52
53
use lib "@prefix@/lib";
use libdb;
use libtestbed;
54

55
56
57
# Be careful not to exit on transient error
$libdb::DBQUERY_MAXTRIES = 30;

58
my $FRISBEELAUNCHER = "$TB/sbin/frisbeelauncher";
59
my $nodereboot	= "$TB/bin/node_reboot";
60
my $schedreload	= "$TB/bin/sched_reload";
61
my $osselect	= "$TB/bin/os_select";
62
my $ping	= "/sbin/ping";
63
my $dbg		= 0;
64
my @row;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
65
my $usedefault  = 1;
66
my $imagename;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
67
my $imageid;
68
my $imagepid    = TB_OPSPID;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
69
my %imageid_row;
70
my @nodes       = ();
71
my %retries	= ();
72
my $mereuser    = 0;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
73
my $waitmode    = 1;
74
my $failures    = 0;
75
my $startwait   = 0;
76
my $maxwait	= 0;
77
my $reboot      = 1;
78
79
80
81
82
83
84

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

$| = 1; #Turn off line buffering on output

85
86
87
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
%options = ();
88
89
90
91
if (! getopts($optlist, \%options)) { usage(); }
if (defined($options{"d"})) { $dbg++; }
if (defined($options{"l"})) { dolisting(); exit(0); }
if (defined($options{"s"})) { $waitmode = 0; }
92
if (defined($options{"r"})) { $reboot = 0; }
93
if (defined($options{"i"}) && defined($options{"m"})) { usage(); }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
94
if (defined($options{"i"})) {
95
96
    $imagename  = $options{"i"};
    $usedefault = 0;
97
98
    if ($imagename =~ /^([-\w\.\+]+)$/) { $imagename = $1; }
    else { die("*** Bad data in $imagename.\n"); }
99
100
101

    if (defined($options{"p"})) {
	$imagepid = $options{"p"};
102
103
	if ($imagepid =~ /^([-\w\.\+]+)$/) { $imagepid = $1; }
	else { die("*** Bad data in $imagepid.\n"); }
104
105
106
107
    }
}
if (defined($options{"m"})) {
    $imageid = $options{"m"};
Leigh B. Stoller's avatar
Leigh B. Stoller committed
108
    $usedefault = 0;
109
110
    if ($imageid =~ /^([-\w\.\+]+)$/) { $imageid = $1; }
    else { die("*** Bad data in $imageid\n"); }
111
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
112
if (defined($options{"e"})) {
113
    if (@ARGV) { usage(); }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
114
    my $pideid = $options{"e"};
115

Leigh B. Stoller's avatar
Leigh B. Stoller committed
116
117
118
119
120
    if ($pideid =~ /([-\w]*),([-\w]*)/) {
	if (! (@nodes = ExpNodes($1, $2))) {
	    die("*** $0:\n".
		"    There are no nodes in $1/$2!\n");
	}
121
	$imagepid = $1;
122
123
    }
    else {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
124
125
	die("Invalid argument to -e option: $pideid\n");
	usage();
126
    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
127
128
}
else {
129
    if (! @ARGV) { usage(); }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
130
131
    # Untaint nodes.
    foreach my $node ( @ARGV ) {
132
133
	if ($node =~ /^([-\@\w]+)$/) { $node = $1; }
	else { die("*** Bad node name: $node.\n"); }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
134
135
	push(@nodes, $node);
    }
136
}
137

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
# VIRTNODE HACK: Virtual nodes are special. Do not reload!
my @temp = ();
foreach my $node ( @nodes ) {
    if (TBIsNodeVirtual($node)) {
	print "*** Skipping virtual node $node ...\n";
	next;
    }
    push(@temp, $node);
}
@nodes = @temp;
if (! @nodes) {
    print "No nodes to load. Exiting ...\n";
    exit(0);
}

153
154
155
# Figure out who called us. Root and admin types can do whatever they
# want. Normal users can only change nodes in experiments in their
# own projects.
156
157
if ($UID && !TBAdmin($UID)) {
    $mereuser = 1;
158

Leigh B. Stoller's avatar
Leigh B. Stoller committed
159
    if (! TBNodeAccessCheck($UID, TB_NODEACCESS_LOADIMAGE, @nodes)) {
160
161
162
163
	die("*** $0:\n".
            "    You do not have permission to load images on one (or more) ".
	    "nodes!\n");
    }
164
165
}

166
167
168
169
170
171
# Convert external name to internal (imageid), and check permission.
if (defined($imagename)) {
    if (! ($imageid = TBImageID($imagepid, $imagename))) {
	die("*** $0:\n".
	    "    No such image $imagename in project $imagepid!\n");
    }
172
}
173

174
175
176
177
if (defined($imageid) && $mereuser &&
    ! TBImageIDAccessCheck($UID, $imageid, TB_IMAGEID_READINFO)) {
    die("*** $0:\n".
	"    You do not have permission to load this image!\n");
178
179
}

180
# Loop for each node.
181
foreach my $node (@nodes) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
182
183
    # Oh, I suppose it would be nice to do this just once per imageid,
    # but that would be a pain. Not in the mood.
184

185
186
187
188
189
190
    # Get default imageid for this node.
    my $default_imageid;
    if (! ($default_imageid = DefaultImageID($node))) {
	die("*** $0:\n".
	    "    No default imageid is defined for $node!\n");
    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
191
    if ($usedefault) {
192
	$imageid = $default_imageid;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
193
    }
194
195

    print STDERR "Using $imageid for for $node\n" if $dbg;
196

Leigh B. Stoller's avatar
Leigh B. Stoller committed
197
198
    my $db_result =
	DBQueryFatal("select * from images where imageid='$imageid'");
199

Leigh B. Stoller's avatar
Leigh B. Stoller committed
200
    if ($db_result->numrows < 1) {
201
202
	die("*** $0:\n".
	    "    No such imageid $imageid is defined in the DB!\n");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
203
204
    }
    %imageid_row = $db_result->fetchhash();
205

206
207
208
209
    my $loadpart       = $imageid_row{'loadpart'};
    my $loadlen        = $imageid_row{'loadlength'};
    my $imagepath      = $imageid_row{'path'};
    my $defosid        = $imageid_row{'default_osid'};
Leigh B. Stoller's avatar
Leigh B. Stoller committed
210

211
212
213
214
215
216
217
218
219
220
    # Check for a few errors early!
    if (!defined($imagepath)) {
	die("*** $0:\n".
	    "    There is no filename associated with $imageid!\n");
    }
    if (! -R $imagepath) {
	die("*** $0:\n".
	    "    $imagepath does not exists or cannot be read!\n");
    }

221
222
223
224
225
226
227
228
229
    #
    # If there's a maxiumum number of concurrent loads listed, check to
    # see if we'll go over the limit, by checking to see how many other
    # nodes are currently booting thie image's default_osid. This is NOT
    # intended to be strong enforcement of license restrictions, just a way
    # to catch mistakes.
    # XXX This could go outside the @nodes loop, but so could most of this
    # stuff
    #
230
231
232
233
    if (!TBImageLoadMaxOkay($imageid,scalar(@nodes),@nodes)) {
	die("*** $0:\n".
	    "    This image has a limited number of maximum concurrent " .
	    "instances\n");
234
235
    }

236
237
238
239
240
241
    #
    # Compute a maxwait time based on the image size plus a constant
    # factor for the reboot cycle. 
    #
    my $sb     = stat($imagepath);
    my $chunks = $sb->size / (1024 * 1024);
242
    $maxwait   = int((($chunks / 100.0) * 30) + (5 * 60));
243

Leigh B. Stoller's avatar
Leigh B. Stoller committed
244
245
    # 0 means load the entire disk.
    my $diskpart = "";
246
247
    if ($loadpart) { $diskpart = "wd0:s${loadpart}"; }
    else  {$diskpart = "wd0"; }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
248
249

    print STDOUT "Changing default OS for $node to $defosid\n";
250
    if (!$TESTMODE) {
251
        system("$osselect -m PXEBOOT $node");
252
        system("$osselect $defosid $node");
253
    }
254

255
    # If loading an image (which is not the default) then
256
257
258
    # schedule a reload for it so that when the experiment is terminated
    # it will get a fresh default image before getting reallocated to
    # another experiment.
259
    if ($imageid ne $default_imageid) {
260
261
262
263
264
265
	if (! TBSetSchedReload($node, $default_imageid)) {
	    print "*** $0:\n".
		  "    WARNING: Could not schedule default reload for $node!";
	}
    }

266
267
268
    # Assign partition table entries for each partition in the image.
    # This is complicated by the fact that an image that covers only
    # part of the slices, should only change the partition table entries
269
    # for the subset of slices that are written to disk.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
270
    my $startpart = $loadpart == 0 ? 1 : $loadpart;
271
    my $endpart   = $startpart + $loadlen;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
272
273
	
    for ($i = $startpart; $i < $endpart; $i++) {
274
275
276
	my $partname = "part${i}_osid";
	if (defined($imageid_row{$partname})) {
	    my $osid = $imageid_row{$partname};
277
278
	    DBQueryFatal("replace into partitions ".
			 "(partition, osid, node_id) ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
279
			 "values('$i', '$osid', '$node')");
280
	} else {
281
	    DBQueryFatal("delete from partitions ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
282
			 "where node_id='$node' and partition='$i'");
283
284
	}
    }
285

Leigh B. Stoller's avatar
Leigh B. Stoller committed
286
    print STDOUT "Setting up reload for $node\n";
287
    if (!$TESTMODE) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
288
        SetupReload($node);
289
    }
290
}
291

292
# Exit if not doing about actual reload.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
293
if ($TESTMODE) {
294
    print STDOUT "OS Reload (Testmode) Done!\n";
295
296
297
    exit 0;
}

298
# Fire off a mass reboot and quit if not in waitmode.
299
if (! $waitmode) {
300
301
302
303
304
305
306
    if ($reboot) {
	print STDOUT "Reload ready. Rebooting nodes at ".`date`;
	system("$nodereboot @nodes");
	$failures = $? >> 8;
    } else {
	$failures = 0;
    }
307
308
309

    print STDOUT "OS Reload (no waiting) Done!\n";
    exit $failures;
310
311
}

312
313
314
315
# The retry vector is initialized to the number of retries we allow per
# node, afterwhich its a fatal error.
foreach my $node (@nodes) {
    $retries{$node} = $MAXRETRIES;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
316
317
}

318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
my @failed=();
while (@nodes) {
  # Reboot them all.
  print "Issuing reboot for @nodes and then waiting ...\n";

  if ($reboot) {
      system("$nodereboot @nodes");
      if ($?) {
	  print "Reboot failed for (some of) @nodes. Quitting!\n";
	  exit ($? >> 8);
      }
  }

  # Now wait for them.
  $startwait = time;
  @failed = WaitTillReloadDone(@nodes);
  @nodes=();
  while (@failed) {
    my $node = shift(@failed);

    if ($retries{$node}) {
      print "*** Trying $node again (resetting/rebooting) ...\n";
      push(@nodes, $node);

      # Possible race with reboot?
      SetupReload($node);

      # Retry until count hits zero.
      $retries{$node} -= 1;
    } else {
      print "*** $node failed too many times. Skipping!\n";
      $failures++;
    }
  }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
352
353
354
355
356
}

print "OS Reload Done! There were $failures failures!\n";
exit($failures);

357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
# Wait for a reload to finish by watching its state
sub WaitTillReloadDone {
    my (@nodes) = @_;
    my %done	= ();
    my $count   = @nodes;
    my @failed  = ();

    foreach my $node ( @nodes ) { $done{$node}  = 0; }

    print STDERR "Waiting for @nodes to finish reloading\n".`date` if $dbg;

    # Start a counter going, relative to the time we rebooted the first
    # node.
    my $waittime  = 0;
    my $minutes   = 0;

    while ($count) {
	# Wait first to make sure reboot is done, and so that we don't
	# wait one more time after everyone is up.
	sleep(5);
	foreach my $node ( @nodes ) {
	    if (! $done{$node}) {
		my ($query_result, @row);
		$query_result =
		    DBQueryFatal("SELECT op_mode FROM nodes ".
				 "where node_id='$node'");
		@row = $query_result->fetchrow_array();

		# We simply wait for the node to leave the reloading opmode
		if ($row[0] ne TBDB_NODEOPMODE_RELOAD) {
		    print STDERR "$node has left reloading mode\n".`date` if $dbg;
		    $count--;
		    $done{$node} = 1;
		    next;
		}
	
		# Soon we will have stated's timeouts take care of
		# rebooting once or twice if we get stuck during
		# reloading.
		$waittime = time - $startwait;
		if ($waittime > $maxwait) {
		    my $t = (int ($waittime / 60));
		    print "*** $node appears wedged; ".
			"its been $t minutes since it was rebooted.\n";

		    $count--;
		    $done{$node} = 1;
		    push(@failed, $node);
		    next;
		}
		if (int($waittime / 60) > $minutes) {
		    $minutes = int($waittime / 60);
		    print "Still waiting for $node to reload - ".
			"its been $minutes minute(s)\n";
		}
	    }
	}
    }
    return @failed;
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
417

418
419
# Setup a reload. Note that imageid is global.
sub SetupReload($) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
420
421
    my ($node) = @_;

422
423
424
425
426
    # Put it in the current_reloads table so that nodes can find out which
    # OS to load
    DBQueryFatal("replace into current_reloads ".
		 "(node_id, image_id) values ('$node', '$imageid')");

427
428
429
430
    system "$osselect -1 -m PXEFRISBEE $node" and
      die "*** Unable to select frisbee OS\n";
    system "$FRISBEELAUNCHER ".($dbg? "-d ":"")."$imageid" and
      die "*** Unable to launch frisbee daemon\n";
431
}
432

433
# Print a listing of imageids.
434
sub dolisting() {
435
    my($query_result);
436

437
438
439
440
    if ($UID && !TBAdmin($UID)) {
	my ($me) = getpwuid($UID);
	$query_result =
	    DBQueryFatal("select distinct i.* from images as i ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
441
			 "left join group_membership as g on g.pid=i.pid ".
442
			 "where g.uid='$me' or i.global ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
443
			 "order by i.pid,i.imageid");
444
    } else {
445
446
447
448
	$query_result =
	    DBQueryFatal("SELECT * FROM images order by imageid");
    }

449
450
451
    if ($query_result->numrows) {
	printf "%-12s %-20s %s\n", "Pid", "Imagename", "Description";
	printf "------------ -------------------- -------------------------\n";
452

453
454
455
456
457
458
459
460
461
	for ($i = 0; $i < $query_result->numrows; $i++) {
	    my %row  = $query_result->fetchhash();
	    my $id   = $row{'imageid'};
	    my $pid  = $row{'pid'};
	    my $name = $row{'imagename'};
	    my $desc = $row{'description'};

	    printf "%-12s %-20s %s\n", $pid, $name, $desc;
	}
462
463
    }
}