os_load.in 13.5 KB
Newer Older
1
#!/usr/bin/perl -wT
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
3
4
5
6
7
8

#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2002 University of Utah and the Flux Group.
# All rights reserved.
#

9
use English;
10
use Getopt::Std;
11

12
#
13
# XXX boss.emulab.net and users.emulab.net wired in. 
14
#     wd0 wired in. Should come from node_types table in DB
15
16
17
# 

#
18
19
20
# Load an image onto a disk. The image must be in the DB images table,
# which defines how/where to load, and what partitions are affected.
# The nodes and partitions tables are updated appropriately. 
21
# 
22
23
sub usage()
{
Leigh B. Stoller's avatar
Leigh B. Stoller committed
24
    print STDOUT
25
26
	"Usage: os_load [-s] [[-p <pid>] -i <imagename>] <node> [node ...]\n".
	"       os_load [-s] [[-p <pid>] -i <imagename>] -e pid,eid\n".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
27
	"       os_load -l\n".
28
29
	"Use -i to specify an image name. Use node default otherwise.\n".
	"Use -m to specify an image ID (internal name, TB admins only!).\n".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
30
	"Use -s to start reload, but do not wait for it to complete.\n".
31
32
	"Use -e to reload all the nodes in an experiment.\n" .
        "Use -l to get a list of images you are permitted to load.\n";
33
34
    exit(-1);
}
35
my  $optlist = "sldrni:e:p:m:b";
36
37
38
39
40

#
# Configure variables
#
my $TB		= "@prefix@";
41
my $TESTMODE    = @TESTMODE@;
42
my $TBOPS       = "@TBOPSEMAIL@";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
43
44
45
my $BOSSADDR	= "@BOSSNODE@";
my $USERADDR	= "@USERNODE@";

46
47
48
49
#
# Max number of simultaneous loads. Will be better with Frisbee.
# 
my $MAXLOADS	= 2;
50

Leigh B. Stoller's avatar
Leigh B. Stoller committed
51
52
53
54
55
56
#
# Max number of retries (per node) before its deemed fatal. This allows
# for the occasional pxeboot failure.
#
my $MAXRETRIES  = 1;

57
58
59
#
# Load the Testbed support stuff. 
#
60
61
62
use lib "@prefix@/lib";
use libdb;
use libtestbed;
63

64
my $NETDISKOSID = "NETDISK-STD";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
65
my $FRISBEEPATH = "$BOSSADDR:/tftpboot/pxeboot.frisbee";
66
my $FRISBEELAUNCHER = "$TB/sbin/frisbeelauncher";
67
my $nodereboot	= "$TB/bin/node_reboot";
68
my $schedreload	= "$TB/bin/sched_reload";
69
my $ping	= "/sbin/ping";
70
my $dbg		= 0;
71
my @row;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
72
my $usedefault  = 1;
73
my $imagename;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
74
my $imageid;
75
my $imagepid    = TB_OPSPID;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
76
my %imageid_row;
77
my @nodes       = ();
Leigh B. Stoller's avatar
Leigh B. Stoller committed
78
my %retries	= ();
79
my $mereuser    = 0;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
80
my $waitmode    = 1;
81
my $failures    = 0;
82
my $startwait   = 0;
83
my $type	= TB_DEFAULT_RELOADTYPE;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
84
my $cmdline     = "";
85
86
87
88
89
90
91
92

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

$| = 1; #Turn off line buffering on output

#
93
94
95
96
97
98
99
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
100
101
102
if (defined($options{"d"})) {
    $dbg++;
}
103
104
105
106
if (defined($options{"l"})) {
    dolisting();
    exit(0);
}
107
if (defined($options{"s"})) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
108
    $waitmode = 0;
109
}
110
if (defined($options{"r"})) {
111
112
113
114
115
116
117
    if (defined($options{"n"})) {
	die "Only one of -r or -n should be given\n";
    }
    $type = TB_RELOADTYPE_FRISBEE;
}
if (defined($options{"n"})) {
    $type = TB_RELOADTYPE_NETDISK;
118
}
119
120
121
122
123

if (defined($options{"i"}) && defined($options{"m"})) {
    usage();
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
124
if (defined($options{"i"})) {
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    $imagename  = $options{"i"};
    $usedefault = 0;

    if ($imagename =~ /^([-\w\.\+]+)$/) {
	$imagename = $1;
    }
    else {
	die("*** Bad data in $imagename.\n");
    }
    if (defined($options{"p"})) {
	$imagepid = $options{"p"};
	
	if ($imagepid =~ /^([-\w\.\+]+)$/) {
	    $imagepid = $1;
	}
	else {
	    die("*** Bad data in $imagepid.\n");
	}
    }
}
if (defined($options{"m"})) {
    $imageid = $options{"m"};
Leigh B. Stoller's avatar
Leigh B. Stoller committed
147
    $usedefault = 0;
148

149
    if ($imageid =~ /^([-\w\.\+]+)$/) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
150
151
152
	$imageid = $1;
    }
    else {
153
	die("*** Bad data in $imageid\n");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
154
    }
155
}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
156
157
158
159
160
161
162
163
164
165
166
if (defined($options{"e"})) {
    if (@ARGV) {
	usage();
    }
    my $pideid = $options{"e"};
    
    if ($pideid =~ /([-\w]*),([-\w]*)/) {
	if (! (@nodes = ExpNodes($1, $2))) {
	    die("*** $0:\n".
		"    There are no nodes in $1/$2!\n");
	}
167
168
    }
    else {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
169
170
	die("Invalid argument to -e option: $pideid\n");
	usage();
171
    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
}
else {
    if (! @ARGV) {
	usage();
    }
    #
    # Untaint nodes.
    #
    foreach my $node ( @ARGV ) {
	if ($node =~ /^([-\@\w]+)$/) {
	    $node = $1;
	}
	else {
	    die("*** Bad node name: $node.\n");
	}
187
    
Leigh B. Stoller's avatar
Leigh B. Stoller committed
188
189
	push(@nodes, $node);
    }
190
}
191

192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#
# VIRTNODE HACK: Virtual nodes are special. Do not reload!
#
my @temp = ();
foreach my $node ( @nodes ) {
    if (TBIsNodeVirtual($node)) {
	print "*** Skipping virtual node $node ...\n";
	next;
    }
    push(@temp, $node);
}
@nodes = @temp;
if (! @nodes) {
    print "No nodes to load. Exiting ...\n";
    exit(0);
}

209
210
211
212
213
#
# Figure out who called us. Root and admin types can do whatever they
# want. Normal users can only change nodes in experiments in their
# own projects.
#
214
215
if ($UID && !TBAdmin($UID)) {
    $mereuser = 1;
216

Leigh B. Stoller's avatar
Leigh B. Stoller committed
217
    if (! TBNodeAccessCheck($UID, TB_NODEACCESS_LOADIMAGE, @nodes)) {
218
219
220
221
	die("*** $0:\n".
            "    You do not have permission to load images on one (or more) ".
	    "nodes!\n");
    }
222
223
224
}

#
225
# Convert external name to internal (imageid), and check permission.
226
#
227
228
229
230
231
if (defined($imagename)) {
    if (! ($imageid = TBImageID($imagepid, $imagename))) {
	die("*** $0:\n".
	    "    No such image $imagename in project $imagepid!\n");
    }
232
}
233

234
235
236
237
if (defined($imageid) && $mereuser &&
    ! TBImageIDAccessCheck($UID, $imageid, TB_IMAGEID_READINFO)) {
    die("*** $0:\n".
	"    You do not have permission to load this image!\n");
238
239
240
}

#
241
# Loop for each node.
242
243
# 
foreach my $node (@nodes) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
244
245
246
247
248
249
    #
    # Oh, I suppose it would be nice to do this just once per imageid,
    # but that would be a pain. Not in the mood.
    #
    
    #
250
    # Get default imageid for this node.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
251
    #
252
253
254
255
256
    my $default_imageid;
    if (! ($default_imageid = DefaultImageID($node))) {
	die("*** $0:\n".
	    "    No default imageid is defined for $node!\n");
    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
257
    if ($usedefault) {
258
	$imageid = $default_imageid;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
259
    }
260
261

    print STDERR "Using $imageid for for $node\n" if $dbg;
262
    
Leigh B. Stoller's avatar
Leigh B. Stoller committed
263
264
265
266
    my $db_result =
	DBQueryFatal("select * from images where imageid='$imageid'");
    
    if ($db_result->numrows < 1) {
267
268
	die("*** $0:\n".
	    "    No such imageid $imageid is defined in the DB!\n");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
269
270
271
272
273
274
275
    }
    %imageid_row = $db_result->fetchhash();
    
    my $loadpart  = $imageid_row{'loadpart'};
    my $loadlen   = $imageid_row{'loadlength'};
    my $imagepath = $imageid_row{'path'};
    my $defosid   = $imageid_row{'default_osid'};
276
    my $shared    = $imageid_row{'shared'};
Leigh B. Stoller's avatar
Leigh B. Stoller committed
277

278
279
280
281
282
283
284
285
286
287
288
289
    #
    # Check for a few errors early!
    #
    if (!defined($imagepath)) {
	die("*** $0:\n".
	    "    There is no filename associated with $imageid!\n");
    }
    if (! -R $imagepath) {
	die("*** $0:\n".
	    "    $imagepath does not exists or cannot be read!\n");
    }

Leigh B. Stoller's avatar
Leigh B. Stoller committed
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
    #
    # 0 means load the entire disk.
    #
    my $diskpart = "";
    if ($loadpart) {
	$diskpart = "wd0:s${loadpart}";
    }
    else {
	$diskpart = "wd0";
    }

    #
    # For now, all testbed default images come from paper and all pid specific
    # images come from plastic:/proj.
    # 
305
    if (! $shared) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
306
	if (! ($imagepath =~ /^\/proj\//)) {
307
308
	    die("*** $0:\n".
		"    Your image must reside in /proj\n");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
309
310
311
312
313
314
315
316
	}
	$cmdline = "${USERADDR}:$imagepath $diskpart";    
    }
    else {
	$cmdline = "${BOSSADDR}:$imagepath $diskpart";
    }
    
    print STDOUT "Changing default OS for $node to $defosid\n";
317
318
319
    if (!$TESTMODE) {
      DBQueryFatal("update nodes set ".
		   "def_boot_osid='$defosid',def_boot_path='' ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
320
		   "where node_id='$node'");
321
    }
322

323
    # Put it in the current_reloads table so that nodes can find out which
Leigh B. Stoller's avatar
Leigh B. Stoller committed
324
    # OS to load
325
    DBQueryFatal("replace into current_reloads ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
326
		 "(node_id, image_id) values ('$node', '$imageid')");
327

328
    #
329
    # If loading an image (which is not the default) then
330
331
332
333
    # schedule a reload for it so that when the experiment is terminated
    # it will get a fresh default image before getting reallocated to
    # another experiment.
    #
334
    if ($imageid ne $default_imageid) {
335
336
337
338
339
340
	if (! TBSetSchedReload($node, $default_imageid)) {
	    print "*** $0:\n".
		  "    WARNING: Could not schedule default reload for $node!";
	}
    }

341
    #
342
343
344
    # Assign partition table entries for each partition in the image.
    # This is complicated by the fact that an image that covers only
    # part of the slices, should only change the partition table entries
345
    # for the subset of slices that are written to disk.
346
    #
Leigh B. Stoller's avatar
Leigh B. Stoller committed
347
    my $startpart = $loadpart == 0 ? 1 : $loadpart;
348
    my $endpart   = $startpart + $loadlen;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
349
350
	
    for ($i = $startpart; $i < $endpart; $i++) {
351
352
353
354
355
	my $partname = "part${i}_osid";
	
	if (defined($imageid_row{$partname})) {
	    my $osid = $imageid_row{$partname};

356
357
	    DBQueryFatal("replace into partitions ".
			 "(partition, osid, node_id) ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
358
			 "values('$i', '$osid', '$node')");
359
360
	}
	else {
361
	    DBQueryFatal("delete from partitions ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
362
			 "where node_id='$node' and partition='$i'");
363
364
	}
    }
365
    
Leigh B. Stoller's avatar
Leigh B. Stoller committed
366
    print STDOUT "Setting up reload for $node\n";
367
    if (!$TESTMODE) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
368
        SetupReload($node);
369
    }
370
}
371

372
#
373
# Exit if not doing about actual reload.
374
#
Leigh B. Stoller's avatar
Leigh B. Stoller committed
375
if ($TESTMODE) {
376
377
378
379
380
381
382
383
    print STDOUT "OS Reload (Setup/Testmode) Done!\n";
    exit 0;
}

#
# Fire off a mass reboot if not in waitmode.
#
if (! $waitmode) {
384
385
    system("$nodereboot @nodes");
    $failures = $? >> 8;
386
387
388

    print STDOUT "OS Reload (no waiting) Done!\n";
    exit $failures;
389
390
}

391
392
393
394
#
# Okay, in waitmode we do a couple at a time and wait for them to come
# back alive before proceeding to the next one.
#
Leigh B. Stoller's avatar
Leigh B. Stoller committed
395
396
397
398
399
400
401
# The retry vector is initialized to the number of retries we allow per
# node, afterwhich its a fatal error.
#
foreach my $node (@nodes) {
    $retries{$node} = $MAXRETRIES;
}

402
while (@nodes) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
403
404
    my @batch  = ();
    my @failed = ();
405
406
    my $i;

407
408
409
410
411
412
413
414
415
416
417
    #
    # If doing a netdisk load, only do a few at a time. For frisbee, we do
    # 'em all
    #
    if ($type eq TB_RELOADTYPE_NETDISK) {
	for ($i = 0; $i < $MAXLOADS && @nodes > 0; $i++) {
	    push(@batch, shift(@nodes));
	}
    } elsif ($type eq TB_RELOADTYPE_FRISBEE) {
	@batch = @nodes;
	@nodes = ();
418
419
    }
    
Leigh B. Stoller's avatar
Leigh B. Stoller committed
420
    print "Issuing reboot for @batch and then waiting ...\n";
421
422

    system("$nodereboot @batch");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
423
    if ($?) {
424
	print "Reboot failed for (some of) @batch. Quitting!\n";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
425
	exit ($? >> 8);
426
427
428
429
430
431
    }

    #
    # Now wait for them.
    #
    $startwait = time;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
432
433
434
435
436
437
438
439
440
441
442
443
444
    @failed = WaitTillReloadDone(@batch);
    while (@failed) {
	my $node = shift(@failed);

	if ($retries{$node}) {
	    print "*** Trying $node again (resetting/rebooting) ...\n";
	    push(@nodes, $node);

	    # Possible race with reboot?
	    SetupReload($node);

	    # Retry until count hits zero.
	    $retries{$node} -= 1;
445
	}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
	else {
	    print "*** $node failed too many times. Skipping!\n";
	    $failures++;
	}
    }
}

print "OS Reload Done! There were $failures failures!\n";
exit($failures);

#
# Wait for a reload to finish. We do this in a rather kludgey manner,
# by waiting for bootinfo to clear the DB state (next_boot_osid).
# 
sub WaitTillReloadDone {
    my (@nodes) = @_;
    my %done	= ();
    my $count   = @nodes;
    my @failed  = ();
465

Leigh B. Stoller's avatar
Leigh B. Stoller committed
466
    #
467
    # Seems like a long time to wait, but it ain't!
Leigh B. Stoller's avatar
Leigh B. Stoller committed
468
    # 
469
    my $maxwait = (60 * 15);
Leigh B. Stoller's avatar
Leigh B. Stoller committed
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543

    foreach my $node ( @nodes ) {
	$done{$node}  = 0;
    }

    print STDERR "Waiting for @nodes to finish reloading\n" if $dbg;

    #
    # Start a counter going, relative to the time we rebooted the first
    # node. 
    # 
    my $waittime  = 0;
    my $minutes   = 0;

    while ($count) {
	foreach my $node ( @nodes ) {
	    if (! $done{$node}) {
		my ($query_result, @row);

		if ($type eq TB_RELOADTYPE_FRISBEE) {
		    $query_result =
			DBQueryFatal("SELECT next_pxe_boot_path FROM nodes ".
				     "where node_id='$node'");
		}
		else {
		    $query_result =
			DBQueryFatal("SELECT next_boot_osid FROM nodes ".
				     "where node_id='$node'");
		}

		@row = $query_result->fetchrow_array();
		if (! $row[0]) {
		    print STDERR "$node alive and well\n" if $dbg;
		    $count--;
		    $done{$node} = 1;
		    next;
		}
	
		$waittime = time - $startwait;
		if ($waittime > $maxwait) {
		    my $t = (int ($waittime / 60));
		    
		    print "*** $node appears wedged; ".
			"its been $t minutes since it was rebooted.\n";

		    $count--;
		    $done{$node} = 1;
		    push(@failed, $node);
		    next;
		}
		if (int($waittime / 60) > $minutes) {
		    $minutes = int($waittime / 60);
		    print "Still waiting for $node - ".
			"its been $minutes minute(s)\n";
		}
	    }
	}
	sleep(5);
    }
    return @failed;
}

#
# Setup a reload. Note that type, imageid, and cmdline are global.
#
sub SetupReload($)
{
    my ($node) = @_;

    if ($type eq TB_RELOADTYPE_FRISBEE) {
	DBQueryFatal("update nodes set ".
		     "next_pxe_boot_path='$FRISBEEPATH'" .
		     "where node_id='$node'");
	
544
	system "$FRISBEELAUNCHER ".($dbg? "-d ":"")."$imageid" and
Leigh B. Stoller's avatar
Leigh B. Stoller committed
545
546
547
548
549
550
551
552
553
554
	    die "*** Unable to launch frisbee daemon\n";
    }
    elsif ($type eq TB_RELOADTYPE_NETDISK) {
	DBQueryFatal("update nodes set ".
		     "next_boot_osid='$NETDISKOSID',".
		     "next_boot_cmd_line='$cmdline' ".
		     "where node_id='$node'");
    }
    else {
	die "*** Unknown reload type ($type)\n";
555
556
    }
}
557

558
559
560
561
562
563
564
565
566
567
568
569
#
# Print a listing of imageids.
#
sub dolisting()
{
    my($query_result);
    
    if ($UID && !TBAdmin($UID)) {
	my ($me) = getpwuid($UID);
	
	$query_result =
	    DBQueryFatal("select distinct i.* from images as i ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
570
571
572
			 "left join group_membership as g on g.pid=i.pid ".
			 "where g.uid='$me' or i.shared ".
			 "order by i.pid,i.imageid");
573
574
575
576
577
578
    }
    else {
	$query_result =
	    DBQueryFatal("SELECT * FROM images order by imageid");
    }

579
580
581
582
583
584
585
586
587
588
589
590
591
    if ($query_result->numrows) {
	printf "%-12s %-20s %s\n", "Pid", "Imagename", "Description";
	printf "------------ -------------------- -------------------------\n";
    
	for ($i = 0; $i < $query_result->numrows; $i++) {
	    my %row  = $query_result->fetchhash();
	    my $id   = $row{'imageid'};
	    my $pid  = $row{'pid'};
	    my $name = $row{'imagename'};
	    my $desc = $row{'description'};

	    printf "%-12s %-20s %s\n", $pid, $name, $desc;
	}
592
593
    }
}