frisbeelauncher.in 9.96 KB
Newer Older
1
#!/usr/bin/perl -wT
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2002, 2004 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
5
6
# All rights reserved.
#
Robert Ricci's avatar
Robert Ricci committed
7
8
9
use Getopt::Std;
use POSIX 'setsid'; # For &daemonize
use Sys::Syslog;
10
use English;
Robert Ricci's avatar
Robert Ricci committed
11

12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This also kills a running frisbee.
#
sub usage()
{
    print "Usage: $0 [-d] [-k] <imageid>\n";
    print "-k:	Kill running frisbee.\n";
    print "-d:	Print debugging output.\n";
    exit(1);
}
my $optlist  = "dk";
my $debug    = 0;
my $killmode = 0;

Robert Ricci's avatar
Robert Ricci committed
26
27
# Configure variables
my $TB		= "@prefix@";
Leigh B. Stoller's avatar
Leigh B. Stoller committed
28
29
my $TBOPS	= "@TBOPSEMAIL@";

30
31
32
33
34
35
#
# Untaint the path
# 
$ENV{'PATH'} = "/bin:/usr/bin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

Leigh B. Stoller's avatar
Leigh B. Stoller committed
36
37
38
39
#
# Turn off line buffering on output
#
$| = 1;
Robert Ricci's avatar
Robert Ricci committed
40
41
42

use lib "@prefix@/lib";
use libdb;
43
use libtestbed;
Robert Ricci's avatar
Robert Ricci committed
44
45
46
47

# Defines
my $FRISBEED	= "$TB/sbin/frisbeed";
my $BASEADDR	= "234.5.6";
48
my $BASEPORT	= "3564";
Robert Ricci's avatar
Robert Ricci committed
49
50
my $LOGFILE	= "$TB/log/frisbeelauncher";

51
52
my $STD_BW	= 72000000; 	# 71.6Mb/sec w/1000HZ kernel
my $USR_BW	= 54000000;	# 53.7Mb/sec w/1000HZ kernel
53

54
55
56
57
58
59
60
61
62
63
64
65
66
#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"k"})) {
    $killmode = 1;
Robert Ricci's avatar
Robert Ricci committed
67
68
69
}
$imageid = shift @ARGV;

70
71
72
#
# Untaint the argument.
#
73
if ($imageid =~ /^([-\@\w\+.]+)$/) {
74
75
76
77
    $imageid = $1;
}
else {
    die("Invalid image '$imageid' contains illegal characters.\n");
78
79
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
80
81
82
# Grab the filename to give to frisbee
my $filename = &get_filename($imageid);

83
#
84
# Make sure that the user has sufficient permissions.
85
#
86
87
88
89
90
if (!TBImageIDAccessCheck($UID, $imageid,
			  ($killmode ? TB_IMAGEID_DESTROY :
			   TB_IMAGEID_READINFO))) {
    die("*** $0:\n".
	"    Not enough permission!\n");
91
92
}

93
94
95
if (!$killmode && ! -R $filename) {
    die("*** You do not have permission to read the image file for\n".
	"imageid $imageid: $filename\n");
Leigh B. Stoller's avatar
Leigh B. Stoller committed
96
97
98
}

#
99
100
# Need to lock the tables here, so we can lock out anyone else from
# messing with the image (and so we can pick an address atomically). 
Leigh B. Stoller's avatar
Leigh B. Stoller committed
101
102
103
#
&lock_tables;

Robert Ricci's avatar
Robert Ricci committed
104
# Try to discover if some other process is handling this address
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
($address, $pid) = get_address($imageid);

if ($killmode) {
    #
    # Killmode. We do some sanity checking cause there is a small race 
    # inherent in trying to use the pid via the DB without locking (below).
    #
    # No point in leaving table locked; not going to modify it.
    #
    unlock_tables();

    # Nothing running.
    exit(0)
	if (!$address && !$pid);

    if (!$address && $pid) {
	#
	# This makes no sense. Just send email.
	#
	my $mesg = "Inconsistent DB state. PID ($pid) but no load address!";
	
	SENDMAIL($TBOPS,
		 "Frisbee Killer Failed!",
		 "Imageid: $imageid\n".
		 $mesg);
	
	die("*** $0:\n".
	    "    $mesg\n");
    }
    if ($address && !$pid) {
	#
	# Okay, minor problem. It is possible we caught the launcher between
	# setting the address and setting the pid. Wait a moment, and then
	# try again. If still no pid, bail.
	#
	sleep(1);
	($address, $pid) = get_address($imageid);

	# Okay, situation resolved itself; other frisbeelauncher bailed.
	exit(0)
	    if (!$address && !$pid);

	#
	# Still inconsistent so bail.
	# 
	if ($address && !$pid) {
	    my $mesg = "Inconsistent DB state. Load address but no PID!";
	
	    SENDMAIL($TBOPS,
		     "Frisbee Killer Failed!",
		     "Imageid: $imageid\n".
		     $mesg);
	
	    die("*** $0:\n".
		"    $mesg\n");
	}
    }

    #
    # Okay, address and pid. We could clear the pid from the DB,
    # preventing another killer from thinking it is running, but not
    # much point since it is not likely to happen. If it turns out to
    # be a problem we can change the way this works.
    #
    unlock_tables();
    if (! kill('TERM', $pid)) {
	SENDMAIL($TBOPS,
		 "Frisbee Killer Failed!",
		 "Failed to stop frisbee daemon for $imageid\n".
		 "Could not kill(TERM) process $pid: $? $!");

	die("*** $0:\n".
	    "    Failed to stop frisbee daemon for $imageid!\n");
    }
    exit(0);
}
Robert Ricci's avatar
Robert Ricci committed
181

Leigh B. Stoller's avatar
Leigh B. Stoller committed
182
183
184
if ($address && &keepbusy($imageid)) {
        &unlock_tables;
	&debug("A server ($address) is already running for image $imageid\n");
Robert Ricci's avatar
Robert Ricci committed
185
186
187
	exit (0);
}

188
189
190
191
192
193
194
195
196
197
198
199
200
# This would be inconsistent.
if ($pid) {
    my $mesg = "Inconsistent DB state. No load address but PID ($pid) set!";
	
    SENDMAIL($TBOPS,
	     "Frisbee Startup Failed!",
	     "Imageid: $imageid\n".
	     $mesg);
	
    die("*** $0:\n".
	"    $mesg\n");
}

Robert Ricci's avatar
Robert Ricci committed
201
202
203
204
205
206
207
208
209
210
211
212
213
# Pick an address: Die if unsucessful, set address and unlock if sucessful
$address = &pick_address;
&debug("Picked address $address\n");

if (!$address) {
	&unlock_tables;
	die "Unable to find a free address to send on\n";
}

&set_address($imageid,$address);
&unlock_tables;

# Run in the background
214
215
216
if (TBBackGround($LOGFILE)) {
    exit(0);
}
Robert Ricci's avatar
Robert Ricci committed
217

218
219
220
221
222
223
224
# Set up a signal handler that will clean up in case we get killed
$SIG{HUP} = $SIG{INT} = $SIG{TERM} = \&cleanup;

# Set our pid. This happens outside the lock which could lead to races,
# but that is unlikely. Look for it above though.
set_pid($imageid, $PID);

225
226
227
228
229
230
231
#
# Drop root permissions, if we have them
#
if ($EUID == 0) {
	$EUID = $UID;
}

232
233
234
235
236
237
238
239
240
241
#
# Select the appropriate bandwidth
#
my $BW;
if ($filename =~ /^$TB\/images/) {
    $BW = $STD_BW;
} else {
    $BW = $USR_BW;
}

Robert Ricci's avatar
Robert Ricci committed
242
243
# Now, we actually launch Frisbee
while (1) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
244
245
246
247
248
249
250
251
252
253
254
        #
        # Each time the server exits, test the busy bit to see if
	# it should keep going. This has to be done with tables locked
	# since another caller is going to bump it.
	#
	&lock_tables();
	if (! &testbusy($imageid)) {
		last;
	}
	&unlock_tables();
	
Robert Ricci's avatar
Robert Ricci committed
255
256
	if ($child_pid = fork()) {
		# Wait for child to exit
Leigh B. Stoller's avatar
Leigh B. Stoller committed
257
258
259
		waitpid($child_pid, 0);

		if ($?) {
260
			SENDMAIL($TBOPS, "Frisbeed Failed!",
Leigh B. Stoller's avatar
Leigh B. Stoller committed
261
262
263
264
265
266
267
268
269
				 "Imageid: $imageid\n".
				 "Address: $address\n\n".
				 "Process $child_pid exited with value $?.\n".
				 "Please look at the syslog for frisbeed!\n\n".
				 "NOTE: Another frisbeed will not start!\n");
			#
			# Dump early. This will leave the address in
			# in the DB, so that another one will not start
			# until the matter is resolved by someone. 
270
271
			#
			set_pid($imageid, 0);
Leigh B. Stoller's avatar
Leigh B. Stoller committed
272
			exit(1);
Robert Ricci's avatar
Robert Ricci committed
273
		}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
274
275
	}
	else {
Robert Ricci's avatar
Robert Ricci committed
276
		# Child branch
Leigh B. Stoller's avatar
Leigh B. Stoller committed
277
278
279
280
281
282
		# The database format for address is host:port - however,
		# we need to give them as seperate arguments to frisbeed.
		if ($address =~ /(.*):(.*)/) {
			my $addr = $1;
			my $port = $2;

283
			if (!exec("$FRISBEED -W $BW -m $addr -p $port $filename")) {
Leigh B. Stoller's avatar
Leigh B. Stoller committed
284
285
				die("$$: Unable to exec $FRISBEED\n");
			}
Robert Ricci's avatar
Robert Ricci committed
286
		}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
287
		die("$$: Bad address format: $address.\n");
Robert Ricci's avatar
Robert Ricci committed
288
289
290
291
	}
}

&clear_address;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
292
&unlock_tables();
Robert Ricci's avatar
Robert Ricci committed
293
294
295
296
297
298
299
300
301
exit(0);

######################################################################
# Subroutines
######################################################################

# Only print if -d option was given. Also add $$ on the beginning of the
# string, to ease debugging
sub debug {
302
	if ($debug) { print "$$: ", @_ };
Robert Ricci's avatar
Robert Ricci committed
303
304
305
306
307
308
}

# Grab the address for the passed-in imageid
sub get_address {
	my ($imageid) = @_;

Leigh B. Stoller's avatar
Leigh B. Stoller committed
309
	my $sth =
310
	    DBQueryFatal("SELECT load_address,frisbee_pid ".
Leigh B. Stoller's avatar
Leigh B. Stoller committed
311
			 "FROM images WHERE imageid='$imageid'");
Robert Ricci's avatar
Robert Ricci committed
312
313
314
315
316
317

	my @row = $sth->fetchrow;
	if (!@row) {
		die "No such imageid: $imageid\n";
	}

318
	return ($row[0], $row[1]);
Robert Ricci's avatar
Robert Ricci committed
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
}

# Grab the filename for the passed-in imageid
sub get_filename {
	my ($imageid) = @_;
	my $image_query = "SELECT path FROM images WHERE " .
		"imageid='$imageid'";

	my $sth = DBQueryFatal($image_query);

	my @row = $sth->fetchrow;
	if (!@row) {
		die "No such imageid: $imageid\n";
	}

	return $row[0];
}


# Lock the tables used in this script - waits indefinitely until it
# succeeds
sub lock_tables {
	while (1) {
		&debug("Locking tables\n");
		my $sth = DBQuery("LOCK TABLES images WRITE"); 
		if (!$sth) {
			print "DB Error locking tables. Waiting a bit ...\n";
			sleep(10);
		} else {
			last;
		}
	}
}

# Unlock the tables used in this script
sub unlock_tables {
	&debug("Unlocking tables\n");
	DBQueryFatal("UNLOCK TABLES"); 
}

# Pick out an address to use
sub pick_address {
	my $address_query = "SELECT load_address FROM images WHERE " .
		"load_address IS NOT NULL and load_address != ''";
	my $sth = DBQueryFatal($address_query);

	my %used_addrs = (); # Loading addresses already taken
	while (@row = $sth->fetchrow) {
367
368
369
370
		$row[0] =~ /^$BASEADDR\.(\d+):(\d+)$/;
		# $1 is the address, $2 the port number
		if ($1 && $2) {
			$used_addrs{$1} = $2;
Robert Ricci's avatar
Robert Ricci committed
371
372
373
374
375
376
		}
	}

	my $address;
	for (my $i = 1; $i < 255; $i++) {
		if (!$used_addrs{$i}) {
377
378
			my $port = $BASEPORT + ($i - 1);
			$address = "${BASEADDR}.${i}:${port}";
Robert Ricci's avatar
Robert Ricci committed
379
380
381
382
383
384
385
386
387
388
			last;
		}
	}

	return $address;
}

# Pass in an imageid, and an address
sub set_address {
	my ($imageid,$address) = @_;
Leigh B. Stoller's avatar
Leigh B. Stoller committed
389
390
391
392
393

	DBQueryFatal("UPDATE images SET load_address='$address',load_busy=1 " .
		     "WHERE imageid='$imageid'");
}

394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
# Pass in an imageid and a pid.
sub set_pid {
	my ($imageid,$pid) = @_;

	DBQueryFatal("UPDATE images SET frisbee_pid=$pid " .
		     "WHERE imageid='$imageid'");
}

# Pass in an imageid and a pid.
sub clear_pid {
	my ($imageid) = @_;

	DBQueryFatal("UPDATE images SET frisbee_pid=0 " .
		     "WHERE imageid='$imageid'");
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
# Bump the busy indicator to keep the frisbeed going.
sub keepbusy($imageid) {
	my ($imageid) = @_;

	DBQueryFatal("UPDATE images SET load_busy=GREATEST(load_busy,1) " .
		     "WHERE imageid='$imageid'");

	return 1;
}

# Test the busy indicator, and set to zero.
sub testbusy($imageid) {
	my ($imageid) = @_;

	my $query_result =
	    DBQueryFatal("select load_busy from images ".
			 "WHERE imageid='$imageid'");

	my @row = $query_result->fetchrow;
	if (!@row) {
	        return 0;
	}
	
	if ($row[0]) {
	    DBQueryFatal("UPDATE images SET load_busy=0 ".
			 "WHERE imageid='$imageid'");
	}
	return $row[0];
Robert Ricci's avatar
Robert Ricci committed
438
439
440
441
442
443
444
445
446
447
448
449
450
}

# Kill off our child process, if started, and clear out registered address
# Also, die off
sub cleanup {
	print STDERR "$$: Killed, cleaning up\n";
	if ($child_pid) {
		kill 15, $child_pid;
	}
	&clear_address;
	exit(1);
}

Leigh B. Stoller's avatar
Leigh B. Stoller committed
451
# Clear out the address (and pid) registered to this process
Robert Ricci's avatar
Robert Ricci committed
452
sub clear_address {
453
	&debug("Clearing out registered load_address and pid\n");
Robert Ricci's avatar
Robert Ricci committed
454
	# Now, clear out the load_address we had set up
455
456
457
	DBQueryFatal("update images set ".
		     "load_address='',load_busy=0,frisbee_pid=0 ".
		     "where imageid='$imageid'");
Robert Ricci's avatar
Robert Ricci committed
458
}