vnode_setup.in 6.52 KB
Newer Older
1
#!/usr/bin/perl -wT
Leigh B. Stoller's avatar
Leigh B. Stoller committed
2
3
4

#
# EMULAB-COPYRIGHT
5
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
Leigh B. Stoller's avatar
Leigh B. Stoller committed
6
7
# All rights reserved.
#
8
9
10
11
12
13
14
15
16
17
18
19
20
use English;
use Getopt::Std;

#
# Set up the vnode state on a virtual (multiplexed) node.
#
# XXX - This script should only be run from os_setup!
#
# The output is all jumbled together since the updates are issued in parallel.
# Might be a pain when debugging. 
# 
sub usage()
{
21
    print STDOUT "Usage: vnode_setup [-f] [-k] <pid> <eid> [node ...]\n";
22
23
24
    exit(-1);
}
my  $optlist = "fdk";
25
26
27
28
29
30
31
32
33
34

#
# We don't want to run this script unless its the real version.
# That is, it must be setuid root. 
#
if ($EUID != 0) {
    die("*** $0:\n".
	"    Must be root! Maybe its a development version?\n");
}

35
36
37
38
39
40
41
#
# Configure variables
#
my $TB		= "@prefix@";
my $TESTMODE    = @TESTMODE@;
my $TBOPS       = "@TBOPSEMAIL@";
my $TBLOGS      = "@TBLOGSEMAIL@";
42
my $CLIENT_BIN  = "@CLIENT_BINDIR@";
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

my $ssh		= "$TB/bin/sshtb -n";
my $debug       = 0;
my $force       = 0;
my $failed      = 0;
my $killmode    = 0;
my $mode        = "setup";
my $dbuid;

#
# Load the Testbed support stuff. 
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;

# un-taint path
$ENV{'PATH'} = '/bin:/usr/bin:/usr/local/bin';
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

# Turn off line buffering on output
$| = 1; 

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
%options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
74
if (@ARGV < 2) {
75
76
77
78
79
80
81
82
83
84
85
86
    usage();
}
if (defined($options{"f"})) {
    $force = 1;
}
if (defined($options{"d"})) {
    $debug = 1;
}
if (defined($options{"k"})) {
    $killmode = 1;
    $mode = "teardown";
}
87
88
my $pid   = shift(@ARGV);
my $eid   = shift(@ARGV);
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

#
# Untaint the arguments.
#
if ($pid =~ /^([-\@\w]+)$/) {
    $pid = $1;
}
else {
    die("*** Bad data in pid: $pid\n");
}	
if ($eid =~ /^([-\@\w]+)$/) {
    $eid = $1;
}
else {
    die("*** Bad data in eid: $eid\n");
}

#
# Verify permission to muck with this experiment.
#
Leigh B. Stoller's avatar
Leigh B. Stoller committed
109
110
111
112
if (!TBAdmin($UID) &&
    !TBExptAccessCheck($UID, $pid, $eid, TB_EXPT_DESTROY)) {
    die("*** $0:\n".
	"    You do not have permission to mess with $pid/$eid!\n");
113
114
115
}

#
116
# Get the list of nodes in this experiment.
117
118
119
# 
my @nodes = ExpNodes($pid, $eid);
if (! @nodes) {
120
121
    die("*** $0:\n".
	"    No allocated nodes in experiment $pid/$eid!\n");
122
}
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    
# Nodes on the command line. Operate only on this set. 
if (@ARGV) {
    my %fulllist = ();

    # Temporary hash list for searching.
    foreach my $node ( @nodes ) {
	$fulllist{$node} = 1;
    }
    @nodes = ();

    foreach my $node ( @ARGV ) {
	if ($node =~ /^([-\@\w]+)$/) {
	    $node = $1;

	    if (!defined($fulllist{$node})) {
		die("*** $0:\n".
		     "   Node $node is not allocated to $pid/$eid!\n");
	    }
	}
	else {
	    die("Bad node name: $node.");
	}
	push(@nodes, $node);
    }
}
my $exptstate = ExpState($pid, $eid);
150
151
152
153

# Just the vnodes mam.
foreach my $node (@nodes) {
    my $pnode;
154
    my $jailed;
155
    my $allocstate;
156
    
157
    if (! TBIsNodeVirtual($node, \$jailed)) {
158
159
	next;
    }
160
161
162
163
    if (! TBPhysNodeID($node, \$pnode)) {
	die("*** $0:\n".
	    "    No physical node for $node!\n");
    }
164

165
166
    TBGetNodeAllocState($node, \$allocstate);

167
168
169
    #
    # On remote nodes, or when forcemode is on, always do the deed.
    # Otherwise, look at experiment state.
170
171
    #
    if (!$force) {
172
173
174
175
	if ($exptstate eq EXPTSTATE_SWAPPING) {
	    #
	    # When swapping, local vnodes go down with the physnode.
	    #
176
177
178
179
180
181
182
183
	    if (! TBIsNodeRemote($node)) {
		print "$node will $mode with local node $pnode.\n";
		next;
	    }
	    elsif ($allocstate eq TBDB_ALLOCSTATE_DOWN) {
		print "$node never booted; skipping.\n";
		next;
	    }
Leigh B. Stoller's avatar
Leigh B. Stoller committed
184
	}
185
186
	elsif ($exptstate eq EXPTSTATE_ACTIVATING) {
	    #
187
188
189
190
191
192
193
194
195
196
197
	    # The allocstate determines if the vnode actually needs to
	    # be setup or torndown. Note that a failed experiment will
	    # cause a bunch of vnodes to be torndown, while in the
	    # ACTIVATING state. See os_setup and assign_wrapper; the
	    # idea is to avoid doing setup/teardown up vnodes on
	    # machines that are rebooting anyway, or that failed.
	    # Complicated by modify which could add/subtract a vnode on
	    # an existing machine, but not reboot the machine. Note that
	    # free (now unused) vnodes will land in RES_TEARDOWN. It is
	    # assumed that these booted okay, and need to be torndown,
	    # even though they are not RES_READY.
198
	    #
199
200
201
202
203
204
205
206
207
208
209
210
211
212
	    if (! TBIsNodeRemote($node)) {
		if ($killmode) {
		    if ($allocstate eq TBDB_ALLOCSTATE_DOWN) {
			print "$node never booted; skipping $mode.\n";
			next;
		    }
		}
		elsif ($allocstate eq TBDB_ALLOCSTATE_RES_READY()) {
		    print "$node will $mode with local node $pnode\n";
		    next;
		}
	    }
	    elsif ($killmode && $allocstate eq TBDB_ALLOCSTATE_DOWN) {
		print "$node never booted; skipping $mode.\n";
213
214
215
		next;
	    }
	}
Leigh B. Stoller's avatar
Leigh B. Stoller committed
216
217
218
219
220
221
222
223
224
    }

    #
    # When setting up a vnode, force its event state into SHUTDOWN since
    # no telling what its initial state is. 
    # 
    if (!$killmode) {
	TBSetNodeEventState($node, TBDB_NODESTATE_SHUTDOWN);
    }
225
226
227
228
229
230
231
232
233
234
235
236
    
    print STDOUT "Doing $mode of vnode $node on $pnode ...\n";

    #
    # Run an ssh command in a child process, protected by an alarm to
    # ensure that the ssh is not hung up forever if the machine is in
    # some funky state.
    # 
    my $syspid = fork();

    if ($syspid) {
	local $SIG{ALRM} = sub { kill("TERM", $syspid); };
237
	alarm 60;
238
239
240
241
242
243
	waitpid($syspid, 0);
	alarm 0;

	print STDERR "vnode $mode on $pnode returned $?.\n" if $debug;
    
	#
244
	# Look for setup failure, reported back through ssh.
245
	# 
246
247
248
249
	if ($?) {
	    my $exitstatus = $?;

	    if ($exitstatus == 256) {
250
		print STDERR "$node is not running sshd.\n" if $debug;
251
252
	    }
	    elsif ($exitstatus == 15) {
253
254
255
256
257
258
		print STDERR "$node is wedged.\n" if $debug;
	    }
	    
	    # Send mail to testbed-ops about it
	    SENDMAIL($TBOPS, "Virtual Node $node $mode failure",
	     "Virtual node $node $mode (on physical node $pnode) in pid/eid\n".
259
	     "$pid/$eid has failed! Exit status was $exitstatus.\n");
260
261
262
263
264
265
266
267
268
269
270
271

	    if (!$killmode) {
		die("*** $0:\n".
		    "    Virtual node $node setup failure!\n");
	    }
	    else {
		warn("*** $0:\n".
		    "    Virtual node $node teardown failure!\n");
	    }
	}
    }
    else {
272
273
274
	my $args = ($killmode ? "-k " : " ");
	$args   .= ($jailed   ? "-j " : " ");
	$args   .= "$node ";
275
276
277
278
	
	# Must change our real UID to root so that ssh will work.
	$UID = 0;
    
279
	exec("$ssh -host $pnode $CLIENT_BIN/vnodesetup $args");
280
281
282
283
284
285
286
287
288
289
	die("*** $0:\n".
	    "    exec failed!\n");
    }
}

#
# Only if all nodes setup okay.
# 
print STDOUT "Vnode Setup Done!\n";
exit 0;