Commit 4d5620c2 authored by Leigh Stoller's avatar Leigh Stoller

Switch to "dynamic" creation of certain virtual nodes (jail/pcvm and

vserver/pcplab). Rather then 10000s of fixed node entries in the DB,
create node entries on the fly as needed (assign_wrapper), and then
delete them when they are no longer used (nfree). When new nodes check
in, no longer create all those nodes table entries (utils/newnode.in
and tbsetup/plab/libplab.py.in).

I've added a new library: db/Node.pm which is something I started a
while back, and decided to commit, along with the support for creating
and deleting virtual nodes. CreateVnodes() creates a new set of nodes,
choosing non-conflicting names in the DB, and then immediately
reserves them to the pid/eid specified. DeleteVnodes takes a list of
vnodes and deletes them from nodes,reserved,etc. This library does a
few other things which I am going to be playing with, so you might
want to go read the comment at the top of the file. Feel free to speak
up. CreateVnodes() is from called assign_wrapper when a node type has
the "isdynamic" property. Otherwise does the existing avail/nalloc
stuff. DeleteVnodes() is called from nfree when the node type has the
isdynamic property.

I've added a script (sql/delvnodes.pl) to run after updating the DB
and software. All free pcvm and pcplab virtual nodes are deleted from
the DB; reserved ones will get deleted whenever their experiment ends.
I've noted all of this in doc/UPDATING, including setting the
isdynamic property on pcvm in the node_types table.

I've left tbsetup/plab/libplab.py.in to create a single pcplab node
for the management sliver (still called -20). We can worry about this
later.

All this for modelnet?
parent 38b6344f
......@@ -17,7 +17,7 @@ SBIN_SCRIPTS = avail inuse showgraph if2port backup webcontrol node_status \
dbcheck interswitch dbboot grabron stategraph newwanode \
idletimes idlemail setsitevar audit changeuid
LIBEXEC_SCRIPTS = webnodelog webnfree webnewwanode webidlemail xmlconvert
LIB_SCRIPTS = libdb.pm
LIB_SCRIPTS = libdb.pm Node.pm
# Stuff installed on plastic.
USERBINS = node_list readycount
......
This diff is collapsed.
......@@ -33,6 +33,7 @@ my $TESTMODE = @TESTMODE@;
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use Node;
my $consetup = "$TB/libexec/console_setup";
my $osselect = "$TB/bin/os_select";
......@@ -46,6 +47,7 @@ my $lockedeid = NFREELOCKED_EID();
my @nodes;
my @freed_nodes=();
my @dynanodes=();
my $error = 0;
$| = 1; # Turn off line buffering on output
......@@ -231,12 +233,14 @@ foreach my $n (@freed_nodes) {
# Find the default values for its node type.
my $result =
DBQueryFatal("select nt.osid,n.eventstate, " .
" nt.isvirtnode, nt.imageable, o.mustclean ".
" nt.isvirtnode, nt.imageable, o.mustclean, ".
" nt.isdynamic ".
" from nodes as n " .
"left join node_types as nt on n.type=nt.type " .
"left join os_info as o on o.osid=n.def_boot_osid ".
"where node_id='$n'");
my ($osid, $estate, $isvirt, $imageable, $clean) = $result->fetchrow_array();
my ($osid, $estate, $isvirt, $imageable, $clean, $isdynamic) =
$result->fetchrow_array();
# See if the OS it was running was marked as mustclean or not. Basically,
# this is an OSKit hack to avoid reloading disks that have not been
......@@ -251,12 +255,22 @@ foreach my $n (@freed_nodes) {
# If def_boot_osid set, then $clean is defined. Otherwise not set
# so default to cleaning node.
$mustclean = $clean;
}
}
#
# If the node is a dynamic virtual node, just save it for later.
# We will call into the Node library to delete it.
#
if ($isdynamic) {
push(@dynanodes, $n);
next;
}
# Clean up interfaces by clearing IPs and/or aliases.
if (! ($n =~ /sh\d+/)) {
# Its not a shark, so clean out all IPs except the control net.
DBQueryWarn("update interfaces set IP='',IPaliases=NULL,mask=NULL,rtabid='0',vnode_id=NULL " .
DBQueryWarn("update interfaces set IP='',IPaliases=NULL,mask=NULL,".
" rtabid='0',vnode_id=NULL " .
"where node_id='$n' and ".
" role='" . TBDB_IFACEROLE_EXPERIMENT() . "'")
|| $error++;
......@@ -364,6 +378,11 @@ foreach my $n (@freed_nodes) {
}
}
# Release dynamic nodes.
if (@dynanodes) {
Node::DeleteVnodes(@dynanodes);
}
######################################################################
# Step 3 - Set up console for freed nodes.
#
......
......@@ -6,6 +6,14 @@ This file is in the same format at the FreeBSD UPDATING file, whis is
to say, in reverse chronological order, with the date of the change
in YYYYMMDD format.
20040625:
After updating to revision 1.257 of sql/database-create.txt,
run sql/devlnodes.pl to clear out all of the non reserved
pcvm nodes. Be sure to do a current install of the software, and
then:
update node_types set isdynamic=1 where type='pcvm';
20040615:
If you update the ports on your boss node, you'll need to take
into accout that FreeBSD is in the process of changing how its
......
#!/usr/bin/perl -w
use English;
use Errno;
use lib "/usr/testbed/lib";
use libdb;
use libtestbed;
DBQueryFatal("lock tables reserved write, nodes write");
my $query_result =
DBQueryFatal("select nodes.node_id from nodes ".
"left join reserved on nodes.node_id=reserved.node_id ".
"where reserved.node_id is null and ".
" (nodes.type='pcvm' or nodes.type='pcplab')");
while (my ($vnodeid) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from reserved where node_id='$vnodeid'");
DBQueryWarn("delete from nodes where node_id='$vnodeid'");
}
DBQueryFatal("unlock tables");
$query_result->dataseek(0);
while (my ($vnodeid) = $query_result->fetchrow_array()) {
DBQueryWarn("delete from node_hostkeys where node_id='$vnodeid'");
DBQueryWarn("delete from node_status where node_id='$vnodeid'");
DBQueryWarn("delete from node_rusage where node_id='$vnodeid'");
}
......@@ -49,7 +49,7 @@ sub usage ()
exit($WRAPPER_FAILED);
}
my $optlist = "vutnfp";
my $verbose = 0;
my $verbose = 1;
my $fixmode = 0;
my $updating = 0;
my $toponly = 0;
......@@ -75,6 +75,7 @@ $| = 1;
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use Node;
#
# assign_wrapper Settings
......@@ -1215,51 +1216,75 @@ foreach my $pnode (keys(%virtnodes)) {
#
if ($numvs) {
#
# Run avail to get the list of virtnodes on the phys node. We
# already know there are enough, since assign knows that.
#
printdb "Asking avail for $numvs for vnodes: @vlist on $pnode\n";
# All vnodes on pnode are dynamic if the first one is.
# We also assume that we do not mix vnode types on a pnode; bad.
#
if (virtnodeisdynamic($vlist[0])) {
# Always use the base type ... node type system sucks.
my $vtype = nodetypetype(virtnodetype($vlist[0]));
#
# Call into library. Be sure to pass impotent mode along.
#
if (Node::CreateVnodes(\@plist,
{"pid" => "$pid", "eid" => "$eid",
"count" => $numvs,
"vtype" => $vtype,
"nodeid" => $pnode,
"verbose" => $verbose,
"impotent" => $impotent}) < 0) {
fatal("Could not allocate vnodes on $pnode");
}
}
else {
#
# Run avail to get the list of virtnodes on the phys node. We
# already know there are enough, since assign knows that.
#
printdb "Asking avail for $numvs for vnodes: @vlist on $pnode\n";
open(AVAIL,"$TBROOT/sbin/avail virtonly=$pnode rand limit=$numvs |")
or fatal("avail failed");
open(AVAIL,"$TBROOT/sbin/avail virtonly=$pnode rand limit=$numvs|")
or fatal("avail failed");
while (<AVAIL>) {
next
if (! /^\|/);
next
if (/node_id/);
while (<AVAIL>) {
next
if (! /^\|/);
next
if (/node_id/);
if ($_ =~ /^\|([-a-zA-Z0-9]+)\s*\|(\w+)\s*\|(\w+)\s*\|$/) {
push(@plist, $1);
}
else {
fatal("Bad line from avail: $_");
if ($_ =~ /^\|([-a-zA-Z0-9]+)\s*\|(\w+)\s*\|(\w+)\s*\|$/) {
push(@plist, $1);
}
else {
fatal("Bad line from avail: $_");
}
}
}
close(AVAIL);
close(AVAIL);
# Sanity check.
if (scalar(@vlist) != scalar(@plist)) {
printdb "avail gave " . scalar(@plist) .
" vnodes: @plist on $pnode\n";
fatal("Could not map some virtual nodes on $pnode");
}
# Sanity check.
if (scalar(@vlist) != scalar(@plist)) {
printdb "avail gave " . scalar(@plist) .
" vnodes: @plist on $pnode\n";
fatal("Could not map some virtual nodes on $pnode");
}
#
# Try to allocate. Note, if this fails we are done for. Okay for now
# since it is never the case that it should fail!
#
if ($impotent) {
print "Selected for $pnode: @plist\n";
print "Skipping physical reservation, as directed.\n";
}
else {
print "Reserving on $pnode: @plist ...\n";
TBDebugTimeStamp("nalloc vnodes on $pnode started");
if (system("nalloc $pid $eid @plist")) {
fatal("Failed to reserve @plist (on $pnode)");
#
# Try to allocate. Note, if this fails we are done
# for. Okay for now since it is never the case that it
# should fail!
#
if ($impotent) {
print "Selected for $pnode: @plist\n";
print "Skipping physical reservation, as directed.\n";
}
else {
print "Reserving on $pnode: @plist ...\n";
TBDebugTimeStamp("nalloc vnodes on $pnode started");
if (system("nalloc $pid $eid @plist")) {
fatal("Failed to reserve @plist (on $pnode)");
}
TBDebugTimeStamp("nalloc vnodes on $pnode finished");
}
TBDebugTimeStamp("nalloc vnodes on $pnode finished");
}
}
......@@ -3070,12 +3095,14 @@ sub interfacespeedmbps($$) {
return $interface_capabilities{$_[0]}->{$_[1] . "_defspeed"} / 1000.0;
}
sub nodetypeistype($) { return exists($node_types{$_[0]}); }
sub nodetypetype($) { return $node_types{$_[0]}->{"type"}; }
sub nodetypeclass($) { return $node_types{$_[0]}->{"class"}; }
sub nodedelayosid($) { return $node_types{$_[0]}->{"delay_osid"}; }
sub nodejailosid($) { return $node_types{$_[0]}->{"jail_osid"}; }
sub nodedefaultosid($) { return $node_types{$_[0]}->{"osid"}; }
sub nodetypeisremote($) { return $node_types{$_[0]}->{"isremotenode"}; }
sub nodetypeisvirt($) { return $node_types{$_[0]}->{"isvirtnode"}; }
sub nodetypeisdynamic($){ return $node_types{$_[0]}->{"isdynamic"}; }
sub nodetypeissub($) { return $node_types{$_[0]}->{"issubnode"}; }
sub nodetypeisplab($) { return $node_types{$_[0]}->{"isplabdslice"}; }
sub nodetypeissim($) { return $node_types{$_[0]}->{"issimnode"}; }
......@@ -3164,6 +3191,7 @@ sub LoadVirtNodes()
my $issub = 0;
my $isplab = 0;
my $issim = 0;
my $isdyn = 0; # Only virtnodes are dynamic.
# If we have a real type or auxtype ...
if (nodetypeistype($type)) {
......@@ -3172,6 +3200,7 @@ sub LoadVirtNodes()
$issub = nodetypeissub($type);
$isplab = nodetypeisplab($type);
$issim = nodetypeissim($type);
$isdyn = nodetypeisdynamic($type);
}
elsif (virttypeisvtype($type)) {
#
......@@ -3185,6 +3214,7 @@ sub LoadVirtNodes()
$issub = nodetypeissub($vtype);
$isplab = nodetypeisplab($vtype);
$issim = nodetypeissim($vtype);
$isdyn = nodetypeisdynamic($vtype);
}
else {
fatal("Improper type $type for node $vname!");
......@@ -3197,7 +3227,8 @@ sub LoadVirtNodes()
$rowref->{"__isvirtnode"} = $isvirt;
$rowref->{"__issubnode"} = $issub;
$rowref->{"__isplabnode"} = $isplab;
$rowref->{"__issimnode"} = $issim;
$rowref->{"__issimnode"} = $issim;
$rowref->{"__isdynamic"} = $isdyn;
$rowref->{"__usewanassign"} = 0;
# The mapped osname to actual osid.
$rowref->{"__osid"} = undef;
......@@ -3357,6 +3388,9 @@ sub virtnodeissubnode($) {
sub virtnodeissim($) {
return virtnodeisvnode($_[0]) && $virt_nodes{$_[0]}->{"__issimnode"};
}
sub virtnodeisdynamic($) {
return virtnodeisvnode($_[0]) && $virt_nodes{$_[0]}->{"__isdynamic"};
}
sub virtnodeisplabnode($) {
return virtnodeisvnode($_[0]) && $virt_nodes{$_[0]}->{"__isplabnode"};
}
......@@ -4266,9 +4300,12 @@ sub LoadPhysNode($)
"left join node_types as nt on nt.type=n.type ".
"where n.node_id='$pnode'");
fatal("No such pnode $pnode in nodes table!")
if (!$query_result->num_rows);
if (!$query_result->num_rows) {
return
if ($impotent);
fatal("No such pnode $pnode in nodes table!")
}
my $rowref = $query_result->fetchrow_hashref();
#
......@@ -4285,7 +4322,7 @@ sub LoadPhysNode($)
# Routing table id for different vnodes on pnode
$rowref->{"__rtabid"} = 0;
}
return $phys_nodes{$pnode};
return;
}
#
......
......@@ -83,7 +83,6 @@ LOCAL_PLAB_LINKTYPE = "pcplabinet2"
# '128.208.4.199', '128.2.198.199', '155.98.35.2',
# '155.98.35.3')
ALLOWED_NODES = ()
NUMVNODES = 20
PLABNODE = "@prefix@/sbin/plabnode"
SSH = "@prefix@/bin/sshtb"
......@@ -505,38 +504,39 @@ class Plab:
vnodetype = "pcplab"
vnodeid = ""
for n in range(NUMVNODES):
vprio = (priority * 100) + (n+1)
sshdport = 38000+(n+1)
vnodeid = "%s-%d" % (nodeid, n+1)
if verbose:
print "Creating vnode %s, priority %d" % (vnodeid, vprio)
pass
DBQueryFatal("insert into nodes"
" (node_id, type, phys_nodeid, role, priority,"
" op_mode, def_boot_osid, update_accounts,"
" allocstate, allocstate_timestamp,"
" eventstate, state_timestamp, sshdport)"
" values (%s, %s, %s, %s, %s,"
" %s, %s, %s, %s, now(), %s, now(), %s)",
(vnodeid, vnodetype, nodeid, 'virtnode', vprio,
'PCVM', defosid, 1,
'FREE_CLEAN',
'SHUTDOWN', sshdport))
DBQueryFatal("insert into node_hostkeys"
" (node_id)"
" values (%s)",
(vnodeid))
DBQueryFatal("insert into node_status"
" (node_id, status, status_timestamp)"
" values (%s, %s, now())",
(vnodeid, 'up'))
# Create a single reserved plab vnode for the managment sliver.
# XXX I left it as "20" cause of all the existing ones.
n = 20
vprio = (priority * 100) + (n+1)
sshdport = 38000+(n+1)
vnodeid = "%s-%d" % (nodeid, n+1)
if verbose:
print "Creating vnode %s, priority %d" % (vnodeid, vprio)
pass
DBQueryFatal("insert into nodes"
" (node_id, type, phys_nodeid, role, priority,"
" op_mode, def_boot_osid, update_accounts,"
" allocstate, allocstate_timestamp,"
" eventstate, state_timestamp, sshdport)"
" values (%s, %s, %s, %s, %s,"
" %s, %s, %s, %s, now(), %s, now(), %s)",
(vnodeid, vnodetype, nodeid, 'virtnode', vprio,
'PCVM', defosid, 1,
'FREE_CLEAN',
'SHUTDOWN', sshdport))
DBQueryFatal("insert into node_hostkeys"
" (node_id)"
" values (%s)",
(vnodeid))
DBQueryFatal("insert into node_status"
" (node_id, status, status_timestamp)"
" values (%s, %s, now())",
(vnodeid, 'up'))
# Put the last vnode created into the special monitoring expt.
DBQueryFatal("insert into reserved"
" (node_id, pid, eid, rsrv_time, vname)"
......
......@@ -315,22 +315,6 @@ NODE: foreach my $node_id (@node_ids) {
"type='pcvm', count=$DEFAULT_PCVM_COUNT");
DBQueryFatal("INSERT INTO node_auxtypes set node_id='$node_id', " .
"type='$vtype', count=$DEFAULT_PCVM_COUNT");
for (my $i = 1; $i <= $NUM_VNODES; $i++) {
my $vpriority = 10000000 + ($nodenum * 1000) + $i;
my $nodename = $node_id;
if (!($nodename =~ s/pc/pcvm/)) {
$nodename = "$nodename-vm";
}
$nodename .= "-$i";
my $jailip = "${IPBASE}.${nodenum}.${i}";
DBQueryFatal("INSERT INTO nodes SET node_id='$nodename', " .
"type='$type', phys_nodeid='$node_id', role='virtnode', " .
"priority='$vpriority', op_mode='PCVM', " .
"eventstate='SHUTDOWN', " .
"def_boot_osid='emulab-ops-FBSD-JAIL', " .
"update_accounts=1, jailflag=1, jailip='$jailip'");
}
}
while (my ($card, $aref) = each %interfaces) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment