Commit 11752432 authored by Leigh B Stoller's avatar Leigh B Stoller
Browse files

Add physical memory accounting for openvz/xen nodes. The total

amount a physical has is stored in the node types table, and the
per-vm memory requirement is stored in the nodes table. ptopgen
adds up usage, and subtracts from the total for the ptop file.
The vtop number comes from a virt_node_attribute table, and we
pass this through to the client side. Note that this is less
important for openvz, more so for XEN.

In the NS file:

	tb-set-node-memory-size $node 1024

Number is in MBs. The mapper defaults this to 128 for openvz and 256
for xen. Maximum is hardwired to 256 and 512 respectively. Need to
think about a good way to configure this in.
parent cb9d504b
......@@ -1546,6 +1546,9 @@ sub rebootable($;$) {
sub bios_waittime($;$) {
return NodeTypeInfo($_[0])->bios_waittime($_[1]);
}
sub memory($;$) {
return NodeTypeInfo($_[0])->memory($_[1]);
}
#
# Perform some updates ...
......@@ -1621,7 +1624,7 @@ sub ClearBootAttributes($)
"update_accounts=0,ipport_next=ipport_low,rtabid=0, ".
"sfshostid=NULL,allocstate='$allocFreeState',boot_errno=0, ".
"destination_x=NULL,destination_y=NULL, ".
"destination_orientation=NULL ".
"destination_orientation=NULL,reserved_memory=0 ".
"where node_id='$node_id'")
or return -1;
......@@ -2092,6 +2095,14 @@ sub CreateVnodes($$$)
return -1;
}
# Need this below.
my $total_memory = $node->memory();
my $node_attributes = $node->GetNodeAttributes();
if (defined($node_attributes) &&
exists($node_attributes->{"physical_ram"})) {
$total_memory = $node_attributes->{"physical_ram"};
}
#
# Assign however many we are told to (typically by assign). Locally
# this is not a problem since nodes are not shared; assign always
......@@ -2176,6 +2187,44 @@ sub CreateVnodes($$$)
if ($opmode eq "ALWAYSUP");
my $allocstate = TBDB_ALLOCSTATE_FREE_CLEAN();
#
# Check memory constraints before we create anything.
#
my $reserved_memory = ();
if (defined($total_memory)) {
foreach my $ref (@tocreate) {
my ($n, $vname) = @{ $ref };
my $vm_memsize;
#
# Look to see if the container can actually get the memory it
# needs. This really only matters on a shared node. On a
# dedicated node, assign will never violate this, but on a
# shared node it could happen if two swapins are running
# at the same time. Like bandwidth.
#
$experiment->GetVirtNodeAttribute($vname,
"VM_MEMSIZE", \$vm_memsize)
== 0 or goto bad;
if ($vm_memsize) {
if ($verbose) {
print STDERR
"$vname is reserving $vm_memsize MB on $node\n";
}
if ($total_memory < $vm_memsize) {
print STDERR "*** CreateVnodes: ".
"no free memory for $vname on $pnode\n";
goto bad;
}
$total_memory -= $vm_memsize;
# remember for below.
$reserved_memory{$vname} = $vm_memsize;
}
}
}
#
# Create a bunch.
#
......@@ -2186,14 +2235,14 @@ sub CreateVnodes($$$)
my ( $jailip, $jailmask );
if ($isjailed) {
$query_result =
DBQueryWarn( "SELECT attrvalue FROM virt_node_attributes " .
"WHERE pid='$pid' AND eid='$eid' AND " .
"vname='$vname' AND " .
"attrkey='routable_control_ip'" );
if( $query_result && $query_result->numrows &&
( $query_result->fetchrow_array() )[ 0 ] eq "true" ) {
my $routable_control_ip;
$experiment->GetVirtNodeAttribute($vname, "routable_control_ip",
\$routable_control_ip)
== 0 or goto bad;
if ($routable_control_ip &&
$routable_control_ip eq "true") {
#
# Grab a public IP address from the free pool, if there
# is one.
......@@ -2274,12 +2323,10 @@ sub CreateVnodes($$$)
"def_boot_osid" => $osid,
"update_accounts" => 1,
"jailflag" => $isjailed);
# This is deprecated and will be removed. We now create real
# interfaces, see below.
if (0 && $isjailed && !$isremote) {
$nodesets{"jailip"} = $jailip;
$nodesets{"jailipmask"} = $JAILIPMASK;
}
if (exists($reserved_memory{$vname})) {
$nodesets{"reserved_memory"} = $reserved_memory{$vname};
}
my $statement = "insert into nodes set ".
join(",", map("$_='" . $nodesets{$_} . "'", keys(%nodesets)));
......
......@@ -28,7 +28,7 @@ use Exporter;
use vars qw(@ISA @EXPORT @EXPORT_OK
$VTOP_FLAGS_UPDATE $VTOP_FLAGS_VERBOSE $VTOP_FLAGS_QUIET
$VTOP_FLAGS_FIXNODES $VTOP_FLAGS_IMPOTENT $VTOP_FLAGS_ALLOCONLY
$VTOP_FLAGS_REGRESSION);
$VTOP_FLAGS_REGRESSION $VTOP_FLAGS_FIXLANNODES);
@ISA = "Exporter";
@EXPORT = qw( );
......@@ -82,10 +82,11 @@ $VTOP_FLAGS_IMPOTENT = 0x08;
$VTOP_FLAGS_REGRESSION = 0x10;
$VTOP_FLAGS_QUIET = 0x20;
$VTOP_FLAGS_ALLOCONLY = 0x40;
$VTOP_FLAGS_FIXLANNODES = 0x80;
@EXPORT_OK = qw($VTOP_FLAGS_UPDATE $VTOP_FLAGS_VERBOSE $VTOP_FLAGS_FIXNODES
$VTOP_FLAGS_IMPOTENT $VTOP_FLAGS_REGRESSION $VTOP_FLAGS_QUIET
$VTOP_FLAGS_ALLOCONLY);
$VTOP_FLAGS_ALLOCONLY $VTOP_FLAGS_FIXLANNODES);
#
# Create an object representing the stuff we need to create the vtop file.
......@@ -133,11 +134,15 @@ sub Create($$$$)
# Mostly for update mode.
$self->{'FIXEDNODES'} = {};
$self->{'CURRENT_V2P'} = {};
$self->{'CURRENT_V2P'} = {};
$self->{'CURRENT_P2V'} = {};
$self->{'CURRENT_V2V'} = {};
$self->{'OLDRSRVCLEAN_FLAG'} = 0;
$self->{'OLDRSRVCLEAN_NODES'} = {};
# This is just for lannodes.
$self->{'CURRENT_V2PMAP'} = {};
# Below is for interpretation of assign results.
$self->{'PNODES'} = {};
$self->{'SOLUTION'} = {};
......@@ -178,6 +183,7 @@ sub results($) { return $_[0]->{'RESULTS'}; }
sub current_v2p($) { return $_[0]->{'CURRENT_V2P'}; }
sub current_p2v($) { return $_[0]->{'CURRENT_P2V'}; }
sub current_v2v($) { return $_[0]->{'CURRENT_V2V'}; }
sub current_v2pmap($) { return $_[0]->{'CURRENT_V2PMAP'}; }
sub pnodes($) { return $_[0]->{'PNODES'}; }
sub fixednodes($) { return $_[0]->{'FIXEDNODES'}; }
sub newreserved($) { return $_[0]->{'NEWRESERVED'}; }
......@@ -200,6 +206,7 @@ sub virt_lan_lans($) { return $_[0]->virt_table("virt_lan_lans"); }
sub virt_paths($) { return $_[0]->virt_table("virt_paths"); }
sub virt_bridges($) { return $_[0]->virt_table("virt_bridges"); }
sub virt_desires($) { return $_[0]->virt_table("virt_node_desires"); }
sub virt_attributes($) { return $_[0]->virt_table("virt_node_attributes"); }
sub virt_startloc($) { return $_[0]->virt_table("virt_node_startloc"); }
sub virt_trafgens($) { return $_[0]->virt_table("virt_trafgens"); }
sub virt_lan_settings($){ return $_[0]->virt_table("virt_lan_settings"); }
......@@ -219,6 +226,7 @@ sub verbose($) { return $_[0]->flags() & $VTOP_FLAGS_VERBOSE; }
sub quiet($) { return $_[0]->flags() & $VTOP_FLAGS_QUIET; }
sub updating($) { return $_[0]->flags() & $VTOP_FLAGS_UPDATE; }
sub fixcurrent($) { return $_[0]->flags() & $VTOP_FLAGS_FIXNODES; }
sub fixlanodes($) { return $_[0]->flags() & $VTOP_FLAGS_FIXLANNODES; }
sub impotent($) { return $_[0]->flags() & $VTOP_FLAGS_IMPOTENT; }
sub alloconly($) { return $_[0]->flags() & $VTOP_FLAGS_ALLOCONLY; }
sub regression($) { return $_[0]->flags() & $VTOP_FLAGS_REGRESSION; }
......@@ -935,6 +943,7 @@ sub interfacespeedmbps($$$)
sub LoadCurrentResources($)
{
my ($self) = @_;
my $exptidx = $self->exptidx();
$self->counters()->{'reserved_simcount'} = 0;
$self->counters()->{'reserved_virtcount'} = 0;
......@@ -1052,6 +1061,18 @@ sub LoadCurrentResources($)
}
}
}
#
# Grab the v2pmap table so we can find out where lan nodes were
# assigned last time.
#
my $query_result =
DBQueryWarn("select vname,node_id from v2pmap where exptidx='$exptidx'");
return -1
if (!$query_result);
while (my ($vname,$nodeid) = $query_result->fetchrow_array()) {
$self->current_v2pmap()->{$vname} = $nodeid;
}
return 0;
}
......@@ -1070,6 +1091,7 @@ sub LoadVirtNodes($)
my $vnode = libvtop::virt_node->Create($self, $virt_node);
my $vname = $vnode->vname();
my $desires = {};
my $attrs = {};
my $startloc = undef;
# Other fields we need.
......@@ -1358,6 +1380,58 @@ sub LoadVirtNodes($)
}
$vnode->_desires($desires);
#
# Add in attributes.
#
foreach my $attr ($self->virt_attributes()->Rows()) {
next
if ($attr->vname() ne $vname);
$attrs->{$attr->attrkey()} = $attr->attrvalue();
if ($attr->attrkey() eq "XEN_MEMSIZE" ||
$attr->attrkey() eq "MEMORYSIZE") {
$vnode->_desires()->{"?+ram"} = $attr->attrvalue();
}
}
$vnode->_atttributes($attrs);
#
# Need to set a default. But this needs to be seen on
# the client too, via virt_node_attributes.
#
# We need a way to associate these defaults with the virtualization
# type, but that is a function of the image. Needs more thought.
#
if ($isvirt) {
my $defmem = 128;
my $maxmem = 128;
if ($vnode->_parent_osinfo()->osname() =~ /xen/i) {
$defmem = 256;
$maxmem = 512;
}
if (!exists($vnode->_desires()->{"?+ram"})) {
$self->printdb("Setting VM memsize to $defmem for $vname\n");
$vnode->_desires()->{"?+ram"} = $defmem;
}
else {
#
# If the user wants a shared node, they are not allowed to
# ask for more then the max, unless its an admin.
#
if ($vnode->_desires()->{"?+ram"} > $maxmem &&
$vnode->_sharedokay() &&
!$self->user()->IsAdmin()) {
tberror("Invalid def_parentosid in " .
$vnode->_osinfo() . "\n");
return -1;
}
}
$self->experiment()->SetVirtNodeAttribute($vname, "VM_MEMSIZE",
$vnode->_desires()->{"?+ram"});
}
#
# And the startloc, but doubt this is used anymore.
#
......@@ -2238,6 +2312,18 @@ sub GenFixNodes($)
$self->createFixedNode($vname, $fixed);
}
}
if ($self->fixlanodes()) {
#
# Add lan node fixnodes.
#
foreach my $lannode (sort(keys(%{ $self->lannodes() }))) {
next
if (!exists($self->current_v2pmap()->{$lannode}));
$self->createFixedNode($lannode,
$self->current_v2pmap()->{$lannode});
}
}
return 0;
}
......@@ -4148,9 +4234,13 @@ sub AddNodeToSolution($$$)
{
my ($self, $virtual, $physical) = @_;
# Skip LAN/Fake nodes.
return 0
if (exists($self->lannodes()->{$virtual}));
#
# We generally ignore lannodes, but lets remember the mapping.
#
if (exists($self->lannodes()->{$virtual})) {
$self->lannodes()->{$virtual} = $physical;
return 0;
}
#
# XXX Must distinguish between local and ProtoGeni resources
......@@ -4482,10 +4572,13 @@ sub ReadRspecSolution($$)
my $node_urn = GeniXML::GetNodeId($ref);
my $virtual = GeniXML::GetVirtualId($ref);
# Skip LAN/Fake nodes.
return 0
if (exists($self->lannodes()->{$virtual}));
#
# We generally ignore lannodes, but lets remember the mapping.
#
if (exists($self->lannodes()->{$virtual})) {
$self->lannodes()->{$virtual} = $node_urn;
return 0;
}
$self->AddNodeToSolution($virtual, $node_urn);
$ifacemap{$virtual} = {};
......@@ -5221,8 +5314,7 @@ sub AllocNodes($)
}
#
# Upload the v2pmap table. The only place I know that cares about
# this table is dohosts() in tmcd.c
# Upload the v2pmap table.
#
foreach my $vnodename (keys(%{ $self->solution_v2p() })) {
#
......@@ -5240,7 +5332,16 @@ sub AllocNodes($)
" vname='$vnodename', node_id='$pnodename'")
or return -1 if (!($self->impotent() || $self->alloconly()));
}
foreach my $vnodename (keys(%{ $self->lannodes() })) {
my $pnodename = $self->lannodes()->{$vnodename};
$self->printdb("v2pmap (lannode): $vnodename $pnodename\n");
DBQueryWarn("insert into v2pmap set ".
" pid='$pid', eid='$eid', exptidx='$idx', ".
" vname='$vnodename', node_id='$pnodename'")
or return -1 if (!($self->impotent() || $self->alloconly()));
}
return 0;
}
......@@ -6373,7 +6474,7 @@ sub InterpLinksAux($)
# If the "lannode" is placed on a node, and that node is
# different than the current node, we have to connect the
# two in the vlan. Typically, the lannode is placed on a
# switch, and this is not an issue. Rob understands this!
# switch, and this is not an issue.
#
if (!$virtnodeA->_onsharednode() &&
! ($member0->_lannode() ne "null" &&
......@@ -7122,7 +7223,10 @@ sub NewVirtIface($$$$;$)
if (!defined($virtiface)) {
return undef;
}
$self->printdb("$virtiface: $member, isvdev:$isvdev, isveth:$isveth\n");
$self->printdb("$virtiface: $member, ".
"isvdev:$isvdev, isveth:$isveth" .
(defined($pport) ? ", pport:$pport" : "") . "\n");
my $newid = $virtiface->unit();
# Record this vinterface mapping.
......@@ -7154,7 +7258,7 @@ sub NewVirtIface($$$$;$)
}
#
# XXX hackery that only Rob and Leigh understand.
# This is to assist with patching up virts. See AddVirtPatch().
# A LAN of vnodes split across multiple physical machines may
# not have the correct physical LAN info coming out of assign
# and may need to be patched up later.
......@@ -7165,7 +7269,7 @@ sub NewVirtIface($$$$;$)
$self->solution_vethmap()->{$lan}->{$pnodename} = []
if (!exists($self->solution_vethmap()->{$lan}->{$pnodename}));
push(@{ $self->solution_vethmap()->{$lan}->{$pnodename} }, $newid);
push(@{ $self->solution_vethmap()->{$lan}->{$pnodename} }, $virtiface);
}
return $virtiface;
}
......@@ -8251,6 +8355,43 @@ sub UploadVlans($)
if ($self->verbose());
}
#
# Patch up virts.
#
foreach my $vname (keys(%{ $self->vlans() })) {
next
if (!exists($self->solution_vethpatch()->{$vname}));
foreach my $pname (keys(%{ $self->solution_vethmap()->{$vname} })) {
next
if (!exists($self->solution_vethpatch()->{$vname}->{$pname}));
my $pport = $self->solution_vethpatch()->{$vname}->{$pname};
my $pnode = $self->pnodes()->{$pname};
foreach my $viface (@{$self->solution_vethmap()->{$vname}->{$pname}}) {
#
# For veth and vlan interfaces, we need to set the
# characteristics of the underlying physical
# interface.
#
my $speed =
$self->interfacespeedmbps(physinterfacetype($pnode, $pport),
"ethernet");
$self->printdb("Virt Patch: ".
"$vname $pname $viface $pport $speed\n");
my $member = $viface->_member();
$viface->_speed($speed);
$viface->_trunk(($viface->type() eq "vlan") ? 1 : 0);
# This is a fake data structure in impotent mode.
$viface->Update({"iface" => $pport})
if (! $self->alloconly());
}
}
}
#
# Upload the port bw to the interfaces table for each iface.
#
......@@ -8388,46 +8529,7 @@ sub UploadVlans($)
}
}
}
#
# Patch up virts.
#
foreach my $vname (keys(%{ $self->vlans() })) {
next
if (!exists($self->solution_vethpatch()->{$vname}));
foreach my $pname (keys(%{ $self->solution_vethmap()->{$vname} })) {
next
if (!exists($self->solution_vethpatch()->{$vname}->{$pname}));
my $pport = $self->solution_vethpatch()->{$vname}->{$pname};
my $pnode = $self->pnodes()->{$pname};
foreach my $vid (@{$self->solution_vethmap()->{$vname}->{$pname}}) {
#
# For veth and vlan interfaces, we need to set the
# characteristics of the underlying physical
# interface.
#
my $speed =
$self->interfacespeedmbps(physinterfacetype($pnode, $pport),
"ethernet");
$self->printdb("Virt Patch: ".
"$vname $pname $vid $pport $speed\n");
if (!($self->impotent() || $self->alloconly())) {
DBQueryWarn("update vinterfaces set iface='$pport' ".
"where node_id='$pname' and unit='$vid'")
or return -1;
DBQueryWarn("update interfaces set " .
" current_speed='$speed' " .
"where node_id='$pname' and iface='$pport'")
or return -1;
}
}
}
}
return 0;
}
#
......
# -*- tcl -*-
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -67,6 +67,7 @@ proc tb-set-usewatunnels {onoff} {}
proc tb-set-wasolver-weights {delay bw plr} {}
proc tb-use-endnodeshaping {onoff} {}
proc tb-force-endnodeshaping {onoff} {}
proc tb-set-node-memory-size {node mem} {}
proc tb-set-multiplexed {link onoff} {}
proc tb-set-endnodeshaping {link onoff} {}
proc tb-set-noshaping {link onoff} {}
......
# -*- tcl -*-
#
# Copyright (c) 2000-2012 University of Utah and the Flux Group.
# Copyright (c) 2000-2013 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -2304,6 +2304,16 @@ proc tb-set-node-routable-ip {node onoff} {
}
}
# This number is MBs.
proc tb-set-node-memory-size {node mem} {
if {[$node info class] != "Node"} {
perror "\[tb-set-node-memory-size] $node is not a node."
return
}
$node add-attribute "MEMORY_SIZE" "$mem"
$node add-desire "?+ram" "$mem"
}
# The following codes are written, in order to use OML to send data to a server
# in an Emulab experiment. It will define a node to run OML server and create
# two files (MP.c and MP.h) for users. (mp represents measurement point in
......
......@@ -506,11 +506,22 @@ while (my ($class,$type,$isvirt) = $result->fetchrow_array) {
# look for the pxe_boot_path pointing to the tpm version of grub.
our %node_usb;
$result = DBQueryFatal("select node_id from node_attributes where attrkey='pxe_boot_path' and attrvalue='/tftpboot/pxeboot_tpm'");
$result = DBQueryFatal("select node_id from node_attributes ".
"where attrkey='pxe_boot_path' and ".
" attrvalue='/tftpboot/pxeboot_tpm'");
while (($node) = $result->fetchrow_array) {
$node_usb{$node} = 1;
}
# Physical RAM overrides.
my %node_ram;
$result = DBQueryFatal("select node_id,attrvalue from node_attributes ".
"where attrkey='physical_ram'");
while (my ($nodeid,$ram) = $result->fetchrow_array) {
$node_ram{$nodeid} = $ram;
}
# Read node_startloc
$result = DBQueryFatal("select node_id,building from node_startloc");
while (($node,$building) = $result->fetchrow_array) {
......@@ -768,6 +779,27 @@ while (my ($node_id,$count) = $result->fetchrow_array) {
$globalcounts{$node_id} = $count;
}
#
# Get the memory usage on each physical node.
# This will be subtracted from whatever the type/node info says.
# Note that we have to exclude nodes from this experiment
# that are already mapped to the node since the user already owns that memory.
#
my %node_ramusage;
if ($virtstuff) {
$result =
DBQueryFatal("select n.phys_nodeid,n.reserved_memory from nodes as n ".
"left join reserved as r on r.node_id=n.node_id ".
"where n.node_id!=n.phys_nodeid ".
(defined($experiment) ?
"and r.exptidx!=" . $experiment->idx() : ""));
while (my ($pnode,$memory) = $result->fetchrow_array()) {
$node_ramusage{$pnode} += $memory;
}
}
# Find available nodes.
#
# This first query deals with just local nodes. Local nodes can host
......@@ -990,6 +1022,15 @@ foreach $node (@nodenames) {
my ($latitude, $longitude, $country);
# per-node override RAM.
$ram = $node_ram{$node} if (exists($node_ram{$node}));
if (exists($node_ramusage{$node})) {
$ram -= $node_ramusage{$node};
# Should this be an error?
$ram = 0
if ($ram < 0);
}
# XXX temporary hack until node reboot avoidance
# is available. Nodes running the FBSD-NSE image
# will have a feature def-osid-fbsd-nse 0.0
......@@ -1159,9 +1200,10 @@ foreach $node (@nodenames) {
if ($trivspeed) {
push @flags, "trivial_bw:$trivspeed";
}
# Add CPU and RAM information
$cpu_ram_features_present++;
if (! $sharing_mode{$node}) {
# Add CPU and RAM information
$cpu_ram_features_present++;
# This number can be use for fine-tuning packing
push @features, "?+virtpercent:100";
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment