libplab.py.in

# -*- python -*-
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2003 University of Utah and the Flux Group.
# All rights reserved.
#

"""
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
"""

import sys
sys.path.append("@prefix@/lib")

import os, time
import string
import traceback
import socket
import httplib
import xml.parsers.expat
import re

from popen2 import Popen4
from warnings import warn

#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice

agents = {'PLC'    : mod_PLC,
          'dslice' : mod_dslice}

#
# output control vars
#
verbose = 0
debug = 0

#
# Constants
#
DEF_AGENT = "PLC";

RENEW_TIME = 2*24*60*60  # Renew two days before lease expires

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
NODEPROBEINT  = 10

TBOPS = "@TBOPSEMAIL@".replace("\\","")
MAILTAG = "@THISHOMEBASE@"

RESERVED_PID = "emulab-ops"
RESERVED_EID = "hwdown"       # start life in hwdown
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"

MAGIC_INET2_GATEWAYS = ("205.124.237.10", )
MAGIC_INET_GATEWAYS = ("205.124.249.123", "205.124.249.113",
                       "205.124.249.121", "205.124.249.115")
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
# right now these are the only 2.0 machines running the new slice interface:
#ALLOWED_NODES = ('198.78.49.59', '18.31.0.213', '169.229.50.85',
#                 '169.229.50.89', '128.112.152.124', '12.46.129.23',
#                 '64.41.221.196', '132.239.17.226', '128.223.6.113',
#                 '128.208.4.199', '128.2.198.199', '155.98.35.2',
#                 '155.98.35.3')
ALLOWED_NODES = ()
NUMVNODES = 20

PLABNODE = "@prefix@/sbin/plabnode"
SSH = "@prefix@/bin/sshtb"
NAMED_SETUP = "@prefix@/sbin/named_setup"

ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"

DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000

#
# var to track failed renewals
#
failedrenew = []

#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)


#
# Plab abstraction
#

#
# Class responsible for parsing planetlab sites file
#
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
            pass
        
        elif name == "HOST":
            self.__hosts.append({'HNAME' : attrs['NAME'],
                                 'IP'    : attrs['IP'],
                                 'NODEID': attrs['NODE_ID'],
                                 'SITE'  : self.__sitename})
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
            pass
        return

        
class Plab:
    def __init__(self, agent=None):
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
        if debug:
            print "Using module: %s" % self.agent.modname
            pass
        pass

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

    def updateNodeEntries(self, ignorenew = False):
        """
        Finds out which Plab nodes are available, and
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
        discovered.

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.

        Deleting nodes that are no longer available may not be the
        best approach due to the overhead of adding new nodes (ie, if
        the node disappears for a while, then comes back). Therefore,
        this is not currently done.
        """
        
        print "Getting available Plab nodes ..."

        avail = []
        try:
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
        # XXX: rewrite to use more elegant exception info gathering.
        except:
            extype, exval, extrace = sys.exc_info()
            print "Error talking to agent: %s: %s" % (extype, exval)
            if debug:
                print extrace
            print "Going back to sleep until next scheduled poll"
            return

        if debug:
            print "Got advertisement list:"
            print avail
            pass

        # Enforce allowed nodes limitation, if any.
        if len(ALLOWED_NODES):
            ravail = []
            for nodeent in avail:
                if nodeent['IP'] in ALLOWED_NODES:
                    ravail.append(nodeent)
                    pass
                pass
            print "Advertisements in allowed nodes list:\n%s" % ravail
            avail = ravail
            pass

        # Get node info we already have.
        known = self.__getKnownPnodes()
        if debug:
            print "Got known pnodes:"
            print known
            pass

        # Create list of nodes to add or update
        toadd = []  # List of node entries to add to DB
        for nodeent in avail:
            # Replace sequences of bad chars in the site entity with
            # a single "-".
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
            nid = nodeent['NODEID']
            # If we don't know about this node, then add it and mark
            # that we are _not_ doing an update.
            if not known.has_key(nid):
                toadd.append((nodeent, False))
                pass
            # If we do know abou this node, check to see if any of its
            # attributes have changed, and if so, add it to the list and
            # mark if for update.
            else:
                kent = known[nid]
                if kent['HNAME'] != nodeent['HNAME'] or \
                   kent['IP']    != nodeent['IP'] or \
                   kent['SITE']  != nodeent['SITE']:
                    toadd.append((nodeent, True))
                    pass
                pass
            pass

        # Process the add/update list - add to, or update in the DB.
        if len(toadd):
            # Are we ignoring new entries?
            if ignorenew:
                if verbose:
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
                    pass
                pass
            # If not ignoring, do the addition/update.
            else:
                addstr = ""
                updstr = ""
                print "There are %d new/changed Plab nodes." % len(toadd)
                for nodeent, update in toadd:
                    # Get the linktype here so we can report it in email.
                    self.__findLinkType(nodeent)
                    if debug:
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
                        pass
                    # Add/update the node in the DB.
                    self.__addNode(nodeent, update)
                    # Rest of block adds a line for the add/update messages.
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
                              (nodeent['NODEID'],
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
                    if update:
                        updstr += nodestr
                        pass
                    else:
                        addstr += nodestr
                        pass
                    pass

                # We need to update DNS since we've added hosts..
                print "Forcing a named map update ..."
                os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
                # Now announce that we've added/updated nodes.
                SENDMAIL(TBOPS,
                         "Plab nodes have been added/updated in the DB.",
                         "The following plab nodes have been added to the DB:\n"
                         "NodeID\t\tHostname\t\tIP\t\tSite\t\tLinktype\n\n"
                         "%s\n\n"
                         "The following plab nodes have been updated in the DB:\n"
                         "NodeID\t\tHostname\t\tIP\t\tSite\n\n"
                         "%s\n\n" % \
                         (addstr, updstr),
                         TBOPS)
                print "Done adding new Plab nodes."
                pass
            pass
        return

    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
        res = DBQueryFatal("select i.node_id, i.IP, w.hostname, w.site"
                           " from interfaces as i"
                           " left join nodes as np on"
                           "  np.node_id = i.node_id"
                           " left join widearea_nodeinfo as w on"
                           "  i.node_id = w.node_id"
                           " where np.type = 'pcplabphys'")
        ret = {}
        for nodeid, ip, hname, site in res:
            nid = nodeid.replace(NODEPREFIX, "")
            ret[nid] = {'HNAME' : hname,
                        'IP'    : ip,
                        'SITE'  : site}
            pass
        return ret
        
    def __findLinkType(self, nodeent):
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

        This can't detect DSL links..
        """
        # Is host international (or flux/emulab local)?
        from socket import gethostbyaddr, getfqdn, herror
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
        # Is host on I2?
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
                nodeent['LINKTYPE'] = "pcplabinet2"
                return

        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
                break
        else:
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']

        # Must be plain 'ole Internet
        nodeent['LINKTYPE'] = "pcplabinet"
        return

    def __addNode(self, nodeent, update = False):
        """
        getFree helper function.  Adds a new Plab pnode and associated
        vnodes to the DB.  linktype should be one of (inet2, inet, intl,
        dsl).

        XXX This duplicates a lot of the functionality of newwanode.
        Note that, very unlike newwanode, the node is initially up,
        since it had to be up to be added in the first place.  This also
        adds some additional fields that newwanode doesn't, and takes
        advantage of the fact that the Plab nodes may be added in bulk.
        """
        # block out common termination signals while adding a node
        osigs = disable_sigs(TERMSIGS)
        defosid, controliface = self.__getNodetypeInfo()
        nodeid = NODEPREFIX + nodeent['NODEID']
        priority = PLABBASEPRIO + int(nodeent['NODEID'])
        hostonly = nodeent['HNAME'].replace(".", "-")
        
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)

        try:
            DBQueryFatal("replace into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
                          'FREE_CLEAN',
                          'ISUP'))

            DBQueryFatal("replace into widearea_nodeinfo"
                         " (node_id, contact_uid, hostname, site)"
                         " values (%s, %s, %s, %s)",
                         (nodeid, 'bnc', nodeent['HNAME'], nodeent['SITE']))

            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'fxp',
                          controliface, 'ctrl'))

            DBQueryFatal("replace into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))

            DBQueryFatal("replace into node_auxtypes"
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, nodeent['LINKTYPE'], 1))
            
            DBQueryFatal("replace into node_auxtypes"
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
            DBQueryFatal("replace into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (nodeid, 'down'))

            # Don't do any vnode additions if we are just updating.
            if update:
                enable_sigs(osigs)
                return

            vnodetype = "pcplab"
            vnodeid = ""
            for n in range(NUMVNODES):
                vprio = (priority * 100) + (n+1)
                sshdport = 38000+(n+1)
                vnodeid = "%s-%d" % (nodeid, n+1)
                if verbose:
                    print "Creating vnode %s, priority %d" % (vnodeid, vprio)
                    pass
                    
                DBQueryFatal("insert into nodes"
                             " (node_id, type, phys_nodeid, role, priority,"
                             "  op_mode, def_boot_osid, update_accounts,"
                             "  allocstate, allocstate_timestamp,"
                             "  eventstate, state_timestamp, sshdport)"
                             " values (%s, %s, %s, %s, %s,"
                             "  %s, %s, %s, %s, now(), %s, now(), %s)",
                             (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                              'PCVM', defosid, 1,
                              'FREE_CLEAN',
                              'SHUTDOWN', sshdport))
                
                DBQueryFatal("insert into node_status"
                             " (node_id, status, status_timestamp)"
                             " values (%s, %s, now())",
                             (vnodeid, 'up'))
                
                pass
                
            # Put the last vnode created into the special monitoring expt.
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
            pass
        
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
                     "\n Please clean up!\n" % tbmsg, TBOPS)
            enable_sigs(osigs)
            raise

        # last but not least, unblock signals
        enable_sigs(osigs)
        return

    def __getNodetypeInfo(self):
        """
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
        it doesn't change.
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
            if debug:
                print "Getting node type info"
                pass
            res = DBQueryFatal("select osid, control_iface"
                               " from node_types"
                               " where type = 'pcplabphys'")
            assert (len(res) == 1), "Failed to get node type info"
            (self.__getNodetypeInfoCache, ) = res
            pass
        
        return self.__getNodetypeInfoCache

    # XXX: deprecated - should probably just be removed
    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
        if debug:
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
            pass
        finally:
            DBQueryFatal("unlock tables")
            pass
        
        return nodeid, priority

    # XXX: might want to just call into slice.renew and let module specific
    # code deal with individual node renew if it tracks at that level.
    def renew(self):
        """
        Renews all of the Plab leases that are going to expire soon.
        """
        
        print "Renewing Plab leases ..."
        # Ugh, MySQL doesn't know UTC until v4.1.1, and unix_timestamp()
        # returns the local time
        now = int(time.mktime(time.gmtime()))
        endtime = now + RENEW_TIME
        res = DBQueryFatal("select NULL, pid, eid,"
                            " unix_timestamp(leaseend) from plab_slices"
                            " where leaseend != NULL and"
                            "  %s > unix_timestamp(leaseend)",
                            (endtime, ))
        res += DBQueryFatal("select node_id, pid, eid,"
                            " unix_timestamp(leaseend) from plab_slice_nodes"
                            " where leaseend != NULL and"
                            "  %s > unix_timestamp(leaseend)",
                            (endtime, ))
        
        loadedSlices = {}
        global failedrenew # XXX
        newfail = []
        failsoon = []
        ret = 0

        for entry in res:
            nodeid, pid, eid, tstamp = entry

            if tstamp <= now:
                if nodeid:
                    print "WARNING: Node lease for %s (%s/%s) has expired!" % \
                          (nodeid, pid, eid)
                    pass
                else:
                    print "WARNING: Slice lease for %s/%s has expired!" % \
                          (pid, eid)
                    pass
                continue

            try:
                slice = loadedSlices[(pid, eid)]
                pass
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
                pass
            
            if nodeid:
                node = slice.loadNode(nodeid)
                ret = node.renew()
                pass
            else:
                try:
                    res, slicemeta, leaselen = \
                         self.agent.renewSlice(slice.slicename)
                    ret = 1
                    pass
                except:
                    ret = 0

            if not ret:
                if nodeid:
                    print "Failed to renew lease for %s (%s/%s)" % \
                          (nodeid, pid, eid)
                    pass
                else:
                    print "Failed to renew lease for %s/%s" % \
                          (pid, eid)
                    pass
                if entry not in failedrenew:
                    newfail.append(entry)
                    pass
                if (tstamp - now) < (2*3600):
                    failsoon.append(entry)
                    pass
                pass
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
                    
        if newfail:
            failedrenew += newfail
            failstr = ""
            for n in newfail:
                if n[0]:
                    failstr += "%s (%s/%s)\n" % n[:3]
                    pass
                else:
                    failstr += "%s/%s\n" % n[1:3]
                    pass
                pass
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
                     "Failed to renew lease on the following nodes:\n%s" %
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
            pass

        if failsoon:
            failstr = ""
            for n in failsoon:
                if n[0]:
                    failstr += "%s (%s/%s): expires: %s\n" % \
                               (n[:3] + (time.ctime(n[3]),))
                    pass
                else:
                    failstr += "%s/%s: expires: %s\n" % \
                               (n[1:3] + (time.ctime(n[3]),))
                    pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases are about to expire",
                     "The following plab leases are about to expire:\n%s" %
                     failstr + "\n\nPlease look into it!", TBOPS)
            pass
        return
    
    pass # end class Plab


#
# Slice abstraction
#

class Slice:

    def __init__(self, plab, pid, eid, slicename = None):
        self.plab = plab
        self.pid, self.eid = pid, eid
        self.slicemeta = None
        self.slicename = slicename
        return
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
        if not self.slicename:
            res = DBQueryFatal("select idx from experiments "
                               "where pid=%s "
                               "and eid=%s",
                               (self.pid, self.eid))
            if not len(res):
                raise RuntimeError, "Didn't get any results while looking for idx"
            eindex = res[0][0]
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass
        
        print "Creating Plab slice %s." % self.slicename
        
        try:
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)

            DBQueryFatal("insert into plab_slices"
                         " (pid, eid, slicename, slicemeta, leaseend)"
                         " values (%s, %s, %s, %s, FROM_UNIXTIME(%s))",
                         (self.pid, self.eid, self.slicename,
                          self.slicemeta, self.leaseend))
            pass
        except:
            self.plab.agent.deleteSlice(self)
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
            raise

        return res

    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.

        XXX This should probably be made lazy, since not all operations
        really need it
        """
        if verbose:
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
        res = DBQueryFatal("select slicename, slicemeta from plab_slices"
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
        ((self.slicename, self.slicemeta), ) = res
        pass

    def renew(self):
        """
        Renews slice lease, if applicable to selected backend agent.
        """
        print "Renewing lease for slice %s" % self.slicename

        ret = 0
        try:
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.renewSlice(self)
            
            DBQueryFatal("replace into plab_slices"
                         " (pid, eid, slicename, slicemeta, leaseend)"
                         " values (%s, %s, %s, %s, FROM_UNIXTIME(%s))",
                         (self.pid, self.eid, self.slicename,
                          self.slicemeta, self.leaseend))
            ret = 1
            pass
        except:
            print "Slice renewal failed!"
            traceback.print_exc()
            ret = 0
            pass
        
        return ret        

    def destroy(self):
        """
        Frees all nodes in this slice and destroys the slice.  Note
        that this will really pound the DB if there are many nodes left
        in the slice, but those should be removed by Emulab before the
        slice is destroyed.
        """
        print "Destroying Plab slice %s." % self.slicename
        res = DBQueryFatal("select node_id from plab_slice_nodes"
                           " where slicename = %s",
                           (self.slicename))
        print "\tRemoving any remaining nodes in slice.."
        for (nodeid,) in res:
            node = self.loadNode(nodeid)
            node.free()
            del node  # Encourage the GC'er

        osigs = disable_sigs(TERMSIGS)

        try:
            self.plab.agent.deleteSlice(self)
            pass
        except:
            print "Failed to delete slice!"
            traceback.print_exc()
            pass
        
        try:
            print "\tRemoving slice DB entry."
            DBQueryFatal("delete from plab_slices where slicename = %s",
                         (self.slicename,))
        except:
            print "Error deleting slice from DB!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error deleting slice from DB",
                     "Slice deletion error:\n\n%s" % tbstr, TBOPS)
            enable_sigs(osigs)
            raise
        
        enable_sigs(osigs)

    def createNode(self, nodeid, force=False):
        """
        Node factory function
        """
        # XXX: KRW - The following is a hack to help me with testing.
        if not nodeid.startswith("plab"):
            IP = socket.gethostbyname(nodeid)
            qres = DBQueryFatal("select n.node_id from nodes as n left join "
                                "interfaces as i on n.node_id = i.node_id "
                                "where i.IP = %s", (IP,))
            assert (len(qres) > 0), "Node does not exist in DB: %s" % nodeid
            nodeid = qres[0][0] + "-20"
        node = Node(self, nodeid)
        node._create(force)
        return node

    def loadNode(self, nodeid):
        """
        Node factory function
        """
        node = Node(self, nodeid)
        node._load()
        return node

#
# Node abstraction
#

class Node:
    def __init__(self, slice, nodeid, pollNode = False):
        self.slice = slice
        self.nodeid = nodeid
        self.IP = self.__findIP()
        self.nodemeta = None
        self.pollNode = pollNode
        return

    # XXX: may want to rethink signal handling here.
    def _create(self, force=False):
        """
        Creates a new node.  This physically allocates the node into the
        slice through the dslice agent and node manager.  Note that no
        node setup is performed.  Don't call this directly, use
        Slice.createNode instead.
        """
        
        # First, make sure there isn't already an entry in the DB
        try:
            self._load()
        except:
            pass
        else:
            if force:
                print "Node entry exists in DB, but creation forced anyway."
            else:
                raise RuntimeError, "Entry for plab node %s already exists " \
                      "in the DB" % self.nodeid
            pass

        print "Creating Plab node %s on %s." % (self.nodeid, self.IP)
        res, self.nodemeta, self.leaselen = \
             self.slice.plab.agent.createNode(self)
        
        DBQueryFatal("replace into plab_slice_nodes"
                     " (pid, eid, slicename, node_id,"
                     " nodemeta, leaseend)"
                     " values (%s, %s, %s, %s, %s, %s)",
                     (self.slice.pid, self.slice.eid,
                      self.slice.slicename, self.nodeid,
                      self.nodemeta, self.leaselen))

        if self.pollNode:
            TIMESTAMP("Waiting for %s to respond" % self.nodeid)
            while True:
                try:
                    self.__perform("/bin/true")
                    pass
                except:
                    time.sleep(NODEPROBEINT)
                    pass
                else: break
                pass
            TIMESTAMP("Node %s ready." % self.nodeid)
            pass        

        TIMESTAMP("createnode finished on %s." % self.nodeid)
        return

    def _load(self):
        """
        Loads an already allocated node from the DB.  Don't call this
        directly, use Slice.loadNode instead.
        """
        if verbose:
            print "Loading node %s" % self.nodeid
        res = DBQueryFatal("select slicename, nodemeta"
                           " from plab_slice_nodes where node_id = %s",
                           (self.nodeid))
        assert (len(res) > 0), \
               "Node %s (slice %s) not found" % \
               (self.nodeid, self.slice.slicename)
        assert (len(res) == 1), \
               "Multiple nodes found for nodeid %s" % self.nodeid
        ((slicename, self.nodemeta), ) = res
        assert (slicename == self.slice.slicename), \
               "Node %s loaded by slice %s, but claims to be in slice %s" % \
               (self.nodeid, self.slice.slicename, slicename)

    def free(self):
        """
        Frees the node and kills the VM.  Note that this does not
        shutdown anything inside the vserver.  Warning: forks a process
        to carry out the actual work!
        """
        res = ForkCmd(self._free, timeout=FREE_TIMEOUT,
                      disable_sigs_parent=TERMSIGS,
                      disable_sigs_child=TERMSIGS)
        return res[0] | res[1]
        
    def _free(self):
        """
        Frees the node and kills the VM.  Note that this does not
        shutdown anything inside the vserver.  Don't call this directly;
        instead, use Node.free()
        """
        deleted = 0
        TIMESTAMP("freenode %s started." % self.nodeid)
        print "Freeing Plab node %s." % self.nodeid
        
        # Remove the DB entry first.
        try:
            DBQueryFatal("delete from plab_slice_nodes where node_id = %s",
                         (self.nodeid,))
        except:
            print "Uh oh, couldn't remove plab sliver record from the DB!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error: Couldn't remove plab vnode from DB",
                     "Unable to delete entry for sliver %s from the DB:"
                     "\n\n%s" % (self.nodeid, tbstr), TBOPS)
            pass

        deleted = self.slice.plab.agent.freeNode(self)
        
        TIMESTAMP("freenode %s finished." % self.nodeid)
        return not deleted

    def renew(self):
        """
        Renew the lease for this node.  Note that this method
        forks and runs another private method to actually do the
        work!