libplab.py.in

# -*- python -*-
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#

"""
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
"""

import sys
sys.path.append("@prefix@/lib")

import os, time
import string
import traceback
import socket
import httplib
import xml.parsers.expat
import re
import calendar

from popen2 import Popen4
from warnings import warn

#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
from mod_PLCNM import mod_PLCNM

agents = {'PLC'    : mod_PLC,
          'dslice' : mod_dslice,
          'PLCNM'  : mod_PLCNM}

#
# output control vars
#
verbose = 0
debug = 0

#
# Constants
#
DEF_AGENT = "PLCNM";

RENEW_TIME = 2*24*60*60  # Renew two days before lease expires

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
NODEPROBEINT  = 30

TBOPS = "@TBOPSEMAIL@".replace("\\","")
MAILTAG = "@THISHOMEBASE@"

RESERVED_PID = "emulab-ops"
RESERVED_EID = "hwdown"       # start life in hwdown
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"

MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18", )
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
LOCAL_PLAB_LINKTYPE = "pcplabinet2"

ALLOWED_NODES = ()
#ALLOWED_NODES = ("128.112.139.80", "128.112.139.72", "169.229.50.4")
IGNORED_NODES = ("10283",)

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)

PLABNODE = "@prefix@/sbin/plabnode"
SSH = "@prefix@/bin/sshtb"
NAMED_SETUP = "@prefix@/sbin/named_setup"

ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"

DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
PLAB_SVC_SLICENAME = "utah_svc_slice"
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"

PLABEXPIREWARN = 2*24*60*60   # two days advance warning.

#
# var to track failed renewals
#
failedrenew = []

#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)


#
# Plab abstraction
#

#
# Class responsible for parsing planetlab sites file
#
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
            pass
        
        elif name == "HOST":
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
            self.__hosts.append({'HNAME' : attrs['NAME'],
                                 'IP'    : attrs['IP'],
                                 'PLABID': attrs['NODE_ID'],
                                 'MAC'   : attrs['MAC'],
                                 'SITE'  : self.__sitename})
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
            pass
        return

        
class Plab:
    def __init__(self, agent=None):
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
        if debug:
            print "Using module: %s" % self.agent.modname
            pass
        pass

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

    def updateNodeEntries(self, ignorenew = False):
        """
        Finds out which Plab nodes are available, and
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
        discovered.

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
        
        print "Getting available Plab nodes ..."

        avail = []
        try:
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
        # XXX: rewrite to use more elegant exception info gathering.
        except:
            extype, exval, extrace = sys.exc_info()
            print "Error talking to agent: %s: %s" % (extype, exval)
            if debug:
                print extrace
            print "Going back to sleep until next scheduled poll"
            return

        if debug:
            print "Got advertisement list:"
            print avail
            pass

        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
        if len(ALLOWED_NODES) or len(IGNORED_NODES):
            allowed = []
            for nodeent in avail:
                if nodeent['PLABID'] in IGNORED_NODES:
                    continue
                elif len(ALLOWED_NODES):
                    if nodeent['IP'] in ALLOWED_NODES:
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
                    pass
                pass
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
            pass

        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

        # Get node info we already have.
        known = self.__getKnownPnodes()
        if debug:
            print "Got known pnodes:"
            print known
            pass

        # Create list of nodes to add or update
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
        for nodeent in avail:
            # Replace sequences of bad chars in the site entity with
            # a single "-".
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
            res = self.__matchPlabNode(nodeent, known)
            if not res:
                toadd.append(nodeent)
                pass
            elif len(res[1]):
                toupdate.append(res)
                pass

        # Process the list of nodes to add
        addstr = ""
        if len(toadd):
            # Are we ignoring new entries?
            if ignorenew:
                if verbose:
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
                    pass
                pass
            # If not ignoring, do the addition/update.
            else:
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
                    # Get the linktype here so we can report it in email.
                    self.__findLinkType(nodeent)
                    if debug:
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
                        pass
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
                              (nodeent['PLABID'],
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
                    addstr += nodestr
                    pass
                pass
            pass

        # Process node attribute updates.
        updstr = ""
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
            for updent in toupdate:
                self.__updateNode(updent)
                # Add a line for the add/update message.
                nodestr = updent[0] + "\n"
                for attr,val in updent[1].items():
                    nodestr += "\t%s:\t%s => %s\n" % (attr,val[0],val[1])
                    pass
                updstr += nodestr + "\n"
                pass
            pass

        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
        return

    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
            if len(same):
                return (nid, diff)
            pass
        return ()

    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
        res = DBQueryFatal("select node_id,plab_id,hostname,IP,mac"
                           " from plab_mapping")
        
        ret = {}
        for nodeid, plabid, hostname, ip, mac in res:
            ret[nodeid] = {'PLABID'  : plabid,
                           'HNAME'   : hostname,
                           'IP'      : ip,
                           'MAC'     : mac}
            pass
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
        return ret

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        uniqattrs = ['PLABID', 'HNAME', 'IP', 'MAC']
        attrs = {}
        dups = {}
        
        for ent in nodelist:
            for attr in uniqattrs:
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
        
    def __findLinkType(self, nodeent):
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

        This can't detect DSL links..
        """
        # Is host international (or flux/emulab local)?
        from socket import gethostbyaddr, getfqdn, herror
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
        # Is host on I2?
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
                nodeent['LINKTYPE'] = "pcplabinet2"
                return

        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
                break
        else:
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']

        # Must be plain 'ole Internet
        nodeent['LINKTYPE'] = "pcplabinet"
        return

    def __addNode(self, nodeent, update = False):
        """
        getFree helper function.  Adds a new Plab pnode and associated
        vnodes to the DB.  linktype should be one of (inet2, inet, intl,
        dsl).

        XXX This duplicates a lot of the functionality of newwanode.
        Note that, very unlike newwanode, the node is initially up,
        since it had to be up to be added in the first place.  This also
        adds some additional fields that newwanode doesn't, and takes
        advantage of the fact that the Plab nodes may be added in bulk.
        """
        # block out common termination signals while adding a node
        osigs = disable_sigs(TERMSIGS)
        defosid, controliface = self.__getNodetypeInfo()
        nidnum, priority = self.__nextFreeNodeid()
        nodeid = "%s%d" % (NODEPREFIX, nidnum)
        vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
        hostonly = nodeent['HNAME'].replace(".", "-")
        
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)

        try:
            DBQueryFatal("replace into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
                          'FREE_CLEAN',
                          'ISUP'))

            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (nodeid))

            DBQueryFatal("replace into widearea_nodeinfo"
                         " (node_id, contact_uid, hostname, site)"
                         " values (%s, %s, %s, %s)",
                         (nodeid, 'bnc', nodeent['HNAME'], nodeent['SITE']))

            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'fxp',
                          controliface, 'ctrl'))

            DBQueryFatal("replace into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))

            DBQueryFatal("replace into node_auxtypes"
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, nodeent['LINKTYPE'], 1))
            
            DBQueryFatal("replace into node_auxtypes"
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
            DBQueryFatal("replace into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (nodeid, 'down'))

            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))

            # Create a single reserved plab vnode for the managment sliver.
            # XXX I left it as "20" cause of all the existing ones.
            # XXXX I set it to 1 due to the above comment (correct?)
            #      since we are re-creating anyway.            
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
                pass
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
            # Put the last vnode created into the special monitoring expt.
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
            pass
        
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
                     "\n Please clean up!\n" % tbmsg, TBOPS)
            enable_sigs(osigs)
            raise

        # last but not least, unblock signals
        enable_sigs(osigs)
        return

    def __updateNode(self, updent):
        """
        Updates changed node attributes in the mapping table.
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}
        nodeid, chattrs = updent
        
        if len(chattrs) > 2:
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
                     "%s has been moved to hwdown." % (chattrs, nodeid)
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
            SENDMAIL(TBOPS,
                     "More than 2 plab node attrs have changed on %s" % nodeid,
                     errmsg,
                     TBOPS)
            raise RuntimeError, errmsg # XXX: maybe don't raise an exception.
        
        updstr = ",".join(map(lambda x: "%s='%s'" % (attrmap[x[0]],x[1][1]),
                              chattrs.items()))
        DBQueryFatal("update plab_mapping set " + updstr + " where node_id=%s",
                     (nodeid))
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

    def __getNodetypeInfo(self):
        """
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
        it doesn't change.
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
            if debug:
                print "Getting node type info"
                pass
            res = DBQueryFatal("select osid, control_iface"
                               " from node_types"
                               " where type = 'pcplabphys'")
            assert (len(res) == 1), "Failed to get node type info"
            (self.__getNodetypeInfoCache, ) = res
            pass
        
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
        if debug:
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
            pass
        finally:
            DBQueryFatal("unlock tables")
            pass
        
        return nodeid, priority

    def renew(self):
        """
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
        """

        now = int(time.mktime(time.gmtime()))
        res = DBQueryFatal("select pid, eid from plab_slices");
        
        loadedSlices = {}
        global failedrenew # XXX
        newfail = []
        failsoon = []
        ret = 0

        print "Renewing Plab leases at %s ..." % time.ctime()

        for (pid, eid) in res:

            try:
                slice = loadedSlices[(pid, eid)]
                pass
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
                pass
            
            res = slice.renew()
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
                if entry not in failedrenew:
                    newfail.append(entry)
                    pass
                if (slice.leaseend - now) < PLABEXPIREWARN:
                    failsoon.append(entry)
                    pass
                pass
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
                    pass
                    
        if newfail:
            failedrenew += newfail
            failstr = ""
            for n in newfail:
                failstr += "%s/%s (expires: %s UTC)\n" % \
                           (n[:2] + (time.ctime(n[2]),))
                pass
            
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
                     "Failed to renew the following leases:\n%s" %
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
            pass

        if failsoon:
            failstr = ""
            for n in failsoon:
                failstr += "%s/%s: (expires: %s UTC)\n" % \
                           (n[:2] + (time.ctime(n[2]),))
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
            pass
        
        return
    
    pass # end class Plab


#
# Slice abstraction
#

class Slice:

    def __init__(self, plab, pid, eid, slicename = None):
        self.plab = plab
        self.pid, self.eid = pid, eid
        self.slicemeta = None
        self.slicename = slicename
        return
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
            self.slicename = PLAB_SVC_SLICENAME
            adminbit = 1
            pass
        
        if not self.slicename:
            res = DBQueryFatal("select idx from experiments "
                               "where pid=%s "
                               "and eid=%s",
                               (self.pid, self.eid))
            if not len(res):
                raise RuntimeError, "Didn't get any results while looking for idx"
            eindex = res[0][0]
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass

        print "Creating Plab slice %s." % self.slicename

        try:
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)

            DBQueryFatal("insert into plab_slices"
                         " (pid, eid, slicename, slicemeta, leaseend, admin)"
                         " values (%s, %s, %s, %s, %s, %s)",
                         (self.pid, self.eid, self.slicename,
                          self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          adminbit))
            pass
        except:
            self.plab.agent.deleteSlice(self)
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
            raise

        return res

    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.

        XXX This should probably be made lazy, since not all operations
        really need it
        """
        if verbose:
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
        res = DBQueryFatal("select slicename, slicemeta, leaseend "
                           " from plab_slices "
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
        ((self.slicename, self.slicemeta, self.leaseend), ) = res
        self.leaseend = calendar.timegm(time.strptime(str(self.leaseend),
                                                      "%Y-%m-%d %H:%M:%S"))
        return

    def renew(self):
        """
        Renews slice lease.  We want this to be the maximum allowed by law..
        Store the expiration time in UTC.
        """
        print "Renewing lease for slice %s" % self.slicename

        try:
            ret = self.plab.agent.renewSlice(self)
            DBQueryFatal("update plab_slices "
                         " set slicemeta=%s, leaseend=%s "
                         " where slicename=%s",
                         (self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          self.slicename))
            pass
        except:
            print "slice.renew: Slice renewal failed:"
            traceback.print_exc()
            ret = 0
            pass
        
        return ret

    def destroy(self):
        """
        Frees all nodes in this slice and destroys the slice.  Note
        that this will really pound the DB if there are many nodes left
        in the slice, but those should be removed by Emulab before the
        slice is destroyed.
        """
        print "Destroying Plab slice %s." % self.slicename
        res = DBQueryFatal("select node_id from plab_slice_nodes"
                           " where slicename = %s",
                           (self.slicename))
        print "\tRemoving any remaining nodes in slice.."
        for (nodeid,) in res:
            node = self.loadNode(nodeid)
            node.free()
            del node  # Encourage the GC'er

        osigs = disable_sigs(TERMSIGS)

        try:
            self.plab.agent.deleteSlice(self)
            pass
        except:
            print "Failed to delete slice!"
            traceback.print_exc()
            pass
        
        try:
            print "\tRemoving slice DB entry."
            DBQueryFatal("delete from plab_slices where slicename = %s",
                         (self.slicename,))
        except:
            print "Error deleting slice from DB!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error deleting slice from DB",
                     "Slice deletion error:\n\n%s" % tbstr, TBOPS)
            enable_sigs(osigs)
            raise
        
        enable_sigs(osigs)

    def createNode(self, nodeid, force=False):
        """
        Node factory function
        """
        # XXX: KRW - The following is a hack to help me with testing.
        if not nodeid.startswith("plab"):
            IP = socket.gethostbyname(nodeid)
            qres = DBQueryFatal("select n.node_id from nodes as n left join "
                                "interfaces as i on n.node_id = i.node_id "
                                "where i.IP = %s", (IP,))
            assert (len(qres) > 0), "Node does not exist in DB: %s" % nodeid
            nodeid = qres[0][0] + "-20"
        node = Node(self, nodeid)
        node._create(force)
        return node

    def loadNode(self, nodeid):
        """
        Node factory function
        """
        node = Node(self, nodeid)
        node._load()
        return node

    def updateSliceMeta(self):
        """
        Grab current slice metadata from Planetlab and store in db
        """
        try:
            self.slicemeta = self.plab.agent.getSliceMeta(self)
            DBQueryFatal("update plab_slices set "
                         "slicemeta=%s where slicename=%s",
                         (self.slicemeta, self.slicename))
            pass
        except:
            print "Error updating slice metadata!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error updating slice metadata",
                     "Slice metadata update error:\n\n%s" % tbstr, TBOPS)
            raise

    def getSliceNodes(self):
        """
        Return a tuple containing the nodes that belong to this slice
        """