libplab.py.in 69.1 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004, 2006, 2007 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5 6 7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9 10 11 12 13 14 15 16 17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18 19 20
"""

import sys
21 22 23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
Kirk Webb committed
25
import traceback
26
import signal
27
import socket
28 29 30
import httplib
import xml.parsers.expat
import re
31
import calendar
32
import shlex
33

34
from popen2 import Popen4
35
from warnings import warn
36

37 38 39 40 41 42 43 44 45 46 47
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
48
from mod_PLCNM import mod_PLCNM
49
from mod_PLC4 import mod_PLC4
50 51

agents = {'PLC'    : mod_PLC,
52
          'dslice' : mod_dslice,
53 54
          'PLCNM'  : mod_PLCNM,
          'PLC4'   : mod_PLC4}
55

Kirk Webb's avatar
Kirk Webb committed
56 57 58 59 60 61 62
#
# Initialize the AOP stuff
#
from aspects import wrap_around
from timer_advisories import initTimeAdvice, timeAdvice
initTimeAdvice("plabtiming")

Kirk Webb's avatar
Kirk Webb committed
63 64 65 66 67 68
#
# output control vars
#
verbose = 0
debug = 0

69 70 71
#
# Constants
#
72
DEF_AGENT = "PLC4";
73 74 75 76
#
# Add a compatibility agent --- libplab will make all PLC calls on this agent
# in addition to the default agent.
#
77
COMPAT_AGENT = "PLCNM";
78
# if this is set, do all PLC calls on COMPAT_AGENT as well as DEF_AGENT
79
compat_mode = 0
80

81
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
82 83 84

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
85
NODEPROBEINT  = 30
86

87 88
USERNODE = "@USERNODE@"
TBOPS = "@TBOPSEMAIL_NOSLASH@"
89
MAILTAG = "@THISHOMEBASE@"
90
SLICE_ALIAS_DIR = "/etc/mail/plab-slice-addrs"
91

92
RESERVED_PID = "emulab-ops"
93
RESERVED_EID = "hwdown"       # start life in hwdown
94 95
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
96

Kirk Webb's avatar
Kirk Webb committed
97 98
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18",
                        "205.124.244.178", )
99 100 101
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
102
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
103
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
104 105 106

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
107

108 109 110 111
# 'critical' node identifiers - those that are actually used to uniquely
# identify a planetlab node
ATTR_CRIT_KEYS = ('HNAME', 'IP', 'PLABID', 'MAC',)

112 113 114 115
# The amount by which latitude and longitude are allowed to differ before we
# classify them ask changed
LATLONG_DELTA = 0.001

116
PLABNODE = "@prefix@/sbin/plabnode"
117
SSH = "@prefix@/bin/sshtb"
118
NAMED_SETUP = "@prefix@/sbin/named_setup"
119
PELAB_PUSH  = "@prefix@/sbin/pelab_opspush"
120

121 122 123 124
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
125 126
IGNORED_NODES_FILE = "@prefix@/etc/plab/IGNOREDNODES"
ALLOWED_NODES_FILE = "@prefix@/etc/plab/ALLOWEDNODES"
127

128
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
129
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
130 131 132 133
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
134
PLAB_SVC_SLICENAME = "utah_svc_slice"
135 136 137 138
PLAB_SVC_SLICEDESC = "Emulab management service slice. Performs periodic " \
                     "checkins with Emulab central, and routes events for " \
                     "other Emulab slices. Slivers in this slice should " \
                     "only interact with other PlanetLab machines, and Emulab."
139 140
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"
141
DEF_SLICE_DESC = "Slice created by Emulab"
142

143 144
PLABEXPIREWARN = 1*WEEK        # one week advance warning for slice expiration.
NODEEXPIREWARN = 2*WEEK+2*DAY  # about two weeks advance warning for slivers.
145

146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
#
# This is a trigger table for dist'ing out multiple rootballs depending on
# which NM is running on the node.  Since we don't know which version is
# running on a node until we call into (DEF|COMPAT)_AGENT and actually connect
# to the node, it's based off classnames.
#
# The table specifies a prefix to the default rootball name; this rootball
# should be placed in the normal location.
#
# If the plcagent class and nodeagent class are both not in the trigger table,
# we simply push the default rootball.
#
# Note that we could store version info in the database, but that doesn't
# really have long-term value.  Plus, we have to be checking sites.xml all the
# time to catch version changes during rollout.  This way, plabmonitord and
# the web interface know exactly what is the version whenever a node is
# contacted for setup.
#
# For now, only NM4agent has a custom tarball, since v4 nodes are in the
# distinct minority right now.
#
from mod_PLC4 import NM4agent

rootball_triggers = { mod_PLC4 : { NM4agent : 'NM4-' } }

171 172 173 174 175
#
# var to track failed renewals
#
failedrenew = []

176 177 178 179 180
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

181 182 183 184 185
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

186

187 188 189
#
# Plab abstraction
#
190

191 192 193 194 195 196 197 198 199 200
#
# Multiple attribute change exception
#
class MultiChangeError(Exception):
    def __init__(self, nodeid, chattrs={}):
        self.nodeid = nodeid
        self.chattrs = chattrs
        pass
    pass

Kirk Webb's avatar
Kirk Webb committed
201

Kirk Webb's avatar
Kirk Webb committed
202 203 204
#
# Class responsible for parsing planetlab sites file
#
205 206 207 208 209 210 211 212
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
213 214
        self.__latitude = 0
        self.__longitude = 0
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
242 243 244
            if attrs.has_key('LATITUDE'):
                self.__latitude = attrs['LATITUDE']
            else:
245
                self.__latitude = 0
246 247 248 249
            if attrs.has_key('LONGITUDE'):
                self.__longitude = attrs['LONGITUDE']
            else:
                self.__longitude = 0
250 251 252
            pass
        
        elif name == "HOST":
253 254 255
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
256 257 258 259 260 261 262 263 264 265 266 267 268
            if not attrs.has_key('BWLIMIT'):
                attrs['BWLIMIT'] = "-1"
                pass
            self.__hosts.append({
                'HNAME'     : attrs['NAME'],
                'IP'        : attrs['IP'],
                'PLABID'    : attrs['NODE_ID'],
                'MAC'       : attrs['MAC'],
                'BWLIMIT'   : attrs['BWLIMIT'],
                'SITE'      : self.__sitename,
                'LATITUDE'  : self.__latitude,
                'LONGITUDE' : self.__longitude
                })
269 270 271 272 273 274 275 276 277 278 279 280
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
281 282
            self.__latitude = 0
            self.__longitude = 0
283 284 285
            pass
        return

Kirk Webb's avatar
Kirk Webb committed
286

287
class Plab:
Kirk Webb's avatar
Kirk Webb committed
288
    def __init__(self, agent=None):
289 290 291
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
292 293
        if compat_mode:
            self.compat_agent = agents[COMPAT_AGENT]()
Kirk Webb's avatar
Kirk Webb committed
294
        if debug:
295
            print "Using module: %s" % self.agent.modname
296 297
            if compat_mode:
                print "COMPAT: Using module: %s" % self.compat_agent.modname
298 299
            pass
        pass
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

317
    def updateNodeEntries(self, ignorenew = False):
318
        """
319
        Finds out which Plab nodes are available, and
320 321 322 323 324
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
325
        discovered.
326 327 328 329 330 331

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
332 333 334
        
        print "Getting available Plab nodes ..."

335
        avail = []
336
        try:
337 338 339
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
340
        # XXX: rewrite to use more elegant exception info gathering.
341 342
        except:
            extype, exval, extrace = sys.exc_info()
343
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
344
            if debug:
345 346 347
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
348

Kirk Webb's avatar
Kirk Webb committed
349
        if debug:
350 351
            print "Got advertisement list:"
            print avail
352
            pass
353

354 355 356
        ignored_nodes = self.__readNodeFile(IGNORED_NODES_FILE)
        allowed_nodes = self.__readNodeFile(ALLOWED_NODES_FILE)

357 358 359 360 361 362
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
363
        if len(allowed_nodes) or len(ignored_nodes):
364
            allowed = []
365
            for nodeent in avail:
366
                if nodeent['PLABID'] in ignored_nodes:
367
                    continue
368 369
                elif len(allowed_nodes):
                    if nodeent['IP'] in allowed_nodes:
370 371 372 373 374
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
375 376
                    pass
                pass
377 378 379 380
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
381
            pass
382

383 384 385 386 387 388 389 390 391 392 393
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

394
        # Get node info we already have.
395
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
396
        if debug:
397 398
            print "Got known pnodes:"
            print known
399
            pass
400

Kirk Webb's avatar
Kirk Webb committed
401
        # Create list of nodes to add or update
402 403
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
404
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
405 406
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
407
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
408 409 410
            # Determine if we already know about this node.
            matchres = self.__matchPlabNode(nodeent, known)
            if not matchres:
411
                toadd.append(nodeent)
412
                pass
413 414
            elif len(matchres[1]):
                toupdate.append((nodeent,matchres))
415
                pass
416
            pass
Kirk Webb's avatar
Kirk Webb committed
417

418 419
        # Process the list of nodes to add
        addstr = ""
420
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
421
            # Are we ignoring new entries?
422
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
423
                if verbose:
424
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
425 426
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
427
            # If not ignoring, do the addition/update.
428
            else:
429 430
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
431
                    # Get the linktype here so we can report it in email.
432
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
433
                    if debug:
434 435
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
436
                        pass
437 438 439
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
440
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
441
                              (nodeent['PLABID'],
442 443 444 445
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
446
                    addstr += nodestr
447
                    pass
448 449
                pass
            pass
450

451
        # Process node updates.
452
        updstr = ""
453
        chgerrstr = ""
454 455
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
456
            for (nodeent,(nodeid,diffattrs)) in toupdate:
457
                try:
458
                    self.__updateNodeMapping(nodeid, diffattrs)
459 460 461 462 463
                    pass
                except MultiChangeError, e:
                    print "%s not updated: Too many attribute changes." % \
                          e.nodeid
                    chgerrstr += "%s:\n" % e.nodeid
464 465
                    for (attr,(old,new)) in e.chattrs.items():
                        chgerrstr += "\t%s:\t%s => %s\n" % (attr,old,new)
466 467 468
                        pass
                    chgerrstr += "\n"
                    continue
469
                self.__updateNode(nodeid, nodeent)
470
                # Add a line for the add/update message.
471 472 473
                nodestr = nodeid + "\n"
                for (attr,(old,new)) in diffattrs.items():
                    nodestr += "\t%s:\t%s => %s\n" % (attr,old,new)
474 475
                    pass
                updstr += nodestr + "\n"
476 477
                pass
            pass
478

479 480 481 482 483 484 485 486
        if chgerrstr:
            SENDMAIL(TBOPS,
                     "Two or more changes detected for some plab nodes",
                     "Two or more distinguishing attributes have changed "
                     "on the following planetlab nodes:\n\n%s\n" % chgerrstr,
                     TBOPS)
            pass

487 488 489 490
        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
491 492
            print "Pushing out site_mapping ..."
            os.spawnl(os.P_WAIT, PELAB_PUSH, PELAB_PUSH)
493 494 495 496 497 498 499 500 501 502 503 504
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
505
        return
506

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
525 526 527
                elif (attr == "LATITUDE") or (attr == "LONGITUDE"):
                    # Special rules for latitude and longitude to avoid
                    # FP errors
528 529 530 531 532 533
                    if (ent[attr] != None and plabent[attr] != None) \
                           and (ent[attr] != "" and plabent[attr] != "") \
                           and ((float(ent[attr]) > \
                                 (float(plabent[attr]) + LATLONG_DELTA)) \
                                or (float(ent[attr]) < \
                                    (float(plabent[attr]) - LATLONG_DELTA))):
534 535 536 537
                        diff[attr] = (ent[attr], plabent[attr])
                    else:
                        same[attr] = ent[attr]
                        pass
538 539 540 541 542 543 544
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
545 546
            # Only consider these to be the same if at least one 'critical'
            # attr is the same
547
            if len(same):
548 549 550
                for attr in same:
                    if attr in ATTR_CRIT_KEYS:
                        return (nid, diff)
551 552 553
            pass
        return ()

554 555 556 557 558
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
559 560
        res = DBQueryFatal("select plab_mapping.node_id,plab_id,"
                           "plab_mapping.hostname,IP,mac,site,latitude,"
561
                           "longitude,bwlimit"
562 563 564 565
                           " from plab_mapping"
                           " left join widearea_nodeinfo on"
                           "    plab_mapping.node_id = "
                           "    widearea_nodeinfo.node_id")
566
        
567
        ret = {}
568 569
        for (nodeid, plabid, hostname, ip, mac, site,
             latitude, longitude, bwlimit) in res:
570 571 572 573 574 575
            ret[nodeid] = {'PLABID'    : plabid,
                           'HNAME'     : hostname,
                           'IP'        : ip,
                           'MAC'       : mac,
                           'SITE'      : site,
                           'LATITUDE'  : latitude,
576 577
                           'LONGITUDE' : longitude,
                           'BWLIMIT'   : bwlimit}
578
            pass
579 580 581 582 583 584 585 586 587 588
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
589
        return ret
590 591 592 593 594 595 596 597 598

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        attrs = {}
        dups = {}
        
        for ent in nodelist:
599
            for attr in ATTR_CRIT_KEYS:
600 601 602 603 604 605 606 607 608 609 610 611 612 613
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
614
        
615
    def __findLinkType(self, nodeent):
616 617 618 619 620 621 622
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

623
        This can't detect DSL links..
624
        """
625
        # Is host international (or flux/emulab local)?
626
        from socket import gethostbyaddr, getfqdn, herror
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
649
        # Is host on I2?
650
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
651 652 653 654 655
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
656 657
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
658

659 660
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
661 662
                nodeent['LINKTYPE'] = "pcplabinet"
                return
663
        else:
664
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
665

Kirk Webb's avatar
Kirk Webb committed
666 667
        # We don't know - must manually classify.
        nodeent['LINKTYPE'] = "*Unknown*"
668
        return
669

670
    def __addNode(self, nodeent):
671
        """
672 673 674
        updateNodeEntries() helper function.  Adds a new Plab pnode and
        associated vnode to the DB.  The argument is a dictionary containing
        the new node's attributes.
675
        """
676
        # Generate/grab variables to be used when creating the node.
677
        defosid, controliface = self.__getNodetypeInfo()
678
        hostonly = nodeent['HNAME'].replace(".", "-")
679 680 681 682 683 684 685 686 687 688 689 690
        nidnum, priority = self.__nextFreeNodeid()
        nodeid = "%s%d" % (NODEPREFIX, nidnum)
        vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)

        # Do the stuff common to both node addition and update first
        # Note that if this fails, we want the exception generated to
        # percolate up to the caller immediately, so don't catch it.
        self.__updateNode(nodeid, nodeent)

        # Now perform stuff specific to node addition
691
        try:
692 693 694
            res_exptidx = TBExptIDX(RESERVED_PID, RESERVED_EID)
            mon_exptidx = TBExptIDX(MONITOR_PID, MONITOR_EID)
            
695
            DBQueryFatal("replace into nodes"
696
                         " (node_id, type, phys_nodeid, role, priority,"
697
                         "  op_mode, def_boot_osid,"
698
                         "  allocstate, allocstate_timestamp,"
699
                         "  eventstate, state_timestamp)"
700
                         " values (%s, %s, %s, %s, %s,"
701 702 703 704
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
705
                          'FREE_CLEAN',
706
                          'ISUP'))
707

708 709 710
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
711
                         (nodeid))
712

713
            DBQueryFatal("replace into reserved"
714
                         " (node_id, exptidx, pid, eid, rsrv_time, vname)"
715
                         " values (%s, %s, %s, %s, now(), %s)",
716 717
                         (nodeid, res_exptidx,
                          RESERVED_PID, RESERVED_EID, hostonly))
718

719 720
            # XXX: This should probably be checked and updated if necessary
            #      when updating.
721
            DBQueryFatal("replace into node_auxtypes"
722 723
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
724
                         (nodeid, nodeent['LINKTYPE'], 1))
725
            
726
            DBQueryFatal("replace into node_auxtypes"
727 728 729 730
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
731
            DBQueryFatal("replace into node_status"
732 733
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
734
                         (nodeid, 'down'))
735

736 737 738 739 740
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
741

742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
            #
            # NowAdd the site_mapping entry for this node.
            #
            
            # See if we know about the associated site - grab idx if so
            siteidx = 0
            nodeidx = 1
            siteres = DBQueryFatal("select site_idx, node_idx from "
                                   " plab_site_mapping where site_name=%s",
                                   nodeent['SITE']);
            if len(siteres):
                # There are already nodes listed for this site, so get
                # the next node id.
                siteidx = siteres[0][0]
                for (foo, idx) in siteres:
                    if idx > nodeidx: nodeidx = idx
                    pass
                nodeidx += 1
                pass
            else:
                # No nodes listed for site, so get the largest site_idx
                # in the DB so far, and increment cuz we're going to add
                # a new one.
                maxres = DBQueryFatal("select MAX(site_idx) from "
                                      " plab_site_mapping")
                try:
                    siteidx = int(maxres[0][0]) + 1
                    pass
                except ValueError:
                    siteidx = 1
                    pass
                pass
            # Create site_mapping entry, optionally creating new site idx
            # via not specifying the site_idx field (field is auto_increment)
            DBQueryFatal("insert into plab_site_mapping "
                         " values (%s, %s, %s, %s)",
                         (nodeent['SITE'], siteidx, nodeid, nodeidx))

780
            # Create a single reserved plab vnode for the managment sliver.
781 782 783 784 785
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
786 787
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
788
                pass
789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
812
            # Put the last vnode created into the special monitoring expt.
813
            DBQueryFatal("insert into reserved"
814
                         " (node_id, exptidx, pid, eid, rsrv_time, vname)"
815
                         " values (%s, %s, %s, %s, now(), %s)",
816 817
                         (vnodeid, mon_exptidx,
                          MONITOR_PID, MONITOR_EID, vnodeid))
818 819
            pass
        
820 821 822 823 824 825
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
826
                     "\n Please clean up!\n" % tbmsg, TBOPS)
827
            raise
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
        return


    def __updateNode(self, nodeid, nodeent):
        """
        updateNodeEntries() helper function.  Updates attributes for plab
        nodes passed in via the nodeent argument.
        """
        # Get the name of the control interface for plab nodes.
        defosid, controliface = self.__getNodetypeInfo()

        haslatlong = (('LATITUDE' in nodeent and 'LONGITUDE' in nodeent) and
            (nodeent['LATITUDE'] != 0 or nodeent['LONGITUDE'] != 0))
        try:
            DBQueryFatal("replace into widearea_nodeinfo"
843 844 845 846 847
                         " (node_id, contact_uid, contact_idx, hostname, site,"
                         "  latitude, longitude, bwlimit)"
                         " values (%s, %s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 'nobody', '0', nodeent['HNAME'],
                          nodeent['SITE'],
848 849 850 851
                          # Poor man's ternary operator
                          haslatlong and nodeent['LATITUDE'] or "NULL",
                          haslatlong and nodeent['LONGITUDE'] or "NULL",
                          nodeent['BWLIMIT']))
852

853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'plab_fake',
                          controliface, 'ctrl'))
            pass
        except:
            print "Error updating PLAB node in DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error updating plab node in DB: %s\n" % nodeid,
                     "Some operation failed while trying to update"
                     " plab node %s in the DB:\n\n%s"
                     "\nPlease clean up!\n" % (nodeid, tbmsg), TBOPS)
            raise
868
        return
869

870

871
    def __updateNodeMapping(self, nodeid, chattrs):
872
        """
873
        Updates changed node attributes in the plab mapping table.
874 875 876
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
877

878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
        # mapping from attrs to column names
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}

        # Get the intersection of mapping (critical) keys with those that
        # have changed.
        changedcritkeys = set(ATTR_CRIT_KEYS) & set(chattrs.keys())
        # nothing to do if none of the mapping attributes have changed.
        if not changedcritkeys:
            return
        # If the node has more than two critical attrs that have changed,
        # then move it to hwdown and raise an exception.
        if len(changedcritkeys) > 2:
            crattrs = {}
            for chkey in changedcritkeys:
                crattrs[chkey] = chattrs[chkey]
896 897
                pass
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
898
                     "%s has been moved to hwdown." % (crattrs, nodeid)
899 900
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
901 902 903 904 905 906 907
            raise MultiChangeError(nodeid, crattrs)

        # Update mapping table entry.
        updstr = ",".join(map(lambda x: "%s='%s'" %
                              (attrmap[x],chattrs[x][1]), changedcritkeys))
        DBQueryFatal("update plab_mapping set %s where node_id='%s'" %
                     (updstr, nodeid))
908 909 910 911 912 913 914
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

915 916
    def __getNodetypeInfo(self):
        """
917 918
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
919
        it doesn't change.
920 921
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
922
            if debug:
923
                print "Getting node type info"
924
                pass
925 926 927 928 929 930 931 932 933

            dbres = DBQueryFatal("select attrkey,attrvalue "
                                 " from node_type_attributes as a "
                                 " where type = 'pcplabphys' and "
                                 "       (a.attrkey='default_osid' or "
                                 "        a.attrkey='control_interface') "
                                 " order by attrkey")
            
            assert (len(dbres) == 2), "Failed to get node type info"
Kirk Webb's avatar
Kirk Webb committed
934 935
            attrdict = {}
            for attrkey, attrvalue in dbres:
Kirk Webb's avatar
Kirk Webb committed
936
                attrdict[attrkey] = attrvalue;
Kirk Webb's avatar
Kirk Webb committed
937 938
                pass
            self.__getNodetypeInfoCache = \
Kirk Webb's avatar
Kirk Webb committed
939 940
                                        (attrdict["default_osid"],
                                         attrdict["control_interface"])
941 942
            pass
        
943 944 945 946 947 948 949
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
950
        if debug:
951 952 953 954 955 956 957 958 959 960
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
961
            pass
962 963
        finally:
            DBQueryFatal("unlock tables")
964 965
            pass
        
966 967
        return nodeid, priority

968 969 970 971 972 973 974 975 976 977 978 979 980
    def __readNodeFile(self, filename):
        """
        Helper function - read in list of nodes from a file, seperated
        by arbitrary amounts of whitespace.  No comments allowed.
        """
        nodelist = []
        if os.access(filename, os.F_OK):
            nodefile = open(filename, "r+")
            nodelist = nodefile.read().split()
            nodefile.close()
            pass
        return nodelist

981
    def renew(self, inpid = None, ineid = None, force = False):
982
        """
983 984 985
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
986
        """
987

988 989
        global failedrenew # XXX
        
990 991 992 993 994 995 996 997 998 999
        now = int(time.time())
        
        if not inpid:
            res = DBQueryFatal("select pid, eid from plab_slices");
            pass
        else:
            if not ineid:
                raise RuntimeError, "renew: Must provide eid with pid."
            res = ((inpid, ineid),)
            pass
Kirk Webb's avatar
Kirk Webb committed
1000
        
1001
        loadedSlices = {}
1002 1003
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
1004 1005
        ret = 0

1006
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
1007

1008
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
1009

1010 1011
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
1012
                pass
1013 1014 1015
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
1016
                pass
1017
            
1018
            res = slice.renew(force)
1019 1020 1021 1022 1023
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
1024 1025
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
1026
                    pass
1027
                if (slice.leaseend - now) < PLABEXPIREWARN:
1028
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
1029 1030
                    pass
                pass
1031 1032 1033
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
1034
                    pass
Kirk Webb's avatar
Kirk Webb committed
1035
                    
1036 1037
        if newfail:
            failedrenew += newfail
1038
            failstr = ""
1039
            for n in newfail:
1040
                failstr += "%s/%s (expires: %s UTC)\n" % \
1041
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
Kirk Webb committed
1042
                pass
1043
            
1044
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
1045
                     "Failed to renew the following leases:\n%s" %
1046
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1047
            pass
1048

1049 1050 1051
        if failsoon:
            failstr = ""
            for n in failsoon:
1052
                failstr += "%s/%s: (expires: %s UTC)\n" % \
1053
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
1054 1055 1056 1057
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1058
            pass
1059

Kirk Webb's avatar
Kirk Webb committed
1060 1061 1062
        return
    
    pass # end class Plab
Kirk Webb's avatar
Kirk Webb committed
1063 1064
# AOP wrappers for class Plab
wrap_around(Plab.createSlice, timeAdvice)
1065 1066 1067 1068 1069

#
# Slice abstraction
#
class Slice:
1070 1071

    def __init__(self, plab, pid, eid, slicename = None):
1072 1073
        self.plab = plab
        self.pid, self.eid = pid, eid
1074
        self.slicemeta = None
1075
        self.exptidx = None
1076
        self.slicename = slicename
1077
        self.description = DEF_SLICE_DESC
1078
        return
1079 1080 1081 1082 1083 1084
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
1085 1086 1087

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
1088 1089
            self.slicename   = PLAB_SVC_SLICENAME
            self.description = PLAB_SVC_SLICEDESC
1090 1091
            adminbit = 1
            pass
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101

        res = DBQueryFatal("select idx, expt_name from experiments "
                           "where pid=%s "
                           "and eid=%s",
                           (self.pid, self.eid))
        if not len(res):
            raise RuntimeError, \
                  "Didn't get any results while looking up info on " \
                  "experiment %s/%s" % (self.pid, self.eid)
        (eindex, descr) = res[0]
1102
        
1103 1104 1105
        if not self.slicename:
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass
1106

1107 1108 1109
        self.description = descr
        self.exptidx = eindex

1110
        print "Creating Plab slice %s." % self.slicename
1111

1112
        try:
Kirk Webb's avatar
Kirk Webb committed
1113 1114
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
            
            # Do compat calls right after main agent calls.
            if compat_mode:
                try:
                    self.plab.compat_agent.createSlice(self)
                except:
                    print "WARNING: compat agent failed in createSlice; " \
                          "\n  watch for inconsistent DB state!"
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
1125

1126 1127
            DBQueryFatal("insert into plab_slices "
                         "(exptidx,pid,eid,slicename,slicemeta,leaseend,admin)"
1128
                         " values (%s, %s, %s, %s, %s, %s, %s)",
1129
                         (self.exptidx, self.pid, self.eid, self.slicename,
1130 1131 1132 1133
                          self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          adminbit))
1134 1135
            pass
        except:
Kirk Webb's avatar
Kirk Webb committed
1136
            self.plab.agent.deleteSlice(self)
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
            
            if compat_mode:
                try:
                    self.plab.compat_agent.deleteSlice(self)
                except:
                    print "WARNING: compat agent failed in deleteSlice; " \
                          "\n  watch for inconsistent DB state!"
                    pass
                pass

1147 1148
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
1149 1150
            raise

1151 1152 1153 1154 1155 1156
        # Setup mailing alias for the slice.  All mail currently filters
        # in via the 'emulabman' alias.  A procmail filter there will
        # redirect it to the appropriate user.
        try:
            qres = DBQueryFatal("select u.uid, u.usr_email from users as u "
                                "left join experiments as e "
1157
                                "on u.uid_idx = e.swapper_idx "
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
                                "where e.pid=%s and e.eid=%s",
                                (self.pid, self.eid))
            if not len(qres):
                raise RuntimeError, \
                      "Didn't get any results while looking up user info " \
                      "for experiment %s/%s" % (self.pid, self.eid)
            (username, usremail) = qres[0]
            command = "%s -host %s /bin/echo %s \> %s/%s" % \
                      (SSH, USERNODE, usremail,
                       SLICE_ALIAS_DIR, self.slicename)
            os.system(command)
        except:
            print "Could not setup email alias for slice: %s!" % self.slicename
            traceback.print_exc()
            pass

1174 1175
        return res

1176 1177 1178 1179
    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.
1180 1181 1182

        XXX This should probably be made lazy, since not all operations
        really need it
1183
        """
Kirk Webb's avatar
Kirk Webb committed
1184
        if verbose:
1185
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
            pass

        # grab our exptidx
        res = DBQueryFatal("select idx, expt_name from experiments "
                           "where pid=%s "
                           "and eid=%s",
                           (self.pid, self.eid))
        if not len(res):
            raise RuntimeError, \
                  "Didn't get any results while looking up info on " \
                  "experiment %s/%s" % (self.pid, self.eid)
        (eindex, descr) = res[0]
        
        if not self.slicename:
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass
        
        self.description = descr
        self.exptidx = eindex
            
1206 1207
        res = DBQueryFatal("select slicename, slicemeta, leaseend "
                           " from plab_slices "
1208 1209 1210 1211 1212 1213
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
1214 1215
        ((self.slicename, self.slicemeta, leaseend), ) = res
        self.leaseend = calendar.timegm(time.strptime(str(leaseend),
1216 1217
                                                      "%Y-%m-%d %H:%M:%S"))
        return
1218

1219
    def renew(self, force = False):
Kirk Webb's avatar
Kirk Webb committed
1220
        """
1221
        Renews slice lease.  We want this to be the maximum allowed by law...
1222
        Store the expiration time in UTC.
Kirk Webb's avatar
Kirk Webb committed
1223 1224 1225 1226
        """
        print "Renewing lease for slice %s" % self.slicename

        try:
1227
            ret = self.plab.agent.renewSlice(self, force)
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237

            if compat_mode:
                try:
                    self.plab.compat_agent.renewSlice(self)
                except:
                    print "WARNING: compat agent failed in renewSlice; " \
                          "\n  watch for inconsistent DB state!"
                    pass
                pass
            
1238 1239 1240 1241 1242 1243 1244
            DBQueryFatal("update plab_slices "
                         " set slicemeta=%s, leaseend=%s "
                         " where slicename=%s",
                         (self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          self.slicename))
Kirk Webb's avatar
Kirk Webb committed
1245 1246
            pass
        except:
1247
            print "slice.renew: Slice renewal failed:"
Kirk Webb's avatar
Kirk Webb committed
1248 1249 1250
            traceback.print_exc()
            ret = 0
            pass
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289

        # Renew individual slivers, if necessary.  Any that fail to renew
        # and are close to expiration need to be noted.  They will be
        # reported after renewal has been attempted on all nodes.
        now = int(time.time())
        nodes = map(lambda x: x[0], self.getSliceNodes())
        reportfailed = []
        for nodeid in nodes:
            try:
                node = self.loadNode(nodeid)
                pass
            except AssertionError:
                print "Node %s doesn't really exist in %s" % \
                      (nodeid, self.slicename)
                continue
            if node.renew():
                print "Failed to renew node: %s (expires %s UTC)" % \
                      (nodeid, time.asctime(time.gmtime(node.leaseend)))
                print "Timediff: %s" % (node.leaseend - now)
                if node.leaseend - now < NODEEXPIREWARN:
                    reportfailed.append((nodeid, node.leaseend))
                    pass
                pass
            del node
            pass

        # Report any nodes that are near to expiration
        if len(reportfailed) > 0:
            tbstr = ""
            for nodeid, leaseend in reportfailed:
                tbstr += "Node: %s, Leaseend: %s UTC\n" % \
                         (nodeid, time.asctime(time.gmtime(leaseend)))
                pass
            SENDMAIL(TBOPS, "Plab nodes in danger of expiration: %s/%s" % \
                     (self.pid, self.eid),
                     "The following slivers in %s/%s will expire "
                     "soon:\n\n%s" % \
                     (self.pid, self.eid, tbstr),
                     TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1290
        
1291
        return ret
Kirk Webb's avatar
Kirk Webb committed
1292

1293 1294 1295 1296 1297 1298 1299
    def destroy(self):
        """
        Frees all nodes in this slice and destroys the slice.  Note
        that this will really pound the DB if there are many nodes left
        in the slice, but those should be removed by Emulab before the
        slice is destroyed.
        """
1300
        print "Destroying Plab slice %s." % self.slicename
1301 1302 1303 1304 1305 1306 1307 1308
        res = DBQueryFatal("select node_id from plab_slice_nodes"
                           " where slicename = %s",
                           (self.slicename))
        print "\tRemoving any remaining nodes in slice.."
        for (nodeid,) in res:
            node = self.loadNode(nodeid)
            node.free()
            del node  # Encourage the GC'er
1309

1310
        osigs = disable_sigs(TERMSIGS)
1311

1312
        try:
Kirk Webb's avatar
Kirk Webb committed
1313
            self.plab.agent.deleteSlice(self)
1314 1315 1316 1317 1318
            pass
        except:
            print "Failed to delete slice!"
            traceback.print_exc()
            pass
1319 1320 1321 1322 1323 1324 1325 1326 1327

        if compat_mode:
            try:
                self.plab.compat_agent.deleteSlice(self)
            except:
                print "WARNING: compat agent failed in deleteSlice; " \
                      "\n  watch for inconsistent DB state!"
                pass
            pass
1328
        
1329
        try:
1330
            print "\tRemoving slice DB entry."
1331 1332 1333 1334 1335 1336
            DBQueryFatal("delete from plab_slices where slicename = %s",
                         (self.slicename,))
        except:
            print "Error deleting slice from DB!"
            tbstr = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error deleting slice from DB",
1337
                     "Slice deletion error:\n\n%s" % tbstr, TBOPS)
1338 1339
            enable_sigs(osigs)
            raise
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350

        try:
            command = "%s -host %s /bin/rm -f %s/%s" % \
                      (SSH, USERNODE, SLICE_ALIAS_DIR, self.slicename)
            os.system(command)
        except:
            print "Could not remove email alias for slice: %s!" % \
                  self.slicename
            traceback.print_exc()
            pass
            
1351 1352
        
        enable_sigs(osigs)
1353

1354
    def createNode(self, nodeid, force=False):
1355 1356 1357
        """
        Node factory function
        """
1358 1359 1360 1361 1362 1363 1364 1365
        # XXX: KRW - The following is a hack to help me with testing.
        if not nodeid.startswith("plab"):
            IP = socket.gethostbyname(nodeid)
            qres = DBQueryFatal("select n.node_id from nodes as n left join "
                                "interfaces as i on n.node_id = i.node_id "
                                "where i.IP = %s", (IP,))
            assert (len(qres) > 0), "Node does not exist in DB: %s" % nodeid
            nodeid = qres[0][0] + "-20"
1366
        node = Node(self, nodeid)
1367
        node._create(force)
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
        return node

    def loadNode(self, nodeid):
        """
        Node factory function
        """
        node = Node(self, nodeid)
        node._load()
        return node

1378 1379 1380 1381 1382
    def updateSliceMeta(self):
        """
        Grab current slice metadata from Planetlab and store in db
        """
        try:
1383 1384 1385
            # XXX: no handling of compat agent here---this is about getting
            # a ticket to the local NM---and there is only one NM per node,
            # so having a compat call here doesn't make a difference.
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397