libplab.py.in 59.8 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2 3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004, 2006, 2007 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5 6 7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9 10 11 12 13 14 15 16 17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18 19 20
"""

import sys
21 22 23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
 
Kirk Webb committed
25
import traceback
Kirk Webb's avatar
 
Kirk Webb committed
26
import signal
Kirk Webb's avatar
 
Kirk Webb committed
27
import socket
Kirk Webb's avatar
Kirk Webb committed
28 29 30
import httplib
import xml.parsers.expat
import re
Kirk Webb's avatar
 
Kirk Webb committed
31
import calendar
Kirk Webb's avatar
 
Kirk Webb committed
32
import shlex
Kirk Webb's avatar
 
Kirk Webb committed
33

Kirk Webb's avatar
 
Kirk Webb committed
34
from popen2 import Popen4
Kirk Webb's avatar
 
Kirk Webb committed
35
from warnings import warn
36

37 38 39 40 41 42 43 44 45 46 47
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
Kirk Webb's avatar
 
Kirk Webb committed
48
from mod_PLCNM import mod_PLCNM
49 50

agents = {'PLC'    : mod_PLC,
Kirk Webb's avatar
 
Kirk Webb committed
51 52
          'dslice' : mod_dslice,
          'PLCNM'  : mod_PLCNM}
53

Kirk Webb's avatar
 
Kirk Webb committed
54 55 56 57 58 59 60
#
# Initialize the AOP stuff
#
from aspects import wrap_around
from timer_advisories import initTimeAdvice, timeAdvice
initTimeAdvice("plabtiming")

Kirk Webb's avatar
Kirk Webb committed
61 62 63 64 65 66
#
# output control vars
#
verbose = 0
debug = 0

67 68 69
#
# Constants
#
Kirk Webb's avatar
 
Kirk Webb committed
70
DEF_AGENT = "PLCNM";
71

72
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
Kirk Webb's avatar
 
Kirk Webb committed
73 74 75

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
76
NODEPROBEINT  = 30
77

Kirk Webb's avatar
 
Kirk Webb committed
78 79
USERNODE = "@USERNODE@"
TBOPS = "@TBOPSEMAIL_NOSLASH@"
80
MAILTAG = "@THISHOMEBASE@"
Kirk Webb's avatar
 
Kirk Webb committed
81
SLICE_ALIAS_DIR = "/etc/mail/plab-slice-addrs"
82

83
RESERVED_PID = "emulab-ops"
84
RESERVED_EID = "hwdown"       # start life in hwdown
85 86
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
87

Kirk Webb's avatar
 
Kirk Webb committed
88 89
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18",
                        "205.124.244.178", )
Kirk Webb's avatar
 
Kirk Webb committed
90 91 92
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
93
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
94
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
Kirk Webb's avatar
 
Kirk Webb committed
95 96 97

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
98

99 100 101 102
# 'critical' node identifiers - those that are actually used to uniquely
# identify a planetlab node
ATTR_CRIT_KEYS = ('HNAME', 'IP', 'PLABID', 'MAC',)

103 104 105 106
# The amount by which latitude and longitude are allowed to differ before we
# classify them ask changed
LATLONG_DELTA = 0.001

Kirk Webb's avatar
 
Kirk Webb committed
107
PLABNODE = "@prefix@/sbin/plabnode"
108
SSH = "@prefix@/bin/sshtb"
109
NAMED_SETUP = "@prefix@/sbin/named_setup"
Kirk Webb's avatar
 
Kirk Webb committed
110
PELAB_PUSH  = "@prefix@/sbin/pelab_opspush"
111

Kirk Webb's avatar
Kirk Webb committed
112 113 114 115
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
Kirk Webb's avatar
 
Kirk Webb committed
116 117
IGNORED_NODES_FILE = "@prefix@/etc/plab/IGNOREDNODES"
ALLOWED_NODES_FILE = "@prefix@/etc/plab/ALLOWEDNODES"
118

119
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
120
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
Kirk Webb's avatar
Kirk Webb committed
121 122 123 124
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
Kirk Webb's avatar
 
Kirk Webb committed
125
PLAB_SVC_SLICENAME = "utah_svc_slice"
Kirk Webb's avatar
 
Kirk Webb committed
126 127 128 129
PLAB_SVC_SLICEDESC = "Emulab management service slice. Performs periodic " \
                     "checkins with Emulab central, and routes events for " \
                     "other Emulab slices. Slivers in this slice should " \
                     "only interact with other PlanetLab machines, and Emulab."
Kirk Webb's avatar
 
Kirk Webb committed
130 131
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"
Kirk Webb's avatar
 
Kirk Webb committed
132
DEF_SLICE_DESC = "Slice created by Emulab"
Kirk Webb's avatar
 
Kirk Webb committed
133

Kirk Webb's avatar
 
Kirk Webb committed
134 135
PLABEXPIREWARN = 1*WEEK        # one week advance warning for slice expiration.
NODEEXPIREWARN = 2*WEEK+2*DAY  # about two weeks advance warning for slivers.
136

137 138 139 140 141
#
# var to track failed renewals
#
failedrenew = []

142 143 144 145 146
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

Kirk Webb's avatar
 
Kirk Webb committed
147 148 149 150 151
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

152

153 154 155
#
# Plab abstraction
#
Kirk Webb's avatar
Kirk Webb committed
156

Kirk Webb's avatar
 
Kirk Webb committed
157 158 159 160 161 162 163 164 165 166
#
# Multiple attribute change exception
#
class MultiChangeError(Exception):
    def __init__(self, nodeid, chattrs={}):
        self.nodeid = nodeid
        self.chattrs = chattrs
        pass
    pass

Kirk Webb's avatar
 
Kirk Webb committed
167

Kirk Webb's avatar
Kirk Webb committed
168 169 170
#
# Class responsible for parsing planetlab sites file
#
Kirk Webb's avatar
Kirk Webb committed
171 172 173 174 175 176 177 178
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
179 180
        self.__latitude = 0
        self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
208 209 210
            if attrs.has_key('LATITUDE'):
                self.__latitude = attrs['LATITUDE']
            else:
Kirk Webb's avatar
 
Kirk Webb committed
211
                self.__latitude = 0
212 213 214 215
            if attrs.has_key('LONGITUDE'):
                self.__longitude = attrs['LONGITUDE']
            else:
                self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
216 217 218
            pass
        
        elif name == "HOST":
Kirk Webb's avatar
 
Kirk Webb committed
219 220 221
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
Kirk Webb's avatar
 
Kirk Webb committed
222 223 224 225 226 227 228 229 230 231 232 233 234
            if not attrs.has_key('BWLIMIT'):
                attrs['BWLIMIT'] = "-1"
                pass
            self.__hosts.append({
                'HNAME'     : attrs['NAME'],
                'IP'        : attrs['IP'],
                'PLABID'    : attrs['NODE_ID'],
                'MAC'       : attrs['MAC'],
                'BWLIMIT'   : attrs['BWLIMIT'],
                'SITE'      : self.__sitename,
                'LATITUDE'  : self.__latitude,
                'LONGITUDE' : self.__longitude
                })
Kirk Webb's avatar
Kirk Webb committed
235 236 237 238 239 240 241 242 243 244 245 246
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
247 248
            self.__latitude = 0
            self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
249 250 251
            pass
        return

Kirk Webb's avatar
 
Kirk Webb committed
252

253
class Plab:
Kirk Webb's avatar
Kirk Webb committed
254
    def __init__(self, agent=None):
255 256 257
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
Kirk Webb's avatar
Kirk Webb committed
258
        if debug:
259 260 261
            print "Using module: %s" % self.agent.modname
            pass
        pass
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

Kirk Webb's avatar
Kirk Webb committed
279
    def updateNodeEntries(self, ignorenew = False):
280
        """
Kirk Webb's avatar
Kirk Webb committed
281
        Finds out which Plab nodes are available, and
282 283 284 285 286
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
Kirk Webb's avatar
Kirk Webb committed
287
        discovered.
288 289 290 291 292 293

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
294 295 296
        
        print "Getting available Plab nodes ..."

297
        avail = []
298
        try:
Kirk Webb's avatar
Kirk Webb committed
299 300 301
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
302
        # XXX: rewrite to use more elegant exception info gathering.
303 304
        except:
            extype, exval, extrace = sys.exc_info()
305
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
306
            if debug:
307 308 309
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
Kirk Webb's avatar
 
Kirk Webb committed
310

Kirk Webb's avatar
Kirk Webb committed
311
        if debug:
312 313
            print "Got advertisement list:"
            print avail
Kirk Webb's avatar
Kirk Webb committed
314
            pass
Kirk Webb's avatar
 
Kirk Webb committed
315

Kirk Webb's avatar
 
Kirk Webb committed
316 317 318
        ignored_nodes = self.__readNodeFile(IGNORED_NODES_FILE)
        allowed_nodes = self.__readNodeFile(ALLOWED_NODES_FILE)

Kirk Webb's avatar
 
Kirk Webb committed
319 320 321 322 323 324
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
Kirk Webb's avatar
 
Kirk Webb committed
325
        if len(allowed_nodes) or len(ignored_nodes):
Kirk Webb's avatar
 
Kirk Webb committed
326
            allowed = []
Kirk Webb's avatar
Kirk Webb committed
327
            for nodeent in avail:
Kirk Webb's avatar
 
Kirk Webb committed
328
                if nodeent['PLABID'] in ignored_nodes:
Kirk Webb's avatar
 
Kirk Webb committed
329
                    continue
Kirk Webb's avatar
 
Kirk Webb committed
330 331
                elif len(allowed_nodes):
                    if nodeent['IP'] in allowed_nodes:
Kirk Webb's avatar
 
Kirk Webb committed
332 333 334 335 336
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
337 338
                    pass
                pass
Kirk Webb's avatar
 
Kirk Webb committed
339 340 341 342
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
Kirk Webb's avatar
Kirk Webb committed
343
            pass
344

Kirk Webb's avatar
 
Kirk Webb committed
345 346 347 348 349 350 351 352 353 354 355
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

356
        # Get node info we already have.
357
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
358
        if debug:
359 360
            print "Got known pnodes:"
            print known
Kirk Webb's avatar
Kirk Webb committed
361
            pass
362

Kirk Webb's avatar
Kirk Webb committed
363
        # Create list of nodes to add or update
Kirk Webb's avatar
 
Kirk Webb committed
364 365
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
Kirk Webb's avatar
Kirk Webb committed
366
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
367 368
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
369
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
Kirk Webb's avatar
 
Kirk Webb committed
370 371 372
            # Determine if we already know about this node.
            matchres = self.__matchPlabNode(nodeent, known)
            if not matchres:
Kirk Webb's avatar
 
Kirk Webb committed
373
                toadd.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
374
                pass
Kirk Webb's avatar
 
Kirk Webb committed
375 376
            elif len(matchres[1]):
                toupdate.append((nodeent,matchres))
Kirk Webb's avatar
Kirk Webb committed
377
                pass
Kirk Webb's avatar
 
Kirk Webb committed
378
            pass
Kirk Webb's avatar
Kirk Webb committed
379

Kirk Webb's avatar
 
Kirk Webb committed
380 381
        # Process the list of nodes to add
        addstr = ""
382
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
383
            # Are we ignoring new entries?
384
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
385
                if verbose:
386
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
Kirk Webb's avatar
Kirk Webb committed
387 388
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
389
            # If not ignoring, do the addition/update.
390
            else:
Kirk Webb's avatar
 
Kirk Webb committed
391 392
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
393
                    # Get the linktype here so we can report it in email.
Kirk Webb's avatar
Kirk Webb committed
394
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
395
                    if debug:
Kirk Webb's avatar
Kirk Webb committed
396 397
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
398
                        pass
Kirk Webb's avatar
 
Kirk Webb committed
399 400 401
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
Kirk Webb's avatar
Kirk Webb committed
402
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
Kirk Webb's avatar
 
Kirk Webb committed
403
                              (nodeent['PLABID'],
Kirk Webb's avatar
Kirk Webb committed
404 405 406 407
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
Kirk Webb's avatar
 
Kirk Webb committed
408
                    addstr += nodestr
Kirk Webb's avatar
Kirk Webb committed
409
                    pass
Kirk Webb's avatar
 
Kirk Webb committed
410 411
                pass
            pass
412

Kirk Webb's avatar
 
Kirk Webb committed
413
        # Process node updates.
Kirk Webb's avatar
 
Kirk Webb committed
414
        updstr = ""
Kirk Webb's avatar
 
Kirk Webb committed
415
        chgerrstr = ""
Kirk Webb's avatar
 
Kirk Webb committed
416 417
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
Kirk Webb's avatar
 
Kirk Webb committed
418
            for (nodeent,(nodeid,diffattrs)) in toupdate:
Kirk Webb's avatar
 
Kirk Webb committed
419
                try:
Kirk Webb's avatar
 
Kirk Webb committed
420
                    self.__updateNodeMapping(nodeid, diffattrs)
Kirk Webb's avatar
 
Kirk Webb committed
421 422 423 424 425
                    pass
                except MultiChangeError, e:
                    print "%s not updated: Too many attribute changes." % \
                          e.nodeid
                    chgerrstr += "%s:\n" % e.nodeid
Kirk Webb's avatar
 
Kirk Webb committed
426 427
                    for (attr,(old,new)) in e.chattrs.items():
                        chgerrstr += "\t%s:\t%s => %s\n" % (attr,old,new)
Kirk Webb's avatar
 
Kirk Webb committed
428 429 430
                        pass
                    chgerrstr += "\n"
                    continue
Kirk Webb's avatar
 
Kirk Webb committed
431
                self.__updateNode(nodeid, nodeent)
Kirk Webb's avatar
 
Kirk Webb committed
432
                # Add a line for the add/update message.
Kirk Webb's avatar
 
Kirk Webb committed
433 434 435
                nodestr = nodeid + "\n"
                for (attr,(old,new)) in diffattrs.items():
                    nodestr += "\t%s:\t%s => %s\n" % (attr,old,new)
Kirk Webb's avatar
 
Kirk Webb committed
436 437
                    pass
                updstr += nodestr + "\n"
Kirk Webb's avatar
Kirk Webb committed
438 439
                pass
            pass
Kirk Webb's avatar
 
Kirk Webb committed
440

Kirk Webb's avatar
 
Kirk Webb committed
441 442 443 444 445 446 447 448
        if chgerrstr:
            SENDMAIL(TBOPS,
                     "Two or more changes detected for some plab nodes",
                     "Two or more distinguishing attributes have changed "
                     "on the following planetlab nodes:\n\n%s\n" % chgerrstr,
                     TBOPS)
            pass

Kirk Webb's avatar
 
Kirk Webb committed
449 450 451 452
        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
Kirk Webb's avatar
 
Kirk Webb committed
453 454
            print "Pushing out site_mapping ..."
            os.spawnl(os.P_WAIT, PELAB_PUSH, PELAB_PUSH)
Kirk Webb's avatar
 
Kirk Webb committed
455 456 457 458 459 460 461 462 463 464 465 466
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
Kirk Webb's avatar
Kirk Webb committed
467
        return
468

Kirk Webb's avatar
 
Kirk Webb committed
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
487 488 489
                elif (attr == "LATITUDE") or (attr == "LONGITUDE"):
                    # Special rules for latitude and longitude to avoid
                    # FP errors
490 491 492 493 494 495
                    if (ent[attr] != None and plabent[attr] != None) \
                           and (ent[attr] != "" and plabent[attr] != "") \
                           and ((float(ent[attr]) > \
                                 (float(plabent[attr]) + LATLONG_DELTA)) \
                                or (float(ent[attr]) < \
                                    (float(plabent[attr]) - LATLONG_DELTA))):
496 497 498 499
                        diff[attr] = (ent[attr], plabent[attr])
                    else:
                        same[attr] = ent[attr]
                        pass
Kirk Webb's avatar
 
Kirk Webb committed
500 501 502 503 504 505 506
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
507 508
            # Only consider these to be the same if at least one 'critical'
            # attr is the same
Kirk Webb's avatar
 
Kirk Webb committed
509
            if len(same):
510 511 512
                for attr in same:
                    if attr in ATTR_CRIT_KEYS:
                        return (nid, diff)
Kirk Webb's avatar
 
Kirk Webb committed
513 514 515
            pass
        return ()

516 517 518 519 520
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
521 522
        res = DBQueryFatal("select plab_mapping.node_id,plab_id,"
                           "plab_mapping.hostname,IP,mac,site,latitude,"
Kirk Webb's avatar
 
Kirk Webb committed
523
                           "longitude,bwlimit"
524 525 526 527
                           " from plab_mapping"
                           " left join widearea_nodeinfo on"
                           "    plab_mapping.node_id = "
                           "    widearea_nodeinfo.node_id")
Kirk Webb's avatar
 
Kirk Webb committed
528
        
529
        ret = {}
Kirk Webb's avatar
 
Kirk Webb committed
530 531
        for (nodeid, plabid, hostname, ip, mac, site,
             latitude, longitude, bwlimit) in res:
532 533 534 535 536 537
            ret[nodeid] = {'PLABID'    : plabid,
                           'HNAME'     : hostname,
                           'IP'        : ip,
                           'MAC'       : mac,
                           'SITE'      : site,
                           'LATITUDE'  : latitude,
Kirk Webb's avatar
 
Kirk Webb committed
538 539
                           'LONGITUDE' : longitude,
                           'BWLIMIT'   : bwlimit}
Kirk Webb's avatar
Kirk Webb committed
540
            pass
Kirk Webb's avatar
 
Kirk Webb committed
541 542 543 544 545 546 547 548 549 550
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
551
        return ret
Kirk Webb's avatar
 
Kirk Webb committed
552 553 554 555 556 557 558 559 560

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        attrs = {}
        dups = {}
        
        for ent in nodelist:
Kirk Webb's avatar
 
Kirk Webb committed
561
            for attr in ATTR_CRIT_KEYS:
Kirk Webb's avatar
 
Kirk Webb committed
562 563 564 565 566 567 568 569 570 571 572 573 574 575
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
Kirk Webb's avatar
 
Kirk Webb committed
576
        
Kirk Webb's avatar
Kirk Webb committed
577
    def __findLinkType(self, nodeent):
578 579 580 581 582 583 584
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

Kirk Webb's avatar
Kirk Webb committed
585
        This can't detect DSL links..
586
        """
587
        # Is host international (or flux/emulab local)?
588
        from socket import gethostbyaddr, getfqdn, herror
Kirk Webb's avatar
Kirk Webb committed
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
611
        # Is host on I2?
Kirk Webb's avatar
Kirk Webb committed
612
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
613 614 615 616 617
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
618 619
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
620

621 622
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
 
Kirk Webb committed
623 624
                nodeent['LINKTYPE'] = "pcplabinet"
                return
625
        else:
Kirk Webb's avatar
Kirk Webb committed
626
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
627

Kirk Webb's avatar
 
Kirk Webb committed
628 629
        # We don't know - must manually classify.
        nodeent['LINKTYPE'] = "*Unknown*"
Kirk Webb's avatar
Kirk Webb committed
630
        return
631

Kirk Webb's avatar
 
Kirk Webb committed
632
    def __addNode(self, nodeent):
633
        """
Kirk Webb's avatar
 
Kirk Webb committed
634 635 636
        updateNodeEntries() helper function.  Adds a new Plab pnode and
        associated vnode to the DB.  The argument is a dictionary containing
        the new node's attributes.
637
        """
Kirk Webb's avatar
 
Kirk Webb committed
638
        # Generate/grab variables to be used when creating the node.
639
        defosid, controliface = self.__getNodetypeInfo()
Kirk Webb's avatar
Kirk Webb committed
640
        hostonly = nodeent['HNAME'].replace(".", "-")
Kirk Webb's avatar
 
Kirk Webb committed
641 642 643 644 645 646 647 648 649 650 651 652
        nidnum, priority = self.__nextFreeNodeid()
        nodeid = "%s%d" % (NODEPREFIX, nidnum)
        vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)

        # Do the stuff common to both node addition and update first
        # Note that if this fails, we want the exception generated to
        # percolate up to the caller immediately, so don't catch it.
        self.__updateNode(nodeid, nodeent)

        # Now perform stuff specific to node addition
Kirk Webb's avatar
 
Kirk Webb committed
653
        try:
Kirk Webb's avatar
Kirk Webb committed
654
            DBQueryFatal("replace into nodes"
655
                         " (node_id, type, phys_nodeid, role, priority,"
Kirk Webb's avatar
 
Kirk Webb committed
656
                         "  op_mode, def_boot_osid,"
657
                         "  allocstate, allocstate_timestamp,"
Kirk Webb's avatar
 
Kirk Webb committed
658
                         "  eventstate, state_timestamp)"
659
                         " values (%s, %s, %s, %s, %s,"
Kirk Webb's avatar
 
Kirk Webb committed
660 661 662 663
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
664
                          'FREE_CLEAN',
Kirk Webb's avatar
 
Kirk Webb committed
665
                          'ISUP'))
666

667 668 669
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
Kirk Webb's avatar
Kirk Webb committed
670
                         (nodeid))
671

Kirk Webb's avatar
Kirk Webb committed
672
            DBQueryFatal("replace into reserved"
Kirk Webb's avatar
 
Kirk Webb committed
673 674
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
675
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))
Kirk Webb's avatar
 
Kirk Webb committed
676

Kirk Webb's avatar
 
Kirk Webb committed
677 678
            # XXX: This should probably be checked and updated if necessary
            #      when updating.
Kirk Webb's avatar
Kirk Webb committed
679
            DBQueryFatal("replace into node_auxtypes"
Kirk Webb's avatar
 
Kirk Webb committed
680 681
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
682
                         (nodeid, nodeent['LINKTYPE'], 1))
Kirk Webb's avatar
 
Kirk Webb committed
683
            
Kirk Webb's avatar
Kirk Webb committed
684
            DBQueryFatal("replace into node_auxtypes"
685 686 687 688
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
Kirk Webb's avatar
Kirk Webb committed
689
            DBQueryFatal("replace into node_status"
690 691
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
692
                         (nodeid, 'down'))
Kirk Webb's avatar
 
Kirk Webb committed
693

Kirk Webb's avatar
 
Kirk Webb committed
694 695 696 697 698
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
699

Kirk Webb's avatar
 
Kirk Webb committed
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
            #
            # NowAdd the site_mapping entry for this node.
            #
            
            # See if we know about the associated site - grab idx if so
            siteidx = 0
            nodeidx = 1
            siteres = DBQueryFatal("select site_idx, node_idx from "
                                   " plab_site_mapping where site_name=%s",
                                   nodeent['SITE']);
            if len(siteres):
                # There are already nodes listed for this site, so get
                # the next node id.
                siteidx = siteres[0][0]
                for (foo, idx) in siteres:
                    if idx > nodeidx: nodeidx = idx
                    pass
                nodeidx += 1
                pass
            else:
                # No nodes listed for site, so get the largest site_idx
                # in the DB so far, and increment cuz we're going to add
                # a new one.
                maxres = DBQueryFatal("select MAX(site_idx) from "
                                      " plab_site_mapping")
                try:
                    siteidx = int(maxres[0][0]) + 1
                    pass
                except ValueError:
                    siteidx = 1
                    pass
                pass
            # Create site_mapping entry, optionally creating new site idx
            # via not specifying the site_idx field (field is auto_increment)
            DBQueryFatal("insert into plab_site_mapping "
                         " values (%s, %s, %s, %s)",
                         (nodeent['SITE'], siteidx, nodeid, nodeidx))

738
            # Create a single reserved plab vnode for the managment sliver.
Kirk Webb's avatar
 
Kirk Webb committed
739 740 741 742 743
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
744 745
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
Kirk Webb's avatar
Kirk Webb committed
746
                pass
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
Kirk Webb's avatar
Kirk Webb committed
770
            # Put the last vnode created into the special monitoring expt.
Kirk Webb's avatar
 
Kirk Webb committed
771 772 773 774
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
Kirk Webb's avatar
Kirk Webb committed
775 776
            pass
        
Kirk Webb's avatar
 
Kirk Webb committed
777 778 779 780 781 782
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
783
                     "\n Please clean up!\n" % tbmsg, TBOPS)
Kirk Webb's avatar
 
Kirk Webb committed
784
            raise
Kirk Webb's avatar
 
Kirk Webb committed
785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
        return


    def __updateNode(self, nodeid, nodeent):
        """
        updateNodeEntries() helper function.  Updates attributes for plab
        nodes passed in via the nodeent argument.
        """
        # Get the name of the control interface for plab nodes.
        defosid, controliface = self.__getNodetypeInfo()

        haslatlong = (('LATITUDE' in nodeent and 'LONGITUDE' in nodeent) and
            (nodeent['LATITUDE'] != 0 or nodeent['LONGITUDE'] != 0))
        try:
            DBQueryFatal("replace into widearea_nodeinfo"
800 801 802 803 804
                         " (node_id, contact_uid, contact_idx, hostname, site,"
                         "  latitude, longitude, bwlimit)"
                         " values (%s, %s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 'nobody', '0', nodeent['HNAME'],
                          nodeent['SITE'],
Kirk Webb's avatar
 
Kirk Webb committed
805 806 807 808
                          # Poor man's ternary operator
                          haslatlong and nodeent['LATITUDE'] or "NULL",
                          haslatlong and nodeent['LONGITUDE'] or "NULL",
                          nodeent['BWLIMIT']))
809

Kirk Webb's avatar
 
Kirk Webb committed
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'plab_fake',
                          controliface, 'ctrl'))
            pass
        except:
            print "Error updating PLAB node in DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error updating plab node in DB: %s\n" % nodeid,
                     "Some operation failed while trying to update"
                     " plab node %s in the DB:\n\n%s"
                     "\nPlease clean up!\n" % (nodeid, tbmsg), TBOPS)
            raise
Kirk Webb's avatar
Kirk Webb committed
825
        return
826

Kirk Webb's avatar
 
Kirk Webb committed
827

Kirk Webb's avatar
 
Kirk Webb committed
828
    def __updateNodeMapping(self, nodeid, chattrs):
Kirk Webb's avatar
 
Kirk Webb committed
829
        """
Kirk Webb's avatar
 
Kirk Webb committed
830
        Updates changed node attributes in the plab mapping table.
Kirk Webb's avatar
 
Kirk Webb committed
831 832 833
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
Kirk Webb's avatar
 
Kirk Webb committed
834

Kirk Webb's avatar
 
Kirk Webb committed
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
        # mapping from attrs to column names
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}

        # Get the intersection of mapping (critical) keys with those that
        # have changed.
        changedcritkeys = set(ATTR_CRIT_KEYS) & set(chattrs.keys())
        # nothing to do if none of the mapping attributes have changed.
        if not changedcritkeys:
            return
        # If the node has more than two critical attrs that have changed,
        # then move it to hwdown and raise an exception.
        if len(changedcritkeys) > 2:
            crattrs = {}
            for chkey in changedcritkeys:
                crattrs[chkey] = chattrs[chkey]
Kirk Webb's avatar
 
Kirk Webb committed
853 854
                pass
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
Kirk Webb's avatar
 
Kirk Webb committed
855
                     "%s has been moved to hwdown." % (crattrs, nodeid)
Kirk Webb's avatar
 
Kirk Webb committed
856 857
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
Kirk Webb's avatar
 
Kirk Webb committed
858 859 860 861 862 863 864
            raise MultiChangeError(nodeid, crattrs)

        # Update mapping table entry.
        updstr = ",".join(map(lambda x: "%s='%s'" %
                              (attrmap[x],chattrs[x][1]), changedcritkeys))
        DBQueryFatal("update plab_mapping set %s where node_id='%s'" %
                     (updstr, nodeid))
Kirk Webb's avatar
 
Kirk Webb committed
865 866 867 868 869 870 871
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

872 873
    def __getNodetypeInfo(self):
        """
874 875
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
876
        it doesn't change.
877 878
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
879
            if debug:
880
                print "Getting node type info"
Kirk Webb's avatar
Kirk Webb committed
881
                pass
882 883 884 885 886 887 888 889 890

            dbres = DBQueryFatal("select attrkey,attrvalue "
                                 " from node_type_attributes as a "
                                 " where type = 'pcplabphys' and "
                                 "       (a.attrkey='default_osid' or "
                                 "        a.attrkey='control_interface') "
                                 " order by attrkey")
            
            assert (len(dbres) == 2), "Failed to get node type info"
Kirk Webb's avatar
 
Kirk Webb committed
891 892
            attrdict = {}
            for attrkey, attrvalue in dbres:
Kirk Webb's avatar
 
Kirk Webb committed
893
                attrdict[attrkey] = attrvalue;
Kirk Webb's avatar
 
Kirk Webb committed
894 895
                pass
            self.__getNodetypeInfoCache = \
Kirk Webb's avatar
 
Kirk Webb committed
896 897
                                        (attrdict["default_osid"],
                                         attrdict["control_interface"])
Kirk Webb's avatar
Kirk Webb committed
898 899
            pass
        
900 901 902 903 904 905 906
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
907
        if debug:
908 909 910 911 912 913 914 915 916 917
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
Kirk Webb's avatar
Kirk Webb committed
918
            pass
919 920
        finally:
            DBQueryFatal("unlock tables")
Kirk Webb's avatar
Kirk Webb committed
921 922
            pass
        
923 924
        return nodeid, priority

Kirk Webb's avatar
 
Kirk Webb committed
925 926 927 928 929 930 931 932 933 934 935 936 937
    def __readNodeFile(self, filename):
        """
        Helper function - read in list of nodes from a file, seperated
        by arbitrary amounts of whitespace.  No comments allowed.
        """
        nodelist = []
        if os.access(filename, os.F_OK):
            nodefile = open(filename, "r+")
            nodelist = nodefile.read().split()
            nodefile.close()
            pass
        return nodelist

Kirk Webb's avatar
 
Kirk Webb committed
938
    def renew(self, inpid = None, ineid = None, force = False):
939
        """
Kirk Webb's avatar
 
Kirk Webb committed
940 941 942
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
943
        """
Kirk Webb's avatar
 
Kirk Webb committed
944

Kirk Webb's avatar
 
Kirk Webb committed
945 946
        global failedrenew # XXX
        
Kirk Webb's avatar
 
Kirk Webb committed
947 948 949 950 951 952 953 954 955 956
        now = int(time.time())
        
        if not inpid:
            res = DBQueryFatal("select pid, eid from plab_slices");
            pass
        else:
            if not ineid:
                raise RuntimeError, "renew: Must provide eid with pid."
            res = ((inpid, ineid),)
            pass
Kirk Webb's avatar
Kirk Webb committed
957
        
958
        loadedSlices = {}
959 960
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
961 962
        ret = 0

Kirk Webb's avatar
 
Kirk Webb committed
963
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
964

Kirk Webb's avatar
 
Kirk Webb committed
965
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
966

967 968
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
969
                pass
970 971 972
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
973
                pass
974
            
Kirk Webb's avatar
 
Kirk Webb committed
975
            res = slice.renew(force)
Kirk Webb's avatar
 
Kirk Webb committed
976 977 978 979 980
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
981 982
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
983
                    pass
Kirk Webb's avatar
 
Kirk Webb committed
984
                if (slice.leaseend - now) < PLABEXPIREWARN:
985
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
986 987
                    pass
                pass
988 989 990
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
Kirk Webb's avatar
 
Kirk Webb committed
991
                    pass
Kirk Webb's avatar
Kirk Webb committed
992
                    
993 994
        if newfail:
            failedrenew += newfail
Kirk Webb's avatar
Kirk Webb committed
995
            failstr = ""
996
            for n in newfail:
Kirk Webb's avatar
 
Kirk Webb committed
997
                failstr += "%s/%s (expires: %s UTC)\n" % \
Kirk Webb's avatar
 
Kirk Webb committed
998
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
Kirk Webb committed
999
                pass
Kirk Webb's avatar
 
Kirk Webb committed
1000
            
1001
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
Kirk Webb's avatar
 
Kirk Webb committed
1002
                     "Failed to renew the following leases:\n%s" %
1003
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1004
            pass
Kirk Webb's avatar
Kirk Webb committed
1005

1006 1007 1008
        if failsoon:
            failstr = ""
            for n in failsoon:
Kirk Webb's avatar
 
Kirk Webb committed
1009
                failstr += "%s/%s: (expires: %s UTC)\n" % \
Kirk Webb's avatar
 
Kirk Webb committed
1010
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
 
Kirk Webb committed
1011 1012 1013 1014
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1015
            pass
Kirk Webb's avatar
 
Kirk Webb committed
1016

Kirk Webb's avatar
Kirk Webb committed
1017 1018 1019
        return
    
    pass # end class Plab
Kirk Webb's avatar
 
Kirk Webb committed
1020 1021
# AOP wrappers for class Plab
wrap_around(Plab.createSlice, timeAdvice)
1022 1023 1024 1025 1026

#
# Slice abstraction
#
class Slice:
Kirk Webb's avatar
 
Kirk Webb committed
1027 1028

    def __init__(self, plab, pid, eid, slicename = None):
1029 1030
        self.plab = plab
        self.pid, self.eid = pid, eid
1031
        self.slicemeta = None
Kirk Webb's avatar
 
Kirk Webb committed
1032
        self.slicename = slicename
Kirk Webb's avatar
 
Kirk Webb committed
1033
        self.description = DEF_SLICE_DESC
Kirk Webb's avatar
 
Kirk Webb committed
1034
        return
1035 1036 1037 1038 1039 1040
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
Kirk Webb's avatar
 
Kirk Webb committed
1041 1042 1043

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
Kirk Webb's avatar
 
Kirk Webb committed
1044 1045
            self.slicename   = PLAB_SVC_SLICENAME
            self.description = PLAB_SVC_SLICEDESC
Kirk Webb's avatar
 
Kirk Webb committed
1046 1047 1048
            adminbit = 1
            pass
        
Kirk Webb's avatar
 
Kirk Webb committed
1049
        if not self.slicename:
Kirk Webb's avatar
 
Kirk Webb committed
1050
            res = DBQueryFatal("select idx, expt_name from experiments "
Kirk Webb's avatar
 
Kirk Webb committed
1051 1052 1053 1054
                               "where pid=%s "
                               "and eid=%s",
                               (self.pid, self.eid))
            if not len(res):
Kirk Webb's avatar
 
Kirk Webb committed
1055 1056 1057 1058
                raise RuntimeError, \
                      "Didn't get any results while looking up info on " \
                      "experiment %s/%s" % (self.pid, self.eid)
            (eindex, descr) = res[0]
Kirk Webb's avatar
 
Kirk Webb committed
1059
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
Kirk Webb's avatar
 
Kirk Webb committed
1060
            self.description = descr
Kirk Webb's avatar
 
Kirk Webb committed
1061
            pass
Kirk Webb's avatar
 
Kirk Webb committed
1062

1063
        print "Creating Plab slice %s." % self.slicename
Kirk Webb's avatar
 
Kirk Webb committed
1064

1065
        try:
Kirk Webb's avatar
Kirk Webb committed
1066 1067 1068 1069
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)

            DBQueryFatal("insert into plab_slices"
Kirk Webb's avatar
 
Kirk Webb committed
1070 1071
                         " (pid, eid, slicename, slicemeta, leaseend, admin)"
                         " values (%s, %s, %s, %s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
1072
                         (self.pid, self.eid, self.slicename,
Kirk Webb's avatar
 
Kirk Webb committed
1073 1074 1075 1076
                          self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          adminbit))
1077 1078
            pass
        except:
Kirk Webb's avatar
Kirk Webb committed
1079
            self.plab.agent.deleteSlice(self)
1080 1081
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
1082 1083
            raise

Kirk Webb's avatar
 
Kirk Webb committed
1084 1085 1086 1087 1088 1089
        # Setup mailing alias for the slice.  All mail currently filters
        # in via the 'emulabman' alias.  A procmail filter there will
        # redirect it to the appropriate user.
        try:
            qres = DBQueryFatal("select u.uid, u.usr_email from users as u "
                                "left join experiments as e "
1090
                                "on u.uid_idx = e.swapper_idx "
Kirk Webb's avatar
 
Kirk Webb committed
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
                                "where e.pid=%s and e.eid=%s",
                                (self.pid, self.eid))
            if not len(qres):
                raise RuntimeError, \
                      "Didn't get any results while looking up user info " \
                      "for experiment %s/%s" % (self.pid, self.eid)
            (username, usremail) = qres[0]
            command = "%s -host %s /bin/echo %s \> %s/%s" % \
                      (SSH, USERNODE, usremail,
                       SLICE_ALIAS_DIR, self.slicename)
            os.system(command)
        except:
            print "Could not setup email alias for slice: %s!" % self.slicename
            traceback.print_exc()
            pass

1107 1108
        return res

1109 1110 1111 1112
    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.
1113 1114 1115

        XXX This should probably be made lazy, since not all operations
        really need it
1116
        """
Kirk Webb's avatar
Kirk Webb committed
1117
        if verbose:
1118
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
Kirk Webb's avatar
 
Kirk Webb committed
1119 1120
        res = DBQueryFatal("select slicename, slicemeta, leaseend "
                           " from plab_slices "
1121 1122 1123 1124 1125 1126
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
Kirk Webb's avatar
 
Kirk Webb committed
1127 1128
        ((self.slicename, self.slicemeta, leaseend), ) = res
        self.leaseend = calendar.timegm(time.strptime(str(leaseend),
Kirk Webb's avatar
 
Kirk Webb committed
1129 1130
                                                      "%Y-%m-%d %H:%M:%S"))
        return
1131

Kirk Webb's avatar
 
Kirk Webb committed
1132
    def renew(self, force = False):
Kirk Webb's avatar
Kirk Webb committed
1133
        """
Kirk Webb's avatar
 
Kirk Webb committed
1134
        Renews slice lease.  We want this to be the maximum allowed by law...
Kirk Webb's avatar
 
Kirk Webb committed
1135
        Store the expiration time in UTC.
Kirk Webb's avatar
Kirk Webb committed
1136 1137 1138 1139
        """
        print "Renewing lease for slice %s" % self.slicename

        try:
Kirk Webb's avatar
 
Kirk Webb committed
1140
            ret = self.plab.agent.renewSlice(self, force)
Kirk Webb's avatar
 
Kirk Webb committed
1141 1142 1143 1144 1145 1146 1147
            DBQueryFatal("update plab_slices "
                         " set slicemeta=%s, leaseend=%s "
                         " where slicename=%s",
                         (self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          self.slicename))
Kirk Webb's avatar
Kirk Webb committed
1148 1149
            pass
        except:
Kirk Webb's avatar
 
Kirk Webb committed
1150
            print "slice.renew: Slice renewal failed:"
Kirk Webb's avatar
Kirk Webb committed
1151 1152 1153
            traceback.print_exc()
            ret = 0
            pass
Kirk Webb's avatar
 
Kirk Webb committed
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

        # Renew individual slivers, if necessary.  Any that fail to renew
        # and are close to expiration need to be noted.  They will be
        # reported after renewal has been attempted on all nodes.
        now = int(time.time())
        nodes = map(lambda x: x[0], self.getSliceNodes())
        reportfailed = []
        for nodeid in nodes:
            try:
                node = self.loadNode(nodeid)
                pass
            except AssertionError:
                print "Node %s doesn't really exist in %s" % \
                      (nodeid, self.slicename)
                continue
            if node.renew():
                print "Failed to renew node: %s (expires %s UTC)" % \
                      (nodeid, time.asctime(time.gmtime(node.leaseend)))
                print "Timediff: %s" % (node.leaseend - now)
                if node.leaseend - now < NODEEXPIREWARN:
                    reportfailed.append((nodeid, node.leaseend))
                    pass
                pass
            del node
            pass

        # Report any nodes that are near to expiration
        if len(reportfailed) > 0:
            tbstr = ""
            for nodeid, leaseend in reportfailed:
                tbstr += "Node: %s, Leaseend: %s UTC\n" % \
                         (nodeid, time.asctime(time.gmtime(leaseend)))
                pass
            SENDMAIL(TBOPS, "Plab nodes in danger of expiration: %s/%s" % \
                     (self.pid, self.eid),
                     "The following slivers in %s/%s will expire "
                     "soon:\n\n%s" % \
                     (self.pid, self.eid, tbstr),
                     TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1193
        
Kirk Webb's avatar
 
Kirk Webb committed
1194
        return ret
Kirk Webb's avatar
Kirk Webb committed
1195

1196 1197 1198 1199 1200 1201 1202
    def destroy(self):
        """
        Frees all nodes in this slice and destroys the slice.  Note
        that this will really pound the DB if there are many nodes left
        in the slice, but those should be removed by Emulab before the
        slice is destroyed.
        """
1203
        print "Destroying Plab slice %s." % self.slicename
1204 1205 1206 1207 1208 1209 1210 1211
        res = DBQueryFatal("select node_id from plab_slice_nodes"
                           " where slicename = %s",
                           (self.slicename))
        print "\tRemoving any remaining nodes in slice.."
        for (nodeid,) in res:
            node = self.loadNode(nodeid)
            node.free()
            del node  # Encourage the GC'er