libplab.py.in 69.1 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2
3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004, 2006, 2007 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5
6
7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9
10
11
12
13
14
15
16
17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18
19
20
"""

import sys
21
22
23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
   
Kirk Webb committed
25
import traceback
Kirk Webb's avatar
   
Kirk Webb committed
26
import signal
Kirk Webb's avatar
   
Kirk Webb committed
27
import socket
Kirk Webb's avatar
Kirk Webb committed
28
29
30
import httplib
import xml.parsers.expat
import re
Kirk Webb's avatar
   
Kirk Webb committed
31
import calendar
Kirk Webb's avatar
   
Kirk Webb committed
32
import shlex
Kirk Webb's avatar
   
Kirk Webb committed
33

Kirk Webb's avatar
   
Kirk Webb committed
34
from popen2 import Popen4
Kirk Webb's avatar
   
Kirk Webb committed
35
from warnings import warn
36

37
38
39
40
41
42
43
44
45
46
47
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
Kirk Webb's avatar
   
Kirk Webb committed
48
from mod_PLCNM import mod_PLCNM
49
from mod_PLC4 import mod_PLC4
50
51

agents = {'PLC'    : mod_PLC,
Kirk Webb's avatar
   
Kirk Webb committed
52
          'dslice' : mod_dslice,
53
54
          'PLCNM'  : mod_PLCNM,
          'PLC4'   : mod_PLC4}
55

Kirk Webb's avatar
   
Kirk Webb committed
56
57
58
59
60
61
62
#
# Initialize the AOP stuff
#
from aspects import wrap_around
from timer_advisories import initTimeAdvice, timeAdvice
initTimeAdvice("plabtiming")

Kirk Webb's avatar
Kirk Webb committed
63
64
65
66
67
68
#
# output control vars
#
verbose = 0
debug = 0

69
70
71
#
# Constants
#
72
DEF_AGENT = "PLC4";
73
74
75
76
#
# Add a compatibility agent --- libplab will make all PLC calls on this agent
# in addition to the default agent.
#
77
COMPAT_AGENT = "PLCNM";
78
# if this is set, do all PLC calls on COMPAT_AGENT as well as DEF_AGENT
79
compat_mode = 0
80

81
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
Kirk Webb's avatar
   
Kirk Webb committed
82
83
84

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
85
NODEPROBEINT  = 30
86

Kirk Webb's avatar
   
Kirk Webb committed
87
88
USERNODE = "@USERNODE@"
TBOPS = "@TBOPSEMAIL_NOSLASH@"
89
MAILTAG = "@THISHOMEBASE@"
Kirk Webb's avatar
   
Kirk Webb committed
90
SLICE_ALIAS_DIR = "/etc/mail/plab-slice-addrs"
91

92
RESERVED_PID = "emulab-ops"
93
RESERVED_EID = "hwdown"       # start life in hwdown
94
95
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
96

Kirk Webb's avatar
   
Kirk Webb committed
97
98
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18",
                        "205.124.244.178", )
Kirk Webb's avatar
   
Kirk Webb committed
99
100
101
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
102
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
103
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
Kirk Webb's avatar
   
Kirk Webb committed
104
105
106

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
107

108
109
110
111
# 'critical' node identifiers - those that are actually used to uniquely
# identify a planetlab node
ATTR_CRIT_KEYS = ('HNAME', 'IP', 'PLABID', 'MAC',)

112
113
114
115
# The amount by which latitude and longitude are allowed to differ before we
# classify them ask changed
LATLONG_DELTA = 0.001

Kirk Webb's avatar
   
Kirk Webb committed
116
PLABNODE = "@prefix@/sbin/plabnode"
117
SSH = "@prefix@/bin/sshtb"
118
NAMED_SETUP = "@prefix@/sbin/named_setup"
Kirk Webb's avatar
   
Kirk Webb committed
119
PELAB_PUSH  = "@prefix@/sbin/pelab_opspush"
120

Kirk Webb's avatar
Kirk Webb committed
121
122
123
124
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
Kirk Webb's avatar
   
Kirk Webb committed
125
126
IGNORED_NODES_FILE = "@prefix@/etc/plab/IGNOREDNODES"
ALLOWED_NODES_FILE = "@prefix@/etc/plab/ALLOWEDNODES"
127

128
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
129
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
Kirk Webb's avatar
Kirk Webb committed
130
131
132
133
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
Kirk Webb's avatar
   
Kirk Webb committed
134
PLAB_SVC_SLICENAME = "utah_svc_slice"
Kirk Webb's avatar
   
Kirk Webb committed
135
136
137
138
PLAB_SVC_SLICEDESC = "Emulab management service slice. Performs periodic " \
                     "checkins with Emulab central, and routes events for " \
                     "other Emulab slices. Slivers in this slice should " \
                     "only interact with other PlanetLab machines, and Emulab."
Kirk Webb's avatar
   
Kirk Webb committed
139
140
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"
Kirk Webb's avatar
   
Kirk Webb committed
141
DEF_SLICE_DESC = "Slice created by Emulab"
Kirk Webb's avatar
   
Kirk Webb committed
142

Kirk Webb's avatar
   
Kirk Webb committed
143
144
PLABEXPIREWARN = 1*WEEK        # one week advance warning for slice expiration.
NODEEXPIREWARN = 2*WEEK+2*DAY  # about two weeks advance warning for slivers.
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
#
# This is a trigger table for dist'ing out multiple rootballs depending on
# which NM is running on the node.  Since we don't know which version is
# running on a node until we call into (DEF|COMPAT)_AGENT and actually connect
# to the node, it's based off classnames.
#
# The table specifies a prefix to the default rootball name; this rootball
# should be placed in the normal location.
#
# If the plcagent class and nodeagent class are both not in the trigger table,
# we simply push the default rootball.
#
# Note that we could store version info in the database, but that doesn't
# really have long-term value.  Plus, we have to be checking sites.xml all the
# time to catch version changes during rollout.  This way, plabmonitord and
# the web interface know exactly what is the version whenever a node is
# contacted for setup.
#
# For now, only NM4agent has a custom tarball, since v4 nodes are in the
# distinct minority right now.
#
from mod_PLC4 import NM4agent

rootball_triggers = { mod_PLC4 : { NM4agent : 'NM4-' } }

171
172
173
174
175
#
# var to track failed renewals
#
failedrenew = []

176
177
178
179
180
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

Kirk Webb's avatar
   
Kirk Webb committed
181
182
183
184
185
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

186

187
188
189
#
# Plab abstraction
#
Kirk Webb's avatar
Kirk Webb committed
190

Kirk Webb's avatar
   
Kirk Webb committed
191
192
193
194
195
196
197
198
199
200
#
# Multiple attribute change exception
#
class MultiChangeError(Exception):
    def __init__(self, nodeid, chattrs={}):
        self.nodeid = nodeid
        self.chattrs = chattrs
        pass
    pass

Kirk Webb's avatar
   
Kirk Webb committed
201

Kirk Webb's avatar
Kirk Webb committed
202
203
204
#
# Class responsible for parsing planetlab sites file
#
Kirk Webb's avatar
Kirk Webb committed
205
206
207
208
209
210
211
212
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
213
214
        self.__latitude = 0
        self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
242
243
244
            if attrs.has_key('LATITUDE'):
                self.__latitude = attrs['LATITUDE']
            else:
Kirk Webb's avatar
   
Kirk Webb committed
245
                self.__latitude = 0
246
247
248
249
            if attrs.has_key('LONGITUDE'):
                self.__longitude = attrs['LONGITUDE']
            else:
                self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
250
251
252
            pass
        
        elif name == "HOST":
Kirk Webb's avatar
   
Kirk Webb committed
253
254
255
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
Kirk Webb's avatar
   
Kirk Webb committed
256
257
258
259
260
261
262
263
264
265
266
267
268
            if not attrs.has_key('BWLIMIT'):
                attrs['BWLIMIT'] = "-1"
                pass
            self.__hosts.append({
                'HNAME'     : attrs['NAME'],
                'IP'        : attrs['IP'],
                'PLABID'    : attrs['NODE_ID'],
                'MAC'       : attrs['MAC'],
                'BWLIMIT'   : attrs['BWLIMIT'],
                'SITE'      : self.__sitename,
                'LATITUDE'  : self.__latitude,
                'LONGITUDE' : self.__longitude
                })
Kirk Webb's avatar
Kirk Webb committed
269
270
271
272
273
274
275
276
277
278
279
280
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
281
282
            self.__latitude = 0
            self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
283
284
285
            pass
        return

Kirk Webb's avatar
   
Kirk Webb committed
286

287
class Plab:
Kirk Webb's avatar
Kirk Webb committed
288
    def __init__(self, agent=None):
289
290
291
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
292
293
        if compat_mode:
            self.compat_agent = agents[COMPAT_AGENT]()
Kirk Webb's avatar
Kirk Webb committed
294
        if debug:
295
            print "Using module: %s" % self.agent.modname
296
297
            if compat_mode:
                print "COMPAT: Using module: %s" % self.compat_agent.modname
298
299
            pass
        pass
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

Kirk Webb's avatar
Kirk Webb committed
317
    def updateNodeEntries(self, ignorenew = False):
318
        """
Kirk Webb's avatar
Kirk Webb committed
319
        Finds out which Plab nodes are available, and
320
321
322
323
324
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
Kirk Webb's avatar
Kirk Webb committed
325
        discovered.
326
327
328
329
330
331

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
332
333
334
        
        print "Getting available Plab nodes ..."

335
        avail = []
336
        try:
Kirk Webb's avatar
Kirk Webb committed
337
338
339
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
340
        # XXX: rewrite to use more elegant exception info gathering.
341
342
        except:
            extype, exval, extrace = sys.exc_info()
343
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
344
            if debug:
345
346
347
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
Kirk Webb's avatar
   
Kirk Webb committed
348

Kirk Webb's avatar
Kirk Webb committed
349
        if debug:
350
351
            print "Got advertisement list:"
            print avail
Kirk Webb's avatar
Kirk Webb committed
352
            pass
Kirk Webb's avatar
   
Kirk Webb committed
353

Kirk Webb's avatar
   
Kirk Webb committed
354
355
356
        ignored_nodes = self.__readNodeFile(IGNORED_NODES_FILE)
        allowed_nodes = self.__readNodeFile(ALLOWED_NODES_FILE)

Kirk Webb's avatar
   
Kirk Webb committed
357
358
359
360
361
362
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
Kirk Webb's avatar
   
Kirk Webb committed
363
        if len(allowed_nodes) or len(ignored_nodes):
Kirk Webb's avatar
   
Kirk Webb committed
364
            allowed = []
Kirk Webb's avatar
Kirk Webb committed
365
            for nodeent in avail:
Kirk Webb's avatar
   
Kirk Webb committed
366
                if nodeent['PLABID'] in ignored_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
367
                    continue
Kirk Webb's avatar
   
Kirk Webb committed
368
369
                elif len(allowed_nodes):
                    if nodeent['IP'] in allowed_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
370
371
372
373
374
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
375
376
                    pass
                pass
Kirk Webb's avatar
   
Kirk Webb committed
377
378
379
380
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
Kirk Webb's avatar
Kirk Webb committed
381
            pass
382

Kirk Webb's avatar
   
Kirk Webb committed
383
384
385
386
387
388
389
390
391
392
393
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

394
        # Get node info we already have.
395
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
396
        if debug:
397
398
            print "Got known pnodes:"
            print known
Kirk Webb's avatar
Kirk Webb committed
399
            pass
400

Kirk Webb's avatar
Kirk Webb committed
401
        # Create list of nodes to add or update
Kirk Webb's avatar
   
Kirk Webb committed
402
403
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
Kirk Webb's avatar
Kirk Webb committed
404
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
405
406
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
407
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
Kirk Webb's avatar
   
Kirk Webb committed
408
409
410
            # Determine if we already know about this node.
            matchres = self.__matchPlabNode(nodeent, known)
            if not matchres:
Kirk Webb's avatar
   
Kirk Webb committed
411
                toadd.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
412
                pass
Kirk Webb's avatar
   
Kirk Webb committed
413
414
            elif len(matchres[1]):
                toupdate.append((nodeent,matchres))
Kirk Webb's avatar
Kirk Webb committed
415
                pass
Kirk Webb's avatar
   
Kirk Webb committed
416
            pass
Kirk Webb's avatar
Kirk Webb committed
417

Kirk Webb's avatar
   
Kirk Webb committed
418
419
        # Process the list of nodes to add
        addstr = ""
420
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
421
            # Are we ignoring new entries?
422
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
423
                if verbose:
424
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
Kirk Webb's avatar
Kirk Webb committed
425
426
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
427
            # If not ignoring, do the addition/update.
428
            else:
Kirk Webb's avatar
   
Kirk Webb committed
429
430
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
431
                    # Get the linktype here so we can report it in email.
Kirk Webb's avatar
Kirk Webb committed
432
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
433
                    if debug:
Kirk Webb's avatar
Kirk Webb committed
434
435
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
436
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
437
438
439
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
Kirk Webb's avatar
Kirk Webb committed
440
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
441
                              (nodeent['PLABID'],
Kirk Webb's avatar
Kirk Webb committed
442
443
444
445
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
Kirk Webb's avatar
   
Kirk Webb committed
446
                    addstr += nodestr
Kirk Webb's avatar
Kirk Webb committed
447
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
448
449
                pass
            pass
450

Kirk Webb's avatar
   
Kirk Webb committed
451
        # Process node updates.
Kirk Webb's avatar
   
Kirk Webb committed
452
        updstr = ""
Kirk Webb's avatar
   
Kirk Webb committed
453
        chgerrstr = ""
Kirk Webb's avatar
   
Kirk Webb committed
454
455
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
Kirk Webb's avatar
   
Kirk Webb committed
456
            for (nodeent,(nodeid,diffattrs)) in toupdate:
Kirk Webb's avatar
   
Kirk Webb committed
457
                try:
Kirk Webb's avatar
   
Kirk Webb committed
458
                    self.__updateNodeMapping(nodeid, diffattrs)
Kirk Webb's avatar
   
Kirk Webb committed
459
460
461
462
463
                    pass
                except MultiChangeError, e:
                    print "%s not updated: Too many attribute changes." % \
                          e.nodeid
                    chgerrstr += "%s:\n" % e.nodeid
Kirk Webb's avatar
   
Kirk Webb committed
464
465
                    for (attr,(old,new)) in e.chattrs.items():
                        chgerrstr += "\t%s:\t%s => %s\n" % (attr,old,new)
Kirk Webb's avatar
   
Kirk Webb committed
466
467
468
                        pass
                    chgerrstr += "\n"
                    continue
Kirk Webb's avatar
   
Kirk Webb committed
469
                self.__updateNode(nodeid, nodeent)
Kirk Webb's avatar
   
Kirk Webb committed
470
                # Add a line for the add/update message.
Kirk Webb's avatar
   
Kirk Webb committed
471
472
473
                nodestr = nodeid + "\n"
                for (attr,(old,new)) in diffattrs.items():
                    nodestr += "\t%s:\t%s => %s\n" % (attr,old,new)
Kirk Webb's avatar
   
Kirk Webb committed
474
475
                    pass
                updstr += nodestr + "\n"
Kirk Webb's avatar
Kirk Webb committed
476
477
                pass
            pass
Kirk Webb's avatar
   
Kirk Webb committed
478

Kirk Webb's avatar
   
Kirk Webb committed
479
480
481
482
483
484
485
486
        if chgerrstr:
            SENDMAIL(TBOPS,
                     "Two or more changes detected for some plab nodes",
                     "Two or more distinguishing attributes have changed "
                     "on the following planetlab nodes:\n\n%s\n" % chgerrstr,
                     TBOPS)
            pass

Kirk Webb's avatar
   
Kirk Webb committed
487
488
489
490
        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
Kirk Webb's avatar
   
Kirk Webb committed
491
492
            print "Pushing out site_mapping ..."
            os.spawnl(os.P_WAIT, PELAB_PUSH, PELAB_PUSH)
Kirk Webb's avatar
   
Kirk Webb committed
493
494
495
496
497
498
499
500
501
502
503
504
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
Kirk Webb's avatar
Kirk Webb committed
505
        return
506

Kirk Webb's avatar
   
Kirk Webb committed
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
525
526
527
                elif (attr == "LATITUDE") or (attr == "LONGITUDE"):
                    # Special rules for latitude and longitude to avoid
                    # FP errors
528
529
530
531
532
533
                    if (ent[attr] != None and plabent[attr] != None) \
                           and (ent[attr] != "" and plabent[attr] != "") \
                           and ((float(ent[attr]) > \
                                 (float(plabent[attr]) + LATLONG_DELTA)) \
                                or (float(ent[attr]) < \
                                    (float(plabent[attr]) - LATLONG_DELTA))):
534
535
536
537
                        diff[attr] = (ent[attr], plabent[attr])
                    else:
                        same[attr] = ent[attr]
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
538
539
540
541
542
543
544
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
545
546
            # Only consider these to be the same if at least one 'critical'
            # attr is the same
Kirk Webb's avatar
   
Kirk Webb committed
547
            if len(same):
548
549
550
                for attr in same:
                    if attr in ATTR_CRIT_KEYS:
                        return (nid, diff)
Kirk Webb's avatar
   
Kirk Webb committed
551
552
553
            pass
        return ()

554
555
556
557
558
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
559
560
        res = DBQueryFatal("select plab_mapping.node_id,plab_id,"
                           "plab_mapping.hostname,IP,mac,site,latitude,"
Kirk Webb's avatar
   
Kirk Webb committed
561
                           "longitude,bwlimit"
562
563
564
565
                           " from plab_mapping"
                           " left join widearea_nodeinfo on"
                           "    plab_mapping.node_id = "
                           "    widearea_nodeinfo.node_id")
Kirk Webb's avatar
   
Kirk Webb committed
566
        
567
        ret = {}
Kirk Webb's avatar
   
Kirk Webb committed
568
569
        for (nodeid, plabid, hostname, ip, mac, site,
             latitude, longitude, bwlimit) in res:
570
571
572
573
574
575
            ret[nodeid] = {'PLABID'    : plabid,
                           'HNAME'     : hostname,
                           'IP'        : ip,
                           'MAC'       : mac,
                           'SITE'      : site,
                           'LATITUDE'  : latitude,
Kirk Webb's avatar
   
Kirk Webb committed
576
577
                           'LONGITUDE' : longitude,
                           'BWLIMIT'   : bwlimit}
Kirk Webb's avatar
Kirk Webb committed
578
            pass
Kirk Webb's avatar
   
Kirk Webb committed
579
580
581
582
583
584
585
586
587
588
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
589
        return ret
Kirk Webb's avatar
   
Kirk Webb committed
590
591
592
593
594
595
596
597
598

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        attrs = {}
        dups = {}
        
        for ent in nodelist:
Kirk Webb's avatar
   
Kirk Webb committed
599
            for attr in ATTR_CRIT_KEYS:
Kirk Webb's avatar
   
Kirk Webb committed
600
601
602
603
604
605
606
607
608
609
610
611
612
613
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
Kirk Webb's avatar
   
Kirk Webb committed
614
        
Kirk Webb's avatar
Kirk Webb committed
615
    def __findLinkType(self, nodeent):
616
617
618
619
620
621
622
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

Kirk Webb's avatar
Kirk Webb committed
623
        This can't detect DSL links..
624
        """
625
        # Is host international (or flux/emulab local)?
626
        from socket import gethostbyaddr, getfqdn, herror
Kirk Webb's avatar
Kirk Webb committed
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
649
        # Is host on I2?
Kirk Webb's avatar
Kirk Webb committed
650
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
651
652
653
654
655
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
656
657
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
658

659
660
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
   
Kirk Webb committed
661
662
                nodeent['LINKTYPE'] = "pcplabinet"
                return
663
        else:
Kirk Webb's avatar
Kirk Webb committed
664
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
665

Kirk Webb's avatar
   
Kirk Webb committed
666
667
        # We don't know - must manually classify.
        nodeent['LINKTYPE'] = "*Unknown*"
Kirk Webb's avatar
Kirk Webb committed
668
        return
669

Kirk Webb's avatar
   
Kirk Webb committed
670
    def __addNode(self, nodeent):
671
        """
Kirk Webb's avatar
   
Kirk Webb committed
672
673
674
        updateNodeEntries() helper function.  Adds a new Plab pnode and
        associated vnode to the DB.  The argument is a dictionary containing
        the new node's attributes.
675
        """
Kirk Webb's avatar
   
Kirk Webb committed
676
        # Generate/grab variables to be used when creating the node.
677
        defosid, controliface = self.__getNodetypeInfo()
Kirk Webb's avatar
Kirk Webb committed
678
        hostonly = nodeent['HNAME'].replace(".", "-")
Kirk Webb's avatar
   
Kirk Webb committed
679
680
681
682
683
684
685
686
687
688
689
690
        nidnum, priority = self.__nextFreeNodeid()
        nodeid = "%s%d" % (NODEPREFIX, nidnum)
        vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
        print "Creating pnode %s as %s, priority %d." % \
              (nodeent['IP'], nodeid, priority)

        # Do the stuff common to both node addition and update first
        # Note that if this fails, we want the exception generated to
        # percolate up to the caller immediately, so don't catch it.
        self.__updateNode(nodeid, nodeent)

        # Now perform stuff specific to node addition
Kirk Webb's avatar
   
Kirk Webb committed
691
        try:
692
693
694
            res_exptidx = TBExptIDX(RESERVED_PID, RESERVED_EID)
            mon_exptidx = TBExptIDX(MONITOR_PID, MONITOR_EID)
            
Kirk Webb's avatar
Kirk Webb committed
695
            DBQueryFatal("replace into nodes"
696
                         " (node_id, type, phys_nodeid, role, priority,"
Kirk Webb's avatar
   
Kirk Webb committed
697
                         "  op_mode, def_boot_osid,"
698
                         "  allocstate, allocstate_timestamp,"
Kirk Webb's avatar
   
Kirk Webb committed
699
                         "  eventstate, state_timestamp)"
700
                         " values (%s, %s, %s, %s, %s,"
Kirk Webb's avatar
   
Kirk Webb committed
701
702
703
704
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
705
                          'FREE_CLEAN',
Kirk Webb's avatar
   
Kirk Webb committed
706
                          'ISUP'))
707

708
709
710
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
Kirk Webb's avatar
Kirk Webb committed
711
                         (nodeid))
712

Kirk Webb's avatar
Kirk Webb committed
713
            DBQueryFatal("replace into reserved"
714
                         " (node_id, exptidx, pid, eid, rsrv_time, vname)"
715
                         " values (%s, %s, %s, %s, now(), %s)",
716
717
                         (nodeid, res_exptidx,
                          RESERVED_PID, RESERVED_EID, hostonly))
Kirk Webb's avatar
   
Kirk Webb committed
718

Kirk Webb's avatar
   
Kirk Webb committed
719
720
            # XXX: This should probably be checked and updated if necessary
            #      when updating.
Kirk Webb's avatar
Kirk Webb committed
721
            DBQueryFatal("replace into node_auxtypes"
Kirk Webb's avatar
   
Kirk Webb committed
722
723
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
724
                         (nodeid, nodeent['LINKTYPE'], 1))
Kirk Webb's avatar
   
Kirk Webb committed
725
            
Kirk Webb's avatar
Kirk Webb committed
726
            DBQueryFatal("replace into node_auxtypes"
727
728
729
730
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
Kirk Webb's avatar
Kirk Webb committed
731
            DBQueryFatal("replace into node_status"
732
733
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
734
                         (nodeid, 'down'))
Kirk Webb's avatar
   
Kirk Webb committed
735

Kirk Webb's avatar
   
Kirk Webb committed
736
737
738
739
740
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
741

Kirk Webb's avatar
   
Kirk Webb committed
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
            #
            # NowAdd the site_mapping entry for this node.
            #
            
            # See if we know about the associated site - grab idx if so
            siteidx = 0
            nodeidx = 1
            siteres = DBQueryFatal("select site_idx, node_idx from "
                                   " plab_site_mapping where site_name=%s",
                                   nodeent['SITE']);
            if len(siteres):
                # There are already nodes listed for this site, so get
                # the next node id.
                siteidx = siteres[0][0]
                for (foo, idx) in siteres:
                    if idx > nodeidx: nodeidx = idx
                    pass
                nodeidx += 1
                pass
            else:
                # No nodes listed for site, so get the largest site_idx
                # in the DB so far, and increment cuz we're going to add
                # a new one.
                maxres = DBQueryFatal("select MAX(site_idx) from "
                                      " plab_site_mapping")
                try:
                    siteidx = int(maxres[0][0]) + 1
                    pass
                except ValueError:
                    siteidx = 1
                    pass
                pass
            # Create site_mapping entry, optionally creating new site idx
            # via not specifying the site_idx field (field is auto_increment)
            DBQueryFatal("insert into plab_site_mapping "
                         " values (%s, %s, %s, %s)",
                         (nodeent['SITE'], siteidx, nodeid, nodeidx))

780
            # Create a single reserved plab vnode for the managment sliver.
Kirk Webb's avatar
   
Kirk Webb committed
781
782
783
784
785
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
786
787
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
Kirk Webb's avatar
Kirk Webb committed
788
                pass
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
Kirk Webb's avatar
Kirk Webb committed
812
            # Put the last vnode created into the special monitoring expt.
Kirk Webb's avatar
   
Kirk Webb committed
813
            DBQueryFatal("insert into reserved"
814
                         " (node_id, exptidx, pid, eid, rsrv_time, vname)"
815
                         " values (%s, %s, %s, %s, now(), %s)",
816
817
                         (vnodeid, mon_exptidx,
                          MONITOR_PID, MONITOR_EID, vnodeid))
Kirk Webb's avatar
Kirk Webb committed
818
819
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
820
821
822
823
824
825
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
826
                     "\n Please clean up!\n" % tbmsg, TBOPS)
Kirk Webb's avatar
   
Kirk Webb committed
827
            raise
Kirk Webb's avatar
   
Kirk Webb committed
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
        return


    def __updateNode(self, nodeid, nodeent):
        """
        updateNodeEntries() helper function.  Updates attributes for plab
        nodes passed in via the nodeent argument.
        """
        # Get the name of the control interface for plab nodes.
        defosid, controliface = self.__getNodetypeInfo()

        haslatlong = (('LATITUDE' in nodeent and 'LONGITUDE' in nodeent) and
            (nodeent['LATITUDE'] != 0 or nodeent['LONGITUDE'] != 0))
        try:
            DBQueryFatal("replace into widearea_nodeinfo"
843
844
845
846
847
                         " (node_id, contact_uid, contact_idx, hostname, site,"
                         "  latitude, longitude, bwlimit)"
                         " values (%s, %s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 'nobody', '0', nodeent['HNAME'],
                          nodeent['SITE'],
Kirk Webb's avatar
   
Kirk Webb committed
848
849
850
851
                          # Poor man's ternary operator
                          haslatlong and nodeent['LATITUDE'] or "NULL",
                          haslatlong and nodeent['LONGITUDE'] or "NULL",
                          nodeent['BWLIMIT']))
852

Kirk Webb's avatar
   
Kirk Webb committed
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'plab_fake',
                          controliface, 'ctrl'))
            pass
        except:
            print "Error updating PLAB node in DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error updating plab node in DB: %s\n" % nodeid,
                     "Some operation failed while trying to update"
                     " plab node %s in the DB:\n\n%s"
                     "\nPlease clean up!\n" % (nodeid, tbmsg), TBOPS)
            raise
Kirk Webb's avatar
Kirk Webb committed
868
        return
869

Kirk Webb's avatar
   
Kirk Webb committed
870

Kirk Webb's avatar
   
Kirk Webb committed
871
    def __updateNodeMapping(self, nodeid, chattrs):
Kirk Webb's avatar
   
Kirk Webb committed
872
        """
Kirk Webb's avatar
   
Kirk Webb committed
873
        Updates changed node attributes in the plab mapping table.
Kirk Webb's avatar
   
Kirk Webb committed
874
875
876
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
Kirk Webb's avatar
   
Kirk Webb committed
877

Kirk Webb's avatar
   
Kirk Webb committed
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
        # mapping from attrs to column names
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}

        # Get the intersection of mapping (critical) keys with those that
        # have changed.
        changedcritkeys = set(ATTR_CRIT_KEYS) & set(chattrs.keys())
        # nothing to do if none of the mapping attributes have changed.
        if not changedcritkeys:
            return
        # If the node has more than two critical attrs that have changed,
        # then move it to hwdown and raise an exception.
        if len(changedcritkeys) > 2:
            crattrs = {}
            for chkey in changedcritkeys:
                crattrs[chkey] = chattrs[chkey]
Kirk Webb's avatar
   
Kirk Webb committed
896
897
                pass
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
Kirk Webb's avatar
   
Kirk Webb committed
898
                     "%s has been moved to hwdown." % (crattrs, nodeid)
Kirk Webb's avatar
   
Kirk Webb committed
899
900
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
Kirk Webb's avatar
   
Kirk Webb committed
901
902
903
904
905
906
907
            raise MultiChangeError(nodeid, crattrs)

        # Update mapping table entry.
        updstr = ",".join(map(lambda x: "%s='%s'" %
                              (attrmap[x],chattrs[x][1]), changedcritkeys))
        DBQueryFatal("update plab_mapping set %s where node_id='%s'" %
                     (updstr, nodeid))
Kirk Webb's avatar
   
Kirk Webb committed
908
909
910
911
912
913
914
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

915
916
    def __getNodetypeInfo(self):
        """
917
918
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
919
        it doesn't change.
920
921
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
922
            if debug:
923
                print "Getting node type info"
Kirk Webb's avatar
Kirk Webb committed
924
                pass
925
926
927
928
929
930
931
932
933

            dbres = DBQueryFatal("select attrkey,attrvalue "
                                 " from node_type_attributes as a "
                                 " where type = 'pcplabphys' and "
                                 "       (a.attrkey='default_osid' or "
                                 "        a.attrkey='control_interface') "
                                 " order by attrkey")
            
            assert (len(dbres) == 2), "Failed to get node type info"
Kirk Webb's avatar
   
Kirk Webb committed
934
935
            attrdict = {}
            for attrkey, attrvalue in dbres:
Kirk Webb's avatar
   
Kirk Webb committed
936
                attrdict[attrkey] = attrvalue;
Kirk Webb's avatar
   
Kirk Webb committed
937
938
                pass
            self.__getNodetypeInfoCache = \
Kirk Webb's avatar
   
Kirk Webb committed
939
940
                                        (attrdict["default_osid"],
                                         attrdict["control_interface"])
Kirk Webb's avatar
Kirk Webb committed
941
942
            pass
        
943
944
945
946
947
948
949
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
950
        if debug:
951
952
953
954
955
956
957
958
959
960
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
Kirk Webb's avatar
Kirk Webb committed
961
            pass
962
963
        finally:
            DBQueryFatal("unlock tables")
Kirk Webb's avatar
Kirk Webb committed
964
965
            pass
        
966
967
        return nodeid, priority

Kirk Webb's avatar
   
Kirk Webb committed
968
969
970
971
972
973
974
975
976
977
978
979
980
    def __readNodeFile(self, filename):
        """
        Helper function - read in list of nodes from a file, seperated
        by arbitrary amounts of whitespace.  No comments allowed.
        """
        nodelist = []
        if os.access(filename, os.F_OK):
            nodefile = open(filename, "r+")
            nodelist = nodefile.read().split()
            nodefile.close()
            pass
        return nodelist

Kirk Webb's avatar
   
Kirk Webb committed
981
    def renew(self, inpid = None, ineid = None, force = False):
982
        """
Kirk Webb's avatar
   
Kirk Webb committed
983
984
985
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
986
        """
Kirk Webb's avatar
   
Kirk Webb committed
987

Kirk Webb's avatar
   
Kirk Webb committed
988
989
        global failedrenew # XXX
        
Kirk Webb's avatar
   
Kirk Webb committed
990
991
992
993
994
995
996
997
998
999
        now = int(time.time())
        
        if not inpid:
            res = DBQueryFatal("select pid, eid from plab_slices");
            pass
        else:
            if not ineid:
                raise RuntimeError, "renew: Must provide eid with pid."
            res = ((inpid, ineid),)
            pass
Kirk Webb's avatar
Kirk Webb committed
1000
        
1001
        loadedSlices = {}
1002
1003
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
1004
1005
        ret = 0

Kirk Webb's avatar
   
Kirk Webb committed
1006
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
1007

Kirk Webb's avatar
   
Kirk Webb committed
1008
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
1009

1010
1011
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
1012
                pass
1013
1014
1015
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
1016
                pass
1017
            
Kirk Webb's avatar
   
Kirk Webb committed
1018
            res = slice.renew(force)
Kirk Webb's avatar
   
Kirk Webb committed
1019
1020
1021
1022
1023
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
1024
1025
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
1026
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
1027
                if (slice.leaseend - now) < PLABEXPIREWARN:
1028
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
1029
1030
                    pass
                pass
1031
1032
1033
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
Kirk Webb's avatar
   
Kirk Webb committed
1034
                    pass
Kirk Webb's avatar
Kirk Webb committed
1035
                    
1036
1037
        if newfail:
            failedrenew += newfail
Kirk Webb's avatar
Kirk Webb committed
1038
            failstr = ""
1039
            for n in newfail:
Kirk Webb's avatar
   
Kirk Webb committed
1040
                failstr += "%s/%s (expires: %s UTC)\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
1041
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
Kirk Webb committed
1042
                pass
Kirk Webb's avatar
   
Kirk Webb committed
1043
            
1044
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
Kirk Webb's avatar
   
Kirk Webb committed
1045
                     "Failed to renew the following leases:\n%s" %
1046
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1047
            pass
Kirk Webb's avatar
Kirk Webb committed
1048

1049
1050
1051
        if failsoon:
            failstr = ""
            for n in failsoon:
Kirk Webb's avatar
   
Kirk Webb committed
1052
                failstr += "%s/%s: (expires: %s UTC)\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
1053
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
   
Kirk Webb committed
1054
1055
1056
1057
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
1058
            pass
Kirk Webb's avatar
   
Kirk Webb committed
1059

Kirk Webb's avatar
Kirk Webb committed
1060
1061
1062
        return
    
    pass # end class Plab
Kirk Webb's avatar
   
Kirk Webb committed
1063
1064
# AOP wrappers for class Plab
wrap_around(Plab.createSlice, timeAdvice)
1065
1066
1067
1068
1069

#
# Slice abstraction
#
class Slice:
Kirk Webb's avatar
   
Kirk Webb committed
1070
1071

    def __init__(self, plab, pid, eid, slicename = None):
1072
1073
        self.plab = plab
        self.pid, self.eid = pid, eid
1074
        self.slicemeta = None
1075
        self.exptidx = None
Kirk Webb's avatar
   
Kirk Webb committed
1076
        self.slicename = slicename
Kirk Webb's avatar
   
Kirk Webb committed
1077
        self.description = DEF_SLICE_DESC
Kirk Webb's avatar
   
Kirk Webb committed
1078
        return
1079
1080
1081
1082
1083
1084
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
Kirk Webb's avatar
   
Kirk Webb committed
1085
1086
1087

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
Kirk Webb's avatar
   
Kirk Webb committed
1088
1089
            self.slicename   = PLAB_SVC_SLICENAME
            self.description = PLAB_SVC_SLICEDESC
Kirk Webb's avatar
   
Kirk Webb committed
1090
1091
            adminbit = 1
            pass
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101

        res = DBQueryFatal("select idx, expt_name from experiments "
                           "where pid=%s "
                           "and eid=%s",
                           (self.pid, self.eid))
        if not len(res):
            raise RuntimeError, \
                  "Didn't get any results while looking up info on " \
                  "experiment %s/%s" % (self.pid, self.eid)
        (eindex, descr) = res[0]
Kirk Webb's avatar
   
Kirk Webb committed
1102
        
Kirk Webb's avatar
   
Kirk Webb committed
1103
1104
1105
        if not self.slicename:
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
            pass
Kirk Webb's avatar
   
Kirk Webb committed
1106

1107
1108
1109
        self.description = descr
        self.exptidx = eindex

1110
        print "Creating Plab slice %s." % self.slicename
Kirk Webb's avatar
   
Kirk Webb committed
1111

1112
        try:
Kirk Webb's avatar
Kirk Webb committed
1113
1114
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
            
            # Do compat calls right after main agent calls.
            if compat_mode:
                try:
                    self.plab.compat_agent.createSlice(self)
                except:
                    print "WARNING: compat agent failed in createSlice; " \
                          "\n  watch for inconsistent DB state!"
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
1125