libplab.py.in 60.1 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2
3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004, 2006 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5
6
7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9
10
11
12
13
14
15
16
17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18
19
20
"""

import sys
21
22
23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
   
Kirk Webb committed
25
import traceback
Kirk Webb's avatar
   
Kirk Webb committed
26
import signal
Kirk Webb's avatar
   
Kirk Webb committed
27
import socket
Kirk Webb's avatar
Kirk Webb committed
28
29
30
import httplib
import xml.parsers.expat
import re
Kirk Webb's avatar
   
Kirk Webb committed
31
import calendar
Kirk Webb's avatar
   
Kirk Webb committed
32
import shlex
Kirk Webb's avatar
   
Kirk Webb committed
33

Kirk Webb's avatar
   
Kirk Webb committed
34
from popen2 import Popen4
Kirk Webb's avatar
   
Kirk Webb committed
35
from warnings import warn
36

37
38
39
40
41
42
43
44
45
46
47
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
Kirk Webb's avatar
   
Kirk Webb committed
48
from mod_PLCNM import mod_PLCNM
49
50

agents = {'PLC'    : mod_PLC,
Kirk Webb's avatar
   
Kirk Webb committed
51
52
          'dslice' : mod_dslice,
          'PLCNM'  : mod_PLCNM}
53

Kirk Webb's avatar
   
Kirk Webb committed
54
55
56
57
58
59
60
#
# Initialize the AOP stuff
#
from aspects import wrap_around
from timer_advisories import initTimeAdvice, timeAdvice
initTimeAdvice("plabtiming")

Kirk Webb's avatar
Kirk Webb committed
61
62
63
64
65
66
#
# output control vars
#
verbose = 0
debug = 0

67
68
69
#
# Constants
#
Kirk Webb's avatar
   
Kirk Webb committed
70
DEF_AGENT = "PLCNM";
71

72
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
Kirk Webb's avatar
   
Kirk Webb committed
73
74
75

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
76
NODEPROBEINT  = 30
77

Kirk Webb's avatar
   
Kirk Webb committed
78
79
USERNODE = "@USERNODE@"
TBOPS = "@TBOPSEMAIL_NOSLASH@"
80
MAILTAG = "@THISHOMEBASE@"
Kirk Webb's avatar
   
Kirk Webb committed
81
SLICE_ALIAS_DIR = "/etc/mail/plab-slice-addrs"
82

83
RESERVED_PID = "emulab-ops"
84
RESERVED_EID = "hwdown"       # start life in hwdown
85
86
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
87

Kirk Webb's avatar
   
Kirk Webb committed
88
89
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18",
                        "205.124.244.178", )
Kirk Webb's avatar
   
Kirk Webb committed
90
91
92
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
93
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
94
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
Kirk Webb's avatar
   
Kirk Webb committed
95
96
97

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
98

99
100
101
102
# 'critical' node identifiers - those that are actually used to uniquely
# identify a planetlab node
ATTR_CRIT_KEYS = ('HNAME', 'IP', 'PLABID', 'MAC',)

103
104
105
106
# The amount by which latitude and longitude are allowed to differ before we
# classify them ask changed
LATLONG_DELTA = 0.001

Kirk Webb's avatar
   
Kirk Webb committed
107
PLABNODE = "@prefix@/sbin/plabnode"
108
SSH = "@prefix@/bin/sshtb"
109
NAMED_SETUP = "@prefix@/sbin/named_setup"
Kirk Webb's avatar
   
Kirk Webb committed
110
PELAB_PUSH  = "@prefix@/sbin/pelab_opspush"
111

Kirk Webb's avatar
Kirk Webb committed
112
113
114
115
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
Kirk Webb's avatar
   
Kirk Webb committed
116
117
IGNORED_NODES_FILE = "@prefix@/etc/plab/IGNOREDNODES"
ALLOWED_NODES_FILE = "@prefix@/etc/plab/ALLOWEDNODES"
118

119
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
120
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
Kirk Webb's avatar
Kirk Webb committed
121
122
123
124
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
Kirk Webb's avatar
   
Kirk Webb committed
125
PLAB_SVC_SLICENAME = "utah_svc_slice"
Kirk Webb's avatar
   
Kirk Webb committed
126
127
128
129
PLAB_SVC_SLICEDESC = "Emulab management service slice. Performs periodic " \
                     "checkins with Emulab central, and routes events for " \
                     "other Emulab slices. Slivers in this slice should " \
                     "only interact with other PlanetLab machines, and Emulab."
Kirk Webb's avatar
   
Kirk Webb committed
130
131
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"
Kirk Webb's avatar
   
Kirk Webb committed
132
DEF_SLICE_DESC = "Slice created by Emulab"
Kirk Webb's avatar
   
Kirk Webb committed
133

Kirk Webb's avatar
   
Kirk Webb committed
134
135
PLABEXPIREWARN = 1*WEEK        # one week advance warning for slice expiration.
NODEEXPIREWARN = 2*WEEK+2*DAY  # about two weeks advance warning for slivers.
136

137
138
139
140
141
#
# var to track failed renewals
#
failedrenew = []

142
143
144
145
146
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

Kirk Webb's avatar
   
Kirk Webb committed
147
148
149
150
151
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

152

153
154
155
#
# Plab abstraction
#
Kirk Webb's avatar
Kirk Webb committed
156

Kirk Webb's avatar
   
Kirk Webb committed
157
158
159
160
161
162
163
164
165
166
#
# Multiple attribute change exception
#
class MultiChangeError(Exception):
    def __init__(self, nodeid, chattrs={}):
        self.nodeid = nodeid
        self.chattrs = chattrs
        pass
    pass

Kirk Webb's avatar
   
Kirk Webb committed
167

Kirk Webb's avatar
Kirk Webb committed
168
169
170
#
# Class responsible for parsing planetlab sites file
#
Kirk Webb's avatar
Kirk Webb committed
171
172
173
174
175
176
177
178
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
179
180
        self.__latitude = 0
        self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
208
209
210
            if attrs.has_key('LATITUDE'):
                self.__latitude = attrs['LATITUDE']
            else:
Kirk Webb's avatar
   
Kirk Webb committed
211
                self.__latitude = 0
212
213
214
215
            if attrs.has_key('LONGITUDE'):
                self.__longitude = attrs['LONGITUDE']
            else:
                self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
216
217
218
            pass
        
        elif name == "HOST":
Kirk Webb's avatar
   
Kirk Webb committed
219
220
221
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
222
223
224
225
226
227
228
            self.__hosts.append({'HNAME'     : attrs['NAME'],
                                 'IP'        : attrs['IP'],
                                 'PLABID'    : attrs['NODE_ID'],
                                 'MAC'       : attrs['MAC'],
                                 'SITE'      : self.__sitename,
                                 'LATITUDE'  : self.__latitude,
                                 'LONGITUDE' : self.__longitude})
Kirk Webb's avatar
Kirk Webb committed
229
230
231
232
233
234
235
236
237
238
239
240
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
241
242
            self.__latitude = 0
            self.__longitude = 0
Kirk Webb's avatar
Kirk Webb committed
243
244
245
            pass
        return

Kirk Webb's avatar
   
Kirk Webb committed
246

247
class Plab:
Kirk Webb's avatar
Kirk Webb committed
248
    def __init__(self, agent=None):
249
250
251
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
Kirk Webb's avatar
Kirk Webb committed
252
        if debug:
253
254
255
            print "Using module: %s" % self.agent.modname
            pass
        pass
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

Kirk Webb's avatar
Kirk Webb committed
273
    def updateNodeEntries(self, ignorenew = False):
274
        """
Kirk Webb's avatar
Kirk Webb committed
275
        Finds out which Plab nodes are available, and
276
277
278
279
280
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
Kirk Webb's avatar
Kirk Webb committed
281
        discovered.
282
283
284
285
286
287

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
288
289
290
        
        print "Getting available Plab nodes ..."

291
        avail = []
292
        try:
Kirk Webb's avatar
Kirk Webb committed
293
294
295
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
296
        # XXX: rewrite to use more elegant exception info gathering.
297
298
        except:
            extype, exval, extrace = sys.exc_info()
299
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
300
            if debug:
301
302
303
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
Kirk Webb's avatar
   
Kirk Webb committed
304

Kirk Webb's avatar
Kirk Webb committed
305
        if debug:
306
307
            print "Got advertisement list:"
            print avail
Kirk Webb's avatar
Kirk Webb committed
308
            pass
Kirk Webb's avatar
   
Kirk Webb committed
309

Kirk Webb's avatar
   
Kirk Webb committed
310
311
312
        ignored_nodes = self.__readNodeFile(IGNORED_NODES_FILE)
        allowed_nodes = self.__readNodeFile(ALLOWED_NODES_FILE)

Kirk Webb's avatar
   
Kirk Webb committed
313
314
315
316
317
318
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
Kirk Webb's avatar
   
Kirk Webb committed
319
        if len(allowed_nodes) or len(ignored_nodes):
Kirk Webb's avatar
   
Kirk Webb committed
320
            allowed = []
Kirk Webb's avatar
Kirk Webb committed
321
            for nodeent in avail:
Kirk Webb's avatar
   
Kirk Webb committed
322
                if nodeent['PLABID'] in ignored_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
323
                    continue
Kirk Webb's avatar
   
Kirk Webb committed
324
325
                elif len(allowed_nodes):
                    if nodeent['IP'] in allowed_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
326
327
328
329
330
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
331
332
                    pass
                pass
Kirk Webb's avatar
   
Kirk Webb committed
333
334
335
336
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
Kirk Webb's avatar
Kirk Webb committed
337
            pass
338

Kirk Webb's avatar
   
Kirk Webb committed
339
340
341
342
343
344
345
346
347
348
349
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

350
        # Get node info we already have.
351
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
352
        if debug:
353
354
            print "Got known pnodes:"
            print known
Kirk Webb's avatar
Kirk Webb committed
355
            pass
356

Kirk Webb's avatar
Kirk Webb committed
357
        # Create list of nodes to add or update
Kirk Webb's avatar
   
Kirk Webb committed
358
359
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
Kirk Webb's avatar
Kirk Webb committed
360
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
361
362
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
363
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
Kirk Webb's avatar
   
Kirk Webb committed
364
365
366
            # Determine if we already know about this node.
            matchres = self.__matchPlabNode(nodeent, known)
            if not matchres:
Kirk Webb's avatar
   
Kirk Webb committed
367
                toadd.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
368
                pass
Kirk Webb's avatar
   
Kirk Webb committed
369
370
            elif len(matchres[1]):
                toupdate.append((nodeent,matchres))
Kirk Webb's avatar
Kirk Webb committed
371
                pass
Kirk Webb's avatar
   
Kirk Webb committed
372
            pass
Kirk Webb's avatar
Kirk Webb committed
373

Kirk Webb's avatar
   
Kirk Webb committed
374
375
        # Process the list of nodes to add
        addstr = ""
376
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
377
            # Are we ignoring new entries?
378
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
379
                if verbose:
380
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
Kirk Webb's avatar
Kirk Webb committed
381
382
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
383
            # If not ignoring, do the addition/update.
384
            else:
Kirk Webb's avatar
   
Kirk Webb committed
385
386
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
387
                    # Get the linktype here so we can report it in email.
Kirk Webb's avatar
Kirk Webb committed
388
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
389
                    if debug:
Kirk Webb's avatar
Kirk Webb committed
390
391
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
392
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
393
394
395
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
Kirk Webb's avatar
Kirk Webb committed
396
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
397
                              (nodeent['PLABID'],
Kirk Webb's avatar
Kirk Webb committed
398
399
400
401
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
Kirk Webb's avatar
   
Kirk Webb committed
402
                    addstr += nodestr
Kirk Webb's avatar
Kirk Webb committed
403
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
404
405
                pass
            pass
406

Kirk Webb's avatar
   
Kirk Webb committed
407
        # Process node updates.
Kirk Webb's avatar
   
Kirk Webb committed
408
        updstr = ""
Kirk Webb's avatar
   
Kirk Webb committed
409
        chgerrstr = ""
Kirk Webb's avatar
   
Kirk Webb committed
410
411
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
Kirk Webb's avatar
   
Kirk Webb committed
412
            for updent,updmapent in toupdate:
Kirk Webb's avatar
   
Kirk Webb committed
413
414
415
416
417
418
419
420
421
422
423
424
                try:
                    self.__updateNodeMapping(updmapent)
                    pass
                except MultiChangeError, e:
                    print "%s not updated: Too many attribute changes." % \
                          e.nodeid
                    chgerrstr += "%s:\n" % e.nodeid
                    for attr,val in e.chattrs.items():
                        chgerrstr += "\t%s:\t%s => %s\n" % (attr,val[0],val[1])
                        pass
                    chgerrstr += "\n"
                    continue
Kirk Webb's avatar
   
Kirk Webb committed
425
                self.__addNode(updent, updmapent)
Kirk Webb's avatar
   
Kirk Webb committed
426
                # Add a line for the add/update message.
Kirk Webb's avatar
   
Kirk Webb committed
427
428
                nodestr = updmapent[0] + "\n"
                for attr,val in updmapent[1].items():
Kirk Webb's avatar
   
Kirk Webb committed
429
430
431
                    nodestr += "\t%s:\t%s => %s\n" % (attr,val[0],val[1])
                    pass
                updstr += nodestr + "\n"
Kirk Webb's avatar
Kirk Webb committed
432
433
                pass
            pass
Kirk Webb's avatar
   
Kirk Webb committed
434

Kirk Webb's avatar
   
Kirk Webb committed
435
436
437
438
439
440
441
442
        if chgerrstr:
            SENDMAIL(TBOPS,
                     "Two or more changes detected for some plab nodes",
                     "Two or more distinguishing attributes have changed "
                     "on the following planetlab nodes:\n\n%s\n" % chgerrstr,
                     TBOPS)
            pass

Kirk Webb's avatar
   
Kirk Webb committed
443
444
445
446
        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
Kirk Webb's avatar
   
Kirk Webb committed
447
448
            print "Pushing out site_mapping ..."
            os.spawnl(os.P_WAIT, PELAB_PUSH, PELAB_PUSH)
Kirk Webb's avatar
   
Kirk Webb committed
449
450
451
452
453
454
455
456
457
458
459
460
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
Kirk Webb's avatar
Kirk Webb committed
461
        return
462

Kirk Webb's avatar
   
Kirk Webb committed
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
481
482
483
                elif (attr == "LATITUDE") or (attr == "LONGITUDE"):
                    # Special rules for latitude and longitude to avoid
                    # FP errors
484
485
486
487
488
489
                    if (ent[attr] != None and plabent[attr] != None) \
                           and (ent[attr] != "" and plabent[attr] != "") \
                           and ((float(ent[attr]) > \
                                 (float(plabent[attr]) + LATLONG_DELTA)) \
                                or (float(ent[attr]) < \
                                    (float(plabent[attr]) - LATLONG_DELTA))):
490
491
492
493
                        diff[attr] = (ent[attr], plabent[attr])
                    else:
                        same[attr] = ent[attr]
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
494
495
496
497
498
499
500
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
501
502
            # Only consider these to be the same if at least one 'critical'
            # attr is the same
Kirk Webb's avatar
   
Kirk Webb committed
503
            if len(same):
504
505
506
                for attr in same:
                    if attr in ATTR_CRIT_KEYS:
                        return (nid, diff)
Kirk Webb's avatar
   
Kirk Webb committed
507
508
509
            pass
        return ()

510
511
512
513
514
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
515
516
517
518
519
520
521
        res = DBQueryFatal("select plab_mapping.node_id,plab_id,"
                           "plab_mapping.hostname,IP,mac,site,latitude,"
                           "longitude"
                           " from plab_mapping"
                           " left join widearea_nodeinfo on"
                           "    plab_mapping.node_id = "
                           "    widearea_nodeinfo.node_id")
Kirk Webb's avatar
   
Kirk Webb committed
522
        
523
        ret = {}
524
525
526
527
528
529
530
531
        for nodeid, plabid, hostname, ip, mac, site, latitude, longitude in res:
            ret[nodeid] = {'PLABID'    : plabid,
                           'HNAME'     : hostname,
                           'IP'        : ip,
                           'MAC'       : mac,
                           'SITE'      : site,
                           'LATITUDE'  : latitude,
                           'LONGITUDE' : longitude}
Kirk Webb's avatar
Kirk Webb committed
532
            pass
Kirk Webb's avatar
   
Kirk Webb committed
533
534
535
536
537
538
539
540
541
542
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
543
        return ret
Kirk Webb's avatar
   
Kirk Webb committed
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        uniqattrs = ['PLABID', 'HNAME', 'IP', 'MAC']
        attrs = {}
        dups = {}
        
        for ent in nodelist:
            for attr in uniqattrs:
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
Kirk Webb's avatar
   
Kirk Webb committed
569
        
Kirk Webb's avatar
Kirk Webb committed
570
    def __findLinkType(self, nodeent):
571
572
573
574
575
576
577
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

Kirk Webb's avatar
Kirk Webb committed
578
        This can't detect DSL links..
579
        """
580
        # Is host international (or flux/emulab local)?
581
        from socket import gethostbyaddr, getfqdn, herror
Kirk Webb's avatar
Kirk Webb committed
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
604
        # Is host on I2?
Kirk Webb's avatar
Kirk Webb committed
605
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
606
607
608
609
610
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
611
612
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
613

614
615
616
617
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
                break
        else:
Kirk Webb's avatar
Kirk Webb committed
618
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
619

Kirk Webb's avatar
   
Kirk Webb committed
620
621
        # We don't know - must manually classify.
        nodeent['LINKTYPE'] = "*Unknown*"
Kirk Webb's avatar
Kirk Webb committed
622
        return
623

Kirk Webb's avatar
   
Kirk Webb committed
624
    def __addNode(self, nodeent, updent = ()):
625
626
627
628
629
        """
        getFree helper function.  Adds a new Plab pnode and associated
        vnodes to the DB.  linktype should be one of (inet2, inet, intl,
        dsl).
        """
Kirk Webb's avatar
   
Kirk Webb committed
630
631
        # block out common termination signals while adding a node
        osigs = disable_sigs(TERMSIGS)
632
        defosid, controliface = self.__getNodetypeInfo()
Kirk Webb's avatar
Kirk Webb committed
633
        hostonly = nodeent['HNAME'].replace(".", "-")
Kirk Webb's avatar
   
Kirk Webb committed
634
        # These will be setup properly below.
Kirk Webb's avatar
   
Kirk Webb committed
635
636
        nidnum = 0
        priority = 0
Kirk Webb's avatar
   
Kirk Webb committed
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
        nodeid = ""
        vnodeprefix = ""

        # Setup nodeid according to whether or not we were passed in an
        # update entry.
        if updent:
            nodeid = updent[0]
            print "Updating node %s" % nodeid
            pass
        else:
            nidnum, priority = self.__nextFreeNodeid()
            nodeid = "%s%d" % (NODEPREFIX, nidnum)
            vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
            print "Creating pnode %s as %s, priority %d." % \
                  (nodeent['IP'], nodeid, priority)
            pass
653

654
655
        haslatlong = (('LATITUDE' in nodeent and 'LONGITUDE' in nodeent) and
            (nodeent['LATITUDE'] != 0 or nodeent['LONGITUDE'] != 0))
Kirk Webb's avatar
   
Kirk Webb committed
656
        try:
Kirk Webb's avatar
   
Kirk Webb committed
657
            DBQueryFatal("replace into widearea_nodeinfo"
658
659
660
661
662
663
664
                         " (node_id, contact_uid, hostname, site, latitude, "
                         "  longitude)"
                         " values (%s, %s, %s, %s, %s, %s)",
                         (nodeid, 'nobody', nodeent['HNAME'], nodeent['SITE'],
                          # Poor man's ternary operator
                          haslatlong and nodeent['LATITUDE'] or "NULL",
                          haslatlong and nodeent['LONGITUDE'] or "NULL"))
Kirk Webb's avatar
   
Kirk Webb committed
665
666
667
668
669

            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
Kirk Webb's avatar
   
Kirk Webb committed
670
                         (nodeid, 0, 1, nodeent['IP'], 'plab_fake',
Kirk Webb's avatar
   
Kirk Webb committed
671
672
673
674
675
676
677
                          controliface, 'ctrl'))

            # Don't do anything else if we are only updating the node
            if updent:
                enable_sigs(osigs)
                return

Kirk Webb's avatar
Kirk Webb committed
678
            DBQueryFatal("replace into nodes"
679
                         " (node_id, type, phys_nodeid, role, priority,"
Kirk Webb's avatar
   
Kirk Webb committed
680
                         "  op_mode, def_boot_osid,"
681
                         "  allocstate, allocstate_timestamp,"
Kirk Webb's avatar
   
Kirk Webb committed
682
                         "  eventstate, state_timestamp)"
683
                         " values (%s, %s, %s, %s, %s,"
Kirk Webb's avatar
   
Kirk Webb committed
684
685
686
687
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
688
                          'FREE_CLEAN',
Kirk Webb's avatar
   
Kirk Webb committed
689
                          'ISUP'))
690

691
692
693
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
Kirk Webb's avatar
Kirk Webb committed
694
                         (nodeid))
695

Kirk Webb's avatar
Kirk Webb committed
696
            DBQueryFatal("replace into reserved"
Kirk Webb's avatar
   
Kirk Webb committed
697
698
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
699
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))
Kirk Webb's avatar
   
Kirk Webb committed
700

Kirk Webb's avatar
   
Kirk Webb committed
701
702
            # XXX: This should probably be checked and updated if necessary
            #      when updating.
Kirk Webb's avatar
Kirk Webb committed
703
            DBQueryFatal("replace into node_auxtypes"
Kirk Webb's avatar
   
Kirk Webb committed
704
705
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
706
                         (nodeid, nodeent['LINKTYPE'], 1))
Kirk Webb's avatar
   
Kirk Webb committed
707
            
Kirk Webb's avatar
Kirk Webb committed
708
            DBQueryFatal("replace into node_auxtypes"
709
710
711
712
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
Kirk Webb's avatar
Kirk Webb committed
713
            DBQueryFatal("replace into node_status"
714
715
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
716
                         (nodeid, 'down'))
Kirk Webb's avatar
   
Kirk Webb committed
717

Kirk Webb's avatar
   
Kirk Webb committed
718
719
720
721
722
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
723

Kirk Webb's avatar
   
Kirk Webb committed
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
            #
            # NowAdd the site_mapping entry for this node.
            #
            
            # See if we know about the associated site - grab idx if so
            siteidx = 0
            nodeidx = 1
            siteres = DBQueryFatal("select site_idx, node_idx from "
                                   " plab_site_mapping where site_name=%s",
                                   nodeent['SITE']);
            if len(siteres):
                # There are already nodes listed for this site, so get
                # the next node id.
                siteidx = siteres[0][0]
                for (foo, idx) in siteres:
                    if idx > nodeidx: nodeidx = idx
                    pass
                nodeidx += 1
                pass
            else:
                # No nodes listed for site, so get the largest site_idx
                # in the DB so far, and increment cuz we're going to add
                # a new one.
                maxres = DBQueryFatal("select MAX(site_idx) from "
                                      " plab_site_mapping")
                try:
                    siteidx = int(maxres[0][0]) + 1
                    pass
                except ValueError:
                    siteidx = 1
                    pass
                pass
            # Create site_mapping entry, optionally creating new site idx
            # via not specifying the site_idx field (field is auto_increment)
            DBQueryFatal("insert into plab_site_mapping "
                         " values (%s, %s, %s, %s)",
                         (nodeent['SITE'], siteidx, nodeid, nodeidx))

762
            # Create a single reserved plab vnode for the managment sliver.
Kirk Webb's avatar
   
Kirk Webb committed
763
764
765
766
767
768
769
770
            # XXX I left it as "20" cause of all the existing ones.
            # XXXX I set it to 1 due to the above comment (correct?)
            #      since we are re-creating anyway.            
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
771
772
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
Kirk Webb's avatar
Kirk Webb committed
773
                pass
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
Kirk Webb's avatar
Kirk Webb committed
797
            # Put the last vnode created into the special monitoring expt.
Kirk Webb's avatar
   
Kirk Webb committed
798
799
800
801
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
Kirk Webb's avatar
Kirk Webb committed
802
803
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
804
805
806
807
808
809
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
810
                     "\n Please clean up!\n" % tbmsg, TBOPS)
Kirk Webb's avatar
   
Kirk Webb committed
811
812
            enable_sigs(osigs)
            raise
813

Kirk Webb's avatar
   
Kirk Webb committed
814
815
        # last but not least, unblock signals
        enable_sigs(osigs)
Kirk Webb's avatar
Kirk Webb committed
816
        return
817

Kirk Webb's avatar
   
Kirk Webb committed
818

Kirk Webb's avatar
   
Kirk Webb committed
819
    def __updateNodeMapping(self, updent):
Kirk Webb's avatar
   
Kirk Webb committed
820
821
822
823
824
        """
        Updates changed node attributes in the mapping table.
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
        # mapping from attrs to which table they belong in
        tablemap = {'PLABID'    : 'plab_mapping',
                    'HNAME'     : 'plab_mapping',
                    'IP'        : 'plab_mapping',
                    'MAC'       : 'plab_mapping',
                    'SITE'      : 'widearea_nodeinfo',
                    'LATITUDE'  : 'widearea_nodeinfo',
                    'LONGITUDE' : 'widearea_nodeinfo',}
        # mapping from attrs to column names
        attrmap = {'plab_mapping' : {'PLABID' : 'plab_id',
                                     'HNAME'  : 'hostname',
                                     'IP'     : 'IP',
                                     'MAC'    : 'mac'},
                   'widearea_nodeinfo' : {'SITE'      : 'site',
                                          'LATITUDE'  : 'latitude',
                                          'LONGITUDE' : 'longitude',}}

Kirk Webb's avatar
   
Kirk Webb committed
842
        nodeid, chattrs = updent
Kirk Webb's avatar
   
Kirk Webb committed
843
                
844
845
846
847
848
849
850
        # seperate out attrs by table
        chattrs_by_table = {}
        for attr in chattrs:
            table = tablemap[attr]
            if table not in chattrs_by_table:
                chattrs_by_table[table] = []
            chattrs_by_table[table].append(attr)
Kirk Webb's avatar
   
Kirk Webb committed
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
            pass

        # If more than 2 attributes have changed in the plab_mapping table,
        # then move the node to hwdown and report the issue.
        if chattrs_by_table.has_key('plab_mapping') and \
               len(chattrs_by_table['plab_mapping']) > 2:
            pmattrs = {}
            for attr in chattrs_by_table['plab_mapping']:
                pmattrs[attr] = chattrs[attr]
                pass
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
                     "%s has been moved to hwdown." % (pmattrs, nodeid)
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
            raise MultiChangeError(nodeid, pmattrs)
866
867
868
869

        # update each table
        for table in chattrs_by_table:
            updstr = ",".join(map(lambda x: "%s='%s'" %
Kirk Webb's avatar
   
Kirk Webb committed
870
871
872
                (attrmap[table][x],chattrs[x][1]), chattrs_by_table[table]))
            DBQueryFatal("update %s set %s where node_id='%s'" %
                         (table, updstr, nodeid))
Kirk Webb's avatar
   
Kirk Webb committed
873
874
875
876
877
878
879
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

880
881
    def __getNodetypeInfo(self):
        """
882
883
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
884
        it doesn't change.
885
886
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
887
            if debug:
888
                print "Getting node type info"
Kirk Webb's avatar
Kirk Webb committed
889
                pass
890
891
892
893
894
895
896
897
898

            dbres = DBQueryFatal("select attrkey,attrvalue "
                                 " from node_type_attributes as a "
                                 " where type = 'pcplabphys' and "
                                 "       (a.attrkey='default_osid' or "
                                 "        a.attrkey='control_interface') "
                                 " order by attrkey")
            
            assert (len(dbres) == 2), "Failed to get node type info"
Kirk Webb's avatar
   
Kirk Webb committed
899
900
            attrdict = {}
            for attrkey, attrvalue in dbres:
Kirk Webb's avatar
   
Kirk Webb committed
901
                attrdict[attrkey] = attrvalue;
Kirk Webb's avatar
   
Kirk Webb committed
902
903
                pass
            self.__getNodetypeInfoCache = \
Kirk Webb's avatar
   
Kirk Webb committed
904
905
                                        (attrdict["default_osid"],
                                         attrdict["control_interface"])
Kirk Webb's avatar
Kirk Webb committed
906
907
            pass
        
908
909
910
911
912
913
914
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
915
        if debug:
916
917
918
919
920
921
922
923
924
925
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
Kirk Webb's avatar
Kirk Webb committed
926
            pass
927
928
        finally:
            DBQueryFatal("unlock tables")
Kirk Webb's avatar
Kirk Webb committed
929
930
            pass
        
931
932
        return nodeid, priority

Kirk Webb's avatar
   
Kirk Webb committed
933
934
935
936
937
938
939
940
941
942
943
944
945
    def __readNodeFile(self, filename):
        """
        Helper function - read in list of nodes from a file, seperated
        by arbitrary amounts of whitespace.  No comments allowed.
        """
        nodelist = []
        if os.access(filename, os.F_OK):
            nodefile = open(filename, "r+")
            nodelist = nodefile.read().split()
            nodefile.close()
            pass
        return nodelist

Kirk Webb's avatar
   
Kirk Webb committed
946
    def renew(self, inpid = None, ineid = None, force = False):
947
        """
Kirk Webb's avatar
   
Kirk Webb committed
948
949
950
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
951
        """
Kirk Webb's avatar
   
Kirk Webb committed
952

Kirk Webb's avatar
   
Kirk Webb committed
953
954
        global failedrenew # XXX
        
Kirk Webb's avatar
   
Kirk Webb committed
955
956
957
958
959
960
961
962
963
964
        now = int(time.time())
        
        if not inpid:
            res = DBQueryFatal("select pid, eid from plab_slices");
            pass
        else:
            if not ineid:
                raise RuntimeError, "renew: Must provide eid with pid."
            res = ((inpid, ineid),)
            pass
Kirk Webb's avatar
Kirk Webb committed
965
        
966
        loadedSlices = {}
967
968
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
969
970
        ret = 0

Kirk Webb's avatar
   
Kirk Webb committed
971
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
972

Kirk Webb's avatar
   
Kirk Webb committed
973
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
974

975
976
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
977
                pass
978
979
980
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
981
                pass
982
            
Kirk Webb's avatar
   
Kirk Webb committed
983
            res = slice.renew(force)
Kirk Webb's avatar
   
Kirk Webb committed
984
985
986
987
988
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
989
990
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
991
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
992
                if (slice.leaseend - now) < PLABEXPIREWARN:
993
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
994
995
                    pass
                pass
996
997
998
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
Kirk Webb's avatar
   
Kirk Webb committed
999
                    pass
Kirk Webb's avatar
Kirk Webb committed
1000
                    
For faster browsing, not all history is shown. View entire blame