libplab.py.in 52.7 KB
Newer Older
1
# -*- python -*-
Kirk Webb's avatar
Kirk Webb committed
2
3
#
# EMULAB-COPYRIGHT
4
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
Kirk Webb's avatar
Kirk Webb committed
5
6
7
# All rights reserved.
#

8
"""
Kirk Webb's avatar
Kirk Webb committed
9
10
11
12
13
14
15
16
17
Library for interfacing with Plab.  This abstracts out the concepts of
Plab central, slices, and nodes.  All data (except static things like
certificates) is kept in the Emulab DB.  Unlike the regular dslice
svm, this one supports dynamically changing which nodes are in a
slice.

This requires an already obtained dslice certficate and key.  By
default it expects to find these in the @prefix@/etc/plab/
subdirectory.
18
19
20
"""

import sys
21
22
23
sys.path.append("@prefix@/lib")

import os, time
24
import string
Kirk Webb's avatar
   
Kirk Webb committed
25
import traceback
Kirk Webb's avatar
   
Kirk Webb committed
26
import signal
Kirk Webb's avatar
   
Kirk Webb committed
27
import socket
Kirk Webb's avatar
Kirk Webb committed
28
29
30
import httplib
import xml.parsers.expat
import re
Kirk Webb's avatar
   
Kirk Webb committed
31
import calendar
Kirk Webb's avatar
   
Kirk Webb committed
32
import shlex
Kirk Webb's avatar
   
Kirk Webb committed
33

Kirk Webb's avatar
   
Kirk Webb committed
34
from popen2 import Popen4
Kirk Webb's avatar
   
Kirk Webb committed
35
from warnings import warn
36

37
38
39
40
41
42
43
44
45
46
47
#
# Testbed and DB access libs
#
from libtestbed import *
from libdb import *

#
# Plab modules to import
#
from mod_PLC import mod_PLC
from mod_dslice import mod_dslice
Kirk Webb's avatar
   
Kirk Webb committed
48
from mod_PLCNM import mod_PLCNM
49
50

agents = {'PLC'    : mod_PLC,
Kirk Webb's avatar
   
Kirk Webb committed
51
52
          'dslice' : mod_dslice,
          'PLCNM'  : mod_PLCNM}
53

Kirk Webb's avatar
Kirk Webb committed
54
55
56
57
58
59
#
# output control vars
#
verbose = 0
debug = 0

60
61
62
#
# Constants
#
Kirk Webb's avatar
   
Kirk Webb committed
63
DEF_AGENT = "PLCNM";
64

65
RENEW_TIME = 2*24*60*60  # Renew two days before lease expires
Kirk Webb's avatar
   
Kirk Webb committed
66
67
68

RENEW_TIMEOUT = 1*60     # give the node manager a minute to respond to renew
FREE_TIMEOUT  = 1*60     # give the node manager a minute to respond to free
69
NODEPROBEINT  = 30
70

71
72
73
TBOPS = "@TBOPSEMAIL@".replace("\\","")
MAILTAG = "@THISHOMEBASE@"

74
RESERVED_PID = "emulab-ops"
75
RESERVED_EID = "hwdown"       # start life in hwdown
76
77
MONITOR_PID  = "emulab-ops"
MONITOR_EID  = "plab-monitor"
78

Kirk Webb's avatar
   
Kirk Webb committed
79
80
81
82
MAGIC_INET2_GATEWAYS = ("205.124.237.10",  "205.124.244.18", )
MAGIC_INET_GATEWAYS =  ("205.124.244.150", "205.124.239.185",
                        "205.124.244.154", "205.124.244.138",
                        "205.124.244.130", )
83
LOCAL_PLAB_DOMAIN = ".flux.utah.edu"
84
LOCAL_PLAB_LINKTYPE = "pcplabinet2"
Kirk Webb's avatar
   
Kirk Webb committed
85
86
87

# allowed nil/unknown values (sentinels).
ATTR_NIL_VALUES = ('None',)
88

Kirk Webb's avatar
   
Kirk Webb committed
89
PLABNODE = "@prefix@/sbin/plabnode"
90
SSH = "@prefix@/bin/sshtb"
91
NAMED_SETUP = "@prefix@/sbin/named_setup"
Kirk Webb's avatar
   
Kirk Webb committed
92
PELAB_PUSH  = "@prefix@/sbin/pelab_opspush"
93

Kirk Webb's avatar
Kirk Webb committed
94
95
96
97
ROOTBALL_URL = "http://localhost:1492/" # ensure this ends in a slash

DEF_PLAB_URL = "www.planet-lab.org"
DEF_SITE_XML = "/xml/sites.xml"
Kirk Webb's avatar
   
Kirk Webb committed
98
99
IGNORED_NODES_FILE = "@prefix@/etc/plab/IGNOREDNODES"
ALLOWED_NODES_FILE = "@prefix@/etc/plab/ALLOWEDNODES"
100

101
DEF_ROOTBALL_NAME = "@PLAB_ROOTBALL@"
102
SLICEPREFIX = "@PLAB_SLICEPREFIX@"
Kirk Webb's avatar
Kirk Webb committed
103
104
105
106
NODEPREFIX  = "plab"

BADSITECHARS = re.compile(r"\W+")
PLABBASEPRIO = 20000
Kirk Webb's avatar
   
Kirk Webb committed
107
PLAB_SVC_SLICENAME = "utah_svc_slice"
Kirk Webb's avatar
   
Kirk Webb committed
108
109
110
111
PLAB_SVC_SLICEDESC = "Emulab management service slice. Performs periodic " \
                     "checkins with Emulab central, and routes events for " \
                     "other Emulab slices. Slivers in this slice should " \
                     "only interact with other PlanetLab machines, and Emulab."
Kirk Webb's avatar
   
Kirk Webb committed
112
113
PLABMON_PID = "emulab-ops"
PLABMON_EID = "plab-monitor"
Kirk Webb's avatar
   
Kirk Webb committed
114
DEF_SLICE_DESC = "Slice created by Emulab"
Kirk Webb's avatar
   
Kirk Webb committed
115

Kirk Webb's avatar
   
Kirk Webb committed
116
117
PLABEXPIREWARN = 1*WEEK        # one week advance warning for slice expiration.
NODEEXPIREWARN = 2*WEEK+2*DAY  # about two weeks advance warning for slivers.
118

119
120
121
122
123
#
# var to track failed renewals
#
failedrenew = []

124
125
126
127
128
#
# Disable line buffering
#
sys.stdout = os.fdopen(sys.stdout.fileno(), sys.stdout.mode, 0)

Kirk Webb's avatar
   
Kirk Webb committed
129
130
131
132
133
#
# Ensure SIGPIPE doesn't bite us:
#
signal.signal(signal.SIGPIPE, signal.SIG_IGN)

134

135
136
137
#
# Plab abstraction
#
Kirk Webb's avatar
Kirk Webb committed
138

Kirk Webb's avatar
Kirk Webb committed
139
140
141
#
# Class responsible for parsing planetlab sites file
#
Kirk Webb's avatar
Kirk Webb committed
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
class siteParser:

    def __init__(self):
        self.parser = xml.parsers.expat.ParserCreate()
        self.parser.StartElementHandler = self.__site_start_elt
        self.parser.EndElementHandler = self.__site_end_elt
        self.__hosts = []
        self.__sitename = ""
        
    def getPlabNodeInfo(self):
                
        conn = httplib.HTTPSConnection(DEF_PLAB_URL)
        conn.request("GET", DEF_SITE_XML)
        res = conn.getresponse()
        if res.status != 200:
            raise RuntimeError, "HTTP Error getting site list:\n" \
                  "Code: %d Reason: %s" % \
                  (res.status, res.reason)
        try:
            self.parser.ParseFile(res)
            pass
        except xml.parsers.expat.ExpatError, e:
            print "Error parsing XML file, lineno: %d, offset: %d:\n%s" % \
                  (e.lineno, e.offset, xml.parsers.expat.ErrorString(e.code))
            raise

        return self.__hosts

    def __site_start_elt(self, name, attrs):
        
        if name == "PLANETLAB_SITES":
            pass
        
        elif name == "SITE":
            self.__sitename = attrs['SHORT_SITE_NAME']
            pass
        
        elif name == "HOST":
Kirk Webb's avatar
   
Kirk Webb committed
180
181
182
            if not attrs.has_key('MAC'):
                attrs['MAC'] = "None"
                pass
Kirk Webb's avatar
Kirk Webb committed
183
184
            self.__hosts.append({'HNAME' : attrs['NAME'],
                                 'IP'    : attrs['IP'],
Kirk Webb's avatar
   
Kirk Webb committed
185
186
                                 'PLABID': attrs['NODE_ID'],
                                 'MAC'   : attrs['MAC'],
Kirk Webb's avatar
Kirk Webb committed
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
                                 'SITE'  : self.__sitename})
            pass
        
        else:
            print "Unknown element in site file: %s: %s" % (name, attrs)
            pass
        
        return

    def __site_end_elt(self, name):
        
        if name == "SITE":
            self.__sitename = "Unknown"
            pass
        return

        
204
class Plab:
Kirk Webb's avatar
Kirk Webb committed
205
    def __init__(self, agent=None):
206
207
208
        if not agent:
            self.agent = agents[DEF_AGENT]()
            pass
Kirk Webb's avatar
Kirk Webb committed
209
        if debug:
210
211
212
            print "Using module: %s" % self.agent.modname
            pass
        pass
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229

    def createSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._create()
        return slice

    def loadSlice(self, pid, eid):
        """
        Slice factory function
        """
        slice = Slice(self, pid, eid)
        slice._load()
        return slice

Kirk Webb's avatar
Kirk Webb committed
230
    def updateNodeEntries(self, ignorenew = False):
231
        """
Kirk Webb's avatar
Kirk Webb committed
232
        Finds out which Plab nodes are available, and
233
234
235
236
237
        update the DB accordingly.  If ignorenew is True, this will only
        make sure that the data in the DB is correct, and not complete.
        If ignorenew is False (the default), this will do a complete
        update of the DB.  However, this can take some time, as
        information about new nodes (such as link type) must be
Kirk Webb's avatar
Kirk Webb committed
238
        discovered.
239
240
241
242
243
244

        Note that this seemingly innocent funciton actually does a lot of
        magic.  This is the main/only way that Plab nodes get into the
        nodes DB, and this list is updated dynamically.  It also gathers
        static data about new nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
245
246
247
        
        print "Getting available Plab nodes ..."

248
        avail = []
249
        try:
Kirk Webb's avatar
Kirk Webb committed
250
251
252
            parser = siteParser()
            avail = parser.getPlabNodeInfo()
            pass
253
        # XXX: rewrite to use more elegant exception info gathering.
254
255
        except:
            extype, exval, extrace = sys.exc_info()
256
            print "Error talking to agent: %s: %s" % (extype, exval)
Kirk Webb's avatar
Kirk Webb committed
257
            if debug:
258
259
260
                print extrace
            print "Going back to sleep until next scheduled poll"
            return
Kirk Webb's avatar
   
Kirk Webb committed
261

Kirk Webb's avatar
Kirk Webb committed
262
        if debug:
263
264
            print "Got advertisement list:"
            print avail
Kirk Webb's avatar
Kirk Webb committed
265
            pass
Kirk Webb's avatar
   
Kirk Webb committed
266

Kirk Webb's avatar
   
Kirk Webb committed
267
268
269
        ignored_nodes = self.__readNodeFile(IGNORED_NODES_FILE)
        allowed_nodes = self.__readNodeFile(ALLOWED_NODES_FILE)

Kirk Webb's avatar
   
Kirk Webb committed
270
271
272
273
274
275
        # Enforce node limitations, if any.
        # XXX: This is ugly - maybe move to a separate function
        #      that takes a list of filter functions.  I know!!
        #      Create a generator out of a set of filter functions
        #      and the initial node list! :-)  Python geek points to me if
        #      I ever get around to it...  KRW
Kirk Webb's avatar
   
Kirk Webb committed
276
        if len(allowed_nodes) or len(ignored_nodes):
Kirk Webb's avatar
   
Kirk Webb committed
277
            allowed = []
Kirk Webb's avatar
Kirk Webb committed
278
            for nodeent in avail:
Kirk Webb's avatar
   
Kirk Webb committed
279
                if nodeent['PLABID'] in ignored_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
280
                    continue
Kirk Webb's avatar
   
Kirk Webb committed
281
282
                elif len(allowed_nodes):
                    if nodeent['IP'] in allowed_nodes:
Kirk Webb's avatar
   
Kirk Webb committed
283
284
285
286
287
                        allowed.append(nodeent)
                        pass
                    pass
                else:
                    allowed.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
288
289
                    pass
                pass
Kirk Webb's avatar
   
Kirk Webb committed
290
291
292
293
            if verbose:
                print "Advertisements in allowed nodes list:\n%s" % allowed
                pass
            avail = allowed
Kirk Webb's avatar
Kirk Webb committed
294
            pass
295

Kirk Webb's avatar
   
Kirk Webb committed
296
297
298
299
300
301
302
303
304
305
306
        # Check for duplicate node attributes (sanity check)
        availdups = self.__findDuplicateAttrs(avail)
        if len(availdups):
            SENDMAIL(TBOPS, "Duplicates in plab advertised node list",
                     "Duplicate attributes:\n"
                     "%s\n\n"
                     "Let plab support know!" % availdups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate attributes in plab node listing:\n%s" % availdups

307
        # Get node info we already have.
308
        known = self.__getKnownPnodes()
Kirk Webb's avatar
Kirk Webb committed
309
        if debug:
310
311
            print "Got known pnodes:"
            print known
Kirk Webb's avatar
Kirk Webb committed
312
            pass
313

Kirk Webb's avatar
Kirk Webb committed
314
        # Create list of nodes to add or update
Kirk Webb's avatar
   
Kirk Webb committed
315
316
        toadd    = []  # List of node entries to add to DB
        toupdate = []  # List of node entries to update in the DB
Kirk Webb's avatar
Kirk Webb committed
317
        for nodeent in avail:
Kirk Webb's avatar
Kirk Webb committed
318
319
            # Replace sequences of bad chars in the site entity with
            # a single "-".
Kirk Webb's avatar
Kirk Webb committed
320
            nodeent['SITE'] = BADSITECHARS.sub("-", nodeent['SITE'])
Kirk Webb's avatar
   
Kirk Webb committed
321
322
323
            # Determine if we already know about this node.
            matchres = self.__matchPlabNode(nodeent, known)
            if not matchres:
Kirk Webb's avatar
   
Kirk Webb committed
324
                toadd.append(nodeent)
Kirk Webb's avatar
Kirk Webb committed
325
                pass
Kirk Webb's avatar
   
Kirk Webb committed
326
327
            elif len(matchres[1]):
                toupdate.append((nodeent,matchres))
Kirk Webb's avatar
Kirk Webb committed
328
                pass
Kirk Webb's avatar
   
Kirk Webb committed
329
            pass
Kirk Webb's avatar
Kirk Webb committed
330

Kirk Webb's avatar
   
Kirk Webb committed
331
332
        # Process the list of nodes to add
        addstr = ""
333
        if len(toadd):
Kirk Webb's avatar
Kirk Webb committed
334
            # Are we ignoring new entries?
335
            if ignorenew:
Kirk Webb's avatar
Kirk Webb committed
336
                if verbose:
337
                    print "%d new Plab nodes, but ignored for now" % len(toadd)
Kirk Webb's avatar
Kirk Webb committed
338
339
                    pass
                pass
Kirk Webb's avatar
Kirk Webb committed
340
            # If not ignoring, do the addition/update.
341
            else:
Kirk Webb's avatar
   
Kirk Webb committed
342
343
                print "There are %d new Plab nodes." % len(toadd)
                for nodeent in toadd:
Kirk Webb's avatar
Kirk Webb committed
344
                    # Get the linktype here so we can report it in email.
Kirk Webb's avatar
Kirk Webb committed
345
                    self.__findLinkType(nodeent)
Kirk Webb's avatar
Kirk Webb committed
346
                    if debug:
Kirk Webb's avatar
Kirk Webb committed
347
348
                        print "Found linktype %s for node %s" % \
                              (nodeent['LINKTYPE'], nodeent['IP'])
Kirk Webb's avatar
Kirk Webb committed
349
                        pass
Kirk Webb's avatar
   
Kirk Webb committed
350
351
352
                    # Add the node.
                    self.__addNode(nodeent)
                    # Add a line for the add/update message.
Kirk Webb's avatar
Kirk Webb committed
353
                    nodestr = "%s\t\t%s\t\t%s\t\t%s\t\t%s\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
354
                              (nodeent['PLABID'],
Kirk Webb's avatar
Kirk Webb committed
355
356
357
358
                               nodeent['IP'],
                               nodeent['HNAME'],
                               nodeent['SITE'],
                               nodeent['LINKTYPE'])
Kirk Webb's avatar
   
Kirk Webb committed
359
                    addstr += nodestr
Kirk Webb's avatar
Kirk Webb committed
360
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
361
362
                pass
            pass
363

Kirk Webb's avatar
   
Kirk Webb committed
364
        # Process node updates.
Kirk Webb's avatar
   
Kirk Webb committed
365
366
367
        updstr = ""
        if len(toupdate):
            print "There are %d plab node updates." % len(toupdate)
Kirk Webb's avatar
   
Kirk Webb committed
368
369
370
            for updent,updmapent in toupdate:
                self.__updateNodeMapping(updmapent)
                self.__addNode(updent, updmapent)
Kirk Webb's avatar
   
Kirk Webb committed
371
                # Add a line for the add/update message.
Kirk Webb's avatar
   
Kirk Webb committed
372
373
                nodestr = updmapent[0] + "\n"
                for attr,val in updmapent[1].items():
Kirk Webb's avatar
   
Kirk Webb committed
374
375
376
                    nodestr += "\t%s:\t%s => %s\n" % (attr,val[0],val[1])
                    pass
                updstr += nodestr + "\n"
Kirk Webb's avatar
Kirk Webb committed
377
378
                pass
            pass
Kirk Webb's avatar
   
Kirk Webb committed
379
380
381
382
383

        if len(toadd) or len(toupdate):
            # We need to update DNS since we've added hosts..
            print "Forcing a named map update ..."
            os.spawnl(os.P_WAIT, NAMED_SETUP, NAMED_SETUP)
Kirk Webb's avatar
   
Kirk Webb committed
384
385
            print "Pushing out site_mapping ..."
            os.spawnl(os.P_WAIT, PELAB_PUSH, PELAB_PUSH)
Kirk Webb's avatar
   
Kirk Webb committed
386
387
388
389
390
391
392
393
394
395
396
397
            # Now announce that we've added/updated nodes.
            SENDMAIL(TBOPS,
                     "Plab nodes have been added/updated in the DB.",
                     "The following plab nodes have been added to the DB:\n"
                     "PlabID\t\tIP\t\tHostname\t\tSite\t\tLinktype\n\n"
                     "%s\n\n"
                     "The following plab nodes have been updated in the DB:\n"
                     "\n%s\n\n" % \
                     (addstr, updstr),
                     TBOPS)
            print "Done adding new Plab nodes."
            pass
Kirk Webb's avatar
Kirk Webb committed
398
        return
399

Kirk Webb's avatar
   
Kirk Webb committed
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
    def __matchPlabNode(self, plabent, knownents):
        """
        Helper function.  Returns a two-element tuple or null.
        Null is returned when the node does not match any in the
        knownents list (none of it's attributes match those of any
        in the list).  If a match (or partial match) is found, a two
        element tuple is returned.  The first element is the emulab
        node id that matched, and the second is a dictionary containing
        thos elements that differed between the two (in the case of a
        partial match).
        """
        for nid in knownents:
            ent = knownents[nid]
            same = {}
            diff = {}
            for attr in ent:
                if ent[attr] in ATTR_NIL_VALUES:
                    continue
                elif ent[attr] == plabent[attr]:
                    same[attr] = ent[attr]
                    pass
                else:
                    diff[attr] = (ent[attr], plabent[attr])
                    pass
                pass
            if len(same):
                return (nid, diff)
            pass
        return ()

430
431
432
433
434
    def __getKnownPnodes(self):
        """
        getFree helper function.  Returns a dict of IP:node_id pairs
        for the Plab nodes that currently exist in the DB.
        """
Kirk Webb's avatar
   
Kirk Webb committed
435
436
437
        res = DBQueryFatal("select node_id,plab_id,hostname,IP,mac"
                           " from plab_mapping")
        
438
        ret = {}
Kirk Webb's avatar
   
Kirk Webb committed
439
440
441
442
443
        for nodeid, plabid, hostname, ip, mac in res:
            ret[nodeid] = {'PLABID'  : plabid,
                           'HNAME'   : hostname,
                           'IP'      : ip,
                           'MAC'     : mac}
Kirk Webb's avatar
Kirk Webb committed
444
            pass
Kirk Webb's avatar
   
Kirk Webb committed
445
446
447
448
449
450
451
452
453
454
        # Check for duplicate node attributes: report any that are found.
        dups = self.__findDuplicateAttrs(ret.values())
        if len(dups):
            SENDMAIL(TBOPS, "Duplicate plab node attributes in the DB!",
                     "Duplicate node attrs:\n"
                     "%s\n\n"
                     "Fix up please!" % dups,
                     TBOPS)
            raise RuntimeError, \
                  "Duplicate node attributes in DB:\n%s" % dups            
455
        return ret
Kirk Webb's avatar
   
Kirk Webb committed
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480

    def __findDuplicateAttrs(self, nodelist):
        """
        Find duplicate node attributes in the node list passed in.
        """
        uniqattrs = ['PLABID', 'HNAME', 'IP', 'MAC']
        attrs = {}
        dups = {}
        
        for ent in nodelist:
            for attr in uniqattrs:
                entry = "%s:%s" % (attr, ent[attr])
                if attrs.has_key(entry) and \
                   ent[attr] not in ATTR_NIL_VALUES:
                    print "Duplicate node attribute: %s" % entry
                    if not dups.has_key(entry):
                        dups[entry] = [attrs[entry],]
                        pass
                    dups[entry].append(ent['PLABID'])
                else:
                    attrs[entry] = ent['PLABID']
                    pass
                pass
            pass
        return dups
Kirk Webb's avatar
   
Kirk Webb committed
481
        
Kirk Webb's avatar
Kirk Webb committed
482
    def __findLinkType(self, nodeent):
483
484
485
486
487
488
489
        """
        getFree helper function.  Figures out the link type of the given
        host.  This first performs a traceroute and checks for the U of
        U's I2 gateway to classify Internet2 hosts.  If this test fails,
        it checks if the hostname is international.  If this test fails,
        this simply specifies an inet link type.

Kirk Webb's avatar
Kirk Webb committed
490
        This can't detect DSL links..
491
        """
492
        # Is host international (or flux/emulab local)?
493
        from socket import gethostbyaddr, getfqdn, herror
Kirk Webb's avatar
Kirk Webb committed
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
        
        if not nodeent.has_key('HNAME'):
            try:
                (hname, ) = gethostbyaddr(ip)
                nodeent['HNAME'] = getfqdn(hname)
                pass
            except herror:
                nodeent['HNAME'] = nodeent['IP']
                print "WARNING: Failed to get hostname for %s" % nodeent['IP']
                pass
            pass
        
        tld = nodeent['HNAME'].split(".")[-1].lower()
        if not tld in ("edu", "org", "net", "com", "gov", "us", "ca"):
            nodeent['LINKTYPE'] = "pcplabintl"
            return
        
        # Is it us?
        if nodeent['HNAME'].endswith(LOCAL_PLAB_DOMAIN):
            nodeent['LINKTYPE'] = LOCAL_PLAB_LINKTYPE
            return
        
516
        # Is host on I2?
Kirk Webb's avatar
Kirk Webb committed
517
        traceroute = os.popen("traceroute -nm 10 -q 1 %s" % nodeent['IP'])
518
519
520
521
522
        trace = traceroute.read()
        traceroute.close()

        for gw in MAGIC_INET2_GATEWAYS:
            if trace.find(gw) != -1:
Kirk Webb's avatar
Kirk Webb committed
523
524
                nodeent['LINKTYPE'] = "pcplabinet2"
                return
525

526
527
528
529
        for gw in MAGIC_INET_GATEWAYS:
            if trace.find(gw) != -1:
                break
        else:
Kirk Webb's avatar
Kirk Webb committed
530
            print "WARNING: Unknown gateway for host %s" % nodeent['IP']
531

532
        # Must be plain 'ole Internet
Kirk Webb's avatar
Kirk Webb committed
533
534
        nodeent['LINKTYPE'] = "pcplabinet"
        return
535

Kirk Webb's avatar
   
Kirk Webb committed
536
    def __addNode(self, nodeent, updent = ()):
537
538
539
540
541
        """
        getFree helper function.  Adds a new Plab pnode and associated
        vnodes to the DB.  linktype should be one of (inet2, inet, intl,
        dsl).
        """
Kirk Webb's avatar
   
Kirk Webb committed
542
543
        # block out common termination signals while adding a node
        osigs = disable_sigs(TERMSIGS)
544
        defosid, controliface = self.__getNodetypeInfo()
Kirk Webb's avatar
Kirk Webb committed
545
        hostonly = nodeent['HNAME'].replace(".", "-")
Kirk Webb's avatar
   
Kirk Webb committed
546
        # These will be setup properly below.
Kirk Webb's avatar
   
Kirk Webb committed
547
548
        nidnum = 0
        priority = 0
Kirk Webb's avatar
   
Kirk Webb committed
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
        nodeid = ""
        vnodeprefix = ""

        # Setup nodeid according to whether or not we were passed in an
        # update entry.
        if updent:
            nodeid = updent[0]
            print "Updating node %s" % nodeid
            pass
        else:
            nidnum, priority = self.__nextFreeNodeid()
            nodeid = "%s%d" % (NODEPREFIX, nidnum)
            vnodeprefix = "%svm%d" % (NODEPREFIX, nidnum)
            print "Creating pnode %s as %s, priority %d." % \
                  (nodeent['IP'], nodeid, priority)
            pass
565

Kirk Webb's avatar
   
Kirk Webb committed
566
        try:
Kirk Webb's avatar
   
Kirk Webb committed
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
            # XXX: Site is not checked for updates.
            DBQueryFatal("replace into widearea_nodeinfo"
                         " (node_id, contact_uid, hostname, site)"
                         " values (%s, %s, %s, %s)",
                         (nodeid, 'nobody', nodeent['HNAME'], nodeent['SITE']))

            DBQueryFatal("replace into interfaces"
                         " (node_id, card, port, IP, interface_type,"
                         " iface, role)"
                         " values (%s, %s, %s, %s, %s, %s, %s)",
                         (nodeid, 0, 1, nodeent['IP'], 'fxp',
                          controliface, 'ctrl'))

            # Don't do anything else if we are only updating the node
            if updent:
                enable_sigs(osigs)
                return

Kirk Webb's avatar
Kirk Webb committed
585
            DBQueryFatal("replace into nodes"
586
                         " (node_id, type, phys_nodeid, role, priority,"
Kirk Webb's avatar
   
Kirk Webb committed
587
                         "  op_mode, def_boot_osid,"
588
                         "  allocstate, allocstate_timestamp,"
Kirk Webb's avatar
   
Kirk Webb committed
589
                         "  eventstate, state_timestamp)"
590
                         " values (%s, %s, %s, %s, %s,"
Kirk Webb's avatar
   
Kirk Webb committed
591
592
593
594
                         "  %s, %s, %s, now(), %s, now())",
                         (nodeid, 'pcplabphys', nodeid,
                          'testnode', priority*100,
                          'ALWAYSUP', defosid,
595
                          'FREE_CLEAN',
Kirk Webb's avatar
   
Kirk Webb committed
596
                          'ISUP'))
597

598
599
600
            DBQueryFatal("replace into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
Kirk Webb's avatar
Kirk Webb committed
601
                         (nodeid))
602

Kirk Webb's avatar
Kirk Webb committed
603
            DBQueryFatal("replace into reserved"
Kirk Webb's avatar
   
Kirk Webb committed
604
605
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
606
                         (nodeid, RESERVED_PID, RESERVED_EID, hostonly))
Kirk Webb's avatar
   
Kirk Webb committed
607

Kirk Webb's avatar
   
Kirk Webb committed
608
609
            # XXX: This should probably be checked and updated if necessary
            #      when updating.
Kirk Webb's avatar
Kirk Webb committed
610
            DBQueryFatal("replace into node_auxtypes"
Kirk Webb's avatar
   
Kirk Webb committed
611
612
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
613
                         (nodeid, nodeent['LINKTYPE'], 1))
Kirk Webb's avatar
   
Kirk Webb committed
614
            
Kirk Webb's avatar
Kirk Webb committed
615
            DBQueryFatal("replace into node_auxtypes"
616
617
618
619
                         " (node_id, type, count)"
                         " values (%s, %s, %s)",
                         (nodeid, 'pcplab', 1))
            
Kirk Webb's avatar
Kirk Webb committed
620
            DBQueryFatal("replace into node_status"
621
622
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
Kirk Webb's avatar
Kirk Webb committed
623
                         (nodeid, 'down'))
Kirk Webb's avatar
   
Kirk Webb committed
624

Kirk Webb's avatar
   
Kirk Webb committed
625
626
627
628
629
            DBQueryFatal("insert into plab_mapping"
                         " (node_id, plab_id, hostname, IP, mac, create_time)"
                         " values (%s, %s, %s, %s, %s, now())",
                         (nodeid, nodeent['PLABID'], nodeent['HNAME'],
                          nodeent['IP'], nodeent['MAC']))
630

Kirk Webb's avatar
   
Kirk Webb committed
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
            #
            # NowAdd the site_mapping entry for this node.
            #
            
            # See if we know about the associated site - grab idx if so
            siteidx = 0
            nodeidx = 1
            siteres = DBQueryFatal("select site_idx, node_idx from "
                                   " plab_site_mapping where site_name=%s",
                                   nodeent['SITE']);
            if len(siteres):
                # There are already nodes listed for this site, so get
                # the next node id.
                siteidx = siteres[0][0]
                for (foo, idx) in siteres:
                    if idx > nodeidx: nodeidx = idx
                    pass
                nodeidx += 1
                pass
            else:
                # No nodes listed for site, so get the largest site_idx
                # in the DB so far, and increment cuz we're going to add
                # a new one.
                maxres = DBQueryFatal("select MAX(site_idx) from "
                                      " plab_site_mapping")
                try:
                    siteidx = int(maxres[0][0]) + 1
                    pass
                except ValueError:
                    siteidx = 1
                    pass
                pass
            # Create site_mapping entry, optionally creating new site idx
            # via not specifying the site_idx field (field is auto_increment)
            DBQueryFatal("insert into plab_site_mapping "
                         " values (%s, %s, %s, %s)",
                         (nodeent['SITE'], siteidx, nodeid, nodeidx))

669
            # Create a single reserved plab vnode for the managment sliver.
Kirk Webb's avatar
   
Kirk Webb committed
670
671
672
673
674
675
676
677
            # XXX I left it as "20" cause of all the existing ones.
            # XXXX I set it to 1 due to the above comment (correct?)
            #      since we are re-creating anyway.            
            n = 1
            vprio = (priority * 100) + n
            sshdport = 38000 + n
            vnodeid = "%s-%d" % (vnodeprefix, n)
            vnodetype = "pcplab"
678
679
            if verbose:
                print "Creating vnode %s, priority %d" % (vnodeid, vprio)
Kirk Webb's avatar
Kirk Webb committed
680
                pass
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
                    
            DBQueryFatal("insert into nodes"
                         " (node_id, type, phys_nodeid, role, priority,"
                         "  op_mode, def_boot_osid, update_accounts,"
                         "  allocstate, allocstate_timestamp,"
                         "  eventstate, state_timestamp, sshdport)"
                         " values (%s, %s, %s, %s, %s,"
                         "  %s, %s, %s, %s, now(), %s, now(), %s)",
                         (vnodeid, vnodetype, nodeid, 'virtnode', vprio,
                          'PCVM', defosid, 1,
                          'FREE_CLEAN',
                          'SHUTDOWN', sshdport))

            DBQueryFatal("insert into node_hostkeys"
                         " (node_id)"
                         " values (%s)",
                         (vnodeid))
            
            DBQueryFatal("insert into node_status"
                         " (node_id, status, status_timestamp)"
                         " values (%s, %s, now())",
                         (vnodeid, 'up'))
            
Kirk Webb's avatar
Kirk Webb committed
704
            # Put the last vnode created into the special monitoring expt.
Kirk Webb's avatar
   
Kirk Webb committed
705
706
707
708
            DBQueryFatal("insert into reserved"
                         " (node_id, pid, eid, rsrv_time, vname)"
                         " values (%s, %s, %s, now(), %s)",
                         (vnodeid, MONITOR_PID, MONITOR_EID, vnodeid))
Kirk Webb's avatar
Kirk Webb committed
709
710
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
711
712
713
714
715
716
        except:
            print "Error adding PLAB node to DB: someone needs to clean up!"
            tbmsg = "".join(traceback.format_exception(*sys.exc_info()))
            SENDMAIL(TBOPS, "Error adding new plab node to DB: %s\n" %
                     nodeid, "Some operation failed while trying to add a"
                     " newly discovered plab node to the DB:\n %s"
717
                     "\n Please clean up!\n" % tbmsg, TBOPS)
Kirk Webb's avatar
   
Kirk Webb committed
718
719
            enable_sigs(osigs)
            raise
720

Kirk Webb's avatar
   
Kirk Webb committed
721
722
        # last but not least, unblock signals
        enable_sigs(osigs)
Kirk Webb's avatar
Kirk Webb committed
723
        return
724

Kirk Webb's avatar
   
Kirk Webb committed
725

Kirk Webb's avatar
   
Kirk Webb committed
726
    def __updateNodeMapping(self, updent):
Kirk Webb's avatar
   
Kirk Webb committed
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
        """
        Updates changed node attributes in the mapping table.
        """
        uid = os.getuid()
        dbuid = uid == 0 and "root" or UNIX2DBUID(uid)
        attrmap = {'PLABID' : 'plab_id',
                   'HNAME'  : 'hostname',
                   'IP'     : 'IP',
                   'MAC'    : 'mac'}
        nodeid, chattrs = updent
        
        if len(chattrs) > 2:
            errmsg = "More than 2 plab node attrs have changed!\n\n%s\n\n" \
                     "%s has been moved to hwdown." % (chattrs, nodeid)
            MarkPhysNodeDown(nodeid)
            TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, errmsg)
            SENDMAIL(TBOPS,
                     "More than 2 plab node attrs have changed on %s" % nodeid,
                     errmsg,
                     TBOPS)
            raise RuntimeError, errmsg # XXX: maybe don't raise an exception.
        
        updstr = ",".join(map(lambda x: "%s='%s'" % (attrmap[x[0]],x[1][1]),
                              chattrs.items()))
        DBQueryFatal("update plab_mapping set " + updstr + " where node_id=%s",
                     (nodeid))
        updmsg = "Plab node %s attributes updated:\n\n%s" % (nodeid, chattrs)
        TBSetNodeLogEntry(nodeid, dbuid, TB_NODELOGTYPE_MISC, updmsg)
        # updateNodeEtries() already sends mail.
        #SENDMAIL(TBOPS,
        #         "Plab node %s attributes updated." % nodeid, updmsg, TBOPS)
        return

760
761
    def __getNodetypeInfo(self):
        """
762
763
        addNode helper function.  Returns a (defosid, controliface) 
        tuple for the Plab pnode type.  Caches the result since
764
        it doesn't change.
765
766
        """
        if not hasattr(self, "__getNodetypeInfoCache"):
Kirk Webb's avatar
Kirk Webb committed
767
            if debug:
768
                print "Getting node type info"
Kirk Webb's avatar
Kirk Webb committed
769
                pass
770
            res = DBQueryFatal("select osid, control_iface"
771
                               " from node_types"
772
773
774
                               " where type = 'pcplabphys'")
            assert (len(res) == 1), "Failed to get node type info"
            (self.__getNodetypeInfoCache, ) = res
Kirk Webb's avatar
Kirk Webb committed
775
776
            pass
        
777
778
779
780
781
782
783
        return self.__getNodetypeInfoCache

    def __nextFreeNodeid(self):
        """
        addNode helper function.  Returns a (nodeid, priority) tuple of
        the next free nodeid and priority for Plab nodes.
        """
Kirk Webb's avatar
Kirk Webb committed
784
        if debug:
785
786
787
788
789
790
791
792
793
794
            print "Getting next free nodeid"
        DBQueryFatal("lock tables nextfreenode write")
        try:
            res = DBQueryFatal("select nextid, nextpri from nextfreenode"
                               " where nodetype = 'pcplab'")
            assert (len(res) == 1), "Unable to find next free nodeid"
            DBQueryFatal("update nextfreenode"
                         " set nextid = nextid + 1, nextpri = nextpri + 1"
                         " where nodetype = 'pcplab'")
            ((nodeid, priority), ) = res
Kirk Webb's avatar
Kirk Webb committed
795
            pass
796
797
        finally:
            DBQueryFatal("unlock tables")
Kirk Webb's avatar
Kirk Webb committed
798
799
            pass
        
800
801
        return nodeid, priority

Kirk Webb's avatar
   
Kirk Webb committed
802
803
804
805
806
807
808
809
810
811
812
813
814
    def __readNodeFile(self, filename):
        """
        Helper function - read in list of nodes from a file, seperated
        by arbitrary amounts of whitespace.  No comments allowed.
        """
        nodelist = []
        if os.access(filename, os.F_OK):
            nodefile = open(filename, "r+")
            nodelist = nodefile.read().split()
            nodefile.close()
            pass
        return nodelist

Kirk Webb's avatar
   
Kirk Webb committed
815
    def renew(self, inpid = None, ineid = None, force = False):
816
        """
Kirk Webb's avatar
   
Kirk Webb committed
817
818
819
        Renews all of the Plab leases regardless of when they expire.  Note
        that all times are handled in the UTC time zone.  We don't trust
        MySQL to do the right thing with times (yet).
820
        """
Kirk Webb's avatar
   
Kirk Webb committed
821

Kirk Webb's avatar
   
Kirk Webb committed
822
823
        global failedrenew # XXX
        
Kirk Webb's avatar
   
Kirk Webb committed
824
825
826
827
828
829
830
831
832
833
        now = int(time.time())
        
        if not inpid:
            res = DBQueryFatal("select pid, eid from plab_slices");
            pass
        else:
            if not ineid:
                raise RuntimeError, "renew: Must provide eid with pid."
            res = ((inpid, ineid),)
            pass
Kirk Webb's avatar
Kirk Webb committed
834
        
835
        loadedSlices = {}
836
837
        newfail = []
        failsoon = []
Kirk Webb's avatar
Kirk Webb committed
838
839
        ret = 0

Kirk Webb's avatar
   
Kirk Webb committed
840
        print "Renewing Plab leases at %s ..." % time.ctime()
Kirk Webb's avatar
Kirk Webb committed
841

Kirk Webb's avatar
   
Kirk Webb committed
842
        for (pid, eid) in res:
Kirk Webb's avatar
Kirk Webb committed
843

844
845
            try:
                slice = loadedSlices[(pid, eid)]
Kirk Webb's avatar
Kirk Webb committed
846
                pass
847
848
849
            except KeyError:
                slice = self.loadSlice(pid, eid)
                loadedSlices[(pid, eid)] = slice
Kirk Webb's avatar
Kirk Webb committed
850
                pass
851
            
Kirk Webb's avatar
   
Kirk Webb committed
852
            res = slice.renew(force)
Kirk Webb's avatar
   
Kirk Webb committed
853
854
855
856
857
            entry = (pid, eid, slice.leaseend)
            
            if not res:
                print "Failed to renew lease for %s/%s" % \
                      entry[:2]
858
859
                if entry not in failedrenew:
                    newfail.append(entry)
Kirk Webb's avatar
Kirk Webb committed
860
                    pass
Kirk Webb's avatar
   
Kirk Webb committed
861
                if (slice.leaseend - now) < PLABEXPIREWARN:
862
                    failsoon.append(entry)
Kirk Webb's avatar
Kirk Webb committed
863
864
                    pass
                pass
865
866
867
            else:
                if entry in failedrenew:
                    failedrenew.remove(entry)
Kirk Webb's avatar
   
Kirk Webb committed
868
                    pass
Kirk Webb's avatar
Kirk Webb committed
869
                    
870
871
        if newfail:
            failedrenew += newfail
Kirk Webb's avatar
Kirk Webb committed
872
            failstr = ""
873
            for n in newfail:
Kirk Webb's avatar
   
Kirk Webb committed
874
                failstr += "%s/%s (expires: %s UTC)\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
875
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
Kirk Webb committed
876
                pass
Kirk Webb's avatar
   
Kirk Webb committed
877
            
878
            SENDMAIL(TBOPS, "Lease renewal(s) failed",
Kirk Webb's avatar
   
Kirk Webb committed
879
                     "Failed to renew the following leases:\n%s" %
880
                     failstr + "\n\nPlease check the plabrenew log", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
881
            pass
Kirk Webb's avatar
Kirk Webb committed
882

883
884
885
        if failsoon:
            failstr = ""
            for n in failsoon:
Kirk Webb's avatar
   
Kirk Webb committed
886
                failstr += "%s/%s: (expires: %s UTC)\n" % \
Kirk Webb's avatar
   
Kirk Webb committed
887
                           (n[:2] + (time.asctime(time.gmtime(n[2])),))
Kirk Webb's avatar
   
Kirk Webb committed
888
889
890
891
                pass
            SENDMAIL(TBOPS, "WARNING: PLAB leases have expired, or will soon",
                     "The following plab leases have expired, or will soon:\n"
                     + failstr + "\n\nPlease look into it!", TBOPS)
Kirk Webb's avatar
Kirk Webb committed
892
            pass
Kirk Webb's avatar
   
Kirk Webb committed
893

Kirk Webb's avatar
Kirk Webb committed
894
895
896
        return
    
    pass # end class Plab
897

898
899
900
901
902
903

#
# Slice abstraction
#

class Slice:
Kirk Webb's avatar
   
Kirk Webb committed
904
905

    def __init__(self, plab, pid, eid, slicename = None):
906
907
        self.plab = plab
        self.pid, self.eid = pid, eid
908
        self.slicemeta = None
Kirk Webb's avatar
   
Kirk Webb committed
909
        self.slicename = slicename
Kirk Webb's avatar
   
Kirk Webb committed
910
        self.description = DEF_SLICE_DESC
Kirk Webb's avatar
   
Kirk Webb committed
911
        return
912
913
914
915
916
917
    
    def _create(self):
        """
        Creates a new slice that initially contains no nodes.  Don't call
        this directly, use Plab.createSlice instead.
        """
Kirk Webb's avatar
   
Kirk Webb committed
918
919
920

        adminbit = 0
        if self.pid == PLABMON_PID and self.eid == PLABMON_EID:
Kirk Webb's avatar
   
Kirk Webb committed
921
922
            self.slicename   = PLAB_SVC_SLICENAME
            self.description = PLAB_SVC_SLICEDESC
Kirk Webb's avatar
   
Kirk Webb committed
923
924
925
            adminbit = 1
            pass
        
Kirk Webb's avatar
   
Kirk Webb committed
926
        if not self.slicename:
Kirk Webb's avatar
   
Kirk Webb committed
927
            res = DBQueryFatal("select idx, expt_name from experiments "
Kirk Webb's avatar
   
Kirk Webb committed
928
929
930
931
                               "where pid=%s "
                               "and eid=%s",
                               (self.pid, self.eid))
            if not len(res):
Kirk Webb's avatar
   
Kirk Webb committed
932
933
934
935
                raise RuntimeError, \
                      "Didn't get any results while looking up info on " \
                      "experiment %s/%s" % (self.pid, self.eid)
            (eindex, descr) = res[0]
Kirk Webb's avatar
   
Kirk Webb committed
936
            self.slicename = "%s_%s" % (SLICEPREFIX, eindex)
Kirk Webb's avatar
   
Kirk Webb committed
937
            self.description = descr
Kirk Webb's avatar
   
Kirk Webb committed
938
            pass
Kirk Webb's avatar
   
Kirk Webb committed
939

940
        print "Creating Plab slice %s." % self.slicename
Kirk Webb's avatar
   
Kirk Webb committed
941

942
        try:
Kirk Webb's avatar
Kirk Webb committed
943
944
945
946
            res, self.slicemeta, self.leaseend = \
                 self.plab.agent.createSlice(self)

            DBQueryFatal("insert into plab_slices"
Kirk Webb's avatar
   
Kirk Webb committed
947
948
                         " (pid, eid, slicename, slicemeta, leaseend, admin)"
                         " values (%s, %s, %s, %s, %s, %s)",
Kirk Webb's avatar
Kirk Webb committed
949
                         (self.pid, self.eid, self.slicename,
Kirk Webb's avatar
   
Kirk Webb committed
950
951
952
953
                          self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
                          adminbit))
954
955
            pass
        except:
Kirk Webb's avatar
Kirk Webb committed
956
            self.plab.agent.deleteSlice(self)
957
958
            DBQueryFatal("delete from plab_slices where slicename=%s",
                         (self.slicename,))
959
960
            raise

961
962
        return res

963
964
965
966
    def _load(self):
        """
        Loads an already allocated slice from the DB.  Don't call this
        directly, use Plab.loadSlice instead.
967
968
969

        XXX This should probably be made lazy, since not all operations
        really need it
970
        """
Kirk Webb's avatar
Kirk Webb committed
971
        if verbose:
972
            print "Loading slice for pid/eid %s/%s" % (self.pid, self.eid)
Kirk Webb's avatar
   
Kirk Webb committed
973
974
        res = DBQueryFatal("select slicename, slicemeta, leaseend "
                           " from plab_slices "
975
976
977
978
979
980
                           " where pid = %s and eid = %s",
                           (self.pid, self.eid))
        assert (len(res) > 0), \
               "No slice found for %s-%s" % (self.pid, self.eid)
        assert (len(res) == 1), \
               "Multiple slices found for %s-%s" % (self.pid, self.eid)
Kirk Webb's avatar
   
Kirk Webb committed
981
982
        ((self.slicename, self.slicemeta, leaseend), ) = res
        self.leaseend = calendar.timegm(time.strptime(str(leaseend),
Kirk Webb's avatar
   
Kirk Webb committed
983
984
                                                      "%Y-%m-%d %H:%M:%S"))
        return
985

Kirk Webb's avatar
   
Kirk Webb committed
986
    def renew(self, force = False):
Kirk Webb's avatar
Kirk Webb committed
987
        """
Kirk Webb's avatar
   
Kirk Webb committed
988
        Renews slice lease.  We want this to be the maximum allowed by law...
Kirk Webb's avatar
   
Kirk Webb committed
989
        Store the expiration time in UTC.
Kirk Webb's avatar
Kirk Webb committed
990
991
992
993
        """
        print "Renewing lease for slice %s" % self.slicename

        try:
Kirk Webb's avatar
   
Kirk Webb committed
994
            ret = self.plab.agent.renewSlice(self, force)
Kirk Webb's avatar
   
Kirk Webb committed
995
996
997
998
999
1000
            DBQueryFatal("update plab_slices "
                         " set slicemeta=%s, leaseend=%s "
                         " where slicename=%s",
                         (self.slicemeta,
                          time.strftime("%Y-%m-%d %H:%M:%S",
                                        time.gmtime(self.leaseend)),
For faster browsing, not all history is shown. View entire blame