Merge remote-tracking branch 'origin/geni-lib'

.gitignore

@@ -10,3 +10,4 @@ pdf/
 *.aux
 *.out
 *.png
+*~

advanced-topics.scrbl

@@ -157,7 +157,7 @@ you can add the @tt{geni-get} client from
 @hyperlink["https://www.emulab.net/downloads/geni-get.tar.gz"]{its
 repository}.)
 
-While @tt{geni-get} supports many options, there are three commands most
+While @tt{geni-get} supports many options, there are four commands most
 useful in the @(tb) context.
 
 @subsection[#:tag "geni-get-client-id"]{Client ID}
@@ -185,3 +185,17 @@ manifest to standard output, including any annotations added during
 instantiation.  For instance, this is an appropriate technique to use to
 query the allocation of a @seclink["dynamic-public-ip"]{dynamic public
 IP address pool}.
+
+@subsection[#:tag "geni-get-key"]{Private key}
+
+As a convenience, @(tb) will automatically generate an RSA private
+key unique to each profile instance.  @tt{geni-get key} will retrieve
+the private half of the keypair, which makes it a useful command for
+profiles bootstraping an authenticated channel.  For instance:
+
+@code-sample["geni-get-key.sh"]
+
+Please note that the private key will be accessible to any user who
+can invoke @tt{geni-get} from within the profile instance.  Therefore, it
+is NOT suitable for an authentication mechanism for privilege within
+a multi-user instance!

apt-manual.scrbl

@@ -62,6 +62,7 @@ of Utah.
 @include-section["repeatable-research.scrbl"]
 @include-section["creating-profiles.scrbl"]
 @include-section["basic-concepts.scrbl"]
+@include-section["geni-lib.scrbl"]
 @include-section["advanced-topics.scrbl"]
 @include-section["hardware.scrbl"]
 @include-section["planned.scrbl"]

cloudlab-manual.scrbl

@@ -2,7 +2,7 @@
 @(require racket/date)
 @(require "defs.rkt")
 
-@title[#:version apt-version
+@title[#:version apt-version #:style main-style
        #:date (date->string (current-date))]{The CloudLab Manual}
 
 @author[
@@ -52,6 +52,7 @@ Take a look at the @seclink["status-notes"]{status notes}, and then
 @include-section["repeatable-research.scrbl"]
 @include-section["creating-profiles.scrbl"]
 @include-section["basic-concepts.scrbl"]
+@include-section["geni-lib.scrbl"]
 @include-section["advanced-topics.scrbl"]
 @include-section["hardware.scrbl"]
 @include-section["planned.scrbl"]

code-sample.css

+.code-sample {
+    display: block;
+    unicode-bidi: embed;
+    font-family: "Source Code Pro", monospace;
+    white-space: pre;
+    border: 1px solid #000000;
+    font-size: 12pt;
+}

code-sample.tex

+\newcommand{\code-sample}[1]{\fbox{\verbatim{#1}}}

code-samples/geni-get-key.sh

+#!/bin/sh
+
+# Create the user SSH directory, just in case.
+mkdir $HOME/.ssh && chmod 700 $HOME/.ssh
+
+# Retrieve the server-generated RSA private key.
+geni-get key > $HOME/.ssh/id_rsa
+chmod 600 $HOME/.ssh/id_rsa
+
+# Derive the corresponding public key portion.
+ssh-keygen -y -f $HOME/.ssh/id_rsa > $HOME/.ssh/id_rsa.pub
+
+# If you want to permit login authenticated by the auto-generated key,
+# then append the public half to the authorized_keys file:
+grep -q -f $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys || cat $HOME/.ssh/id_rsa_pub >> $HOME/.ssh/authorized_keys

code-samples/geni-lib-node-ips.py

+"""An example of constructing a profile with node IP addresses specified
+manually.
+
+Instructions:
+Wait for the profile instance to start, and then log in to either VM via the
+ssh ports specified below.  (Note that even though the EXPERIMENTAL
+data plane interfaces will use the addresses given in the profile, you
+will still connect over the control plane interfaces using addresses
+given by the testbed.  The data plane addresses are for intra-experiment
+communication only.)
+"""
+
 import geni.portal as portal
 import geni.rspec.pg as pg 
 
@@ -29,4 +41,4 @@ link.addInterface(iface2)
 
 rspec.addResource(link)
 
-pc.printRequestRSpec(rspec)
+pc.printRequestRSpec(rspec)
\ No newline at end of file

code-samples/geni-lib-os-install-scripts.py

+"""An example of constructing a profile with install and execute services.
+manually.
+
+Instructions:
+Wait for the profile instance to start, and then log in to the VM via the
+ssh port specified below.  The install and execute services are handled
+automatically during profile instantiation, with no manual intervention
+required.
+"""
+
 import geni.portal as portal
 import geni.rspec.pg as pg 
 
@@ -10,8 +20,9 @@ node1 = pg.XenVM("node1")
 node1.disk_image = "<URL to disk image>"
 
 # Install and execute scripts on the VM
-node1.addService(pg.Install(url="<URL to the tarball file>", path="local"))
-node1.addService(pg.Execute(shell="bash", command="<Path to executable>"))
+node1.addService(pg.Install(url="http://example.org/sample.tar.gz", path="/local"))
+node1.addService(pg.Execute(shell="bash", command="/local/example.sh"))
 
 rspec.addResource(node1)
-pc.printRequestRSpec(rspec)
+
\ No newline at end of file
+pc.printRequestRSpec(rspec)

code-samples/geni-lib-parameters.py

+"""An example of using parameters to construct a profile with a variable
+number of nodes.
+
+Instructions:
+Wait for the profile instance to start, and then log in to one or more of
+the VMs via the ssh port(s) specified below.
+"""
+
+# Import the Portal object.
+import geni.portal as portal
+# Import the ProtoGENI library.
+import geni.rspec.pg as pg
+
+# Create the Portal context.
+pc = portal.Context()
+
+# Describe the parameter(s) this profile script can accept.
+pc.defineParameter( "n", "Number of VMs", portal.ParameterType.INTEGER, 1 )
+
+# Retrieve the values the user specifies during instantiation.
+params = pc.bindParameters()
+
+# Create a Request object to start building the RSpec.
+rspec = pg.Request()
+
+# Check parameter validity.
+if params.n < 1 or params.n > 8:
+    pc.reportError( portal.ParameterError( "You must choose at least 1 and no more than 8 VMs." ) )
+
+for i in range( params.n ):
+    # Create a XenVM and add it to the RSpec.
+    node = pg.XenVM( "node" + str( i ) )
+    rspec.addResource( node )
+
+# Print the RSpec to the enclosing page.
+pc.printRequestRSpec(rspec)

code-samples/geni-lib-single-pc.py

+"""An example of constructing a profile with a single raw PC.
+
+Instructions:
+Wait for the profile instance to start, and then log in to the host via the
+ssh port specified below.
+"""
+
 # Import the Portal object.
 import geni.portal as portal
 # Import the ProtoGENI library.
@@ -12,6 +19,6 @@ rspec = pg.Request()
 # Create a raw PC and add it to the RSpec.
 node = pg.RawPC("node")
 rspec.addResource(node)
- 
+
 # Print the RSpec to the enclosing page.
 pc.printRequestRSpec(rspec)

code-samples/geni-lib-single-vm.py

+"""An example of constructing a profile with a single Xen VM.
+
+Instructions:
+Wait for the profile instance to start, and then log in to the VM via the
+ssh port specified below.  (Note that in this case, you will need to access
+the VM through a high port on the physical host, since we have not requested
+a public IP address for the VM itself.)
+"""
+
 # Import the Portal object.
 import geni.portal as portal
 # Import the ProtoGENI library.
@@ -12,6 +21,6 @@ rspec = pg.Request()
 # Create a XenVM and add it to the RSpec.
 node = pg.XenVM("node")
 rspec.addResource(node)
- 
+
 # Print the RSpec to the enclosing page.
-pc.printRequestRSpec(rspec)
+pc.printRequestRSpec(rspec)
\ No newline at end of file

code-samples/geni-lib-two-arm-lan.py

+"""An example of constructing a profile with two ARM64 nodes connected by a LAN.
+
+Instructions:
+Wait for the profile instance to start, and then log in to either host via the
+ssh ports specified below.
+"""
+
 import geni.portal as portal
 import geni.rspec.pg as pg 
 

code-samples/geni-lib-two-vm-lan.py

+"""An example of constructing a profile with two VMs connected by a LAN.
+
+Instructions:
+Wait for the profile instance to start, and then log in to either VM via the
+ssh ports specified below.
+"""
+
 import geni.portal as portal
 import geni.rspec.pg as pg 
 
 pc = portal.Context()
 rspec = pg.Request()
 
-# Create a XenVM nodes.
+# Create two XenVM nodes.
 node1 = pg.XenVM("node1")
 node2 = pg.XenVM("node2")
 
@@ -25,4 +32,4 @@ link.addInterface(iface2)
 # Add the link to the RSpec.
 rspec.addResource(link)
 
-pc.printRequestRSpec(rspec)
+pc.printRequestRSpec(rspec)
\ No newline at end of file

creating-profiles.scrbl

@@ -69,7 +69,7 @@ be rebooted in order to take consistent snapshots of the disk.
 For the time being, this process only works for single-node profiles; we will
 add support for multi-node profiles in the future.
 
-@subsection{Creating the Profile}
+@subsection[#:tag "creating-the-profile"]{Creating the Profile}
 
 @itemlist[ #:style 'ordered
     @instructionstep["Create an experiment"]{Create an experiment using the
@@ -216,63 +216,6 @@ for the node, set commands to be run when the node boots, etc. To unselect
 the current node or link, and return to the palette on the left, simply
 click a blank area of the canvas.
 
-@section[#:tag "geni-lib"]{Describing a profile with python and @tt{geni-lib}}
-
-@margin-note{This feature is currently in @bold{alpha testing} and may change
-in the future.}
-
-@tt{geni-lib} is a tool that allows users to generate @seclink["rspecs"]{RSpec} files from Python
-code.  An @bold{experimental} feature of @(tb) is the ability to use @tt{geni-lib}
-scripts as the definition of a profile, rather then the more primitive
-RSpec format. When you supply a @tt{geni-lib} script on the
-@seclink["creating-profiles"]{Create Profile} page, your script is uploaded
-to the server so that it can be executed in the @tt{geni-lib} environment. This
-allows the script to be verified for correctness, and also produces the 
-equivalent RSpec representation that you can view if you so desire.
-
-@screenshot["create-geni-lib-empty.png"]
-
-When you provide a @tt{geni-lib} script, you will see a slightly different set
-of buttons on the @seclink["creating-profiles"]{Create Profile} page; next
-to the ``Source'' button there is an ``XML'' button that will pop up the
-RSpec XML for you to look at. The XML is read-only; if you want to change
-the profile, you will need to change the python source code that is
-displayed when you click on the ``Source'' button. Each time you change the
-python source code, the script is uploaded to the server and processed. Be
-sure to save your changes if you are 
-@seclink["updating-profiles"]{updating an existing profile}.
-
-The following examples demonstrate basic @tt{geni-lib} usage. More information
-about @tt{geni-lib} and additional examples, can be found in the
-@hyperlink["https://bitbucket.org/barnstorm/geni-lib/src"]{@tt{geni-lib} repository}. Its full documentation is online at
-@link["http://geni-lib.readthedocs.org/"]{geni-lib.readthedocs.org}.
-
-@subsection[#:tag "geni-lib-example-single-vm"]{A single XEN VM node}
-
-@code-sample["geni-lib-single-vm.py"]
-
-@subsection[#:tag "geni-lib-example-single-pc"]{A single physical host}
-
-@code-sample["geni-lib-single-pc.py"]
-
-@subsection[#:tag "geni-lib-example-two-vm-lan"]{Two XenVM nodes with a LAN between them}
-
-@code-sample["geni-lib-two-vm-lan.py"]
-
-@subsection[#:tag "geni-lib-example-two-arm-lan"]{Two ARM64 servers in a LAN}
-
-@code-sample["geni-lib-two-arm-lan.py"]
-
-@subsection[#:tag "geni-lib-example-node-ips"]{Set a specific IP address on each node}
-
-@code-sample["geni-lib-node-ips.py"]
-
-
-@subsection[#:tag "geni-lib-example-os-install-scripts"]{Specify an operating system and set install and execute scripts}
-
-@code-sample["geni-lib-os-install-scripts.py"]
-
-
 @section[#:tag "creating-from-scratch"]{Creating a profile from scratch}
 
 @future-work["planned-easier-profiles"]

defs.rkt

 #lang racket/base
 (require scribble/base)
+(require scribble/core)
 (require scribble/decode)
 (require scribble/manual)
+(require scribble/private/defaults)
+(require scribble/html-properties)
+(require scribble/latex-properties)
 (require racket/class)
 (require racket/draw)
 (require racket/system)
@@ -11,6 +15,11 @@
 
 (provide (all-defined-out))
 
+(define main-style
+  (make-style "main-body"
+              (list (js-style-addition "highlight.pack.js")
+                    (make-css-addition "highlight-default.css"))))
+
 ; Check to see if we are building Apt or CloudLab documentation
 (define tb-mode
   (cond
@@ -156,5 +165,10 @@
 (define (ssh)
   (tt "ssh"))
 
+(define code-sample-style
+  (make-style "code-sample"
+              (list (make-css-addition "code-sample.css")
+                    (make-tex-addition "code-sample.tex"))))
+
 (define (code-sample filename)
-  (code-inset (verbatim (file->string (string-append "code-samples/" filename)))))
+  (elem #:style code-sample-style (file->string (string-append "code-samples/" filename))))

emulab-manual.scrbl

@@ -40,6 +40,7 @@ you can apply to start a new project.
 @include-section["repeatable-research.scrbl"]
 @include-section["creating-profiles.scrbl"]
 @include-section["basic-concepts.scrbl"]
+@include-section["geni-lib.scrbl"]
 @include-section["advanced-topics.scrbl"]
 @include-section["hardware.scrbl"]
 @include-section["planned.scrbl"]

geni-lib.scrbl

+#lang scribble/manual
+
+@(require "defs.rkt")
+
+@title[#:tag "geni-lib" #:style main-style #:version apt-version]{Describing a profile with python and @tt{geni-lib}}
+
+@tt{geni-lib} is a tool that allows users to generate @seclink["rspecs"]{RSpec} files from Python
+code.  @(tb) offers the ability to use @tt{geni-lib}
+scripts as the definition of a profile, rather then the more primitive
+RSpec format. When you supply a @tt{geni-lib} script on the
+@seclink["creating-profiles"]{Create Profile} page, your script is uploaded
+to the server so that it can be executed in the @tt{geni-lib} environment. This
+allows the script to be verified for correctness, and also produces the 
+equivalent RSpec representation that you can view if you so desire.
+
+@screenshot["create-geni-lib-empty.png"]
+
+When you provide a @tt{geni-lib} script, you will see a slightly different set
+of buttons on the @seclink["creating-profiles"]{Create Profile} page; next
+to the ``Source'' button there is an ``XML'' button that will pop up the
+RSpec XML for you to look at. The XML is read-only; if you want to change
+the profile, you will need to change the python source code that is
+displayed when you click on the ``Source'' button. Each time you change the
+python source code, the script is uploaded to the server and processed. Be
+sure to save your changes if you are 
+@seclink["updating-profiles"]{updating an existing profile}.
+
+The following examples demonstrate basic @tt{geni-lib} usage. More information
+about @tt{geni-lib} and additional examples, can be found in the
+@hyperlink["https://bitbucket.org/barnstorm/geni-lib/src"]{@tt{geni-lib} repository}. Its full documentation is online at
+@link["http://geni-lib.readthedocs.org/"]{geni-lib.readthedocs.org}.
+
+@section[#:tag "geni-lib-example-single-vm"]{A single XEN VM node}
+
+@code-sample["geni-lib-single-vm.py"]
+
+This example demonstrates the two most important objects: the @bold{portal
+context} (acquired with the @tt{portal.Context()} constructor in the
+@tt{geni.portal} module), and the @bold{request RSpec} created with
+@tt{pg.Request()} from @tt{geni.rspec.pg}.  These fundamental objects
+are central to essentially all @(tb) @tt{geni-lib} profiles.
+
+Once the request object has been created, arbitrary resources may be
+added to it using the RSpec's @tt{addResource} method.  In this example,
+just a single node (created with the @tt{pg.XenVM()} constructor,
+asking for a single VM identified by the name "node") is requested,
+but in more complicated profiles, the @tt{addResource} call could be
+repeated as many times as necessary with different resources.
+
+The final action the @tt{geni-lib} script performs is to generate the
+XML representation of the request RSpec, with the @tt{printRequestRSpec}
+call on the last line.  This has the effect of communicating the
+description of all the resources requested by the profile back to
+@(tb).
+
+You will also notice that the profile begins with a string literal
+(to be precise, it is a Python
+@link["http://www.python.org/dev/peps/pep-0257/"]{docstring}).  The initial
+text will also be used as the
+@seclink["creating-the-profile"]{profile description}; the text following
+the @tt{Instructions:} line will be used as the corresponding
+@seclink["creating-the-profile"]{instructions}.  This documentation
+is so important that adding the description to the profile is mandatory.
+(Using a docstring like this is not the only way to produce the description
+and instructions, although it is the most convenient.)
+
+This simple example has now demonstrated all the important elements of
+a @tt{geni-lib} profile.  The portal context and request RSpec objects,
+the final @tt{printRequestRSpec} call, and the docstring description
+and instructions are ``boilerplate'' constructions, and you will probably
+include similar or identical versions of them in every @tt{geni-lib}
+profile you create unless you are doing something quite unusual.  The
+@tt{addResource} call, however, and its corresponding @tt{XenVM} objects
+in this example, is very much specific to this example and you should
+expect to tailor this portion of the script in each profile.
+
+@section[#:tag "geni-lib-example-single-pc"]{A single physical host}
+
+@code-sample["geni-lib-single-pc.py"]
+
+As mentioned above, most of these simple examples consist of boilerplate
+@tt{geni-lib} fragments, and indeed the portal context and request RSpec
+operations are unchanged from the previous script.  The big difference,
+though (other than the updated documentation) is that in this case the
+@tt{RawPC()} constructor from the @tt{pg} module was invoked instead of
+the @tt{XenVM()}.  As you might expect, the new profile will request a
+physical host instead of a virtual one.  (A side effect of using a
+real machine is that it automatically comes with a unique public IP address,
+where the VM used in the earlier example did not.  Profiles can
+@seclink["public-ip-access"]{request public IP addresses} for VMs too,
+though it does not happen by default.)
+
+@section[#:tag "geni-lib-example-two-vm-lan"]{Two XenVM nodes with a LAN between them}
+
+@code-sample["geni-lib-two-vm-lan.py"]
+
+This example demonstrates two important @tt{geni-lib} concepts: first,
+adding more than a single node to the request (which is a relatively
+straightforward matter of calling more than one node object constructor,
+being careful to use a different identifier each time, and then invoking
+@tt{addResource} once per node).  It also shows how to add @bold{links}
+between nodes, which is slightly more involved than the nodes themselves,
+because the network topology must be described in enough detail to be
+unambiguous.
+
+Note that every node which will be included in a network requires
+at least one @bold{interface}.  (This is accomplished in the example
+above with the @tt{addInterface} method available on the node object.)
+Next, the link itself is constructed (see the invocation of @tt{pg.LAN}),
+and then both of the previously described interfaces are added to the
+link with its @tt{addInterface} method.  Note that the @bold{link's}
+@tt{addInterface} and the @bold{node's} @tt{addInterface} are distinct
+methods, even though they share the same name!  They are also both
+strictly necessary to describe the topology: although it might seem
+redundant to add the same interfaces in more than one place in this case,
+observe that in a more complicated profile containing more than one
+network, establishing the correct correspondence between interfaces and
+links would become critical.
+
+Lastly, the network object itself is a kind of resource, and it must
+be added to the request RSpec just as the nodes were.  Calling
+@tt{addResource} and passing the link object takes care of this.
+
+@section[#:tag "geni-lib-example-two-arm-lan"]{Two ARM64 servers in a LAN}
+
+@code-sample["geni-lib-two-arm-lan.py"]
+
+We now come to demonstrate requesting particular properties of nodes---until
+now, all nodes had been either @tt{XenVM()}s or @tt{RawPC()}s and (although
+they might have had @seclink["geni-lib-example-two-vm-lan"]{interfaces added})
+nothing further was said about them.  @tt{geni-lib} allows the user to
+specify various details about the nodes, and this example makes use
+of the @tt{hardware_type} property.  The @tt{hardware_type} can be set
+to a string describing the type of physical machine onto which the logical
+node can be mapped: in this case, the string is @tt{"m400"}, which means
+a ProLiant Moonshot m400 host (an ARM64 server).  Obviously, such a
+profile cannot be instantiated on a cluster without a sufficient quantity
+of appropriate machines!  (This profile was written with the
+Utah CloudLab cluster in mind.)  @(tb)
+will indicate a list of suitable clusters when the user attempts to
+instantiate the profile, so he or she is not forced to find one by
+trial and error.
+
+@section[#:tag "geni-lib-example-node-ips"]{Set a specific IP address on each node}
+
+@code-sample["geni-lib-node-ips.py"]
+
+Some of the available qualifiers on requested nodes are specified by
+manipulating attributes within the node (or interface) object directly.  The
+@tt{hardware_type} in the
+@seclink["geni-lib-example-two-arm-lan"]{previous example} is one
+such case, as is the @tt{component_id} here.  (Note that the @tt{component_id}
+in this example is applied to an interface, although it is also possible to
+specify @tt{component_id}s on nodes, too, to request a particular
+physical host.)
+
+Other modifications to requests require dedicated methods.  For instance,
+see the @tt{addAddress()} calls made on each of the two interfaces above.
+In each case, an @tt{IPv4Address} object is obtained from the appropriate
+constructor (the parameters are the address and the netmask, respectively),
+and then added to the corresponding interface.
+
+@section[#:tag "geni-lib-example-os-install-scripts"]{Specify an operating system and