Commit 7a0b96d2 authored by Robert Ricci's avatar Robert Ricci

Merge branch 'master' into link-samples-to-profiles

parents 40658164 03098e22
......@@ -9,5 +9,4 @@ pdf/
*.log
*.aux
*.out
*.png
*~
stages:
- fetch
- prep
- build
image: gitlab.flux.utah.edu:4567/emulab/testbed-manual:latest
fetch:
stage: fetch
script: "((cd geni-lib && hg pull && hg update 0.9-EMULAB && cd ..) || hg clone https://bitbucket.org/emulab/geni-lib && cd geni-lib && hg update 0.9-EMULAB && cd ../)"
artifacts:
paths:
- geni-lib
expire_in: 20 minutes
prep:
stage: prep
script: "cd geni-lib/docs && gmake html"
artifacts:
paths:
- geni-lib/docs/build/html
expire_in: 20 minutes
stages:
- build
- deploy
apt:
stage: build
script: "env PYTHONPATH=geni-lib/ gmake apt"
script: "((cd geni-lib && hg pull && hg update 0.9-EMULAB && cd ..) || hg clone https://bitbucket.org/emulab/geni-lib && cd geni-lib && hg update 0.9-EMULAB && cd ../) && env PYTHONPATH=geni-lib/ make apt"
artifacts:
paths:
- apt-manual/
cloudlab:
stage: build
script: "env PYTHONPATH=geni-lib/ gmake cloudlab"
script: "((cd geni-lib && hg pull && hg update 0.9-EMULAB && cd ..) || hg clone https://bitbucket.org/emulab/geni-lib && cd geni-lib && hg update 0.9-EMULAB && cd ..) && env PYTHONPATH=geni-lib/ make cloudlab"
artifacts:
paths:
- cloudlab-manual/
emulab:
stage: build
script: "env PYTHONPATH=geni-lib/ gmake emulab"
script: "((cd geni-lib && hg pull && hg update 0.9-EMULAB && cd ..) || hg clone https://bitbucket.org/emulab/geni-lib && cd geni-lib && hg update 0.9-EMULAB && cd ..) && env PYTHONPATH=geni-lib/ make emulab"
artifacts:
paths:
- emulab-manual/
phantomnet:
stage: build
script: "env PYTHONPATH=geni-lib/ gmake phantomnet"
script: "((cd geni-lib && hg pull && hg update 0.9-EMULAB && cd ..) || hg clone https://bitbucket.org/emulab/geni-lib && cd geni-lib && hg update 0.9-EMULAB && cd .. ) && env PYTHONPATH=geni-lib/ make phantomnet"
artifacts:
paths:
- phantomnet-manual/
pages:
stage: deploy
dependencies:
- apt
- cloudlab
- emulab
- phantomnet
script: "mkdir public/; cp -r apt-manual cloudlab-manual emulab-manual phantomnet-manual public/; cp toplevel-toc.html public/index.html"
artifacts:
paths:
- public/
FROM ubuntu:latest
MAINTAINER Robert Ricci "ricci@cs.utah.edu"
RUN apt-get update
RUN apt-get install -y racket python python-pip mercurial make git
RUN pip install --upgrade sphinx
RUN pip install sphinx_rtd_theme
RUN pip install lxml
\ No newline at end of file
......@@ -23,24 +23,35 @@ pdf: pdf/apt-manual.pdf pdf/cloudlab-manual.pdf pdf/phantomnet-manual.pdf \
apt: $(SOURCES)
-mkdir apt-manual/
cd $(GENI_LIB_DIR)/docs && $(MAKE) html
-rm -rf apt-manual/geni-lib/
cp -r $(GENI_LIB_DIR)/docs/build/html apt-manual/geni-lib
scribble --htmls ++style apt-manual.css ++arg --geni-lib-dir ++arg $(GENI_LIB_DIR) $(TOPLEVEL).scrbl
cloudlab: $(SOURCES)
-mkdir cloudlab-manual/
cd $(GENI_LIB_DIR)/docs && $(MAKE) html
-rm -rf cloudlab-manual/geni-lib/
cp -r $(GENI_LIB_DIR)/docs/build/html cloudlab-manual/geni-lib
scribble --htmls ++style cloudlab-manual.css ++arg --clab ++arg --geni-lib-dir ++arg $(GENI_LIB_DIR) cloudlab-manual.scrbl
cloudlab-ga: $(SOURCES)
-mkdir cloudlab-manual/
cd $(GENI_LIB_DIR)/docs && $(MAKE) html
-rm -rf cloudlab-manual/geni-lib/
cp -r $(GENI_LIB_DIR)/docs/build/html cloudlab-manual/geni-lib
scribble --htmls ++style cloudlab-manual.css ++arg --ga ++arg --clab ++arg --geni-lib-dir ++arg $(GENI_LIB_DIR) cloudlab-manual.scrbl
phantomnet: $(SOURCES)
-mkdir phantomnet-manual/
cd $(GENI_LIB_DIR)/docs && $(MAKE) html
-rm -rf phantomnet-manual/geni-lib/
cp -r $(GENI_LIB_DIR)/docs/build/html phantomnet-manual/geni-lib
scribble --htmls ++style phantomnet-manual.css ++arg --pnet ++arg --geni-lib-dir ++arg $(GENI_LIB_DIR) phantomnet-manual.scrbl
emulab: $(SOURCES)
-mkdir emulab-manual/
cd $(GENI_LIB_DIR)/docs && $(MAKE) html
-rm -rf emulab-manual/geni-lib/
cp -r $(GENI_LIB_DIR)/docs/build/html emulab-manual/geni-lib
scribble --htmls ++style emulab-manual.css ++arg --elab ++arg --geni-lib-dir ++arg $(GENI_LIB_DIR) emulab-manual.scrbl
......@@ -84,7 +95,7 @@ install-all: install-live install-cloudlab install-phantomnet install-emulab
install-live: apt
-rsync -v -az -e ssh --delete apt-manual/ $(INSTALL_LIVE)
install-cloudlab: cloudlab
install-cloudlab: cloudlab-ga
-rsync -v -az -e ssh --delete cloudlab-manual/ $(INSTALL_CLAB)
install-phantomnet: phantomnet
......
......@@ -126,7 +126,7 @@ expires reminding you to copy your data off or request an extension.
@subsection[#:tag "extending"]{Extending Experiments}
If you need more time to run an experiment, you may use the ``extend'' button
If you need more time to run an experiment, you may use the ``Extend'' button
on the experiment's page. You will be presented with a dialog that allows you
to select how much longer you need the experiment. Longer time periods require
more extensive appoval processes. Short extensions are auto-approved, while
......
......@@ -47,6 +47,7 @@ Take a look at the @seclink["status-notes"]{status notes}, and then
@include-section["repeatable-research.scrbl"]
@include-section["creating-profiles.scrbl"]
@include-section["basic-concepts.scrbl"]
@include-section["reservations.scrbl"]
@include-section["geni-lib.scrbl"]
@include-section["advanced-topics.scrbl"]
@include-section["hardware.scrbl"]
......
......@@ -65,16 +65,21 @@ information about how they work.
@itemlist[#:style 'ordered
@instructionstep["Start Experiment"]{
@screenshot["tutorial/start-experiment-menu.png"]
After logging in, you are taken to your main status
@link["https://www.cloudlab.us/user-dashboard.php"]{dashboard}.
Select ``Start Experiment'' from
the ``Experiments'' menu.
}
@instructionstep["Select a profile"]{
@screenshot["tutorial/start-experiment.png"]
@margin-note{After logging in, you will be taken to the
``@link["https://www.cloudlab.us/instantiate.php"]{Start an
Experiment}'' page automatically if you do not have any current,
running experiments. You can get back to this page at any time by
selecting the ``Start Experiment'' link from the ``Actions'' menu.}
The ``Start an Experiment'' page is where you will select a profile
to instantiate. We will use the @bold{OpenStack} profile; if
it is not selected, follow
......@@ -223,7 +228,7 @@ If an experiment has ``startup services'' (programs that run at the beginning
of the experiment to set it up), their status is indicated by a small icon in
the upper right corner of the node. You can mouse over this icon to see a
description of the current status. In this profile, the startup services
on the compute node(s) and network manager typically complete quickly, but
on the compute node(s) typically complete quickly, but
the control node may take much longer.
It is important to note that every node in @(tb) has at least @italic{two}
......@@ -252,7 +257,7 @@ work.
@subsection{Manifest View}
The final default tab shows a
The third default tab shows a
@link["http://groups.geni.net/geni/wiki/GENIExperimenter/RSpecs#ManifestRSpec"]{manifest} detailing the hardware that has been assigned to you. This is the
@seclink["rspecs"]{``request'' RSpec} that is used to define the profile,
annotated with details of the hardware that was chosen to instantiate your
......@@ -266,6 +271,28 @@ directly from this manifest; it is parsed and laid out in-browser.}
@screenshot["tutorial/experiment-manifest.png"]
@subsection{Graphs View}
The final default tab shows a page of CPU load and network traffic
graphs for the nodes in your experiment. On a freshly-created
experiment, it may take several minutes for the first data to appear.
After clicking on the ``Graphs'' tab the first time, a small reload icon
will appear on the tab, which you can click to refresh the data and
regenerate the graphs. For instance, here is the load average graph for
an OpenStack experiment running this profile for over 6 hours. Scroll
past this screenshot to see the control and experiment network traffic
graphs. In your experiment, you'll want to wait 20-30 minutes before
expecting to see anything interesting.
@screenshot["tutorial/experiment-graphs.png"]
Here are the control network and experiment network packet graphs at the
same time. The spikes at the beginning are produced by OpenStack setup
and configuration, as well as the simple OpenStack tasks you'll perform
later in this profile, like adding a VM.
@screenshot["tutorial/experiment-graphs-nets.png"]
@subsection[#:tag "openstack-tutorial-actions"]{Actions}
In both the topology and list views, you have access to several actions that
......@@ -373,7 +400,8 @@ command line interfaces or other APIs as well.}
@instructionstep["Log in to the OpenStack web interface"
#:screenshot "tutorial/os-login.png"]{
Log in using the username @tt{admin} and the password shown in
the instructions for the profile.
the instructions for the profile. Use the domain name @tt{default}
if prompted for a domain.
@margin-note{This profile generates a new (random) password for every
experiment.}
......@@ -393,37 +421,41 @@ command line interfaces or other APIs as well.}
@instructionstep["Set Basic Settings For the Instance"]{
There a few settings you will need to make in order to launch your
instance:
instance. These instructions are for the launch wizard in the
OpenStack @tt{Mitaka}, but the different release wizards are all
similar in function and required information.
Use the tabs in the column on the left of the launch
wizard to add the required information (i.e., @tt{Details},
@tt{Source}, @tt{Flavor}, @tt{Networks}, and @tt{Key Pair}),
as shown in the following screenshots:
@screenshot["tutorial/os-launch-basic.png"]
@itemlist[
@item{Pick any ``Instance Name'' you wish}
]
@item{Set the ``Flavor'' to @tt{m1.small}---the disk for the
default @tt{m1.tiny} instance is too small for the image we
will be using, and since we have only one compute node, we
want to avoid using up too many of its resources.}
@screenshot["tutorial/os-launch-source.png"]
@itemlist[
@item{For the ``Image Name'', select ``trusty-server''}
@item{For the ``Instance Boot Source'', select ``Boot from
image''}
@item{For the ``Image Name'', select ``trusty-server''}
]
]
@bold{Important:} If you do not see any available images,
the image import script may not have finished yet; wait a few
minutes, reload the page, and try again.
}
@instructionstep["Set an SSH Keypair"
#:screenshot "tutorial/os-launch-key.png"]{
On the ``Access & Security'' tab, you will add an @(ssh) keypair
to log into your node. If you configured an @(ssh) key in your GENI
account, you should find it as one of the options in this list. If
not, you can add a new keypair with the red button to the right of the
list. Alternately, you
can skip this step and use a password for login later.
@screenshot["tutorial/os-launch-flavor.png"]
@itemlist[
@item{Set the ``Flavor'' to @tt{m1.small}---the disk for the
default @tt{m1.tiny} instance is too small for the image we
will be using, and since we have only one compute node, we
want to avoid using up too many of its resources.}
]
}
@instructionstep["Add a Network to the Instance"
......@@ -443,10 +475,24 @@ command line interfaces or other APIs as well.}
started. Unfortunately, reloading does not solve this, you will need
to log out and back in.
}
@instructionstep["Launch, and Wait For Your Instance to Boot"
@instructionstep["Set an SSH Keypair"
#:screenshot "tutorial/os-launch-finish.png"]{
Click the ``Launch'' button, and wait for the status on the
On the ``Key Pair'' tab (or ``Access & Security'' in previous versions), you will add an @(ssh) keypair
to log into your node. If you configured an @(ssh) key in your GENI
account, you should find it as one of the options in this
list. You can filter the list by typing your Cloudlab username,
or a portion of it, into the filter box to reduce the size of
the list. (By default, we load in public keys for all users in the
project in which you created your experiment, for convenience --
thus the list can be long.) If you don't see your keypair,
you can add a new one with the red button to the right of the
list. Alternately, you
can skip this step and use a password for login later.
}
@instructionstep["Launch, and Wait For Your Instance to Boot"]{
Click the ``Launch'' button on the ``Key Pair'' wizard page, and wait for the status on the
instances page to show up as ``Active''.
}
......@@ -475,8 +521,12 @@ command line interfaces or other APIs as well.}
newly-allocated address with this instance.}
]
@margin-note{The public address is tunneled by the @tt{nm} (network manager) node
from the control network to the experiment network.}
@margin-note{The public address is tunneled by the @tt{ctl}
(controller) node from the control network to the experiment network.
(In older OpenStack profile versions, or depending the profile
parameters specified to the current profile version, the public
address may instead be tunneled by the @tt{nm} (network manager) node,
in a split controller/network manager OpenStack deployment.)}
You will now see your instance's public address on the ``Instances''
page, and should be able to ping this address from your laptop or
......@@ -515,7 +565,8 @@ Some things to try:
@itemlist[
@item{Run @tt{ps -ef} on the @tt{ctl} to see the list of OpenStack
services running}
@item{Run @tt{ifconfig} on the @tt{nm} node to see the various
@item{Run @tt{ifconfig} on the @tt{ctl} node (or the @tt{nm} node,
if your experiment has one), to see the various
bridges and tunnels that have been brought to support the networking in
your cloud}
@item{Run @tt{sudo virsh list @literal{--}all} on @tt{cp1} to see the VMs that
......
"""This profile demonstrates how to mount a copy of a dataset in a file system. You can make changes to the dataset, but they are *temporary** (temporary means any changes will be lost when you terminate your experiment). You must explicitly request that the changes be saved if you want them to persist.
Instructions:
Log into your node, your dataset is mounted at `/mydata`.
"""
# Import the Portal object.
import geni.portal as portal
# Import the ProtoGENI library.
import geni.rspec.pg as rspec
# Import the emulab extensions library.
import geni.rspec.emulab
# Create a portal context.
pc = portal.Context()
# Create a Request object to start building the RSpec.
request = pc.makeRequestRSpec()
# Allocate a node and ask for a copy of a dataset to be mounted at /mydata
node = request.RawPC("node")
bs = node.Blockstore("bs", "/mydata")
# A demonstration dataset, there is nothing in the file system.
bs.dataset = "urn:publicid:IDN+emulab.net:testbed+imdataset+pgimdat"
# Print the RSpec to the enclosing page.
pc.printRequestRSpec()
"""This profile demonstrates how to add some extra *local* disk space on your node. In general nodes have much more disk space then what you see with `df` when you log in. That extra space is in unallocated partitions or additional disk drives. An *ephemeral blockstore* is how you ask for some of that space to be allocated and mounted as a **temporary** filesystem (temporary means it will be lost when you terminate your experiment).
Instructions:
Log into your node, your **temporary** file system in mounted at `/mydata`.
"""
# Import the Portal object.
import geni.portal as portal
# Import the ProtoGENI library.
import geni.rspec.pg as rspec
# Import the emulab extensions library.
import geni.rspec.emulab
# Create a portal context.
pc = portal.Context()
# Create a Request object to start building the RSpec.
request = pc.makeRequestRSpec()
# Allocate a node and ask for a 30GB file system mounted at /mydata
node = request.RawPC("node")
bs = node.Blockstore("bs", "/mydata")
bs.size = "30GB"
# Print the RSpec to the enclosing page.
pc.printRequestRSpec()
......@@ -25,27 +25,31 @@ physical) described by the RSpec.
@section[#:tag "creating-from-existing"]{Creating a profile from an existing one}
The easiest way to create a new profile is by cloning an existing one and
The easiest way to create a new profile is by cloning or copying an existing one and
customizing it to your needs. The basic steps are:
@itemlist[ #:style 'ordered
@item{Find an existing profile that's closest to your needs}
@item{Create an experiment using that profile}
@item{Log into the node(s) in your experiment to install your software,
configure the operating system, etc.}
@item{``Clone'' the experiment to create a new profile}
]
When you @bold{clone} an experiment, you are taking an existing experiment, including a snapshot of the disk, and creating a new profile based on it. The new profile will be identical to the profile that experiment was based on in all other respects. Cloning only works on experiments with a single node.
If you @bold{copy} a profile, you are creating a new profile that is identical in every way to an existing profile. You may or may not have a running experiment using the source profile. And if you do have a running experiment, it does not impact the copy. After copying a profile, you can then modify it for your own use. And if you instantiate the copy, you can then take snapshots of disk images and use them in future version of your copy. Any profile that you have access to may be copied.
@subsection[#:tag "profile-creation-preparation"]{Preparation and precautions}
To create profiles, you need to be a @seclink["register"]{registered user}.
Creating a profile can take a while, so we recommend that you
Cloning a profile can take a while, so we recommend that you
@seclink["experiments"]{extend your experiment} while creating it, and contact
us if you are worried your experiment might expire before you're done creating
your profile. We also strongly recommend testing your profile fully before
terminating the experiment you're creating it from.
When cloning, your home directory is @bold{not} included in the disk image snapshot! You will need to install your code and data elsewhere in the image. We recommend /local/. Keep in mind that others who use your profile are going to have their own accounts, so make sure that nothing in your image makes assumptions about the username, home directory, etc. of the user running it.
When cloning, be aware the only the contents of disk (not running process, etc.) are stored as part of the profile, and as part of the creation process, your node(s) will be rebooted in order to take consistent snapshots of the disk.
For the time being, cloning only works for single-node profiles; we will add support for multi-node profiles in the future.
When copying a profile, remember that the disk images of a currently running experiment are not saved. If you want to customize the disk images using copy, you must copy the profile first, then instantiate your copy, then take snapshots of the modified disk image in your experiment.
@apt-only{
If you want your profile to be usable by @seclink["guest-users"]{guest
users}, keep in mind that there are @seclink["guest-users"]{several
......@@ -56,22 +60,10 @@ terminating the experiment you're creating it from.
the profile.
}
Your home directory is @bold{not} included in the disk image snapshot! You
will need to install your code and data elsewhere in the image. We recommend
@code{/local/}. Keep in mind that others who use your profile are going to have
their own accounts, so make sure that nothing in your image makes assumptions
about the username, home directory, etc. of the user running it.
Be aware the only the contents of disk (not running process, etc.) are stored
as part of the profile, and as part of the creation process, your node(s) will
be rebooted in order to take consistent snapshots of the disk.
For the time being, this process only works for single-node profiles; we will
add support for multi-node profiles in the future.
@subsection[#:tag "creating-the-profile"]{Creating the Profile}
@subsection[#:tag"cloning-a-profile"]{Cloning a Profile}
@itemlist[ #:style 'ordered
@instructionstep["Create an experiment"]{Create an experiment using the
profile that is most similar to the one you want to build. Usually, this
will be one of our facility-provided profiles with a generic installation
......@@ -95,6 +87,39 @@ add support for multi-node profiles in the future.
your running experiment.
}
@instructionstep["Create Profile"]{
You will be taken to a complete profile form and should fill it out as described below.
}
]
@subsection[#:tag"copying-a-profile"]{Copying a Profile}
@itemlist[ #:style 'ordered
@instructionstep["Choose a profile"
#:screenshot "begin-experiment.png" ]{
Find the profile you wish to clone using the “Start Experiment” selector. Then you can click “Show Profile” to clone the profile directly or you can instantiate the profile if you wish to create an experiment first. Both profiles themselves and active experiments can be copied.
}
@instructionstep["Copy the profile or experiment"
#:screenshot "copy-button.png" ]{
While logged in, both your experiment page and the show profile page will have a copy button. Clicking this button will create a profile based on that profile or experiment.
This button only copies the rspec or genilib script. No state in the active experiment is preserved.
}
@instructionstep["Create Profile"]{
You will be taken to a complete profile form and should fill it out as described below.
}
]
@subsection[#:tag "creating-the-profile"]{Creating the Profile}
After copying or cloning a profile (see above) or selecting the menu option to create a new profile from scratch, you will need to fill out the profile creation form in order to complete the creation process.
@itemlist[ #:style 'ordered
@instructionstep["Fill out information for the new profile"]{
After clicking on the ``clone'' button, you will see a form that
allows you to view and edit the basic information associated with your
......@@ -144,7 +169,7 @@ add support for multi-node profiles in the future.
ready! It will now show up in your ``My Profiles'' list.}
@instructionstep["Test your profile"]{Before terminating your experiment
(or letting it expire), we strongly recommend testing out the profile. If
(or letting it expire), we strongly recommend testing out the new profile. If
you elected to make it publicly visible, it will be listed in the profile
selection dialog on the front page of @url[(apturl)]. If not,
you can instantiate it from the listing in your ``My Profiles'' page. If
......@@ -216,6 +241,74 @@ for the node, set commands to be run when the node boots, etc. To unselect
the current node or link, and return to the palette on the left, simply
click a blank area of the canvas.
@section[#:tag "repo-based-profiles"]{Repository-Based Profiles}
You can turn any public @tt{git} repository (including those hosted on GitHub)
into a @(tb) profile. Simply place a @tt{geni-lib} script
named @tt{profile.py} into the top-level directory of your repository. When
you create a new profile, you can provide the URL for your repository. The URL
needs to be a @tt{http://} or @tt{https://} URL, and the CloudLab portal needs
to be able to clone the repository without authentication.
@margin-note{
Note that CloudLab is not a @tt{git} hosting service; while we do keep a
cache of your repository, we don't guarantee that the profile will continue
to work if the original repository becomes unavailable. We also have limits
on the size of the repositories that we will clone.
}
When you intantiate a repository-based profile, the repository will be cloned
into the directory @tt{/local/repository} on all nodes in the experiment. This
means that you can keep source code, startup scripts, etc. in your repository
and refrence them from @tt{profile.py}. @(tb) sets the @tt{pull} URL for all
of these clones to be your ``upstream'' repository, and attempts to set
a suitable @tt{push} URL for (it assumes that the hosting service uses @tt{ssh}
for pushes, and uses the @tt{git@"@"<hostname>:user/repo} convention). As a
result, @tt{git pull} and @tt{git push} should be connected to your repository.
There is an example repository on GitHub at
@hyperlink["https://github.com/emulab/my-profile"]{@tt{https://github.com/emulab/my-profile}}; if you don't already have a @tt{git} repository created, a good
way to get started is to for this one and crate a new profile pointing at your