From 6b7181875719cb4c7cfd4385d9ab7e14e007a8b5 Mon Sep 17 00:00:00 2001
From: Robert Ricci <ricci@cs.utah.edu>
Date: Tue, 3 Mar 2015 13:59:38 -0700
Subject: [PATCH] Update eval form for 2015

---
 assignments/papers3/evals/paper-eval.tex | 47 ++++++++++++++----------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/assignments/papers3/evals/paper-eval.tex b/assignments/papers3/evals/paper-eval.tex
index daea424..9215f7d 100644
--- a/assignments/papers3/evals/paper-eval.tex
+++ b/assignments/papers3/evals/paper-eval.tex
@@ -30,27 +30,35 @@
 \section{Questions to Answer}
 
 \question{List the questions you think the paper needs to answer in order
-    to persuasively argue that it has accomplished its goals. Which of
-    these questions can be answered by evaluation? For those that can, what
-    type of evaluation would you find most convincing? What factors would
-    you expect to see investigated, and over what ranges (order of magnitude)?
-    What metrics would you expect to see used? For questions that cannot
-    be answered with a typical evaluation, what kinds of arguments would you
-    find persuasive?}
+    to persuasively argue that it has accomplished its goals. Please number
+    the questions in order to refer to them later. For each question, state:
+    \begin{itemize}
+        \item Can the question be answered with an empirical evaluation?\\
+            If so:
+            \begin{itemize}
+                \item What type of evaluation (simulation, analytic,
+                    measurement) would you find convincing?
+                \item What factors would you expect to see investigated, and
+                    over what ranges (order of magnitude)?
+                \item What metrics would you expect to see used? 
+            \end{itemize}
+            If not:
+            \begin{itemize}
+                \item What kinds of arguments would you find persuasive?
+            \end{itemize}
+    \end{itemize}
+}
 
 \section{System Under Test}
 
-\question{What did the authors chose as the boundaries for the System Under
-    Test? Did they define them clearly? If not, try defining them yourself,
-    either based on clues in the paper or based on what you think the
-    boundaries should be.}
-
-\section{Parameters and Factors}
-
-\question{What (system and/or workload) parameters do the authors list? Did
-    they include any that were not necessary, or leave out any you think
-    to be important? Which of the parameters did they vary as factors,
-    and do you think there are any unnecessary or left-out factors?}
+\question{
+    \begin{itemize}
+        \item List the inputs and outputs to the System Under Test
+        \item What components (software, hardware, network,  etc.) are
+            necessary to complete the process of turning the inputs into
+            outputs?
+    \end{itemize}
+}
 
 \section{Major Evaluations}
 
@@ -63,13 +71,14 @@
             \item Factor(s) varied
             \item Workload used
             \item Conclusion / outcome of the evaluation
+            \item Which, if any, of your questions from Part 3 did it address?
         \end{itemize}
     }
 
 \section{Questions Revisited}
 
 \question{Looking back at the list of questions, which were answered to your
-    satisfaction, and which were not?}
+    satisfaction, and which were not? (Refer to them by number) }
 
 \section{Goals Revisited}
 
-- 
GitLab