lecturenotes.tex 2.6 KB
Newer Older
1 2
\documentclass{article}[12pt]

Robert Ricci's avatar
Robert Ricci committed
3 4
\input{../../texstuff/fonts.sty}
\input{../../texstuff/notepaper.sty}
5 6 7 8
\usepackage{outlines}

\title{CS6963 Lecture \#20}
\author{Robert Ricci}
Robert Ricci's avatar
Robert Ricci committed
9
\date{April 16, 2015}
10 11 12 13 14 15 16

\begin{document}

\maketitle

\begin{outline}

17 18
\1 Project status updates

19 20 21 22 23 24 25 26 27 28 29
\1 Review of topics covered
    \2 Overall point of systems evaluation
        \3 Providing data necessary to make a decision
        \3 Presenting it on a convincing manner
    \2 Understand what you are measuring
        \3 Understand what make convincing evidence 
        \3 Understand what you need to measure and how to measure it
    \2 Understand the system under test
        \3 Where are the boundaries?
        \3 What is inside those boundaries that you are actually measuring?
        \3 How to the SUT boundaries relate to a deployment environment?
30
    \2 Evaluation should be a part of the research and development process
31 32 33 34 35 36 37
        \3 Convince yourself with data, not just bias
        \3 Often need preliminary evaluations
        \3 Understand the scope and limitations of your work
        \3 The more evaluation you do along the way, the less biased you are
            likely to be
    \2 Recognize the strengths and weaknesses in evaluations that you read
        \3 Think actively about what you need to see to be convinced
38
        \3 Look for biases or basic mistakes
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
    \2 Common mistakes in systems evaluation
        \3 No goals or biased goals
        \3 Ignoring significant factors
        \3 Analysis without understanding the problem
        \3 No sensitivity analysis
        \3 Ignoring variability
    \2 Use the statistical tools available to you
        \3 For selecting the number of runs
        \3 For understanding the confidence in your results
        \3 For showing difference or sameness in results
        \3 The question is ``have you found something real?''
    \2 Use the tools available to you
        \3 Reproducibility is a big deal---not just for others, but for
            yourself too
        \3 Assuming your environment is fragile forces you to build more
            reproducible research
        \3 The closer you can get to ``experiment as function call'', the 
            happier you will be
        \3 Keep track of everything 
58 59
    \2 You are running experiments all the time, the only question is whether
        you are learning anything from them
60

61 62 63 64 65 66
\1 For next time
    \2 Weekly 3 due Friday midnight
    \2 Papers 4 due next Tuesday
    \2 Talks next Thursday (Anil Kumar, Jithu, George, Jonathon)
        \3 Schedule a time to meet with me
        \3 Any questions about what's expected?
67 68 69 70

\end{outline}

\end{document}