\documentclass[nohyper,justified]{tufte-handout}
%\VignetteEngine{knitr::knitr}
%\VignetteIndexEntry{Sampling Properties of Variable Selection}
% \usepackage{tufte-larger}
\usepackage{amsmath, amssymb,txfonts,bm}
\usepackage{scrhack}
\usepackage{nextpage}
\usepackage{fancyvrb}
\usepackage[unicode=true,pdfusetitle,
 bookmarks=true,bookmarksnumbered=true,bookmarksopen=true,bookmarksopenlevel=2,
 breaklinks=true,pdfborder={0 0 1},backref=false,colorlinks=false]
 {hyperref}
\hypersetup{pdfstartview=FitH}
\usepackage{breakurl}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% LyX specific LaTeX commands.

\title{Functions that Demonstrate the Sampling Properties of Variable
     Selection}
\author{John Maindonald}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.
\renewcommand{\textfraction}{0.05}
\renewcommand{\topfraction}{0.8}
\renewcommand{\bottomfraction}{0.8}
\renewcommand{\floatpagefraction}{0.75}

\usepackage[buttonsize=1em]{animate}

\makeatother

\newenvironment{itemizz}%
  {\begin{itemize}%
    \setlength{\itemsep}{2pt}%
    \setlength{\parskip}{2pt}}%
  {\end{itemize}}

% Macros for typesetting the documentation
\newcommand{\hlred}[1]{\textcolor{Maroon}{#1}}% prints in red
\newcommand{\hangleft}[1]{\makebox[0pt][r]{#1}}
\newcommand{\hairsp}{\hspace{1pt}}% hair space
\newcommand{\hquad}{\hskip0.5em\relax}% half quad space
\newcommand{\TODO}{\textcolor{red}{\bf TODO!}\xspace}
\newcommand{\ie}{\textit{i.\hairsp{}e.}\xspace}
\newcommand{\eg}{\textit{e.\hairsp{}g.}\xspace}
\newcommand{\na}{\quad--}% used in tables for N/A cells
\providecommand{\XeLaTeX}{X\lower.5ex\hbox{\kern-0.15em\reflectbox{E}}\kern-0.1em\LaTeX}
\newcommand{\tXeLaTeX}{\XeLaTeX\index{XeLaTeX@\protect\XeLaTeX}}
% \index{\texttt{\textbackslash xyz}@\hangleft{\texttt{\textbackslash}}\texttt{xyz}}
\newcommand{\tuftebs}{\symbol{'134}}% a backslash in tt type in OT1/T1
\newcommand{\doccmdnoindex}[2][]{\texttt{\tuftebs#2}}% command name -- adds backslash automatically (and doesn't add cmd to the index)
\newcommand{\doccmddef}[2][]{%
  \hlred{\texttt{\tuftebs#2}}\label{cmd:#2}%
  \ifthenelse{\isempty{#1}}%
    {% add the command to the index
      \index{#2 command@\protect\hangleft{\texttt{\tuftebs}}\texttt{#2}}% command name
    }%
    {% add the command and package to the index
      \index{#2 command@\protect\hangleft{\texttt{\tuftebs}}\texttt{#2} (\texttt{#1} package)}% command name
      \index{#1 package@\texttt{#1} package}\index{packages!#1@\texttt{#1}}% package name
    }%
}% command name -- adds backslash automatically
\newcommand{\doccmd}[2][]{%
  \texttt{\tuftebs#2}%
  \ifthenelse{\isempty{#1}}%
    {% add the command to the index
      \index{#2 command@\protect\hangleft{\texttt{\tuftebs}}\texttt{#2}}% command name
    }%
    {% add the command and package to the index
      \index{#2 command@\protect\hangleft{\texttt{\tuftebs}}\texttt{#2} (\texttt{#1} package)}% command name
      \index{#1 package@\texttt{#1} package}\index{packages!#1@\texttt{#1}}% package name
    }%
}% command name -- adds backslash automatically
\newcommand{\docopt}[1]{\ensuremath{\langle}\textrm{\textit{#1}}\ensuremath{\rangle}}% optional command argument
\newcommand{\docarg}[1]{\textrm{\textit{#1}}}% (required) command argument
\newenvironment{docspec}{\begin{quotation}\ttfamily\parskip0pt\parindent0pt\ignorespaces}{\end{quotation}}% command specification environment
\newcommand{\docenv}[1]{\texttt{#1}\index{#1 environment@\texttt{#1} environment}\index{environments!#1@\texttt{#1}}}% environment name
\newcommand{\docenvdef}[1]{\hlred{\texttt{#1}}\label{env:#1}\index{#1 environment@\texttt{#1} environment}\index{environments!#1@\texttt{#1}}}% environment name
\newcommand{\docpkg}[1]{\texttt{#1}\index{#1 package@\texttt{#1} package}\index{packages!#1@\texttt{#1}}}% package name
\newcommand{\doccls}[1]{\texttt{#1}}% document class name
\newcommand{\docclsopt}[1]{\texttt{#1}\index{#1 class option@\texttt{#1} class option}\index{class options!#1@\texttt{#1}}}% document class option name
\newcommand{\docclsoptdef}[1]{\hlred{\texttt{#1}}\label{clsopt:#1}\index{#1 class option@\texttt{#1} class option}\index{class options!#1@\texttt{#1}}}% document class option name defined
\newcommand{\docmsg}[2]{\bigskip\begin{fullwidth}\noindent\ttfamily#1\end{fullwidth}\medskip\par\noindent#2}
\newcommand{\docfilehook}[2]{\texttt{#1}\index{file hooks!#2}\index{#1@\texttt{#1}}}
\newcommand{\doccounter}[1]{\texttt{#1}\index{#1 counter@\texttt{#1} counter}}

\setcounter{secnumdepth}{2}

% Prints the month name (e.g., January) and the year (e.g., 2008)
\newcommand{\monthyear}{%
  \ifcase\month\or January\or February\or March\or April\or May\or June\or
  July\or August\or September\or October\or November\or
  December\fi\space\number\year
}

\renewcommand{\maketitlepage}[0]{%
  \cleardoublepage%
  {%
  \sffamily%
  \begin{fullwidth}%
  \fontsize{18}{20}\selectfont\par\noindent\textcolor{darkgray}{{\thanklessauthor}}%
  \vspace{11.5pc}%
%  \fontsize{27}{30}\selectfont\par\noindent\textcolor{darkgray}{{\thanklesstitle}}%
  \fontsize{40}{45}\selectfont\par\noindent\textcolor{darkgray}{{\thanklesstitle}}%
  \vfill%
  \fontsize{14}{16}\selectfont\par\noindent\allcaps{\thanklesspublisher}%
  \end{fullwidth}%
  }
  \thispagestyle{empty}%
  \clearpage%
}


\begin{document}

\bmdefine\bX{\mathbf{X}}
\bmdefine\bP{\mathbf{P}}
\bmdefine\sfX{\bm{\textsf{\textmd{X}}}}
\def\sx{\textsf{x}}
\bmdefine\bI{\mathbf{I}}
\def\bW{\boldsymbol{W}}
\def\by{\boldsymbol{y}}
\def\bSigma{\boldsymbol{\Sigma}}
\def\beps{\boldsymbol{\epsilon}}
\def\bbeta{\boldsymbol{\beta}}
\def\bmu{\boldsymbol{\mu}}
\def\bb{\boldsymbol{b}}
\def\bx{\boldsymbol{x}}

<<setup, include=FALSE, cache=FALSE>>=
library(knitr)
options(replace.assign=TRUE,width=50)
opts_chunk$set(fig.path='figs/gph-', cache.path='cache/gph-',
               fig.align='center', dev='pdf', fig.width=5,
               fig.height=5, fig.show='hold', cache=TRUE, par=TRUE)
knit_hooks$set(par=function(before, options, envir){
if (before && options$fig.show!='none') par(mar=c(4,4,1.6,.1),
              cex.lab=.95,cex.axis=.9,mgp=c(2,.7,0),tcl=-.3)
}, crop=hook_pdfcrop)
@

\maketitle

The function \texttt{varselect()} in the {\em leaps} package can be
used for variable selection.  Available appraoches are {\em forward},
{\em backward} and {\em exhaustive} selection.  The {\em DAAG} package
has the functions \texttt{bestsetNoise()} and \texttt{bsnVaryNvar()}
that are designed to give insight on the sampling properties of output
from the function \texttt{lm()}, when one of these variable selection
approaches has been used to choose the explanatory variables that
appear in the model.

<<loadDAAG>>=
library(DAAG, quietly=TRUE, warn.conflicts=FALSE)
@ %

\section{Selection of a specified number of explanatory variables}
The function \texttt{bestsetNoise()} ({\em DAAG}) can be used to
experiment with the behaviour of various variable selection techniques
with data that is purely noise.  \Citet[Section 6.5, pp.~197-198]{m-b}
gives examples from the use of this function.  For example, try:
<<bestset, eval=FALSE, echo=TRUE>>=
bestsetNoise(m=100, n=40, nvmax=3)
bestsetNoise(m=100, n=40, method="backward", nvmax=3)
@ %
The analyses will typically yield a model that, if assessed using
output from R's function \texttt{lm()}, appears to have highly (but
spuriously) statistically significant explanatory power, with one or
more coefficients that appear (again spuriously) significant at a
level of around $p$=0.01 or less.

The function \texttt{bestsetNoise()} has provision to specify the
model matrix.  Model matrices with uncorrelated columns of independent
Normal data, which is the default, are not a good match to most
practical situations.

\section{Change with the Number of Variables Available for Selection}

As above, datasets of random normal data were created, always with 100
observations and with the number of variables varying between 3 and
50.  For three variables, there was no selection, while in other cases
the ``best'' three variables were selected, by exhaustive search.
Figure \ref{fig:exhaust} plots the p-values for the 3 variables that
were selected against the total number of variables. The fitted line
estimates the median $p$-value.

\begin{figure}
<<exhaust, eval=TRUE, echo=FALSE, fig.width=3.25, fig.height=3.25, out.width="0.6\\textwidth">>=
## Code
library(quantreg, quietly=TRUE)
library(splines, quietly=TRUE)
set.seed(37)   # Use to reproduce graph that is shown
bsnVaryNvar(m=100, nvar=3:50, nvmax=3)
@ %
\caption{$p$-values from R's \texttt{lm()} function, versus number of
  variables available for selection, when the ``best'' 3 variables
  were selected by exhaustive search.  A curve has been added that
  estimates the median $p$-value, as a function of \texttt{nvar}.  The
  function \texttt{bsnVaryNvar()} that is used for the calculations
  makes repeated calls to \texttt{bestsetNoise().
  Similar results will be obtained from use of forward or backward
  selection.}\label{fig:exhaust}}
\end{figure}

<<exhaust, eval=FALSE, echo=TRUE>>=
@ %

When all 3 variables are taken, the $p$-values are expected to average
0.5.  Notice that, for selection of the best 3 variables out of 10,
the median $p$-value has reduced to about 0.1.

\begin{thebibliography}{99}
  \bibitem[Maindonald \& Braun(2010)]{m-b} \textsc{Maindonald, J.~H.\
  and Braun, W.J.} 2010.  \textit{Data Analysis and Graphics Using R --
    An Example-Based Approach\/},  3$^{rd}$ edition,
  Cambridge University Press.\newline
\url{http://www.maths.anu.edu.au/~johnm/r-book.html}
\end{thebibliography}

\end{document}
