\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\w{\vec\theta}

\def\sp{{s^{\prime}}}

\def\ap{{a^{\prime}}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau \in \string\@lb\space +1,-1,0\string\@rb\space $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline808: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 810
$\mathaccent "705F \theta _1\in [-4\pi , 4\pi ]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline810: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 812
$\mathaccent "705F \theta _2\in [-9\pi , 9\pi ]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline812: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$m_1=m_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline814: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_1=l_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline816: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_{c1}=l_{c2}=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline818: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$I_1=I_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline820: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$g=9.8$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline822: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline824: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{10}
\stepcounter{chapter}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline830: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=backgammon.ps,rheight=2in,width=3.54in}}

  \vspace*{-.1in}
  
  \label{backgammon}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline832: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline834: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=TD-Gammon.ps,rheight=2.1in,width=4.136in}}

  \vspace*{-.1in}
  
  \label{TD-Gammon}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(n-3)/2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline844: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$n/2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline846: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$n/15$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline850: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$h(j)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline854: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}h(j) = \sigma\left(\sum_i w_{ij} \phi(i)\right) = {1 \over 1+e^{-\sum_i w_{ij}
\phi(i)}},
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\phi(i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline858: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$w_{ij}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline862: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline866: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 298
\label{nonlinearTDrule}
\vec{\theta}_{t+1} = \vec{\theta}_t + \alpha\Bigl[r_{t+1}+\gamma V_t(s_{t+1})-
V_t(s_t)\Bigr] \vec e_{t},
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 868
$\vec{\theta}_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline868: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e_{t}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline870: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline872: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}%

\vec e_{t} = \mbox{$\gamma$}
 \mbox{$\lambda$}
 \vec e_{t-1} + \nabla_{\w_t}V_t(s_t),
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e_0=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline878: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline880: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s_{t+1}) - V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline882: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_0, s_1, s_2, \ldots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline884: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{table}{\footnotesize
\begin{tabular}{|c|c|c|c|c|}
\hline
{\bf Program}	& {\bf Hidden} & {\bf Training}&{\bf Opponents}&{\bf Results}\\  
{\bf }   & {\bf Units}  &  {\bf Games}  &        &     \\ 
\hline\hline 
TD-Gam 0.0 & 40 & 300,000 & Other Programs & Tied for Best \\ 
\hline 
TD-Gam 1.0 & 80 & 300,000 & Robertie, Magriel, ...& $-13$~pts\,/\,51~games\\ 
\hline 
TD-Gam 2.0 & 40 & 800,000 &Var.\ Grandmasters&  $-7$~pts\,/\,38~games \\  
\hline
TD-Gam 2.1 & 80 & 1,500,000 & Robertie &$-1$~pts\,/\,40~games\\  
\hline 
TD-Gam 3.0 & 80 & 1,500,000 & Kazaros &$+6$~pts\,/\,20~games\\  \hline
\end{tabular}}

\label{TD-GammonTable}
\end{table}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=Samuelbackup.ps,rheight=2.3in,width=3.61in}}

  
  \label{fig:Samuelbackup}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=acrobot.ps,rheight=1.9in,width=1.58in}}


\label{fig:acrobot}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau\in\{+1,-1,0\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline900: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 902
$\dot\theta_1\in [-4\pi, 4\pi]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline902: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 904
$\dot\theta_2\in [-9\pi, 9\pi]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline904: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$m_1=m_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline906: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_1=l_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline908: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_{c1}=l_{c2}=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline910: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$I_1=I_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline912: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$g=9.8$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline914: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau\in\{+1,-1,0\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline900: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 902
$\dot\theta_1\in [-4\pi, 4\pi]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline902: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 904
$\dot\theta_2\in [-9\pi, 9\pi]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline904: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$m_1=m_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline906: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_1=l_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline908: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$l_{c1}=l_{c2}=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline910: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$I_1=I_2=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline912: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$g=9.8$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline914: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}% latex2html id marker 370
\begin{eqnarray*}
\ddot\theta_1 &=& -d_1^{-1}(d_2\ddot\theta_2 + \phi_1)\\ 
\ddot\theta_2 &=& \left(m_2l_{c2}^2 + I_2 - {d_2^2\over d_1}
      \right)^{-1} \left(\tau + {d_2\over d_1} \phi_1 - \phi_2 \right)\\ 
d_1 &=& m_1 l^2_{c1} + m_2(l_1^2 + l^2_{c2} + 2 l_1 l_{c2} \cos\theta_2)
              + I_1 + I_2) \\ 
d_2 &=& m_2 (l^2_{c2}+l_1l_{c2} \cos\theta_2) + I_2 \\ 
\phi_1 &=& -m_2l_1l_{c2}\dot\theta_2^2sin\theta_2 - 2m_2l_1l_{c2}
               \dot\theta_2\dot\theta_1sin\theta_2 \\ 
       &~& ~~~+~ (m_1 l_{c1}+m_2 l_1)g\cos (\theta_1-\pi/2) + \phi_2\\ 
\phi_2 &=& m_2l_{c2}g\cos(\theta_1 + \theta_2 - \pi/2)
\end{eqnarray*}
  
\label{fig:acrobot-equations}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline918: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline920: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline926: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 928
$\theta_1,\dot\theta_1,
\theta_2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline928: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 930
$\dot\theta_2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline930: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$[0,2\pi]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline932: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$6
\times
  7 \times
  6 \times
  7 = 1764$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline934: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 938
$\alpha=0.2/48$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline938: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda=0.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline940: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline942: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline944: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline946: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline948: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline948: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=acrobot-graph.ps,rheight=2.11in,width=3.71in}}


\label{fig:acrobot-graph}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=spider.ps,rheight=2.75in,width=5.03in}}


\label{fig:spider}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$10^{22}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline960: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=elevators.ps,rheight=2.7in,width=4.08in}}

  
  \label{fig:elevators}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{displaymath}R_t = \sum_{k=0}^\infty \gamma^k r_{t+k+1}
\qquad\mbox{\rm becomes}\qquad
R_t = \int_{0}^{\infty} e^{-\beta\tau} r_{t+\tau} \, d\tau,
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline962: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+\tau}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline964: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$t+\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline966: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta>0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline968: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma\in[0,1)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline970: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$t_1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline976: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$t_2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline978: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline980: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 957
Q(s,a) \gets Q(s,a) + \alpha \left[ \int_{t_1}^{t_2} e^{-\beta(\tau-t_1)}
r_\tau
\, d\tau
+ e^{-\beta(t_2-t_1)} \min_{a^{\prime}}  Q({s^{\prime}} ,{a^{\prime}} ) - Q(s,a) \right].
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e^{-\beta(t_2-t_1)}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline984: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=elevator-results.ps,rheight=1.9in,width=5.45in}}

  
  \label{elevator-results}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \psfig{figure=cells.ps,rheight=.41in,width=3.53in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$70^{49}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline992: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline996: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}R_t = \int_{0}^{\infty} e^{-\beta\tau} r_{t+\tau} \, d\tau,
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta>0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline998: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1000: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=cell-graph.ps,rheight=2.3in,width=3.98in}}

  
  \label{fig:cell-graph}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta V$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1008: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\exp(-\Delta V/T)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1010: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1014: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-0.001$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1016: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1018: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1020: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_f$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1026: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-RDF(s_f)-0.001(N-1)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1030: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-0.001$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1032: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0.001$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1034: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0.001$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1036: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1038: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1040: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1042: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1044: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1046: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=anytimeCmp.ps,rheight=2.7in,width=5.0in}}

  
  \label{fig:ZandD-NIPSFig4}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=anytimeCmpStp.ps,rheight=2.7in,width=5.0in}}

  
  \label{fig:ZandD-NIPSFig5}
\end{figure}
}

\stepcounter{chapter}

\end{document}
